Add support for another kind of index in HTTP backend

Rafid · August 30, 2024, 1:26pm

What is the problem you are having with rclone?

rclone failed to show files & folders from a specific website. I suspect the website is javascript based. I've attached the page source from the website below for inspection. If it's javascript based, is there any way to use rclone?
Note: I've tried both no_slash & no_head, not working.

Run the command 'rclone version' and share the full output of the command.

rclone v1.67.0

os/version: Microsoft Windows 11 Pro 24H2 (64 bit)
os/kernel: 10.0.26100.1457 (x86_64)
os/type: windows
os/arch: amd64
go/version: go1.22.4
go/linking: static
go/tags: cmount

Which cloud storage system are you using? (eg Google Drive)

http

The command you were trying to run (eg `rclone copy /tmp remote:tmp`)

rclone lsd CDN:

Please run 'rclone config redacted' and share the full output. If you get command not found, please make sure to update rclone.

[TEST]
type = http
url = http://cdn.dflix.live:5050

A log from the command that you were trying to run with the `-vv` flag

2024/08/29 00:07:35 DEBUG : rclone: Version "v1.67.0" starting with parameters ["C:\\rclone\\rclone.exe" "lsd" "CDN:" "-vv"]
2024/08/29 00:07:35 DEBUG : Creating backend with remote "CDN:"
2024/08/29 00:07:35 DEBUG : Using config file from "C:\\Users\\afshe\\AppData\\Roaming\\rclone\\rclone.conf"
2024/08/29 00:07:35 DEBUG : Root: http://cdn.dflix.live:5050/
2024/08/29 00:07:35 NOTICE: Time may be set wrong - time from "cdn.dflix.live:5050" is -9m8.4272683s different from this computer
2024/08/29 00:07:35 DEBUG : 4 go routines active

Website page source [http://cdn.dflix.live:5050/]


<!DOCTYPE html>
<html lang="en" translate="no">
  <head>
    <script src="https://polyfill.io/v3/polyfill.min.js?features=String.prototype.replaceAll"></script>
    <meta charset="utf-8" >
    <meta name="viewport" content="width=device-width, initial-scale=1" >
    <meta name="referrer" content="same-origin" >
    <meta name="generator" content="AList V3" >
    <meta name="theme-color" content="#000000" >
    <meta name="google" content="notranslate" >
    
    <meta name="apple-mobile-web-app-capable" content="yes" >
    <meta name="apple-mobile-web-app-title" content="AList" >
    <link       rel="apple-touch-icon"
      href="https://jsd.nn.ci/gh/alist-org/logo@main/logo.png"
    >
    <script       src="https://g.alicdn.com/IMM/office-js/1.1.5/aliyun-web-office-sdk.min.js"
      async
    ></script>
    <link       rel="shortcut icon"
      type="image/ico"
      href="http://dflix.live/favicon.ico"
    >
    <title>DFLIX</title>
    <script>
      window.ALIST = {
        cdn: '',
        monaco_cdn: undefined,
        base_path: '/',
        api: undefined,
        main_color: '#D71B5D',
      }
      window.__dynamic_base__ = window.ALIST.cdn || ""
    </script>
    
    
    <script type="module">try{import.meta.url;import("_").catch(()=>1);}catch(e){}window.__vite_is_modern_browser=true;</script>
    <script type="module">!function(){if(window.__vite_is_modern_browser)return;console.warn("vite: loading legacy build because dynamic import or import.meta.url is unsupported, syntax error above should be ignored");var e=document.getElementById("vite-legacy-polyfill"),n=document.createElement("script");n.src= window.__dynamic_base__+e.getAttribute('data-src'),n.onload=function(){System.import( window.__dynamic_base__+document.getElementById('vite-legacy-entry').getAttribute('data-src'))},document.body.appendChild(n)}();</script>
    <script>
(function(){
var preloads = [{"parentTagName":"head","tagName":"link","attrs":{"href":"/static/manifest.json","rel":"manifest"}},{"parentTagName":"head","tagName":"script","attrs":{"type":"module","crossorigin":"","src":"/assets/index.52083170.js"}},{"parentTagName":"head","tagName":"link","attrs":{"rel":"stylesheet","href":"/assets/index.15d8d646.css"}}];
function setAttribute(target, attrs) {
for (var key in attrs) {
  target.setAttribute(key, attrs[key]);
}
return target;
};
for(var i = 0; i < preloads.length; i++){
var item = preloads[i]
var childNode = document.createElement(item.tagName);
setAttribute(childNode, item.attrs)
if( window.__dynamic_base__) {
  if(item.tagName == 'link') {
    setAttribute(childNode, { href:  window.__dynamic_base__ + item.attrs.href })
  } else if (item.tagName == 'script') {
    setAttribute(childNode, { src:  window.__dynamic_base__ + item.attrs.src })
  }
}
document.getElementsByTagName(item.parentTagName)[0].appendChild(childNode);
}
})();
</script>
</head>
  <body>
    <noscript>You need to enable JavaScript to run this app.</noscript>
    <div id="root"></div>

    
    <div id="custom-styles">
  <ul>
    <li><a href="http://stream.dflix.live">EMBY SERVER</a></li>
 
  </ul>
</div>
<marquee style="background: #D71B5D; margin-top: 10px; padding: 5px; font-size: 22px; font-family: sans-serif; color: white; border-radius: 100px;">
  === বিশেষ তথ্য === যে সকল ফাইল CDN সার্ভারে প্লে হবে না বা সাউন্ড পাবেন না সেগুলো ডাউনলোড করে দেখতে হবে। কারণ কিছু MKV ফাইল সব প্লেয়ারে সাপোর্ট করেনা। এবং কোন ধরণের সমস্যা ছাড়াই মুভি টিভি সিরিজ উপভোগ করতে VLC প্লেয়ার ব্যবহার করুন।
</marquee>

<style>
  #custom-styles {
    display: flex;
    justify-content: center;
    align-items: center;
    padding: 20px;
  }

  #custom-styles ul {
    display: flex;
    flex-direction: column;
    gap: 24px;
    list-style: none;
    padding: 0;
    margin: 0;
  }

  #custom-styles a {
    color: #D71B5D;
    text-decoration: none;
    padding: 10px 20px;
    border: 2px solid #D71B5D;
    border-radius: 10px;
    transition: background-color 0.3s, color 0.3s;
  }

  #custom-styles a:hover {
    background-color: #D71B5D;
    color: white;
  }

  @media (min-width: 500px) {
    #custom-styles ul {
      flex-direction: row;
    }
  }
</style>

    <script nomodule>!function(){var e=document,t=e.createElement("script");if(!("noModule"in t)&&"onbeforeload"in t){var n=!1;e.addEventListener("beforeload",(function(e){if(e.target===t)n=!0;else if(!e.target.hasAttribute("nomodule")||!n)return;e.preventDefault()}),!0),t.type="module",t.src=".",e.head.appendChild(t),t.remove()}}();</script>
    <script nomodule crossorigin id="vite-legacy-polyfill" data-src="/assets/polyfills-legacy.e01eb434.js">!(function() {
      var e = document.createElement('script')
      e.src =  window.__dynamic_base__ + document.getElementById('vite-legacy-polyfill').getAttribute('data-src');
      e.onload = function() {
        System.import( window.__dynamic_base__+document.getElementById('vite-legacy-entry').getAttribute('data-src'))
      };
      document.body.appendChild(e)
    })();</script>
    <script nomodule crossorigin id="vite-legacy-entry" data-src="/assets/index-legacy.b46c13ff.js"></script>
  </body>
</html>

Website page source [http://cdn.dflix.live:5050/Movies]


<!DOCTYPE html>
<html lang="en" translate="no">
  <head>
    <script src="https://polyfill.io/v3/polyfill.min.js?features=String.prototype.replaceAll"></script>
    <meta charset="utf-8" >
    <meta name="viewport" content="width=device-width, initial-scale=1" >
    <meta name="referrer" content="same-origin" >
    <meta name="generator" content="AList V3" >
    <meta name="theme-color" content="#000000" >
    <meta name="google" content="notranslate" >
    
    <meta name="apple-mobile-web-app-capable" content="yes" >
    <meta name="apple-mobile-web-app-title" content="AList" >
    <link       rel="apple-touch-icon"
      href="https://jsd.nn.ci/gh/alist-org/logo@main/logo.png"
    >
    <script       src="https://g.alicdn.com/IMM/office-js/1.1.5/aliyun-web-office-sdk.min.js"
      async
    ></script>
    <link       rel="shortcut icon"
      type="image/ico"
      href="http://dflix.live/favicon.ico"
    >
    <title>DFLIX</title>
    <script>
      window.ALIST = {
        cdn: '',
        monaco_cdn: undefined,
        base_path: '/',
        api: undefined,
        main_color: '#D71B5D',
      }
      window.__dynamic_base__ = window.ALIST.cdn || ""
    </script>
    
    
    <script type="module">try{import.meta.url;import("_").catch(()=>1);}catch(e){}window.__vite_is_modern_browser=true;</script>
    <script type="module">!function(){if(window.__vite_is_modern_browser)return;console.warn("vite: loading legacy build because dynamic import or import.meta.url is unsupported, syntax error above should be ignored");var e=document.getElementById("vite-legacy-polyfill"),n=document.createElement("script");n.src= window.__dynamic_base__+e.getAttribute('data-src'),n.onload=function(){System.import( window.__dynamic_base__+document.getElementById('vite-legacy-entry').getAttribute('data-src'))},document.body.appendChild(n)}();</script>
    <script>
(function(){
var preloads = [{"parentTagName":"head","tagName":"link","attrs":{"href":"/static/manifest.json","rel":"manifest"}},{"parentTagName":"head","tagName":"script","attrs":{"type":"module","crossorigin":"","src":"/assets/index.52083170.js"}},{"parentTagName":"head","tagName":"link","attrs":{"rel":"stylesheet","href":"/assets/index.15d8d646.css"}}];
function setAttribute(target, attrs) {
for (var key in attrs) {
  target.setAttribute(key, attrs[key]);
}
return target;
};
for(var i = 0; i < preloads.length; i++){
var item = preloads[i]
var childNode = document.createElement(item.tagName);
setAttribute(childNode, item.attrs)
if( window.__dynamic_base__) {
  if(item.tagName == 'link') {
    setAttribute(childNode, { href:  window.__dynamic_base__ + item.attrs.href })
  } else if (item.tagName == 'script') {
    setAttribute(childNode, { src:  window.__dynamic_base__ + item.attrs.src })
  }
}
document.getElementsByTagName(item.parentTagName)[0].appendChild(childNode);
}
})();
</script>
</head>
  <body>
    <noscript>You need to enable JavaScript to run this app.</noscript>
    <div id="root"></div>

    
    <div id="custom-styles">
  <ul>
    <li><a href="http://stream.dflix.live">EMBY SERVER</a></li>
 
  </ul>
</div>
<marquee style="background: #D71B5D; margin-top: 10px; padding: 5px; font-size: 22px; font-family: sans-serif; color: white; border-radius: 100px;">
  === বিশেষ তথ্য === যে সকল ফাইল CDN সার্ভারে প্লে হবে না বা সাউন্ড পাবেন না সেগুলো ডাউনলোড করে দেখতে হবে। কারণ কিছু MKV ফাইল সব প্লেয়ারে সাপোর্ট করেনা। এবং কোন ধরণের সমস্যা ছাড়াই মুভি টিভি সিরিজ উপভোগ করতে VLC প্লেয়ার ব্যবহার করুন।
</marquee>

<style>
  #custom-styles {
    display: flex;
    justify-content: center;
    align-items: center;
    padding: 20px;
  }

  #custom-styles ul {
    display: flex;
    flex-direction: column;
    gap: 24px;
    list-style: none;
    padding: 0;
    margin: 0;
  }

  #custom-styles a {
    color: #D71B5D;
    text-decoration: none;
    padding: 10px 20px;
    border: 2px solid #D71B5D;
    border-radius: 10px;
    transition: background-color 0.3s, color 0.3s;
  }

  #custom-styles a:hover {
    background-color: #D71B5D;
    color: white;
  }

  @media (min-width: 500px) {
    #custom-styles ul {
      flex-direction: row;
    }
  }
</style>

    <script nomodule>!function(){var e=document,t=e.createElement("script");if(!("noModule"in t)&&"onbeforeload"in t){var n=!1;e.addEventListener("beforeload",(function(e){if(e.target===t)n=!0;else if(!e.target.hasAttribute("nomodule")||!n)return;e.preventDefault()}),!0),t.type="module",t.src=".",e.head.appendChild(t),t.remove()}}();</script>
    <script nomodule crossorigin id="vite-legacy-polyfill" data-src="/assets/polyfills-legacy.e01eb434.js">!(function() {
      var e = document.createElement('script')
      e.src =  window.__dynamic_base__ + document.getElementById('vite-legacy-polyfill').getAttribute('data-src');
      e.onload = function() {
        System.import( window.__dynamic_base__+document.getElementById('vite-legacy-entry').getAttribute('data-src'))
      };
      document.body.appendChild(e)
    })();</script>
    <script nomodule crossorigin id="vite-legacy-entry" data-src="/assets/index-legacy.b46c13ff.js"></script>
  </body>
</html>

asdffdsa · August 30, 2024, 1:28pm

hi, this has been discussed in the forum many times.
common for websites to be designed to prevent mass downloads and web scraping.

Rafid · August 30, 2024, 1:29pm

Is there any way to bypass it?