Rclone Move With Python Wrapper

Im using a python script to move my files to cloud, script freeze some time during moving, maybe someone help me to find real issue,

from datetime import datetime
import os
from rich import print
import shutil
from rclone_python import rclone
import concurrent.futures
import time
import random

PATH_TO_INSPECT = "/www/wwwroot/videos/rclone2/tmp"
# PATH_TO_INSPECT = 'tmp'
MINIMUM_SECONDS_SINCE_LAST_MODIFICATION=10
TEMP_LOCATION_BASE = "/www/wwwroot/videos/rclone2/move"
# TEMP_LOCATION_BASE = 'move'
SLEEP_IN_SECONDS=20
PARALLEL_THREAD = 3
REMOTE_LIST = ['box1', 'box2', 'box3', 'box4', 'box5', 'box6', 'box7', 'box8', 'box9', 'box10']
TAB = []

def move_and_rclone(BASE_NAME):
    global TAB
    if len(TAB) == len(REMOTE_LIST):
        TAB.clear()

    DATE = datetime.now().strftime('%Y/%m')
    OUTPUT_DIR = TEMP_LOCATION_BASE
    if not os.path.exists(OUTPUT_DIR):
        os.mkdir(OUTPUT_DIR)
    print(f'[green][+][/green] moving {BASE_NAME} to {OUTPUT_DIR}/{BASE_NAME}')
    source_folder = f'{PATH_TO_INSPECT}/{BASE_NAME}'
    destination_folder = f'{OUTPUT_DIR}/{BASE_NAME}'
    shutil.move(source_folder, destination_folder)

    while True:
        for i in range(len(REMOTE_LIST)):
            selected_remote = REMOTE_LIST[i]
            if selected_remote not in TAB:
                TAB.append(selected_remote)
                break
        check_remote_exists = rclone.check_remote_existing(selected_remote)
        if check_remote_exists:
            print(f'[green][+][/green] Remote [yellow]{selected_remote}[/yellow] selected for {BASE_NAME}')
            break
        else:
            print(f'[red][-][/red] Remote [yellow]{selected_remote}[/yellow] does not exists')
    DROPBOX_PATH = f'{selected_remote}:box/{DATE}/{BASE_NAME}'
    print(f'[green][+][/green] rcloning {destination_folder} -> {DROPBOX_PATH}')
    rclone.move(destination_folder,DROPBOX_PATH, ignore_existing=False, args=['--delete-empty-src-dirs','--tpslimit 12','--tpslimit-burst 12','--transfers 64','--dropbox-batch-size 1000','--dropbox-batch-timeout 10s','--checkers 32','--checksum','--log-file /www/wwwroot/videos/rclone2/rclone2.log'])

if __name__=="__main__":
    while True:
        tmp_folders = os.listdir(PATH_TO_INSPECT)
        with concurrent.futures.ThreadPoolExecutor(max_workers=PARALLEL_THREAD) as executor:
            executor.map(move_and_rclone, tmp_folders)

        move_folders = os.listdir(TEMP_LOCATION_BASE)
        for move_folder in move_folders:
            move_full_dir = f'{TEMP_LOCATION_BASE}/{move_folder}'
            move_files = os.listdir(move_full_dir)
            if len(move_files) == 0:
                os.rmdir(move_full_dir)

        print('\n')
        print(f'[green][+][/green] Sleeping for {SLEEP_IN_SECONDS} seconds')
        time.sleep(SLEEP_IN_SECONDS)



    

Thanks

Hi Marc,

Moved your post to "Help and support", the "How to" category is for guides on how to do something.

1 Like

please post
---the output of rclone version
--- the rclone debug log.

for debug log do i need to add -vv at the end of rclone move cmd line?

Thanks

@Yamiraan,
please answer the all questions in the help and support template.