Im using a python script to move my files to cloud, script freeze some time during moving, maybe someone help me to find real issue,
from datetime import datetime
import os
from rich import print
import shutil
from rclone_python import rclone
import concurrent.futures
import time
import random
PATH_TO_INSPECT = "/www/wwwroot/videos/rclone2/tmp"
# PATH_TO_INSPECT = 'tmp'
MINIMUM_SECONDS_SINCE_LAST_MODIFICATION=10
TEMP_LOCATION_BASE = "/www/wwwroot/videos/rclone2/move"
# TEMP_LOCATION_BASE = 'move'
SLEEP_IN_SECONDS=20
PARALLEL_THREAD = 3
REMOTE_LIST = ['box1', 'box2', 'box3', 'box4', 'box5', 'box6', 'box7', 'box8', 'box9', 'box10']
TAB = []
def move_and_rclone(BASE_NAME):
global TAB
if len(TAB) == len(REMOTE_LIST):
TAB.clear()
DATE = datetime.now().strftime('%Y/%m')
OUTPUT_DIR = TEMP_LOCATION_BASE
if not os.path.exists(OUTPUT_DIR):
os.mkdir(OUTPUT_DIR)
print(f'[green][+][/green] moving {BASE_NAME} to {OUTPUT_DIR}/{BASE_NAME}')
source_folder = f'{PATH_TO_INSPECT}/{BASE_NAME}'
destination_folder = f'{OUTPUT_DIR}/{BASE_NAME}'
shutil.move(source_folder, destination_folder)
while True:
for i in range(len(REMOTE_LIST)):
selected_remote = REMOTE_LIST[i]
if selected_remote not in TAB:
TAB.append(selected_remote)
break
check_remote_exists = rclone.check_remote_existing(selected_remote)
if check_remote_exists:
print(f'[green][+][/green] Remote [yellow]{selected_remote}[/yellow] selected for {BASE_NAME}')
break
else:
print(f'[red][-][/red] Remote [yellow]{selected_remote}[/yellow] does not exists')
DROPBOX_PATH = f'{selected_remote}:box/{DATE}/{BASE_NAME}'
print(f'[green][+][/green] rcloning {destination_folder} -> {DROPBOX_PATH}')
rclone.move(destination_folder,DROPBOX_PATH, ignore_existing=False, args=['--delete-empty-src-dirs','--tpslimit 12','--tpslimit-burst 12','--transfers 64','--dropbox-batch-size 1000','--dropbox-batch-timeout 10s','--checkers 32','--checksum','--log-file /www/wwwroot/videos/rclone2/rclone2.log'])
if __name__=="__main__":
while True:
tmp_folders = os.listdir(PATH_TO_INSPECT)
with concurrent.futures.ThreadPoolExecutor(max_workers=PARALLEL_THREAD) as executor:
executor.map(move_and_rclone, tmp_folders)
move_folders = os.listdir(TEMP_LOCATION_BASE)
for move_folder in move_folders:
move_full_dir = f'{TEMP_LOCATION_BASE}/{move_folder}'
move_files = os.listdir(move_full_dir)
if len(move_files) == 0:
os.rmdir(move_full_dir)
print('\n')
print(f'[green][+][/green] Sleeping for {SLEEP_IN_SECONDS} seconds')
time.sleep(SLEEP_IN_SECONDS)
Thanks