Code cleanup
This commit is contained in:
parent
a2922b90a9
commit
4b958968d1
|
@ -1,12 +1,11 @@
|
|||
import os
|
||||
import sqlite3
|
||||
from threading import Thread
|
||||
from urllib.parse import urlparse
|
||||
from collections import namedtuple
|
||||
from b2.api import B2Api
|
||||
import sys
|
||||
from itertools import islice, filterfalse
|
||||
from concurrent.futures import ThreadPoolExecutor, Future
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import re
|
||||
|
||||
# import logging
|
||||
|
@ -27,9 +26,11 @@ The manager will iterate the set of FileInfos, and pass each to the dest
|
|||
Dest will upload the file, and inform the manager it was completed
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class B2SyncManager(object):
|
||||
|
||||
workers = 10
|
||||
workers = 15
|
||||
|
||||
def __init__(self, source_module, dest_module, exclude_res=None):
|
||||
self.src = source_module
|
||||
|
@ -88,15 +89,14 @@ class B2SyncManager(object):
|
|||
# Files will be marked as seen as they are processed
|
||||
# Later, unseen files will be purged
|
||||
c = self.db.cursor()
|
||||
row = c.execute("UPDATE 'files' SET seen=0;")
|
||||
c.execute("UPDATE 'files' SET seen=0;")
|
||||
c.close()
|
||||
|
||||
chunk_size = 1000
|
||||
|
||||
files_source = filterfalse( # if rel_path matches any of the REs, the filter is True and the file is skipped
|
||||
lambda x: any([pattern.match(x.rel_path) for pattern in self.exclude_res]),
|
||||
self.src
|
||||
)
|
||||
self.src)
|
||||
|
||||
while True:
|
||||
chunk = list(islice(files_source, chunk_size))
|
||||
|
@ -119,7 +119,6 @@ class B2SyncManager(object):
|
|||
|
||||
row = c.execute("SELECT * FROM 'files' WHERE `path` = ?;", (f.rel_path,)).fetchone()
|
||||
|
||||
|
||||
if not row or row['mtime'] < f.mtime:
|
||||
|
||||
print("Uploading:", f.rel_path)
|
||||
|
@ -175,6 +174,7 @@ class LocalProvider(Provider):
|
|||
Iterates files on local disk
|
||||
"""
|
||||
max_chunk_size = 8*1024*1024
|
||||
|
||||
def __init__(self, local_path):
|
||||
super(LocalProvider, self).__init__()
|
||||
self.local_path = local_path
|
||||
|
@ -219,6 +219,7 @@ class Reciever(object):
|
|||
def purge_file(self, file_path):
|
||||
raise NotImplemented()
|
||||
|
||||
|
||||
class B2Reciever(Reciever):
|
||||
|
||||
max_chunk_size = 256*1024
|
||||
|
@ -235,7 +236,6 @@ class B2Reciever(Reciever):
|
|||
self.bucket = self.api.get_bucket_by_name(self.bucket_name)
|
||||
|
||||
def put_file(self, file_info):
|
||||
#print(">>> {}".format(file_info.abs_path))
|
||||
dest_path = os.path.join(self.path, file_info.rel_path).lstrip('/')
|
||||
self.bucket.upload_local_file(
|
||||
file_info.abs_path,
|
||||
|
@ -246,7 +246,6 @@ class B2Reciever(Reciever):
|
|||
dest_path = os.path.join(self.path, file_path).lstrip('/')
|
||||
self.delete_by_path(dest_path)
|
||||
|
||||
|
||||
def delete_by_path(self, file_path):
|
||||
for f in self.bucket.list_file_versions(start_filename=file_path, max_entries=100)["files"]:
|
||||
if f["fileName"] == file_path:
|
||||
|
|
Loading…
Reference in New Issue