initial commit - bare uploading works
This commit is contained in:
commit
b5b4579d74
|
@ -0,0 +1 @@
|
||||||
|
__version__ = "0.0.0"
|
|
@ -0,0 +1,197 @@
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
from threading import Thread
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from collections import namedtuple
|
||||||
|
from b2.api import B2Api
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, Future
|
||||||
|
"""
|
||||||
|
How it works:
|
||||||
|
|
||||||
|
B2SyncManager manages the transfer
|
||||||
|
|
||||||
|
It holes a src and dest object, src objects provide an iterable of FileInfos.
|
||||||
|
|
||||||
|
The manager will iterate the set of FileInfos, and pass each to the dest
|
||||||
|
|
||||||
|
Dest will upload the file, and inform the manager it was completed
|
||||||
|
|
||||||
|
"""
|
||||||
|
class B2SyncManager(object):
|
||||||
|
|
||||||
|
threads = 5
|
||||||
|
|
||||||
|
def __init__(self, source_module, dest_module):
|
||||||
|
self.src = source_module
|
||||||
|
self.dest = dest_module
|
||||||
|
self.db = sqlite3.connect('./sync.db')
|
||||||
|
self.db.row_factory = B2SyncManager.dict_factory
|
||||||
|
self.db.isolation_level = None # TBD - does it hurt perf?
|
||||||
|
self._init_db()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def dict_factory(cursor, row):
|
||||||
|
d = {}
|
||||||
|
for idx, col in enumerate(cursor.description):
|
||||||
|
d[col[0]] = row[idx]
|
||||||
|
return d
|
||||||
|
|
||||||
|
def _init_db(self):
|
||||||
|
c = self.db.cursor()
|
||||||
|
|
||||||
|
def table_exists(table_name):
|
||||||
|
c.execute("SELECT * FROM SQLITE_MASTER WHERE `type`='table' AND `name`=?", (table_name,))
|
||||||
|
tables = c.fetchall()
|
||||||
|
if len(tables)==0:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
tables = {
|
||||||
|
"files": """
|
||||||
|
CREATE TABLE `files` (
|
||||||
|
`path` varchar(1024),
|
||||||
|
`mtime` INTEGER,
|
||||||
|
`size` INTEGER
|
||||||
|
);"""
|
||||||
|
}
|
||||||
|
|
||||||
|
for table_name, table_create_query in tables.items():
|
||||||
|
if not table_exists(table_name):
|
||||||
|
c.execute(table_create_query)
|
||||||
|
|
||||||
|
c.close()
|
||||||
|
|
||||||
|
def sync(self):
|
||||||
|
print("Syncing from {} to {}".format(self.src, self.dest))
|
||||||
|
for f in self.src:
|
||||||
|
c = self.db.cursor()
|
||||||
|
|
||||||
|
row = c.execute("SELECT * FROM 'files' WHERE `path` = ?;", (f.rel_path,)).fetchone()
|
||||||
|
|
||||||
|
if not row or row['mtime'] < f.mtime:
|
||||||
|
|
||||||
|
print("Uploading: ", f.rel_path, end='')
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
self.dest.put_file(f)
|
||||||
|
f.fp.close()
|
||||||
|
|
||||||
|
print(" ok")
|
||||||
|
|
||||||
|
# The file was uploaded, commit it to the db
|
||||||
|
|
||||||
|
c.execute("INSERT INTO 'files' VALUES(?, ?, ?);", (f.rel_path, f.mtime, f.size,))
|
||||||
|
|
||||||
|
else:
|
||||||
|
print("Skipping: ", f.rel_path)
|
||||||
|
|
||||||
|
c.close()
|
||||||
|
|
||||||
|
|
||||||
|
class Provider(object):
|
||||||
|
"""
|
||||||
|
Base class file queue iterable
|
||||||
|
"""
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
FileInfo = namedtuple('FileInfo', ['abs_path', 'rel_path', 'size', 'mtime', 'fp'])
|
||||||
|
|
||||||
|
class LocalProvider(Provider):
|
||||||
|
"""
|
||||||
|
Iterates files on local disk
|
||||||
|
"""
|
||||||
|
max_chunk_size = 8*1024*1024
|
||||||
|
def __init__(self, local_path):
|
||||||
|
super(LocalProvider, self).__init__()
|
||||||
|
self.local_path = local_path
|
||||||
|
self.current_set = (None, [], [])
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
self.walker = os.walk(self.local_path)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __next__(self):
|
||||||
|
if len(self.current_set[2]) > 0:
|
||||||
|
file_abs_path = os.path.join(self.current_set[0], self.current_set[2].pop())
|
||||||
|
relative_path = file_abs_path[len(self.local_path):]
|
||||||
|
return FileInfo(
|
||||||
|
file_abs_path,
|
||||||
|
relative_path,
|
||||||
|
os.path.getsize(file_abs_path),
|
||||||
|
int(os.path.getmtime(file_abs_path)),
|
||||||
|
open(file_abs_path, 'rb')
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.current_set = self.walker.__next__()
|
||||||
|
self.current_set[1].sort(reverse=True)
|
||||||
|
self.current_set[2].sort(reverse=True)
|
||||||
|
return self.__next__()
|
||||||
|
|
||||||
|
|
||||||
|
class B2Provider(Provider):
|
||||||
|
"""
|
||||||
|
Iterates files in bucket
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, accountId, appKey, bucketId, bucketBasePath):
|
||||||
|
super(B2Provider, self).__init__()
|
||||||
|
raise NotImplemented()
|
||||||
|
|
||||||
|
|
||||||
|
class Reciever(object):
|
||||||
|
"""
|
||||||
|
Base class for destinations
|
||||||
|
"""
|
||||||
|
|
||||||
|
class B2Reciever(Reciever):
|
||||||
|
|
||||||
|
max_chunk_size = 256*1024
|
||||||
|
|
||||||
|
def __init__(self, bucket, path, account_id, app_key):
|
||||||
|
super(B2Reciever, self).__init__()
|
||||||
|
self.bucket = bucket
|
||||||
|
self.path = path
|
||||||
|
self.account_id = account_id
|
||||||
|
self.app_key = app_key
|
||||||
|
|
||||||
|
self.api = B2Api()
|
||||||
|
self.api.authorize_account('production', self.account_id, self.app_key)
|
||||||
|
self.bucket = self.api.get_bucket_by_name(self.bucket)
|
||||||
|
|
||||||
|
def put_file(self, file_info):
|
||||||
|
#print(">>> {}".format(file_info.abs_path))
|
||||||
|
dest_path = os.path.join(self.path, file_info.rel_path).lstrip('/')
|
||||||
|
self.bucket.upload_local_file(
|
||||||
|
file_info.abs_path,
|
||||||
|
dest_path
|
||||||
|
)
|
||||||
|
|
||||||
|
def sync(source_uri, dest_uri, account_id, app_key):
|
||||||
|
source = urlparse(source_uri)
|
||||||
|
dest = urlparse(dest_uri)
|
||||||
|
|
||||||
|
syncer = B2SyncManager(source_uri, dest_uri)
|
||||||
|
|
||||||
|
source_provider = None
|
||||||
|
dest_receiver = None
|
||||||
|
|
||||||
|
if source.scheme == '': # Plain file URI
|
||||||
|
source_provider = LocalProvider(source.path)
|
||||||
|
else:
|
||||||
|
raise Exception("Sources other than local file paths not supported")
|
||||||
|
|
||||||
|
if dest.scheme == 'b2': # Plain file URI
|
||||||
|
dest_receiver = B2Reciever(bucket=dest.netloc, path=dest.path, account_id=account_id, app_key=app_key)
|
||||||
|
else:
|
||||||
|
raise Exception("Dests other than B2 URIs not yet supported")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
assert source_provider is not None
|
||||||
|
assert dest_receiver is not None
|
||||||
|
|
||||||
|
syncer = B2SyncManager(source_provider, dest_receiver)
|
||||||
|
syncer.sync()
|
|
@ -0,0 +1,20 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from b2mirror import mirror
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Sync data to/from B2")
|
||||||
|
|
||||||
|
parser.add_argument("-s", "--source", required=True, help="Source URI")
|
||||||
|
parser.add_argument("-d", "--dest", required=True, help="Dest URI")
|
||||||
|
|
||||||
|
parser.add_argument("-a", "--account-id", required=True, help="Backblaze account ID")
|
||||||
|
parser.add_argument("-k", "--app-key", required=True, help="Backblaze application key")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
mirror.sync(args.source, args.dest, args.account_id, args.app_key)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -0,0 +1,13 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from setuptools import setup
|
||||||
|
from b2mirror import __version__
|
||||||
|
|
||||||
|
setup(name='b2mirror',
|
||||||
|
version=__version__,
|
||||||
|
description='Tool for syc',
|
||||||
|
url='http://gitlab.xmopx.net/dave/b2mirror',
|
||||||
|
author='dpedu',
|
||||||
|
author_email='dave@davepedu.com',
|
||||||
|
packages=['b2mirror'],
|
||||||
|
scripts=['bin/b2mirror'],
|
||||||
|
zip_safe=False)
|
Loading…
Reference in New Issue