From aafe1a05ace27720b65ecffe29c6b30fbee8ab7d Mon Sep 17 00:00:00 2001 From: dave Date: Sun, 24 Jul 2016 13:00:32 -0700 Subject: [PATCH] clean up logging and interfaces --- b2mirror/b2plugin.py | 8 ++++---- b2mirror/base.py | 14 ++++++++++++-- b2mirror/mirror.py | 18 ++++++++++++++++-- bin/b2mirror | 7 ++++++- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/b2mirror/b2plugin.py b/b2mirror/b2plugin.py index 82acd1f..1f0a86a 100644 --- a/b2mirror/b2plugin.py +++ b/b2mirror/b2plugin.py @@ -146,13 +146,13 @@ class B2Reciever(Reciever): row = c.execute("SELECT * FROM 'files' WHERE `path` = ?;", (f.rel_path,)).fetchone() if self.should_transfer(row, f): - print("Uploading:", f.rel_path) + self.log.info("Uploading: %s", f.rel_path) try: # upload the file. if a row existed it means there may be historic copies of the file already there result = self.put_file(f, purge_historics=row is not None) except: - print("Failed:", f.rel_path) - print("Unexpected error:", sys.exc_info()[0]) + self.log.error("Failed:", f.rel_path) + self.log.error("Unexpected error:", sys.exc_info()[0]) raise # The file was uploaded, commit it to the db @@ -181,7 +181,7 @@ class B2Reciever(Reciever): with closing(self.db.cursor()) as c_del: for purge_file in c.execute("SELECT * FROM 'files' WHERE seen=0;"): - print("Delete on remote: ", purge_file["path"]) + self.log.info("Delete on remote: %s", purge_file["path"]) self.purge_file(purge_file["path"]) c_del.execute("DELETE FROM 'files' WHERE path=?;", (purge_file["path"],)) diff --git a/b2mirror/base.py b/b2mirror/base.py index 847077d..1e97256 100644 --- a/b2mirror/base.py +++ b/b2mirror/base.py @@ -19,10 +19,20 @@ class Reciever(object): """ Base class for destinations """ - def put_file(self, file_info, purge_historics=False): + def xfer_file(self, f): + """ + Future-called function that handles a single file. This method should: + - check if the file is eligible for transfer (not in ignore list etc) + - return Result.skipped if skipped + - transfer the file and return Result.ok or Result.failed + """ raise NotImplemented() - def purge_file(self, file_path): + def purge(self): + """ + Delete files on the remote that were not found when scanning the local tree. This will always be called AFTER + scanning through the local tree and calling xfer_file for every file. + """ raise NotImplemented() def teardown(self): diff --git a/b2mirror/mirror.py b/b2mirror/mirror.py index 898ebbc..d5a0cf2 100644 --- a/b2mirror/mirror.py +++ b/b2mirror/mirror.py @@ -46,6 +46,8 @@ class B2SyncManager(object): self.workers = workers + self.log.info("Initialized with %s workers, %s ignores", self.workers, len(self.exclude_res)) + def sync(self): """ Sync the source to the dest. First uploads new local files, then cleans dead files from the remote. @@ -55,14 +57,15 @@ class B2SyncManager(object): # Phase 2 - Delete files on the remote missing locally self.purge_remote() # Phase 3 - Tear down the src/dest modules - self.src.teardown() - self.dest.teardown() + self.cleanup() def sync_up(self): """ Sync local files to the remote. All files in the DB will be marked as unseen. When a file is found locally it is again marked as seen. This state later used to clear deleted files from the destination """ + self.log.info("beginning upload phase") + chunk_size = 1000 # if rel_path matches any of the REs, the filter is True and the file is skipped @@ -89,10 +92,18 @@ class B2SyncManager(object): During upload phase it is expected that destination modules track state of what files have been seen on the local end. When local scan + upload is complete, the module uses this state to purge dead files on the remote. """ + self.log.info("beginning remote purge phase") self.dest.purge() + def cleanup(self): + self.log.info("beginning cleanp phase") + self.src.teardown() + self.dest.teardown() + def sync(source_uri, dest_uri, account_id, app_key, workers=10, exclude=[], compare_method="mtime"): + log = logging.getLogger("mirror") + source = urlparse(source_uri) dest = urlparse(dest_uri) @@ -113,6 +124,9 @@ def sync(source_uri, dest_uri, account_id, app_key, workers=10, exclude=[], comp assert source_provider is not None assert dest_receiver is not None + log.info("Source: %s", source_provider) + log.info("Dest: %s", dest_receiver) + syncer = B2SyncManager(source_provider, dest_receiver, workers=workers, exclude_res=exclude) syncer.sync() diff --git a/bin/b2mirror b/bin/b2mirror index c8d210d..64a302c 100755 --- a/bin/b2mirror +++ b/bin/b2mirror @@ -8,10 +8,13 @@ import logging def main(): - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.DEBUG, + format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") logging.getLogger("requests").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) + log = logging.getLogger("main") + parser = argparse.ArgumentParser(description="Sync data to/from B2") parser.add_argument("-i", "--size", help="Compare by size instead of mtime", action="store_true", default=False) @@ -29,6 +32,8 @@ def main(): ignore_res = [re.compile(i) for i in args.exclude] if args.exclude else [] + log.debug("Running with params: %s", args) + mirror.sync( args.source, args.dest,