Command line client for automated backups
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

223 lines
7.9 KiB

  1. #!/usr/bin/env python3
  2. import argparse
  3. from configparser import ConfigParser
  4. from urllib.parse import urlparse
  5. from os.path import normpath, join, exists
  6. from os import chmod, chown, stat
  7. from enum import Enum
  8. import subprocess
  9. from requests import get,put
  10. SSH_KEY_PATH = '/root/.ssh/datadb.key'
  11. RSYNC_DEFAULT_ARGS = ['rsync', '-avzr', '--exclude=.datadb.lock', '--whole-file', '--one-file-system', '--delete', '-e', 'ssh -i {} -p 4874 -o StrictHostKeyChecking=no'.format(SSH_KEY_PATH)]
  12. DATADB_HTTP_API = 'http://datadb.services.davepedu.com:4875/cgi-bin/'
  13. class SyncStatus(Enum):
  14. "Data is on local disk"
  15. DATA_AVAILABLE = 1
  16. "Data is not on local disk"
  17. DATA_MISSING = 2
  18. def restore(profile, conf, force=False): #remote_uri, local_dir, identity='/root/.ssh/datadb.key'
  19. """
  20. Restore data from datadb
  21. """
  22. # Sanity check: If the lockfile exists we assume the data is already there, so we wouldn't want to call rsync again
  23. # as it would wipe out local changes. This can be overridden with --force
  24. assert (status(profile, conf) == SyncStatus.DATA_MISSING) or force, "Data already exists (Use --force?)"
  25. original_perms = stat(conf["dir"])
  26. dest = urlparse(conf["uri"])
  27. if dest.scheme == 'rsync':
  28. args = RSYNC_DEFAULT_ARGS[:]
  29. # Request backup server to prepare the backup, the returned dir is what we sync from
  30. rsync_path = get(DATADB_HTTP_API+'get_backup', params={'proto':'rsync', 'name':profile}).text.rstrip()
  31. # Add rsync source path
  32. args.append('nexus@{}:{}'.format(dest.netloc, normpath(rsync_path)+'/'))
  33. # Add local dir
  34. args.append(normpath(conf["dir"])+'/')
  35. print("Rsync restore call: {}".format(' '.join(args)))
  36. subprocess.check_call(args)
  37. elif dest.scheme == 'archive':
  38. # http request backup server
  39. # download tarball
  40. args_curl = ['curl', '-s', '-v', '-XGET', '{}get_backup?proto=archive&name={}'.format(DATADB_HTTP_API, profile)]
  41. # unpack
  42. args_tar = ['tar', 'zxv', '-C', normpath(conf["dir"])+'/']
  43. print("Tar restore call: {} | {}".format(' '.join(args_curl), ' '.join(args_tar)))
  44. dl = subprocess.Popen(args_curl, stdout=subprocess.PIPE)
  45. extract = subprocess.Popen(args_tar, stdin=dl.stdout)
  46. dl.wait()
  47. extract.wait()
  48. # TODO: convert to pure python?
  49. assert dl.returncode == 0, "Could not download archive"
  50. assert extract.returncode == 0, "Could not extract archive"
  51. # Restore original permissions on data dir
  52. # TODO store these in conf file
  53. chmod(conf["dir"], original_perms.st_mode)
  54. chown(conf["dir"], original_perms.st_uid, original_perms.st_gid)
  55. # TODO apply other permissions
  56. def backup(profile, conf, force=False):
  57. """
  58. Backup data to datadb
  59. """
  60. # Sanity check: If the lockfile doesn't exist we assume the data is missing, so we wouldn't want to call rsync
  61. # again as it would wipe out the backup.
  62. assert (status(profile, conf) == SyncStatus.DATA_AVAILABLE) or force, "Data is missing (Use --force?)"
  63. dest = urlparse(conf["uri"])
  64. if dest.scheme == 'rsync':
  65. args = RSYNC_DEFAULT_ARGS[:]
  66. # Add local dir
  67. args.append(normpath(conf["dir"])+'/')
  68. # Hit backupdb via http to retreive absolute path of rsync destination of remote server
  69. rsync_path = get(DATADB_HTTP_API+'new_backup', params={'proto':'rsync', 'name':profile, 'keep':conf["keep"]}).text.rstrip()
  70. # Add rsync source path
  71. args.append(normpath('nexus@{}:{}'.format(dest.netloc, rsync_path))+'/')
  72. #print("Rsync backup call: {}".format(' '.join(args)))
  73. subprocess.check_call(args)
  74. elif dest.scheme == 'archive':
  75. # CD to local source dir
  76. # create tarball
  77. # http PUT file to backup server
  78. args_tar = ['tar', '--exclude=.datadb.lock', '-zcv', './']
  79. args_curl = ['curl', '-v', '-XPUT', '--data-binary', '@-', '{}new_backup?proto=archive&name={}&keep={}'.format(DATADB_HTTP_API, profile, conf["keep"])]
  80. print("Tar backup call: {} | {}".format(' '.join(args_tar), ' '.join(args_curl)))
  81. compress = subprocess.Popen(args_tar, stdout=subprocess.PIPE, cwd=normpath(conf["dir"])+'/')
  82. upload = subprocess.Popen(args_curl, stdin=compress.stdout)
  83. compress.wait()
  84. upload.wait()
  85. # TODO: convert to pure python?
  86. assert compress.returncode == 0, "Could not create archive"
  87. assert upload.returncode == 0, "Could not upload archive"
  88. def status(profile, conf):
  89. """
  90. Check status of local dir - if the lock file is in place, we assume the data is there
  91. """
  92. lockfile = join(conf["dir"], '.datadb.lock')
  93. if exists(lockfile):
  94. return SyncStatus.DATA_AVAILABLE
  95. return SyncStatus.DATA_MISSING
  96. def main():
  97. """
  98. Excepts a config file at /etc/datadb.ini. Example:
  99. ----------------------------
  100. [gyfd]
  101. uri=
  102. dir=
  103. keep=
  104. auth=
  105. restore_preexec=
  106. restore_postexec=
  107. export_preexec=
  108. export_postexec=
  109. ----------------------------
  110. Each [section] defines one backup task.
  111. Fields:
  112. *uri*: Destination/source for this instance's data. Always fits the following format:
  113. <procotol>://<server>/<backup name>
  114. Valid protocols:
  115. rsync - rsync executed over SSH. The local dir will be synced with the remote backup dir using rsync.
  116. archive - tar archives transported over HTTP. The local dir will be tarred and PUT to the backup server's remote dir via http.
  117. *dir*: Local dir for this backup
  118. *keep*: Currently unused. Number of historical copies to keep on remote server
  119. *auth*: Currently unused. Username:password string to use while contacting the datadb via HTTP.
  120. *restore_preexec*: Shell command to exec before pulling/restoring data
  121. *restore_postexec*: Shell command to exec after pulling/restoring data
  122. *export_preexec*: Shell command to exec before pushing data
  123. *export_postexec*: Shell command to exec after pushing data
  124. """
  125. # Load profiles
  126. config = ConfigParser()
  127. config.read("/etc/datadb.ini")
  128. config = {section:{k:config[section][k] for k in config[section]} for section in config.sections()}
  129. parser = argparse.ArgumentParser(description="Backupdb Agent depends on config: /etc/datadb.ini")
  130. parser.add_argument('--force', default=False, action='store_true', help='force restore operation if destination data already exists')
  131. parser.add_argument('profile', type=str, choices=config.keys(), help='Profile to restore')
  132. #parser.add_argument('-i', '--identity',
  133. # help='Ssh keyfile to use', type=str, default='/root/.ssh/datadb.key')
  134. #parser.add_argument('-r', '--remote',
  135. # help='Remote server (rsync://...)', type=str, required=True)
  136. #parser.add_argument('-l', '--local_dir',
  137. # help='Local path', type=str, required=True)
  138. subparser_modes = parser.add_subparsers(dest='mode', help='modes (only "rsync")')
  139. subparser_backup = subparser_modes.add_parser('backup', help='backup to datastore')
  140. subparser_restore = subparser_modes.add_parser('restore', help='restore from datastore')
  141. subparser_status = subparser_modes.add_parser('status', help='get info for profile')
  142. args = parser.parse_args()
  143. if args.mode == 'restore':
  144. restore(args.profile, config[args.profile], force=args.force)
  145. elif args.mode == 'backup':
  146. backup(args.profile, config[args.profile])
  147. elif args.mode == 'status':
  148. info = status(args.profile, config[args.profile])
  149. print(SyncStatus(info))
  150. else:
  151. parser.print_usage()
  152. if __name__ == '__main__':
  153. main()