separate emit function

backupdb/server.py

@@ -112,29 +112,29 @@ class AppWeb(object):
             keys.append(key['Key'])
             size += key['Size']
 
+        # keys is a sorted list of bucket paths to emit
         keys.sort()
         print(size)
-        # keys is a sorted list of bucket paths to emit
 
         cherrypy.response.headers.update({"Content-Length": str(size),
                                           "Content-Type": 'application/x-gzip',
                                           "Content-Disposition": 'attachment; filename="backup.tar.gz"'})
 
-        def emit():
-            for key in keys:
-                # print("getting", key)
-                o = self.s3.get_object(Bucket=self.bucket_name, Key=key)
-
-                while True:
-                    chunk = o["Body"].read(256 * 1024)
-                    if not chunk:
-                        break
-                    yield chunk
-
-        return emit()
+        return self.emit(keys)
 
     get_backup._cp_config = {'response.stream': True}
 
+    def emit(self, keys):
+        for key in keys:
+            # print("getting", key)
+            o = self.s3.get_object(Bucket=self.bucket_name, Key=key)
+
+            while True:
+                chunk = o["Body"].read(256 * 1024)
+                if not chunk:
+                    break
+                yield chunk
+
 
 def main():
     import argparse

default

@@ -11,6 +11,7 @@ server {
     fastcgi_temp_path /data/data/datadb/tmp/;
     proxy_temp_path /data/data/datadb/tmp/;
 
+
     #auth_basic "Restricted";
     #auth_basic_user_file /data/htpasswd;
 
@@ -25,9 +26,10 @@ server {
         fastcgi_pass  unix:/tmp/fcgiwrap.socket;
         include /etc/nginx/fastcgi_params;
         fastcgi_param SCRIPT_FILENAME  /usr/share/backupdb$fastcgi_script_name;
-        fastcgi_read_timeout 600s;
-        fastcgi_send_timeout 600s;
+        fastcgi_read_timeout 7200s;
+        fastcgi_send_timeout 7200s;
         client_max_body_size 0;
+        fastcgi_buffering off;
     }
 
     location /api/ {

scripts/new_backup

@@ -16,6 +16,7 @@ from random import randint
 from time import time
 from hashlib import md5
 from glob import iglob
+import logging
 import json
 
 
@@ -225,6 +226,7 @@ def handle_req():
 
 if __name__ == "__main__":
     try:
+        #logging.basicConfig(level=logging.WARNING)
         handle_req()
     except Exception as e:
         start_response(status_code=("500", "Internal server error"))