From 388f10fb226c3d2904a3b108d8b69f437086ba27 Mon Sep 17 00:00:00 2001 From: dave Date: Fri, 12 Apr 2019 14:39:34 -0700 Subject: [PATCH 1/4] Mysql as backend instead of flat files --- nodepupper/nodeops.py | 8 ++++++-- requirements.txt | 13 +++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/nodepupper/nodeops.py b/nodepupper/nodeops.py index b69229f..8f2398b 100644 --- a/nodepupper/nodeops.py +++ b/nodepupper/nodeops.py @@ -1,5 +1,7 @@ import ZODB -import ZODB.FileStorage +from relstorage.storage import RelStorage +from relstorage.options import Options +from relstorage.adapters.mysql import MySQLAdapter import persistent import persistent.list import persistent.mapping @@ -41,7 +43,9 @@ class NClassAttachment(persistent.Persistent): class NodeOps(object): def __init__(self, db_path): - self.storage = ZODB.FileStorage.FileStorage(db_path) + self.mysql = MySQLAdapter(host="localhost", user="zodb", passwd="zodb", db="zodb", + options=Options(keep_history=False)) + self.storage = RelStorage(adapter=self.mysql) self.db = ZODB.DB(self.storage) with self.db.transaction() as c: diff --git a/requirements.txt b/requirements.txt index 9fc62a5..fa408e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,35 @@ appdirs==1.4.3 +asn1crypto==0.24.0 backports.functools-lru-cache==1.5 BTrees==4.5.1 certifi==2018.11.29 +cffi==1.12.2 chardet==3.0.4 cheroot==6.5.2 CherryPy==18.0.1 +cryptography==2.6.1 idna==2.8 jaraco.functools==1.20 Jinja2==2.10 MarkupSafe==1.0 more-itertools==4.3.0 +perfmetrics==2.0 persistent==4.4.2 portend==2.3 +pycparser==2.19 +PyMySQL==0.9.3 pytz==2018.5 PyYAML==3.13 +RelStorage==2.1.1 requests==2.21.0 six==1.11.0 tempora==1.13 -transaction==2.2.1 +transaction==2.4.0 urllib3==1.24.1 zc.lockfile==1.3.0 ZConfig==3.3.0 -ZODB==5.4.0 +zdaemon==4.3 +ZEO==5.2.1 +ZODB==5.5.1 zodbpickle==1.0.2 zope.interface==4.5.0 From 12f6b562a0cae001662a3b42ff12db7200b39f0d Mon Sep 17 00:00:00 2001 From: dave Date: Sun, 14 Apr 2019 17:04:24 -0700 Subject: [PATCH 2/4] database uri option --- nodepupper/daemon.py | 11 ++++++++--- nodepupper/nodeops.py | 7 +++++-- requirements.txt | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/nodepupper/daemon.py b/nodepupper/daemon.py index c0aca8a..3e29eef 100644 --- a/nodepupper/daemon.py +++ b/nodepupper/daemon.py @@ -3,10 +3,10 @@ import cherrypy import logging from nodepupper.nodeops import NodeOps, NObject, NClass, NClassAttachment from jinja2 import Environment, FileSystemLoader, select_autoescape -from nodepupper.common import pwhash -import math from urllib.parse import urlparse +import math import yaml +import sys APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) @@ -278,7 +278,8 @@ def main(): parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on") # parser.add_argument('-l', '--library', default="./library", help="library path") # parser.add_argument('-c', '--cache', default="./cache", help="cache path") - parser.add_argument('-s', '--database', default="./pupper.db", help="path to persistent sqlite database") + parser.add_argument('-s', '--database', default=os.environ.get('DATABASE_URI', None), + help="mysql:// connection uri") parser.add_argument('--debug', action="store_true", help="enable development options") args = parser.parse_args() @@ -286,6 +287,10 @@ def main(): logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") + if not args.database: + print("--database or $DATABASE_URI is required") + sys.exit(2) + library = NodeOps(args.database) tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates" diff --git a/nodepupper/nodeops.py b/nodepupper/nodeops.py index 8f2398b..c18b52f 100644 --- a/nodepupper/nodeops.py +++ b/nodepupper/nodeops.py @@ -1,3 +1,4 @@ +from urllib.parse import urlparse import ZODB from relstorage.storage import RelStorage from relstorage.options import Options @@ -42,8 +43,10 @@ class NClassAttachment(persistent.Persistent): class NodeOps(object): - def __init__(self, db_path): - self.mysql = MySQLAdapter(host="localhost", user="zodb", passwd="zodb", db="zodb", + def __init__(self, db_uri): + uri = urlparse(db_uri) + + self.mysql = MySQLAdapter(host=uri.hostname, user=uri.username, passwd=uri.password, db=uri.path[1:], options=Options(keep_history=False)) self.storage = RelStorage(adapter=self.mysql) self.db = ZODB.DB(self.storage) diff --git a/requirements.txt b/requirements.txt index fa408e6..7f02456 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ pycparser==2.19 PyMySQL==0.9.3 pytz==2018.5 PyYAML==3.13 -RelStorage==2.1.1 +-e git+ssh://git@git.davepedu.com:223/dave/relstorage.git@5ae138b71a0eeb740208e61cfd1662c49738ee45#egg=RelStorage requests==2.21.0 six==1.11.0 tempora==1.13 From 52650a49d15aba2938f4b7fbba24ed1b7cd7e737 Mon Sep 17 00:00:00 2001 From: dave Date: Sun, 14 Apr 2019 17:20:27 -0700 Subject: [PATCH 3/4] config dumper --- nodepupper/cli.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nodepupper/cli.py b/nodepupper/cli.py index 437ba50..a7560d4 100644 --- a/nodepupper/cli.py +++ b/nodepupper/cli.py @@ -72,6 +72,8 @@ def main(): spr_delc = spr_action.add_parser("delclass", help="delete a class") spr_delc.add_argument("cls", help="name of class to delete") + spr_dump = spr_action.add_parser("dump", help="dump the database") + args = parser.parse_args() r = requests.session() @@ -118,6 +120,17 @@ def main(): elif args.action == "delclass": r.delete(args.host.rstrip("/") + "/api/class/" + args.cls).raise_for_status() + elif args.action == "dump": + nodes = yaml.load(r.get(args.host.rstrip("/") + "/api/node").text)["nodes"] + + dump = {"classes": yaml.load(r.get(args.host.rstrip("/") + "/api/class").text)["classes"], + "nodes": {}} + + for nodename in nodes: + dump["nodes"][nodename] = yaml.load(getnode(nodename)) + + print(yaml.dump(dump, default_flow_style=False)) + if __name__ == "__main__": main() From 855cbcc4abfb10974a12e7761c2d1aa67de0c641 Mon Sep 17 00:00:00 2001 From: dave Date: Mon, 15 Apr 2019 21:36:44 -0700 Subject: [PATCH 4/4] import/export cli. v0.0.2 --- Dockerfile | 2 -- nodepupper/cli.py | 21 +++++++++++++++++++++ nodepupper/daemon.py | 3 ++- setup.py | 2 +- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2dcb968..7bb5d8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,6 @@ RUN pip3 install -U pip && \ python3 setup.py install && \ useradd --uid 1000 app -VOLUME /data/ - ADD start /start ENTRYPOINT ["/start"] diff --git a/nodepupper/cli.py b/nodepupper/cli.py index a7560d4..f890516 100644 --- a/nodepupper/cli.py +++ b/nodepupper/cli.py @@ -8,6 +8,9 @@ import tempfile import subprocess +import sys + + APPNAME = "npcli" CONFDIR = user_config_dir(APPNAME) CONFPATH = os.path.join(CONFDIR, "conf.json") @@ -74,6 +77,9 @@ def main(): spr_dump = spr_action.add_parser("dump", help="dump the database") + spr_import = spr_action.add_parser("import", help="import a database dump") + spr_import.add_argument("fname", help="db dump yaml file to import") + args = parser.parse_args() r = requests.session() @@ -131,6 +137,21 @@ def main(): print(yaml.dump(dump, default_flow_style=False)) + elif args.action == "import": + with open(args.fname) as f: + dump = yaml.load(f) + + for clsname in dump["classes"]: + r.put(args.host.rstrip("/") + "/api/class/" + clsname).raise_for_status() + + # just make the nodes first + for nodename, nodebody in dump["nodes"].items(): + putnode(nodename, yaml.dump({"body": {}, "classes": {}, "parents": []})).raise_for_status() + + # then fill out node bodies to avoid missing parent ordering issues + for nodename, nodebody in dump["nodes"].items(): + putnode(nodename, yaml.dump(nodebody)).raise_for_status() + if __name__ == "__main__": main() diff --git a/nodepupper/daemon.py b/nodepupper/daemon.py index 3e29eef..86e8fd5 100644 --- a/nodepupper/daemon.py +++ b/nodepupper/daemon.py @@ -206,7 +206,8 @@ class ClassesApi(object): def PUT(self, cls): with self.nodes.db.transaction() as c: - c.root.classes[cls] = NClass(cls) + if cls not in c.root.classes: + c.root.classes[cls] = NClass(cls) def DELETE(self, cls): with self.nodes.db.transaction() as c: diff --git a/setup.py b/setup.py index f89b109..4533b31 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from setuptools import setup import os -__version__ = "0.0.1" +__version__ = "0.0.2" with open(os.path.join(os.path.dirname(__file__), "requirements.txt")) as f: __requirements__ = [line.strip() for line in f.readlines()]