dave
/
z-hypervisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add disk API

This commit is contained in:
dave 2016-12-28 18:43:07 -08:00
parent 6332ef5865
commit b59cc8c634
9 changed files with 279 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
README.md
View File

@ -25,14 +25,6 @@ Install
HTTP API
========
*GET /api/v1/create_disk*
Create a storate disk to use with machines. Params:
- datastore: datastore name such as 'default'
- name: arbitrary disk name like 'ubuntu-root.bin'
- size: size in megabytes
- fmt: format, raw or qcow2
*GET /api/v1/machine/:id/start*
Start a machine given its id
@ -53,10 +45,21 @@ HTTP API
Create a new machine or update an existing machine. Params:
- machine_id: alphanumeric name for the name
- machine_type: type of virtualization to run the machine with
- machine_spec: serialized json object describing the machine. See the 'spec' key of example/ubuntu.json
*DELETE /api/v1/machine/:id*
Delete a machine give its id
*GET /api/v1/disk/:id*
List all disks or a specific disk if passed
*PUT /api/v1/disk/:id*
Create a storate disk to use with machines. Params:
- disk_spec: serialized json object describing the disk. See the 'spec' key of example/ubuntu-root.json and example/ubuntu-iso.json
*DELETE /api/v1/disk/:id*
Delete a disk by ID

11
example/ubuntu-iso.json Normal file
View File

@ -0,0 +1,11 @@
{
"disk_id": "ubuntu-14.04.5_x64.iso",
"spec": {
"options": {
"type": "iso",
"datastore": "default"
},
"properties": {
}
}
}

13
example/ubuntu-root.json Normal file
View File

@ -0,0 +1,13 @@
{
"disk_id": "ubuntu-root.bin",
"spec": {
"options": {
"type": "qdisk",
"datastore": "default"
},
"properties": {
"size": 8192,
"fmt": "qcow2"
}
}
}

7
example/ubuntu.json
View File

@ -3,6 +3,7 @@
"machine_type": "q",
"spec": {
"options": {
"type": "q",
"autostart": true,
"respawn": true
},
@ -11,14 +12,12 @@
"mem": 1024,
"drives": [
{
"file": "ubuntu-root.bin",
"datastore": "default",
"disk": "ubuntu-root",
"index": 0,
"if": "virtio"
},
{
"file": "ubuntu-14.04.iso",
"datastore": "default",
"disk": "multipreseed-14.04.iso",
"index": 1,
"media": "cdrom"
}

100
zhypervisor/api/api.py
View File

@ -1,7 +1,6 @@
import cherrypy
import logging
import json
import subprocess
from threading import Thread
@ -70,6 +69,7 @@ class ZApiV1(Mountable):
def __init__(self, root):
super().__init__(conf={
"/machine": {'request.dispatch': cherrypy.dispatch.MethodDispatcher()},
"/disk": {'request.dispatch': cherrypy.dispatch.MethodDispatcher()},
# "/task": {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}, # @TODO this conf belongs in the child
# "/logs": {
# 'tools.staticdir.on': True,
@ -79,6 +79,7 @@ class ZApiV1(Mountable):
})
self.root = root
self.machine = ZApiMachines(self.root)
self.disk = ZApiDisks(self.root)
# self.task = BSApiTask(self.root)
# self.control = BSApiControl(self.root)
# self.socket = ApiWebsockets(self.root)
@ -87,18 +88,6 @@ class ZApiV1(Mountable):
def index(self):
yield "It works!"
@cherrypy.expose
def create_disk(self, datastore, name, size, fmt):
"""
WORKAROUND for creating qemu disks
TODO replace me
"""
assert fmt in ["qcow2", "raw"], "Disk format is invalid"
assert name.endswith(".bin"), "Disk must be named <something>.bin"
self.root.master.create_disk(datastore, name, fmt, size)
return name
@cherrypy.popargs("machine_id")
class ZApiMachineStop(object):
@ -180,10 +169,10 @@ class ZApiMachines():
self.restart = ZApiMachineRestart(self.root)
@cherrypy.tools.json_out()
def GET(self, machine_id=None, action=None, summary=False):
def GET(self, machine_id=None, summary=False):
"""
Get a list of all machines or specific one if passed
:param task_id: task to retrieve
:param machine_id: machine to retrieve
"""
summary = summary in [True, 'True', 'true', 'yes', '1', 1]
@ -192,8 +181,7 @@ class ZApiMachines():
machine = {"machine_id": _machine_id,
"_status": machine_spec.machine.get_status()}
if not summary:
machine.update({"machine_type": machine_spec.machine_type,
"spec": machine_spec.serialize()})
machine.update({"spec": machine_spec.serialize()})
machines[_machine_id] = machine
if machine_id is not None:
@ -205,11 +193,10 @@ class ZApiMachines():
return list(machines.values())
@cherrypy.tools.json_out()
def PUT(self, machine_id, machine_type, machine_spec):
def PUT(self, machine_id, machine_spec):
"""
Create a new machine or update an existing machine
:param machine_id: id of machine to create or modify
:param machine_type: set machine type (currently, only "q")
'param machine_spec: json dictionary describing the machine. see the 'spec' key of example/banutoo.json
"""
@ -218,7 +205,7 @@ class ZApiMachines():
"Machine must be stopped to modify"
machine_spec = json.loads(machine_spec)
self.root.master.add_machine(machine_id, machine_type, machine_spec, write=True)
self.root.master.add_machine(machine_id, machine_spec, write=True)
return machine_id
def DELETE(self, machine_id):
@ -234,3 +221,76 @@ class ZApiMachines():
self.root.master.remove_machine(machine_id)
return machine_id
@cherrypy.popargs("disk_id")
class ZApiDisks():
"""
Endpoint for managing disks
"""
exposed = True
def __init__(self, root):
"""
Endpoint to modify disks. PUT and DELETE require the disk not be attached.
TODO how to attach/detach?
"""
self.root = root
@cherrypy.tools.json_out()
def GET(self, disk_id=None, summary=False):
"""
Get a list of disks or a specific one if passed
:param disk_id: task to retrieve
"""
summary = summary in [True, 'True', 'true', 'yes', '1', 1]
disks = {}
for _disk_id, disk_spec in self.root.master.disks.items():
disk = {"disk_id": _disk_id}
# "_status": machine_spec.machine.get_status()} attached / detached ?
# if not summary:
# machine.update({"machine_type": machine_spec.machine_type,
# "spec": machine_spec.serialize()})
disk.update({"spec": disk_spec.serialize()})
disks[_disk_id] = disk
if disk_id is not None:
try:
return [disks[disk_id]]
except KeyError:
raise cherrypy.HTTPError(status=404)
else:
return list(disks.values())
@cherrypy.tools.json_out()
def PUT(self, disk_id, disk_spec):
"""
Create a new disk or update an existing disk
:param disk_id: id of disk to create or modify
'param disk_spec: json dictionary describing the disk. see the 'spec' key of example/ubuntu-root.json
"""
assert disk_id not in self.root.master.disks or \
self.root.master.disks[disk_id].get_status() == "idle", \
"Disk must not be attached to modify" # TODO to a running machine?
# TODO move asserts out of the API
disk_spec = json.loads(disk_spec)
self.root.master.add_disk(disk_id, disk_spec, write=True)
return disk_id
def DELETE(self, disk_id):
"""
Delete a disk. Raises 404 if no such disk exists. Raises error if disk is not idle (detached)
:param disk_id: ID of disk to remove
"""
try:
assert self.root.master.disks[disk_id].get_status() == "idle", \
"Disk must be detached to delete"
except KeyError:
raise cherrypy.HTTPError(status=404)
self.root.master.remove_disk(disk_id)
return disk_id

57
zhypervisor/clients/qmachine.py
View File

@ -5,6 +5,7 @@ from time import sleep
from threading import Thread
from zhypervisor.util import TapDevice, Machine
from zhypervisor.util import ZDisk
class QMachine(Machine):
@ -124,18 +125,21 @@ class QMachine(Machine):
"""
Inspect props.drives expecting a format like: {"file": "/tmp/ubuntu.qcow2", "index": 0, "if": "virtio"}
"""
drives = []
for drive in self.spec.properties.get("drives", []):
drive_info = dict(drive)
drives.append("-drive")
args = []
for attached_drive in self.spec.properties.get("drives", []):
args.append("-drive")
# translate datastore paths if neede
if "file" in drive_info:
drive_info["file"] = self.get_datastore_path(drive_info["datastore"], drive_info["file"])
del drive_info["datastore"]
disk_ob = self.spec.master.disks[attached_drive["disk"]]
drives.append(QMachine.format_args(drive_info))
return drives
drive_args = {"file": disk_ob.get_path()}
for option in ["if", "index", "media"]:
if option in attached_drive:
drive_args[option] = attached_drive[option]
args.append(QMachine.format_args((drive_args)))
return args
@staticmethod
def format_args(d):
@ -152,3 +156,36 @@ class QMachine(Machine):
if not args:
return None
return ','.join(args)
class QDisk(ZDisk):
def init(self):
"""
Create a QEMU-formatted virtual disk
"""
disk_path = self.get_path()
assert not os.path.exists(disk_path), "Disk already exists!"
img_args = ["qemu-img", "create", "-f", self.properties["fmt"], disk_path, "{}M".format(int(self.properties["size"]))]
logging.info("Creating disk with: %s", str(img_args))
subprocess.check_call(img_args)
def validate(self):
assert self.disk_id.endswith(".bin"), "QDisks names must end with '.bin'"
def delete(self):
os.unlink(self.get_path())
class IsoDisk(ZDisk):
pass
# TODO make this do more nothing
def validate(self):
assert self.disk_id.endswith(".iso"), "IsoDisk names must end with '.iso'"
def init(self):
assert os.path.exists(self.get_path()), "ISO must already exist!"
def delete(self):
pass

107
zhypervisor/daemon.py
View File

@ -4,17 +4,17 @@ import json
import signal
import logging
import argparse
import subprocess
from time import sleep
from concurrent.futures import ThreadPoolExecutor
from glob import iglob
from threading import Thread
from concurrent.futures import ThreadPoolExecutor
from zhypervisor.logging import setup_logging
from zhypervisor.machine import MachineSpec
from zhypervisor.clients.qmachine import QDisk, IsoDisk
from zhypervisor.util import ZDisk
from zhypervisor.api.api import ZApi
from pprint import pprint
class ZHypervisorDaemon(object):
def __init__(self, config):
@ -27,6 +27,7 @@ class ZHypervisorDaemon(object):
"""
self.config = config # JSON config listing, mainly, datastore paths
self.datastores = {} # Mapping of datastore name -> objects
self.disks = {} # Mapping of disk name -> objects
self.machines = {} # Mapping of machine name -> objects
self.running = True
@ -34,6 +35,9 @@ class ZHypervisorDaemon(object):
self.init_datastores()
self.state = ZConfig(self.datastores["default"])
# Set up disks
self.init_disks()
# start API
self.api = ZApi(self)
@ -48,13 +52,20 @@ class ZHypervisorDaemon(object):
for name, info in self.config["datastores"].items():
self.datastores[name] = ZDataStore(name, info["path"], info.get("init", False))
def init_disks(self):
"""
Load all disks and ensure reachability
"""
for disk in self.state.get_disks():
self.add_disk(disk["disk_id"], {"options": disk["options"], "properties": disk["properties"]})
def init_machines(self):
"""
Per machine in the on-disk state, create a machine object
"""
for machine_info in self.state.get_machines():
machine_id = machine_info["machine_id"]
self.add_machine(machine_id, machine_info["machine_type"], machine_info["spec"])
self.add_machine(machine_id, machine_info["spec"])
def signal_handler(self, signum, frame):
"""
@ -85,28 +96,43 @@ class ZHypervisorDaemon(object):
# Below here are methods external forces may use to manipulate disks
def create_disk(self, datastore, name, fmt, size=None):
def add_disk(self, disk_id, disk_spec, write=False):
"""
Create a disk. Disks represent arbitrary storage
@TODO support formats passed by runnable modules
:param datastore: datastore to store the disk in
:param name: name for the disk
:param size: size of the disk, in mb, if applicable
:param format: format of the disk
Create a disk
"""
disk_path = self.datastores[datastore].get_filepath(name)
assert not os.path.exists(disk_path), "Disk already exists!"
img_args = ["qemu-img", "create", "-f", fmt, disk_path, "{}M".format(int(size))]
logging.info("Creating disk with: %s", str(img_args))
subprocess.check_call(img_args)
assert disk_id not in self.disks, "Cannot update disks"
disk_type = disk_spec["options"]["type"]
disk_datastore = disk_spec["options"]["datastore"]
datastore = self.datastores[disk_datastore]
if disk_type == "qdisk":
disk = QDisk(datastore, disk_id, disk_spec)
elif disk_type == "iso":
disk = IsoDisk(datastore, disk_id, disk_spec)
else:
raise Exception("Unknown disk type: {}".format(disk_type))
disk = ZDisk(datastore, disk_id, disk_spec)
if not disk.exists():
disk.init()
assert disk.exists(), "Disk file path is missing: {}".format(disk.get_path())
self.disks[disk_id] = disk
if write:
self.state.write_disk(disk_id, disk_spec)
def remove_disk(self, disk_id):
"""
Remove a disk from the system
"""
assert self.disks[disk_id].get_status() == "idle", "Disk must be idle to delete"
self.disks[disk_id].delete()
del self.disks[disk_id]
self.state.remove_disk(disk_id)
# Below here are methods external forces may use to manipulate machines
def add_machine(self, machine_id, machine_type, machine_spec, write=False):
def add_machine(self, machine_id, machine_spec, write=False):
"""
Create or update a machine.
:param machine_id: alphanumeric id of machine to modify/create
:param machine_type: runnable type e.g. "q"
:param machine_spec: dictionary of machine options - see example/ubuntu.json
:param write: commit machinge changes to on-disk state
"""
@ -116,12 +142,12 @@ class ZHypervisorDaemon(object):
machine.options = machine_spec["options"]
machine.properties = machine_spec["properties"]
else:
machine = MachineSpec(self, machine_id, machine_type, machine_spec)
machine = MachineSpec(self, machine_id, machine_spec)
self.machines[machine_id] = machine
# Update if necessary
if write:
self.state.write_machine(machine_id, machine_type, machine_spec)
self.state.write_machine(machine_id, machine_spec)
# Launch if machine is an autostarted machine
if machine.options.get("autostart", False) and machine.machine.get_status() == "stopped":
@ -181,35 +207,35 @@ class ZConfig(object):
self.datastore = datastore
self.machine_data_dir = self.datastore.get_filepath("machines")
self.disk_data_dir = self.datastore.get_filepath("disks")
for d in [self.machine_data_dir]:
for d in [self.machine_data_dir, self.disk_data_dir]:
os.makedirs(d, exist_ok=True)
def get_machines(self):
"""
Return config of all machines on disk
Return list of all machines on hypervisor
"""
machines = []
logging.info("Looking for machines in {}".format(self.machine_data_dir))
for mach_name in os.listdir(self.machine_data_dir):
with open(os.path.join(self.machine_data_dir, mach_name), "r") as f:
logging.info("Looking for machine configs in {}".format(self.machine_data_dir))
for f_name in iglob(self.machine_data_dir + '/*.json'):
with open(f_name, "r") as f:
machines.append(json.load(f))
return machines
def write_machine(self, machine_id, machine_type, machine_spec):
def write_machine(self, machine_id, machine_spec):
"""
Write a machine's config to the disk. Params similar to elsewhere.
"""
with open(os.path.join(self.machine_data_dir, "{}.json".format(machine_id)), "w") as f:
json.dump({"machine_id": machine_id,
"machine_type": machine_type,
"spec": machine_spec}, f, indent=4)
def write_machine_o(self, machine_obj):
"""
Similar to write_machine, but accepts a MachineSpec object
"""
self.write_machine(machine_obj.machine_id, machine_obj.machine_type, machine_obj.serialize())
self.write_machine(machine_obj.machine_id, machine_obj.serialize())
def remove_machine(self, machine_id):
"""
@ -218,6 +244,27 @@ class ZConfig(object):
json_path = os.path.join(self.machine_data_dir, "{}.json".format(machine_id))
os.unlink(json_path)
def get_disks(self):
"""
Return list of all disks on the hypervisor
"""
disks = []
logging.info("Looking for disk configs in {}".format(self.disk_data_dir))
for f_name in iglob(self.disk_data_dir + '/*.json'):
with open(f_name, "r") as f:
disks.append(json.load(f))
return disks
def write_disk(self, disk_id, disk_spec):
with open(os.path.join(self.disk_data_dir, "{}.json".format(disk_id)), "w") as f:
disk = {"disk_id": disk_id,
"options": disk_spec["options"],
"properties": disk_spec["properties"]}
json.dump(disk, f, indent=4)
def remove_disk(self, disk_id):
os.unlink(os.path.join(self.disk_data_dir, "{}.json".format(disk_id)))
def main():
setup_logging()

7
zhypervisor/machine.py
View File

@ -7,7 +7,7 @@ class MachineSpec(object):
"""
Represents a machine we may control
"""
def __init__(self, master, machine_id, machine_type, spec):
def __init__(self, master, machine_id, spec):
"""
Initialize options and properties of the machine. More importantly, initialize the self.machine object which
should be a subclass of zhypervisor.util.Machine.
@ -15,16 +15,15 @@ class MachineSpec(object):
logging.info("Initting machine %s", machine_id)
self.master = master
self.machine_id = machine_id
self.machine_type = machine_type
self.options = spec["options"]
self.properties = spec["properties"]
# TODO replace if/else with better system
if machine_type == "q":
if self.options["type"] == "q":
self.machine = QMachine(self)
else:
raise Exception("Unknown machine type: {}".format(machine_type))
raise Exception("Unknown machine type: {}".format(self.options["type"]))
def start(self):
"""

33
zhypervisor/util.py
View File

@ -1,5 +1,6 @@
import os
import json
from random import randint
@ -63,3 +64,35 @@ class Machine(object):
Resolve the filesystem path for a path in the given datastore
"""
return self.spec.master.datastores.get(datastore_name).get_filepath(*paths)
class ZDisk(object):
def __init__(self, datastore, disk_id, spec):
self.datastore = datastore
self.disk_id = disk_id
self.options = spec["options"]
self.properties = spec["properties"]
self.validate()
def validate(self):
pass
def get_path(self):
return self.datastore.get_filepath(os.path.join("disks", self.disk_id))
def exists(self):
path = self.get_path()
return os.path.exists(path)
def init(self):
os.makedirs(self.get_path(), exist_ok=True)
def serialize(self):
return {"options": self.options,
"properties": self.properties}
def get_status(self):
return "idle" # TODO
def delete(self):
raise NotImplemented()