Refactor and add docs

This commit is contained in:
dave 2016-09-04 19:28:22 -07:00
parent e3def18c79
commit 2226434f1d
5 changed files with 177 additions and 113 deletions

6
.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
.DS_Store
__pycache__
build
dist
lenv
loginjector.egg-info/

66
README.md Normal file
View File

@ -0,0 +1,66 @@
loginjector
===========
**Retrieve logs from docker containers in real time.**
Not all programs support sending logs to a remote server, so logs in containers tend to be lost by lazy sysadmins. This
is a tool that attempts to fix this, by leveraging rsyslog.
By specifying a list of log paths in the container or auto detection from a built-in list, loginjector will generate a
rsyslog config within the container and spawn rsyslogd. Simultaneously, loginjector listens on UDP ports to receive log
entires sent by containers and writes them to disk on the host.
**Assumptions**
* The rsyslogd binary is available in the container at /usr/sbin/rsyslogd (this is stander for ubuntu base images)
* Docker is using it's default networking strategy
**Installation**
* `git clone ssh://git@gitlab.davepedu.com:222/dave/loginjector.git`
* `cd loginjector`
* `python3 setup.py install`
**Running**
* `loginjector_daemon -s unix://var/run/docker.sock -o /var/log/container/`
(The above arguments are actually the defaults and need not be specified)
**Specifying custom paths**
Add the `-c <file>` argument where `<file>` is a json or yml file structured like:
```
{
"container_name": {
"app_name": ["/log/path.log", "/another/log/path.log"],
"another_app": [ ... ]
},
"another_container": {
}
}
```
**Container bake-in**
If you're a docker image creator, you can add a file to your image containing log paths.
Add to your image a file at the path `/.loghint` containing:
```
{
"app_name": ["/log/path.log", "/another/log/path.log"],
"another_app": [ ... ]
}
```
**TODO**
- Implement the custom path option displayed above
- Implement the loghint file mentioned above

View File

@ -1,19 +0,0 @@
*Status:* good idea
```
make a logger-injector using https://docker-py.readthedocs.io/en/latest/api/#execute
- runs on docker host
- lists running containers
- per container, look for processes (optionally, a hint file in the container) that we know where to look for logs for (psutil, or container-fs://.logs)
- generate syslogd confs to broadcast these logs somewhere else
- execute syslogd in the container
- just spawn it or if we detect supervisor, try to insert it?
- wait for the container to exit
- maybe poll for syslogd still running?
polling docker for containers seems expensive so
- poll every minute normally
- if a container dies, poll every 5 seconds until it returns
- but not for more than 5 minutes
```

View File

@ -17,50 +17,7 @@ from docker import Client
from jinja2 import Environment
DEFAULT_TEMPLATE = """
$PrivDropToUser syslog
$PrivDropToGroup syslog
$template myFormat,"%rawmsg%\\n"
# $ActionFileDefaultTemplate myFormat
#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog
#
# Provide file listening
#
module(load="imfile")
#
# Begin logs
#
{% for logfile in logfiles %}
#
# {{ logfile }}
#
input(type="imfile"
File="{{ logfile.path }}"
statefile="{{ logfile.statefile }}"
Tag="{{ logfile.program }}-{{ logfile.logname }}"
Severity="{{ logfile.program }}"
facility="local0")
if ($syslogtag == "{{ logfile.program }}-{{ logfile.logname }}") then {
local0.* @{{ logfile.dest_ip }}:{{ logfile.dest_port }};myFormat
}
{% endfor %}
*.* /var/log/syslog
"""
from loginjector.template import DEFAULT_TEMPLATE
def shell():
@ -69,9 +26,10 @@ def shell():
[logging.getLogger(mute).setLevel(logging.ERROR) for mute in ["docker", "requests"]]
parser = argparse.ArgumentParser(description="Python logging daemon")
parser.add_argument('-s', '--socket', required=True, help="Path or URL to docker daemon socket")
parser.add_argument('-s', '--socket', default="unix://var/run/docker.sock",
help="Path or URL to docker daemon socket")
# parser.add_argument('-t', '--template', required=False, help="Path to syslog template")
parser.add_argument('-o', '--output', required=True, help="Path to host log output dir")
parser.add_argument('-o', '--output', default="/var/log/container/", help="Path to host log output dir")
args = parser.parse_args()
@ -132,10 +90,7 @@ class LogInjectorDaemon(object):
def run(self):
"""
Start all service threads:
change_listner: subscribes to docker's event api and listens for containers stopping/starting
message_recvr: udp listener that receives log messages from containers
Start all service threads and init listeners on preexisting containers
"""
change_listner = Thread(target=self.listen_events, daemon=True)
@ -144,6 +99,7 @@ class LogInjectorDaemon(object):
message_recvr = Thread(target=self.listen_udp, daemon=True)
message_recvr.start()
# Get listing of existing containers and spawn the log listener on each
containers = self.docker.containers()
for container in containers:
@ -162,9 +118,9 @@ class LogInjectorDaemon(object):
def listen_udp(self):
"""
Loop through active loggers. If there's data on the line, read it. This is meant to be ran as a Thread
UDP listener thread. Loop through active loggers. If there's data on the line, read it
"""
while True:
while self.alive:
with self.loggers_lock:
socket_fnos = list(self.loggers.keys())
readable, _, dead = select(socket_fnos, [], socket_fnos, 0.2)
@ -191,20 +147,24 @@ class LogInjectorDaemon(object):
os.fsync(f.fileno()) # is this necessary since we're closing the file?l
def listen_events(self):
try:
for e in self.docker.events(filters=LogInjectorDaemon.EVENT_FILTERS_STOPSTART):
event = json.loads(e.decode('UTF-8'))
# logging.info("event: {}".format(str(event)))
if event["status"] == "start":
logging.info("{}: got start event".format(event["id"]))
Thread(target=self.relisten_on, args=(event["id"],)).start()
"""
Docker change listener thread. Subscribes to docker's event api and respond to containers stopping/starting
"""
for e in self.docker.events(filters=LogInjectorDaemon.EVENT_FILTERS_STOPSTART):
event = json.loads(e.decode('UTF-8'))
self.handle_event(event)
elif event["status"] == "stop":
logging.info("{}: got stop event".format(event["id"]))
Thread(target=self.end_listen_on, args=(event["id"],)).start()
def handle_event(self, event):
"""
Handle an event received from docker
"""
logging.info("{}: got {} event".format(event["id"], event["status"]))
except KeyboardInterrupt:
logging.warning("Stopped listening for events")
if event["status"] == "start":
Thread(target=self.relisten_on, args=(event["id"],)).start()
elif event["status"] == "stop":
Thread(target=self.end_listen_on, args=(event["id"],)).start()
def end_listen_on(self, container_id):
"""
@ -251,12 +211,31 @@ class LogInjectorDaemon(object):
modules_found = self.find_logs(ps_lines)
logging.info("{}: logs detected: {}".format(container_id, str(modules_found)))
modules_use = self.use_builtins.intersection({k for k, v in modules_found.items() if v})
modules_use = list(self.use_builtins.intersection({k for k, v in modules_found.items() if v}))
logging.info("{}: using: {}".format(container_id, str(modules_use)))
logfiles = []
for mod in modules_use:
if len(modules_use) == 0:
logging.info("{}: no log files found, exiting".format(container_id))
return None
syslog_conf = self.render_template(container_id, self.template, modules_use)
# transfer syslog conf
self.write_in_container(container_id, "/etc/rsyslog.conf", syslog_conf)
# start syslog
logging.info("{}: spawning rsyslogd".format(container_id))
self.exec_in_container(container_id, '/usr/sbin/rsyslogd')
def render_template(self, container_id, template_contents, log_modules):
"""
Create a rsyslog config from template
"""
# prepare template vars - only a list of detected log files
logfiles = []
for mod in log_modules:
for path in self.detectors[mod].paths:
original_logname = os.path.basename(path["path"])
# add local listener
@ -273,19 +252,8 @@ class LogInjectorDaemon(object):
"dest_port": new_port,
"container_id": container_id}]
if len(logfiles) == 0:
logging.info("{}: no log files found, exiting".format(container_id))
return
# generate syslog config
syslog_conf = Environment().from_string(self.template).render(logfiles=logfiles)
# transfer syslog conf
self.write_in_container(container_id, "/etc/rsyslog.conf", syslog_conf)
# start syslog
logging.info("{}: spawning rsyslogd".format(container_id))
self.exec_in_container(container_id, '/usr/sbin/rsyslogd')
return Environment().from_string(template_contents).render(logfiles=logfiles)
def get_container_name(self, container_id):
container_info = self.docker.inspect_container(container_id)
@ -295,20 +263,16 @@ class LogInjectorDaemon(object):
# strip leading slash
raw_name = raw_name[1:]
# hacky lazy loading
# hack: lazy loading of bridge ip - we must listen for udp packets on the docker bridge interface, so we need
# the IP for binding. Lazily set it after the first container is fetched from the docker host, as this will
# always happen before any udp binding
if not self.docker_bridge_ip:
self.set_bridge_ip(container_info["NetworkSettings"]["Networks"]["bridge"]["Gateway"])
bridge_ip = container_info["NetworkSettings"]["Networks"]["bridge"]["Gateway"]
logging.info("Found bridge ip: {}".format(bridge_ip))
self.docker_bridge_ip = bridge_ip
return raw_name
def set_bridge_ip(self, bridge_ip):
"""
We must listen for udp packets on the docker bridge interface, so we need the IP for binding. Lazily set it
after the first container is fetched from the docker host, as this will always happen before any udp binding
"""
logging.info("Found bridge ip: {}".format(bridge_ip))
self.docker_bridge_ip = bridge_ip
def add_udp_listener(self, container_id, program, original_logname):
"""
Listen on a random UDP socket and create a new listener. A listener is an association between a udp port and
@ -346,16 +310,19 @@ class LogInjectorDaemon(object):
return {name: hits[name] for name in self.detectors.keys()}
def exec_in_container(self, container, cmd):
e = self.docker.exec_create(container=container, cmd=cmd)
def exec_in_container(self, container_id, cmd_str):
"""
Execute a command in a container
"""
e = self.docker.exec_create(container=container_id, cmd=cmd_str)
return self.docker.exec_start(e["Id"])
def write_in_container(self, container, path, contents):
def write_in_container(self, container_id, path, contents):
"""
This is ugly and sucks
"""
logging.info("{}: writing {} bytes to container's {}".format(container, len(contents), path))
logging.info("{}: writing {} bytes to container's {}".format(container_id, len(contents), path))
if type(contents) != bytes:
contents = contents.encode('UTF-8')
@ -367,7 +334,7 @@ class LogInjectorDaemon(object):
chunk = []
for byte in contents[chunk_size * i:chunk_size * i + chunk_size]:
chunk.append('\\\\x' + hex(byte)[2:])
self.exec_in_container(container,
self.exec_in_container(container_id,
"bash -c -- 'printf {} {} {}'".format(''.join(chunk),
">" if i == 0 else ">>",
path))

44
loginjector/template.py Normal file
View File

@ -0,0 +1,44 @@
DEFAULT_TEMPLATE = """
$PrivDropToUser syslog
$PrivDropToGroup syslog
$template myFormat,"%rawmsg%\\n"
# $ActionFileDefaultTemplate myFormat
#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog
#
# Provide file listening
#
module(load="imfile")
#
# Begin logs
#
{% for logfile in logfiles %}
#
# {{ logfile }}
#
input(type="imfile"
File="{{ logfile.path }}"
statefile="{{ logfile.statefile }}"
Tag="{{ logfile.program }}-{{ logfile.logname }}"
Severity="{{ logfile.program }}"
facility="local0")
if ($syslogtag == "{{ logfile.program }}-{{ logfile.logname }}") then {
local0.* @{{ logfile.dest_ip }}:{{ logfile.dest_port }};myFormat
}
{% endfor %}
*.* /var/log/syslog
"""