Refactor and add docs
This commit is contained in:
parent
e3def18c79
commit
2226434f1d
|
@ -0,0 +1,6 @@
|
||||||
|
.DS_Store
|
||||||
|
__pycache__
|
||||||
|
build
|
||||||
|
dist
|
||||||
|
lenv
|
||||||
|
loginjector.egg-info/
|
|
@ -0,0 +1,66 @@
|
||||||
|
loginjector
|
||||||
|
===========
|
||||||
|
|
||||||
|
**Retrieve logs from docker containers in real time.**
|
||||||
|
|
||||||
|
Not all programs support sending logs to a remote server, so logs in containers tend to be lost by lazy sysadmins. This
|
||||||
|
is a tool that attempts to fix this, by leveraging rsyslog.
|
||||||
|
|
||||||
|
By specifying a list of log paths in the container or auto detection from a built-in list, loginjector will generate a
|
||||||
|
rsyslog config within the container and spawn rsyslogd. Simultaneously, loginjector listens on UDP ports to receive log
|
||||||
|
entires sent by containers and writes them to disk on the host.
|
||||||
|
|
||||||
|
**Assumptions**
|
||||||
|
|
||||||
|
* The rsyslogd binary is available in the container at /usr/sbin/rsyslogd (this is stander for ubuntu base images)
|
||||||
|
* Docker is using it's default networking strategy
|
||||||
|
|
||||||
|
**Installation**
|
||||||
|
|
||||||
|
* `git clone ssh://git@gitlab.davepedu.com:222/dave/loginjector.git`
|
||||||
|
* `cd loginjector`
|
||||||
|
* `python3 setup.py install`
|
||||||
|
|
||||||
|
|
||||||
|
**Running**
|
||||||
|
|
||||||
|
* `loginjector_daemon -s unix://var/run/docker.sock -o /var/log/container/`
|
||||||
|
|
||||||
|
(The above arguments are actually the defaults and need not be specified)
|
||||||
|
|
||||||
|
|
||||||
|
**Specifying custom paths**
|
||||||
|
|
||||||
|
Add the `-c <file>` argument where `<file>` is a json or yml file structured like:
|
||||||
|
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"container_name": {
|
||||||
|
"app_name": ["/log/path.log", "/another/log/path.log"],
|
||||||
|
"another_app": [ ... ]
|
||||||
|
},
|
||||||
|
"another_container": {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
**Container bake-in**
|
||||||
|
|
||||||
|
If you're a docker image creator, you can add a file to your image containing log paths.
|
||||||
|
|
||||||
|
Add to your image a file at the path `/.loghint` containing:
|
||||||
|
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"app_name": ["/log/path.log", "/another/log/path.log"],
|
||||||
|
"another_app": [ ... ]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
**TODO**
|
||||||
|
|
||||||
|
- Implement the custom path option displayed above
|
||||||
|
- Implement the loghint file mentioned above
|
19
README.txt
19
README.txt
|
@ -1,19 +0,0 @@
|
||||||
*Status:* good idea
|
|
||||||
|
|
||||||
```
|
|
||||||
make a logger-injector using https://docker-py.readthedocs.io/en/latest/api/#execute
|
|
||||||
- runs on docker host
|
|
||||||
- lists running containers
|
|
||||||
- per container, look for processes (optionally, a hint file in the container) that we know where to look for logs for (psutil, or container-fs://.logs)
|
|
||||||
- generate syslogd confs to broadcast these logs somewhere else
|
|
||||||
- execute syslogd in the container
|
|
||||||
- just spawn it or if we detect supervisor, try to insert it?
|
|
||||||
- wait for the container to exit
|
|
||||||
- maybe poll for syslogd still running?
|
|
||||||
|
|
||||||
|
|
||||||
polling docker for containers seems expensive so
|
|
||||||
- poll every minute normally
|
|
||||||
- if a container dies, poll every 5 seconds until it returns
|
|
||||||
- but not for more than 5 minutes
|
|
||||||
```
|
|
|
@ -17,50 +17,7 @@ from docker import Client
|
||||||
|
|
||||||
from jinja2 import Environment
|
from jinja2 import Environment
|
||||||
|
|
||||||
|
from loginjector.template import DEFAULT_TEMPLATE
|
||||||
DEFAULT_TEMPLATE = """
|
|
||||||
$PrivDropToUser syslog
|
|
||||||
$PrivDropToGroup syslog
|
|
||||||
|
|
||||||
$template myFormat,"%rawmsg%\\n"
|
|
||||||
# $ActionFileDefaultTemplate myFormat
|
|
||||||
|
|
||||||
#
|
|
||||||
# Where to place spool and state files
|
|
||||||
#
|
|
||||||
$WorkDirectory /var/spool/rsyslog
|
|
||||||
|
|
||||||
#
|
|
||||||
# Provide file listening
|
|
||||||
#
|
|
||||||
|
|
||||||
module(load="imfile")
|
|
||||||
|
|
||||||
#
|
|
||||||
# Begin logs
|
|
||||||
#
|
|
||||||
|
|
||||||
{% for logfile in logfiles %}
|
|
||||||
#
|
|
||||||
# {{ logfile }}
|
|
||||||
#
|
|
||||||
|
|
||||||
input(type="imfile"
|
|
||||||
File="{{ logfile.path }}"
|
|
||||||
statefile="{{ logfile.statefile }}"
|
|
||||||
Tag="{{ logfile.program }}-{{ logfile.logname }}"
|
|
||||||
Severity="{{ logfile.program }}"
|
|
||||||
facility="local0")
|
|
||||||
|
|
||||||
if ($syslogtag == "{{ logfile.program }}-{{ logfile.logname }}") then {
|
|
||||||
local0.* @{{ logfile.dest_ip }}:{{ logfile.dest_port }};myFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
{% endfor %}
|
|
||||||
|
|
||||||
*.* /var/log/syslog
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def shell():
|
def shell():
|
||||||
|
@ -69,9 +26,10 @@ def shell():
|
||||||
[logging.getLogger(mute).setLevel(logging.ERROR) for mute in ["docker", "requests"]]
|
[logging.getLogger(mute).setLevel(logging.ERROR) for mute in ["docker", "requests"]]
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Python logging daemon")
|
parser = argparse.ArgumentParser(description="Python logging daemon")
|
||||||
parser.add_argument('-s', '--socket', required=True, help="Path or URL to docker daemon socket")
|
parser.add_argument('-s', '--socket', default="unix://var/run/docker.sock",
|
||||||
|
help="Path or URL to docker daemon socket")
|
||||||
# parser.add_argument('-t', '--template', required=False, help="Path to syslog template")
|
# parser.add_argument('-t', '--template', required=False, help="Path to syslog template")
|
||||||
parser.add_argument('-o', '--output', required=True, help="Path to host log output dir")
|
parser.add_argument('-o', '--output', default="/var/log/container/", help="Path to host log output dir")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
@ -132,10 +90,7 @@ class LogInjectorDaemon(object):
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""
|
"""
|
||||||
Start all service threads:
|
Start all service threads and init listeners on preexisting containers
|
||||||
|
|
||||||
change_listner: subscribes to docker's event api and listens for containers stopping/starting
|
|
||||||
message_recvr: udp listener that receives log messages from containers
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
change_listner = Thread(target=self.listen_events, daemon=True)
|
change_listner = Thread(target=self.listen_events, daemon=True)
|
||||||
|
@ -144,6 +99,7 @@ class LogInjectorDaemon(object):
|
||||||
message_recvr = Thread(target=self.listen_udp, daemon=True)
|
message_recvr = Thread(target=self.listen_udp, daemon=True)
|
||||||
message_recvr.start()
|
message_recvr.start()
|
||||||
|
|
||||||
|
# Get listing of existing containers and spawn the log listener on each
|
||||||
containers = self.docker.containers()
|
containers = self.docker.containers()
|
||||||
|
|
||||||
for container in containers:
|
for container in containers:
|
||||||
|
@ -162,9 +118,9 @@ class LogInjectorDaemon(object):
|
||||||
|
|
||||||
def listen_udp(self):
|
def listen_udp(self):
|
||||||
"""
|
"""
|
||||||
Loop through active loggers. If there's data on the line, read it. This is meant to be ran as a Thread
|
UDP listener thread. Loop through active loggers. If there's data on the line, read it
|
||||||
"""
|
"""
|
||||||
while True:
|
while self.alive:
|
||||||
with self.loggers_lock:
|
with self.loggers_lock:
|
||||||
socket_fnos = list(self.loggers.keys())
|
socket_fnos = list(self.loggers.keys())
|
||||||
readable, _, dead = select(socket_fnos, [], socket_fnos, 0.2)
|
readable, _, dead = select(socket_fnos, [], socket_fnos, 0.2)
|
||||||
|
@ -191,20 +147,24 @@ class LogInjectorDaemon(object):
|
||||||
os.fsync(f.fileno()) # is this necessary since we're closing the file?l
|
os.fsync(f.fileno()) # is this necessary since we're closing the file?l
|
||||||
|
|
||||||
def listen_events(self):
|
def listen_events(self):
|
||||||
try:
|
"""
|
||||||
for e in self.docker.events(filters=LogInjectorDaemon.EVENT_FILTERS_STOPSTART):
|
Docker change listener thread. Subscribes to docker's event api and respond to containers stopping/starting
|
||||||
event = json.loads(e.decode('UTF-8'))
|
"""
|
||||||
# logging.info("event: {}".format(str(event)))
|
for e in self.docker.events(filters=LogInjectorDaemon.EVENT_FILTERS_STOPSTART):
|
||||||
if event["status"] == "start":
|
event = json.loads(e.decode('UTF-8'))
|
||||||
logging.info("{}: got start event".format(event["id"]))
|
self.handle_event(event)
|
||||||
Thread(target=self.relisten_on, args=(event["id"],)).start()
|
|
||||||
|
|
||||||
elif event["status"] == "stop":
|
def handle_event(self, event):
|
||||||
logging.info("{}: got stop event".format(event["id"]))
|
"""
|
||||||
Thread(target=self.end_listen_on, args=(event["id"],)).start()
|
Handle an event received from docker
|
||||||
|
"""
|
||||||
|
logging.info("{}: got {} event".format(event["id"], event["status"]))
|
||||||
|
|
||||||
except KeyboardInterrupt:
|
if event["status"] == "start":
|
||||||
logging.warning("Stopped listening for events")
|
Thread(target=self.relisten_on, args=(event["id"],)).start()
|
||||||
|
|
||||||
|
elif event["status"] == "stop":
|
||||||
|
Thread(target=self.end_listen_on, args=(event["id"],)).start()
|
||||||
|
|
||||||
def end_listen_on(self, container_id):
|
def end_listen_on(self, container_id):
|
||||||
"""
|
"""
|
||||||
|
@ -251,12 +211,31 @@ class LogInjectorDaemon(object):
|
||||||
modules_found = self.find_logs(ps_lines)
|
modules_found = self.find_logs(ps_lines)
|
||||||
logging.info("{}: logs detected: {}".format(container_id, str(modules_found)))
|
logging.info("{}: logs detected: {}".format(container_id, str(modules_found)))
|
||||||
|
|
||||||
modules_use = self.use_builtins.intersection({k for k, v in modules_found.items() if v})
|
modules_use = list(self.use_builtins.intersection({k for k, v in modules_found.items() if v}))
|
||||||
logging.info("{}: using: {}".format(container_id, str(modules_use)))
|
logging.info("{}: using: {}".format(container_id, str(modules_use)))
|
||||||
|
|
||||||
logfiles = []
|
if len(modules_use) == 0:
|
||||||
for mod in modules_use:
|
logging.info("{}: no log files found, exiting".format(container_id))
|
||||||
|
return None
|
||||||
|
|
||||||
|
syslog_conf = self.render_template(container_id, self.template, modules_use)
|
||||||
|
|
||||||
|
# transfer syslog conf
|
||||||
|
self.write_in_container(container_id, "/etc/rsyslog.conf", syslog_conf)
|
||||||
|
|
||||||
|
# start syslog
|
||||||
|
logging.info("{}: spawning rsyslogd".format(container_id))
|
||||||
|
self.exec_in_container(container_id, '/usr/sbin/rsyslogd')
|
||||||
|
|
||||||
|
def render_template(self, container_id, template_contents, log_modules):
|
||||||
|
"""
|
||||||
|
Create a rsyslog config from template
|
||||||
|
"""
|
||||||
|
|
||||||
|
# prepare template vars - only a list of detected log files
|
||||||
|
logfiles = []
|
||||||
|
|
||||||
|
for mod in log_modules:
|
||||||
for path in self.detectors[mod].paths:
|
for path in self.detectors[mod].paths:
|
||||||
original_logname = os.path.basename(path["path"])
|
original_logname = os.path.basename(path["path"])
|
||||||
# add local listener
|
# add local listener
|
||||||
|
@ -273,19 +252,8 @@ class LogInjectorDaemon(object):
|
||||||
"dest_port": new_port,
|
"dest_port": new_port,
|
||||||
"container_id": container_id}]
|
"container_id": container_id}]
|
||||||
|
|
||||||
if len(logfiles) == 0:
|
|
||||||
logging.info("{}: no log files found, exiting".format(container_id))
|
|
||||||
return
|
|
||||||
|
|
||||||
# generate syslog config
|
# generate syslog config
|
||||||
syslog_conf = Environment().from_string(self.template).render(logfiles=logfiles)
|
return Environment().from_string(template_contents).render(logfiles=logfiles)
|
||||||
|
|
||||||
# transfer syslog conf
|
|
||||||
self.write_in_container(container_id, "/etc/rsyslog.conf", syslog_conf)
|
|
||||||
|
|
||||||
# start syslog
|
|
||||||
logging.info("{}: spawning rsyslogd".format(container_id))
|
|
||||||
self.exec_in_container(container_id, '/usr/sbin/rsyslogd')
|
|
||||||
|
|
||||||
def get_container_name(self, container_id):
|
def get_container_name(self, container_id):
|
||||||
container_info = self.docker.inspect_container(container_id)
|
container_info = self.docker.inspect_container(container_id)
|
||||||
|
@ -295,20 +263,16 @@ class LogInjectorDaemon(object):
|
||||||
# strip leading slash
|
# strip leading slash
|
||||||
raw_name = raw_name[1:]
|
raw_name = raw_name[1:]
|
||||||
|
|
||||||
# hacky lazy loading
|
# hack: lazy loading of bridge ip - we must listen for udp packets on the docker bridge interface, so we need
|
||||||
|
# the IP for binding. Lazily set it after the first container is fetched from the docker host, as this will
|
||||||
|
# always happen before any udp binding
|
||||||
if not self.docker_bridge_ip:
|
if not self.docker_bridge_ip:
|
||||||
self.set_bridge_ip(container_info["NetworkSettings"]["Networks"]["bridge"]["Gateway"])
|
bridge_ip = container_info["NetworkSettings"]["Networks"]["bridge"]["Gateway"]
|
||||||
|
logging.info("Found bridge ip: {}".format(bridge_ip))
|
||||||
|
self.docker_bridge_ip = bridge_ip
|
||||||
|
|
||||||
return raw_name
|
return raw_name
|
||||||
|
|
||||||
def set_bridge_ip(self, bridge_ip):
|
|
||||||
"""
|
|
||||||
We must listen for udp packets on the docker bridge interface, so we need the IP for binding. Lazily set it
|
|
||||||
after the first container is fetched from the docker host, as this will always happen before any udp binding
|
|
||||||
"""
|
|
||||||
logging.info("Found bridge ip: {}".format(bridge_ip))
|
|
||||||
self.docker_bridge_ip = bridge_ip
|
|
||||||
|
|
||||||
def add_udp_listener(self, container_id, program, original_logname):
|
def add_udp_listener(self, container_id, program, original_logname):
|
||||||
"""
|
"""
|
||||||
Listen on a random UDP socket and create a new listener. A listener is an association between a udp port and
|
Listen on a random UDP socket and create a new listener. A listener is an association between a udp port and
|
||||||
|
@ -346,16 +310,19 @@ class LogInjectorDaemon(object):
|
||||||
|
|
||||||
return {name: hits[name] for name in self.detectors.keys()}
|
return {name: hits[name] for name in self.detectors.keys()}
|
||||||
|
|
||||||
def exec_in_container(self, container, cmd):
|
def exec_in_container(self, container_id, cmd_str):
|
||||||
e = self.docker.exec_create(container=container, cmd=cmd)
|
"""
|
||||||
|
Execute a command in a container
|
||||||
|
"""
|
||||||
|
e = self.docker.exec_create(container=container_id, cmd=cmd_str)
|
||||||
return self.docker.exec_start(e["Id"])
|
return self.docker.exec_start(e["Id"])
|
||||||
|
|
||||||
def write_in_container(self, container, path, contents):
|
def write_in_container(self, container_id, path, contents):
|
||||||
"""
|
"""
|
||||||
This is ugly and sucks
|
This is ugly and sucks
|
||||||
"""
|
"""
|
||||||
|
|
||||||
logging.info("{}: writing {} bytes to container's {}".format(container, len(contents), path))
|
logging.info("{}: writing {} bytes to container's {}".format(container_id, len(contents), path))
|
||||||
|
|
||||||
if type(contents) != bytes:
|
if type(contents) != bytes:
|
||||||
contents = contents.encode('UTF-8')
|
contents = contents.encode('UTF-8')
|
||||||
|
@ -367,7 +334,7 @@ class LogInjectorDaemon(object):
|
||||||
chunk = []
|
chunk = []
|
||||||
for byte in contents[chunk_size * i:chunk_size * i + chunk_size]:
|
for byte in contents[chunk_size * i:chunk_size * i + chunk_size]:
|
||||||
chunk.append('\\\\x' + hex(byte)[2:])
|
chunk.append('\\\\x' + hex(byte)[2:])
|
||||||
self.exec_in_container(container,
|
self.exec_in_container(container_id,
|
||||||
"bash -c -- 'printf {} {} {}'".format(''.join(chunk),
|
"bash -c -- 'printf {} {} {}'".format(''.join(chunk),
|
||||||
">" if i == 0 else ">>",
|
">" if i == 0 else ">>",
|
||||||
path))
|
path))
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
|
||||||
|
DEFAULT_TEMPLATE = """
|
||||||
|
$PrivDropToUser syslog
|
||||||
|
$PrivDropToGroup syslog
|
||||||
|
|
||||||
|
$template myFormat,"%rawmsg%\\n"
|
||||||
|
# $ActionFileDefaultTemplate myFormat
|
||||||
|
|
||||||
|
#
|
||||||
|
# Where to place spool and state files
|
||||||
|
#
|
||||||
|
$WorkDirectory /var/spool/rsyslog
|
||||||
|
|
||||||
|
#
|
||||||
|
# Provide file listening
|
||||||
|
#
|
||||||
|
|
||||||
|
module(load="imfile")
|
||||||
|
|
||||||
|
#
|
||||||
|
# Begin logs
|
||||||
|
#
|
||||||
|
|
||||||
|
{% for logfile in logfiles %}
|
||||||
|
#
|
||||||
|
# {{ logfile }}
|
||||||
|
#
|
||||||
|
|
||||||
|
input(type="imfile"
|
||||||
|
File="{{ logfile.path }}"
|
||||||
|
statefile="{{ logfile.statefile }}"
|
||||||
|
Tag="{{ logfile.program }}-{{ logfile.logname }}"
|
||||||
|
Severity="{{ logfile.program }}"
|
||||||
|
facility="local0")
|
||||||
|
|
||||||
|
if ($syslogtag == "{{ logfile.program }}-{{ logfile.logname }}") then {
|
||||||
|
local0.* @{{ logfile.dest_ip }}:{{ logfile.dest_port }};myFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
*.* /var/log/syslog
|
||||||
|
|
||||||
|
"""
|
Loading…
Reference in New Issue