dave
/
logcollector
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Misc fixes

This commit is contained in:
dave 2017-11-09 19:34:12 -08:00
parent 61c2a8cc79
commit 75da8c7a71
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sh
View File

@ -1,4 +1,4 @@
#!/bin/sh #!/bin/sh -ex
dpkg-buildpackage -us -uc -b dpkg-buildpackage -us -uc -b

12
src/archive.go
View File

@ -22,6 +22,7 @@ var (
cmd_import_dir = cmd_import.Flag("dir", "dir containing raw znc log files").Short('d').Required().String() cmd_import_dir = cmd_import.Flag("dir", "dir containing raw znc log files").Short('d').Required().String()
cmd_import_output = cmd_import.Flag("output", "dir to place created archives").Short('o').Required().String() cmd_import_output = cmd_import.Flag("output", "dir to place created archives").Short('o').Required().String()
cmd_import_all = cmd_import.Flag("all", "Import all log files, not only channels").Bool() cmd_import_all = cmd_import.Flag("all", "Import all log files, not only channels").Bool()
cmd_import_parallel = cmd_import.Flag("parallel", "How many importers can run in parallel").Short('p').Default("4").Int()
cmd_inspect = kingpin.Command("inspect", "Enumerate the contents of archives") cmd_inspect = kingpin.Command("inspect", "Enumerate the contents of archives")
cmd_inspect_fpath = cmd_inspect.Flag("file", "log archive file to inspect").Short('f').Required().String() cmd_inspect_fpath = cmd_inspect.Flag("file", "log archive file to inspect").Short('f').Required().String()
@ -110,6 +111,7 @@ func archive_log(logs []LogInfo, archive_path string) {
archive := CombinedLogfile{ archive := CombinedLogfile{
fpath: archive_path, fpath: archive_path,
} }
archive.Parse()
// For each log // For each log
for _, log := range logs { for _, log := range logs {
// Load the log into a LogPortion // Load the log into a LogPortion
@ -139,7 +141,7 @@ func archive_log(logs []LogInfo, archive_path string) {
// Entrypoint for the `import` command. Given an srcdir, scan it for log files. The log files will be sorted by channel // Entrypoint for the `import` command. Given an srcdir, scan it for log files. The log files will be sorted by channel
// and combined into an archive file per channel, placed in `outdir`. The `impall` flag determines whether only channel // and combined into an archive file per channel, placed in `outdir`. The `impall` flag determines whether only channel
// logs will be imported. If `true`, non-channel logs, such as PMs or server messages, will be archived too. // logs will be imported. If `true`, non-channel logs, such as PMs or server messages, will be archived too.
func cmd_import_do(srcdir string, outdir string, impall bool) { func cmd_import_do(srcdir string, outdir string, impall bool, parallel int) {
raw_logs := discover_logs(srcdir) raw_logs := discover_logs(srcdir)
// Sort logs by channel // Sort logs by channel
@ -153,7 +155,7 @@ func cmd_import_do(srcdir string, outdir string, impall bool) {
fmt.Printf("Discovered %v raw logs\n\n", len(raw_logs)) fmt.Printf("Discovered %v raw logs\n\n", len(raw_logs))
// For each channel // For each channel
wg := sizedwaitgroup.New(4) // TODO num cores wg := sizedwaitgroup.New(parallel)
for channel, logs := range bychannel { for channel, logs := range bychannel {
fmt.Printf("Reading %v portions for %s\n", len(logs), channel) fmt.Printf("Reading %v portions for %s\n", len(logs), channel)
@ -190,8 +192,8 @@ func cmd_inspect_do(fpath string) {
[]string{"network", log.Network}, []string{"network", log.Network},
[]string{"portions", strconv.Itoa(len(log.portions))}, []string{"portions", strconv.Itoa(len(log.portions))},
[]string{"lines", strconv.Itoa(log.TotalLines())}, []string{"lines", strconv.Itoa(log.TotalLines())},
[]string{"start", lmin.Format("2006-01-02")}, []string{"start", lmin.Format(ARCHTIMEFMT2)},
[]string{"end", lmax.Format("2006-01-02")}, []string{"end", lmax.Format(ARCHTIMEFMT2)},
} }
layout := &tabulate.Layout{Headers:[]string{"property", "value"}, Format:tabulate.SimpleFormat} layout := &tabulate.Layout{Headers:[]string{"property", "value"}, Format:tabulate.SimpleFormat}
asText, _ := tabulate.Tabulate(table, layout) asText, _ := tabulate.Tabulate(table, layout)
@ -236,7 +238,7 @@ func cmd_split_do(srcpath string, destdir string) {
func main() { func main() {
switch kingpin.Parse() { switch kingpin.Parse() {
case "import": case "import":
cmd_import_do(*cmd_import_dir, *cmd_import_output, *cmd_import_all) cmd_import_do(*cmd_import_dir, *cmd_import_output, *cmd_import_all, *cmd_import_parallel)
case "inspect": case "inspect":
cmd_inspect_do(*cmd_inspect_fpath) cmd_inspect_do(*cmd_inspect_fpath)
case "slice": case "slice":