From 158a5fa20cf24aeca46459fb3b491a15f85de385 Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 14 Jul 2018 19:23:50 -0700 Subject: [PATCH] Geoip + elasticsearch setup --- src/Makefile | 36 ++++-- src/elasticsearch-template.json | 73 ++++++++++++ src/elasticsearch.c | 70 +++++++++++ src/elasticsearch.h | 2 + src/gdb.sh | 2 +- src/geo.c | 68 +++++++++++ src/geo.h | 8 ++ src/get-geoip.sh | 10 ++ src/helpers.c | 8 ++ src/helpers.h | 3 +- src/main.c | 133 +-------------------- src/msgbuffer.c | 90 +++++++++++++++ src/msgbuffer.h | 12 ++ src/msgbuffer.test | Bin 0 -> 12912 bytes src/pfparser.c | 28 +++++ src/put-elasticsearch-template.sh | 6 + src/server.c | 186 ++++++++++++++++++++++++++++++ src/server.h | 1 + 18 files changed, 597 insertions(+), 139 deletions(-) create mode 100644 src/elasticsearch-template.json create mode 100644 src/elasticsearch.c create mode 100644 src/elasticsearch.h create mode 100644 src/geo.c create mode 100644 src/geo.h create mode 100755 src/get-geoip.sh create mode 100644 src/helpers.c create mode 100644 src/msgbuffer.c create mode 100644 src/msgbuffer.h create mode 100755 src/msgbuffer.test create mode 100755 src/put-elasticsearch-template.sh create mode 100644 src/server.c create mode 100644 src/server.h diff --git a/src/Makefile b/src/Makefile index 5fb3793..d3c9dd6 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,24 +1,40 @@ CC=cc -CFLAGS := -g -I. -Wall -Wpedantic -I ../deps/build/usr/local/include/ -LDFLAGS = -L ../deps/build/usr/local/lib/ -LDLIBS = -ljson-c -lcurl -lpthread +override CFLAGS += -g -I. -Wall -Wpedantic -I ../deps/build/usr/local/include/ +LIBPATH = ../deps/build/usr/local/lib/ +LDFLAGS = -L $(LIBPATH) +LDLIBS = -ljson-c -lcurl -lpthread -lGeoIP LDFLAGS += $(LDLIBS) CFLAGS_STATIC = $(CFLAGS) --static -OBJ=main.o pfparser.o sysparser.o +OBJ=helpers.o pfparser.o sysparser.o msgbuffer.o geo.o elasticsearch.o server.o +TESTS=$(patsubst %.c,%.test,$(wildcard tests/*.c)) + %.o: %.c $(DEPS) $(CC) -c -o $@ $< $(CFLAGS) -csyslog: $(OBJ) +csyslog: $(OBJ) main.o $(CC) -o $@ $^ $(LDFLAGS) -.PHONY: clean -clean: - rm -vf *.o csyslog - -static: $(OBJ) +static: $(OBJ) main.o $(CC) -o csyslog $^ $(LDFLAGS) --static .PHONY: docker docker: static sudo docker build -t csyslog . + + +.PHONY: clean +clean: cleantests + rm -vf *.o csyslog + +.PHONY: cleantests +cleantests: + rm -vf tests/*.test + + +$(TESTS): tests/%.test: tests/%.c $(OBJ) + $(CC) -o $@ $(OBJ) tests/$*.c $(CFLAGS) $(LDFLAGS) + +.PHONY: tests +tests: $(TESTS) + @bash -c 'for test in $^ ; do printf "======== %-30s ========\n" $$test ; LD_LIBRARY_PATH=$(LIBPATH) ./$$test ; printf "%48s\n" "RC: $$?" ; done' diff --git a/src/elasticsearch-template.json b/src/elasticsearch-template.json new file mode 100644 index 0000000..e6f0358 --- /dev/null +++ b/src/elasticsearch-template.json @@ -0,0 +1,73 @@ +{ + "template": "firewall-*", + "index_patterns": [ + "firewall-*" + ], + "settings": { + "number_of_replicas": 0, + "number_of_shards": 8 + }, + "mappings": { + "event": { + "_source": { + "enabled": true + }, + "properties": { + "action": { + "type": "keyword" + }, + "app": { + "type": "keyword" + }, + "date": { + "type": "date" + }, + "dest_addr": { + "type": "ip" + }, + "dest_port": { + "type": "long" + }, + "interface": { + "type": "keyword" + }, + "ipversion": { + "type": "short" + }, + "length": { + "type": "long" + }, + "protocol_id": { + "type": "short" + }, + "src_addr": { + "type": "ip" + }, + "src_city": { + "type": "keyword" + }, + "src_country": { + "type": "keyword" + }, + "src_country_code": { + "type": "keyword" + }, + "src_port": { + "type": "long" + }, + "src_region": { + "type": "keyword" + }, + "src_state": { + "type": "keyword" + }, + "srcloc": { + "type": "geo_point" + }, + "ttl": { + "type": "long" + } + } + } + } +} diff --git a/src/elasticsearch.c b/src/elasticsearch.c new file mode 100644 index 0000000..d939657 --- /dev/null +++ b/src/elasticsearch.c @@ -0,0 +1,70 @@ +#include +#include + + +size_t write_data(char *dbit, size_t size, size_t nmemb, void *user_data) { + char *data = (char*)user_data; + static size_t data_size = 0; + size_t n = size * nmemb; + memcpy(data + data_size, dbit, n); + data_size += n; + data[data_size] = '\0'; + return n; +} + + +void test_curl() { + CURL *curl; + CURLcode res; + char data[50000] = ""; + + curl = curl_easy_init(); + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.1.120:8298/"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data); + + res = curl_easy_perform(curl); + + curl_easy_cleanup(curl); + curl_global_cleanup(); + printf("%d %s", res, data); +} + + + +int put_events(char* data) { + CURL *curl; + CURLcode res; + char response[50000] = ""; + + curl = curl_easy_init(); // check this and all of these curl functions + curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.1.120:8298/_bulk"); + curl_easy_setopt(curl, CURLOPT_POST, 1); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); + // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + + struct curl_slist *headers=NULL; + headers = curl_slist_append(headers, "Content-Type: application/json"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + res = curl_easy_perform(curl); + + curl_slist_free_all(headers); + + curl_easy_cleanup(curl); + curl_global_cleanup(); + + if(res != CURLE_OK) { + printf("%d %s\n", res, response); + } + + return res == CURLE_OK ? 0 : 1; +} + + + + diff --git a/src/elasticsearch.h b/src/elasticsearch.h new file mode 100644 index 0000000..3b4965d --- /dev/null +++ b/src/elasticsearch.h @@ -0,0 +1,2 @@ +void test_curl(); +int put_events(char* data); diff --git a/src/gdb.sh b/src/gdb.sh index 4108d10..178b266 100755 --- a/src/gdb.sh +++ b/src/gdb.sh @@ -1 +1 @@ -gdb --args ./csyslog 4200 +gdb -ex=r --args ./csyslog 4200 diff --git a/src/geo.c b/src/geo.c new file mode 100644 index 0000000..c8c2483 --- /dev/null +++ b/src/geo.c @@ -0,0 +1,68 @@ +#include +#include +#include "geo.h" + + +GeoIP *gi = NULL; +GeoIP *gi6 = NULL; + + +void geo_init() { + gi = GeoIP_open("GeoLiteCity.dat", GEOIP_INDEX_CACHE); + gi6 = GeoIP_open("GeoLiteCityv6.dat", GEOIP_INDEX_CACHE); + if (gi == NULL || gi6 == NULL) { + fprintf(stderr, "Error opening geoip databases\n"); + exit(1); + } +} + +void geo_close() { + GeoIP_delete(gi); + GeoIP_delete(gi6); +} + +GeoIPRecord* geo_get(char* ip) { + return GeoIP_record_by_name(gi, (const char *)ip); // GeoIP_record_by_name_v6 + // must be freed later with GeoIPRecord_delete() +} + +GeoIPRecord* geo_get6(char* ip) { + return GeoIP_record_by_name_v6(gi6, (const char *)ip); + // must be freed later with GeoIPRecord_delete() +} + +const char* geo_country_name(GeoIPRecord* rec) { + return GeoIP_country_name_by_id(gi, GeoIP_id_by_code(rec->country_code)); +} + +#ifdef TEST + +static const char * _mk_NA( const char * p ){ + return p ? p : "N/A"; +} + +int main(int argc, char** argv) { + geo_init(); + char* host = "24.4.129.164"; + char* host6 = "2601:647:4701:733:5bf:f3c2:f2b2:9c1f"; + + GeoIPRecord *gir = GeoIP_record_by_name(gi, (const char *) host); // GeoIP_record_by_name_v6 + // GeoIPRecord *gir = GeoIP_record_by_name_v6(gi, (const char *) host6); + + printf("%s\t%s\t%s\t%s\t%s\t%s\t%f\t%f\t%d\t%d\n", host, + _mk_NA(gir->country_code), + _mk_NA(gir->region), + _mk_NA(GeoIP_region_name_by_code(gir->country_code, gir->region)), + _mk_NA(gir->city), + _mk_NA(gir->postal_code), + gir->latitude, + gir->longitude, + gir->metro_code, + gir->area_code); + + GeoIPRecord_delete(gir); + + geo_close(); +} + +#endif diff --git a/src/geo.h b/src/geo.h new file mode 100644 index 0000000..860d11d --- /dev/null +++ b/src/geo.h @@ -0,0 +1,8 @@ +#include +#include + +void geo_init(); +void geo_close(); +GeoIPRecord* geo_get(char* ip); +GeoIPRecord* geo_get6(char* ip); +const char* geo_country_name(GeoIPRecord* rec); diff --git a/src/get-geoip.sh b/src/get-geoip.sh new file mode 100755 index 0000000..88449a3 --- /dev/null +++ b/src/get-geoip.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -x +set -e + +wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz +wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz + +rm -vf *.dat +gunzip --keep Geo*.gz diff --git a/src/helpers.c b/src/helpers.c new file mode 100644 index 0000000..8f0c0ee --- /dev/null +++ b/src/helpers.c @@ -0,0 +1,8 @@ +#include +#include + + +void panic(const char* s) { + perror(s); + exit(1); +} diff --git a/src/helpers.h b/src/helpers.h index e538499..e056fa8 100644 --- a/src/helpers.h +++ b/src/helpers.h @@ -1,4 +1,3 @@ - /*Convert a defined token to a string*/ #define _STR(x) #x @@ -6,3 +5,5 @@ Note: #x does not work in context, 2nd layer is required ?? http://www.decompile.com/cpp/faq/file_and_line_error_string.htm*/ #define STR(x) _STR(x) + +void panic(const char* s); diff --git a/src/main.c b/src/main.c index 4efc659..6d5023d 100644 --- a/src/main.c +++ b/src/main.c @@ -1,148 +1,27 @@ #include #include -#include -#include -#include #include -#include #include -#include -#include "helpers.h" -#include "sysparser.h" -#include - -#include - -void panic(const char* s) { - perror(s); - exit(1); -} +#include "server.h" -/*defined here as they are used in conjunction with the shutdown signal handler*/ -int running = 1; -int sock_fd; - - -void sig_handler(int signum) { - printf("\nExiting on signal %s\n", strsignal(signum)); - running = 0; /* shut down the main loop */ - shutdown(sock_fd, SHUT_RDWR); /* break the listener socket */ - close(sock_fd); -} - - -/*UDP server bits mostly lifted from https://cs.nyu.edu/~mwalfish/classes/16sp/classnotes/handout01.pdf*/ int main(int argc, char** argv) { if (argc != 2) { fprintf(stderr, "usage: %s \n", argv[0]); exit(1); } - signal(SIGTERM, sig_handler); - signal(SIGINT, sig_handler); - /*Parse port number to integer*/ char* portend; unsigned int portl; portl = strtol(argv[1], &portend, 10); - if (portend == NULL) panic("strtol"); + if (portend == NULL) { + fprintf(stderr, "usage: %s \n", argv[0]); + exit(1); + } assert(portl < USHRT_MAX); unsigned short port = (unsigned short)portl; - /*Create socket*/ - if ((sock_fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) - panic("socket"); - - /*Set socket options*/ - int one = 1; - setsockopt(sock_fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); - - /*Bind socket*/ - struct sockaddr_in my_addr, my_peer_addr; - memset(&my_addr, 0, sizeof(my_addr)); - my_addr.sin_family = AF_INET; - my_addr.sin_addr.s_addr = INADDR_ANY; - my_addr.sin_port = htons(port); /*host to network endianess for a short - converts a *s*hort from the *h*ost's to *n*etwork's endianness*/ - if (bind(sock_fd, (struct sockaddr*)&my_addr, sizeof(struct sockaddr_in)) < 0) - panic("bind failed"); - - time_t cur_t = {0}; - struct tm cur_time = {0}; - - socklen_t addrlen = sizeof(struct sockaddr_in); - char msg[4096]; - while (running) { - int size_recvd; - if ((size_recvd = recvfrom(sock_fd, /* socket */ - msg, /* buffer */ - sizeof(msg), /* buffer length */ - 0, /* no flags */ - (struct sockaddr*)&my_peer_addr, /* who’s sending */ - &addrlen /* length of buffer to receive peer info */ - )) < 0) { - if (running) panic("recvfrom"); - else break; /*sock was closed by exit signal*/ - } - assert(size_recvd < sizeof(msg)); /*messages can't be longer than our buffer. TODO if they are longer we should - dump it and wait until the next loop. if the next buffer is some portion of a too-long message, we can expect - the various parsing below to fail.*/ - - assert(addrlen == sizeof(struct sockaddr_in)); - /*printf("\nGot message: %s\n", msg);*/ - - /*TODO should we check that msg[size_recvd] == \0 ? - printf("From host %s src port %d got message %.*s\n", - inet_ntoa(my_peer_addr.sin_addr), ntohs(my_peer_addr.sin_port), size_recvd, msg);*/ - struct SysMessage result; - memset(&result, 0, sizeof(result)); /* Doing this or setting result above to `= {};` makes valgrind happy */ - /*printf("\nsize: %lu\n\n", sizeof(result)); // curious how big the struct gets - // printf("msg[size_recvd] is: %d", msg[size_recvd]);*/ - msg[size_recvd] = '\0'; /*We receive 1 full string at a time*/ - - /*parse syslog message into fields*/ - if(sysmsg_parse(&result, msg) != 0) { - printf("Failed to parse message: %s", msg); - } else { - /*printf("syslog message is valid:\n\tpriority: %d\n\tapplication: %s\n\tDate: %s %d %02d:%02d:%02d\n", - result.priority, - result.application, - result.date.month, - result.date.day, - result.date.hour, - result.date.minute, - result.date.second);*/ - - /*parse MSG field into pfsense data*/ - pf_data fwdata = {0}; - //memset(&fwdata, 0, sizeof(fwdata)); - if(pfdata_parse(msg, &fwdata) != 0) { - printf("Failed to parse pfsense data: %s\n\n", msg); - } else { - // pfdata_print(&fwdata); - - cur_t = time(NULL); - cur_time = *localtime(&cur_t); - - char date_formtted[32]; - sprintf(date_formtted, "%04d-%02d-%02dT%02d:%02d:%02dZ", - cur_time.tm_year + 1900, - month2num(result.date.month), - result.date.day, - result.date.hour, - result.date.minute, - result.date.second); - - json_object* jobj = json_object_new_object(); - add_strfield(jobj, "date", date_formtted); - add_strfield(jobj, "app", result.application); - pfdata_to_json(&fwdata, jobj); - printf("%s\n",json_object_to_json_string(jobj)); - json_object_put(jobj); - - } - } - } - + run_server(port); exit(EXIT_SUCCESS); } diff --git a/src/msgbuffer.c b/src/msgbuffer.c new file mode 100644 index 0000000..62fbc18 --- /dev/null +++ b/src/msgbuffer.c @@ -0,0 +1,90 @@ +#include +#include +#include +#include + + +struct msgbuffer_entry { + char* data; + struct msgbuffer_entry* next; +}; + + +struct msgbuffer_entry* msgbuffer = NULL; + + +void buff_push(char* data) { + struct msgbuffer_entry* oldhead = msgbuffer; + msgbuffer = malloc(sizeof(struct msgbuffer_entry)); + msgbuffer->data = data; // or strdup(data) if data should be moved to heap + msgbuffer->next = oldhead; +} + + +char* buff_pop() { + // assert(msgbuffer != NULL); // Popped when no items present + if(msgbuffer == NULL) return NULL; + char* result = msgbuffer->data; + struct msgbuffer_entry* oldhead = msgbuffer; + msgbuffer = msgbuffer->next; + free(oldhead); + return result; +} + + +int buff_count() { + int len = 0; + struct msgbuffer_entry* current = msgbuffer; + while (current != NULL) { + current = current->next; + len++; + } + return len; +} + + +void buff_freeall() { + struct msgbuffer_entry* current = msgbuffer; + struct msgbuffer_entry* next = msgbuffer; + while (current != NULL) { + next = current->next; + free(current->data); // maybe not desirable in reuse + free(current); + current = next; + } + msgbuffer = NULL; +} + + +char* buff_pop_head() { + // assert(msgbuffer != NULL); // Popped when no items present + if(msgbuffer == NULL) return NULL; + char* result = NULL; + struct msgbuffer_entry* previous = msgbuffer; + struct msgbuffer_entry* current = msgbuffer; + while (current->next != NULL) { + previous = current; + current = current->next; + } + result = current->data; + previous->next = NULL; + if(current == msgbuffer) msgbuffer = NULL; + free(current); + return result; +} + + +void buff_push_head(char* data) { + struct msgbuffer_entry* new = malloc(sizeof(struct msgbuffer_entry)); + new->data = data; + new->next = NULL; + if(msgbuffer == NULL) { + msgbuffer = new; + } else { + struct msgbuffer_entry* current = msgbuffer; + while (current->next != NULL) { + current = current->next; + } + current -> next = new; + } +} diff --git a/src/msgbuffer.h b/src/msgbuffer.h new file mode 100644 index 0000000..544e1ba --- /dev/null +++ b/src/msgbuffer.h @@ -0,0 +1,12 @@ + +void buff_push(char* data); + +char* buff_pop(); + +int buff_count(); + +void buff_freeall(); + +char* buff_pop_head(); + +void buff_push_head(char* data); diff --git a/src/msgbuffer.test b/src/msgbuffer.test new file mode 100755 index 0000000000000000000000000000000000000000..f54c37b219359cf98e73da8da6cb431af9dde2ad GIT binary patch literal 12912 zcmeHNdvH|M89$pP5MG;rJOW~Kv7iWLc^ZQf$tKxwQ^TW>h!60x$!?OBO*ZbM4OE77 zBD5GZFhox9&TzwdFr^PO|gz2}~Dz~4|;;BpC0ZgHoe((Z{0snU#<%X9{$N>qr+_`g9c z5F|E1Vv1j-2)HVZYST=MH0}l^y@~<_J>6wm;UtYe} z8;^Qd%XX7}lHItuN!CRD&7mcqXW=4@_}_SQ>HXU;O)?iY9l7`RgT;Fud-NzxBxmN4-VMyigA=mtzb3dxXa!37&W0&Uq#6z@7Kihy&+3$j+z(pF)66CpAv)xfldJ<-qAa;56XC`Q9ep z^A21$RjQnG;0T?N= zC%xhG8KlL>sZ6d8pOfXssZ1do9+2h7s7x*opOocCsZ6d8_scTrw;n}Vrd99Pr(Su$ zO#RL5eQRWEV?$rzK7`rqo9>pR@{h2TnSD2OE)|#65~p%63QcBT;X|P2x=}Orc48iy zvXqKw%Jf~C^R3?EvPY!-U4zt$u`=}6kzn7gtEHJ+u0Y>BbH#0@M$9wm4W{d?`O1|< zDGYvCstisegV(axQ#lMVkz8T+Rt{0FiTs&n95rSx<(Ic?(aY~dM{UF1#=Id*Xn zCea=e-!Vl!l=5F!c{l^n-u)vY`Jy>gxECfUjkVW+;Wjdi_LgD%{2c@5+4T@XS8qTb9bXHQ zzDwFGT(3o^$j}V;kl>Ji6ogv$1?3_2!*wbsC_!B zM34$wP`2q|1nmL_V}6I;u9s{finhl%9u?e$*ZqMr!|iW-57Jc0&4b$fwStw%`^e4SObeCY_Sct*bLj+X^7bombG`;^2Ow|4C_$(;wPERg)PeT zhiIS_-C>Kq1-4>7EL$&!00K+R@!L|rg%le+ZfhKe#=(IDPe}b22FE6=;xgge;k&y& z^_s8Ahq-a_#F=vHTTXPlFZF4C>f?r6Xx1n)&wT1m-%REYzMn8w{h8;hhSVnwsgG+@ zANVq}-Y|OyUFN#KB;TdE{{CIQoxWYZ2YlAx(XlEo4&nwNd-7>O&m@}Kh3JU4H78qJ zLosj5Sl(DzX>4w4Xb`Q@sAvwvaP1QhCA&p}UM*G{)l$QV<6XiC$8R?l2d9X&dC|%3 zp+L~+igtB{f{=?(5&3i{I|Za$f}FAdTVw!`>w5Pc^zJ(ySJgi=nE}v#&@|{tP!(dBI_jEKLhrE)aFwEN#}erp@V6xp?Z=ih9I` z`FAW`v*<>G$u1rr;w0pp?-^9rPF&BRJ`+^-!z^8bR*|%){Hvm5f{!3Z>vo`-S@Y#s*fh_($o4)`tPJq8B zi{EPV*Mt8Y_#|i7@1E--0gA~^;LoG)LZn~qDfv!8wWsum+wU>@CRBTte7n%}tUNT) z^i=dt+T^K~es1jZvU9)a-){9hx$`y9NlK}%suO!fAlcwKK=(Dx8ZmuLeCX`UkX{>k;a z%-<5`>oV`7EXd)1bR`od=$Kv{>uSURzOho`J&{wI&wCyx6kG0NQMOci{x4{pc5BKh17ZT^2ocz-$9;6xqxIhx+2X}P9#n&#?#UYYmFHrCYKZY*hP zPIe}e#%D-$5l2y#U6jz_*DBo%*LmQ5x#L83^%@BZq3tA zZsD=dT{y(i-eUW{&c?BK#1nyo;TALOa{~?-c4mqtJW)6pZgHJGPdH%6&k{U8WaG1O z9>Q=!GVd01?EBftbPL?QMn3scQIZ$u*=x)S&3hBXhnX=%<*4}uh}+SeczQH`E(c$z zah_L|6(Mdz#)a96{h|6O)!}mGi9-i)s+Z?!lBKj4mn%;^j{(o;|C3U_MD*)^B)=(L ztG%ZsZ>E@5EXA_^^4XW)XyH$8Khif`m$&o*B(fOUQ6^O+Gd#zAs zBDNQ&b^;wCE12x)*b5a$$%14eOVqi~XIWdf`5PPWwd$HS*EH5|*=$+f)$1zOz((CR z-zLB1-&||qTu-)@ps;H1-R#>`Uz5#|r-%UIM3?0^wFR?wo3J)EY^nA&SX=7qw)-2c zMqhPzy7R8{3Xdej<@L@c!_`Ps*iLcu^Hpieqwo5(r~mCaHci^+I9 zdop$kDq9d81@r2ZliH4?LsB?T1;y<1PgXD4<~ z^H|vx*1kQb@wl#E9@4U1SDT`|MT$yrJc~Px&TZvAj3vceqIkj6C&se9$UY*L z%@f}E-i}0|88i`7w4HM}W)_Nd32$dK5%T(~>z5}2ZMxLfne;X%!#FD#4hk=o+5?EY zH@LSG_7qLT6z8jF)s%>kjP924G9 zyS^RTgD}D+rK;`=rNMdB5ahHNbNwejYHk3K^$9zjrfT#DbdU>emAQ7;7J{pWn}!vj61Z^v>?}?+x|ziezlh4^)SSN{kwiiv)l+v^toj(|6ntfbwuu@rJneO})%-EB*%44rPH#Pqw!JN0=T#FYJ& zO8Db4(;w#O^Lq)?$2f4>0N2a*p2*SX??lhBI|8qI|yq;s4&KdBz^}hgpT4U(j#$NaF`R`nLE;D@ #include #include "pfparser.h" +#include "geo.h" int pfdata_parse(char* message, pf_data* result) { @@ -310,6 +311,17 @@ void add_strfield(json_object* obj, char* name, char* value) { } +void add_doublefield(json_object* obj, char* name, double value) { + json_object *number = json_object_new_double(value); + json_object_object_add(obj, name, number); +} + + +const char* null_unknown(const char* p){ + return p ? p : "unknown"; +} + + int pfdata_to_json(pf_data* data, json_object* obj) { /* Populate the passed json_object obj with data from from pf_data data. @@ -352,5 +364,21 @@ int pfdata_to_json(pf_data* data, json_object* obj) { } } + GeoIPRecord* ginfo = (data->ipversion == 4 ? geo_get(data->src_addr) + : geo_get6(data->src_addr)); + if(ginfo != NULL) { + json_object* srcloc = json_object_new_object(); + json_object_object_add(obj, "srcloc", srcloc); + add_doublefield(srcloc, "lat", ginfo->latitude); + add_doublefield(srcloc, "lon", ginfo->longitude); + add_strfield(obj, "src_country", (char*)null_unknown(geo_country_name(ginfo))); + add_strfield(obj, "src_country_code", (char*)null_unknown(ginfo->country_code)); + add_strfield(obj, "src_region", (char*)null_unknown(ginfo->region)); + add_strfield(obj, "src_state", (char*)null_unknown(GeoIP_region_name_by_code(ginfo->country_code, ginfo->region))); + add_strfield(obj, "src_city", (char*)null_unknown(ginfo->city)); + } + + GeoIPRecord_delete(ginfo); + return 0; } diff --git a/src/put-elasticsearch-template.sh b/src/put-elasticsearch-template.sh new file mode 100755 index 0000000..b394bc1 --- /dev/null +++ b/src/put-elasticsearch-template.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e +set -x + +curl -X PUT "http://homeapps1:8298/_template/firewall" -H Content-Type: application/json -d @elasticsearch-template.json diff --git a/src/server.c b/src/server.c new file mode 100644 index 0000000..b1c2657 --- /dev/null +++ b/src/server.c @@ -0,0 +1,186 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "helpers.h" +#include "sysparser.h" +#include "msgbuffer.h" +#include "geo.h" +#include "elasticsearch.h" + + +#define BUFF_MAX 5 + +/*defined here as they are used in conjunction with the shutdown signal handler*/ +int running = 1; +int sock_fd; + + +void sig_handler(int signum) { + printf("\nExiting on signal %s\n", strsignal(signum)); + running = 0; /* shut down the main loop */ + shutdown(sock_fd, SHUT_RDWR); /* break the listener socket */ + close(sock_fd); +} + + +time_t cur_t = {0}; +struct tm cur_time = {0}; + + +int handle_message(char* msg) { + /*TODO should we check that msg[size_recvd] == \0 ? + printf("From host %s src port %d got message %.*s\n", + inet_ntoa(my_peer_addr.sin_addr), ntohs(my_peer_addr.sin_port), size_recvd, msg);*/ + struct SysMessage result; + memset(&result, 0, sizeof(result)); /* Doing this or setting result above to `= {};` makes valgrind happy */ + /*printf("\nsize: %lu\n\n", sizeof(result)); // curious how big the struct gets + // printf("msg[size_recvd] is: %d", msg[size_recvd]);*/ + + /*parse syslog message into fields*/ + if(sysmsg_parse(&result, msg) != 0) { + printf("Failed to parse message: %s", msg); + } else { + /*printf("syslog message is valid:\n\tpriority: %d\n\tapplication: %s\n\tDate: %s %d %02d:%02d:%02d\n", + result.priority, + result.application, + result.date.month, + result.date.day, + result.date.hour, + result.date.minute, + result.date.second);*/ + + /*parse MSG field into pfsense data*/ + pf_data fwdata = {0}; + //memset(&fwdata, 0, sizeof(fwdata)); + if(pfdata_parse(msg, &fwdata) != 0) { + printf("Failed to parse pfsense data: %s\n\n", msg); + } else { + // pfdata_print(&fwdata); + + cur_t = time(NULL); + cur_time = *localtime(&cur_t); + + char date_formtted[32]; + sprintf(date_formtted, "%04d-%02d-%02dT%02d:%02d:%02dZ", + cur_time.tm_year + 1900, + month2num(result.date.month), + result.date.day, + result.date.hour, + result.date.minute, + result.date.second); + + json_object* jobj = json_object_new_object(); + add_strfield(jobj, "date", date_formtted); + add_strfield(jobj, "app", result.application); + pfdata_to_json(&fwdata, jobj); + const char* json_msg = json_object_to_json_string(jobj); + // printf("%s\n", json_msg); + buff_push(strdup(json_msg)); // Copy message to heap and push to buffer + json_object_put(jobj); + } + } + return 0; +} + + +void clear_buffer() { + char* header = "{\"index\": {\"_index\": \"firewall-test\", \"_type\": \"event\"}}\n"; + int header_size = strlen(header); + + // Calculate how large the payload will be + int num_messages = buff_count(); + char* messages[num_messages]; + int message_size = 0; + for(int i=0; i BUFF_MAX) { + printf("\n"); + clear_buffer(); + } + } + + printf("Clearing buffer, freeing %d entries\n", buff_count()); + buff_freeall(); + geo_close(); + return 1; +} diff --git a/src/server.h b/src/server.h new file mode 100644 index 0000000..f1fa1bd --- /dev/null +++ b/src/server.h @@ -0,0 +1 @@ +int run_server(int port);