From 1066539872aaa84ac98aa680fc1eb22b024cee9b Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 26 Jun 2021 10:51:33 -0700 Subject: [PATCH] initial commit --- .dockerignore | 1 + Dockerfile | 14 +++++++++ Makefile | 9 ++++++ cron.yml | 26 ++++++++++++++++ main.tf | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ notes.txt | 2 ++ requirements.txt | 12 +++++++ scrape.py | 71 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 216 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 cron.yml create mode 100644 main.tf create mode 100644 notes.txt create mode 100644 requirements.txt create mode 100644 scrape.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..70538b2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +./testenv/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..21066b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:focal + +RUN apt-get update && \ + apt-get install -y python3-pip + +ADD requirements.txt /tmp/requirements.txt + +RUN pip3 install -r /tmp/requirements.txt + +ADD scrape.py /usr/local/bin/scrape.py + +RUN chmod +x /usr/local/bin/scrape.py + +ENTRYPOINT ["/usr/local/bin/scrape.py"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5aa9bde --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ + +.PHONY: image +image: + docker build -t osrsgamestats . + +.PHONY: push +push: image + docker tag osrsgamestats dockermirror:5000/dpedu/osrsgamestats + docker push dockermirror:5000/dpedu/osrsgamestats diff --git a/cron.yml b/cron.yml new file mode 100644 index 0000000..5d8e62d --- /dev/null +++ b/cron.yml @@ -0,0 +1,26 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: osrsgamestats + namespace: jobs +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Replace + startingDeadlineSeconds: 60 + jobTemplate: + spec: + template: + spec: + containers: + - name: scrape + image: dockermirror:5000/dpedu/osrsgamestats + imagePullPolicy: Always + env: + - name: INFLUX_HOST + value: "influx01" + - name: INFLUX_PORT + value: "8086" + - name: INFLUX_DB + value: osrsgamestats + restartPolicy: Never diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..94df72f --- /dev/null +++ b/main.tf @@ -0,0 +1,81 @@ +terraform { + backend "local" {} + required_version = ">= 0.13" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.3.2" + } + } + experiments = [module_variable_optional_attrs] +} + + +provider "kubernetes" { + config_path = "~/.kube/config" + config_context = "scck8s" +} + +variable "influx_url" { + type = string + description = "influxdb connection url" +} + +variable "namespace" { + type = string +} + +variable "instance_name" { + type = string + default = "osrsgamestats" +} + +variable "image" { + type = string + default = "dockermirror:5000/dpedu/osrsgamestats" +} + +variable "suspend" { + type = bool + default = false +} + + +variable "schedule" { + type = string + description = "kube cron expression" + default = "*/10 * * * *" +} + +resource "kubernetes_cron_job" "osrsgamestats" { + metadata { + name = var.instance_name + namespace = var.namespace + } + spec { + schedule = var.schedule + concurrency_policy = "Replace" + starting_deadline_seconds = 60 + suspend = var.suspend + job_template { + metadata {} + spec { + template { + metadata {} + spec { + restart_policy = "Never" + container { + name = "scrape" + image = var.image + image_pull_policy = "Always" + env { + name = "RSSCRAPE_INFLUX_URL" + value = var.influx_url + } + } + } + } + } + } + } +} diff --git a/notes.txt b/notes.txt new file mode 100644 index 0000000..5c7f6fc --- /dev/null +++ b/notes.txt @@ -0,0 +1,2 @@ +# import twitch +# helix = twitch.Helix('zfiq9befa2btjxorcieajqmvvdtu5l', '3jtwgkmmf9ko39ytqjq19murthefhq') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ce568ad --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +certifi==2020.12.5 +chardet==4.0.0 +idna==2.10 +influxdb==5.3.1 +msgpack==1.0.2 +python-dateutil==2.8.1 +pytz==2021.1 +requests==2.25.1 +Rx==3.1.1 +six==1.15.0 +twitch-python==0.0.19 +urllib3==1.26.3 diff --git a/scrape.py b/scrape.py new file mode 100644 index 0000000..ac52d99 --- /dev/null +++ b/scrape.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import sys +import requests +import re +from influxdb import InfluxDBClient +from urllib.parse import urlparse +import os +import logging + + +def get_rs_players(session): + r = session.get("https://www.runescape.com/player_count.js?varname=iPlayerCount&callback=jQuery3600011173447649542423_1624513024284&_=1624513024285") + matches = re.compile(r'.+\((\d+)\)').findall(r.text) + return int(matches[0]) + + +def get_osrs_players(session): + r = session.get("https://oldschool.runescape.com") + matches = re.compile(r'There are currently ([\d,]+) people playing').findall(r.text) + return int(matches[0].replace(",", "")) + + +def get_influx(): + url_s = os.environ.get("RSSCRAPE_INFLUX_URL") + if not url_s: + logging.critical("must set RSSCRAPE_INFLUX_URL") + sys.exit(1) + + url = urlparse(url_s) + + db_name = url.path[1:] + + influx = InfluxDBClient(url.hostname, url.port, url.username, url.password) + influx.create_database(db_name) + influx.switch_database(db_name) + + return influx + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") + influx = get_influx() + + s = requests.session() + s.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:85.0) Gecko/20100101 Firefox/85.0" + s.headers["DNT"] = "1" + + all_players = get_rs_players(s) + osrs_players = get_osrs_players(s) + + rs3_players = all_players - osrs_players + + body = [ + { + "measurement": "players", + "tags": {}, + "fields": { + "rs3": rs3_players, + "osrs": osrs_players, + "both_games": all_players, + } + }, + ] + + print(body) + influx.write_points(body) + + +if __name__ == "__main__": + main()