commit 1066539872aaa84ac98aa680fc1eb22b024cee9b Author: dave Date: Sat Jun 26 10:51:33 2021 -0700 initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..70538b2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +./testenv/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..21066b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:focal + +RUN apt-get update && \ + apt-get install -y python3-pip + +ADD requirements.txt /tmp/requirements.txt + +RUN pip3 install -r /tmp/requirements.txt + +ADD scrape.py /usr/local/bin/scrape.py + +RUN chmod +x /usr/local/bin/scrape.py + +ENTRYPOINT ["/usr/local/bin/scrape.py"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5aa9bde --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ + +.PHONY: image +image: + docker build -t osrsgamestats . + +.PHONY: push +push: image + docker tag osrsgamestats dockermirror:5000/dpedu/osrsgamestats + docker push dockermirror:5000/dpedu/osrsgamestats diff --git a/cron.yml b/cron.yml new file mode 100644 index 0000000..5d8e62d --- /dev/null +++ b/cron.yml @@ -0,0 +1,26 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: osrsgamestats + namespace: jobs +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Replace + startingDeadlineSeconds: 60 + jobTemplate: + spec: + template: + spec: + containers: + - name: scrape + image: dockermirror:5000/dpedu/osrsgamestats + imagePullPolicy: Always + env: + - name: INFLUX_HOST + value: "influx01" + - name: INFLUX_PORT + value: "8086" + - name: INFLUX_DB + value: osrsgamestats + restartPolicy: Never diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..94df72f --- /dev/null +++ b/main.tf @@ -0,0 +1,81 @@ +terraform { + backend "local" {} + required_version = ">= 0.13" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.3.2" + } + } + experiments = [module_variable_optional_attrs] +} + + +provider "kubernetes" { + config_path = "~/.kube/config" + config_context = "scck8s" +} + +variable "influx_url" { + type = string + description = "influxdb connection url" +} + +variable "namespace" { + type = string +} + +variable "instance_name" { + type = string + default = "osrsgamestats" +} + +variable "image" { + type = string + default = "dockermirror:5000/dpedu/osrsgamestats" +} + +variable "suspend" { + type = bool + default = false +} + + +variable "schedule" { + type = string + description = "kube cron expression" + default = "*/10 * * * *" +} + +resource "kubernetes_cron_job" "osrsgamestats" { + metadata { + name = var.instance_name + namespace = var.namespace + } + spec { + schedule = var.schedule + concurrency_policy = "Replace" + starting_deadline_seconds = 60 + suspend = var.suspend + job_template { + metadata {} + spec { + template { + metadata {} + spec { + restart_policy = "Never" + container { + name = "scrape" + image = var.image + image_pull_policy = "Always" + env { + name = "RSSCRAPE_INFLUX_URL" + value = var.influx_url + } + } + } + } + } + } + } +} diff --git a/notes.txt b/notes.txt new file mode 100644 index 0000000..5c7f6fc --- /dev/null +++ b/notes.txt @@ -0,0 +1,2 @@ +# import twitch +# helix = twitch.Helix('zfiq9befa2btjxorcieajqmvvdtu5l', '3jtwgkmmf9ko39ytqjq19murthefhq') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ce568ad --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +certifi==2020.12.5 +chardet==4.0.0 +idna==2.10 +influxdb==5.3.1 +msgpack==1.0.2 +python-dateutil==2.8.1 +pytz==2021.1 +requests==2.25.1 +Rx==3.1.1 +six==1.15.0 +twitch-python==0.0.19 +urllib3==1.26.3 diff --git a/scrape.py b/scrape.py new file mode 100644 index 0000000..ac52d99 --- /dev/null +++ b/scrape.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import sys +import requests +import re +from influxdb import InfluxDBClient +from urllib.parse import urlparse +import os +import logging + + +def get_rs_players(session): + r = session.get("https://www.runescape.com/player_count.js?varname=iPlayerCount&callback=jQuery3600011173447649542423_1624513024284&_=1624513024285") + matches = re.compile(r'.+\((\d+)\)').findall(r.text) + return int(matches[0]) + + +def get_osrs_players(session): + r = session.get("https://oldschool.runescape.com") + matches = re.compile(r'There are currently ([\d,]+) people playing').findall(r.text) + return int(matches[0].replace(",", "")) + + +def get_influx(): + url_s = os.environ.get("RSSCRAPE_INFLUX_URL") + if not url_s: + logging.critical("must set RSSCRAPE_INFLUX_URL") + sys.exit(1) + + url = urlparse(url_s) + + db_name = url.path[1:] + + influx = InfluxDBClient(url.hostname, url.port, url.username, url.password) + influx.create_database(db_name) + influx.switch_database(db_name) + + return influx + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") + influx = get_influx() + + s = requests.session() + s.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:85.0) Gecko/20100101 Firefox/85.0" + s.headers["DNT"] = "1" + + all_players = get_rs_players(s) + osrs_players = get_osrs_players(s) + + rs3_players = all_players - osrs_players + + body = [ + { + "measurement": "players", + "tags": {}, + "fields": { + "rs3": rs3_players, + "osrs": osrs_players, + "both_games": all_players, + } + }, + ] + + print(body) + influx.write_points(body) + + +if __name__ == "__main__": + main()