initial commit

.dockerignore

@@ -0,0 +1 @@
+./testenv/*

Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:focal
+
+RUN apt-get update && \
+    apt-get install -y python3-pip
+
+ADD requirements.txt /tmp/requirements.txt
+
+RUN pip3 install -r /tmp/requirements.txt
+
+ADD scrape.py /usr/local/bin/scrape.py
+
+RUN chmod +x /usr/local/bin/scrape.py
+
+ENTRYPOINT ["/usr/local/bin/scrape.py"]

Makefile

@@ -0,0 +1,9 @@
+
+.PHONY: image
+image:
+	docker build -t osrsgamestats .
+
+.PHONY: push
+push: image
+	docker tag osrsgamestats dockermirror:5000/dpedu/osrsgamestats
+	docker push dockermirror:5000/dpedu/osrsgamestats

cron.yml

@@ -0,0 +1,26 @@
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: osrsgamestats
+  namespace: jobs
+spec:
+  schedule: "*/10 * * * *"
+  concurrencyPolicy: Replace
+  startingDeadlineSeconds: 60
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: scrape
+            image: dockermirror:5000/dpedu/osrsgamestats
+            imagePullPolicy: Always
+            env:
+            - name: INFLUX_HOST
+              value: "influx01"
+            - name: INFLUX_PORT
+              value: "8086"
+            - name: INFLUX_DB
+              value: osrsgamestats
+          restartPolicy: Never

main.tf

@@ -0,0 +1,81 @@
+terraform {
+  backend "local" {}
+  required_version = ">= 0.13"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.3.2"
+    }
+  }
+  experiments = [module_variable_optional_attrs]
+}
+
+
+provider "kubernetes" {
+  config_path    = "~/.kube/config"
+  config_context = "scck8s"
+}
+
+variable "influx_url" {
+  type        = string
+  description = "influxdb connection url"
+}
+
+variable "namespace" {
+  type = string
+}
+
+variable "instance_name" {
+  type    = string
+  default = "osrsgamestats"
+}
+
+variable "image" {
+  type    = string
+  default = "dockermirror:5000/dpedu/osrsgamestats"
+}
+
+variable "suspend" {
+  type    = bool
+  default = false
+}
+
+
+variable "schedule" {
+  type        = string
+  description = "kube cron expression"
+  default     = "*/10 * * * *"
+}
+
+resource "kubernetes_cron_job" "osrsgamestats" {
+  metadata {
+    name      = var.instance_name
+    namespace = var.namespace
+  }
+  spec {
+    schedule                  = var.schedule
+    concurrency_policy        = "Replace"
+    starting_deadline_seconds = 60
+    suspend                   = var.suspend
+    job_template {
+      metadata {}
+      spec {
+        template {
+          metadata {}
+          spec {
+            restart_policy = "Never"
+            container {
+              name              = "scrape"
+              image             = var.image
+              image_pull_policy = "Always"
+              env {
+                name  = "RSSCRAPE_INFLUX_URL"
+                value = var.influx_url
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

notes.txt

@@ -0,0 +1,2 @@
+# import twitch
+# helix = twitch.Helix('zfiq9befa2btjxorcieajqmvvdtu5l', '3jtwgkmmf9ko39ytqjq19murthefhq')

requirements.txt

@@ -0,0 +1,12 @@
+certifi==2020.12.5
+chardet==4.0.0
+idna==2.10
+influxdb==5.3.1
+msgpack==1.0.2
+python-dateutil==2.8.1
+pytz==2021.1
+requests==2.25.1
+Rx==3.1.1
+six==1.15.0
+twitch-python==0.0.19
+urllib3==1.26.3

scrape.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import sys
+import requests
+import re
+from influxdb import InfluxDBClient
+from urllib.parse import urlparse
+import os
+import logging
+
+
+def get_rs_players(session):
+    r = session.get("https://www.runescape.com/player_count.js?varname=iPlayerCount&callback=jQuery3600011173447649542423_1624513024284&_=1624513024285")
+    matches = re.compile(r'.+\((\d+)\)').findall(r.text)
+    return int(matches[0])
+
+
+def get_osrs_players(session):
+    r = session.get("https://oldschool.runescape.com")
+    matches = re.compile(r'There are currently ([\d,]+) people playing').findall(r.text)
+    return int(matches[0].replace(",", ""))
+
+
+def get_influx():
+    url_s = os.environ.get("RSSCRAPE_INFLUX_URL")
+    if not url_s:
+        logging.critical("must set RSSCRAPE_INFLUX_URL")
+        sys.exit(1)
+
+    url = urlparse(url_s)
+
+    db_name = url.path[1:]
+
+    influx = InfluxDBClient(url.hostname, url.port, url.username, url.password)
+    influx.create_database(db_name)
+    influx.switch_database(db_name)
+
+    return influx
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
+    influx = get_influx()
+
+    s = requests.session()
+    s.headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:85.0) Gecko/20100101 Firefox/85.0"
+    s.headers["DNT"] = "1"
+
+    all_players = get_rs_players(s)
+    osrs_players = get_osrs_players(s)
+
+    rs3_players = all_players - osrs_players
+
+    body = [
+        {
+            "measurement": "players",
+            "tags": {},
+            "fields": {
+                "rs3": rs3_players,
+                "osrs": osrs_players,
+                "both_games": all_players,
+            }
+        },
+    ]
+
+    print(body)
+    influx.write_points(body)
+
+
+if __name__ == "__main__":
+    main()