misc py cleanup

osrsscrape/scrape.py

@@ -12,25 +12,6 @@ import traceback
 from collections import namedtuple
 
 
-timeout = int(os.environ.get("RSSCRAPE_TIMEOUT", 60))
-
-
-def get_influx():
-    influx_host = os.environ.get("INFLUX_HOST")
-    influx_port = os.environ.get("INFLUX_PORT")
-    influx_db = os.environ.get("INFLUX_DB")
-
-    if not all([influx_host, influx_port, influx_db]):
-        logging.critical("must set INFLUX_HOST, INFLUX_PORT and INFLUX_DB")
-        sys.exit(1)
-
-    influx = InfluxDBClient(influx_host, influx_port)  # user, password
-    influx.create_database(influx_db)
-    influx.switch_database(influx_db)
-
-    return influx
-
-
 # order of the skills in the stats data
 STATS = ["Overall",
          "Attack",
@@ -57,6 +38,9 @@ STATS = ["Overall",
          "Hunter",
          "Construction"]
 
+
+# level exp amounts
+# TODO 100+ virtual levels
 LEVELS = [0, 83, 174, 276, 388, 512, 650, 801, 969, 1154,
           1358, 1584, 1833, 2107, 2411, 2746, 3115, 3523, 3973, 4470,
           5018, 5624, 6291, 7028, 7842, 8740, 9730, 10824, 12031, 13363,
@@ -66,13 +50,18 @@ LEVELS = [0, 83, 174, 276, 388, 512, 650, 801, 969, 1154,
           302288, 333804, 368599, 407015, 449428, 496254, 547953, 605032, 668051, 737627,
           814445, 899257, 992895, 1096278, 1210421, 1336443, 1475581, 1629200, 1798808, 1986068,
           2192818, 2421087, 2673114, 2951373, 3258594, 3597792, 3972294, 4385776, 4842295, 5346332,
-          5902831, 6517253, 7195629, 7944614, 8771558, 9684577, 10692629, 11805606, 13034431]
+          5902831, 6517253, 7195629, 7944614, 8771558, 9684577, 10692629, 11805606, 13034431,  # 99
+          #14391160,  # 100
+          #15889109, 17542976, 19368992, 21385073, 23611006, 26068632, 28782069, 31777943, 35085654, 38737661,
+          #42769801, 47221641, 52136869, 57563718, 63555443, 70170840, 77474828, 85539082, 94442737, 104273167,
+          #115126838, 127110260, 140341028, 154948977, 171077457, 188884740, 200000000, ]
+          ]
 
 
-StatData = namedtuple("StatData", "skill rank level xp xp_next")
+StatData = namedtuple("StatData", "skill rank level xp xp_next")  # level_virtual xp_next_virtual
 
 
-def get_player(name):
+def get_player(name, timeout):
 
     data = None
 
@@ -119,49 +108,3 @@ def get_player(name):
             xp_next = LEVELS[level] - xp
         ret.append(StatData(STATS[i].lower(), int(row[0]), level, xp, xp_next))
     return ret
-
-
-def main():
-    logging.basicConfig(level=logging.WARNING,
-                        format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
-
-    influx = get_influx()
-
-    player_name = os.environ["PLAYER_NAME"]
-
-    if not player_name:
-        logging.critical("must set PLAYER_NAME")
-        sys.exit(1)
-
-    player_name = player_name.lower()
-
-    stats = get_player(player_name)
-
-    if stats is None:
-        print("failed to fetch player!")
-        sys.exit(1)
-
-    body = []
-
-    for measurement in ("rank", "level", "xp", "xp_next"):
-        fields = {}
-        for stat in stats:
-            fields[stat.skill] = getattr(stat, measurement)
-
-        blob = {"measurement": measurement,
-                "tags": {"username": player_name},
-                "fields": fields}
-
-        body.append(blob)
-
-    # import json
-    # print(json.dumps(body, indent=4))
-    # return
-
-    influx.write_points(body)
-
-    print("done!")
-
-
-if __name__ == '__main__':
-    main()

requirements.txt

@@ -1,9 +1,11 @@
-certifi==2019.11.28
-chardet==3.0.4
-idna==2.8
-influxdb==5.2.3
+certifi==2021.5.30
+chardet==4.0.0
+click==8.0.1
+idna==2.10
+influxdb==5.3.1
+msgpack==1.0.2
 python-dateutil==2.8.1
-pytz==2019.3
-requests==2.22.0
-six==1.13.0
-urllib3==1.25.7
+pytz==2021.1
+requests==2.25.1
+six==1.16.0
+urllib3==1.26.5