From 2bdece8b9e8ff71e954553b512cda925fc2ee44d Mon Sep 17 00:00:00 2001 From: dave Date: Wed, 8 Apr 2020 22:56:07 -0700 Subject: [PATCH] support multiple stock apis --- docs/api/modules/stockplay.rst | 46 ++--- pyircbot/modules/StockPlay.py | 312 +++++++++++++++++++++++--------- tests/modules/test_stockplay.py | 72 ++++++++ 3 files changed, 323 insertions(+), 107 deletions(-) create mode 100644 tests/modules/test_stockplay.py diff --git a/docs/api/modules/stockplay.rst b/docs/api/modules/stockplay.rst index e2a7a3d..2e9c8a8 100644 --- a/docs/api/modules/stockplay.rst +++ b/docs/api/modules/stockplay.rst @@ -15,13 +15,12 @@ Considering the daily limit means, when evenly spread, we can sent a request *no `(24 * 60 * 60 / 500)` - and therefore, the value of *bginterval* must be some value larger than 173, as this value will completely consume the daily limit. -When trading, the price of the traded symbol is allowed to be *tcachesecs* seconds old before the API will be used to -fetch a more recent price. This value must be balanced against *bginterval* depending on your trade frequency +When trading, the price of the traded symbol is allowed to be *trade_cache_seconds* seconds old before the API will be +used to fetch a more recent price. This value must be balanced against *bginterval* depending on your trade frequency and variety. Background or batch-style tasks that rely on symbol prices run afoul with the above constraints - but in a -magnified way as they rely on api-provided data to calculate player stats across many players at a time. The -*rcachesecs* setting controls the maximum price age before the API is hit. +magnified way as they rely on api-provided data to calculate player stats across many players at a time. Commands @@ -39,8 +38,7 @@ Commands Get a report on the calling player's portfolio. Another player's name can be passed as an argument to retrieve information about a player other than the calling player. Finally, the 'full' argument can be added to retrieve a - full listing of the player's holdings. Per the above, values based on symbol prices may be delayed based on the - *rcachesecs* config setting. + full listing of the player's holdings. Config @@ -51,13 +49,20 @@ Config { "startbalance": 10000, "tradedelay": 0, - "apikey": "xxxxxxxxxxxxxx", - "tcachesecs": 300, - "rcachesecs": 14400, + "trade_cache_seconds": 300, "bginterval": 300, - "midnight_offset": 0, "announce_trades": false, - "announce_channel": "#trades" + "announce_channel": "#trades", + "providers": [ + { + "provider": "iexcloud", + "apikey": "xxxxxxxxxxxxxxx" + }, + { + "provider": "alphavantage", + "apikey": "xxxxxxxxxxxxxxx" + } + ] } .. cmdoption:: startbalance @@ -71,22 +76,21 @@ Config NOT IMPLEMENTED -.. cmdoption:: apikey +.. cmdoption:: providers - API key from https://www.alphavantage.co/support/#api-key + A list of providers to fetch stock data from -.. cmdoption:: tcachesecs + Supported providers: + + * https://www.alphavantage.co/ + * https://iexcloud.io/ + +.. cmdoption:: trade_cache_seconds When performing a trade, how old of a cached symbol price is permitted before fetching from API. Recommended ~30 minutes (1800) -.. cmdoption:: rcachesecs - - When calculating a portfolio report, how old of a cached symbol price is permitted before fetching from API. - - Recommended ~4 hours (14400) - .. cmdoption:: bginterval Symbol prices are updated in the background. This is necessary because fetching a portfolio report may require @@ -94,7 +98,7 @@ Config fetching a report would take multiple minutes with more than 5 symbols, which would not work. For this reason, we update symbols at a low interval in the background. Every *bginterval* seconds, a task will be - started that updates the price of symbols older than *rcachesecs*. + started that updates the price of the oldest symbol. Estimated 5 minute (300), but likely will need tuning depending on playerbase diff --git a/pyircbot/modules/StockPlay.py b/pyircbot/modules/StockPlay.py index b530745..5f6bb32 100644 --- a/pyircbot/modules/StockPlay.py +++ b/pyircbot/modules/StockPlay.py @@ -136,12 +136,15 @@ class StockPlay(ModuleBase): # `data` text # );""") + self.cache = PriceCache(self) + # Last time the interval tasks were executed self.task_time = 0 # background work executor thread self.asyncq = Queue() self.running = True + self.trader = Thread(target=self.trader_background) self.trader.start() @@ -196,7 +199,7 @@ class StockPlay(ModuleBase): c.execute("UPDATE stockplay_prices SET attempt_time=? WHERE symbol=?;", (time(), updatesym)) if updatesym: - self.get_price(updatesym, 0) + self.cache.get_price(updatesym, 0) except Exception: traceback.print_exc() @@ -262,19 +265,18 @@ class StockPlay(ModuleBase): trade.symbol)) # Update quote price try: - symprice = self.get_price(trade.symbol, self.config["tcachesecs"]) + price = self.cache.get_price(trade.symbol, self.config["trade_cache_seconds"]) except Exception: traceback.print_exc() self.bot.act_PRIVMSG(trade.replyto, "{}: invalid symbol or api failure, trade aborted!" .format(trade.nick)) return - if symprice is None: + if price is None: self.bot.act_PRIVMSG(trade.replyto, "{}: invalid symbol '{}'".format(trade.nick, trade.symbol)) return # invalid stock - if not symprice and trade.buy: - self.bot.act_PRIVMSG(trade.replyto, "{}: trading is halted on '{}'".format(trade.nick, trade.symbol)) - return + + symprice = price.price # calculate various prices needed # symprice -= Decimal("0.0001") # for testing dust collection @@ -394,9 +396,10 @@ class StockPlay(ModuleBase): (nick, )).fetchall(): # the API limits us to 5 requests per minute or 500 requests per day or about 1 request every 173s # The background thread updates the oldest price every 5 minutes. Here, we allow even very stale quotes - # because it's simply impossible to request fresh data for every stock right now. Recommended rcachesecs - # is 86400 (1 day) - symprice = Decimal(self.get_price(row["symbol"], -1)) + # because it's simply impossible to request fresh data for every stock right now. + print("build_report: processing", row["symbol"]) + price = self.cache.get_price(row["symbol"], -1) + symprice = price.price holding_value += symprice * row["count"] avgbuy = self.calc_user_avgbuy(nick, row["symbol"]) symbol_count.append((row["symbol"], @@ -435,83 +438,6 @@ class StockPlay(ModuleBase): self.task_time = now self.record_nightly_balances() - def get_price(self, symbol, thresh=None): - """ - Get symbol price, with quote being at most $thresh seconds old - """ - return self.get_priceinfo_cached(symbol, thresh or 60)["price"] - - def get_priceinfo_cached(self, symbol, thresh): - """ - Return the cached symbol price if it's more recent than the last 15 minutes - Otherwise, fetch the price then cache and return it. - """ - cached = self._get_cache_priceinfo(symbol, thresh) - if not cached: - cached = self.fetch_priceinfo(symbol) - if cached: - self._set_cache_priceinfo(symbol, cached) - - numfields = set(['open', 'high', 'low', 'price', 'volume', 'change', 'previous close']) - return {k: Decimal(v) if k in numfields else v for k, v in cached.items()} - - def _set_cache_priceinfo(self, symbol, data): - with closing(self.sql.getCursor()) as c: - c.execute("REPLACE INTO stockplay_prices (symbol, attempt_time, time, data) VALUES (?, ?, ?, ?)", - (symbol, time(), time(), json.dumps(data))) - - def _get_cache_priceinfo(self, symbol, thresh): - with closing(self.sql.getCursor()) as c: - row = c.execute("SELECT * FROM stockplay_prices WHERE symbol=?", - (symbol, )).fetchone() - if not row: - return - if thresh != -1 and time() - row["time"] > thresh: - return - return json.loads(row["data"]) - - def fetch_priceinfo(self, symbol): - """ - Request a stock quote from the API. The API provides the format:: - - {'Global Quote': { - {'01. symbol': 'MSFT', - '02. open': '104.3900', - '03. high': '105.7800', - '04. low': '104.2603', - '05. price': '105.6700', - '06. volume': '21461093', - '07. latest trading day':'2019-02-08', - '08. previous close':'105.2700', - '09. change': '0.4000', - '10. change percent': '0.3800%'}} - - Reformat as:: - - {'symbol': 'AMD', - 'open': '22.3300', - 'high': '23.2750', - 'low': '22.2700', - 'price': '23.0500', - 'volume': '78129280', - 'latest trading day': '2019-02-08', - 'previous close': '22.6700', - 'change': '0.3800', - 'change percent': '1.6762%'} - """ - keys = set(['symbol', 'open', 'high', 'low', 'price', 'volume', - 'latest trading day', 'previous close', 'change', 'change percent']) - self.log.info("fetching api quote for symbol: {}".format(symbol)) - data = get("https://www.alphavantage.co/query", - params={"function": "GLOBAL_QUOTE", - "symbol": symbol, - "apikey": self.config["apikey"]}, - timeout=10).json() - data = data["Global Quote"] - if not data: - return None - return {k[4:]: v for k, v in data.items() if k[4:] in keys} - def checksym(self, s): """ Validate that a string looks like a stock symbol @@ -672,3 +598,217 @@ class StockPlay(ModuleBase): total = int((data["cash"] + data["holding_value"]) * 100) self.log.info("Recording {} daily balance for {}".format(now, row["nick"])) c.execute("INSERT INTO stockplay_balance_history VALUES (?, ?, ?)", (row["nick"], now, total)) + + +class PriceCache(object): + def __init__(self, mod): + self.sql = mod.sql + self.log = mod.log + self.mod = mod + self.providers = [] + + self.configure_providers(mod.config["providers"]) + self.which_provider = dict() + self.unsupported = set() + + def configure_providers(self, config): + for provider in config: + self.providers.append(PROVIDER_TYPES[provider["provider"]](provider, self.log)) + + def get_price(self, symbol, thresh): + if symbol in self.unsupported: + return + symbol = symbol.upper() + # load from cache + price = self._load_priceinfo(symbol) + # if present and meets thresh + if price and (thresh == -1 or time() - price.time < thresh): + return price + + return self.api_fetch(symbol) + + def api_fetch(self, symbol): + fetched = None + + if symbol in self.which_provider: + fetched = self.which_provider[symbol].get_price(symbol) + + if not fetched: + for provider in self.providers: + try: + fetched = provider.get_price(symbol) + self.which_provider[symbol] = provider + break + except NotSupported as nse: + self.unsupported.update([symbol]) + self.log.info("provider {}: {}".format(provider.__class__.__name__, nse)) + + if not fetched: + self.log.critical("unsupported symbol: %s", symbol) + return + + self._store_priceinfo(fetched) + + return fetched + + def _store_priceinfo(self, price): + with closing(self.sql.getCursor()) as c: + c.execute("REPLACE INTO stockplay_prices (symbol, attempt_time, time, data) VALUES (?, ?, ?, ?)", + (price.symbol, price.time, time(), price.to_json())) + + def _load_priceinfo(self, symbol): + with closing(self.sql.getCursor()) as c: + row = c.execute("SELECT * FROM stockplay_prices WHERE symbol=?", + (symbol, )).fetchone() + if not row: + return + return Price.from_json(row["data"]) + + +class Price(object): + def __init__(self, symbol, price, time_): + self.symbol = symbol.upper() + self.price = round(Decimal(price), 4) + self.time = time_ + + def to_json(self): + return json.dumps({ + "symbol": self.symbol, + "price": str(self.price), + "time": self.time, + }) + + @staticmethod + def from_json(data): + data = json.loads(data) + return Price(data["symbol"].upper(), data["price"], data.get("time", 0)) + + +class PriceProvider(object): + def __init__(self, config, logger): + """ + config:: + + { + "provider": "name", + "apikey": "xxxxxxxxxxxxxx", + } + + """ + self.config = config + self.log = logger + + def get_price(self, symbol): + """ + :return: tuple of: + + * price (as a Decimal) + * next_after (the time() after which the next background call should happen) + + or raise: + + NotSupported - if the symbol isnt supported + """ + raise NotImplementedError() + + +class IEXCloudProvider(PriceProvider): + def get_price(self, symbol): + """ + Request a stock quote from the API. The API provides the format:: + + {"symbol": "AAPL", + "companyName": "Apple, Inc.", + "calculationPrice": "close", + "open": 184.7, + "openTime": 1552656600847, + "close": 186.12, + "closeTime": 1552680000497, + "high": 187.33, + "low": 183.74, + "latestPrice": 186.12, + "latestSource": "Close", + "latestTime": "March 15, 2019", + "latestUpdate": 1552680000497, + "latestVolume": 39141464, + "iexRealtimePrice": 186.195, + "iexRealtimeSize": 100, + "iexLastUpdated": 1552679999536, + "delayedPrice": 186.124, + "delayedPriceTime": 1552680900008, + "extendedPrice": 185.92, + "extendedChange": -0.2, + "extendedChangePercent": -0.00107, + "extendedPriceTime": 1552693471549, + "previousClose": 183.73, + "change": 2.39, + "changePercent": 0.01301, + "iexMarketPercent": 0.021849182749015213, + "iexVolume": 855209, + "avgTotalVolume": 25834564, + "iexBidPrice": 0, + "iexBidSize": 0, + "iexAskPrice": 0, + "iexAskSize": 0, + "marketCap": 877607913600, + "peRatio": 15.17, + "week52High": 233.47, + "week52Low": 142, + "ytdChange": 0.176447} + """ + + self.log.info("{}: fetching api quote for symbol: {}".format(self.__class__.__name__, symbol)) + + response = get("https://cloud.iexapis.com/beta/stock/{}/quote".format(symbol.lower()), + params={"token": self.config["apikey"]}, + timeout=10) + + if response.status_code != 200: + if response.status_code == 404: + raise NotSupported(symbol) + else: + response.raise_for_status() + + data = response.json() + return Price(symbol, Decimal(data["latestPrice"]), int(time())) + + +class AlphaVantProvider(PriceProvider): + def get_price(self, symbol): + """ + Request a stock quote from the API. The API provides the format:: + + {'Global Quote': { + {'01. symbol': 'MSFT', + '02. open': '104.3900', + '03. high': '105.7800', + '04. low': '104.2603', + '05. price': '105.6700', + '06. volume': '21461093', + '07. latest trading day':'2019-02-08', + '08. previous close':'105.2700', + '09. change': '0.4000', + '10. change percent': '0.3800%'}} + """ + self.log.info("{}: fetching api quote for symbol: {}".format(self.__class__.__name__, symbol)) + + data = get("https://www.alphavantage.co/query", + params={"function": "GLOBAL_QUOTE", + "symbol": symbol, + "apikey": self.config["apikey"]}, + timeout=10).json() + + if "Global Quote" not in data: + raise NotSupported(symbol) + + return Price(symbol, Decimal(data["Global Quote"]["05. price"]), int(time())) + + +PROVIDER_TYPES = { + "iexcloud": IEXCloudProvider, + "alphavantage": AlphaVantProvider +} + + +class NotSupported(Exception): + pass diff --git a/tests/modules/test_stockplay.py b/tests/modules/test_stockplay.py new file mode 100644 index 0000000..0638fe5 --- /dev/null +++ b/tests/modules/test_stockplay.py @@ -0,0 +1,72 @@ +import pytest +from contextlib import closing +from tests.lib import * # NOQA - fixtures +from time import sleep, time +import datetime + + +@pytest.fixture +def stockbot(fakebot): + """ + Provide a bot loaded with the Calc module. Clear the database. + """ + fakebot.botconfig["module_configs"]["StockPlay"] = { + "startbalance": 10000, + "tradedelay": 0, + "tcachesecs": 120, + "bginterval": 45, + "announce_trades": True, + "announce_channel": "#trades", + "providers": [ + { + "provider": "iexcloud", + "apikey": "xxxxxxxxxxxxxxxxxxxxxx", + "background_interval": 1 + }, + { + "provider": "alphavantage", + "apikey": "xxxxxxxxxxxxxxxxxxxxxx", + "background_interval": 1 + } + ] + } + fakebot.loadmodule("SQLite") + # with closing(fakebot.moduleInstances["SQLite"].opendb("remind.db")) as db: + # db.query("DROP TABLE IF EXISTS `reminders`;") + # fakebot.loadmodule("Remind") + + # os.system("cp /Users/dave/code/pyircbot-work/examples/data2/data/SQLite/stockplay.db {}".format(fakebot.moduleInstances["SQLite"].getFilePath())) + # os.system("ln -s /Users/dave/code/pyircbot-work/examples/data2/data/SQLite/stockplay.db {}".format(fakebot.moduleInstances["SQLite"].getFilePath())) + + fakebot.loadmodule("StockPlay") + return fakebot + + +# @pytest.mark.slow +# def test_stockplay(stockbot): +# sp = stockbot.moduleInstances["StockPlay"] +# # import pdb +# # pdb.set_trace() +# # print(sp.cache) +# # print(sp.cache.get_price("AmD", 60)) +# # print(sp.cache.get_price("AmD", 60)) +# # print(sp.cache.get_price("AmD", 60)) +# # print(sp.cache.get_price("nut", 60)) + +# symbols = set() + +# with closing(sp.sql.getCursor()) as c: +# for row in c.execute("SELECT * FROM stockplay_holdings").fetchall(): +# symbols.update([row["symbol"].lower()]) + +# print(symbols) + +# # # symbols = "a bah chk crm cron f fb mdla nio too tsla".split() +# for symbol in symbols: +# p = sp.cache.get_price(symbol, 0) +# if not p: +# print("not supported:", symbol) +# continue +# print(symbol, "age: ", time() - p.time) + +# # print(sp.cache.get_price("gigl", 60))