From 3fc4ea055c9f7b29adcfe8da20575057dec750f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 15:41:39 +0000 Subject: [PATCH 01/31] Threads and waitress instead of gevent (M9) discord.py cannot run under gevent, and gevent's monkey patching is the one thing that kept the bot from plain threads; with the config swapped in one assignment, state edits locked and the scheduler locked (M1, M2, M6), it can go. - bot.py starts the scheduler thread, and serves HTTP with waitress only when some network's webhook is set (bound before Telegram is given the URL); with none set, the main thread only waits for signals. SIGHUP and SIGUSR1 are plain signal handlers, installed last as before, and the log says when they are in ("finbot is running"). - The repeat check gets a lock: waitress answers on several threads, and a resent update could race its first copy. - Charts are drawn one at a time under a re-entrant lock instead of gevent's one-thread pool, so a chart function may call another; render.preload() imports matplotlib at start. - pywsgi's access log (which printed the secret webhook path) goes; with debug on, each request is logged by network and status. - waitress moves to requirements-optional.txt, imported only when HTTP is served; CI installs both files. A test imports every module and starts bot.py with the optional libraries blocked. - Tests wait for each update's worker thread instead of sleeping, and the hub-stall tests go: threads preempt. The fixture quotes share one timestamp, so quotes fetched in parallel threads still tie on their dates. util/record_outputs.py: 180 steps, identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- .github/workflows/ci.yml | 2 +- CLAUDE.md | 12 ++++--- README.md | 4 ++- bot.py | 66 +++++++++++++++++++++++++-------------- lib/jobrun.py | 5 ++- lib/jobs.py | 2 +- lib/render.py | 30 +++++++----------- requirements-optional.txt | 3 ++ requirements.txt | 1 - tests/common.py | 18 ++++++++--- tests/test_charts.py | 54 ++++++++++++-------------------- tests/test_edges.py | 52 +++++++++++++++++++++++++++--- tests/test_inbound.py | 15 +++++---- tests/test_scheduler.py | 38 ++++++++++------------ 14 files changed, 175 insertions(+), 127 deletions(-) create mode 100644 requirements-optional.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 080ef50d..a4412729 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: cache: pip - if: steps.changes.outputs.docs_only != 'true' run: | - pip install -r requirements.txt + pip install -r requirements.txt -r requirements-optional.txt python -m compileall -q *.py lib util python -m unittest discover -s tests -v -b diff --git a/CLAUDE.md b/CLAUDE.md index 597836fa..23e4d77e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm pull requests that change only README.md, CLAUDE.md, `doc/*.md` or `img/*.png` (install, compile and tests are skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). - Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); - `requirements.txt` is for pip/uv users and CI. + `requirements.txt` is for pip/uv users and CI. The libraries in `requirements-optional.txt` are each imported only + inside the code of the mode that needs it, never at module load (a test starts finbot with them all blocked). - A new preference goes in three places: a `Prefs` field with its default (`lib/config.py`; its type says how it is read), `defaults/preferences.conf`, and the README's preferences table. The tests catch a key missing from the template or a default that differs, but not the README (no test may read README.md: CI skips the tests for a @@ -35,8 +36,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm record real requests. ## Code map (open the one file you need) -- `bot.py`: WSGI entry (route, verify, parse, then a thread of `lib/worker.py` per message or button press); SIGHUP - reload and SIGUSR1 cache flush, each in a thread. `lib/inbound.py`: a class per network that reads its updates +- `bot.py`: WSGI entry (route, verify, parse, then a thread of `lib/worker.py` per message or button press), served by + waitress only when some network's webhook is set; plain threads, no gevent; SIGHUP reload and SIGUSR1 cache flush, + each in a thread. `lib/inbound.py`: a class per network that reads its updates (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back), apart from a class per way they arrive (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an update however it arrived. @@ -102,8 +104,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm never print there (a test checks); only the util/ tools, run by hand, do. - A report gives its chart as a `render.Chart` spec, not a drawing: `deliver()` draws it only for a chat that takes charts, through `render.draw()`, which caches the image by the spec's hash (and the drawing code's). -- A chart renderer in `lib/render.py` takes its `Chart` and is listed in `RENDERERS` by kind; it gets `@_drawn` (the - bot draws in one native thread) and must not call another `@_drawn` function, or that thread waits on itself. +- A chart renderer in `lib/render.py` takes its `Chart` and is listed in `RENDERERS` by kind; it gets `@_drawn`, which + draws one chart at a time (matplotlib's caches are shared by every thread). - A new command: a `Command` in `COMMANDS` (its names, and what its arguments may hold), whose usage, section and buttons make its `.help` lines and keyboard buttons, and the README command list. diff --git a/README.md b/README.md index 1a33be11..a02401ca 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,10 @@ compares the two before the steps. * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: ``` -croniter python-dateutil requests gevent pandas matplotlib numpy +croniter python-dateutil requests pandas matplotlib numpy ``` +* For a network in http mode (its webhook URL set), also `waitress` (Debian: `python3-waitress`); the optional modules are + in `requirements-optional.txt`, each imported only by the mode that needs it. ### Supported commands diff --git a/bot.py b/bot.py index e7f4ab2e..c8e59dab 100755 --- a/bot.py +++ b/bot.py @@ -5,12 +5,8 @@ # ignored until startup is done (the reload handler is installed last): its default action kills the bot signal.signal(signal.SIGHUP, signal.SIG_IGN) signal.signal(signal.SIGUSR1, signal.SIG_IGN) # likewise the cache flush -import gevent.monkey -gevent.monkey.patch_all() -import collections, json, os, re, sys, traceback +import collections, json, os, re, sys, threading, traceback from pathlib import Path -from gevent import pywsgi -import threading from lib import config from lib import inbound @@ -26,6 +22,7 @@ AUTHENTICATED = 'finbot.authenticated' # set once it has passed that network's check MAX_BODY = 1 << 20 # bytes in a request body, at most: an update from Telegram or Slack takes a few thousand _updates_seen = collections.OrderedDict() # (service, update id) of the last updates taken, oldest first +_updates_lock = threading.Lock() # updates arrive on several threads: a resent one may race its first copy def seen_before(service, update_id, keep=1000): """Whether this update was taken already: Telegram's update_id, Slack's event_id. Slack resends an event it got no @@ -34,11 +31,12 @@ def seen_before(service, update_id, keep=1000): if update_id is None: return False key = (service, update_id) - if key in _updates_seen: - return True - _updates_seen[key] = None - if len(_updates_seen) > keep: - _updates_seen.popitem(last=False) + with _updates_lock: + if key in _updates_seen: + return True + _updates_seen[key] = None + if len(_updates_seen) > keep: + _updates_seen.popitem(last=False) return False def main(environ, start_response): @@ -68,7 +66,11 @@ def handle(environ, start_response): comes from that network (before its body is read, when the proof is a header), read the update (CONTENT_LENGTH bytes, up to MAX_BODY) and take it (take): the network gets its answer at once.""" def respond(response): - """Answer the request: a status line and a body.""" + """Answer the request: a status line and a body (logged with the debug preference, naming the network, not the + path).""" + if config.prefs.debug: + print(f"[http] request for {environ.get(ROUTED) or 'no network'} answered {response.status}", + file=sys.stderr) start_response(response.status, [('Content-type', 'application/json')]) return [response.body] def print_headers(): @@ -118,7 +120,7 @@ def take(network, update): return inbound.OK def dispatch(event): - """Hand a message or a button press to lib/worker, in a thread of its own.""" + """Hand a message or a button press to lib/worker, in a thread of its own, which it returns.""" if isinstance(event, inbound.Press): print(f"[{event.service.capitalize()} button]:", event.user, "chat", event.chat_id, event.data, file=sys.stderr) target = worker.process_callback @@ -126,7 +128,9 @@ def dispatch(event): if config.prefs.debug: print(f"[{event.service.capitalize()}]:", event.user, event.text, file=sys.stderr) target = worker.process_request - threading.Thread(target=target, args=(event,)).start() + thread = threading.Thread(target=target, args=(event,)) + thread.start() + return thread def git_version(checkout=None): """Short hash of the checked-out commit, read straight from the .git directory: no git binary to run, and no @@ -187,6 +191,14 @@ def flush_cache(*args): print('ERROR', delivery.error_line(e, 'cache flush'), file=sys.stderr) jobs.start_thread(jobs.warm_now, scheduler) +def http_server(): + """The HTTP server for the networks that post to finbot (those whose webhook is set), bound to ip and port, or None + when none does. waitress is imported only then. Raises OSError when the port cannot be had.""" + if not any(hook.path() for hook in inbound.WEBHOOKS): + return None + import waitress + return waitress.create_server(main, host=config.env.ip, port=config.env.port) + if __name__ == '__main__': print(f'Starting finbot {git_version()}', file=sys.stderr) try: @@ -212,10 +224,16 @@ def flush_cache(*args): sys.exit(1) else: print('Scheduled reports are off (scheduler=False)', file=sys.stderr) - render.render_off_hub() # charts drawn in a thread of their own: no chat waits while another's chart is drawn - httpd = pywsgi.WSGIServer((config.env.ip, config.env.port), main) - httpd.secure_repr = False if config.prefs.debug else None - print(f'Opening socket on http://{config.env.ip}:{config.env.port}', file=sys.stderr) + render.preload() + try: + server = http_server() # bound before Telegram is given the URL + except OSError as e: + print(e, file=sys.stderr) + sys.exit(1) + if server: + print(f'Opening socket on http://{config.env.ip}:{server.effective_port}', file=sys.stderr) + else: + print('No webhook is set: finbot takes no HTTP requests', file=sys.stderr) if config.env.telegramBotToken and config.env.telegramOutgoingWebhook: try: inbound.TELEGRAM.bot_name = telegram.register() # here only: the job processes import lib.telegram too @@ -228,10 +246,10 @@ def flush_cache(*args): jobs.current = scheduler # for the .jobs chat command threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs # Startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron - gevent.signal_handler(signal.SIGHUP, in_thread(reload_all)) - gevent.signal_handler(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache - try: - httpd.serve_forever() - except OSError as e: - print(e, file=sys.stderr) - sys.exit(1) + signal.signal(signal.SIGHUP, in_thread(reload_all)) + signal.signal(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache + print('finbot is running: SIGHUP reloads, SIGUSR1 flushes the cache', file=sys.stderr) + if server: + server.run() + while True: # the threads do the work; this one waits for signals + signal.pause() diff --git a/lib/jobrun.py b/lib/jobrun.py index adcea41f..9c07b9ae 100644 --- a/lib/jobrun.py +++ b/lib/jobrun.py @@ -1,8 +1,7 @@ """Runs one scheduled command in its own process: `python -m lib.jobrun lib.reports. ''`. -The bot starts this for every scheduled or manual run, so a job's CPU-heavy work (charts, pandas) happens off the -bot's event loop and cannot hold up incoming webhooks. Failures are reported to the channel and give the exit status -REPORTED.""" +The bot starts this for every scheduled or manual run, so a job's work happens in a process of its own. Failures are +reported to the channel and give the exit status REPORTED.""" import signal if __name__ == '__main__': # a reload is for the bot, but `systemctl kill -s HUP` signals every process of the unit diff --git a/lib/jobs.py b/lib/jobs.py index e6db402d..76b8c567 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -78,7 +78,7 @@ def stop(proc): def start_thread(target, *args): - """Run target(*args) in a daemon thread of its own (a greenlet, under gevent's monkey patching).""" + """Run target(*args) in a daemon thread of its own.""" threading.Thread(target=target, args=args, daemon=True).start() diff --git a/lib/render.py b/lib/render.py index c74e44cc..85b4c2d2 100644 --- a/lib/render.py +++ b/lib/render.py @@ -1,12 +1,12 @@ -"""Charts drawn as PNGs with matplotlib: the bot draws them in one native thread, off the gevent hub (render_off_hub). A -report describes its chart as a Chart, which draw() draws only for a chat that takes charts, and once: the image is -cached by the spec's hash.""" +"""Charts drawn as PNGs with matplotlib, one at a time (_drawn). A report describes its chart as a Chart, which draw() +draws only for a chat that takes charts, and once: the image is cached by the spec's hash.""" import dataclasses import functools import hashlib import importlib.metadata import io import json +import threading from pathlib import Path from lib import config # first: sets MPLCONFIGDIR before matplotlib is imported (lazily, by the chart functions) from lib import cache @@ -57,31 +57,23 @@ def style_date_axis(ax, start, end, ink): UP, DOWN, FLAT = '#34d399', '#fb7185', PALETTE[2] # a rise, a fall, neither: a price line's colour by its change BAR_UP = CATEGORIES[0] # bars rise in blue; they fall in DOWN -_render_pool = None # set by render_off_hub(): the bot draws its charts in one native thread +_drawing = threading.RLock() # one chart at a time: matplotlib's caches are shared by every thread -def render_off_hub(): - """For the bot: draw charts in one native thread from now on, so that the gevent hub keeps serving every other chat - while one is drawn (the shorts pictogram takes over a second; in the hub that froze everything). One thread, because - several at once deadlock matplotlib under gevent's patched locks. matplotlib is imported here, at start, instead of - by the first chart. Job processes and the tests draw in place.""" - global _render_pool +def preload(): + """For the bot, at start: import matplotlib now, instead of when the first chart is asked for.""" import importlib - from gevent.threadpool import ThreadPool for module in ('matplotlib.backends.backend_agg', 'matplotlib.dates', 'matplotlib.figure', 'matplotlib.patches', 'matplotlib.path', 'matplotlib.ticker'): importlib.import_module(module) - _render_pool = ThreadPool(1) def _drawn(draw): - """A chart function that draws in the render pool when there is one. Only for a function that calls none of the - others: the pool's one thread waiting on itself would never finish.""" + """A chart function that draws while no other thread does (it may call another such function).""" @functools.wraps(draw) - def in_pool(*args, **kwargs): - """Draw in the render thread when the bot has one (render_off_hub), else here.""" - if _render_pool is None: + def alone(*args, **kwargs): + """Draw, holding the drawing lock.""" + with _drawing: return draw(*args, **kwargs) - return _render_pool.apply(draw, args, kwargs) - return in_pool + return alone def _figure(facecolor): """A 1280x800 figure (Telegram's photo size, so it is never resampled) and its Agg canvas.""" diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 00000000..6f848c1c --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1,3 @@ +# Each is needed only by one way of receiving updates, and imported only when that way is configured. CI installs them +# all; an install needs only those of the modes it uses (Debian: the python3-* package named). +waitress>=3.0.2 # any network in http mode (python3-waitress) diff --git a/requirements.txt b/requirements.txt index 4fbb9d88..2b6ef139 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ # Targets Python 3.13.5 (see .python-version). Lower bounds are the first releases with 3.13 support. croniter>=1.3.15 -gevent>=24.11.1 matplotlib>=3.9.2,<4 numpy>=2.1.0,<3 pandas>=2.2.3,<3 # 3.x not yet tried diff --git a/tests/common.py b/tests/common.py index d11f9e61..272587b8 100644 --- a/tests/common.py +++ b/tests/common.py @@ -12,7 +12,6 @@ os.environ.update(telegramOutgoingWebhook='https://example.com/telegram', telegramOutgoingToken='tok', telegramChatID='1') -# first, and once: it monkey-patches with gevent, which must happen before requests imports ssl and never mid-test import bot from lib import answer, buttons, cache, charts, config, delivery, dialects, formats, inbound, links, markets, render from lib import sharesight, shortman, state, telegram, transports, watchlist, worker, yahoo @@ -23,6 +22,7 @@ (yahoo, 'price_history'), (yahoo, 'historic_high'), (sharesight, '_portfolios'), (sharesight, '_trades'), (sharesight, '_performance'))} REAL_GET_HOLDINGS = watchlist.with_holdings +REAL_DISPATCH = bot.dispatch # FinbotCase notes the worker thread of each update, so that a test can wait for it def bare_environment(): @@ -64,17 +64,20 @@ def skip_if_old_env(case): STATES = ('REGULAR', 'PRE', 'POST', 'CLOSED') +NOW = time.time() # one moment for every fixture quote: fetched together in threads, they still tie on their dates + + def market_entry(ticker): """A Yahoo quote for a fixture ticker; T0 to T24 vary in market state, move, rating and so on, so that every threshold, sort and branch has rows.""" i = int(ticker[1:]) if ticker[1:].isdigit() else 0 # varied data so threshold/sort/sell branches all have rows - return {'marketState': STATES[i % 4], 'regularMarketTime': time.time(), 'quoteType': 'EQUITY', + return {'marketState': STATES[i % 4], 'regularMarketTime': NOW, 'quoteType': 'EQUITY', 'profile_title': 'Co ' + ticker, 'profile_exchange': 'NasdaqGS', 'percent_change': (i - 12) * 1.5, 'percent_change_premarket': (i - 12) * 1.5, 'percent_change_postmarket': (i - 12) * 1.5, 'market_cap': 1e9, 'price_to_earnings_trailing': 9, 'price_to_earnings_forward': 9, 'beta': 2.0, 'short_percent': 20, 'fiftyTwoWeekHigh': 120.0, 'fiftyTwoWeekLow': 80.0, - 'regularMarketPrice': 100.0, 'earnings_date': time.time() + 86400 * 3, - 'ex_dividend_date': time.time() + 86400 * 3, 'recommend': 'buy' if i % 2 else 'sell', + 'regularMarketPrice': 100.0, 'earnings_date': NOW + 86400 * 3, + 'ex_dividend_date': NOW + 86400 * 3, 'recommend': 'buy' if i % 2 else 'sell', 'recommend_index': 1.2 if i % 2 else 4.2, 'recommend_analysts': 8, 'price_to_earnings_peg': 1.2, 'currency': 'USD'} @@ -88,6 +91,11 @@ def patch(self, target, attr, value): p.start() self.addCleanup(p.stop) + def join_workers(self): + """Wait for the worker thread of every update taken so far (bot.dispatch starts one each).""" + for thread in self.workers: + thread.join(10) + def env(self, **changes): """Startup settings for this test: config.env with these fields changed.""" self.patch(config, 'env', dataclasses.replace(config.env, **changes)) @@ -106,6 +114,8 @@ def setUp(self): self.addCleanup(tmp.cleanup) self.env(data_dir=tmp.name, cache_dir=tmp.name, webhooks={'telegram': 'https://api.telegram.org/botX/'}) self.patch(bot, '_updates_seen', collections.OrderedDict()) # each test starts with no update taken + self.workers = [] + self.patch(bot, 'dispatch', lambda event: self.workers.append(REAL_DISPATCH(event))) self.sent = [] self.markups = [] # reply_markup of each photo sent self.errors = [] diff --git a/tests/test_charts.py b/tests/test_charts.py index 9407a57d..4c300904 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -4,8 +4,7 @@ 3. one golden image per chart type, compared with a tolerance. Refresh after an intended restyle with: UPDATE_GOLDEN=1 python -m unittest tests.test_charts """ -import datetime, os, time, unittest -import gevent +import datetime, os, threading, time, unittest from unittest import mock import numpy as np import pandas as pd @@ -158,41 +157,28 @@ def test_a_long_title_is_shrunk_to_fit(self): self.check_image(render.draw_line(line(price_rows(), title)), 'long line title') -class RenderPool(unittest.TestCase): - """The bot draws charts in one native thread (render.render_off_hub): the hub keeps serving while one is drawn.""" - - def setUp(self): - """Draw in the render thread, as the bot does, for this test only.""" - render.render_off_hub() - self.addCleanup(setattr, render, '_render_pool', None) +class DrawingLock(unittest.TestCase): + """Charts asked for from several threads at once are drawn one at a time (render._drawn).""" def test_charts_asked_for_at_once_all_finish(self): - """Six charts asked for at once all finish: one render thread, as several deadlock matplotlib under gevent.""" + """Six charts asked for from six threads at once all finish, whole.""" pie = lambda title: render.Chart('pie', title, rows=(('a', 1.0), ('b', 2.0))) - jobs = [gevent.spawn(render.draw_pie, pie(f't{i}')) for i in range(4)] - jobs += [gevent.spawn(bars, MIXED)] - jobs += [gevent.spawn(uncached, render.Chart('bars', 'T', rows=tuple(MIXED)))] - gevent.joinall(jobs, timeout=120) - self.assertTrue(all(job.successful() for job in jobs), - 'a chart did not finish (several native threads deadlock matplotlib)') - self.assertTrue(all(decode(job.value).size == (1280, 800) for job in jobs)) - - def test_the_hub_keeps_serving_while_a_chart_is_drawn(self): - """While the shorts pictogram draws (about a second), other greenlets run on: no gap over half a second.""" - gaps, done = [], [] - def ticker(): - """Note the time between wake-ups until the chart is done.""" - last = time.monotonic() - while not done: - gevent.sleep(0.01) - now = time.monotonic() - gaps.append(now - last) - last = now - tick = gevent.spawn(ticker) - shorts([(f'T{i}', 60.0 - 2 * i) for i in range(24)]) # about a second of drawing - done.append(True) - tick.join() - self.assertLess(max(gaps), 0.5) # drawn in the hub, the ticker stopped for the whole chart + draws = [lambda i=i: render.draw_pie(pie(f't{i}')) for i in range(4)] + draws += [lambda: bars(MIXED), lambda: uncached(render.Chart('bars', 'T', rows=tuple(MIXED)))] + images = {} + threads = [threading.Thread(target=lambda n=n, draw=draw: images.__setitem__(n, draw())) + for n, draw in enumerate(draws)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(120) + self.assertEqual(sorted(images), list(range(6)), 'a chart did not finish') + self.assertTrue(all(decode(image).size == (1280, 800) for image in images.values())) + + def test_one_chart_function_may_call_another(self): + """The lock is re-entrant: a drawn function that calls another drawn one finishes.""" + inner = render._drawn(lambda: 'drawn') + self.assertEqual(render._drawn(inner)(), 'drawn') class Contracts(unittest.TestCase): diff --git a/tests/test_edges.py b/tests/test_edges.py index b25ec1e1..a0cf7f05 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -360,6 +360,52 @@ def test_a_flush_fills_the_cache_again_at_once(self): self.assertEqual(started, [(bot.jobs.warm_now, ('the scheduler',))]) +OPTIONAL = ('waitress', 'websocket', 'discord', 'nacl') # each imported only by the mode that needs it + + +class OptionalLibraries(unittest.TestCase): + """An install without the optional libraries (requirements-optional.txt) runs every mode that does not need them.""" + + def blocked(self, data): + """The environment of a process for which no optional library can be imported, and that has no webhook set.""" + stubs = Path(data, 'stubs') + stubs.mkdir() + for name in OPTIONAL: + (stubs / f'{name}.py').write_text('raise ImportError("blocked by the test")\n') + env = bare_environment() + env.update(DATA_DIR=data, port='0', scheduler='False', PYTHONPATH=str(stubs)) + return env + + def test_every_module_imports_without_them(self): + """bot and every module of lib and lib.reports import with the optional libraries blocked, and none of them is + imported.""" + code = ('import importlib, pkgutil, sys, bot, lib, lib.reports\n' + 'for package in (lib, lib.reports):\n' + ' for module in pkgutil.iter_modules(package.__path__, package.__name__ + "."):\n' + ' importlib.import_module(module.name)\n' + f'print(sorted(name for name in {OPTIONAL!r} if sys.modules.get(name)))') + with tempfile.TemporaryDirectory() as data: + out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=self.blocked(data), capture_output=True, + text=True, timeout=60) + self.assertEqual(out.stdout.strip().splitlines()[-1:], ['[]'], out.stderr) + + def test_finbot_starts_without_them(self): + """With no webhook set, bot.py starts with the optional libraries blocked: no HTTP server, so no waitress.""" + skip_if_old_env(self) + with tempfile.TemporaryDirectory() as data: + proc = subprocess.Popen([sys.executable, '-u', 'bot.py'], cwd=ROOT, env=self.blocked(data), + stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True) + self.addCleanup(proc.wait) + self.addCleanup(proc.kill) + lines = [] + for line in proc.stderr: + lines.append(line) + if line.startswith('finbot is running'): + break + self.assertIsNone(proc.poll(), ''.join(lines)) + self.assertIn('No webhook is set: finbot takes no HTTP requests', ''.join(lines)) + + class StartupSignals(unittest.TestCase): """A reload (SIGHUP) that arrives while the bot is still starting must not kill it.""" @@ -382,10 +428,9 @@ def test_sighup_during_startup_is_ignored_and_works_afterwards(self): lines = [] for line in proc.stderr: lines.append(line) - if line.startswith('Opening socket'): + if line.startswith('finbot is running'): # the handlers are in break self.assertIsNone(proc.poll(), ''.join(lines)) - time.sleep(1) # the handler goes in just before serving proc.send_signal(signal.SIGHUP) for line in proc.stderr: # a reload now runs (there is no preferences.conf here, so it says so) lines.append(line) @@ -420,11 +465,10 @@ def test_usr1_flushes_the_cache_and_keeps_the_state(self): lines = [] for line in proc.stderr: lines.append(line) - if line.startswith('Opening socket'): + if line.startswith('finbot is running'): # the handlers are in break self.assertIsNone(proc.poll(), ''.join(lines)) self.assertEqual(len(os.listdir(cache)), 4) # nothing flushed yet - time.sleep(1) # the handler goes in just before serving proc.send_signal(signal.SIGUSR1) for line in proc.stderr: lines.append(line) diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 71641921..546a23a5 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -2,7 +2,6 @@ and user text escaped where it arrives.""" import contextlib, hashlib, hmac, io, json, time, unittest from urllib.parse import urlencode -import gevent from tests.common import * @@ -134,7 +133,7 @@ def test_a_telegram_update_sent_again_runs_once(self): 'message': {'message_id': 1, 'chat': {'id': 5}, 'from': {'first_name': 'A'}, 'text': '.price'}} for _ in range(2): # Telegram sends an update again when it got no answer self.assertEqual(self.request(body=body, headers=self.SECRET)[1], '200 OK') - gevent.sleep(0.05) + self.join_workers() self.assertEqual(len(calls), 1) def test_an_update_is_taken_however_it_arrived(self): @@ -149,7 +148,7 @@ def test_an_update_is_taken_however_it_arrived(self): for _ in range(2): self.assertEqual(bot.take(inbound.TELEGRAM, {'update_id': 70, 'message': message}), inbound.OK) self.assertEqual(bot.take(inbound.SLACK, event), inbound.OK) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() self.assertEqual([(m.service, m.chat_id, m.text) for m in got], [('telegram', '5', '.price'), ('slack', 'C1', '.trades')]) @@ -188,7 +187,7 @@ def event(self, event_id=None, **event): status = [] with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): bot.main(wsgi_environ('/slack', body), lambda s, *a, **k: status.append(s)) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() return status[-1] def press(self, payload): @@ -197,7 +196,7 @@ def press(self, payload): with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): body = urlencode({'payload': json.dumps(payload)}).encode() bot.main(wsgi_environ('/slack', body), lambda s, *a, **k: status.append(s)) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() return status[-1] def test_a_button_press_reaches_the_worker(self): @@ -284,7 +283,7 @@ def post(self, secret='sss', age=0, token='stok', headers=None, tamper=False): status = [] with contextlib.redirect_stderr(io.StringIO()): bot.main(env, lambda s, *a, **k: status.append(s)) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() return status[-1] def test_a_button_press_is_signed_over_its_form(self): @@ -299,7 +298,7 @@ def test_a_button_press_is_signed_over_its_form(self): answers = [] with contextlib.redirect_stderr(io.StringIO()): bot.main(wsgi_environ('/slack', sent, **headers), lambda s, *a, **k: answers.append(s)) - gevent.sleep(0.05) + self.join_workers() self.assertEqual(answers, [status]) self.assertEqual(presses, [inbound.Press('slack', 't1', 'C1', '1.5', 'c|h|w|T1', '@rob')]) @@ -343,7 +342,7 @@ def test_a_telegram_name_is_escaped_where_it_arrives(self): env = wsgi_environ('/telegram', json.dumps(body).encode(), HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN='tok') with contextlib.redirect_stderr(io.StringIO()): bot.main(env, lambda *a, **k: None) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() self.assertEqual((requests[0].user, requests[0].user_name), ('@A<b>&',) * 2) # the handle and the name diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9668777b..ac460781 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -4,7 +4,6 @@ import zoneinfo from pathlib import Path from unittest import mock -import gevent from tests.common import * from lib import jobs from lib.reports import cleanup, podcasts, price, sharesight_warm, yahoo_warm @@ -754,23 +753,6 @@ def test_a_report_script_loads_neither_pandas_nor_the_scheduler(self): env={**os.environ, 'DATA_DIR': data}) self.assertEqual(out.stdout.strip().splitlines()[-1:], ['[]'], out.stderr) - def test_the_event_loop_keeps_running_while_a_child_works(self): - """Waiting for a child does not stop the event loop.""" - self.sleeper(1) - ticks = [] - def ticker(): - """Tick five times, 0.1 seconds apart.""" - for _ in range(5): - gevent.sleep(0.1) - ticks.append(time.monotonic()) - greenlet = gevent.spawn(ticker) - started = time.monotonic() - self.assertEqual(jobs.run_child('cleanup.py', cleanup, {}), 0) - greenlet.join() - self.assertGreater(time.monotonic() - started, 0.9) - self.assertEqual(len(ticks), 5) # the loop was not blocked by the child - self.assertLess(ticks[-1] - started, 0.8) - def test_a_child_still_running_at_its_deadline_is_stopped(self): """A child still running after JOB_TIMEOUT is stopped, rather than left running, and TimeoutError raised.""" self.patch(jobs, 'JOB_TIMEOUT', 0.3) @@ -782,11 +764,23 @@ def test_a_child_still_running_at_its_deadline_is_stopped(self): self.assertIsNotNone(self.children[0].poll()) # the child was stopped, not left running def test_another_interruption_stops_the_child_too(self): - """An interruption while a child runs (a greenlet killed, say) stops the child as well.""" + """An interruption while a child runs (a KeyboardInterrupt, say) stops the child as well.""" self.sleeper(30) - with self.assertRaises(gevent.Timeout): - with gevent.Timeout(0.3): - jobs.run_child('cleanup.py', cleanup, {}) + start = jobs.subprocess.Popen # the sleeper's + def interrupted(args, **kw): + """The sleeper, whose wait for the job is interrupted.""" + child = start(args, **kw) + wait = child.wait + def wait_once(timeout=None): + """Interrupted while waiting for the job; stop()'s own waits go through.""" + if timeout == jobs.JOB_TIMEOUT: + raise KeyboardInterrupt + return wait(timeout) + child.wait = wait_once + return child + self.patch(jobs.subprocess, 'Popen', interrupted) + with self.assertRaises(KeyboardInterrupt): + jobs.run_child('cleanup.py', cleanup, {}) self.assertIsNotNone(self.children[0].poll()) def test_each_command_of_a_line_has_a_deadline_of_its_own(self): From 4b8729fe8eae1dd80b9189d35e5c454c004f67d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 15:50:05 +0000 Subject: [PATCH 02/31] A button press is acknowledged where it arrives, not by the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each pull mode to come acknowledges a press its own way as it arrives (Slack's socket ack, Discord's interaction callback), so the rule moves out of the worker into one place, bot.take, the same for every network and every way an update arrives. - inbound.Network.acknowledge(press), by default nothing: taking the update is the acknowledgement (Slack's 200). bot.take calls it for a press after the repeat check and before the worker has it, so a resent press is not acknowledged twice. - Telegram's answers the callback query with a toast made from the button's data alone (buttons.toast): "Loading 1Y…" for a period button, "Running…" for a .jobs button. - Transport.acknowledge and the worker's three calls go, and with them the stale .jobs button's "Buttons updated, try again": the list redrawn in place already says it. - util/record_outputs.py acknowledges its presses as bot.take does, and presses a .jobs button too. util/record_outputs.py: 182 steps, 1 differs (that stale .jobs button's toast). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 11 ++++++----- bot.py | 5 ++++- lib/buttons.py | 14 +++++++++++--- lib/inbound.py | 13 ++++++++++++- lib/transports.py | 7 ------- lib/worker.py | 14 +++++--------- tests/common.py | 7 +++++++ tests/test_charts.py | 2 +- tests/test_edges.py | 1 - tests/test_inbound.py | 18 ++++++++++++++++++ tests/test_reports.py | 2 +- tests/test_scheduler.py | 19 ++++++++++--------- tests/test_util.py | 11 +++++++++++ util/record_outputs.py | 18 ++++++++++++++++-- 14 files changed, 102 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 23e4d77e..70a7ae0c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,9 +39,10 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - `bot.py`: WSGI entry (route, verify, parse, then a thread of `lib/worker.py` per message or button press), served by waitress only when some network's webhook is set; plain threads, no gevent; SIGHUP reload and SIGUSR1 cache flush, each in a thread. `lib/inbound.py`: a class per network that reads its updates - (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back), apart from a class per way they - arrive (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an - update however it arrived. + (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back) and acknowledges a press where + the network waits for that (`acknowledge`: Telegram's toast), apart from a class per way they arrive + (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an update + however it arrived, and is the one place a press is acknowledged (the worker never does). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a @@ -58,8 +59,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm `SlackBot`, `SlackWebhook`, `DiscordWebhook`): splitting, captions, charts, buttons, edits, and the typing notice (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop before posting. `lib/telegram.py`: Bot API calls (`call`, `try_call`). -- `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), and what a - pressed button's data asks for (`parse`); each transport draws them. `lib/charts.py`: a chart with period buttons, +- `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), what a + pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. `lib/charts.py`: a chart with period buttons, built by its kind's builder (`build`) for the command's first view (`worker.chart_reply`) and for each press. - `lib/dialects.py`: each network's markup and message sizes. `lib/markets.py`: one table by market code (flag, currency, Yahoo suffix, Google code; Sharesight's codes, so TSE is Toronto), and `code(name)` for any name an exchange diff --git a/bot.py b/bot.py index c8e59dab..ba6fa2cb 100755 --- a/bot.py +++ b/bot.py @@ -109,13 +109,16 @@ def print_headers(): def take(network, update): """An update from a network (an inbound.Network reads it), however it arrived: what it says goes to lib/worker in a - thread of its own (dispatch), unless it is a repeat. Returns the inbound.Response for the network.""" + thread of its own (dispatch), unless it is a repeat; a button press is acknowledged first, the one place that does. + Returns the inbound.Response for the network.""" event = network.parse(update) if isinstance(event, inbound.Response): return event if seen_before(network.service, network.update_id(update)): print(f"[{network.service}]: update {network.update_id(update)} seen already, not run again", file=sys.stderr) return inbound.OK + if isinstance(event, inbound.Press): + network.acknowledge(event) # Telegram's toast ("Loading 1Y…") shows while the worker builds the answer dispatch(event) return inbound.OK diff --git a/lib/buttons.py b/lib/buttons.py index cf8b0982..c93b53ff 100644 --- a/lib/buttons.py +++ b/lib/buttons.py @@ -1,6 +1,6 @@ -"""Buttons under a message, and what their data says when one is pressed (parse): a chart's period buttons (1D | 7D | -1M | 1Y | Max), whose data carries what its builder needs (charts.BUILDERS), and a .jobs command's button. Each -transport draws Button rows its own way.""" +"""Buttons under a message, what their data says when one is pressed (parse), and what a press is told at once +(toast): a chart's period buttons (1D | 7D | 1M | 1Y | Max), whose data carries what its builder needs +(charts.BUILDERS), and a .jobs command's button. Each transport draws Button rows its own way.""" import hashlib import typing @@ -87,3 +87,11 @@ def parse(data): if kind == 'c' and len(parts) == 3 and parts[0] in BUTTONS and parts[1] in BUTTONS[parts[0]]: return ChartPress(*parts) return None + +def toast(data): + """What a press is told at once, from the button's data alone (the network acknowledges a press before finbot + knows more): 'Loading 1Y…' for a period button, 'Running…' for a .jobs command's, None for data of neither.""" + pressed = parse(data) + if isinstance(pressed, JobPress): + return 'Running…' + return f"Loading {PERIODS[pressed.period][0]}…" if pressed else None diff --git a/lib/inbound.py b/lib/inbound.py index 65f7adb5..15fd8e98 100644 --- a/lib/inbound.py +++ b/lib/inbound.py @@ -2,7 +2,7 @@ (Telegram, Slack), and a class per way of arriving takes them in (a webhook: the path the network posts to, how it proves itself, the body), so that another way (a pull mode) can hand its updates to the same reader. An update becomes a Message to answer, a Press of a button, or a Response the network gets straight back (a challenge, or nothing to -do).""" +do). The reader also acknowledges a press where the network waits for that (Telegram), however it arrived.""" import dataclasses import hashlib import hmac @@ -13,7 +13,9 @@ import time from urllib.parse import parse_qs, urlparse +from lib import buttons from lib import config +from lib import telegram @dataclasses.dataclass @@ -76,6 +78,10 @@ def parse(self, update): """What the update (a dict) says: a Message, a Press, or a Response to give straight back.""" raise NotImplementedError + def acknowledge(self, press): + """Tell the network a press was taken, where it waits for a call to say so. Here nothing: taking the update is + the acknowledgement (Slack's is the 200, or the socket's ack).""" + class Telegram(Network): """Telegram's updates: messages and button presses.""" @@ -122,6 +128,11 @@ def parse(self, update): return UNHANDLED return Message(self.service, chat_id, user, text, self.bot_name, user_name, str(message.get('message_id', ''))) + def acknowledge(self, press): + """answerCallbackQuery: the client's spinner stops, and a toast made from the button's data says what is coming + (buttons.toast: 'Loading 1Y…'). A failure is only logged.""" + telegram.answerCallbackQuery(press.press_id, buttons.toast(press.data)) + class Slack(Network): """Slack's Events API (messages) and its interactivity (a button pressed).""" diff --git a/lib/transports.py b/lib/transports.py index a4e05cde..e3f3d0b7 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -117,9 +117,6 @@ def typing(self): def typing_done(self): """Take the typing notice off, where it lasts until taken off (Slack's ⏳).""" - def acknowledge(self, press, text=None): - """Answer a button press, where the network waits for that (Telegram, with `text` as a toast).""" - def reshow(self, message, buttons): """Show a chart message's period (the button marked) again as it was posted lately, where the network keeps that: True when it did. Here never: each period is drawn and sent anew.""" @@ -225,10 +222,6 @@ def typing(self): """sendChatAction: "typing…" for a few seconds.""" telegram.try_call('sendChatAction', data={'chat_id': self.where, 'action': 'typing'}) - def acknowledge(self, press, text=None): - """answerCallbackQuery: the client's spinner stops, and `text` shows as a toast.""" - telegram.answerCallbackQuery(press, text) - def pin(self, answers): """Pin the messages that these answers are (Telegram's), last first, so that the first ends up on top, without the notification.""" diff --git a/lib/worker.py b/lib/worker.py index db14f579..3c252e17 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -43,7 +43,7 @@ def reply(self, lines, chart=None, buttons=None): def process_job_callback(press, pressed): """A command button of .jobs (an inbound.Press; pressed, what its data says): run that command now and answer in - the chat.""" + the chat. A button from an older list runs nothing: the list is redrawn in place as it is now.""" service = dialects.of(press.service) chat = delivery.reply_to(service, press.chat_id) try: @@ -51,11 +51,9 @@ def process_job_callback(press, pressed): raise jobs.ScheduleError('the scheduler is off') command = jobs.current.pick(pressed.number, pressed.checksum) # only a command .jobs offers; anything else fails except jobs.ScheduleError: - chat.acknowledge(press.press_id, 'Buttons updated, try again') payload, markup = about.prepare_jobs_payload(service, '', chat.takes_buttons) # the list now, for the stale one chat.edit_text(press.message_id, '\n'.join(payload), markup or []) return - chat.acknowledge(press.press_id, f'Running {jobs.display(command[0])}…'[:100]) try: line = about.job_line(jobs.current, *jobs.current.run_command(command)) delivery.deliver(chat, delivery.Report([line])) @@ -63,17 +61,15 @@ def process_job_callback(press, pressed): delivery.report_error(e, chat, context='jobs') def process_callback(press): - """A button pressed (an inbound.Press): run a scheduled command from .jobs, or rebuild a chart message for the - chosen period and edit it in place.""" + """A button pressed (an inbound.Press, which bot.take acknowledged as it arrived): run a scheduled command from + .jobs, or rebuild a chart message for the chosen period and edit it in place. Data of neither is ignored.""" pressed = buttons.parse(press.data) if isinstance(pressed, buttons.JobPress): return process_job_callback(press, pressed) - service = dialects.of(press.service) - chat = delivery.reply_to(service, press.chat_id) - # answering now shows Telegram's toast bubble ("Loading 1Y…") while the new chart is built - chat.acknowledge(press.press_id, f"Loading {buttons.PERIODS[pressed.period][0]}…" if pressed else None) if not pressed: return + service = dialects.of(press.service) + chat = delivery.reply_to(service, press.chat_id) try: args = buttons.resolve(pressed.ref) markup = buttons.keyboard(pressed.kind, args, pressed.period) diff --git a/tests/common.py b/tests/common.py index 272587b8..8a524533 100644 --- a/tests/common.py +++ b/tests/common.py @@ -96,6 +96,13 @@ def join_workers(self): for thread in self.workers: thread.join(10) + def press_button(self, data, chat_id=55, message_id=9): + """Press a button under one of the bot's Telegram messages, as the update arrives: bot.take acknowledges it + (answerCallbackQuery) and hands it to the worker, whose thread this waits for.""" + bot.take(inbound.TELEGRAM, {'callback_query': {'id': 'cb', 'from': {'username': 'u'}, 'data': data, + 'message': {'message_id': message_id, 'chat': {'id': chat_id}}}}) + self.join_workers() + def env(self, **changes): """Startup settings for this test: config.env with these fields changed.""" self.patch(config, 'env', dataclasses.replace(config.env, **changes)) diff --git a/tests/test_charts.py b/tests/test_charts.py index 4c300904..7f6c2599 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -4,7 +4,7 @@ 3. one golden image per chart type, compared with a tolerance. Refresh after an intended restyle with: UPDATE_GOLDEN=1 python -m unittest tests.test_charts """ -import datetime, os, threading, time, unittest +import datetime, os, threading, unittest from unittest import mock import numpy as np import pandas as pd diff --git a/tests/test_edges.py b/tests/test_edges.py index a0cf7f05..7db1ea6a 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -146,7 +146,6 @@ def no_history(t, *a, **k): self.assertEqual(self.command('.compare T1 T2'), [('text', ['no price history for T1'])]) self.assertEqual(self.command('.history T1'), [('text', ['no price history for T1'])]) self.sent.clear() - self.patch(telegram, 'answerCallbackQuery', lambda *a, **k: None) worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', 'c|h|m|T1')) self.assertEqual((self.sent, self.errors), ([('text', ['no price history for T1'])], [])) diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 546a23a5..350e7e85 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -166,6 +166,24 @@ def test_button_presses_are_logged(self): bot.main(wsgi_environ('/telegram', body, **secret), lambda *a, **k: None) self.assertIn(f'[Telegram button]: {who} chat 55 j|7|abcd', err.getvalue()) + def test_a_press_is_acknowledged_once_before_the_worker_gets_it(self): + """bot.take acknowledges a press, the one place that does: Telegram's with the toast its data makes, before the + worker has it; a resent copy is neither acknowledged nor run again. Slack's needs no call: the answer to the + update is the acknowledgement.""" + order = [] + self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: order.append(('acknowledged', cid, text))) + self.patch(worker, 'process_callback', lambda press: order.append(('worker', press.service, press.data))) + update = {'update_id': 71, 'callback_query': {'id': 'q1', 'from': {'username': 'u'}, 'data': 'c|h|y|T1', + 'message': {'message_id': 9, 'chat': {'id': 55}}}} + with contextlib.redirect_stderr(io.StringIO()): + for _ in range(2): + self.assertEqual(bot.take(inbound.TELEGRAM, update), inbound.OK) + self.join_workers() + self.assertEqual(bot.take(inbound.SLACK, PRESS), inbound.OK) + self.join_workers() + self.assertEqual(order, [('acknowledged', 'q1', 'Loading 1Y…'), ('worker', 'telegram', 'c|h|y|T1'), + ('worker', 'slack', 'c|h|w|T1')]) + class SlackEvents(FinbotCase): """Slack sends every message in a subscribed channel, the bot's own posts included.""" diff --git a/tests/test_reports.py b/tests/test_reports.py index d6ca0cd8..1bf1b6bf 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -551,7 +551,7 @@ def press(self, data): self.patch(transports.TelegramBot, 'edit_chart', lambda dest, msg, img, caption, buttons=None: edits.append((caption.split('\n')[0], transports.TelegramBot.markup(buttons)))) self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) - worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', data)) + self.press_button(data) return edits, toasts def test_pressing_1d_rebuilds_todays_list_and_moves_the_marker(self): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ac460781..ba3122e6 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -591,13 +591,13 @@ def test_button_press_runs_that_command(self): markup = about.prepare_jobs_payload(TELEGRAM, '', tappable=True)[1] data = next(b.data for row in markup for b in row if b.label.startswith('7.')) self.sent.clear() - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', data)) + self.press_button(data) self.assertEqual(self.ran, [('price', {'mode': 'close'})]) self.assertEqual(self.sent, [('text', ['7. price close: done'])]) - self.assertEqual(sent, ['Running price close…']) + self.assertEqual(sent, ['Running…']) # one acknowledgement, from the data alone: the worker sends none def stale_buttons(self): - """Capture the toast and the in-place edit that a refused button press produces.""" + """Capture the toasts and the in-place edits that refused button presses produce.""" self.toasts, self.edits = [], [] self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: self.toasts.append(text)) self.patch(transports.TelegramBot, 'edit_text', lambda dest, mid, text, buttons=None: @@ -614,9 +614,9 @@ def test_forged_button_data_never_runs_anything_else(self): 'j|8|' + good.upper() + 'x', 'j|99999999999999999999|' + good): self.sent.clear() self.toasts.clear(); self.edits.clear() - worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', data)) + self.press_button(data) self.assertEqual(self.ran, [], data) - self.assertEqual(self.toasts, ['Buttons updated, try again'], data) + self.assertEqual(self.toasts, ['Running…'], data) # from the data alone, as the press arrived self.assertEqual(len(self.edits), 1, data) self.assertEqual(self.sent, [], data) # no new message, the old one is edited # cleanup and podcasts are scheduled but cannot be picked, by number or by their checksum @@ -628,13 +628,14 @@ def test_forged_button_data_never_runs_anything_else(self): self.assertEqual(self.ran, [('trades', {})]) def test_button_from_an_old_schedule_updates_the_message(self): - """A stale button runs nothing; the message gets the current list and buttons (scheduler off: a notice).""" + """A stale button runs nothing; the message gets the current list and buttons (scheduler off: a notice), which + says it all: the toast is the one any job button gets as it arrives, with no second one from the worker.""" self.stale_buttons() - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|7|0000')) - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|9|0000')) + self.press_button('j|7|0000') + self.press_button('j|9|0000') self.assertEqual(self.ran, []) self.assertEqual(self.sent, []) - self.assertEqual(self.toasts, ['Buttons updated, try again'] * 2) + self.assertEqual(self.toasts, ['Running…'] * 2) chat, mid, text, markup = self.edits[0] self.assertEqual((chat, mid), ('55', '9')) # the message with the old buttons is edited in place self.assertIn('7. price close', text) diff --git a/tests/test_util.py b/tests/test_util.py index 5965c47f..00459347 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -120,6 +120,17 @@ def test_button_data_reads_back(self): with self.subTest(data=data): self.assertIsNone(buttons.parse(data)) + def test_a_press_is_told_what_is_coming_from_its_data_alone(self): + """The toast a press gets as it arrives: the period a period button loads, 'Running…' for any .jobs button + (the worker checks it later), and none for data of neither.""" + self.assertEqual([buttons.toast(button.data) for button in buttons.keyboard('l', ['5', '0'])[0]], + ['Loading 1D…', 'Loading 7D…', 'Loading 1M…', 'Loading 1Y…', 'Loading Max…']) + self.assertEqual(buttons.toast(buttons.job_button(8, 'trades', 'abcd').data), 'Running…') + self.assertEqual(buttons.toast('j|99|0000'), 'Running…') + for data in ('c|f|d|', 'c|zz|m|T1', '', 'x'): + with self.subTest(data=data): + self.assertIsNone(buttons.toast(data)) + def test_strip_url(self): """A link's text is its address without the scheme or www.""" for url, text in (('https://www.apple.com', 'apple.com'), ('http://example.org/a?b', 'example.org/a?b'), diff --git a/util/record_outputs.py b/util/record_outputs.py index 2fc1a2a5..d14dce04 100755 --- a/util/record_outputs.py +++ b/util/record_outputs.py @@ -23,7 +23,8 @@ '.performance 7d', '.performance 1y', '.performance Nope', '.pie Rob', '.pie Rob sector', '.pie', '.compare T1 T2', '.compare T1 T2 3m', '.compare T1', '.history T1', '.history', '.watchlist', '.watchlist add MSFT', '.watchlist add MSFT', '.watchlist del MSFT', '.watchlist frob X', '.T1', '.jobs', 'hi @bot', 'noise'] -BUTTONS = ['c|h|m|T1', 'c|h|x|T1', 'c|p|w|T1', 'c|c|y|T1,T2', 'c|f|y|', 'c|l|d|3,0', 'c|l|m|0,10', 'c|zz|m|T1'] +BUTTONS = ['c|h|m|T1', 'c|h|x|T1', 'c|p|w|T1', 'c|c|y|T1,T2', 'c|f|y|', 'c|l|d|3,0', 'c|l|m|0,10', 'c|zz|m|T1', + 'j|1|0000'] SCHEDULED = ['trades.py', 'price.py midsession', 'price.py interday', 'price.py premarket', 'price.py close', 'price.py 7', 'cal.py earnings', 'cal.py ex-dividend', 'performance.py 7', 'rating.py', 'shorts.py', 'milestone.py', 'reminder.py'] @@ -62,6 +63,19 @@ def handing(func, kind, *fields): return lambda: func(*fields) return lambda: func(getattr(common.inbound, kind)(*fields)) + def pressing(service, *fields): + """A step that presses a button: acknowledged first where the tree's bot.take does that (Network.acknowledge), + then handed to the worker as handing() does.""" + run = handing(worker.process_callback, 'Press', service, *fields) + network = getattr(getattr(common, 'inbound', None), service.upper(), None) + if not hasattr(network, 'acknowledge'): + return run + def press(): + """Acknowledge the press, then hand it to the worker.""" + network.acknowledge(common.inbound.Press(service, *fields)) + run() + return press + class Record(common.FinbotCase): """The steps, as one test, so that FinbotCase's fixtures (and their cleanup) apply.""" def runTest(self): @@ -107,7 +121,7 @@ def step(label, run): for service, chat, message in (('telegram', '55', '9'), ('slack', 'C1', '1.5')): for data in BUTTONS: label = f'button {data}' if service == 'telegram' else f'{service} button {data}' - step(label, handing(worker.process_callback, 'Press', service, 'cb', chat, message, data)) + step(label, pressing(service, 'cb', chat, message, data)) self.env(webhooks=WEBHOOKS, telegramChatID='1') for where in ('', ' (slackChannelID)'): # Slack's incoming webhook, then the channel as the bot if where: From bb303d3ff4a24cd95ed0fbb2864bb6602d503f09 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 16:03:41 +0000 Subject: [PATCH 03/31] Telegram long polling: commands with no server (pull mode) Telegram's mode follows its keys: with telegramOutgoingWebhook set it posts its updates to finbot as before (http mode); without it, finbot fetches them (pull mode), so a bot token and a chat id are all a Telegram bot needs for commands and buttons. - lib/pull.py, the pull modes: a listener per network without a webhook, each in a thread of its own handing its updates to bot.take, so reading them, the repeat check and acknowledging a press are the webhook's. TelegramPoll asks getUpdates for the updates after the last one taken, held open up to 50 s (the request waits 65), and waits out failures (a 409 when a webhook is set or another finbot polls, the network down) from 5 s, doubling up to a minute. An update whose handling fails is reported to the default chats, as a failed webhook request is, and not fetched again. - telegram.register() runs as finbot starts in both modes: it deletes the webhook (keeping the updates not delivered yet), sets it again in http mode, and names the bot. Its log no longer shows the webhook URL, whose path is a secret. - Telegram charts and .jobs lists carry their buttons in both modes. - A start-up warning (and check_config, which now shows the warnings of the file it checks) for telegramOutgoingToken without the webhook. - doc/telegram.md: pull mode and HTTP mode in place of reports only and the interactive bot; the README, the finbot.env template and CLAUDE.md. util/record_outputs.py: 182 steps, identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 21 +++--- README.md | 10 +-- bot.py | 8 ++- defaults/finbot.env | 7 +- doc/telegram.md | 65 +++++++++-------- lib/config.py | 17 +++-- lib/pull.py | 95 +++++++++++++++++++++++++ lib/telegram.py | 25 ++++--- lib/transports.py | 6 +- tests/test_config.py | 16 ++++- tests/test_pull.py | 138 +++++++++++++++++++++++++++++++++++++ tests/test_reports.py | 10 +-- tests/test_telegram_api.py | 28 +++++--- util/check_config.py | 12 ++++ 14 files changed, 372 insertions(+), 86 deletions(-) create mode 100644 lib/pull.py create mode 100644 tests/test_pull.py diff --git a/CLAUDE.md b/CLAUDE.md index 70a7ae0c..07f7fd87 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back) and acknowledges a press where the network waits for that (`acknowledge`: Telegram's toast), apart from a class per way they arrive (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an update - however it arrived, and is the one place a press is acknowledged (the worker never does). + however it arrived, and is the one place a press is acknowledged (the worker never does). `lib/pull.py`: the pull + modes, a listener per network whose webhook URL is not set (`TelegramPoll`: getUpdates), each in a thread of its own + handing its updates to `bot.take`, waiting longer after each failure in a row (`backoff`). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a @@ -60,8 +62,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop before posting. `lib/telegram.py`: Bot API calls (`call`, `try_call`). - `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), what a - pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. `lib/charts.py`: a chart with period buttons, - built by its kind's builder (`build`) for the command's first view (`worker.chart_reply`) and for each press. + pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. + `lib/charts.py`: a chart with period buttons, built by its kind's builder (`build`) for the command's first view + (`worker.chart_reply`) and for each press. - `lib/dialects.py`: each network's markup and message sizes. `lib/markets.py`: one table by market code (flag, currency, Yahoo suffix, Google code; Sharesight's codes, so TSE is Toronto), and `code(name)` for any name an exchange goes by. `lib/formats.py`: names, units, dates, percents in words. `lib/links.py`: stock links. `lib/render.py`: @@ -82,12 +85,12 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm them; `temp_dir(self)` is a directory removed after the test, `bare_environment()` the environment without finbot's settings. - Which test file: commands and scheduled reports `test_reports`; failures, bad input, secrets in logs, signals - `test_edges`; webhook requests, Slack events and signatures `test_inbound`; Yahoo `test_yahoo`; Sharesight - `test_sharesight`; posting to each network `test_sending`; registering Telegram's webhook `test_telegram_api`; the - schedule file and job arguments `test_schedule`; the scheduler and manual runs `test_scheduler`; config - `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, - cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, and the docstring, stdout and - line-width checks `test_finbot`. + `test_edges`; webhook requests, Slack events and signatures `test_inbound`; the pull modes `test_pull`; Yahoo + `test_yahoo`; Sharesight `test_sharesight`; posting to each network `test_sending`; registering Telegram's webhook + `test_telegram_api`; the schedule file and job arguments `test_schedule`; the scheduler and manual runs + `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, + formats, links, buttons, cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, and the + docstring, stdout and line-width checks `test_finbot`. ## Conventions - Read settings as `config.env.` / `config.prefs.` where they are used, never into a default or module diff --git a/README.md b/README.md index a02401ca..cb098639 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ compares the two before the steps. ### Requirements * Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key * A Telegram bot, a Slack app or a Discord webhook (see the network's guide, above) -* Interactive commands require a public web server with domain name and matching certificate to proxy chat commands to the bot (see the [Telegram](doc/telegram.md#interactive-bot) and [Slack](doc/slack.md#interactive-bot) guides). +* Telegram commands need no server: finbot fetches its updates (pull mode). Slack's commands, and Telegram's in HTTP mode, need a public web server with a domain name and a matching certificate to proxy chat commands to the bot (see the [Telegram](doc/telegram.md#http-mode) and [Slack](doc/slack.md#interactive-bot) guides). * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: ``` @@ -94,7 +94,7 @@ Trade notifications are peformed by polling the Sharesight trades API on a sched The other various reports can also run on a schedule (e.g. daily or weekly), or on demand through the interactive bot. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. Depending on how they're triggered, they will either report to all configured chat networks, or reply to the chat which triggered them. -The interactive bot component requires you to host a web service on a domain with a trusted certificate. It subscribes to push updates from native Slack apps / Telegram bots, and reacts to certain regex seen in chat. It provides: +The interactive bot answers commands in your chats. On Telegram it fetches its updates itself, with no server to host (pull mode); Slack, and Telegram in HTTP mode, post them to a web service you host on a domain with a trusted certificate. It provides: * Stock lookup (financials, company profile, price charts) * Group maintainable watch list, picked up by the various reports * Listing of portfolios and portfolio performance @@ -472,11 +472,11 @@ of the chat network keys it needs. | `sharesight_client_secret` | | Sharesight API access | | `slack_webhook` | | Slack incoming webhook, for outbound reports | | `discord_webhook` | | Discord webhook, for outbound reports | -| `telegramBotToken` | | Telegram bot token, for outbound reports | +| `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | | `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | | `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`); `slack_webhook` is then not used | -| `telegramOutgoingWebhook` | | Telegram webhook path the bot accepts (interactive mode) | -| `telegramOutgoingToken` | | Secret token Telegram must present (interactive mode) | +| `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | +| `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | | `slackOutgoingWebhook` | | Slack webhook path the bot accepts (interactive mode) | | `slackOutgoingToken` | | Token Slack must present (interactive mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | | `slackSigningSecret` | | Secret Slack signs each request with (interactive mode) | diff --git a/bot.py b/bot.py index ba6fa2cb..8ea3be86 100755 --- a/bot.py +++ b/bot.py @@ -13,6 +13,7 @@ from lib import worker from lib import jobs from lib import net +from lib import pull from lib import telegram from lib import delivery from lib import render @@ -237,17 +238,18 @@ def http_server(): print(f'Opening socket on http://{config.env.ip}:{server.effective_port}', file=sys.stderr) else: print('No webhook is set: finbot takes no HTTP requests', file=sys.stderr) - if config.env.telegramBotToken and config.env.telegramOutgoingWebhook: + if config.env.telegramBotToken: try: inbound.TELEGRAM.bot_name = telegram.register() # here only: the job processes import lib.telegram too except Exception as e: - print('ERROR', delivery.error_line(e, 'telegram webhook'), file=sys.stderr) + print('ERROR', delivery.error_line(e, 'telegram'), file=sys.stderr) sys.exit(1) - if config.env.telegramBotToken: threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background if scheduler: jobs.current = scheduler # for the .jobs chat command threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs + for listener in pull.listeners(): # the networks without a webhook: finbot fetches their updates + threading.Thread(target=listener.run, args=(take,), daemon=True).start() # Startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron signal.signal(signal.SIGHUP, in_thread(reload_all)) signal.signal(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache diff --git a/defaults/finbot.env b/defaults/finbot.env index cdbb8783..de12d3b0 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -15,7 +15,7 @@ # /var/lib/finbot). # DATA_DIR=/var/lib/finbot -# The listening socket +# The listening socket, for a network in http mode (its webhook URL set below) ip=127.0.0.1 port=5000 @@ -32,7 +32,7 @@ sharesight_code= sharesight_client_id= sharesight_client_secret= -# Chat networks: outbound +# Chat networks. A Telegram bot token is enough for reports and commands: finbot fetches its updates (pull mode) slack_webhook= discord_webhook= telegramBotToken= @@ -40,7 +40,8 @@ slackBotToken= # Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken; slack_webhook is then unused) slackChannelID= -# Chat networks: inbound (interactive mode) +# Chat networks: inbound over HTTPS (http mode). Telegram: set both to have it post its updates to your server +# instead of finbot fetching them telegramOutgoingWebhook= telegramOutgoingToken= slackOutgoingWebhook= diff --git a/doc/telegram.md b/doc/telegram.md index 82520fc2..78482470 100644 --- a/doc/telegram.md +++ b/doc/telegram.md @@ -1,57 +1,64 @@ # Telegram -finbot talks to Telegram as a bot user that you create with BotFather. The same bot can post the scheduled reports -only, or also answer commands in your chats: the second builds on the first. +finbot talks to Telegram as a bot user that you create with BotFather. -## Reports only, or the interactive bot? +## Pull mode or HTTP mode? -| | Reports only | Interactive bot | +The bot posts the scheduled reports and answers commands in your chats either way. What differs is how it hears from +Telegram: + +| | Pull mode (the default) | HTTP mode | |---|---|---| -| Scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest | ✓ | ✓ | -| Charts with the reports | ✓ | ✓ | -| Chat commands: `.price`, `.history`, `.performance` and the rest ([the list](../README.md#supported-commands)) | ✗ | ✓ | -| Period buttons under charts, and `.jobs` buttons | ✗ | ✓ | -| "typing…" while a command works | ✗ | ✓ | -| A keyboard of the common commands in a direct chat with the bot | ✗ | ✓ | -| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| How messages and button presses reach finbot | finbot asks Telegram for them (long polling) | Telegram posts them to your HTTPS server | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate, on port 443, 80, 88 or 8443 | | `finbot.env` keys | `telegramBotToken`, `telegramChatID` | Those two, plus `telegramOutgoingWebhook` and `telegramOutgoingToken` | -Start with [Reports only](#reports-only). For commands, go on to [Interactive bot](#interactive-bot) afterwards. +Everything else is the same in both: the scheduled reports with their charts, the chat commands (`.price`, `.history`, +`.performance` and the rest, [the list](../README.md#supported-commands)), period buttons under charts and `.jobs` +buttons, "typing…" while a command works, and a keyboard of the common commands in a direct chat with the bot. + +`telegramOutgoingWebhook` chooses the mode: set, HTTP mode; empty, pull mode. A restart switches. As finbot starts in +pull mode it deletes the webhook, and in HTTP mode it sets it; the updates sent meanwhile are kept by Telegram and +arrive after the switch. -## Reports only +## Pull mode 1. Create the bot: message [@BotFather](https://telegram.me/BotFather), send `/newbot` and answer its questions. It replies with the bot's token: put it in `finbot.env` as `telegramBotToken`. 2. Add the bot to the group or channel the reports should go to. In a channel, make it an administrator, so that it can post. -3. Stop strangers adding your bot to their groups: send BotFather `/setjoingroups`, pick the bot and choose `Disable`. +3. Let the bot read the chat: make it an administrator of the group. A bot that is not an administrator sees only the + commands that start with `/`, and finbot's start with `.`. Alternatively, send BotFather `/setprivacy` and choose + `Disable`. +4. Stop strangers adding your bot to their groups: send BotFather `/setjoingroups`, pick the bot and choose `Disable`. Do this after step 2, as it stops you adding the bot to groups too. -4. Put the chat's id in `finbot.env` as `telegramChatID`. +5. Put the chat's id in `finbot.env` as `telegramChatID`. * A group's id is negative. A channel's or a supergroup's is negative and 13 characters long: prepend `-100` if yours is shorter. - * To find it, send `/start@` in the chat, then open - `https://api.telegram.org/bot/getUpdates` in a browser and look for `"chat":{"id":`. This works - only while the bot has no webhook, so before you set up the interactive bot. + * To find it, stop finbot first (`sudo systemctl stop finbot`): Telegram hands each update to one reader only. Send + `/start@` in the chat, then open `https://api.telegram.org/bot/getUpdates` + in a browser and look for `"chat":{"id":`. If the bot is in HTTP mode, getUpdates refuses while the webhook is + set: open `https://api.telegram.org/bot/deleteWebhook` first (finbot sets it again as it + starts). * A group's id changes when Telegram turns it into a supergroup, which some changes to its settings do. Update `telegramChatID` then: finbot does not follow the change. -5. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the chat. +6. Start finbot (`sudo systemctl restart finbot`). The log says `telegram: long polling for updates`, and the scheduled + reports now arrive in the chat. +7. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. ``` telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA telegramChatID=-1001000000000 ``` -## Interactive bot +## HTTP mode Telegram delivers the messages in your chats, and each button press, to finbot over HTTPS. finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name with a valid certificate. Telegram posts only to ports 443, 80, 88 and 8443. -1. Set up the bot for [reports only](#reports-only), above. -2. Let the bot read the chat: make it an administrator of the group. A bot that is not an administrator sees only the - commands that start with `/`, and finbot's start with `.`. Alternatively, send BotFather `/setprivacy` and choose - `Disable`. -3. Put finbot behind an HTTPS server. An nginx example, which lets in only Telegram's addresses: +1. Set up the bot as for [pull mode](#pull-mode), above. +2. Put finbot behind an HTTPS server. An nginx example, which lets in only Telegram's addresses: ``` server { listen 8443 ssl; @@ -73,12 +80,12 @@ with a valid certificate. Telegram posts only to ports 443, 80, 88 and 8443. and reloads nginx. Run it once as root before nginx loads this configuration, or put it in `/etc/cron.daily/` to keep the list current. `util/iptables_telegram_cron` goes further, in the firewall: of the traffic to ports 443 and 8443 it lets in only Telegram's networks on 8443 and drops the rest, so nothing else can be served on those ports. -4. In `finbot.env`, set `telegramOutgoingWebhook` to the URL Telegram should post to, and `telegramOutgoingToken` to a +3. In `finbot.env`, set `telegramOutgoingWebhook` to the URL Telegram should post to, and `telegramOutgoingToken` to a password of your choosing: 1 to 256 characters, of letters, digits, `_` and `-`. Telegram sends the password with every update, and finbot turns away a request without it. -5. Restart finbot. As it starts, it registers the URL with Telegram, unregistering it first, which also wakes up a stale - connection. If Telegram refuses, finbot does not start, and the log says why. -6. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. +4. Restart finbot. As it starts, it sets the webhook again, deleting it first, which also wakes up a stale connection. + If Telegram refuses, finbot does not start, and the log says why. +5. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. ``` telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA diff --git a/lib/config.py b/lib/config.py index 7a22a430..dd719b74 100644 --- a/lib/config.py +++ b/lib/config.py @@ -419,17 +419,22 @@ def retired_notes(preferences): for key in RETIRED if key in preferences] -def startup_warnings(): - """Settings in finbot.env that do not stop the start but keep part of the bot from working, a line each.""" +def startup_warnings(settings=None): + """Settings in finbot.env (an Env; the running one by default) that do not stop the start but keep part of the bot + from working, or do nothing, a line each.""" + s = settings or env warnings = [] - if env.slackOutgoingWebhook and not env.slackBotToken: + if s.slackOutgoingWebhook and not s.slackBotToken: warnings.append('slackOutgoingWebhook is set but slackBotToken is not: Slack commands will get no reply') - if env.slackChannelID and not env.slackBotToken: + if s.slackChannelID and not s.slackBotToken: warnings.append('slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot') - if env.slackOutgoingWebhook and not (env.slackSigningSecret or env.slackOutgoingToken): + if s.slackOutgoingWebhook and not (s.slackSigningSecret or s.slackOutgoingToken): warnings.append('slackOutgoingWebhook is set but neither slackSigningSecret nor slackOutgoingToken is:' ' every Slack request will be refused') - if env.telegramOutgoingWebhook and not env.telegramOutgoingToken: + if s.telegramOutgoingWebhook and not s.telegramOutgoingToken: warnings.append('telegramOutgoingWebhook is set but telegramOutgoingToken is not: every Telegram update will be' ' refused') + if s.telegramOutgoingToken and not s.telegramOutgoingWebhook: + warnings.append('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram' + ' updates (pull mode), and the token is not used') return warnings diff --git a/lib/pull.py b/lib/pull.py new file mode 100644 index 00000000..cc1f1b38 --- /dev/null +++ b/lib/pull.py @@ -0,0 +1,95 @@ +"""The pull modes: a network that finbot.env gives no webhook URL is asked for its updates instead, by a listener in a +thread of its own, which hands each update to bot.take(network, update) as the webhook would, so that reading it, the +repeat check and acknowledging a press are the same both ways. Each listener keeps going through failures, waiting +longer after each one in a row (backoff).""" +import sys +import time + +from lib import config +from lib import delivery +from lib import inbound +from lib import net +from lib import telegram + +BACKOFF = (5, 60) # seconds to wait after a failure: the first, doubled after each further one, up to the second + + +def backoff(failures): + """The seconds to wait after this many failures in a row.""" + first, most = BACKOFF + return min(most, first * 2 ** (failures - 1)) + + +class Listener: + """Updates fetched from a network, for its reader; run() hands them to take() until the process ends.""" + network = None # the reader, an inbound.Network + failures = 0 # in a row + + def run(self, take): + """Fetch and hand on updates for good, waiting as poll() says between rounds.""" + print(f"{self.network.service}: {self.describe()}", file=sys.stderr) + while True: + time.sleep(self.poll(take)) + + def describe(self): + """How it listens, for the log as it starts.""" + return 'listening' + + def poll(self, take): + """One round: fetch what there is, hand each update to take(network, update), and return the seconds to wait + before the next round.""" + raise NotImplementedError + + def failed(self, what, e): + """Note a failure in a row, log it, and return the wait before trying again.""" + self.failures += 1 + wait = backoff(self.failures) + print(f"{self.network.service}: {what} failed, trying again in {wait} s:", net.redact(e), file=sys.stderr) + return wait + + def hand_on(self, take, update): + """take(network, update), a failure reported as bot.main reports one of an authenticated request: the update + came from the network, so the default chats are told.""" + try: + take(self.network, update) + except Exception as e: + delivery.report_error(e, context=self.network.service) + + +class TelegramPoll(Listener): + """Telegram's long polling: getUpdates, which Telegram holds open until there are updates or POLL_SECONDS have + passed, asking for those after the last one taken (offset), which also tells Telegram that one arrived. No webhook + may be set meanwhile: telegram.register() deletes it as finbot starts.""" + network = inbound.TELEGRAM + POLL_SECONDS = 50 # how long Telegram may hold a getUpdates open + READ_SECONDS = 65 # the request's own timeout for the answer: longer than Telegram holds it + offset = None # the update_id after the last one taken + + def describe(self): + """The mode, for the log.""" + return 'long polling for updates (no telegramOutgoingWebhook)' + + def poll(self, take): + """One getUpdates: each update to take, in order; then no wait, or the back-off after a failure (a 409 says a + webhook is set, or another finbot polls with the same token).""" + params = {'timeout': self.POLL_SECONDS, 'allowed_updates': ['message', 'callback_query']} + if self.offset is not None: + params['offset'] = self.offset + try: + updates = telegram.call('getUpdates', json=params, timeout=(net.CONNECT_TIMEOUT, self.READ_SECONDS)) + except Exception as e: + return self.failed('getUpdates', e) + self.failures = 0 + for update in updates: + if isinstance(update.get('update_id'), int): + self.offset = update['update_id'] + 1 # taken, even if handling it fails: it is not fetched again + self.hand_on(take, update) + return 0 + + +def listeners(): + """The listeners finbot.env asks for: each network with its bot token and no webhook URL.""" + found = [] + if config.env.telegramBotToken and not config.env.telegramOutgoingWebhook: + found.append(TelegramPoll()) + return found diff --git a/lib/telegram.py b/lib/telegram.py index 6920f751..37103dc7 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -22,27 +22,30 @@ def try_call(method, **kwargs): print(f"Telegram {method} failed:", net.redact(e), file=sys.stderr) def setWebhook(): - """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken.""" - print("registering", config.env.telegramOutgoingWebhook, file=sys.stderr) + """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken. The + log does not show the URL: its path is a secret.""" + print("telegram: setting the webhook (telegramOutgoingWebhook)", file=sys.stderr) params = {'url': config.env.telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), 'secret_token': config.env.telegramOutgoingToken} return call('setWebhook', params=params) -def delWebhook(): - """Have Telegram stop posting updates.""" - print("deregistering", config.env.telegramOutgoingWebhook, file=sys.stderr) - return call('setWebhook', params={'url': ''}) # unsubscribe +def deleteWebhook(): + """Have Telegram stop posting updates. Those not delivered yet are kept, for the webhook set next or getUpdates.""" + print("telegram: deleting the webhook", file=sys.stderr) + return call('deleteWebhook', json={'drop_pending_updates': False}) def getMe(): """The bot's own account: its username is the bot's name in chats.""" return call('getMe') def register(): - """Called once by bot.py as it starts, never on import (the job processes and the tests import this module too): - unregister and register the webhook again, which wakes up a stale connection on Telegram's side, then return the - bot's @username. Raises RuntimeError, or a requests error, if any of the three calls fails.""" - delWebhook() - setWebhook() + """Called once by bot.py as it starts, never on import (the job processes and the tests import this module too). + With telegramOutgoingWebhook (http mode): delete the webhook and set it again, which wakes up a stale connection on + Telegram's side. Without it (pull mode): delete any webhook, so that getUpdates may take the updates, those not + delivered yet included. Returns the bot's @username. Raises RuntimeError, or a requests error, if a call fails.""" + deleteWebhook() + if config.env.telegramOutgoingWebhook: + setWebhook() return '@' + getMe()['username'] def pinChatMessage(chat_id, message_id): diff --git a/lib/transports.py b/lib/transports.py index e3f3d0b7..7ecd0fc1 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -144,11 +144,7 @@ class TelegramBot(Transport): name = 'telegram' dialect = dialects.TELEGRAM takes_charts = shows_typing = True - - @property - def takes_buttons(self): - """Buttons only where their presses reach finbot: with telegramOutgoingWebhook, Telegram posts them to it.""" - return bool(config.env.telegramOutgoingWebhook) + takes_buttons = True # their presses reach finbot either way: posted to telegramOutgoingWebhook, or polled for @staticmethod def markup(buttons): diff --git a/tests/test_config.py b/tests/test_config.py index 1dd7cfaa..bdc8c452 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -181,6 +181,17 @@ def test_startup_file_linting(self): path.write_text('port=80\n') self.assertEqual(check_config.check(path, Path(common.temp_dir(self), 'none')), []) + def test_check_config_gives_the_startup_warnings_of_a_valid_file(self): + """check_config shows what bot.py would warn about at start for the file it checks: here a Telegram token for + a webhook that is not set (pull mode).""" + path = Path(common.temp_dir(self), 'finbot.env') + path.write_text('telegramOutgoingToken=tok\n') + self.assertEqual(check_config.check(path, Path(common.temp_dir(self), 'none')), []) + self.assertEqual(check_config.warnings(path), ['telegramOutgoingToken is set but telegramOutgoingWebhook is' + ' not: finbot fetches its Telegram updates (pull mode), and the token is not used']) + path.write_text('telegramOutgoingToken=tok\ntelegramOutgoingWebhook=https://example.com/t\n') + self.assertEqual(check_config.warnings(path), []) + class Reload(unittest.TestCase): def setUp(self): @@ -419,10 +430,13 @@ def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(sel telegram_refused = ('telegramOutgoingWebhook is set but telegramOutgoingToken is not: every Telegram update' ' will be refused') no_bot = 'slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot' + unused = ('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram updates' + ' (pull mode), and the token is not used') cases = (({}, []), ({'slackBotToken': ''}, [no_reply]), ({'slackOutgoingToken': ''}, [slack_refused]), ({'slackChannelID': 'C9'}, []), ({'slackChannelID': 'C9', 'slackBotToken': ''}, [no_reply, no_bot]), ({'slackOutgoingToken': '', 'slackSigningSecret': 's'}, []), - ({'telegramOutgoingToken': ''}, [telegram_refused]), ({'telegramOutgoingWebhook': ''}, [])) + ({'telegramOutgoingToken': ''}, [telegram_refused]), ({'telegramOutgoingWebhook': ''}, [unused]), + ({'telegramOutgoingWebhook': '', 'telegramOutgoingToken': ''}, [])) for changes, warnings in cases: with self.subTest(changes=changes), \ mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**hooks, **changes})): diff --git a/tests/test_pull.py b/tests/test_pull.py new file mode 100644 index 00000000..763d9fca --- /dev/null +++ b/tests/test_pull.py @@ -0,0 +1,138 @@ +"""The pull modes: a network without a webhook URL is asked for its updates, which reach the same reader as the +webhook's. Telegram's long polling.""" +import contextlib, io, json, unittest +from tests.common import * +from lib import pull + + +def callback_query(update_id, data='c|h|y|T1'): + """A Telegram update: a button pressed under message 9 in chat 55.""" + return {'update_id': update_id, 'callback_query': {'id': f'q{update_id}', 'from': {'username': 'rob'}, + 'data': data, 'message': {'message_id': 9, 'chat': {'id': 55}}}} + + +class Stop(Exception): + """Raised by a test's fake to end a listener's loop.""" + + +class TelegramPolling(FinbotCase): + """getUpdates in a loop: each update handed on in order, the next asked for after it, and failures waited out.""" + + def setUp(self): + """Telegram in pull mode: a bot token and no webhook URL; telegram.call answers from self.answers.""" + super().setUp() + self.env(telegramBotToken='123:abc', telegramOutgoingWebhook='') + self.calls, self.answers = [], [] + def call(method, **kwargs): + """Note the call, and give the next answer (an exception is raised).""" + self.calls.append((method, kwargs)) + answer = self.answers.pop(0) + if isinstance(answer, Exception): + raise answer + return answer + self.patch(telegram, 'call', call) + + def test_each_update_is_handed_on_in_order_and_the_next_asked_for_after_it(self): + """Each update goes to take in order; the next getUpdates asks for those after the last (which also tells + Telegram it arrived), waits up to 50 seconds for one, and gives the request 65.""" + taken = [] + message = {'update_id': 10, 'message': {'message_id': 1, 'chat': {'id': 5}, 'from': {'first_name': 'A'}, + 'text': '.price'}} + self.answers += [[message, callback_query(11)], []] + poll = pull.TelegramPoll() + self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) + self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) + self.assertEqual(taken, [(inbound.TELEGRAM, message), (inbound.TELEGRAM, callback_query(11))]) + allowed = ['message', 'callback_query'] + self.assertEqual(self.calls, [ + ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed}, 'timeout': (5, 65)}), + ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed, 'offset': 12}, 'timeout': (5, 65)})]) + + def test_failures_are_waited_out_longer_each_time_and_logged_without_the_token(self): + """A failed getUpdates (a 409 when a webhook is set or another finbot polls, the network down) waits 5 seconds, + then twice as long each time up to a minute; a success starts again from 5. The log never shows the token.""" + conflict = RuntimeError('Telegram getUpdates: Conflict: terminated by other getUpdates request') + down = ConnectionError("Max retries exceeded with url: /bot123:SECRET-token/getUpdates") + self.answers += [conflict, down, conflict, conflict, conflict, conflict, [], down] + poll = pull.TelegramPoll() + err = io.StringIO() + with contextlib.redirect_stderr(err): + waits = [poll.poll(lambda *a: None) for _ in range(8)] + self.assertEqual(waits, [5, 10, 20, 40, 60, 60, 0, 5]) + self.assertIn('telegram: getUpdates failed, trying again in 5 s: Telegram getUpdates: Conflict', err.getvalue()) + self.assertNotIn('SECRET', err.getvalue()) + + def test_an_update_that_fails_is_reported_once_and_not_fetched_again(self): + """An update whose handling fails is reported to the default chats, as a failed webhook request is, and the + ones after it still go; the next getUpdates asks for those after all of them.""" + taken = [] + def take(network, update): + """Fail on update 20, note the rest.""" + if update['update_id'] == 20: + raise KeyError('chat') + taken.append(update['update_id']) + self.answers += [[callback_query(20), callback_query(21)], []] + poll = pull.TelegramPoll() + poll.poll(take) + poll.poll(take) + self.assertEqual(taken, [21]) + self.assertEqual(self.errors, ["⚠️ telegram: KeyError: 'chat'"]) + self.assertEqual(self.calls[1][1]['json']['offset'], 22) + + def test_run_polls_for_good_waiting_as_each_round_says(self): + """run() logs the mode, then polls round after round, sleeping what each round returns.""" + rounds, slept = iter([0, 5, 0]), [] + poll = pull.TelegramPoll() + self.patch(poll, 'poll', lambda take: next(rounds)) + def sleep(seconds): + """Note the wait; end the loop after three.""" + slept.append(seconds) + if len(slept) == 3: + raise Stop + self.patch(pull.time, 'sleep', sleep) + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(Stop): + poll.run(bot.take) + self.assertEqual(slept, [0, 5, 0]) + self.assertIn('telegram: long polling for updates (no telegramOutgoingWebhook)', err.getvalue()) + + def test_a_listener_for_each_network_without_a_webhook(self): + """Telegram is polled with its bot token and no telegramOutgoingWebhook; with the URL, or without a token, it + is not.""" + self.assertEqual([type(listener) for listener in pull.listeners()], [pull.TelegramPoll]) + self.env(telegramOutgoingWebhook='https://example.com/telegram') + self.assertEqual(pull.listeners(), []) + self.env(telegramBotToken='', telegramOutgoingWebhook='') + self.assertEqual(pull.listeners(), []) + + def test_a_press_polled_for_is_the_press_the_webhook_gets(self): + """The same button press, posted to the webhook (http mode) or polled for (pull mode), reaches the worker as the + same Press, and is acknowledged the same way.""" + presses, toasts = [], [] + self.patch(worker, 'process_callback', presses.append) + self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) + body = json.dumps(callback_query(80)).encode() + with contextlib.redirect_stderr(io.StringIO()): + self.env(telegramOutgoingWebhook='https://example.com/telegram') + bot.main(wsgi_environ('/telegram', body, HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN='tok'), lambda *a: None) + self.env(telegramOutgoingWebhook='') + update = callback_query(81) + update['callback_query']['id'] = 'q80' # the same press, as getUpdates gives it + self.answers.append([update]) + pull.TelegramPoll().poll(bot.take) + self.join_workers() + self.assertEqual(len(presses), 2) + self.assertEqual(presses[0], presses[1]) + self.assertEqual(toasts, ['Loading 1Y…'] * 2) + + +class Backoff(unittest.TestCase): + """The wait after failures in a row.""" + + def test_it_doubles_from_five_seconds_up_to_a_minute(self): + """5, 10, 20, 40, then a minute for good.""" + self.assertEqual([pull.backoff(n) for n in range(1, 8)], [5, 10, 20, 40, 60, 60, 60]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_reports.py b/tests/test_reports.py index 1bf1b6bf..ba5f86f7 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -533,17 +533,17 @@ def test_a_long_list_is_stored_and_only_the_newest_are_kept(self): with self.assertRaises(KeyError): buttons.resolve(refs[0]) - def test_no_buttons_where_their_presses_would_go_nowhere(self): - """Without telegramOutgoingWebhook (reports only), Telegram hears nothing from finbot's buttons, so a chart, a - scheduled one included, comes without them.""" + def test_telegram_charts_have_buttons_in_both_modes(self): + """A Telegram chart, a scheduled one included, has its period buttons whether Telegram posts their presses to + telegramOutgoingWebhook (http mode) or finbot fetches them (pull mode).""" self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}, telegramChatID='1') label, kwargs = price.job(['7']) # as scheduled: price.py 7 - for hook, buttons in (('https://example.com/telegram', True), ('', False)): + for hook in ('https://example.com/telegram', ''): with self.subTest(telegramOutgoingWebhook=hook): self.env(telegramOutgoingWebhook=hook) self.markups.clear() price.run(**kwargs) - self.assertEqual(bool(self.markups and self.markups[0]), buttons) + self.assertTrue(self.markups and self.markups[0]) def press(self, data): """Press a period button: (the edits made, the toasts shown).""" diff --git a/tests/test_telegram_api.py b/tests/test_telegram_api.py index 4c2ef64d..457ae87c 100644 --- a/tests/test_telegram_api.py +++ b/tests/test_telegram_api.py @@ -1,4 +1,4 @@ -"""Telegram's Bot API: the webhook registered once, as bot.py starts.""" +"""Telegram's Bot API: the webhook set or deleted once, as bot.py starts.""" import contextlib, io, os, subprocess, unittest from tests.common import * @@ -13,17 +13,27 @@ def setUp(self): self.env(telegramOutgoingWebhook='https://example.com/telegram') - def test_register_unregisters_registers_and_names_the_bot(self): - """register() removes the webhook, sets it again and returns the bot's @name.""" + def test_register_sets_the_webhook_again_or_deletes_it_and_names_the_bot(self): + """register() deletes the webhook, keeping the updates not delivered yet; sets it again with + telegramOutgoingWebhook (http mode), or leaves it unset for getUpdates (pull mode); and returns the bot's @name. + The log never shows the URL: its path is a secret.""" calls = [] def post(url, **kwargs): - """Record the method and URL, and answer like Telegram.""" - calls.append((url.rsplit('/', 1)[1], kwargs.get('params', {}).get('url'))) + """Record the method and what it was given, and answer like Telegram.""" + calls.append((url.rsplit('/', 1)[1], kwargs.get('params', {}).get('url') or kwargs.get('json'))) return Resp({'ok': True, 'result': {'username': 'finbot'} if url.endswith('getMe') else True}) self.patch(telegram.net, 'post', post) - with contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(telegram.register(), '@finbot') - self.assertEqual(calls, [('setWebhook', ''), ('setWebhook', 'https://example.com/telegram'), ('getMe', None)]) + kept = ('deleteWebhook', {'drop_pending_updates': False}) + for hook, expected in (('https://example.com/telegram', [kept, ('setWebhook', 'https://example.com/telegram'), + ('getMe', None)]), ('', [kept, ('getMe', None)])): + with self.subTest(telegramOutgoingWebhook=hook): + self.env(telegramOutgoingWebhook=hook) + calls.clear() + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(telegram.register(), '@finbot') + self.assertEqual(calls, expected) + self.assertNotIn('example.com', err.getvalue()) def test_a_refused_call_raises(self): """A call Telegram refuses raises with its description.""" @@ -88,7 +98,7 @@ def test_the_bot_will_not_start_unregistered_and_never_logs_the_token(self): out = subprocess.run([sys.executable, 'bot.py'], cwd=ROOT, env=env, capture_output=True, text=True, timeout=120) self.assertEqual(out.returncode, 1, out.stderr) - self.assertIn('ERROR ⚠️ telegram webhook:', out.stderr) + self.assertIn('ERROR ⚠️ telegram:', out.stderr) self.assertNotIn('SECRET-token', out.stderr + out.stdout) diff --git a/util/check_config.py b/util/check_config.py index 75da666d..12c354b9 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -39,6 +39,16 @@ def check(startup_path, preferences_path): return problems +def warnings(startup_path): + """What a valid finbot.env sets that keeps part of the bot from working or does nothing, a line each: what bot.py + logs at start.""" + try: + values = config.read_startup_file(startup_path) + return config.startup_warnings(config.startup_values(lambda name, default=None: values.get(name, default))) + except config.ConfigError: + return [] + + def notes(preferences_path): """The keys of a valid preferences.conf that do nothing any more, a line each.""" try: @@ -59,6 +69,8 @@ def notes(preferences_path): if not problems: print('config: ok', args.startup, args.preferences if Path(args.preferences).is_file() else '(no preferences.conf, defaults)') + for line in warnings(args.startup): + print('config: warning:', line) for line in notes(args.preferences): print('config:', line) sys.exit(1 if problems else 0) From 6afa9bb9d83b1a8eb159b41b5426f61ff417093a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 16:13:34 +0000 Subject: [PATCH 04/31] Slack Socket Mode: commands with no server (pull mode) Slack's mode follows its keys as Telegram's does: with slackOutgoingWebhook set it posts to finbot as before (http mode); without it, and with the new slackAppToken (an app-level xapp- token), finbot opens a socket to Slack. - pull.SlackSocket: apps.connections.open gives the socket's URL; each envelope is acked at once, before anything else, and its payload (the body Slack would post to the webhook) goes to bot.take, so the reader and the event_id repeat check are the webhook's. Slack's disconnect opens another socket at once; silence for 30 s sends a ping, and as long again with no answer fails the socket, which is opened again after the back-off. - pull.WebSocket wraps websocket-client, imported only when a socket opens (python3-websocket, now in requirements-optional.txt). - Slack chats take buttons with the bot token in either mode. - transports.slack_api(as_app=True) calls with the app-level token. - Start-up warnings: slackAppToken without slackBotToken, beside slackOutgoingWebhook (unused), or not an xapp- token. Slack's tokens are redacted from logs. - doc/slack.md: Socket Mode and HTTP mode, and the steps they share; the README, the finbot.env template and CLAUDE.md. util/record_outputs.py: 182 steps, identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 5 +- README.md | 16 ++-- defaults/finbot.env | 2 + doc/slack.md | 93 +++++++++++++++------ lib/config.py | 21 +++-- lib/net.py | 5 +- lib/pull.py | 116 ++++++++++++++++++++++++++- lib/transports.py | 20 ++--- requirements-optional.txt | 1 + tests/test_config.py | 13 ++- tests/test_pull.py | 165 +++++++++++++++++++++++++++++++++++++- 11 files changed, 398 insertions(+), 59 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 07f7fd87..e7f80940 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,8 +43,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm the network waits for that (`acknowledge`: Telegram's toast), apart from a class per way they arrive (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an update however it arrived, and is the one place a press is acknowledged (the worker never does). `lib/pull.py`: the pull - modes, a listener per network whose webhook URL is not set (`TelegramPoll`: getUpdates), each in a thread of its own - handing its updates to `bot.take`, waiting longer after each failure in a row (`backoff`). + modes, a listener per network whose webhook URL is not set (`TelegramPoll`: getUpdates; `SlackSocket`: Socket Mode, + over websocket-client), each in a thread of its own handing its updates to `bot.take`, waiting longer after each + failure in a row (`backoff`). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a diff --git a/README.md b/README.md index cb098639..50bb8677 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,15 @@ compares the two before the steps. ### Requirements * Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key * A Telegram bot, a Slack app or a Discord webhook (see the network's guide, above) -* Telegram commands need no server: finbot fetches its updates (pull mode). Slack's commands, and Telegram's in HTTP mode, need a public web server with a domain name and a matching certificate to proxy chat commands to the bot (see the [Telegram](doc/telegram.md#http-mode) and [Slack](doc/slack.md#interactive-bot) guides). +* Commands need no server: finbot fetches its updates from Telegram and Slack (pull mode). In HTTP mode instead, a network posts them to a public web server with a domain name and a matching certificate, which proxies them to the bot (see the [Telegram](doc/telegram.md#http-mode) and [Slack](doc/slack.md#http-mode) guides). * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: ``` croniter python-dateutil requests pandas matplotlib numpy ``` -* For a network in http mode (its webhook URL set), also `waitress` (Debian: `python3-waitress`); the optional modules are - in `requirements-optional.txt`, each imported only by the mode that needs it. +* For a network in http mode (its webhook URL set), also `waitress` (Debian: `python3-waitress`); for Slack in Socket + Mode, `websocket-client` (`python3-websocket`). The optional modules are in `requirements-optional.txt`, each imported + only by the mode that needs it. ### Supported commands @@ -94,7 +95,7 @@ Trade notifications are peformed by polling the Sharesight trades API on a sched The other various reports can also run on a schedule (e.g. daily or weekly), or on demand through the interactive bot. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. Depending on how they're triggered, they will either report to all configured chat networks, or reply to the chat which triggered them. -The interactive bot answers commands in your chats. On Telegram it fetches its updates itself, with no server to host (pull mode); Slack, and Telegram in HTTP mode, post them to a web service you host on a domain with a trusted certificate. It provides: +The interactive bot answers commands in your chats. It fetches its updates from Telegram and Slack itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: * Stock lookup (financials, company profile, price charts) * Group maintainable watch list, picked up by the various reports * Listing of portfolios and portfolio performance @@ -474,12 +475,13 @@ of the chat network keys it needs. | `discord_webhook` | | Discord webhook, for outbound reports | | `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | | `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | +| `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | | `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`); `slack_webhook` is then not used | | `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | | `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | -| `slackOutgoingWebhook` | | Slack webhook path the bot accepts (interactive mode) | -| `slackOutgoingToken` | | Token Slack must present (interactive mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | -| `slackSigningSecret` | | Secret Slack signs each request with (interactive mode) | +| `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | +| `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | +| `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | ### `preferences.conf` items diff --git a/defaults/finbot.env b/defaults/finbot.env index de12d3b0..45f4135a 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -39,6 +39,8 @@ telegramBotToken= slackBotToken= # Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken; slack_webhook is then unused) slackChannelID= +# Slack's app-level token (xapp-) for commands without a server: finbot opens a socket to Slack (Socket Mode) +slackAppToken= # Chat networks: inbound over HTTPS (http mode). Telegram: set both to have it post its updates to your server # instead of finbot fetching them diff --git a/doc/slack.md b/doc/slack.md index 038d6150..1933309e 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -13,8 +13,8 @@ the bot posts them itself, charts and all (`slackChannelID`). | Charts in the replies to commands | n/a | ✓, uploaded into the channel | | A sign that finbot is working on a command | ✗ | ✓: ⏳ on the command's message, Slack having no typing notice for bots | | Period buttons under a chart, which redraw it for that period in place | ✗ | ✓ | -| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | -| `finbot.env` keys | `slack_webhook` | That one, plus `slackOutgoingWebhook`, `slackSigningSecret` and `slackBotToken` (and `slackChannelID`, for charts in the scheduled reports) | +| Public HTTPS server | Not needed | Not needed in Socket Mode (the default); needed in HTTP mode | +| `finbot.env` keys | `slack_webhook` | That one, plus `slackBotToken` and `slackAppToken` (Socket Mode) or `slackOutgoingWebhook` and `slackSigningSecret` (HTTP mode); and `slackChannelID`, for charts in the scheduled reports | Start with [Incoming webhook](#incoming-webhook). For commands, go on to [Interactive bot](#interactive-bot) afterwards, in the same app. @@ -37,11 +37,45 @@ can post them with their charts. ## Interactive bot +### Socket Mode or HTTP mode? + +The bot answers commands either way. What differs is how it hears from Slack: + +| | Socket Mode (the default) | HTTP mode | +|---|---|---| +| How messages and button presses reach finbot | Over a socket that finbot opens to Slack | Slack posts them to your HTTPS server | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| Python module | `websocket-client` (Debian: `python3-websocket`) | `waitress` (Debian: `python3-waitress`) | +| `finbot.env` keys | `slackBotToken`, `slackAppToken` | `slackBotToken`, `slackOutgoingWebhook`, `slackSigningSecret` | + +`slackOutgoingWebhook` chooses the mode: set, HTTP mode; empty, Socket Mode. The app's own _Socket Mode_ setting must +agree, as Slack sends to one or the other: to finbot's socket while it is on, to the Request URLs while it is off. To +switch, change both and restart finbot. + +### Socket Mode + +1. Set up the [incoming webhook](#incoming-webhook), above. +2. Under _Settings > Socket Mode_, turn _Enable Socket Mode_ on. Slack asks for an app-level token: name it (finbot, + say) and _Generate_; it has the `connections:write` scope it needs. Copy the token, which starts with `xapp-`, into + `finbot.env` as `slackAppToken`. It is also under _Basic Information > App-Level Tokens_. +3. Install the socket module: `sudo apt install python3-websocket` (or `pip install websocket-client`). +4. Go on with [the events, the buttons and the bot token](#the-events-the-buttons-and-the-bot-token), below. In Socket + Mode there is no Request URL to enter. + +``` +slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA +slackAppToken=xapp-your-app-level-token +slackBotToken=xoxb-your-bot-token +slackChannelID=C0123456789 +``` + +### HTTP mode + Slack delivers the messages that mention the bot (or every message in a channel, as you choose) to finbot over HTTPS. finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name with a valid certificate. -1. Set up the [incoming webhook](#incoming-webhook), above. +1. Set up the [incoming webhook](#incoming-webhook), above, and leave _Settings > Socket Mode_ off. 2. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests come from: ``` @@ -70,38 +104,45 @@ on a domain name with a valid certificate. * The older _Verification Token_, set as `slackOutgoingToken`, still works in place of the signing secret, but Slack has deprecated it: the token travels in every request, where the signing secret never does. 4. In `finbot.env`, set `slackOutgoingWebhook` to the URL Slack should post to, and restart finbot, so that it can - answer Slack's check of the URL in the next step. -5. Under _Event Subscriptions_: - * Turn _Enable Events_ on, and enter the same URL as the _Request URL_. Slack checks it at once, and finbot answers. + answer Slack's check of the URL in the next steps. +5. Go on with [the events, the buttons and the bot token](#the-events-the-buttons-and-the-bot-token), below, entering + this URL as the _Request URL_ where it asks for one. + +``` +slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA +slackOutgoingWebhook=https://www.example.com:8443/slack +slackSigningSecret=0123456789abcdef0123456789abcdef +slackBotToken=xoxb-your-bot-token +slackChannelID=C0123456789 +``` + +### The events, the buttons and the bot token + +1. Under _Event Subscriptions_: + * Turn _Enable Events_ on. In HTTP mode, enter the URL as the _Request URL_: Slack checks it at once, and finbot + answers. * Under _Subscribe to bot events_, add `app_mention`: the bot sees messages that mention it, such as `@finbot price`. * Or add `message.channels` instead, for the bot to see every message in its channels and answer the commands that start with `.`. Not both: the bot would answer a mention twice. - * For direct messages to the bot, add `message.im`, and tick _App Home > Allow users to send Slash commands and - messages from the messages tab_. + * For direct messages to the bot, add `message.im` (Slack adds the `im:history` scope it needs), and tick + _App Home > Allow users to send Slash commands and messages from the messages tab_. * _Save Changes_. -6. Under _Interactivity & Shortcuts_, turn _Interactivity_ on, enter the same URL as the _Request URL_, and _Save - Changes_. Slack posts there the presses of the period buttons under a chart; without it, Slack answers a press with - an error. -7. Under _OAuth & Permissions > Scopes > Bot Token Scopes_, add `chat:write`, `files:write`, `files:read` and +2. Under _Interactivity & Shortcuts_, turn _Interactivity_ on (in HTTP mode, with the same URL as the _Request URL_) + and _Save Changes_. Slack sends finbot the presses of the period buttons under a chart; without it, Slack answers a + press with an error. +3. Under _OAuth & Permissions > Scopes > Bot Token Scopes_, add `chat:write`, `files:write`, `files:read` and `reactions:write`. The upload of a chart needs both `files` scopes, and the ⏳ on a command that takes a while `reactions:write`. -8. At the top of _OAuth & Permissions_, _Install to Workspace_ (or _Reinstall to Workspace_ when you changed scopes of +4. At the top of _OAuth & Permissions_, _Install to Workspace_ (or _Reinstall to Workspace_ when you changed scopes of an installed app), and copy the _Bot User OAuth Token_ into `finbot.env` as `slackBotToken`. finbot replies with it; without it, commands get no reply, and the log says so at startup and for each reply. -9. Restart finbot, and invite the bot to the channel: `/invite @finbot`. -10. Send `@finbot help` in the channel (or `.help`, with `message.channels`). -11. For charts in the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: - the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot then posts the - scheduled reports there, as it posts its replies, and `slack_webhook` is no longer used. - -``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA -slackOutgoingWebhook=https://www.example.com:8443/slack -slackSigningSecret=0123456789abcdef0123456789abcdef -slackBotToken=xoxb-your-bot-token -slackChannelID=C0123456789 -``` +5. Restart finbot, and invite the bot to the channel: `/invite @finbot`. In Socket Mode the log says + `slack: Socket Mode (no slackOutgoingWebhook)`. +6. Send `@finbot help` in the channel (or `.help`, with `message.channels`). +7. For charts in the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: + the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot then posts the + scheduled reports there, as it posts its replies, and `slack_webhook` is no longer used. A chart is uploaded with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`). With its period buttons, it goes in a message of blocks: the report, the chart (uploaded into no channel, and shown by diff --git a/lib/config.py b/lib/config.py index dd719b74..0fdee6b4 100644 --- a/lib/config.py +++ b/lib/config.py @@ -17,11 +17,11 @@ STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', 'sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', - 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', - 'telegramOutgoingToken', 'telegramOutgoingWebhook') + 'slackAppToken', 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', + 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', - 'discord_webhook', 'slackBotToken', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', - 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed + 'discord_webhook', 'slackBotToken', 'slackAppToken', 'slackOutgoingToken', 'slackOutgoingWebhook', + 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' RETIRED = ('alliterate', 'chunk_maxlines', 'performance_use_sharesight') # preferences.conf keys that do nothing any # more: accepted, with a note at start to delete them @@ -192,6 +192,7 @@ class Env: telegramOutgoingWebhook: str telegramOutgoingToken: str slackBotToken: str + slackAppToken: str # xapp-: Socket Mode, Slack's pull mode (without slackOutgoingWebhook) slackChannelID: str # with slackBotToken, scheduled reports go to this channel as the bot, in place of slack_webhook slackOutgoingWebhook: str slackOutgoingToken: str @@ -230,6 +231,7 @@ def startup_values(getenv): telegramOutgoingWebhook=r.str('telegramOutgoingWebhook', ''), telegramOutgoingToken=r.str('telegramOutgoingToken', ''), slackBotToken=r.str('slackBotToken', ''), + slackAppToken=r.str('slackAppToken', ''), slackChannelID=r.str('slackChannelID', ''), slackOutgoingWebhook=r.str('slackOutgoingWebhook', ''), slackOutgoingToken=r.str('slackOutgoingToken', ''), @@ -424,8 +426,15 @@ def startup_warnings(settings=None): from working, or do nothing, a line each.""" s = settings or env warnings = [] - if s.slackOutgoingWebhook and not s.slackBotToken: - warnings.append('slackOutgoingWebhook is set but slackBotToken is not: Slack commands will get no reply') + for key in ('slackOutgoingWebhook', 'slackAppToken'): + if getattr(s, key) and not s.slackBotToken: + warnings.append(f'{key} is set but slackBotToken is not: Slack commands will get no reply') + if s.slackAppToken and s.slackOutgoingWebhook: + warnings.append('slackAppToken is set but so is slackOutgoingWebhook: Slack posts its updates there' + ' (http mode), and the app-level token is not used') + if s.slackAppToken and not s.slackAppToken.startswith('xapp-'): + warnings.append("slackAppToken does not start with xapp-: it must be an app-level token (the app's Basic" + ' Information, App-Level Tokens), not the bot token') if s.slackChannelID and not s.slackBotToken: warnings.append('slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot') if s.slackOutgoingWebhook and not (s.slackSigningSecret or s.slackOutgoingToken): diff --git a/lib/net.py b/lib/net.py index c5e59fa7..2ae4141a 100644 --- a/lib/net.py +++ b/lib/net.py @@ -29,9 +29,10 @@ def timeouts(kwargs): return {'timeout': (CONNECT_TIMEOUT, config.prefs.http_timeout), **kwargs} def redact(text): - """text (or an exception) without the secrets that request URLs, headers and error messages carry: Telegram bot - tokens, Slack and Discord webhook paths, bearer tokens, and token, crumb, key, secret and auth parameters.""" + """text (or an exception) without the secrets that request URLs, headers and error messages carry: Telegram and + Slack tokens, Slack and Discord webhook paths, bearer tokens, and token, crumb, key, secret and auth parameters.""" text = re.sub(r'bot\d+:[\w-]+', 'bot', str(text)) + text = re.sub(r'\bx(?:ox[a-z]|app)-[\w-]+', '', text) # Slack's tokens text = re.sub(r'(/services/|/api/webhooks/)[^\s\'"?&]+', r'\1', text) text = re.sub(r'(?i)(bearer\s+)[\w.~+/=-]+', r'\1', text) return re.sub(r'(?i)(token|crumb|auth\w*|key|secret)=[^&\s\'"]+', r'\1=', text) diff --git a/lib/pull.py b/lib/pull.py index cc1f1b38..50d2d081 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -1,7 +1,8 @@ """The pull modes: a network that finbot.env gives no webhook URL is asked for its updates instead, by a listener in a thread of its own, which hands each update to bot.take(network, update) as the webhook would, so that reading it, the repeat check and acknowledging a press are the same both ways. Each listener keeps going through failures, waiting -longer after each one in a row (backoff).""" +longer after each one in a row (backoff). Telegram's long polling and Slack's Socket Mode.""" +import json import sys import time @@ -10,6 +11,7 @@ from lib import inbound from lib import net from lib import telegram +from lib import transports BACKOFF = (5, 60) # seconds to wait after a failure: the first, doubled after each further one, up to the second @@ -87,9 +89,117 @@ def poll(self, take): return 0 +class WebSocket: + """A WebSocket connection, through websocket-client: imported here only, as Slack's pull mode is the one thing that + needs it (Debian's python3-websocket).""" + + def __init__(self, url): + """Connect to url, waiting net.CONNECT_TIMEOUT seconds at most for the server.""" + import websocket + self.library = websocket + self.ws = websocket.create_connection(url, timeout=net.CONNECT_TIMEOUT) + + def receive(self, timeout): + """The next message's text; '' for a ping or a pong, which only say the line is alive; None when nothing came + within timeout seconds. Raises ConnectionError when the server closes the socket.""" + self.ws.settimeout(timeout) + try: + opcode, data = self.ws.recv_data(control_frame=True) + except self.library.WebSocketTimeoutException: + return None + if opcode == self.library.ABNF.OPCODE_CLOSE: + raise ConnectionError('the server closed the socket') + if opcode == self.library.ABNF.OPCODE_TEXT: + return data.decode('utf-8') if isinstance(data, bytes) else data + return '' + + def send(self, text): + """Send a text message.""" + self.ws.send(text) + + def ping(self): + """Send a ping, which the server answers with a pong.""" + self.ws.ping() + + def close(self): + """Close the socket; a goodbye that fails is of no matter.""" + try: + self.ws.close(timeout=1) + except Exception: + pass + + +class SlackSocket(Listener): + """Slack's Socket Mode: a WebSocket to the URL that apps.connections.open gives for the app-level token + (slackAppToken). Each event or button press comes in an envelope, acked at once, before anything else (Slack sends + again what is not acked within 3 seconds); inside it is what Slack would post to slackOutgoingWebhook. Slack closes + a socket now and then, saying so first (disconnect), and another is opened at once.""" + network = inbound.SLACK + PING_SECONDS = 30 # silence this long sends a ping; as long again after it, and the socket is taken for dead + TAKEN = ('events_api', 'interactive') # the envelopes whose payload goes to the reader + + def describe(self): + """The mode, for the log.""" + return 'Socket Mode (no slackOutgoingWebhook)' + + def open(self): + """The URL of a new socket, from apps.connections.open.""" + return transports.slack_api('apps.connections.open', as_app=True)['url'] + + def connect(self, url): + """A WebSocket connected to url.""" + return WebSocket(url) + + def poll(self, take): + """One socket, from opening it to Slack closing it: then another at once; after a failure, the back-off.""" + try: + connection = self.connect(self.open()) + except Exception as e: + return self.failed('opening a socket', e) + try: + reason = self.listen(connection, take) + except Exception as e: + return self.failed('the socket', e) + finally: + connection.close() + print(f"slack: Slack is closing the socket ({reason}), opening another", file=sys.stderr) + return 0 + + def listen(self, connection, take): + """Read envelopes until Slack says it will close the socket, and return its reason: each acked, then its payload + handed on. A ping goes out after PING_SECONDS of silence; as long again with nothing, not even the pong, raises + TimeoutError.""" + pinged = False + while True: + text = connection.receive(self.PING_SECONDS) + if text is None: + if pinged: + raise TimeoutError(f'nothing from Slack for {2 * self.PING_SECONDS} s, even after a ping') + connection.ping() + pinged = True + continue + pinged = False + if not text: # a ping or a pong + continue + envelope = json.loads(text) + if envelope.get('envelope_id'): + connection.send(json.dumps({'envelope_id': envelope['envelope_id']})) + kind = envelope.get('type') + if kind == 'hello': + self.failures = 0 + elif kind == 'disconnect': + return envelope.get('reason') or 'no reason given' + elif kind in self.TAKEN: + self.hand_on(take, envelope.get('payload') or {}) + + def listeners(): - """The listeners finbot.env asks for: each network with its bot token and no webhook URL.""" + """The listeners finbot.env asks for: each network with its bot token and no webhook URL (Slack's also needs its + app-level token).""" + env = config.env found = [] - if config.env.telegramBotToken and not config.env.telegramOutgoingWebhook: + if env.telegramBotToken and not env.telegramOutgoingWebhook: found.append(TelegramPoll()) + if env.slackBotToken and env.slackAppToken and not env.slackOutgoingWebhook: + found.append(SlackSocket()) return found diff --git a/lib/transports.py b/lib/transports.py index 7ecd0fc1..f7cdb972 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -250,9 +250,9 @@ def shows_typing(self): @property def takes_buttons(self): - """Buttons only where their presses reach finbot: with the bot token and slackOutgoingWebhook, the URL that the - app's interactivity posts to as its events do.""" - return bool(config.env.slackBotToken and config.env.slackOutgoingWebhook) + """Buttons only where their presses reach finbot: with the bot token, and the app's interactivity posting to + slackOutgoingWebhook (http mode) or sending over the socket (Socket Mode, with slackAppToken).""" + return bool(config.env.slackBotToken and (config.env.slackOutgoingWebhook or config.env.slackAppToken)) def typing(self): """Put ⏳ on the message being answered, once (the bot needs the reactions:write scope); a failure is only @@ -478,12 +478,14 @@ def post_chart(self, image, caption, buttons=None): return None -def slack_api(method, **data): - """POST a Slack Web API method as the bot (form-encoded) and return the answer; an error raises RuntimeError.""" - if not config.env.slackBotToken: - raise RuntimeError(f"Slack {method}: needs slackBotToken in finbot.env") - r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + config.env.slackBotToken}, - data=data) +def slack_api(method, as_app=False, **data): + """POST a Slack Web API method (form-encoded) as the bot, or with the app-level token (as_app, for Socket Mode's + apps.connections.open), and return the answer; an error raises RuntimeError.""" + key = 'slackAppToken' if as_app else 'slackBotToken' + token = getattr(config.env, key) + if not token: + raise RuntimeError(f"Slack {method}: needs {key} in finbot.env") + r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + token}, data=data) try: output = r.json() except ValueError: diff --git a/requirements-optional.txt b/requirements-optional.txt index 6f848c1c..a5e00b03 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,3 +1,4 @@ # Each is needed only by one way of receiving updates, and imported only when that way is configured. CI installs them # all; an install needs only those of the modes it uses (Debian: the python3-* package named). waitress>=3.0.2 # any network in http mode (python3-waitress) +websocket-client>=1.8.0 # Slack in pull mode, Socket Mode (python3-websocket) diff --git a/tests/test_config.py b/tests/test_config.py index bdc8c452..9f8cbfc4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,7 +34,7 @@ def test_the_two_key_sets_are_disjoint_and_cover_everything_config_reads(self): def test_secrets_are_startup_keys(self): """Every secret is a startup key: preferences.conf holds none.""" self.assertTrue(set(config.SECRET_KEYS) <= set(config.STARTUP_KEYS)) - self.assertEqual(len(config.SECRET_KEYS), 12) + self.assertEqual(len(config.SECRET_KEYS), 13) def test_unset_preferences_take_their_defaults(self): """An unset preference takes its field's default, as it is: reading tidies nothing away.""" @@ -432,11 +432,20 @@ def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(sel no_bot = 'slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot' unused = ('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram updates' ' (pull mode), and the token is not used') + app_no_reply = 'slackAppToken is set but slackBotToken is not: Slack commands will get no reply' + app_unused = ('slackAppToken is set but so is slackOutgoingWebhook: Slack posts its updates there (http mode),' + ' and the app-level token is not used') + not_app = ("slackAppToken does not start with xapp-: it must be an app-level token (the app's Basic" + ' Information, App-Level Tokens), not the bot token') cases = (({}, []), ({'slackBotToken': ''}, [no_reply]), ({'slackOutgoingToken': ''}, [slack_refused]), ({'slackChannelID': 'C9'}, []), ({'slackChannelID': 'C9', 'slackBotToken': ''}, [no_reply, no_bot]), ({'slackOutgoingToken': '', 'slackSigningSecret': 's'}, []), ({'telegramOutgoingToken': ''}, [telegram_refused]), ({'telegramOutgoingWebhook': ''}, [unused]), - ({'telegramOutgoingWebhook': '', 'telegramOutgoingToken': ''}, [])) + ({'telegramOutgoingWebhook': '', 'telegramOutgoingToken': ''}, []), + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1'}, []), + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1', 'slackBotToken': ''}, [app_no_reply]), + ({'slackAppToken': 'xapp-1'}, [app_unused]), + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xoxb-1'}, [not_app])) for changes, warnings in cases: with self.subTest(changes=changes), \ mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**hooks, **changes})): diff --git a/tests/test_pull.py b/tests/test_pull.py index 763d9fca..1228ea8a 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -1,6 +1,7 @@ """The pull modes: a network without a webhook URL is asked for its updates, which reach the same reader as the -webhook's. Telegram's long polling.""" -import contextlib, io, json, unittest +webhook's. Telegram's long polling, Slack's Socket Mode.""" +import contextlib, io, json, sys, types, unittest +from urllib.parse import urlencode from tests.common import * from lib import pull @@ -126,6 +127,166 @@ def test_a_press_polled_for_is_the_press_the_webhook_gets(self): self.assertEqual(toasts, ['Loading 1Y…'] * 2) +SLACK_PRESS = {'type': 'block_actions', 'token': 'stok', 'trigger_id': 't1', 'user': {'id': 'U1', 'username': 'rob'}, + 'container': {'type': 'message', 'message_ts': '1.5', 'channel_id': 'C1'}, + 'actions': [{'action_id': 'c|h|w|T1', 'value': 'c|h|w|T1', 'type': 'button'}]} # a period button pressed +SLACK_EVENT = {'type': 'event_callback', 'event_id': 'Ev1', 'authorizations': [{'user_id': 'UBOT'}], + 'event': {'type': 'message', 'channel': 'C1', 'user': 'U1', 'text': '.price', 'ts': '1.2'}} + + +class FakeSocket: + """A connection that gives the messages it was made with, a dict sent as JSON, and notes what is sent and when.""" + + def __init__(self, messages, log): + """messages: what receive() gives in turn (dicts as their JSON, None for a silence); log: a list to note in.""" + self.messages, self.log = list(messages), log + + def receive(self, timeout): + """The next message; when there are none left, the socket fails.""" + if not self.messages: + raise ConnectionError('the test socket has no more') + message = self.messages.pop(0) + return json.dumps(message) if isinstance(message, dict) else message + + def send(self, text): + """Note an ack.""" + self.log.append(('ack', json.loads(text)['envelope_id'])) + + def ping(self): + """Note a ping.""" + self.log.append(('ping',)) + + def close(self): + """Note the close.""" + self.log.append(('close',)) + + +class SlackSocketMode(FinbotCase): + """Socket Mode: envelopes acked before anything else, their payloads to the same reader as the webhook's, a socket + opened again when Slack closes it, and a silent one taken for dead.""" + + def setUp(self): + """Slack in pull mode: the bot and app-level tokens, no webhook URL; apps.connections.open answers a URL.""" + super().setUp() + self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') + self.log = [] + self.patch(transports, 'slack_api', lambda method, as_app=False, **data: + self.log.append((method, as_app)) or {'ok': True, 'url': 'wss://example.com/link'}) + + def socket(self, *messages): + """A SlackSocket whose connections give these messages.""" + listener = pull.SlackSocket() + self.patch(listener, 'connect', lambda url: self.log.append(('connect', url)) or FakeSocket(messages, self.log)) + return listener + + def test_each_envelope_is_acked_before_its_payload_is_handed_on(self): + """After hello, an event and a button press each get their ack first, then go to take as Slack would post + them; Slack's disconnect ends the socket, and the next round opens another at once.""" + take = lambda network, update: self.log.append(('take', network.service, update['type'])) + listener = self.socket({'type': 'hello'}, + {'envelope_id': 'e1', 'type': 'events_api', 'payload': SLACK_EVENT}, + {'envelope_id': 'e2', 'type': 'interactive', 'payload': SLACK_PRESS}, + {'type': 'disconnect', 'reason': 'refresh_requested'}) + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(listener.poll(take), 0) + self.assertEqual(self.log, [('apps.connections.open', True), ('connect', 'wss://example.com/link'), + ('ack', 'e1'), ('take', 'slack', 'event_callback'), ('ack', 'e2'), ('take', 'slack', 'block_actions'), + ('close',)]) + self.assertIn('slack: Slack is closing the socket (refresh_requested), opening another', err.getvalue()) + + def test_a_silent_socket_is_pinged_then_taken_for_dead(self): + """Silence sends a ping; a pong (or anything) keeps the socket; silence after a ping fails it, and the next + round waits the back-off.""" + listener = self.socket({'type': 'hello'}, None, '', None, None) + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(listener.poll(lambda *a: None), 5) + self.assertEqual([entry for entry in self.log if entry[0] in ('ping', 'close')], [('ping',), ('ping',), + ('close',)]) + self.assertIn('slack: the socket failed, trying again in 5 s: nothing from Slack for 60 s', err.getvalue()) + + def test_failures_wait_longer_each_time_and_hello_starts_again(self): + """A socket that cannot be opened, or fails, waits longer each time; one that says hello starts again from 5.""" + self.patch(transports, 'slack_api', mock.Mock(side_effect=RuntimeError('Slack apps.connections.open:' + ' invalid_auth'))) + listener = pull.SlackSocket() + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual([listener.poll(lambda *a: None) for _ in range(3)], [5, 10, 20]) + self.assertIn('slack: opening a socket failed, trying again in 5 s: Slack apps.connections.open:' + ' invalid_auth', err.getvalue()) + self.patch(transports, 'slack_api', lambda *a, **k: {'url': 'wss://example.com/link'}) + self.patch(listener, 'connect', lambda url: FakeSocket([{'type': 'hello'}], self.log)) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(listener.poll(lambda *a: None), 5) # the socket failed, after a hello + + def test_a_press_over_the_socket_is_the_press_the_webhook_gets(self): + """The same button press, posted to slackOutgoingWebhook (http mode) or sent over the socket, reaches the worker + as the same Press; an event already taken over one is not run again over the other (its event_id).""" + presses, messages = [], [] + self.patch(worker, 'process_callback', presses.append) + self.patch(worker, 'process_request', messages.append) + self.env(slackOutgoingWebhook='https://example.com/slack', slackOutgoingToken='stok') + with contextlib.redirect_stderr(io.StringIO()): + body = urlencode({'payload': json.dumps(SLACK_PRESS)}).encode() + bot.main(wsgi_environ('/slack', body), lambda *a: None) + bot.main(wsgi_environ('/slack', json.dumps({**SLACK_EVENT, 'token': 'stok'}).encode()), lambda *a: None) + self.env(slackOutgoingWebhook='') + self.socket({'type': 'hello'}, {'envelope_id': 'e1', 'type': 'interactive', 'payload': SLACK_PRESS}, + {'envelope_id': 'e2', 'type': 'events_api', 'payload': SLACK_EVENT}, + {'type': 'disconnect', 'reason': 'warning'}).poll(bot.take) + self.join_workers() + self.assertEqual(len(presses), 2) + self.assertEqual(presses[0], presses[1]) + self.assertEqual([m.text for m in messages], ['.price']) + + def test_the_websocket_is_imported_only_to_connect_and_read_as_slack_needs(self): + """websocket-client is imported when a socket is opened, not before; its frames read as text, '' for a pong, + None for a timeout, and a close raises ConnectionError.""" + class Timeout(Exception): + """websocket-client's timeout.""" + frames = [(1, b'{"type": "hello"}'), (10, b''), Timeout(), (8, b'')] + class WS: + """A websocket-client connection that gives the frames.""" + def settimeout(self, timeout): + """Note the timeout.""" + self.timeout = timeout + def recv_data(self, control_frame=False): + """The next frame, with control frames (pings, pongs) too.""" + frame = frames.pop(0) + if isinstance(frame, Exception): + raise frame + return frame + stub = types.SimpleNamespace(create_connection=lambda url, timeout: WS(), WebSocketTimeoutException=Timeout, + ABNF=types.SimpleNamespace(OPCODE_TEXT=1, OPCODE_CLOSE=8)) + self.patch(sys, 'modules', {**sys.modules, 'websocket': stub}) + socket = pull.WebSocket('wss://example.com/link') + self.assertEqual([socket.receive(30) for _ in range(3)], ['{"type": "hello"}', '', None]) + with self.assertRaises(ConnectionError): + socket.receive(30) + + def test_slack_is_listened_to_with_both_tokens_and_no_webhook(self): + """The socket is opened with the bot and app-level tokens and no slackOutgoingWebhook; with the URL (http mode), + or without either token, it is not.""" + self.env(telegramBotToken='') + self.assertEqual([type(listener) for listener in pull.listeners()], [pull.SlackSocket]) + for changes in ({'slackOutgoingWebhook': 'https://example.com/slack'}, {'slackAppToken': ''}, + {'slackBotToken': ''}): + with self.subTest(changes=changes): + self.env(**changes) + self.assertEqual(pull.listeners(), []) + self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') + + def test_slack_buttons_in_both_modes_only(self): + """A Slack chat takes buttons with the bot token and either mode (the webhook or the app-level token), and not + with the bot token alone: their presses would reach nobody.""" + http = {'slackAppToken': '', 'slackOutgoingWebhook': 'https://example.com/slack'} + bot_only = {'slackAppToken': ''} + for changes, takes in (({}, True), (http, True), (bot_only, False), ({'slackBotToken': ''}, False)): + with self.subTest(changes=changes): + self.env(**changes) + self.assertEqual(transports.SlackBot('C1').takes_buttons, takes) + self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') + + class Backoff(unittest.TestCase): """The wait after failures in a row.""" From 86346c88948813c1d6df8672f07059dfe152bd43 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 16:37:16 +0000 Subject: [PATCH 05/31] The Discord bot, over the gateway or the interactions endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discord gets a bot (discordBotToken) that answers commands and posts the scheduled reports to discordChannelID, charts and buttons and all. Its mode follows its keys: with discordOutgoingWebhook (the app's Interactions Endpoint URL) and discordPublicKey, Discord posts to finbot (http mode); without them, finbot keeps to the gateway (pull mode). - lib/discord.py: REST calls as the bot, 429s waited out as Discord asks, and the slash commands registered at start in both modes, put only when Discord's differ (it limits how many may be made in a day). - Slash commands follow the .help lines (worker.slash_commands): /price, /dividend and /earnings, /buy and /sell, /forwardpe, /help, and /stock and /profile, the new names of .SYMBOL (.stock NVDA works on every network). Each takes its arguments as one text option, in servers and in DMs. On Discord, .help lists them as slash commands. - inbound.Discord reads an interaction the same from the endpoint and the gateway (a slash command is the message it stands for, a component press a Press), and the gateway's messages as Slack's. An interaction is acknowledged with the answer Discord must have within 3 s ("thinking…" for a command, nothing to see for a press); the work waits in the answer's `then` until it is out: after waitress has sent it (bot.Sent), or after the gateway listener has posted it to the callback. - inbound.DiscordInteractions checks each request's Ed25519 signature with PyNaCl, imported only there, and refuses a bad one with the 401 Discord checks for. - pull.DiscordGateway runs discord.py's client (imported only there) in its own thread, with the message and Message Content intents, taking the raw MESSAGE_CREATE and INTERACTION_CREATE events off its event loop. - transports.DiscordBot: no previews, no pings, charts attached, buttons as components in rows of five, the typing notice for a message command; a slash command's first message replaces its "thinking…" (deleted if it says nothing) and the rest follow up; a pressed message is edited through the press's interaction. - Start-up warnings for the Discord keys; Discord's bot and interaction tokens are redacted from logs. discord.py and PyNaCl join requirements-optional.txt. - util/record_outputs.py records Discord's commands and presses too. - doc/discord.md: the bot in both modes, and the webhook for reports only; the README, the finbot.env template and CLAUDE.md. util/record_outputs.py: the four .help steps differ (.stock SYMBOL), and the new Discord steps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 27 ++- README.md | 28 ++- bot.py | 36 +++- defaults/finbot.env | 7 + doc/discord.md | 114 +++++++++- lib/config.py | 25 ++- lib/delivery.py | 23 ++- lib/discord.py | 88 ++++++++ lib/inbound.py | 132 ++++++++++-- lib/net.py | 9 +- lib/pull.py | 84 +++++++- lib/reports/about.py | 11 +- lib/transports.py | 120 ++++++++++- lib/worker.py | 67 ++++-- requirements-optional.txt | 2 + tests/test_config.py | 22 +- tests/test_discord.py | 425 ++++++++++++++++++++++++++++++++++++++ tests/test_edges.py | 4 +- util/record_outputs.py | 18 +- 19 files changed, 1143 insertions(+), 99 deletions(-) create mode 100644 lib/discord.py create mode 100644 tests/test_discord.py diff --git a/CLAUDE.md b/CLAUDE.md index e7f80940..e6a3816a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,8 +44,11 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an update however it arrived, and is the one place a press is acknowledged (the worker never does). `lib/pull.py`: the pull modes, a listener per network whose webhook URL is not set (`TelegramPoll`: getUpdates; `SlackSocket`: Socket Mode, - over websocket-client), each in a thread of its own handing its updates to `bot.take`, waiting longer after each - failure in a row (`backoff`). + over websocket-client; `DiscordGateway`: discord.py's raw events), each in a thread of its own handing its updates to + `bot.take`, waiting longer after each failure in a row (`backoff`). An answer the network must have before any reply + (Discord's deferred interaction) carries the work in its `then`: started once the HTTP answer is sent (`bot.Sent`) + or, over the gateway, posted to its callback. `lib/discord.py`: Discord's REST calls (`call`, rate limits waited + out) and the slash commands registered at start (`register`, from `worker.slash_commands()`). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a @@ -59,9 +62,11 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). - `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or the default chats), `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per way of posting (`TelegramBot`, - `SlackBot`, `SlackWebhook`, `DiscordWebhook`): splitting, captions, charts, buttons, edits, and the typing notice - (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop before posting. - `lib/telegram.py`: Bot API calls (`call`, `try_call`). + `SlackBot`, `DiscordBot`, `SlackWebhook`, `DiscordWebhook`): splitting, captions, charts, buttons, edits, and the + typing notice (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop + before posting. `DiscordBot`: a slash command's first message replaces its "thinking…" through its `interaction` + (application id/token), and a pressed message is edited through the press's. `lib/telegram.py`: Bot API calls + (`call`, `try_call`). - `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), what a pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. `lib/charts.py`: a chart with period buttons, built by its kind's builder (`build`) for the command's first view @@ -86,12 +91,12 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm them; `temp_dir(self)` is a directory removed after the test, `bare_environment()` the environment without finbot's settings. - Which test file: commands and scheduled reports `test_reports`; failures, bad input, secrets in logs, signals - `test_edges`; webhook requests, Slack events and signatures `test_inbound`; the pull modes `test_pull`; Yahoo - `test_yahoo`; Sharesight `test_sharesight`; posting to each network `test_sending`; registering Telegram's webhook - `test_telegram_api`; the schedule file and job arguments `test_schedule`; the scheduler and manual runs - `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, - formats, links, buttons, cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, and the - docstring, stdout and line-width checks `test_finbot`. + `test_edges`; webhook requests, Slack events and signatures `test_inbound`; the pull modes `test_pull`; the Discord + bot `test_discord`; Yahoo `test_yahoo`; Sharesight `test_sharesight`; posting to each network `test_sending`; + registering Telegram's webhook `test_telegram_api`; the schedule file and job arguments `test_schedule`; the scheduler + and manual runs `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); + markets, formats, links, buttons, cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, + and the docstring, stdout and line-width checks `test_finbot`. ## Conventions - Read settings as `config.env.` / `config.prefs.` where they are used, never into a default or module diff --git a/README.md b/README.md index 50bb8677..00d2eb9a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ * Ex-dividend date warnings * Highly shorted stock warnings (AU, US) * Interday and premarket price movements over a defined threshold -* Interactive chat commands (Slack & Telegram): +* Interactive chat commands (Telegram, Slack & Discord): * All the above reports * Price history graph * Stock lookup with price/valuation related stats and warnings @@ -27,14 +27,14 @@ What finbot can do on each network. Telegram takes everything; the others take w | Feature | Telegram | Slack | Discord | |---|---|---|---| -| Chat commands | ✓ | ✓ | ✗ (a webhook only posts) | -| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots) | ✗ | -| Charts in replies | ✓ | ✓ with `slackBotToken` | n/a | +| Chat commands | ✓ | ✓ | ✓ with the bot: slash commands (`/price`), and messages (`.price`) over the gateway | +| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots) | ✓ for a message; "thinking…" for a slash command | +| Charts in replies | ✓ | ✓ with `slackBotToken` | ✓ | | Charts in scheduled reports | ✓ | ✓ with `slackChannelID` and `slackBotToken` (the incoming webhook takes text only) | ✓ | -| Period and job buttons | ✓ | Period buttons, with the interactive bot | ✗ | +| Period and job buttons | ✓ | Period buttons, with the interactive bot | ✓ with the bot | | No link previews | ✓ | ✓ | ✓ | | Linked tickers | ✓ | ✓ | ✓ | -| Charts updated in place by their buttons | ✓ | ✓ | ✗ | +| Charts updated in place by their buttons | ✓ | ✓ | ✓ | **Setting up a network:** [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md). Each guide covers the whole setup of that network, and where there is a choice between reports only and the interactive bot, it @@ -44,7 +44,7 @@ compares the two before the steps. ### Requirements * Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key -* A Telegram bot, a Slack app or a Discord webhook (see the network's guide, above) +* A Telegram bot, a Slack app or a Discord bot (see the network's guide, above) * Commands need no server: finbot fetches its updates from Telegram and Slack (pull mode). In HTTP mode instead, a network posts them to a public web server with a domain name and a matching certificate, which proxies them to the bot (see the [Telegram](doc/telegram.md#http-mode) and [Slack](doc/slack.md#http-mode) guides). * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: @@ -52,8 +52,9 @@ compares the two before the steps. croniter python-dateutil requests pandas matplotlib numpy ``` * For a network in http mode (its webhook URL set), also `waitress` (Debian: `python3-waitress`); for Slack in Socket - Mode, `websocket-client` (`python3-websocket`). The optional modules are in `requirements-optional.txt`, each imported - only by the mode that needs it. + Mode, `websocket-client` (`python3-websocket`); for Discord's gateway, `discord.py` (`python3-discord`), and for its + http mode, `PyNaCl` (`python3-nacl`). The optional modules are in `requirements-optional.txt`, each imported only by + the mode that needs it. ### Supported commands @@ -64,6 +65,7 @@ Where: * _symbol_ is a Yahoo Finance symbol; a bare one Yahoo does not know is tried again on your exchange (`exchange_suffix`: `BHP` may be `BHP.AX`) ``` .symbol +.stock symbol .beta .buy .sell @@ -89,7 +91,7 @@ Where: .who symbol ``` -`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. +`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command, as Discord registers them: `/stock symbol` (or `/profile symbol`) for `.symbol`, `/forwardpe` for `.forward pe`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying your configured chat networks of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. @@ -472,7 +474,7 @@ of the chat network keys it needs. | `sharesight_client_id` | | Sharesight API access | | `sharesight_client_secret` | | Sharesight API access | | `slack_webhook` | | Slack incoming webhook, for outbound reports | -| `discord_webhook` | | Discord webhook, for outbound reports | +| `discord_webhook` | | Discord webhook, for outbound reports (not used with `discordBotToken` and `discordChannelID`) | | `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | | `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | | `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | @@ -482,6 +484,10 @@ of the chat network keys it needs. | `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | | `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | | `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | +| `discordBotToken` | | Discord bot token: replies, and commands over the gateway unless `discordOutgoingWebhook` is set | +| `discordChannelID` | | Channel id that scheduled reports go to on Discord as the bot, charts and buttons and all (with `discordBotToken`); `discord_webhook` is then not used | +| `discordOutgoingWebhook` | | The Interactions Endpoint URL Discord posts slash commands and button presses to (HTTP mode); empty, the gateway (pull mode) | +| `discordPublicKey` | | The app's public key, which checks the signature of each interaction (HTTP mode) | ### `preferences.conf` items diff --git a/bot.py b/bot.py index 8ea3be86..9a7587b5 100755 --- a/bot.py +++ b/bot.py @@ -5,10 +5,11 @@ # ignored until startup is done (the reload handler is installed last): its default action kills the bot signal.signal(signal.SIGHUP, signal.SIG_IGN) signal.signal(signal.SIGUSR1, signal.SIG_IGN) # likewise the cache flush -import collections, json, os, re, sys, threading, traceback +import collections, dataclasses, json, os, re, sys, threading, traceback from pathlib import Path from lib import config +from lib import discord from lib import inbound from lib import worker from lib import jobs @@ -68,12 +69,12 @@ def handle(environ, start_response): bytes, up to MAX_BODY) and take it (take): the network gets its answer at once.""" def respond(response): """Answer the request: a status line and a body (logged with the debug preference, naming the network, not the - path).""" + path). Work that must wait for the answer starts once the server has sent it.""" if config.prefs.debug: print(f"[http] request for {environ.get(ROUTED) or 'no network'} answered {response.status}", file=sys.stderr) start_response(response.status, [('Content-type', 'application/json')]) - return [response.body] + return Sent([response.body], response.then) if response.then else [response.body] def print_headers(): """Log the request headers, Telegram's secret header redacted.""" for name, value in sorted(environ.items()): @@ -104,22 +105,35 @@ def print_headers(): shown = {k: '' if k == 'token' else v for k, v in update.items()} # Slack's verification token print("warning:", why, file=sys.stderr) print(f"[{source.service}]: refused update", json.dumps(shown, indent=4, default=str), file=sys.stderr) - return respond(inbound.FORBIDDEN) + return respond(source.REFUSED) environ[AUTHENTICATED] = True return respond(take(source.network, update)) +class Sent(list): + """A response body that starts some work once the server has sent it: WSGI calls close() then.""" + def __init__(self, body, then): + """body: the chunks; then: the work.""" + super().__init__(body) + self.then = then + + def close(self): + """The answer is out: start the work.""" + self.then() + def take(network, update): """An update from a network (an inbound.Network reads it), however it arrived: what it says goes to lib/worker in a - thread of its own (dispatch), unless it is a repeat; a button press is acknowledged first, the one place that does. - Returns the inbound.Response for the network.""" + thread of its own (dispatch), unless it is a repeat. It is acknowledged first, the one place that does + (Network.acknowledge: Telegram's toast for a press). Returns the inbound.Response for the network; when the network + must have it before any reply (Discord's interactions), the work waits in its `then`, for whoever delivers it.""" event = network.parse(update) if isinstance(event, inbound.Response): return event if seen_before(network.service, network.update_id(update)): print(f"[{network.service}]: update {network.update_id(update)} seen already, not run again", file=sys.stderr) return inbound.OK - if isinstance(event, inbound.Press): - network.acknowledge(event) # Telegram's toast ("Loading 1Y…") shows while the worker builds the answer + answer = network.acknowledge(event) + if answer is not None: + return dataclasses.replace(answer, then=lambda: dispatch(event)) dispatch(event) return inbound.OK @@ -245,6 +259,12 @@ def http_server(): print('ERROR', delivery.error_line(e, 'telegram'), file=sys.stderr) sys.exit(1) threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background + if config.env.discordBotToken: + try: + inbound.DISCORD.bot_id = discord.register(worker.slash_commands()) + except Exception as e: + print('ERROR', delivery.error_line(e, 'discord'), file=sys.stderr) + sys.exit(1) if scheduler: jobs.current = scheduler # for the .jobs chat command threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs diff --git a/defaults/finbot.env b/defaults/finbot.env index 45f4135a..7b15ef59 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -41,11 +41,18 @@ slackBotToken= slackChannelID= # Slack's app-level token (xapp-) for commands without a server: finbot opens a socket to Slack (Socket Mode) slackAppToken= +# Discord's bot token: reports, and commands over the gateway (pull mode); and the channel id for scheduled reports +discordBotToken= +discordChannelID= # Chat networks: inbound over HTTPS (http mode). Telegram: set both to have it post its updates to your server # instead of finbot fetching them telegramOutgoingWebhook= telegramOutgoingToken= +# Discord: the Interactions Endpoint URL and the app's public key, for slash commands and buttons over HTTPS instead of +# the gateway +discordOutgoingWebhook= +discordPublicKey= slackOutgoingWebhook= slackOutgoingToken= slackSigningSecret= diff --git a/doc/discord.md b/doc/discord.md index 82db2477..c1560a3d 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -1,15 +1,110 @@ # Discord -finbot reaches Discord through a webhook, which posts the scheduled reports into a channel. A webhook only posts, so -there is no choice to make: on Discord, finbot does not answer commands. For commands, set up finbot on -[Telegram](telegram.md) or [Slack](slack.md) as well. +finbot talks to Discord as the bot user of a Discord application that you create in the Developer Portal. The bot +posts the scheduled reports into a channel and answers commands: slash commands (`/price`) in both modes, and messages +(`.price`, `@finbot price`) over the gateway. A Discord webhook, which only posts, can still carry the scheduled +reports on its own (see [Webhook](#webhook-reports-only)). -What the webhook carries: -* the scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest; -* their charts, attached to the messages; -* linked tickers, without link previews, and without pinging anyone. +## Gateway mode or HTTP mode? -## Webhook +The bot does the same either way. What differs is how it hears from Discord: + +| | Gateway mode (the default) | HTTP mode | +|---|---|---| +| How commands and button presses reach finbot | Over Discord's gateway, a connection finbot keeps open | Discord posts them to your HTTPS server | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| Commands | Slash commands (`/price`), and messages (`.price`, `@finbot price`) | Slash commands (`/price`): Discord posts no messages | +| Python modules | `discord.py` (Debian: `python3-discord`) | `PyNaCl` and `waitress` (Debian: `python3-nacl`, `python3-waitress`) | +| `finbot.env` keys | `discordBotToken`, `discordChannelID` | Those two, plus `discordOutgoingWebhook` and `discordPublicKey` | + +In both: the scheduled reports with their charts; period buttons under charts and `.jobs` buttons, which redraw their +message in place; linked tickers, without link previews and without pinging anyone; and direct messages with the bot. +A command in a message shows "finbot is typing…" while it works; a slash command shows Discord's "finbot is +thinking…" until its answer takes its place. + +`discordOutgoingWebhook` chooses the mode: set, HTTP mode; empty, gateway mode. Discord sends the slash commands and +button presses to the app's _Interactions Endpoint URL_ while it is set, and over the gateway while it is empty, so +change both, and restart finbot, to switch. + +## Gateway mode + +1. In the [Developer Portal](https://discord.com/developers/applications), choose _New Application_ and name it + (finbot, say). +2. Under _Bot_: + * _Reset Token_, and copy the token into `finbot.env` as `discordBotToken`. + * Turn _Message Content Intent_ on, for the bot to read `.price` and `@finbot price` in your channels. Without it, + Discord refuses the connection, and the log says so. + * Turn _Public Bot_ off, so that only you can add the bot to servers. +3. Under _OAuth2 > URL Generator_, tick the scopes `bot` and `applications.commands`, then the bot permissions _View + Channels_, _Send Messages_, _Attach Files_ and _Read Message History_. Open the URL it makes, and add the bot to your + server. +4. In Discord, turn on _User Settings > Advanced > Developer Mode_; then right-click the channel for the scheduled + reports, _Copy Channel ID_, and put it in `finbot.env` as `discordChannelID`. +5. Install discord.py: `sudo apt install python3-discord` (or `pip install discord.py`). +6. Restart finbot (`sudo systemctl restart finbot`). As it starts, it registers the slash commands, which can take a + few minutes to show in Discord the first time, and the log says `discord: gateway (no discordOutgoingWebhook)`. +7. Send `/help` (or `.help`) in the channel, or in a direct message to the bot. + +``` +discordBotToken=your-bot-token +discordChannelID=1000000000000000000 +``` + +## HTTP mode + +Discord posts the slash commands and button presses to finbot over HTTPS, each signed with the app's key. finbot +listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a +domain name with a valid certificate. The bot shows as offline in Discord, as it keeps no gateway connection. + +1. Set up the bot as for [gateway mode](#gateway-mode), steps 1 to 4. The _Message Content Intent_ is not needed. +2. Install PyNaCl, which checks the signatures, and waitress, the web server: `sudo apt install python3-nacl + python3-waitress` (or `pip install PyNaCl waitress`). +3. Put finbot behind an HTTPS server. An nginx example: + ``` + server { + listen 8443 ssl; + + server_name www.example.com; + ssl_certificate /etc/letsencrypt/live/www.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/www.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location /discord { + proxy_pass http://127.0.0.1:5000/discord; + } + } + ``` + Discord publishes no list of its addresses, so the signature on each request is what keeps strangers out: finbot + refuses a request without a good one. +4. In `finbot.env`, set `discordOutgoingWebhook` to the URL Discord should post to, and `discordPublicKey` to the app's + _General Information > Public Key_. Restart finbot, so that it can answer Discord's check in the next step. +5. In the Developer Portal, set _General Information > Interactions Endpoint URL_ to the same URL, and _Save Changes_. + Discord checks it at once, sending a signed request and one with a bad signature, which finbot refuses as Discord + expects. +6. Send `/help` in the channel, or in a direct message to the bot. + +``` +discordBotToken=your-bot-token +discordChannelID=1000000000000000000 +discordOutgoingWebhook=https://www.example.com:8443/discord +discordPublicKey=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +``` + +## Slash commands + +finbot registers a slash command for each line of `.help`, as it starts, and again only when the list has changed: +`/price`, `/dividend` and `/earnings`, `/buy` and `/sell`, `/forwardpe` for `.forward pe`, `/stock` (or `/profile`) for +`.SYMBOL`, and the rest. Each takes its arguments, if any, as one text option, as the message would: `/price` with +`5 7d` is `.price 5 7d`. They work in the server's channels and in a direct message with the bot. + +A message holds up to 2,000 characters, so a long report arrives as several messages. A chart goes as an attachment of +the message that carries its report, with its period buttons under it. + +## Webhook (reports only) + +Without the bot, a webhook can post the scheduled reports into a channel, with their charts. It takes no commands and +no buttons. With `discordBotToken` and `discordChannelID`, the bot posts the scheduled reports itself, and the webhook +is not used. 1. In Discord, open _Server Settings > Integrations > Webhooks_ and choose _New Webhook_. 2. Name it (finbot, say), pick the channel for the reports, and _Copy Webhook URL_. @@ -20,5 +115,4 @@ What the webhook carries: discord_webhook=https://discord.com/api/webhooks/1009998000000000000/aaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbb ``` -A URL ending in `/slack`, from older setups, works as it is: finbot uses Discord's own endpoint either way. A message -holds up to 2,000 characters, so a long report arrives as several messages. +A URL ending in `/slack`, from older setups, works as it is: finbot uses Discord's own endpoint either way. diff --git a/lib/config.py b/lib/config.py index 0fdee6b4..a4037dc7 100644 --- a/lib/config.py +++ b/lib/config.py @@ -18,10 +18,12 @@ STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', 'sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', 'slackAppToken', 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', - 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') + 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook', 'discordBotToken', 'discordChannelID', + 'discordOutgoingWebhook', 'discordPublicKey') SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', 'slackAppToken', 'slackOutgoingToken', 'slackOutgoingWebhook', - 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed + 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook', 'discordBotToken', + 'discordOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' RETIRED = ('alliterate', 'chunk_maxlines', 'performance_use_sharesight') # preferences.conf keys that do nothing any # more: accepted, with a note at start to delete them @@ -197,6 +199,10 @@ class Env: slackOutgoingWebhook: str slackOutgoingToken: str slackSigningSecret: str + discordBotToken: str + discordChannelID: str # with discordBotToken, scheduled reports go to this channel as the bot + discordOutgoingWebhook: str # the interactions endpoint URL (http mode); unset, finbot keeps to the gateway + discordPublicKey: str # the app's key, which checks the signature of each interaction posted to the endpoint sharesight_auth: dict webhooks: dict # service -> outbound URL: Telegram's Bot API base, the Slack and Discord incoming webhooks @@ -236,6 +242,10 @@ def startup_values(getenv): slackOutgoingWebhook=r.str('slackOutgoingWebhook', ''), slackOutgoingToken=r.str('slackOutgoingToken', ''), slackSigningSecret=r.str('slackSigningSecret', ''), + discordBotToken=r.str('discordBotToken', ''), + discordChannelID=r.str('discordChannelID', ''), + discordOutgoingWebhook=r.str('discordOutgoingWebhook', ''), + discordPublicKey=r.str('discordPublicKey', ''), sharesight_auth={ "grant_type": 'client_credentials', "code": getenv('sharesight_code'), @@ -446,4 +456,15 @@ def startup_warnings(settings=None): if s.telegramOutgoingToken and not s.telegramOutgoingWebhook: warnings.append('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram' ' updates (pull mode), and the token is not used') + if s.discordOutgoingWebhook and not s.discordBotToken: + warnings.append('discordOutgoingWebhook is set but discordBotToken is not: Discord commands will get no reply') + if s.discordChannelID and not s.discordBotToken: + warnings.append('discordChannelID is set but discordBotToken is not: scheduled reports cannot go there as the' + ' bot') + if s.discordOutgoingWebhook and not re.fullmatch(r'[0-9a-fA-F]{64}', s.discordPublicKey): + warnings.append("discordOutgoingWebhook is set but discordPublicKey is not the app's public key (64 hex" + ' digits): every Discord interaction will be refused') + if s.discordPublicKey and not s.discordOutgoingWebhook: + warnings.append('discordPublicKey is set but discordOutgoingWebhook is not: finbot keeps to the Discord gateway' + ' (pull mode), and the key is not used') return warnings diff --git a/lib/delivery.py b/lib/delivery.py index ebdf48c4..44a802cc 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -20,20 +20,21 @@ class Report: buttons: list = None -REPLIES = {'telegram': transports.TelegramBot, 'slack': transports.SlackBot} # a reply into the chat that asked +REPLIES = {'telegram': transports.TelegramBot, 'slack': transports.SlackBot, 'discord': transports.DiscordBot} -def reply_to(service, chat, message=None): - """The destination of a reply into a chat (to `message`, the one being answered, if given), or None for a network - that takes none.""" +def reply_to(service, chat, message=None, interaction='', pressed=False): + """The destination of a reply into a chat: to `message`, the one being answered, if given; through `interaction` + where the network answers one (Discord's slash command, or with pressed, its button press). None for a network that + takes no replies.""" kind = REPLIES.get(service) - return kind(chat, message) if kind else None + return kind(chat, message, interaction, pressed) if kind else None def default_chats(): """Where a scheduled report goes, in this order: Telegram's telegramChatID; Slack's slackChannelID as the bot, with - slackBotToken (so that charts go too), else its incoming webhook; Discord's webhook. Raises RuntimeError when there - is nowhere to post.""" + slackBotToken (so that charts go too), else its incoming webhook; Discord's discordChannelID as the bot, with + discordBotToken (buttons too), else its webhook. Raises RuntimeError when there is nowhere to post.""" env = config.env chats = [] if env.telegramChatID and 'telegram' in env.webhooks: @@ -42,11 +43,13 @@ def default_chats(): chats.append(transports.SlackBot(env.slackChannelID)) elif 'slack' in env.webhooks: chats.append(transports.SlackWebhook(env.webhooks['slack'])) - if 'discord' in env.webhooks: + if env.discordChannelID and env.discordBotToken: + chats.append(transports.DiscordBot(env.discordChannelID)) + elif 'discord' in env.webhooks: chats.append(transports.DiscordWebhook(env.webhooks['discord'])) if not chats: - raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID, slack_webhook or' - ' discord_webhook') + raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID, discordChannelID,' + ' slack_webhook or discord_webhook') return chats diff --git a/lib/discord.py b/lib/discord.py new file mode 100644 index 00000000..4c6a7891 --- /dev/null +++ b/lib/discord.py @@ -0,0 +1,88 @@ +"""Discord's REST API as the bot (discordBotToken): calls that wait out Discord's rate limits, and the slash commands +registered as finbot starts. The same in both modes: the gateway and the interactions endpoint only bring updates in.""" +import sys +import time + +from lib import config +from lib import net + +API = 'https://discord.com/api/v10' +TRIES = 3 # a call that Discord rate-limits is tried this many times at most +LONGEST_WAIT = 10 # seconds: a rate limit asking for longer is waited this long +SUPPRESS_EMBEDS = 4 # message flag: no link previews +NO_MENTIONS = {'parse': []} # text that looks like @everyone or a mention pings nobody +GUILD, BOT_DM = 0, 1 # where a slash command may be used: in servers, and in a DM with the bot +OPTION = 'arguments' # the name of a slash command's one option: its arguments as text + + +def call(method, path, **kwargs): + """A REST call as the bot: Discord's answer as JSON, or None when it is empty. A 429 waits as long as Discord asks + (LONGEST_WAIT at most) and tries again. Raises RuntimeError with Discord's reason when it refuses, or a requests + error.""" + headers = {'Authorization': 'Bot ' + config.env.discordBotToken, **kwargs.pop('headers', {})} + for attempt in range(1, TRIES + 1): + r = net.request(method, API + path, headers=headers, **kwargs) + if r.status_code != 429 or attempt == TRIES: + break + time.sleep(min(retry_after(r), LONGEST_WAIT)) + if r.status_code >= 400: + raise RuntimeError(f"Discord {method} {path}: {r.status_code} {reason(r)}") + try: + return r.json() + except ValueError: # empty: 204 No Content + return None + + +def try_call(method, path, **kwargs): + """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" + try: + return call(method, path, **kwargs) + except Exception as e: + print(f"Discord {method} failed:", net.redact(e), file=sys.stderr) + + +def retry_after(response): + """The seconds a rate-limited answer asks to wait (its retry_after), or one when it does not say.""" + try: + return float(response.json().get('retry_after')) + except (ValueError, TypeError, AttributeError): + return 1.0 + + +def reason(response): + """Discord's reason for a refusal: its message and code, else the start of the body.""" + try: + answer = response.json() + return f"{answer['message']} ({answer.get('code')})" + except (ValueError, TypeError, KeyError): + return response.text[:200] + + +def command(name, description, arguments): + """A slash command as Discord takes it: a chat command usable in servers and in a DM with the bot, and the arguments + it takes, if any, as one optional text option that they describe.""" + option = {'type': 3, 'name': OPTION, 'description': arguments[:100], 'required': False} + return {'name': name, 'type': 1, 'description': description[:100], 'contexts': [GUILD, BOT_DM], + 'integration_types': [0], 'options': [option] if arguments else []} + + +def summary(commands): + """What of a list of slash commands Discord may differ on from ours, in a form to compare: an answer from Discord + carries ids and versions besides.""" + return sorted((c['name'], c.get('description'), sorted(c.get('contexts') or []), + [(o['name'], o.get('description'), o.get('required', False), o.get('autocomplete', False)) + for o in c.get('options') or []]) for c in commands) + + +def register(commands): + """Called once by bot.py as it starts, in both modes: make the application's slash commands these ([(name, + description, arguments)], from worker.slash_commands), replacing them all only when they differ (Discord limits how + many may be made in a day). Returns the bot's user id: a message that mentions it is for the bot. Raises + RuntimeError, or a requests error.""" + application = call('GET', '/applications/@me')['id'] + wanted = [command(*entry) for entry in commands] + path = f'/applications/{application}/commands' + if summary(call('GET', path) or []) != summary(wanted): + call('PUT', path, json=wanted) + print(f"discord: {len(wanted)} slash commands registered", file=sys.stderr) + return str(call('GET', '/users/@me')['id']) diff --git a/lib/inbound.py b/lib/inbound.py index 15fd8e98..3ab8bef8 100644 --- a/lib/inbound.py +++ b/lib/inbound.py @@ -1,8 +1,9 @@ """What the networks send the bot. Reading an update is apart from how it arrives: a class per network reads its updates -(Telegram, Slack), and a class per way of arriving takes them in (a webhook: the path the network posts to, how it -proves itself, the body), so that another way (a pull mode) can hand its updates to the same reader. An update becomes -a Message to answer, a Press of a button, or a Response the network gets straight back (a challenge, or nothing to -do). The reader also acknowledges a press where the network waits for that (Telegram), however it arrived.""" +(Telegram, Slack, Discord), and a class per way of arriving takes them in (a webhook: the path the network posts to, +how it proves itself, the body), so that another way (a pull mode) can hand its updates to the same reader. An update +becomes a Message to answer, a Press of a button, or a Response the network gets straight back (a challenge, or nothing +to do). The reader also acknowledges an update where the network waits for that, however it arrived: Telegram's press +with a call, Discord's interaction with the answer it must have before any reply.""" import dataclasses import hashlib import hmac @@ -28,6 +29,7 @@ class Message: bot_name: str # the bot's @name, which can start a command user_name: str # what to call them message_id: str = '' # the message itself: Slack shows ⏳ on it while finbot works + interaction: str = '' # Discord's slash command: its application id and token, to answer through @dataclasses.dataclass @@ -39,13 +41,16 @@ class Press: message_id: str data: str user: str = '' # who pressed it, for the log + interaction: str = '' # Discord's press: its application id and token, to edit the message pressed through @dataclasses.dataclass class Response: - """What the network gets back at once: a status and a body.""" + """What the network gets back at once: a status and a body; and the work to start once it has them (`then`), when + the network must have this answer before any reply (Discord's deferred answer to an interaction).""" status: str = '200 OK' body: bytes = b'' + then: object = dataclasses.field(default=None, compare=False) OK = Response() @@ -53,6 +58,7 @@ class Response: UNHANDLED = Response(body=b'

Unhandled

') # a kind it might, which this code does not know BAD_REQUEST = Response('400 Bad Request', b'

Bad Request

') FORBIDDEN = Response('403 Forbidden', b'

Unauthorized

') +UNAUTHORIZED = Response('401 Unauthorized', b'

Unauthorized

') # what Discord expects for a bad signature NOT_FOUND = Response('404 Not Found', b'

404

') TOO_LARGE = Response('413 Payload Too Large', b'

Payload Too Large

') @@ -78,9 +84,11 @@ def parse(self, update): """What the update (a dict) says: a Message, a Press, or a Response to give straight back.""" raise NotImplementedError - def acknowledge(self, press): - """Tell the network a press was taken, where it waits for a call to say so. Here nothing: taking the update is - the acknowledgement (Slack's is the 200, or the socket's ack).""" + def acknowledge(self, event): + """Tell the network an update was taken (a Message or a Press), where it waits for that: None when the answer to + the update says nothing more, or the Response the network must have before any reply. Here None: taking the + update is the acknowledgement (Slack's is the 200, or the socket's ack).""" + return None class Telegram(Network): @@ -128,10 +136,12 @@ def parse(self, update): return UNHANDLED return Message(self.service, chat_id, user, text, self.bot_name, user_name, str(message.get('message_id', ''))) - def acknowledge(self, press): - """answerCallbackQuery: the client's spinner stops, and a toast made from the button's data says what is coming - (buttons.toast: 'Loading 1Y…'). A failure is only logged.""" - telegram.answerCallbackQuery(press.press_id, buttons.toast(press.data)) + def acknowledge(self, event): + """A press: answerCallbackQuery, so that the client's spinner stops and a toast made from the button's data says + what is coming (buttons.toast: 'Loading 1Y…'). A failure is only logged. Nothing goes back with the update.""" + if isinstance(event, Press): + telegram.answerCallbackQuery(event.press_id, buttons.toast(event.data)) + return None class Slack(Network): @@ -184,9 +194,77 @@ def press(self, update): container['message_ts'], value, who) +class Discord(Network): + """Discord's interactions (a slash command, a button pressed, the endpoint's check), which read the same from the + interactions endpoint and the gateway, and the gateway's messages.""" + service = 'discord' + bot_id = None # the bot's user id, from discord.register() as bot.py starts: a message mentioning it is for the bot + PING, COMMAND, COMPONENT = 1, 2, 3 # interaction types + PONG, THINKING, LATER = 1, 5, 6 # answers: to the check, "thinking…" until the reply, nothing to see (a press) + + def update_id(self, update): + """The interaction's id, or the message's.""" + return update.get('id') + + def parse(self, update): + """An interaction (it has a token) or a gateway message.""" + return self.interaction(update) if 'token' in update else self.message(update) + + @property + def bot_name(self): + """The bot as a message names it, @ and its user id (Discord's <@id> mention, read as Slack's are).""" + return f'@{self.bot_id}' if self.bot_id else None + + def interaction(self, update): + """The endpoint's check (a PING, answered at once), a slash command as the message it stands for (/price 7d), + or a button pressed under one of the bot's messages; anything else, UNSUPPORTED.""" + kind = update.get('type') + if kind == self.PING: + return Response(body=json.dumps({'type': self.PONG}).encode()) + person = (update.get('member') or {}).get('user') or update.get('user') or {} + channel = str(update.get('channel_id') or (update.get('channel') or {}).get('id') or '') + through = f"{update.get('application_id')}/{update['token']}" + data = update.get('data') or {} + if kind == self.COMMAND and data.get('name'): + given = ' '.join(str(option.get('value', '')) for option in data.get('options') or []).strip() + text = f"/{data['name']} {given}".strip() + return Message(self.service, channel, f"<@{person.get('id')}>", text, self.bot_name, self.name(person), + interaction=through) + message = update.get('message') or {} + if kind == self.COMPONENT and data.get('custom_id') and message.get('id'): + return Press(self.service, str(update.get('id')), channel, str(message['id']), data['custom_id'], + '@' + str(person.get('username') or person.get('id') or 'unknown'), through) + return UNSUPPORTED + + def message(self, update): + """A message a person wrote (MESSAGE_CREATE); a mention <@id> reads as @id, as Slack's do. One by a bot (this + one's replies included) or a webhook is OK: answering a bot could loop.""" + author = update.get('author') or {} + if author.get('bot') or update.get('webhook_id') or not author.get('id') or 'content' not in update: + return OK + text = re.sub(r'<@!?(\d+)>', r'@\1', update['content']) + return Message(self.service, str(update.get('channel_id')), f"<@{author['id']}>", text, self.bot_name, + self.name(author), str(update.get('id', ''))) + + @staticmethod + def name(person): + """What to call someone: their display name, else their username.""" + return person.get('global_name') or person.get('username') or 'unknown' + + def acknowledge(self, event): + """An interaction's answer, which Discord must have within 3 seconds and before any reply: "thinking…" for a + slash command, until the reply replaces it; nothing to see for a press, whose message is edited after. A + gateway message needs none.""" + if not event.interaction: + return None + answer = self.THINKING if isinstance(event, Message) else self.LATER + return Response(body=json.dumps({'type': answer}).encode()) + + class Webhook: """Updates a network posts to the bot, for its reader (a Network); subclasses give the path it posts to and how it proves itself.""" + REFUSED = FORBIDDEN # the answer to a request that does not prove itself def __init__(self, network): """network: the reader of its updates.""" self.network = network @@ -267,8 +345,34 @@ def decode(self, body): return super().decode(body) -TELEGRAM, SLACK = Telegram(), Slack() # the readers -WEBHOOKS = (TelegramWebhook(TELEGRAM), SlackWebhook(SLACK)) +class DiscordInteractions(Webhook): + """Discord posting interactions to discordOutgoingWebhook (the app's Interactions Endpoint URL), each signed with + the app's Ed25519 key, whose public half is discordPublicKey.""" + REFUSED = UNAUTHORIZED # Discord checks that a bad signature gets a 401, and drops a URL that answers otherwise + + def path(self): + """The path of discordOutgoingWebhook.""" + return urlparse(config.env.discordOutgoingWebhook).path if config.env.discordOutgoingWebhook else '' + + def refuse(self, environ, body, update): + """The request must carry Discord's signature of its timestamp and body under discordPublicKey, checked with + PyNaCl (imported here only: Discord's http mode is the one thing that needs it).""" + signature = environ.get('HTTP_X_SIGNATURE_ED25519', '') + timestamp = environ.get('HTTP_X_SIGNATURE_TIMESTAMP', '') + if not signature or not timestamp: + return "Discord signature not present" + from nacl.exceptions import BadSignatureError + from nacl.signing import VerifyKey + try: + VerifyKey(bytes.fromhex(config.env.discordPublicKey)).verify(timestamp.encode() + body, + bytes.fromhex(signature)) + except (BadSignatureError, ValueError): + return "Discord signature incorrect" + return None + + +TELEGRAM, SLACK, DISCORD = Telegram(), Slack(), Discord() # the readers +WEBHOOKS = (TelegramWebhook(TELEGRAM), SlackWebhook(SLACK), DiscordInteractions(DISCORD)) def route(path): diff --git a/lib/net.py b/lib/net.py index 2ae4141a..332860ca 100644 --- a/lib/net.py +++ b/lib/net.py @@ -23,6 +23,10 @@ def post(url, **kwargs): """requests.post through the shared session, which reuses connections; timed out as timeouts() says.""" return session.post(url, **timeouts(kwargs)) +def request(method, url, **kwargs): + """A request of any method through the shared session, which reuses connections; timed out as timeouts() says.""" + return session.request(method, url, **timeouts(kwargs)) + def timeouts(kwargs): """The request's arguments with a timeout, unless it has its own: CONNECT_TIMEOUT to connect, then http_timeout for the answer.""" @@ -30,9 +34,12 @@ def timeouts(kwargs): def redact(text): """text (or an exception) without the secrets that request URLs, headers and error messages carry: Telegram and - Slack tokens, Slack and Discord webhook paths, bearer tokens, and token, crumb, key, secret and auth parameters.""" + Slack tokens, Slack and Discord webhook paths, Discord's bot and interaction tokens, bearer tokens, and token, + crumb, key, secret and auth parameters.""" text = re.sub(r'bot\d+:[\w-]+', 'bot', str(text)) text = re.sub(r'\bx(?:ox[a-z]|app)-[\w-]+', '', text) # Slack's tokens text = re.sub(r'(/services/|/api/webhooks/)[^\s\'"?&]+', r'\1', text) + text = re.sub(r'(/(?:webhooks|interactions)/\d+/)[^\s\'"?&/]+', r'\1', text) # Discord's API + text = re.sub(r'\b[\w-]{23,28}\.[\w-]{6,7}\.[\w-]{27,}', '', text) # a Discord bot token text = re.sub(r'(?i)(bearer\s+)[\w.~+/=-]+', r'\1', text) return re.sub(r'(?i)(token|crumb|auth\w*|key|secret)=[^&\s\'"]+', r'\1=', text) diff --git a/lib/pull.py b/lib/pull.py index 50d2d081..9b5737b2 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -1,13 +1,16 @@ """The pull modes: a network that finbot.env gives no webhook URL is asked for its updates instead, by a listener in a thread of its own, which hands each update to bot.take(network, update) as the webhook would, so that reading it, the repeat check and acknowledging a press are the same both ways. Each listener keeps going through failures, waiting -longer after each one in a row (backoff). Telegram's long polling and Slack's Socket Mode.""" +longer after each one in a row (backoff). Telegram's long polling, Slack's Socket Mode and Discord's gateway.""" +import asyncio import json import sys +import threading import time from lib import config from lib import delivery +from lib import discord from lib import inbound from lib import net from lib import telegram @@ -50,13 +53,21 @@ def failed(self, what, e): return wait def hand_on(self, take, update): - """take(network, update), a failure reported as bot.main reports one of an authenticated request: the update - came from the network, so the default chats are told.""" + """take(network, update), and the answer delivered where the network must have it before any reply. A failure + is reported as bot.main reports one of an authenticated request: the update came from the network, so the + default chats are told.""" try: - take(self.network, update) + answer = take(self.network, update) + if getattr(answer, 'then', None): + self.answer(update, answer) except Exception as e: delivery.report_error(e, context=self.network.service) + def answer(self, update, answer): + """Deliver an answer that the network must have before any reply, then start the work waiting on it. Only + Discord's interactions have one (DiscordGateway).""" + answer.then() + class TelegramPoll(Listener): """Telegram's long polling: getUpdates, which Telegram holds open until there are updates or POLL_SECONDS have @@ -193,6 +204,69 @@ def listen(self, connection, take): self.hand_on(take, envelope.get('payload') or {}) +class DiscordGateway(Listener): + """Discord's gateway, through discord.py: imported here only, as Discord's pull mode is the one thing that needs it + (Debian's python3-discord). Its client keeps the connection, reconnecting by itself; finbot takes the raw + MESSAGE_CREATE and INTERACTION_CREATE events, which read as the interactions endpoint's, each in a thread of its + own, off the client's event loop. The gateway takes no answers: an interaction's goes to its callback over REST.""" + network = inbound.DISCORD + TAKEN = ('MESSAGE_CREATE', 'INTERACTION_CREATE') + + def describe(self): + """The mode, for the log.""" + return 'gateway (no discordOutgoingWebhook)' + + def poll(self, take): + """Run discord.py's client until it stops, which it does only on a failure it does not get over (a token Discord + refuses, or an intent the app is not allowed): then the back-off.""" + try: + asyncio.run(self.run_client(take)) + except Exception as e: + return self.failed('the gateway', e) + return self.failed('the gateway', 'the client stopped') + + async def run_client(self, take): + """Start the client with the bot token, and close it however it ends.""" + client = self.client(take) + async with client: + await client.start(config.env.discordBotToken) + + def client(self, take): + """discord.py's client, with the intents for messages in servers and DMs, and their text (Message Content, which + the app must be allowed in the Developer Portal); its raw events go to receive().""" + import discord as library + library.VoiceClient.warn_nacl = False # finbot has no voice: no warning that PyNaCl is missing + intents = library.Intents.default() + intents.message_content = True + client = library.Client(intents=intents, enable_debug_events=True) + + @client.event + async def on_socket_raw_receive(raw): + """A gateway message, as it arrived.""" + self.receive(raw, take) + + @client.event + async def on_ready(): + """Connected: the failures in a row are over.""" + self.failures = 0 + return client + + def receive(self, raw, take): + """A raw gateway message: an event finbot takes goes to hand_on in a thread of its own, so that the client's + event loop never waits on finbot; the rest is the client's business.""" + try: + message = json.loads(raw) + except (TypeError, ValueError): + return + if message.get('op') == 0 and message.get('t') in self.TAKEN and isinstance(message.get('d'), dict): + threading.Thread(target=self.hand_on, args=(take, message['d']), daemon=True).start() + + def answer(self, update, answer): + """Post an interaction's answer to its callback, then start the work waiting on it.""" + discord.call('POST', f"/interactions/{update['id']}/{update['token']}/callback", json=json.loads(answer.body)) + answer.then() + + def listeners(): """The listeners finbot.env asks for: each network with its bot token and no webhook URL (Slack's also needs its app-level token).""" @@ -202,4 +276,6 @@ def listeners(): found.append(TelegramPoll()) if env.slackBotToken and env.slackAppToken and not env.slackOutgoingWebhook: found.append(SlackSocket()) + if env.discordBotToken and not env.discordOutgoingWebhook: + found.append(DiscordGateway()) return found diff --git a/lib/reports/about.py b/lib/reports/about.py index b794220f..7cbc520e 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -1,5 +1,6 @@ """What finbot can do: the .help text, and the .jobs listing of the schedule (with running a job from it).""" from lib import buttons +from lib import config from lib import formats from lib import jobs @@ -46,8 +47,12 @@ def job_line(scheduler, label, outcome): def prepare_help(service, botName, sections): """The .help text: each section's command lines under its heading (worker.help_sections() has them), then the bot's - @name as another way to start a command. botName is the bot's @name.""" - example = '<' + botName + '> trades' if service == 'slack' else botName + ' price' + @name as another way to start a command. botName is the bot's @name. On Discord the lines are slash commands, and + the other ways, which are messages, work only over the gateway.""" + if service == 'discord': + example = None if config.env.discordOutgoingWebhook else f'.price, or <{botName}> price' + else: + example = '<' + botName + '> trades' if service == 'slack' else botName + ' price' return formats.sections(*([service.bold(heading + ":")] + lines for heading, lines in sections), - [service.bold("Alternative syntax:"), example], + [service.bold("Alternative syntax:"), example] if example else [], [f"Full help on {service.link('https://github.com/robdevops/finbot', 'GitHub')}"]) diff --git a/lib/transports.py b/lib/transports.py index f7cdb972..b682da43 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -11,6 +11,7 @@ from lib import config from lib import dialects +from lib import discord from lib import net from lib import state from lib import telegram @@ -35,10 +36,14 @@ class Transport: _notices = None # the thread sending the notices _quiet = None # set to stop them - def __init__(self, where, message=None): - """where: the chat id, or the incoming webhook's URL; message: the id of the message being answered, if any.""" + def __init__(self, where, message=None, interaction='', pressed=False): + """where: the chat id, or the incoming webhook's URL; message: the id of the message being answered, if any; + interaction: what the network answers through, where it has one (Discord's application id and token), for a + slash command, or for a button press (pressed).""" self.where = where self.message = message + self.interaction = interaction + self.pressed = pressed def __repr__(self): """: the kind and where, for logs and test failures.""" @@ -478,6 +483,117 @@ def post_chart(self, image, caption, buttons=None): return None +class DiscordBot(Transport): + """A Discord channel or DM, as the bot (discordBotToken): no link previews, no pings, charts attached, buttons as + components, edits in place, the typing notice. A slash command's first message replaces Discord's "thinking…" and + the rest follow it up, through its interaction; a pressed message is edited through the press's.""" + name = 'discord' + dialect = dialects.DISCORD + takes_charts = takes_buttons = True + + def __init__(self, where, message=None, interaction='', pressed=False): + """As Transport; a press has no "thinking…" to replace, so its new messages go into the channel.""" + super().__init__(where, message, interaction, pressed) + self.answered = pressed # the slash command's "thinking…" replaced + + @property + def shows_typing(self): + """The typing notice for a command in a message; a slash command shows Discord's "thinking…" instead.""" + return not self.interaction + + def typing(self): + """POST /typing: "finbot is typing…" for ten seconds, or until it posts.""" + discord.try_call('POST', f'/channels/{self.where}/typing') + + def typing_done(self): + """A slash command over with nothing said: its "thinking…" deleted, or it would show for 15 minutes.""" + if self.interaction and not self.answered: + self.answered = True + discord.try_call('DELETE', f'/webhooks/{self.interaction}/messages/@original') + + @staticmethod + def extras(buttons): + """What a message carries besides its text: no link previews, no pings, and its buttons as components.""" + extra = {'flags': discord.SUPPRESS_EMBEDS, 'allowed_mentions': discord.NO_MENTIONS} + if buttons and not isinstance(buttons, dict): + extra['components'] = components(buttons) + return extra + + def post_text(self, text, buttons=None): + """Post one message (see post).""" + return self.post({'content': text, **self.extras(buttons)}) + + def post_chart(self, image, caption, buttons=None): + """Post a chart, attached to the report as its message (see post).""" + attached = {'content': caption, 'attachments': [{'id': 0, 'filename': 'chart.png'}], **self.extras(buttons)} + return self.post(attached, image) + + def post(self, payload, image=None): + """Post one message: a slash command's first replaces its "thinking…" and the rest follow it up; otherwise it + goes into the channel. Discord's message; MAYBE_SENT when a chart's post timed out; None when it failed + (logged).""" + if self.interaction and not self.answered: + method, path = 'PATCH', f'/webhooks/{self.interaction}/messages/@original' + elif self.interaction and not self.pressed: + method, path = 'POST', f'/webhooks/{self.interaction}' + else: + method, path = 'POST', f'/channels/{self.where}/messages' + try: + answer = discord.call(method, path, **message_body(payload, image)) + except requests.exceptions.ReadTimeout as e: + self.answered = True # it may have arrived + print("Discord post timed out, it may still arrive:", net.redact(e), file=sys.stderr) + return MAYBE_SENT if image is not None else None + except Exception as e: + print("Discord post to", self.where, "failed:", net.redact(e), file=sys.stderr) + return None + self.answered = True + print("OK discord message to", self.where, file=sys.stderr) + return answer or {} + + def message_path(self, message): + """Where a message is edited: through the press's interaction where there is one (a message that answered a + slash command can be edited only so), else in the channel as the bot.""" + if self.interaction: + return f'/webhooks/{self.interaction}/messages/@original' + return f'/channels/{self.where}/messages/{message}' + + def edit_chart(self, message, image, caption, buttons=None): + """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Discord refuses.""" + if hasattr(image, 'seek'): + image.seek(0) + payload = {'content': fit_caption(caption, self.dialect), 'attachments': [{'id': 0, 'filename': 'chart.png'}], + 'components': components(buttons or [])} + discord.call('PATCH', self.message_path(message), **message_body(payload, image)) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + payload = {'content': text} + if buttons is not None: + payload['components'] = components(buttons) + discord.try_call('PATCH', self.message_path(message), json=payload) + + +DISCORD_ROW = 5 # buttons in a row of components, and rows in a message: Discord's limits + + +def components(buttons): + """Rows of buttons.Button as Discord's action rows: DISCORD_ROW buttons to a row and as many rows at most (the + buttons past that are left out), the one shown now blurple and the rest grey.""" + flat = [button for row in buttons for button in row][:DISCORD_ROW * DISCORD_ROW] + return [{'type': 1, 'components': [{'type': 2, 'style': 1 if button.active else 2, 'label': button.label[:80], + 'custom_id': button.data} for button in flat[i:i + DISCORD_ROW]]} for i in range(0, len(flat), DISCORD_ROW)] + + +def message_body(payload, image=None): + """A Discord message's request arguments: its JSON; with a chart, a form of the JSON and the PNG (bytes, so that a + rate-limited call can send it again).""" + if image is None: + return {'json': payload} + data = image.read() if hasattr(image, 'read') else bytes(image) + return {'data': {'payload_json': json.dumps(payload)}, 'files': {'files[0]': ('chart.png', data, 'image/png')}} + + def slack_api(method, as_app=False, **data): """POST a Slack Web API method (form-encoded) as the bot, or with the app-level token (as_app, for Socket Mode's apps.connections.open), and return the answer; an error raises RuntimeError.""" diff --git a/lib/worker.py b/lib/worker.py index 3c252e17..1dfc3995 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -31,10 +31,11 @@ class Request: user_name: str # what to call them bot_name: str message_id: str = None # the message being answered + interaction: str = '' # what the network answers through, where it has one (Discord's slash command) def __post_init__(self): """The chat's transport, which also keeps the typing notice going.""" - self.dest = delivery.reply_to(self.service, self.chat_id, self.message_id) + self.dest = delivery.reply_to(self.service, self.chat_id, self.message_id, self.interaction) def reply(self, lines, chart=None, buttons=None): """Answer in the chat (see delivery.deliver): the lines, perhaps a chart (drawn, or a spec drawn only where it @@ -45,7 +46,7 @@ def process_job_callback(press, pressed): """A command button of .jobs (an inbound.Press; pressed, what its data says): run that command now and answer in the chat. A button from an older list runs nothing: the list is redrawn in place as it is now.""" service = dialects.of(press.service) - chat = delivery.reply_to(service, press.chat_id) + chat = delivery.reply_to(service, press.chat_id, interaction=press.interaction, pressed=True) try: if jobs.current is None: # the scheduler is off now raise jobs.ScheduleError('the scheduler is off') @@ -69,7 +70,7 @@ def process_callback(press): if not pressed: return service = dialects.of(press.service) - chat = delivery.reply_to(service, press.chat_id) + chat = delivery.reply_to(service, press.chat_id, interaction=press.interaction, pressed=True) try: args = buttons.resolve(pressed.ref) markup = buttons.keyboard(pressed.kind, args, pressed.period) @@ -90,7 +91,7 @@ def process_request(message): none gets no answer. Arguments the command cannot take get its usage, and any failure a one-line reply in the chat.""" request = Request(dialects.of(message.service), message.chat_id, message.user, message.user_name, message.bot_name, - message.message_id) + message.message_id, message.interaction) command, args = read(message.text, message.bot_name) if command: try: @@ -104,6 +105,8 @@ def process_request(message): delivery.report_error(e, request.dest, context=message.text.split()[0][:30]) finally: request.dest.end_typing() + elif message.interaction: # a slash command that reads as none: its notice taken off (Discord's "thinking…") + request.dest.end_typing() if message.service == 'telegram': try: ensure_keyboard(message.chat_id) # keep DM keyboards current after the reply @@ -216,11 +219,13 @@ def on_watchlist(request, args): request.reply(holdings.prepare_watchlist(request.service, request.user)) def on_help(request, args): - """.help: every command; in a Telegram DM the command keyboard too.""" + """.help: every command (on Discord as its slash command, which works in both modes); in a Telegram DM the command + keyboard too.""" markup = None if request.service == 'telegram' and int(request.chat_id) > 0: # positive chat ids are DMs; groups are negative markup = command_keyboard() - request.reply(about.prepare_help(request.service, request.bot_name, help_sections()), buttons=markup) + sections = help_sections(slash=request.service == 'discord') + request.reply(about.prepare_help(request.service, request.bot_name, sections), buttons=markup) if markup: mark_keyboard(request.chat_id) # this chat now has the current keyboard @@ -357,8 +362,14 @@ def on_jobs(request, args): request.reply(payload, buttons=markup) def on_profile(request, args): - """.SYMBOL: the company's profile.""" - request.reply(profile.prepare_profile_payload(request.service, request.user, yahoo.symbol(args.word))) + """.SYMBOL, or .stock (.profile) SYMBOL: the company's profile.""" + if args.word.lower() in PROFILE.names: + if len(args.words) != 1: + raise Usage + symbol = ticker(args) + else: + symbol = yahoo.symbol(args.word) + request.reply(profile.prepare_profile_payload(request.service, request.user, symbol)) @dataclasses.dataclass(frozen=True) class Command: @@ -382,7 +393,8 @@ class Command: CHANGE = {'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), 'delete') # of the watchlist BY = dict.fromkeys(('sector', 'sectors'), 'sector') | dict.fromkeys(('industry', 'industries'), 'industry') # .pie's BUY_SELL = ('buy', 'sell', *(a + sep + b for a, b in (('buy', 'sell'), ('sell', 'buy')) for sep in ('', '&', '/', '+'))) -PROFILE = Command((), on_profile, ('.SYMBOL',), COMPANY) # any other first word that may be a ticker +PROFILE = Command(('stock', 'profile'), on_profile, ('.SYMBOL', '.stock SYMBOL'), COMPANY) # any first word that is no +# command's name but may be a ticker calls it too COMMANDS = [ # .help lists a section's commands in this order. The odd spellings (earrings, prince, hospital...) are # kept on purpose. @@ -419,10 +431,39 @@ class Command: PROFILE, ] -def help_sections(): - """The sections of .help: (heading, the .help lines of its commands, in table order), in SECTIONS order.""" - return [(section, [line for command in COMMANDS if command.section == section for line in command.usage]) - for section in SECTIONS] +def help_sections(slash=False): + """The sections of .help: (heading, the .help lines of its commands, in table order), in SECTIONS order; with slash, + each line as its slash command (/forwardpe for .forward pe, /stock SYMBOL for .SYMBOL).""" + def shown(command, line): + """The line, or its first slash command.""" + forms = slash_forms(command, line) if slash else [] + return f'/{forms[0][0]} {forms[0][1]}'.strip() if forms else line + return [(section, list(dict.fromkeys(shown(command, line) for command in COMMANDS if command.section == section + for line in command.usage))) for section in SECTIONS] + +SLASH_NAME = re.compile(r'[a-z0-9_-]{1,32}') # what a slash command may be named, on every network + +def slash_forms(command, line): + """The slash commands of one of a command's .help lines: [(name, the arguments it shows)]. One, named by the + command's name the line starts with where a slash command may be so named, else by the command's first name that it + may (.forward pe is /forwardpe); a line that starts with no name (.SYMBOL) has one under each of them.""" + valid = [name for name in command.names if SLASH_NAME.fullmatch(name)] + words = line.removeprefix('.').split() + named = next((name for name in command.names if [w.lower() for w in words[:len(name.split())]] == name.split()), + None) + if named is None: + return [(name, ' '.join(words)) for name in valid] + return [(named if named in valid else valid[0], ' '.join(words[len(named.split()):]))] if valid else [] + +def slash_commands(): + """The slash commands to register, each with its description and the arguments it shows: [(name, description, + arguments)], from the .help lines (slash_forms), and /help.""" + found = {'help': ('help', 'every command, and what it takes', '')} + for command in COMMANDS: + for line in command.usage: + for name, arguments in slash_forms(command, line): + found.setdefault(name, (name, f'{name} {arguments}'.strip(), arguments)) + return list(found.values()) def command_keyboard(per_row=3): """Persistent reply keyboard of the commands that have buttons (Telegram DMs), alphabetical, row by row; a press diff --git a/requirements-optional.txt b/requirements-optional.txt index a5e00b03..aeb22238 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -2,3 +2,5 @@ # all; an install needs only those of the modes it uses (Debian: the python3-* package named). waitress>=3.0.2 # any network in http mode (python3-waitress) websocket-client>=1.8.0 # Slack in pull mode, Socket Mode (python3-websocket) +discord.py>=2.5.0 # Discord in pull mode, the gateway (python3-discord) +PyNaCl>=1.5.0 # Discord in http mode, the interactions' signatures (python3-nacl) diff --git a/tests/test_config.py b/tests/test_config.py index 9f8cbfc4..670489ba 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,7 +34,7 @@ def test_the_two_key_sets_are_disjoint_and_cover_everything_config_reads(self): def test_secrets_are_startup_keys(self): """Every secret is a startup key: preferences.conf holds none.""" self.assertTrue(set(config.SECRET_KEYS) <= set(config.STARTUP_KEYS)) - self.assertEqual(len(config.SECRET_KEYS), 13) + self.assertEqual(len(config.SECRET_KEYS), 15) def test_unset_preferences_take_their_defaults(self): """An unset preference takes its field's default, as it is: reading tidies nothing away.""" @@ -451,6 +451,26 @@ def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(sel mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**hooks, **changes})): self.assertEqual(config.startup_warnings(), warnings) + def test_discord_settings_that_leave_part_of_it_unable_to_work_are_warned_about(self): + """A startup warning for Discord's endpoint or channel without the bot token, the endpoint without the app's + public key (every interaction refused), and the key without the endpoint (the gateway is used).""" + key = 'ab' * 32 + http = dict(discordBotToken='x', discordOutgoingWebhook='https://example.com/discord', discordPublicKey=key) + no_reply = 'discordOutgoingWebhook is set but discordBotToken is not: Discord commands will get no reply' + no_bot = 'discordChannelID is set but discordBotToken is not: scheduled reports cannot go there as the bot' + refused = ("discordOutgoingWebhook is set but discordPublicKey is not the app's public key (64 hex digits):" + ' every Discord interaction will be refused') + unused = ('discordPublicKey is set but discordOutgoingWebhook is not: finbot keeps to the Discord gateway (pull' + ' mode), and the key is not used') + cases = (({}, []), ({'discordBotToken': ''}, [no_reply]), ({'discordChannelID': '77'}, []), + ({'discordChannelID': '77', 'discordBotToken': '', 'discordOutgoingWebhook': ''}, [no_bot, unused]), + ({'discordPublicKey': 'not-hex'}, [refused]), ({'discordPublicKey': ''}, [refused]), + ({'discordOutgoingWebhook': ''}, [unused]), ({'discordOutgoingWebhook': '', 'discordPublicKey': ''}, [])) + for changes, warnings in cases: + with self.subTest(changes=changes), \ + mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**http, **changes})): + self.assertEqual(config.startup_warnings(), warnings) + def test_startup_report_is_a_short_count_never_names_or_values(self): """The startup report counts the startup keys set, and never names them or shows their values.""" with mock.patch.dict(os.environ, {'slack_webhook': 'https://secret.example/abc', 'port': '5000'}, clear=True): diff --git a/tests/test_discord.py b/tests/test_discord.py new file mode 100644 index 00000000..269840d3 --- /dev/null +++ b/tests/test_discord.py @@ -0,0 +1,425 @@ +"""The Discord bot: interactions and messages read as the other networks' updates, the interactions endpoint and its +signatures, REST calls and their rate limits, messages with charts and buttons, slash commands from the command table, +and the gateway.""" +import contextlib, io, json, sys, types, unittest +from tests.common import * +from lib import discord, pull + +try: + from nacl.signing import SigningKey +except ImportError: # PyNaCl is optional: only Discord's http mode needs it + SigningKey = None + + +def interaction(kind, **fields): + """An interaction of this type in channel 77, from Rob in a server (fields replace the defaults).""" + return {'id': '900', 'application_id': '42', 'token': 'tok', 'type': kind, 'channel_id': '77', + 'member': {'user': {'id': '5', 'username': 'rob', 'global_name': 'Rob'}}, **fields} + + +COMMAND = interaction(2, data={'name': 'price', 'options': [{'name': 'arguments', 'type': 3, 'value': '5 7d'}]}) +PRESS = interaction(3, id='901', data={'custom_id': 'c|h|y|T1', 'component_type': 2}, message={'id': '333'}) +MESSAGE = {'id': '700', 'channel_id': '77', 'author': {'id': '5', 'username': 'rob'}, 'content': '<@99> price 7d'} + + +class Now: + """threading.Thread that runs its target at start(), so that a test needs no waiting.""" + def __init__(self, target, args=(), daemon=None): + """The target and its arguments.""" + self.target, self.args = target, args + + def start(self): + """Run the target now.""" + self.target(*self.args) + + +class Reading(FinbotCase): + """An interaction reads the same however it arrives; a gateway message reads as Slack's do.""" + + def setUp(self): + """The bot's user id is 99.""" + super().setUp() + self.patch(inbound.DISCORD, 'bot_id', '99') + + def test_a_slash_command_is_the_message_it_stands_for(self): + """/price with its arguments is the message '/price 5 7d', answered through its interaction; in a DM the person + is the interaction's user.""" + expected = inbound.Message('discord', '77', '<@5>', '/price 5 7d', '@99', 'Rob', '', '42/tok') + self.assertEqual(inbound.DISCORD.parse(COMMAND), expected) + dm = {k: v for k, v in COMMAND.items() if k != 'member'} | {'user': {'id': '5', 'username': 'rob'}} + self.assertEqual(inbound.DISCORD.parse(dm), dataclasses.replace(expected, user_name='rob')) + self.assertEqual(inbound.DISCORD.parse({**COMMAND, 'data': {'name': 'beta'}}).text, '/beta') + + def test_a_press_is_a_press_of_its_message(self): + """A button pressed is a Press of its message, its custom_id the button's data.""" + self.assertEqual(inbound.DISCORD.parse(PRESS), + inbound.Press('discord', '901', '77', '333', 'c|h|y|T1', '@rob', '42/tok')) + + def test_a_message_by_a_person_reads_with_its_mentions_as_slacks(self): + """A gateway message's mentions read as @id, so that '@bot price 7d' calls .price; one by a bot or a webhook + (this bot's replies included) is not answered.""" + message = inbound.DISCORD.parse(MESSAGE) + self.assertEqual(message, inbound.Message('discord', '77', '<@5>', '@99 price 7d', '@99', 'rob', '700')) + command, args = worker.read(message.text, message.bot_name) + self.assertEqual((command.names[0], args.days), ('price', 7)) + for other in ({**MESSAGE, 'author': {'id': '99', 'bot': True}}, {**MESSAGE, 'webhook_id': '1'}): + self.assertIs(inbound.DISCORD.parse(other), inbound.OK) + + def test_the_endpoints_check_and_other_interactions(self): + """A PING gets its PONG at once; an interaction of another kind is not taken.""" + self.assertEqual(json.loads(inbound.DISCORD.parse(interaction(1)).body), {'type': 1}) + self.assertIs(inbound.DISCORD.parse(interaction(5)), inbound.UNSUPPORTED) + + def test_an_interaction_is_answered_before_its_work_starts(self): + """A slash command is answered "thinking…" and a press with nothing to see, and the work waits in the answer + until whoever delivers it starts it; a gateway message needs no answer, and its work starts at once.""" + taken = [] + self.patch(worker, 'process_request', taken.append) + self.patch(worker, 'process_callback', taken.append) + answer = bot.take(inbound.DISCORD, COMMAND) + self.assertEqual(json.loads(answer.body), {'type': 5}) + self.join_workers() + self.assertEqual(taken, []) + answer.then() + self.assertEqual(json.loads(bot.take(inbound.DISCORD, PRESS).body), {'type': 6}) + self.assertIs(bot.take(inbound.DISCORD, MESSAGE), inbound.OK) + self.join_workers() + self.assertEqual([event.text if isinstance(event, inbound.Message) else event.data for event in taken], + ['/price 5 7d', '@99 price 7d']) + + +@unittest.skipUnless(SigningKey, 'PyNaCl is not installed') +class Endpoint(FinbotCase): + """Discord's interactions endpoint: each request signed with the app's key; a bad signature gets the 401 Discord + looks for.""" + + def setUp(self): + """Discord in http mode, with a key of the test's own.""" + super().setUp() + self.key = SigningKey.generate() + self.env(discordBotToken='x', discordOutgoingWebhook='https://example.com/discord', + discordPublicKey=self.key.verify_key.encode().hex()) + self.taken = [] + self.patch(worker, 'process_request', self.taken.append) + self.patch(worker, 'process_callback', self.taken.append) + + def post(self, update, signature=None, timestamp='1700000000'): + """Post an interaction to the endpoint, signed unless a signature is given: (status, answer, body object).""" + body = json.dumps(update).encode() + if signature is None: + signature = self.key.sign(timestamp.encode() + body).signature.hex() + status = [] + with contextlib.redirect_stderr(io.StringIO()): + out = bot.main(wsgi_environ('/discord', body, HTTP_X_SIGNATURE_ED25519=signature, + HTTP_X_SIGNATURE_TIMESTAMP=timestamp), lambda s, *a: status.append(s)) + return status[0], b''.join(out), out + + def test_a_signed_check_gets_its_pong(self): + """Discord's check of the URL, signed, gets its PONG.""" + status, answer, _ = self.post(interaction(1)) + self.assertEqual((status, json.loads(answer)), ('200 OK', {'type': 1})) + + def test_a_bad_or_missing_signature_gets_401(self): + """A request signed with another key, or with no signature, is refused with a 401 and taken no further.""" + other = SigningKey.generate().sign(b'1700000000' + json.dumps(COMMAND).encode()).signature.hex() + self.assertEqual(self.post(COMMAND, signature=other)[0], '401 Unauthorized') + self.assertEqual(self.post(COMMAND, signature='')[0], '401 Unauthorized') + self.join_workers() + self.assertEqual(self.taken, []) + + def test_the_work_starts_once_discord_has_its_answer(self): + """A slash command is answered "thinking…" (a press, nothing to see); the work starts only once the server has + sent that answer, which it says by closing the body.""" + status, answer, out = self.post(COMMAND) + self.assertEqual((status, json.loads(answer)), ('200 OK', {'type': 5})) + self.join_workers() + self.assertEqual(self.taken, []) + out.close() + self.join_workers() + self.assertEqual([message.text for message in self.taken], ['/price 5 7d']) + self.assertEqual(json.loads(self.post(PRESS)[1]), {'type': 6}) + + +class Sending(FinbotCase): + """DiscordBot: what each message, chart, edit and typing notice sends, and the rate limits waited out.""" + + def setUp(self): + """The bot token set, and Discord's API answered from self.answers (else 200 and a message).""" + super().setUp() + self.env(discordBotToken='x', discordChannelID='77') + self.calls, self.answers = [], [] + def request(method, url, **kwargs): + """Note the call, and give the next answer.""" + self.calls.append((method, url.removeprefix(discord.API), kwargs)) + return self.answers.pop(0) if self.answers else Resp({'id': '1'}) + self.patch(discord.net, 'request', request) + self.patch(transports.Transport, 'text', REAL_TEXT) + self.patch(transports.Transport, 'chart', REAL_CHART) + + def test_a_message_has_no_previews_or_pings_and_its_buttons_in_rows_of_five(self): + """A message suppresses link previews and pings nobody; its buttons go in rows of five, the one shown now + blurple, and past 25 they are left out.""" + row = [buttons.Button(f'B{i}', f'c|h|w|{i}', active=i == 0) for i in range(12)] + with contextlib.redirect_stderr(io.StringIO()): + transports.DiscordBot('77').text(['hello'], [row[:7], row[7:]]) + method, path, kwargs = self.calls[0] + self.assertEqual((method, path), ('POST', '/channels/77/messages')) + payload = kwargs['json'] + self.assertEqual((payload['content'], payload['flags'], payload['allowed_mentions']), ('hello', 4, + {'parse': []})) + self.assertEqual([len(r['components']) for r in payload['components']], [5, 5, 2]) + self.assertEqual([b['style'] for b in payload['components'][0]['components']], [1, 2, 2, 2, 2]) + self.assertEqual(len(transports.components([[buttons.Button('x', 'y')] * 30])), 5) + + def test_a_chart_is_attached_to_its_report(self): + """A chart goes as an attachment of the message that carries the report and the buttons.""" + with contextlib.redirect_stderr(io.StringIO()): + transports.DiscordBot('77').chart(io.BytesIO(b'png'), 'caption', [[buttons.Button('7D', 'c|h|w|T1')]]) + method, path, kwargs = self.calls[0] + payload = json.loads(kwargs['data']['payload_json']) + self.assertEqual((method, path, payload['content'], payload['attachments']), ('POST', '/channels/77/messages', + 'caption', [{'id': 0, 'filename': 'chart.png'}])) + self.assertEqual(kwargs['files']['files[0]'], ('chart.png', b'png', 'image/png')) + self.assertEqual(payload['components'][0]['components'][0]['custom_id'], 'c|h|w|T1') + + def test_a_slash_commands_answer_replaces_its_thinking_and_the_rest_follow_up(self): + """A slash command's first message replaces Discord's "thinking…", through its interaction, and the rest + follow it up; there is no typing notice besides.""" + chat = transports.DiscordBot('77', interaction='42/tok') + self.assertFalse(chat.shows_typing) + with contextlib.redirect_stderr(io.StringIO()): + chat.text(['x' * 1500, 'y' * 1500]) + self.assertEqual([(m, p) for m, p, _ in self.calls], [('PATCH', '/webhooks/42/tok/messages/@original'), + ('POST', '/webhooks/42/tok')]) + chat.typing_done() + self.assertEqual(len(self.calls), 2) # answered: nothing to delete + + def test_a_slash_command_that_says_nothing_has_its_thinking_deleted(self): + """A slash command over without a word, or one that reads as no command, has its "thinking…" deleted, or it + would show for 15 minutes.""" + transports.DiscordBot('77', interaction='42/tok').typing_done() + worker.process_request(inbound.Message('discord', '77', '<@5>', '/nothing@else', '@99', 'Rob', '', '42/t2')) + self.assertEqual([(m, p) for m, p, _ in self.calls], [('DELETE', '/webhooks/42/tok/messages/@original'), + ('DELETE', '/webhooks/42/t2/messages/@original')]) + + def test_a_press_edits_its_message_through_its_interaction(self): + """A press edits its message through its own interaction (a message that answered a slash command can be + edited only so); a new message about it goes into the channel.""" + chat = transports.DiscordBot('77', interaction='42/tok', pressed=True) + chat.edit_chart('333', io.BytesIO(b'png'), 'caption', [[buttons.Button('1Y', 'c|h|y|T1', True)]]) + chat.edit_text('333', 'list', []) + with contextlib.redirect_stderr(io.StringIO()): + chat.text(['Too few prices']) + self.assertEqual([(m, p) for m, p, _ in self.calls], [('PATCH', '/webhooks/42/tok/messages/@original'), + ('PATCH', '/webhooks/42/tok/messages/@original'), ('POST', '/channels/77/messages')]) + self.assertEqual(json.loads(self.calls[0][2]['data']['payload_json'])['attachments'], + [{'id': 0, 'filename': 'chart.png'}]) + self.assertEqual(self.calls[1][2]['json'], {'content': 'list', 'components': []}) + + def test_a_command_in_a_message_shows_typing(self): + """A command in a message (the gateway's) shows "typing…" in its channel.""" + chat = transports.DiscordBot('77', '700') + self.assertTrue(chat.shows_typing) + chat.typing() + self.assertEqual(self.calls[0][:2], ('POST', '/channels/77/typing')) + + def test_rate_limits_are_waited_out_and_refusals_raise(self): + """A 429 waits as long as Discord asks, then tries again, three times at most; a refusal raises with Discord's + reason; the token in an interaction's path is never logged.""" + slept = [] + self.patch(discord.time, 'sleep', slept.append) + self.answers += [Resp({'retry_after': 0.25}, 429), Resp({'id': '5'})] + self.assertEqual(discord.call('GET', '/users/@me'), {'id': '5'}) + self.assertEqual(slept, [0.25]) + self.answers += [Resp({'retry_after': 99}, 429)] * 3 + with self.assertRaisesRegex(RuntimeError, '429'): + discord.call('GET', '/users/@me') + self.assertEqual(slept, [0.25, 10, 10]) + self.answers.append(Resp({'message': 'Missing Access', 'code': 50001}, 403)) + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertIsNone(discord.try_call('PATCH', '/webhooks/42/s3cret-token/messages/@original')) + self.assertIn('403 Missing Access (50001)', err.getvalue()) + self.assertNotIn('s3cret-token', err.getvalue()) + + def test_scheduled_reports_go_to_the_channel_with_their_buttons(self): + """With discordChannelID and the bot token, scheduled reports go to that channel as the bot, buttons and all, + in place of discord_webhook.""" + self.env(webhooks={'discord': 'https://discord.com/api/webhooks/1/abc'}) + chats = delivery.default_chats() + self.assertEqual([(type(chat).__name__, chat.where) for chat in chats], [('DiscordBot', '77')]) + self.assertTrue(chats[0].takes_buttons and chats[0].takes_charts) + self.env(discordBotToken='') + self.assertEqual([type(chat).__name__ for chat in delivery.default_chats()], ['DiscordWebhook']) + + +class Commands(FinbotCase): + """The slash commands, one per .help line, from the command table, as every network registers them.""" + + def test_each_help_line_is_a_slash_command_that_reads_as_its_command(self): + """Each .help line has a slash command, named as a slash command may be (.forward pe is /forwardpe), and + /help, /stock and /profile besides; each reads as its command, its arguments where it takes any.""" + commands = worker.slash_commands() + names = [name for name, _, _ in commands] + self.assertEqual(len(names), len(set(names))) + for expected in ('help', 'dividend', 'earnings', 'buy', 'sell', 'session', 'premarket', 'forwardpe', 'stock', + 'profile', 'jobs'): + self.assertIn(expected, names) + for name, description, arguments in commands: + with self.subTest(name=name): + self.assertRegex(name, r'^[a-z0-9_-]{1,32}$') + self.assertTrue(0 < len(description) <= 100) + command, args = worker.read(f'/{name} {arguments}'.strip(), BOT) + self.assertIsNotNone(command) + self.assertEqual(command.handler, (worker.COMMANDS[0] if name == 'help' else command).handler) + self.assertEqual(dict((n, a) for n, _, a in commands)['price'], '[percent|SYMBOL|top|bottom] [period]') + self.assertEqual(dict((n, a) for n, _, a in commands)['beta'], '') + + def test_stock_and_profile_are_the_profile_command(self): + """.stock SYMBOL and /profile SYMBOL are .SYMBOL; without a symbol, the usage.""" + self.assertEqual(self.command('.stock T1'), self.command('.T1')) + self.assertEqual(self.command('/profile T1'), self.command('.T1')) + self.assertEqual(self.command('.stock'), [('text', ['Usage: .SYMBOL', 'Usage: .stock SYMBOL'])]) + + def test_registering_replaces_the_commands_only_when_they_differ(self): + """Registering puts the commands only when Discord's differ from them, and gives the bot's user id.""" + wanted = [discord.command('price', 'price [period]', '[period]')] + calls = [] + def call(method, path, **kwargs): + """Answer as Discord would, the commands as registered now.""" + calls.append((method, path)) + return {'/applications/@me': {'id': '42'}, '/users/@me': {'id': '99'}}.get(path, registered) + self.patch(discord, 'call', call) + registered = [{**wanted[0], 'id': '1', 'version': '2', 'application_id': '42'}] + self.assertEqual(discord.register([('price', 'price [period]', '[period]')]), '99') + self.assertNotIn(('PUT', '/applications/42/commands'), calls) + registered = [] + with contextlib.redirect_stderr(io.StringIO()): + discord.register([('price', 'price [period]', '[period]')]) + self.assertIn(('PUT', '/applications/42/commands'), calls) + + def test_help_on_discord_lists_the_slash_commands(self): + """.help on Discord lists each command as its slash command, which works in both modes; the messages that also + work are given only over the gateway.""" + self.patch(inbound.DISCORD, 'bot_id', '99') + sent = [] + self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: sent.append(lines) or []) + message = inbound.Message('discord', '77', '<@5>', '/help', '@99', 'Rob', '', '42/tok') + worker.process_request(message) + self.assertIn('/forwardpe [SYMBOL|top|bottom|negative]', sent[0]) + self.assertIn('/stock SYMBOL', sent[0]) + self.assertIn('.price, or <@99> price', sent[0]) + self.env(discordOutgoingWebhook='https://example.com/discord') + worker.process_request(message) + self.assertNotIn('.price, or <@99> price', sent[1]) + + +class FakeClient: + """discord.py's Client, enough for the gateway listener: its handlers get the raw messages of the test.""" + raw = [] + + def __init__(self, intents, enable_debug_events=False): + """Note how it was made.""" + self.intents, self.debug, self.handlers = intents, enable_debug_events, {} + FakeClient.made = self + + def event(self, handler): + """Register a handler by its name.""" + self.handlers[handler.__name__] = handler + return handler + + async def __aenter__(self): + """Enter.""" + return self + + async def __aexit__(self, *exc): + """Leave: closed.""" + self.closed = True + + async def start(self, token): + """Connect: ready, then the raw messages, then fail as Discord does on a refused token.""" + self.token = token + await self.handlers['on_ready']() + for message in self.raw: + await self.handlers['on_socket_raw_receive'](json.dumps(message)) + raise RuntimeError('Improper token has been passed.') + + +class Gateway(FinbotCase): + """The gateway through discord.py: its raw events taken as the endpoint's, an interaction answered over REST.""" + + def setUp(self): + """Discord in pull mode, with discord.py replaced by the fake client, and threads that run at once.""" + super().setUp() + self.env(discordBotToken='x', discordOutgoingWebhook='') + intents = types.SimpleNamespace(message_content=False) + library = types.SimpleNamespace(Client=FakeClient, VoiceClient=types.SimpleNamespace(warn_nacl=True), + Intents=types.SimpleNamespace(default=lambda: intents)) + self.library = library + self.patch(sys, 'modules', {**sys.modules, 'discord': library}) + self.patch(pull, 'threading', types.SimpleNamespace(Thread=Now)) # the listener's threads only + self.calls = [] + self.patch(discord, 'call', lambda method, path, **kwargs: self.calls.append((method, path, kwargs))) + + def test_messages_and_interactions_are_taken_and_an_interaction_answered_first(self): + """The client asks for the message intents and raw events; MESSAGE_CREATE and INTERACTION_CREATE go to take, the + rest is the client's; an interaction's answer goes to its callback before its work starts; a client that stops + waits the back-off.""" + taken, order = [], [] + def take(network, update): + """Note the update; answer an interaction as the reader would, its work noted when started.""" + taken.append(update.get('content') or update['type']) + if 'token' in update: + return inbound.Response(body=b'{"type": 5}', then=lambda: order.append(('work', len(self.calls)))) + return inbound.OK + FakeClient.raw = [{'op': 11}, {'op': 0, 't': 'READY', 'd': {}}, + {'op': 0, 't': 'MESSAGE_CREATE', 'd': MESSAGE}, {'op': 0, 't': 'INTERACTION_CREATE', 'd': COMMAND}] + listener = pull.DiscordGateway() + listener.failures = 3 + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(listener.poll(take), 5) # ready reset the failures; then the refused token + client = FakeClient.made + self.assertTrue(client.debug and client.intents.message_content and client.closed) + self.assertFalse(self.library.VoiceClient.warn_nacl) + self.assertEqual(taken, ['<@99> price 7d', 2]) + self.assertEqual(self.calls, [('POST', '/interactions/900/tok/callback', {'json': {'type': 5}})]) + self.assertEqual(order, [('work', 1)]) + self.assertIn('discord: the gateway failed, trying again in 5 s: Improper token', err.getvalue()) + + @unittest.skipUnless(SigningKey, 'PyNaCl is not installed') + def test_a_press_over_the_gateway_is_the_press_the_endpoint_gets(self): + """The same press, posted to the endpoint (http mode) or sent over the gateway (pull mode), reaches the worker + as the same Press.""" + presses = [] + self.patch(worker, 'process_callback', presses.append) + key = SigningKey.generate() + self.env(discordOutgoingWebhook='https://example.com/discord', discordPublicKey=key.verify_key.encode().hex()) + body = json.dumps(PRESS).encode() + signature = key.sign(b'1700000000' + body).signature.hex() + with contextlib.redirect_stderr(io.StringIO()): + bot.main(wsgi_environ('/discord', body, HTTP_X_SIGNATURE_ED25519=signature, + HTTP_X_SIGNATURE_TIMESTAMP='1700000000'), lambda *a: None).close() + self.join_workers() + self.patch(bot, '_updates_seen', collections.OrderedDict()) # a restart in the other mode + self.env(discordOutgoingWebhook='') + FakeClient.raw = [{'op': 0, 't': 'INTERACTION_CREATE', 'd': PRESS}] + pull.DiscordGateway().poll(bot.take) + self.join_workers() + self.assertEqual(len(presses), 2) + self.assertEqual(presses[0], presses[1]) + self.assertEqual(self.calls[0][1:], ('/interactions/901/tok/callback', {'json': {'type': 6}})) + + def test_discord_is_listened_to_with_its_token_and_no_endpoint(self): + """The gateway is used with the bot token and no discordOutgoingWebhook; with the URL (http mode), or without + the token, it is not.""" + self.env(telegramBotToken='') + self.assertEqual([type(listener) for listener in pull.listeners()], [pull.DiscordGateway]) + self.env(discordOutgoingWebhook='https://example.com/discord') + self.assertEqual(pull.listeners(), []) + self.env(discordOutgoingWebhook='', discordBotToken='') + self.assertEqual(pull.listeners(), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_edges.py b/tests/test_edges.py index 7db1ea6a..f734f814 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -219,9 +219,9 @@ def test_a_command_is_its_whole_name_in_any_case(self): ('.price @bot', 'price')): with self.subTest(message=message): self.assertEqual(self.args(message)[0], (name,)) - for message in ('.planet', '.shortman', '.pricey'): + for message in ('.planet', '.shortman', '.pricey', '.stock T1', '/profile T1'): with self.subTest(message=message): - self.assertEqual(self.args(message)[0], (), message) # .SYMBOL + self.assertEqual(self.args(message)[0], ('stock',), message) # .SYMBOL self.assertEqual(self.args('.price @bot')[1].words, []) for message in ('price 7d', '/price@otherbot', '...', '. ?'): with self.subTest(message=message): diff --git a/util/record_outputs.py b/util/record_outputs.py index d14dce04..3cb2daa7 100755 --- a/util/record_outputs.py +++ b/util/record_outputs.py @@ -81,12 +81,12 @@ class Record(common.FinbotCase): def runTest(self): """Record what each step posts: the real sending code runs, and net.post notes each request.""" posts = [] - def post(url, **kwargs): - """Note a request, the typing notices aside (Telegram's, Slack's ⏳: timing, not output), and answer - like the APIs do.""" - if 'sendChatAction' in url or '/reactions.' in url: + def post(url, method='POST', **kwargs): + """Note a request, the typing notices aside (Telegram's, Slack's ⏳, Discord's: timing, not output), and + answer like the APIs do.""" + if 'sendChatAction' in url or '/reactions.' in url or url.endswith('/typing'): return common.mock.Mock(status_code=200, json=lambda: {'ok': True, 'result': True}) - entry = {'url': url.replace('botX/', 'bot/')} + entry = {'url': url.replace('botX/', 'bot/'), **({'method': method} if method != 'POST' else {})} for key in ('json', 'data', 'params'): body = kwargs.get(key) if isinstance(body, dict): @@ -99,10 +99,14 @@ def post(url, **kwargs): answer = {**ANSWER, 'result': {'message_id': len(posts), 'chat': {'id': 5}}} return common.mock.Mock(status_code=200, json=lambda: answer, text='OK') self.patch(net, 'post', post) + if hasattr(net, 'request'): # Discord's calls, since the Discord bot + self.patch(net, 'request', lambda method, url, **kwargs: post(url, method, **kwargs)) for target, name, real in real_sending: # what the fixture replaces with recorders, put back self.patch(target, name, real) self.patch(common.time, 'sleep', lambda seconds: None) # no pauses between a long report's messages self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') # Slack's buttons too + if 'discordBotToken' in {field.name for field in common.dataclasses.fields(common.config.env)}: + self.env(discordBotToken='discord-test') steps = [] def step(label, run): """Run one step and note what it posted, or what it raised.""" @@ -114,11 +118,11 @@ def step(label, run): except Exception as e: raised = f'{type(e).__name__}: {e}' steps.append({'step': label, 'posts': list(posts), 'raised': raised}) - for service, chat, user in (('telegram', '55', '@u'), ('slack', 'C1', '<@U1>')): + for service, chat, user in (('telegram', '55', '@u'), ('slack', 'C1', '<@U1>'), ('discord', '77', '<@5>')): for text in COMMANDS: step(f'{service} {text}', handing(worker.process_request, 'Message', service, chat, user, text, common.BOT, 'U')) - for service, chat, message in (('telegram', '55', '9'), ('slack', 'C1', '1.5')): + for service, chat, message in (('telegram', '55', '9'), ('slack', 'C1', '1.5'), ('discord', '77', '333')): for data in BUTTONS: label = f'button {data}' if service == 'telegram' else f'{service} button {data}' step(label, pressing(service, 'cb', chat, message, data)) From dc54f31158c6a8ba41bbaa4e8013c7b24ad64c01 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:01:38 +0000 Subject: [PATCH 06/31] Incoming webhooks out: every network posts as its bot, buttons and all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each network now has a bot that listens (pull mode by default), so the outgoing-only incoming webhooks go, and with them the conditions on what a chat can carry. - slack_webhook and discord_webhook retire. RETIRED is one table for both files, each key with what to do instead: a retired key never stops a start (check_config accepts it in finbot.env too) and gets a note at start and from check_config (slack_webhook: set slackChannelID with slackBotToken). split_env drops them with the same note. - The SlackWebhook and DiscordWebhook transports and _post_json go; the scheduled reports go to each network's home chat (telegramChatID, slackChannelID, discordChannelID) as its bot, each with its own token. - One rule for buttons: every bot transport takes charts and buttons, a plain class attribute, whichever way its updates arrive. A start-up warning says when slackBotToken has neither Socket Mode nor a webhook, so that Slack's commands and buttons would get no answer. - env.webhooks becomes env.telegram_api, the Bot API's base for the token. - A test runs the six networks and modes, in a channel and in a DM: a chart and a .jobs list carry their buttons, in the chat that asked, and each command shows its network's notice (typing, Slack's ⏳, Discord's "thinking…" for a slash command). - util/record_outputs.py records the scheduled reports to the home chats as the bots, for older commits too. - The guides lose the webhooks; the README, the finbot.env template and CLAUDE.md follow. util/record_outputs.py: 247 steps, identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 14 +-- README.md | 22 ++-- defaults/finbot.env | 4 +- doc/discord.md | 20 +--- doc/slack.md | 56 ++------- lib/config.py | 57 ++++----- lib/delivery.py | 24 ++-- lib/telegram.py | 2 +- lib/transports.py | 99 ++-------------- tests/common.py | 3 +- tests/test_config.py | 67 +++++++---- tests/test_discord.py | 9 +- tests/test_edges.py | 14 ++- tests/test_pull.py | 11 -- tests/test_reports.py | 48 ++++---- tests/test_sending.py | 236 +++++++++++++++++++++---------------- tests/test_telegram_api.py | 2 +- tests/test_yahoo.py | 3 +- util/check_config.py | 16 ++- util/record_outputs.py | 22 ++-- util/split_env.py | 2 + 21 files changed, 329 insertions(+), 402 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e6a3816a..ea1dcdfb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,13 +60,13 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm milestone reminder podcasts cleanup sharesight_warm yahoo_warm`; the top-level `price.py` and the rest are shims that run them. Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). -- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or the default chats), - `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per way of posting (`TelegramBot`, - `SlackBot`, `DiscordBot`, `SlackWebhook`, `DiscordWebhook`): splitting, captions, charts, buttons, edits, and the - typing notice (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop - before posting. `DiscordBot`: a slash command's first message replaces its "thinking…" through its `interaction` - (application id/token), and a pressed message is edited through the press's. `lib/telegram.py`: Bot API calls - (`call`, `try_call`). +- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or each network's home chat + as its bot), `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per network's bot + (`TelegramBot`, `SlackBot`, `DiscordBot`), each taking charts and buttons: splitting, captions, charts, buttons, + edits, and the typing notice (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and + `chart()` stop before posting. `DiscordBot`: a slash command's first message replaces its "thinking…" through its + `interaction` (application id/token), and a pressed message is edited through the press's. `lib/telegram.py`: Bot API + calls (`call`, `try_call`). - `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), what a pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. `lib/charts.py`: a chart with period buttons, built by its kind's builder (`build`) for the command's first view diff --git a/README.md b/README.md index 00d2eb9a..77cc5da9 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,18 @@ What finbot can do on each network. Telegram takes everything; the others take w | Feature | Telegram | Slack | Discord | |---|---|---|---| -| Chat commands | ✓ | ✓ | ✓ with the bot: slash commands (`/price`), and messages (`.price`) over the gateway | +| Chat commands | ✓ | ✓ | ✓: slash commands (`/price`), and messages (`.price`) over the gateway | | "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots) | ✓ for a message; "thinking…" for a slash command | -| Charts in replies | ✓ | ✓ with `slackBotToken` | ✓ | -| Charts in scheduled reports | ✓ | ✓ with `slackChannelID` and `slackBotToken` (the incoming webhook takes text only) | ✓ | -| Period and job buttons | ✓ | Period buttons, with the interactive bot | ✓ with the bot | +| Charts in replies | ✓ | ✓ | ✓ | +| Charts in scheduled reports | ✓ | ✓ | ✓ | +| Period and job buttons | ✓ | ✓ | ✓ | | No link previews | ✓ | ✓ | ✓ | | Linked tickers | ✓ | ✓ | ✓ | | Charts updated in place by their buttons | ✓ | ✓ | ✓ | **Setting up a network:** [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md). Each guide -covers the whole setup of that network, and where there is a choice between reports only and the interactive bot, it -compares the two before the steps. +covers the whole setup of that network's bot, and compares its two modes (finbot fetching its updates, or the network +posting them to your HTTPS server) before the steps. ![Screenshot of showing trade notifications on Slack](img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") @@ -473,19 +473,17 @@ of the chat network keys it needs. | `sharesight_code` | | Sharesight API access | | `sharesight_client_id` | | Sharesight API access | | `sharesight_client_secret` | | Sharesight API access | -| `slack_webhook` | | Slack incoming webhook, for outbound reports | -| `discord_webhook` | | Discord webhook, for outbound reports (not used with `discordBotToken` and `discordChannelID`) | | `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | | `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | | `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | -| `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`); `slack_webhook` is then not used | +| `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`) | | `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | | `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | | `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | | `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | | `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | | `discordBotToken` | | Discord bot token: replies, and commands over the gateway unless `discordOutgoingWebhook` is set | -| `discordChannelID` | | Channel id that scheduled reports go to on Discord as the bot, charts and buttons and all (with `discordBotToken`); `discord_webhook` is then not used | +| `discordChannelID` | | Channel id that scheduled reports go to on Discord as the bot, charts and buttons and all (with `discordBotToken`) | | `discordOutgoingWebhook` | | The Interactions Endpoint URL Discord posts slash commands and button presses to (HTTP mode); empty, the gateway (pull mode) | | `discordPublicKey` | | The app's public key, which checks the signature of each interaction (HTTP mode) | @@ -559,8 +557,8 @@ sharesight_client_secret=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e9981 ### Chat networks setup Follow the guide for each network you use: [Telegram](doc/telegram.md), [Slack](doc/slack.md) or -[Discord](doc/discord.md). It covers the bot or webhook, its `finbot.env` keys, and for chat commands the HTTPS server in -front of finbot. +[Discord](doc/discord.md). It covers the bot, its `finbot.env` keys, and for HTTP mode the HTTPS server in front of +finbot. ### Portfolios Portfolios are auto-discovered, including other people's portfolios which are shared to you. To exclude specific portfolios, add their IDs to `exclude_portfolios` in the `preferences.conf`: diff --git a/defaults/finbot.env b/defaults/finbot.env index 7b15ef59..910f156b 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -33,11 +33,9 @@ sharesight_client_id= sharesight_client_secret= # Chat networks. A Telegram bot token is enough for reports and commands: finbot fetches its updates (pull mode) -slack_webhook= -discord_webhook= telegramBotToken= slackBotToken= -# Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken; slack_webhook is then unused) +# Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken) slackChannelID= # Slack's app-level token (xapp-) for commands without a server: finbot opens a socket to Slack (Socket Mode) slackAppToken= diff --git a/doc/discord.md b/doc/discord.md index c1560a3d..e675c337 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -2,8 +2,7 @@ finbot talks to Discord as the bot user of a Discord application that you create in the Developer Portal. The bot posts the scheduled reports into a channel and answers commands: slash commands (`/price`) in both modes, and messages -(`.price`, `@finbot price`) over the gateway. A Discord webhook, which only posts, can still carry the scheduled -reports on its own (see [Webhook](#webhook-reports-only)). +(`.price`, `@finbot price`) over the gateway. ## Gateway mode or HTTP mode? @@ -99,20 +98,3 @@ finbot registers a slash command for each line of `.help`, as it starts, and aga A message holds up to 2,000 characters, so a long report arrives as several messages. A chart goes as an attachment of the message that carries its report, with its period buttons under it. - -## Webhook (reports only) - -Without the bot, a webhook can post the scheduled reports into a channel, with their charts. It takes no commands and -no buttons. With `discordBotToken` and `discordChannelID`, the bot posts the scheduled reports itself, and the webhook -is not used. - -1. In Discord, open _Server Settings > Integrations > Webhooks_ and choose _New Webhook_. -2. Name it (finbot, say), pick the channel for the reports, and _Copy Webhook URL_. -3. Put the URL in `finbot.env` as `discord_webhook`. -4. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the channel. - -``` -discord_webhook=https://discord.com/api/webhooks/1009998000000000000/aaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbb -``` - -A URL ending in `/slack`, from older setups, works as it is: finbot uses Discord's own endpoint either way. diff --git a/doc/slack.md b/doc/slack.md index 1933309e..7599f8d6 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -1,45 +1,15 @@ # Slack -finbot reaches Slack through a Slack app of your own. The app's incoming webhook posts the scheduled reports into a -channel; the app's bot also answers commands. The bot builds on the webhook, which carries the scheduled reports unless -the bot posts them itself, charts and all (`slackChannelID`). +finbot reaches Slack through a Slack app of your own, whose bot posts the scheduled reports into a channel, charts and +all, and answers commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)), with +charts in its replies, ⏳ on a command's message while it works (Slack has no typing notice for bots), and period +buttons under a chart that redraw it in place. -## Incoming webhook, or the interactive bot? - -| | Incoming webhook | Interactive bot (with the webhook) | -|---|---|---| -| Scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest | ✓, as text | ✓, as text through the webhook, or with their charts as the bot (`slackChannelID`) | -| Chat commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)) | ✗ | ✓ | -| Charts in the replies to commands | n/a | ✓, uploaded into the channel | -| A sign that finbot is working on a command | ✗ | ✓: ⏳ on the command's message, Slack having no typing notice for bots | -| Period buttons under a chart, which redraw it for that period in place | ✗ | ✓ | -| Public HTTPS server | Not needed | Not needed in Socket Mode (the default); needed in HTTP mode | -| `finbot.env` keys | `slack_webhook` | That one, plus `slackBotToken` and `slackAppToken` (Socket Mode) or `slackOutgoingWebhook` and `slackSigningSecret` (HTTP mode); and `slackChannelID`, for charts in the scheduled reports | - -Start with [Incoming webhook](#incoming-webhook). For commands, go on to [Interactive bot](#interactive-bot) -afterwards, in the same app. - -## Incoming webhook - -1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick - your workspace. -2. Under _Features > Incoming Webhooks_, turn _Activate Incoming Webhooks_ on, then _Add New Webhook to Workspace_ and - pick the channel for the reports. -3. Copy the webhook URL into `finbot.env` as `slack_webhook`. -4. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the channel. - -``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA -``` - -An incoming webhook takes text only, so the scheduled reports come without charts. The [interactive bot](#interactive-bot) -can post them with their charts. - -## Interactive bot +## The bot ### Socket Mode or HTTP mode? -The bot answers commands either way. What differs is how it hears from Slack: +The bot does the same either way. What differs is how it hears from Slack: | | Socket Mode (the default) | HTTP mode | |---|---|---| @@ -54,7 +24,8 @@ switch, change both and restart finbot. ### Socket Mode -1. Set up the [incoming webhook](#incoming-webhook), above. +1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick + your workspace. 2. Under _Settings > Socket Mode_, turn _Enable Socket Mode_ on. Slack asks for an app-level token: name it (finbot, say) and _Generate_; it has the `connections:write` scope it needs. Copy the token, which starts with `xapp-`, into `finbot.env` as `slackAppToken`. It is also under _Basic Information > App-Level Tokens_. @@ -63,7 +34,6 @@ switch, change both and restart finbot. Mode there is no Request URL to enter. ``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA slackAppToken=xapp-your-app-level-token slackBotToken=xoxb-your-bot-token slackChannelID=C0123456789 @@ -75,7 +45,8 @@ Slack delivers the messages that mention the bot (or every message in a channel, finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name with a valid certificate. -1. Set up the [incoming webhook](#incoming-webhook), above, and leave _Settings > Socket Mode_ off. +1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick + your workspace. Leave _Settings > Socket Mode_ off. 2. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests come from: ``` @@ -109,7 +80,6 @@ on a domain name with a valid certificate. this URL as the _Request URL_ where it asks for one. ``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA slackOutgoingWebhook=https://www.example.com:8443/slack slackSigningSecret=0123456789abcdef0123456789abcdef slackBotToken=xoxb-your-bot-token @@ -140,9 +110,9 @@ slackChannelID=C0123456789 5. Restart finbot, and invite the bot to the channel: `/invite @finbot`. In Socket Mode the log says `slack: Socket Mode (no slackOutgoingWebhook)`. 6. Send `@finbot help` in the channel (or `.help`, with `message.channels`). -7. For charts in the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: - the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot then posts the - scheduled reports there, as it posts its replies, and `slack_webhook` is no longer used. +7. For the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: the + _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot posts the scheduled + reports there, as it posts its replies. A chart is uploaded with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`). With its period buttons, it goes in a message of blocks: the report, the chart (uploaded into no channel, and shown by diff --git a/lib/config.py b/lib/config.py index a4037dc7..b1e00e38 100644 --- a/lib/config.py +++ b/lib/config.py @@ -16,17 +16,23 @@ REPO_DIR = Path(__file__).resolve().parent.parent STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', 'sharesight_code', - 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', - 'slackAppToken', 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', - 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook', 'discordBotToken', 'discordChannelID', - 'discordOutgoingWebhook', 'discordPublicKey') -SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', - 'discord_webhook', 'slackBotToken', 'slackAppToken', 'slackOutgoingToken', 'slackOutgoingWebhook', - 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook', 'discordBotToken', - 'discordOutgoingWebhook') # never printed + 'sharesight_client_id', 'sharesight_client_secret', 'slackBotToken', 'slackAppToken', 'slackChannelID', + 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', + 'telegramOutgoingWebhook', 'discordBotToken', 'discordChannelID', 'discordOutgoingWebhook', 'discordPublicKey') +SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slackBotToken', 'slackAppToken', + 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', + 'telegramOutgoingWebhook', 'discordBotToken', 'discordOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' -RETIRED = ('alliterate', 'chunk_maxlines', 'performance_use_sharesight') # preferences.conf keys that do nothing any -# more: accepted, with a note at start to delete them +RETIRED = { # keys of either file that do nothing any more: accepted, so that they never stop a start, with a note at + # start saying what to do instead + 'alliterate': 'delete the line', + 'chunk_maxlines': 'delete the line', + 'performance_use_sharesight': 'delete the line (price performance comes from Yahoo)', + 'slack_webhook': 'the scheduled reports go to slackChannelID as the bot (slackBotToken); set those, and delete the' + ' line', + 'discord_webhook': 'the scheduled reports go to discordChannelID as the bot (discordBotToken); set those, and' + ' delete the line', +} # set by the bot for its job processes: the preferences arrive in the environment, do not read the file PASSED = 'FINBOT_PREFERENCES_PASSED' OLD_ENV = REPO_DIR / '.env' # the single config file of old versions, which python-dotenv loaded; nothing reads it now @@ -110,8 +116,8 @@ def read_startup_file(path): text = read_text(path) if text is None: raise ConfigError(f'{path} does not exist') - values, errors = parse_assignments(text, path, - lambda key: key_problem(key, STARTUP_KEYS, PREFERENCE_KEYS, PREFERENCES_FILE)) + values, errors = parse_assignments(text, path, lambda key: None if key in RETIRED else + key_problem(key, STARTUP_KEYS, PREFERENCE_KEYS, PREFERENCES_FILE)) if errors: raise ConfigError(errors) return values @@ -195,7 +201,7 @@ class Env: telegramOutgoingToken: str slackBotToken: str slackAppToken: str # xapp-: Socket Mode, Slack's pull mode (without slackOutgoingWebhook) - slackChannelID: str # with slackBotToken, scheduled reports go to this channel as the bot, in place of slack_webhook + slackChannelID: str # with slackBotToken, scheduled reports go to this channel as the bot slackOutgoingWebhook: str slackOutgoingToken: str slackSigningSecret: str @@ -204,7 +210,7 @@ class Env: discordOutgoingWebhook: str # the interactions endpoint URL (http mode); unset, finbot keeps to the gateway discordPublicKey: str # the app's key, which checks the signature of each interaction posted to the endpoint sharesight_auth: dict - webhooks: dict # service -> outbound URL: Telegram's Bot API base, the Slack and Discord incoming webhooks + telegram_api: str # the Bot API's base URL for telegramBotToken ('' without one): a method's name follows it @property def preferences_path(self): @@ -217,14 +223,7 @@ def startup_values(getenv): r = _Reader(getenv) data_dir = getenv('DATA_DIR') or (getenv('STATE_DIRECTORY') or '').split(':')[0] or 'var' data_dir = str(REPO_DIR / data_dir) # a relative one is in the checkout, wherever finbot starts - hooks = {} - if getenv('slack_webhook'): - hooks['slack'] = getenv('slack_webhook').rstrip('/') - if getenv('discord_webhook'): - # the native endpoint, not the Slack-compatible one - hooks['discord'] = getenv('discord_webhook').rstrip('/').removesuffix('/slack') - if getenv('telegramBotToken'): - hooks['telegram'] = 'https://api.telegram.org/bot' + getenv('telegramBotToken').rstrip('/') + '/' + token = (getenv('telegramBotToken') or '').rstrip('/') env = Env( data_dir=data_dir, cache_dir=data_dir + '/cache', @@ -253,7 +252,7 @@ def startup_values(getenv): "client_secret": getenv('sharesight_client_secret'), "redirect_uri": 'urn:ietf:wg:oauth:2.0:oob' }, - webhooks=hooks) + telegram_api=f'https://api.telegram.org/bot{token}/' if token else '') r.check() return env @@ -422,13 +421,12 @@ def startup_report(): else: lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, using the defaults; reload will have nothing to' ' re-read') - return lines + retired_notes(prefs.source) + return lines + retired_notes(os.environ, 'finbot.env') + retired_notes(prefs.source) -def retired_notes(preferences): - """A line for each key of a preferences.conf ({key: value}) that does nothing any more.""" - return [f'{key} in {PREFERENCES_FILE} does nothing any more: delete the line' - for key in RETIRED if key in preferences] +def retired_notes(values, file=PREFERENCES_FILE): + """A line for each key of a file's {key: value} that does nothing any more, saying what to do instead.""" + return [f'{key} in {file} does nothing any more: {instead}' for key, instead in RETIRED.items() if key in values] def startup_warnings(settings=None): @@ -436,6 +434,9 @@ def startup_warnings(settings=None): from working, or do nothing, a line each.""" s = settings or env warnings = [] + if s.slackBotToken and not (s.slackAppToken or s.slackOutgoingWebhook): + warnings.append('slackBotToken is set but neither slackAppToken nor slackOutgoingWebhook is: finbot hears' + ' nothing from Slack, so its commands and buttons get no answer') for key in ('slackOutgoingWebhook', 'slackAppToken'): if getattr(s, key) and not s.slackBotToken: warnings.append(f'{key} is set but slackBotToken is not: Slack commands will get no reply') diff --git a/lib/delivery.py b/lib/delivery.py index 44a802cc..98af07fa 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -32,24 +32,18 @@ def reply_to(service, chat, message=None, interaction='', pressed=False): def default_chats(): - """Where a scheduled report goes, in this order: Telegram's telegramChatID; Slack's slackChannelID as the bot, with - slackBotToken (so that charts go too), else its incoming webhook; Discord's discordChannelID as the bot, with - discordBotToken (buttons too), else its webhook. Raises RuntimeError when there is nowhere to post.""" + """Where a scheduled report goes: each network's home chat (telegramChatID, slackChannelID, discordChannelID), as + its bot. Raises RuntimeError when there is nowhere to post.""" env = config.env chats = [] - if env.telegramChatID and 'telegram' in env.webhooks: - chats.append(transports.TelegramBot(env.telegramChatID)) - if env.slackChannelID and env.slackBotToken: - chats.append(transports.SlackBot(env.slackChannelID)) - elif 'slack' in env.webhooks: - chats.append(transports.SlackWebhook(env.webhooks['slack'])) - if env.discordChannelID and env.discordBotToken: - chats.append(transports.DiscordBot(env.discordChannelID)) - elif 'discord' in env.webhooks: - chats.append(transports.DiscordWebhook(env.webhooks['discord'])) + for chat, token, kind in ((env.telegramChatID, env.telegramBotToken, transports.TelegramBot), + (env.slackChannelID, env.slackBotToken, transports.SlackBot), + (env.discordChannelID, env.discordBotToken, transports.DiscordBot)): + if chat and token: + chats.append(kind(chat)) if not chats: - raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID, discordChannelID,' - ' slack_webhook or discord_webhook') + raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID or discordChannelID,' + ' each with its bot token') return chats diff --git a/lib/telegram.py b/lib/telegram.py index 37103dc7..5392687e 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -5,7 +5,7 @@ def call(method, **kwargs): """POST a Bot API method and return its result. Raises RuntimeError with Telegram's reason when it is not ok.""" - response = net.post(config.env.webhooks['telegram'] + method, **kwargs) + response = net.post(config.env.telegram_api + method, **kwargs) try: output = response.json() except ValueError: diff --git a/lib/transports.py b/lib/transports.py index b682da43..58683106 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -1,7 +1,7 @@ -"""How a message gets to each network: a class per way of posting, made for one destination (a chat, or an incoming -webhook). A transport knows its dialect, what it can carry (charts, buttons, a typing notice) and the network's size -limits; text() and chart() are the same everywhere, and each network posts one message its own way (post_text, -post_chart).""" +"""How a message gets to each network: a class per network's bot, made for one destination (a chat). A transport knows +its dialect, what it can carry (charts, buttons, a typing notice) and the network's size limits; text() and chart() are +the same everywhere, and each network posts one message its own way (post_text, post_chart). Every bot takes buttons: +whichever way its network's updates arrive, their presses reach finbot.""" import json import sys import threading @@ -17,11 +17,9 @@ from lib import telegram MAYBE_SENT = object() # a read timeout on the request that posts a chart: it may have arrived, so no text copy follows -SUPPRESS_EMBEDS = 4 # Discord message flag: no link previews SLACK_CHARTS = 'finbot_slack_charts.json' # Slack's chart messages: the file each shows, and the charts uploaded SLACK_CHARTS_KEPT = 500 # chart messages noted, newest last: an older one shows its file for good SLACK_REUSE_SECONDS = 15 * 60 # a period pressed within this long of its chart's upload shows that upload again -NO_MENTIONS = {'parse': []} # Discord: text that looks like @everyone or a mention pings nobody class Transport: @@ -29,15 +27,15 @@ class Transport: typing notice going while a command works.""" name = '' # the network, in logs dialect = None - takes_charts = False # a chart, with the report as its caption - takes_buttons = False # buttons under a message - shows_typing = False # a typing notice while a command works + takes_charts = True # a chart, with the report as its caption + takes_buttons = True # buttons under a message + shows_typing = True # a typing notice while a command works TYPING = (0.5, 7, 10) # the notice: first after 0.5 s (a quick answer gets none), then every 7 s, 10 times at most _notices = None # the thread sending the notices _quiet = None # set to stop them def __init__(self, where, message=None, interaction='', pressed=False): - """where: the chat id, or the incoming webhook's URL; message: the id of the message being answered, if any; + """where: the chat id; message: the id of the message being answered, if any; interaction: what the network answers through, where it has one (Discord's application id and token), for a slash command, or for a button press (pressed).""" self.where = where @@ -127,29 +125,11 @@ def reshow(self, message, buttons): that: True when it did. Here never: each period is drawn and sent anew.""" return False - def _post_json(self, url, payload, headers=None): - """POST a JSON message: the answer ({} when it is not JSON), or None when the network refused it or failed.""" - try: - r = net.post(url, headers={'Content-type': 'application/json', **(headers or {})}, json=payload) - except Exception as e: - print("Failure sending to", self.name + ":", net.redact(e), file=sys.stderr) - return None - if r.status_code != 200: - print(r.status_code, "error outbound to", self.name, file=sys.stderr) - return None - print(r.status_code, "OK outbound to", self.name, file=sys.stderr) - try: - return r.json() - except ValueError: - return {} - class TelegramBot(Transport): """A Telegram chat via the Bot API: HTML, silent, no link previews; charts, buttons, edits, the typing notice.""" name = 'telegram' dialect = dialects.TELEGRAM - takes_charts = shows_typing = True - takes_buttons = True # their presses reach finbot either way: posted to telegramOutgoingWebhook, or polled for @staticmethod def markup(buttons): @@ -243,21 +223,10 @@ class SlackBot(Transport): WORKING = 'hourglass_flowing_sand' # ⏳: Slack has no typing notice for bots, so a reaction on the message stands in working = None # True while ⏳ is on the message, False when putting it there failed - @property - def takes_charts(self): - """Charts are uploaded with the bot token, so only with one.""" - return bool(config.env.slackBotToken) - @property def shows_typing(self): - """⏳ on the message being answered, with the bot token.""" - return bool(self.message and config.env.slackBotToken) - - @property - def takes_buttons(self): - """Buttons only where their presses reach finbot: with the bot token, and the app's interactivity posting to - slackOutgoingWebhook (http mode) or sending over the socket (Socket Mode, with slackAppToken).""" - return bool(config.env.slackBotToken and (config.env.slackOutgoingWebhook or config.env.slackAppToken)) + """⏳ on the message being answered: a scheduled report answers none.""" + return bool(self.message) def typing(self): """Put ⏳ on the message being answered, once (the bot needs the reactions:write scope); a failure is only @@ -436,60 +405,12 @@ def edit_text(self, message, text, buttons=None): print("Slack chat.update failed:", net.redact(e), file=sys.stderr) -class SlackWebhook(Transport): - """A Slack incoming webhook: text only, into the channel it was made for.""" - name = 'slack' - dialect = dialects.SLACK - - def post_text(self, text, buttons=None): - """Post to the webhook, without link previews.""" - return self._post_json(self.where, {'text': text, 'unfurl_links': False, 'unfurl_media': False}) - - -class DiscordWebhook(Transport): - """A Discord webhook's native endpoint (the Slack one takes no flags): no previews, no pings, charts attached.""" - name = 'discord' - dialect = dialects.DISCORD - takes_charts = True - - def url(self): - """The webhook URL that answers with the message posted (200) instead of an empty 204.""" - return self.where + ('&' if '?' in self.where else '?') + 'wait=true' - - def post_text(self, text, buttons=None): - """Post the message.""" - return self._post_json(self.url(), {'content': text, 'flags': SUPPRESS_EMBEDS, 'allowed_mentions': NO_MENTIONS}) - - def post_chart(self, image, caption, buttons=None): - """The chart as an attachment, the report as its message: Discord's answer, MAYBE_SENT, or None if refused.""" - payload = {'content': caption, 'flags': SUPPRESS_EMBEDS, 'allowed_mentions': NO_MENTIONS, - 'attachments': [{'id': 0, 'filename': 'chart.png'}]} - try: - r = net.post(self.url(), data={'payload_json': json.dumps(payload)}, - files={'files[0]': ('chart.png', image, 'image/png')}) - except requests.exceptions.ReadTimeout as e: - print("Discord chart timed out, it may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT - except Exception as e: - print("Discord chart failed:", net.redact(e), file=sys.stderr) - return None - if r.status_code == 200: - print(r.status_code, "OK discord chart", file=sys.stderr) - try: - return r.json() or {} - except ValueError: - return {} - print(r.status_code, "error discord chart", net.redact(r.text[:300]), file=sys.stderr) - return None - - class DiscordBot(Transport): """A Discord channel or DM, as the bot (discordBotToken): no link previews, no pings, charts attached, buttons as components, edits in place, the typing notice. A slash command's first message replaces Discord's "thinking…" and the rest follow it up, through its interaction; a pressed message is edited through the press's.""" name = 'discord' dialect = dialects.DISCORD - takes_charts = takes_buttons = True def __init__(self, where, message=None, interaction='', pressed=False): """As Transport; a press has no "thinking…" to replace, so its new messages go into the channel.""" diff --git a/tests/common.py b/tests/common.py index 8a524533..098677a5 100644 --- a/tests/common.py +++ b/tests/common.py @@ -119,7 +119,8 @@ def setUp(self): self.patch(__import__('requests').Session, 'request', mock.Mock(return_value=fake)) # no real network traffic tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) - self.env(data_dir=tmp.name, cache_dir=tmp.name, webhooks={'telegram': 'https://api.telegram.org/botX/'}) + self.env(data_dir=tmp.name, cache_dir=tmp.name, telegramBotToken='X', + telegram_api='https://api.telegram.org/botX/') self.patch(bot, '_updates_seen', collections.OrderedDict()) # each test starts with no update taken self.workers = [] self.patch(bot, 'dispatch', lambda event: self.workers.append(REAL_DISPATCH(event))) diff --git a/tests/test_config.py b/tests/test_config.py index 670489ba..7777147c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,7 +34,7 @@ def test_the_two_key_sets_are_disjoint_and_cover_everything_config_reads(self): def test_secrets_are_startup_keys(self): """Every secret is a startup key: preferences.conf holds none.""" self.assertTrue(set(config.SECRET_KEYS) <= set(config.STARTUP_KEYS)) - self.assertEqual(len(config.SECRET_KEYS), 15) + self.assertEqual(len(config.SECRET_KEYS), 13) def test_unset_preferences_take_their_defaults(self): """An unset preference takes its field's default, as it is: reading tidies nothing away.""" @@ -84,7 +84,7 @@ def test_unknown_key_gets_a_hint_and_a_line_number(self): def test_a_startup_key_in_preferences_says_where_it_belongs(self): """A startup key in preferences.conf is sent to finbot.env, without printing its value.""" - errors = self.errors('slack_webhook=https://x\nip=0.0.0.0\ntelegramChatID=5\nscheduler=False\n') + errors = self.errors('slackOutgoingWebhook=https://x\nip=0.0.0.0\ntelegramChatID=5\nscheduler=False\n') self.assertEqual(len(errors), 4) for line in errors: self.assertIn('belongs in finbot.env (a change needs a restart)', line) @@ -132,15 +132,26 @@ def test_a_blank_value_means_the_default(self): self.assertEqual(config.preference_values(config._getenv_for({})).exchange_suffix, 'AX') # unset: the default def test_a_retired_key_is_accepted_and_said_to_do_nothing(self): - """A key finbot no longer reads (alliterate, say) does not stop a start, as an unknown key would: it is ignored, - and the log at start says to delete it.""" + """A key finbot no longer reads (alliterate in preferences.conf, slack_webhook in finbot.env, say) does not stop + a start, as an unknown key would: it is ignored, and the log at start (and check_config) says what to do + instead.""" path = Path(common.temp_dir(self), 'preferences.conf') path.write_text(''.join(f'{key}=True\n' for key in config.RETIRED) + 'future_days=45\n') values = config.read_preferences(path) self.assertEqual(config.preference_values(config._getenv_for(values)).future_days, 45) - notes = [f'{key} in preferences.conf does nothing any more: delete the line' for key in config.RETIRED] + notes = [f'{key} in preferences.conf does nothing any more: {instead}' + for key, instead in config.RETIRED.items()] self.assertEqual(config.retired_notes(values), notes) self.assertEqual(check_config.notes(path), config.retired_notes(values)) + startup = Path(common.temp_dir(self), 'finbot.env') + startup.write_text('slack_webhook=https://hooks.slack.com/services/x\nport=80\n') + self.assertEqual(check_config.check(startup, path), []) + self.assertEqual(check_config.notes(path, startup)[0], 'slack_webhook in finbot.env does nothing any more:' + ' the scheduled reports go to slackChannelID as the bot (slackBotToken); set those, and delete the line') + with mock.patch.dict(os.environ, {'discord_webhook': 'https://discord.com/api/webhooks/1/x'}): + report = config.startup_report() + self.assertIn('discord_webhook in finbot.env does nothing any more: the scheduled reports go to' + ' discordChannelID as the bot (discordBotToken); set those, and delete the line', report) def test_unreadable_file_is_an_error_naming_it(self): """A preferences file that cannot be read is an error that names it.""" @@ -170,11 +181,11 @@ def test_the_old_multiplier_name_is_gone_everywhere(self): def test_startup_file_linting(self): """util/check_config.py reports bad lines in finbot.env first, then bad values, and nothing for a good file.""" path = Path(common.temp_dir(self), 'finbot.env') - path.write_text('ip=1.2.3.4\nfuture_days=3\nslak_webhook=x\nport=http\n') + path.write_text('ip=1.2.3.4\nfuture_days=3\nslackBotTokn=x\nport=http\n') problems = check_config.check(path, Path(common.temp_dir(self), 'none')) self.assertEqual(len(problems), 2) # the two bad lines; the bad value is only found once the lines are valid self.assertIn('future_days belongs in preferences.conf (it is reloadable)', problems[0]) - self.assertIn('unknown key slak_webhook (did you mean slack_webhook?)', problems[1]) + self.assertIn('unknown key slackBotTokn (did you mean slackBotToken?)', problems[1]) path.write_text('port=http\n') nowhere = Path(common.temp_dir(self), 'none') self.assertIn("port='http' is not a whole number", check_config.check(path, nowhere)[0]) @@ -260,7 +271,7 @@ def test_an_error_applies_nothing(self): self.write('future_days=45\n') config.reload() for text in ('future_days=3\nprice_percent=abc\n', 'future_days=3\nfutur_days=4\n', - 'future_days=3\nslack_webhook=x\n'): + 'future_days=3\nslackBotToken=x\n'): self.write(text) with self.assertRaises(config.ConfigError): config.reload() @@ -357,7 +368,7 @@ def test_a_broken_preferences_file_refuses_to_start(self): """A bad value, unknown key, startup key or malformed line in preferences.conf stops the start, saying why.""" for prefs, expect in (('future_days=abc\n', "future_days='abc' is not a whole number"), ('futur_days=1\n', 'unknown key futur_days'), - ('slack_webhook=x\n', 'belongs in finbot.env'), ('what\n', 'expected KEY=value')): + ('slackBotToken=x\n', 'belongs in finbot.env'), ('what\n', 'expected KEY=value')): out = self.run_config(prefs=prefs) self.assertEqual(out.returncode, 1, prefs) self.assertIn('ERROR config:', out.stderr) @@ -412,18 +423,20 @@ def test_a_leftover_env_in_the_checkout_refuses_to_start(self): self.assertIn('/util/split_env.py, which writes finbot.env.new and preferences.conf.new', caught.exception.errors[1]) - def test_a_discord_webhook_uses_the_native_endpoint(self): - """A Discord webhook URL is used as is, its old /slack suffix removed.""" - # the old setup's suffix - for url in ('https://discord.com/api/webhooks/1/abc', 'https://discord.com/api/webhooks/1/abc/slack/'): - values = config.startup_values(lambda name, default=None: url if name == 'discord_webhook' else default) - self.assertEqual(values.webhooks['discord'], 'https://discord.com/api/webhooks/1/abc') + def test_the_telegram_api_is_the_bot_tokens(self): + """Telegram's Bot API base is made from the bot token (a trailing slash dropped), and is empty without one.""" + for token, api in (('123:abc', 'https://api.telegram.org/bot123:abc/'), ('123:abc/', + 'https://api.telegram.org/bot123:abc/'), ('', '')): + values = config.startup_values(lambda name, default=None: token if name == 'telegramBotToken' else default) + self.assertEqual(values.telegram_api, api) def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(self): """A startup warning for Slack commands or slackChannelID without slackBotToken (no reply, no reports there), and for an inbound webhook without its secret (every request would be refused).""" hooks = dict(slackOutgoingWebhook='https://example.com/slack', slackOutgoingToken='stok', slackBotToken='xoxb-1', telegramOutgoingWebhook='https://example.com/telegram', telegramOutgoingToken='tok') + deaf = ('slackBotToken is set but neither slackAppToken nor slackOutgoingWebhook is: finbot hears nothing from' + ' Slack, so its commands and buttons get no answer') no_reply = 'slackOutgoingWebhook is set but slackBotToken is not: Slack commands will get no reply' slack_refused = ('slackOutgoingWebhook is set but neither slackSigningSecret nor slackOutgoingToken is:' ' every Slack request will be refused') @@ -445,7 +458,8 @@ def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(sel ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1'}, []), ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1', 'slackBotToken': ''}, [app_no_reply]), ({'slackAppToken': 'xapp-1'}, [app_unused]), - ({'slackOutgoingWebhook': '', 'slackAppToken': 'xoxb-1'}, [not_app])) + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xoxb-1'}, [not_app]), + ({'slackOutgoingWebhook': ''}, [deaf])) for changes, warnings in cases: with self.subTest(changes=changes), \ mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**hooks, **changes})): @@ -473,13 +487,14 @@ def test_discord_settings_that_leave_part_of_it_unable_to_work_are_warned_about( def test_startup_report_is_a_short_count_never_names_or_values(self): """The startup report counts the startup keys set, and never names them or shows their values.""" - with mock.patch.dict(os.environ, {'slack_webhook': 'https://secret.example/abc', 'port': '5000'}, clear=True): + with mock.patch.dict(os.environ, {'slackOutgoingWebhook': 'https://secret.example/abc', 'port': '5000'}, + clear=True): lines = config.startup_report() keys = len(config.STARTUP_KEYS) self.assertEqual(lines[0], f'startup environment provides 2 of {keys} startup keys (values hidden)') self.assertLessEqual(len('config: ' + lines[0]), 90) # the path in the second line depends on where DATA_DIR is self.assertNotIn('secret.example', ''.join(lines)) - self.assertNotIn('slack_webhook', ''.join(lines)) + self.assertNotIn('slackOutgoingWebhook', ''.join(lines)) class SigHup(unittest.TestCase): @@ -524,19 +539,21 @@ def test_reload_with_the_scheduler_off_still_reloads_preferences(self): class Tools(unittest.TestCase): def test_split_env(self): """split_env sorts an old .env into startup keys and prefs, renames volatile_multiplier, lists what it drops.""" - startup, prefs, notes = split_env.split('slack_webhook=x\nfuture_days=3\nvolatile_multiplier=2\nslackToken=y\n' - 'STATE_DIRECTORY=/a\nbogus=1\nnot a setting\n# c\nport=80\n') - self.assertEqual(startup, ['slack_webhook=x', 'port=80']) + startup, prefs, notes = split_env.split('slackBotToken=x\nfuture_days=3\nvolatile_multiplier=2\nslackToken=y\n' + 'STATE_DIRECTORY=/a\nbogus=1\nnot a setting\n# c\nport=80\nslack_webhook=z\n') + self.assertEqual(startup, ['slackBotToken=x', 'port=80']) self.assertEqual(prefs, ['future_days=3', 'volatility_multiplier=2']) - self.assertEqual(len(notes), 4) + self.assertEqual(len(notes), 5) self.assertTrue(any('slackToken dropped' in n for n in notes) and any('unknown key bogus' in n for n in notes)) + self.assertIn('line 10: slack_webhook dropped: the scheduled reports go to slackChannelID', notes[-1]) def test_split_env_reads_the_old_dotenv_format_too(self): """split_env reads the old dotenv format: quotes, export, comments after values and # inside URLs.""" - old = ("# c\nfuture_days = '30' # note\nexport slack_webhook = \"https://x/#frag\"\nport = 80 # comment\nbare\n" - "discord_webhook=http://y/#z\n\n") + old = ("# c\nfuture_days = '30' # note\nexport slackOutgoingWebhook = \"https://x/#frag\"\nport = 80 # note\n" + "bare\ndiscordOutgoingWebhook=http://y/#z\n\n") startup, prefs, notes = split_env.split(old) - self.assertEqual(startup, ['slack_webhook=https://x/#frag', 'port=80', 'discord_webhook=http://y/#z']) + self.assertEqual(startup, ['slackOutgoingWebhook=https://x/#frag', 'port=80', + 'discordOutgoingWebhook=http://y/#z']) self.assertEqual(prefs, ['future_days=30']) self.assertEqual(notes, ['.env line 5: not KEY=value, left out']) diff --git a/tests/test_discord.py b/tests/test_discord.py index 269840d3..94f0dda4 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -243,14 +243,15 @@ def test_rate_limits_are_waited_out_and_refusals_raise(self): self.assertNotIn('s3cret-token', err.getvalue()) def test_scheduled_reports_go_to_the_channel_with_their_buttons(self): - """With discordChannelID and the bot token, scheduled reports go to that channel as the bot, buttons and all, - in place of discord_webhook.""" - self.env(webhooks={'discord': 'https://discord.com/api/webhooks/1/abc'}) + """With discordChannelID and the bot token, scheduled reports go to that channel as the bot, buttons and all; + without the token, nowhere on Discord.""" + self.env(telegramChatID='') chats = delivery.default_chats() self.assertEqual([(type(chat).__name__, chat.where) for chat in chats], [('DiscordBot', '77')]) self.assertTrue(chats[0].takes_buttons and chats[0].takes_charts) self.env(discordBotToken='') - self.assertEqual([type(chat).__name__ for chat in delivery.default_chats()], ['DiscordWebhook']) + with self.assertRaisesRegex(RuntimeError, 'nowhere to post'): + delivery.default_chats() class Commands(FinbotCase): diff --git a/tests/test_edges.py b/tests/test_edges.py index f734f814..57d2b1cb 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -288,18 +288,20 @@ def stderr_of(self, call): return err.getvalue() def test_a_failed_send_logs_no_token(self): - """A failed send logs no Telegram token, Slack webhook path or bot token.""" + """A failed send logs no Telegram, Slack or Discord token, nor the token of a Discord interaction.""" def refuse(url, **kwargs): """Fail like requests does, the URL and headers in the message.""" raise ConnectionError(f"Max retries exceeded with url: {url} (headers {kwargs.get('headers')})") self.patch(transports.net, 'post', refuse) - self.env(slackBotToken='xoxb-SECRET', webhooks={'telegram': self.SECRET_URL.split('sendMessage')[0]}) + self.patch(transports.net, 'request', lambda method, url, **kwargs: refuse(url, **kwargs)) + token = 'MTAwMDAwMDAwMDAwMDAwMDAwMDA.SECRET.' + 'a' * 38 # a Discord bot token's shape + self.env(slackBotToken='xoxb-SECRET', telegram_api=self.SECRET_URL.split('sendMessage')[0], + discordBotToken=token) log = self.stderr_of(lambda: transports.TelegramBot('1').post_text('hi')) log += self.stderr_of(lambda: transports.SlackBot('C1').post_text('hi')) - hook = transports.SlackWebhook('https://hooks.slack.com/services/T0/B0/SECRETpath') - log += self.stderr_of(lambda: hook.post_text('hi')) - self.assertIn('Failure sending', log) - for secret in ('SECRET-tok_en', 'SECRETpath', 'xoxb-SECRET'): + log += self.stderr_of(lambda: transports.DiscordBot('77', interaction='42/SECRET-interaction').post_text('hi')) + self.assertIn('failed', log) + for secret in ('SECRET-tok_en', 'xoxb-SECRET', 'SECRET-interaction', token): self.assertNotIn(secret, log) def test_the_traceback_of_a_reported_error_is_redacted(self): diff --git a/tests/test_pull.py b/tests/test_pull.py index 1228ea8a..fbe14316 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -275,17 +275,6 @@ def test_slack_is_listened_to_with_both_tokens_and_no_webhook(self): self.assertEqual(pull.listeners(), []) self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') - def test_slack_buttons_in_both_modes_only(self): - """A Slack chat takes buttons with the bot token and either mode (the webhook or the app-level token), and not - with the bot token alone: their presses would reach nobody.""" - http = {'slackAppToken': '', 'slackOutgoingWebhook': 'https://example.com/slack'} - bot_only = {'slackAppToken': ''} - for changes, takes in (({}, True), (http, True), (bot_only, False), ({'slackBotToken': ''}, False)): - with self.subTest(changes=changes): - self.env(**changes) - self.assertEqual(transports.SlackBot('C1').takes_buttons, takes) - self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') - class Backoff(unittest.TestCase): """The wait after failures in a row.""" diff --git a/tests/test_reports.py b/tests/test_reports.py index ba5f86f7..76e60aac 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,4 +1,5 @@ """Every chat command and scheduled report produces output on mocked data.""" +import contextlib import unittest import datetime import re @@ -339,7 +340,7 @@ def test_only_the_scheduled_earnings_report_is_pinned(self): self.patch(transports.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) self.command('.earnings') self.assertEqual(pins, []) - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}, telegramChatID='1') + self.env(telegramChatID='1') cal.run(event='ex-dividend') cal.run(event='earnings') self.assertEqual(pins, ['1']) @@ -455,32 +456,34 @@ def test_an_interactive_rating_records_nothing(self): rating.run(transports.TelegramBot('5')) self.assertIsNone(state.load('finbot_rating.json')) - def test_slack_webhook_without_a_bot_token_gets_scheduled_reports(self): - """Scheduled reports reach Slack through its incoming webhook, which takes no channel and no token. They used to - send Telegram's chat id along, which made write() need slackBotToken and crash without one, before Telegram got - its copy.""" - self.env(slackBotToken=False) - - self.env(webhooks={**config.env.webhooks, 'slack': 'https://hooks.slack.com/services/T/B/x'}) + def test_scheduled_reports_reach_each_home_chat_with_its_own_token(self): + """Scheduled reports reach Telegram's and Slack's home chats, each with its own bot's credentials: Slack's + request carries the Slack token and channel, and Telegram's chat id goes only to Telegram.""" + self.env(slackBotToken='xoxb-test', slackChannelID='C9') self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) + self.patch(render, 'draw', lambda chart: io.BytesIO(b'png')) posts = [] def post(url, **kwargs): - """Record (host, whether a bot token was sent, the channel) and answer ok.""" - posts.append((url.split('/')[2], 'Authorization' in kwargs.get('headers', {}), - (kwargs.get('json') or {}).get('channel'))) - return mock.Mock(status_code=200, json=lambda: {'ok': True, 'result': {'message_id': 1, 'chat': {'id': 1}}}) + """Record (host, the Slack token sent, the channel or chat) and answer ok.""" + data = kwargs.get('data') or kwargs.get('json') or {} + posts.append((url.split('/')[2], kwargs.get('headers', {}).get('Authorization'), + data.get('channel') or data.get('channel_id') or data.get('chat_id'))) + return mock.Mock(status_code=200, text='OK', json=lambda: {'ok': True, 'upload_url': 'https://files.x/u', + 'file_id': 'F1', 'ts': '1.5', 'result': {'message_id': 1, 'chat': {'id': 1}}}) self.patch(transports.net, 'post', post) for report in (price.run, shorts.run, milestone.run): with self.subTest(report=report): posts.clear() - report() - self.assertIn(('hooks.slack.com', False, None), posts) - self.assertTrue(any(host == 'api.telegram.org' for host, _, _ in posts)) + with contextlib.redirect_stderr(io.StringIO()): + report() + self.assertIn(('slack.com', 'Bearer xoxb-test', 'C9'), posts) + self.assertTrue(any(host == 'api.telegram.org' and chat == '1' for host, _, chat in posts)) + self.assertFalse(any(host == 'slack.com' and chat == '1' for host, _, chat in posts)) def test_nowhere_to_post_is_an_error(self): """A scheduled report with nowhere to post is an error that says so.""" - self.env(webhooks={}) + self.env(telegramChatID='') with self.assertRaisesRegex(RuntimeError, 'nowhere to post'): price.run() @@ -536,7 +539,7 @@ def test_a_long_list_is_stored_and_only_the_newest_are_kept(self): def test_telegram_charts_have_buttons_in_both_modes(self): """A Telegram chart, a scheduled one included, has its period buttons whether Telegram posts their presses to telegramOutgoingWebhook (http mode) or finbot fetches them (pull mode).""" - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}, telegramChatID='1') + self.env(telegramChatID='1') label, kwargs = price.job(['7']) # as scheduled: price.py 7 for hook in ('https://example.com/telegram', ''): with self.subTest(telegramOutgoingWebhook=hook): @@ -686,11 +689,12 @@ def test_unknown_or_missing_portfolio_lists_known(self): self.assertEqual(self.command('.pie'), [('text', ['Usage: .pie PORTFOLIO [sector]', 'Portfolios: Rob'])]) self.assertEqual(self.command('.pie Nope'), [('text', ['Nope portfolio not found. I only know about:', 'Rob'])]) - def test_a_service_that_takes_no_images_gets_text(self): - """Slack without the bot token takes no images: the report goes as text.""" - self.env(slackBotToken=False) - worker.Request(SLACK, 'C1', '@u', 'U', BOT).reply(['x'], render.Chart('pie', 't', rows=(('a', 1.0),))) - self.assertEqual(self.sent[-1][0], 'text') + def test_with_the_graph_preference_off_a_chart_goes_as_text(self): + """With graph off, a report's chart is left out and its lines go as text, on every network.""" + self.prefs(graph=False) + for service in (TELEGRAM, SLACK, DISCORD): + worker.Request(service, 'C1', '@u', 'U', BOT).reply(['x'], render.Chart('pie', 't', rows=(('a', 1.0),))) + self.assertEqual(self.sent[-1], ('text', ['x'])) if __name__ == '__main__': diff --git a/tests/test_sending.py b/tests/test_sending.py index f2a61d9b..1ddaabf1 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -3,6 +3,7 @@ import contextlib, io, json, threading, unittest from unittest import mock from tests.common import * +from lib import jobs class TelegramSending(FinbotCase): @@ -60,11 +61,10 @@ def test_error_reports_are_escaped_for_telegram_and_slack(self): """An error line is escaped for Telegram's HTML and Slack (the same), and left as it is for Discord.""" sent = [] self.patch(transports.Transport, 'text', REAL_TEXT) - for kind in (transports.TelegramBot, transports.SlackWebhook, transports.DiscordWebhook): + for kind in (transports.TelegramBot, transports.SlackBot, transports.DiscordBot): self.patch(kind, 'post_text', lambda dest, text, buttons=None: sent.append((dest.name, text))) - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/', 'slack': 'https://hooks.slack.com/x', - 'discord': 'https://discord.com/x/slack'}) - self.env(telegramChatID='5') + self.env(telegramChatID='5', slackChannelID='C9', slackBotToken='xoxb-1', discordChannelID='77', + discordBotToken='x') error = RuntimeError("404 for ZZ at https://x/?a=1&crumb=abc; '<' not supported between ") with contextlib.redirect_stderr(io.StringIO()): @@ -122,36 +122,30 @@ def upload_steps(self): return [Resp({'ok': True, 'upload_url': self.UPLOAD_URL, 'file_id': 'F1'}), Resp(None, text='OK - 3'), Resp({'ok': True, 'files': [{'id': 'F1'}]})] - def test_where_charts_can_go(self): - """Charts always go to Telegram and Discord; Slack gets them as the bot, with the bot token, but not through its - incoming webhook.""" - self.env(slackBotToken='xoxb-test') - - kinds = (transports.TelegramBot, transports.SlackBot, transports.SlackWebhook, transports.DiscordWebhook) - self.assertEqual([kind(None).takes_charts for kind in kinds], [True, True, False, True]) - self.env(slackBotToken=False) - self.assertFalse(transports.SlackBot(None).takes_charts) - - def test_scheduled_reports_go_to_slack_as_the_bot_with_slackChannelID(self): - """With slackChannelID and the bot token, scheduled reports reach Slack as the bot in place of the incoming - webhook, which it does not need; without the token, the webhook carries them.""" - hook = 'https://hooks.slack.com/services/x' - telegram = config.env.webhooks['telegram'] - self.env(webhooks={'telegram': telegram, 'slack': hook}, telegramChatID='1', slackChannelID='C9', - slackBotToken='xoxb-test') - self.assertEqual(list(map(repr, delivery.default_chats())), ['', '']) - self.env(slackBotToken='') - self.assertEqual(list(map(repr, delivery.default_chats())), ['', f'']) - self.env(webhooks={'telegram': telegram}, slackBotToken='xoxb-test') - self.assertEqual(list(map(repr, delivery.default_chats())), ['', '']) - self.env(webhooks={}, telegramChatID='') # the bot alone - self.assertEqual(list(map(repr, delivery.default_chats())), ['']) + def test_every_bot_takes_charts_and_buttons(self): + """Every network's bot takes charts and buttons, whichever way its updates arrive: their presses reach finbot, + and no setting holds them back.""" + self.env(slackBotToken='', slackAppToken='', slackOutgoingWebhook='', telegramOutgoingWebhook='') + for kind in (transports.TelegramBot, transports.SlackBot, transports.DiscordBot): + with self.subTest(kind=kind.__name__): + self.assertTrue(kind('1').takes_charts and kind('1').takes_buttons) + + def test_scheduled_reports_go_to_each_networks_home_chat_as_its_bot(self): + """Scheduled reports go to each network's home chat (telegramChatID, slackChannelID, discordChannelID), as its + bot: a chat without its bot token is left out.""" + self.env(telegramChatID='1', slackChannelID='C9', slackBotToken='xoxb-test', discordChannelID='77', + discordBotToken='x') + self.assertEqual(list(map(repr, delivery.default_chats())), ['', '', + '']) + self.env(slackBotToken='', telegramBotToken='') + self.assertEqual(list(map(repr, delivery.default_chats())), ['']) def test_slack_replies_upload_their_charts(self): - """A Slack reply uploads its chart in three steps (bot token), the report as its message, with no text copy.""" + """A Slack reply whose chart has no buttons uploads it in three steps (bot token), the report as its message, + with no text copy (a chart with buttons goes in a Block Kit message: SlackButtons).""" self.env(slackBotToken='xoxb-test') - for cmd, heading in (('.beta', '*Beta over 1.5'), ('.price', '*≥ 5% at close'), ('.compare T1 T2', '*')): + for cmd, heading in (('.beta', '*Beta over 1.5'), ('.pie Rob', '*')): with self.subTest(cmd=cmd): calls = self.slack_answers(*self.upload_steps()) self.sent.clear(); self.errors.clear() @@ -193,13 +187,9 @@ def test_a_slack_upload_that_fails_goes_out_as_text(self): delivery.deliver(transports.SlackBot('C1'), delivery.Report(['*Heading*', 'line 2'], CHART)) self.assertEqual(self.sent, expect) - def test_scheduled_reports_to_slack_carry_charts_only_as_the_bot(self): - """Scheduled reports reach Slack's incoming webhook as text; the Slack bot (slackChannelID) and Telegram get - their charts.""" - self.env(slackBotToken='xoxb-test') - - # an incoming webhook cannot upload - self.env(webhooks={**config.env.webhooks, 'slack': 'https://hooks.slack.com/services/x'}) + def test_scheduled_reports_carry_their_charts_to_every_home_chat(self): + """A scheduled report's chart goes to each network's home chat, as its bot.""" + self.env(slackBotToken='xoxb-test', slackChannelID='C9', discordBotToken='x', discordChannelID='77') sent = [] self.patch(transports.Transport, 'chart', lambda dest, image, caption, buttons=None: sent.append((dest.name, 'chart')) or {'ok': True}) @@ -209,30 +199,7 @@ def test_scheduled_reports_to_slack_carry_charts_only_as_the_bot(self): with self.subTest(report=report): sent.clear() report() - self.assertEqual(sorted(set(sent)), [('slack', 'text'), ('telegram', 'chart')]) - self.env(slackChannelID='C9') # as the bot, the report's chart goes too - sent.clear() - report() - self.assertEqual(sorted(set(sent)), [('slack', 'chart'), ('telegram', 'chart')]) - self.env(slackChannelID='') - - def test_slack_without_a_bot_token_gets_text_and_no_chart_is_drawn(self): - """Without the bot token, Slack replies are text and no chart is drawn for them.""" - self.env(slackBotToken=False) - - def refuse(*a, **k): - """Fail: no chart should be drawn.""" - raise AssertionError('a chart was drawn for Slack') - self.patch(render, 'draw', refuse) # every chart is drawn there - self.env(webhooks={**config.env.webhooks, 'slack': 'https://hooks.slack.com/services/x'}) - for cmd in ('.beta', '.pie Rob', '.buy', '.sell', '.peg', '.price', '.price top', '.price T1 7d', - '.performance 7d', '.shorts', '.history T1', '.compare T1 T2'): - with self.subTest(cmd=cmd): - self.sent.clear(); self.errors.clear() - worker.process_request(inbound.Message('slack', 'C1', '<@U1>', cmd, BOT, 'U')) - self.assertEqual(self.errors, []) - self.assertTrue(self.sent) - self.assertEqual({kind for kind, *_ in self.sent}, {'text'}) + self.assertEqual(sorted(set(sent)), [('discord', 'chart'), ('slack', 'chart'), ('telegram', 'chart')]) def test_the_graph_preference_off_sends_no_chart_anywhere(self): """graph=False means no charts, as the README says; it used to hold back only the price-line charts.""" @@ -277,31 +244,26 @@ def fake(url, **kwargs): self.assertEqual(self.sent, expect) -class DiscordWebhook(FinbotCase): - """Discord's own webhook endpoint (the Slack-like one takes no flags): no previews, no pings, charts attached.""" - URL = 'https://discord.com/api/webhooks/1/abc' +class DiscordSending(FinbotCase): + """Discord's home channel, as the bot: a scheduled report's chart attached, within Discord's 2,000 characters.""" def setUp(self): - """Discord as the only destination, sent through the real path, its posts recorded.""" + """Discord as the only destination, sent through the real path, its calls recorded.""" super().setUp() - self.env(webhooks={'discord': self.URL}) + self.env(telegramChatID='', discordBotToken='x', discordChannelID='77') self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) - self.posts = [] - self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'id': '1'})) - - def test_text_goes_to_the_native_endpoint_without_previews_or_pings(self): - """Text goes to the native endpoint with ?wait=true, without previews or mentions.""" - transports.DiscordWebhook(self.URL).post_text('hello @everyone') - url, kwargs = self.posts[0] - self.assertEqual(url, self.URL + '?wait=true') # without wait Discord answers 204, which counted as a failure - self.assertEqual(kwargs['json'], {'content': 'hello @everyone', 'flags': 4, 'allowed_mentions': {'parse': []}}) + self.calls = [] + self.patch(transports.discord.net, 'request', lambda method, url, **kwargs: self.calls.append((url, kwargs)) + or Resp({'id': '1'})) def test_a_scheduled_report_carries_its_chart(self): - """A scheduled Discord report has its chart attached and the report as its message, within 2,000 characters.""" - price.run() - (url, kwargs), = self.posts - self.assertEqual(url, self.URL + '?wait=true') + """A scheduled Discord report has its chart attached and the report as its message, within 2,000 characters, + without previews or pings.""" + with contextlib.redirect_stderr(io.StringIO()): + price.run() + (url, kwargs), = self.calls + self.assertEqual(url, 'https://discord.com/api/v10/channels/77/messages') payload = json.loads(kwargs['data']['payload_json']) self.assertEqual((payload['flags'], payload['allowed_mentions'], payload['attachments'][0]['filename']), (4, {'parse': []}, 'chart.png')) @@ -309,13 +271,14 @@ def test_a_scheduled_report_carries_its_chart(self): self.assertLessEqual(len(payload['content']), 2000) name, image, kind = kwargs['files']['files[0]'] self.assertEqual((name, kind), ('chart.png', 'image/png')) - self.assertTrue(image.read().startswith(b'\x89PNG')) + self.assertTrue(image.startswith(b'\x89PNG')) def test_a_long_report_is_split_by_size(self): """A long report is split into messages of at most 2,000 characters.""" line = 'x' * 150 - transports.DiscordWebhook(self.URL).text([line] * 30) - sizes = [len(kwargs['json']['content']) for _, kwargs in self.posts] + with contextlib.redirect_stderr(io.StringIO()): + transports.DiscordBot('77').text([line] * 30) + sizes = [len(kwargs['json']['content']) for _, kwargs in self.calls] self.assertGreater(len(sizes), 2) self.assertTrue(all(size <= 2000 for size in sizes), sizes) # 20 lines a chunk used to make 3,019 characters @@ -325,7 +288,7 @@ def test_links_are_masked_links(self): class SlackSending(FinbotCase): - """Slack replies go through chat.postMessage with the bot token; scheduled reports through the incoming webhook.""" + """Slack's messages go through chat.postMessage with the bot token, replies and scheduled reports alike.""" def setUp(self): """The real send path, its posts recorded.""" @@ -336,16 +299,14 @@ def setUp(self): self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'ok': True})) def test_link_previews_are_turned_off_in_the_body(self): - """Link previews are turned off in the body, where Slack reads them, for replies and for the webhook.""" + """Link previews are turned off in the body, where Slack reads them.""" self.env(slackBotToken='xoxb-test') - transports.SlackBot('C1').post_text('hi') - transports.SlackWebhook('https://hooks.slack.com/services/T/B/x').post_text('hi') - (_, reply), (_, hook) = self.posts + with contextlib.redirect_stderr(io.StringIO()): + transports.SlackBot('C1').post_text('hi') + (_, reply), = self.posts self.assertEqual((reply['data']['unfurl_links'], reply['data']['unfurl_media']), ('false', 'false')) - self.assertEqual((hook['json']['unfurl_links'], hook['json']['unfurl_media']), (False, False)) - for kwargs in (reply, hook): # they used to go in the headers, where Slack does not look - self.assertNotIn('unfurl_links', kwargs['headers']) + self.assertNotIn('unfurl_links', reply['headers']) # they used to go in the headers, where Slack does not look def test_a_reply_slack_refuses_is_logged_as_failed(self): """Slack refuses with HTTP 200 and ok false (the bot is not in the channel, say): a failure, logged as one.""" @@ -370,7 +331,7 @@ def test_a_reply_without_a_bot_token_is_logged_not_sent(self): def test_a_command_at_work_shows_an_hourglass_on_its_message(self): """While a command works, ⏳ is on the message it answers (Slack has no typing notice for bots); it comes off - once the answer is out, so the answer does not wait for it. Only with that message and the bot token.""" + once the answer is out, so the answer does not wait for it. Only with that message.""" self.env(slackBotToken='xoxb-test') self.patch(transports.Transport, 'TYPING', (0, 7, 10)) added = threading.Event() @@ -389,8 +350,6 @@ def post(url, **kwargs): reaction = {'channel': 'C1', 'timestamp': '1.2', 'name': 'hourglass_flowing_sand'} self.assertEqual((self.posts[0][1]['data'], self.posts[2][1]['data']), (reaction, reaction)) self.assertFalse(transports.SlackBot('C1').shows_typing) # no message to put it on - self.env(slackBotToken='') - self.assertFalse(transports.SlackBot('C1', '1.2').shows_typing) def test_an_hourglass_slack_refuses_is_tried_once(self): """⏳ that Slack refuses (a missing scope) is logged and not tried again, and nothing is taken off.""" @@ -417,7 +376,7 @@ def test_the_slack_api_says_what_is_missing(self): class SlackButtons(FinbotCase): """A chart's period buttons on Slack: a Block Kit message (the report, the chart uploaded into no channel, the - buttons) that a press redraws in place, where the presses reach finbot (the bot token and slackOutgoingWebhook).""" + buttons) that a press redraws in place, in either mode.""" UPLOAD_URL = 'https://files.slack.com/upload/v1/abc' def setUp(self): @@ -549,12 +508,14 @@ def test_buttons_slack_will_not_show_leave_a_plain_chart(self): self.assertEqual(self.posts[-1][1]['data']['channel_id'], 'C1') self.assertEqual((self.waits, self.sent, self.errors), ([1, 2, 4], [], [])) - def test_without_slackOutgoingWebhook_a_chart_has_no_buttons(self): - """Without slackOutgoingWebhook, no press could reach finbot, so a chart goes without buttons.""" - self.env(slackOutgoingWebhook='') + def test_in_socket_mode_a_chart_has_its_buttons_too(self): + """In Socket Mode (no slackOutgoingWebhook) a press reaches finbot over the socket, so a chart has its buttons + as in http mode.""" + self.env(slackOutgoingWebhook='', slackAppToken='xapp-1') worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.price', BOT, 'U')) - self.assertEqual(self.calls(), ['files.getUploadURLExternal', 'upload', 'files.completeUploadExternal']) - self.assertEqual(self.posts[-1][1]['data']['channel_id'], 'C1') + self.assertEqual(self.calls(), ['files.getUploadURLExternal', 'upload', 'files.completeUploadExternal', + 'chat.postMessage']) + self.assertEqual(self.blocks('chat.postMessage')[-1]['type'], 'actions') def test_a_long_report_takes_several_sections(self): """A report longer than a section takes holds is split into sections, and an empty one takes none.""" @@ -581,5 +542,82 @@ def post(url, **kwargs): self.assertEqual(self.errors, []) + +class EveryNetworkAndMode(FinbotCase): + """One rule for the six networks and modes, in a channel and in a DM: a chart has its period buttons and a .jobs + list its buttons, a command shows the network's notice that finbot is at work, and the reply goes to the chat that + asked.""" + MODES = { + ('telegram', 'http'): {'telegramOutgoingWebhook': 'https://example.com/telegram'}, + ('telegram', 'pull'): {'telegramOutgoingWebhook': ''}, + ('slack', 'http'): {'slackBotToken': 'xoxb-1', 'slackOutgoingWebhook': 'https://example.com/slack'}, + ('slack', 'pull'): {'slackBotToken': 'xoxb-1', 'slackAppToken': 'xapp-1', 'slackOutgoingWebhook': ''}, + ('discord', 'http'): {'discordBotToken': 'x', 'discordOutgoingWebhook': 'https://example.com/discord', + 'discordPublicKey': 'ab' * 32}, + ('discord', 'pull'): {'discordBotToken': 'x', 'discordOutgoingWebhook': ''}, + } + CHATS = {'telegram': ('-1001', '55'), 'slack': ('C1', 'D1'), 'discord': ('77', '88')} # a channel, then a DM + + def setUp(self): + """A schedule for .jobs to list, and what each command posts noted with its destination.""" + super().setUp() + self.patch(inbound.DISCORD, 'bot_id', '99') + schedule = Path(temp_dir(self), 'schedule.cron') + schedule.write_text('*/20 * * * Mon-Fri trades.py\n') + scheduler = jobs.Scheduler(lambda: schedule) + scheduler.load() + self.patch(jobs, 'current', scheduler) + self.posted = [] + self.patch(transports.Transport, 'chart', lambda dest, image, caption, buttons=None: + self.posted.append((dest, buttons)) or {'ok': True}) + self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: self.posted.append((dest, buttons)) + or []) + self.updates = 0 + + def update(self, network, mode, chat, text): + """The update the network sends for a message in this mode: Discord's http mode has only the slash command.""" + self.updates += 1 + n, person = self.updates, {'id': '5', 'username': 'rob', 'first_name': 'Rob'} + if network == 'telegram': + message = {'message_id': n, 'chat': {'id': int(chat)}, 'from': person, 'text': text} + return {'update_id': n, 'message': message} + if network == 'slack': + return {'type': 'event_callback', 'event_id': f'Ev{n}', 'authorizations': [{'user_id': 'UBOT'}], + 'event': {'type': 'message', 'channel': chat, 'user': 'U1', 'text': text, 'ts': f'{n}.1'}} + if mode == 'pull': + return {'id': str(700 + n), 'channel_id': chat, 'author': person, 'content': text} + name, _, arguments = text.removeprefix('.').partition(' ') + options = [{'name': 'arguments', 'type': 3, 'value': arguments}] if arguments else [] + who = {'user': person} if chat == '88' else {'member': {'user': person}} + return {'id': str(900 + n), 'application_id': '42', 'token': f't{n}', 'type': 2, 'channel_id': chat, **who, + 'data': {'name': name, 'options': options}} + + def test_every_network_in_both_modes_in_a_channel_and_a_dm(self): + """A chart and a .jobs list each come with their buttons, in the chat that asked; the command showed the notice + that finbot is at work: "typing…" (Telegram, a Discord message), ⏳ on the message (Slack), or Discord's + "thinking…" for a slash command, which its answer replaces.""" + for (network, mode), keys in self.MODES.items(): + for chat in self.CHATS[network]: + with self.subTest(network=network, mode=mode, chat=chat): + self.env(**keys) + self.posted.clear(); self.typing.clear() + answers = [] + with contextlib.redirect_stderr(io.StringIO()): + for text in ('.price', '.jobs'): + answer = bot.take(getattr(inbound, network.upper()), self.update(network, mode, chat, text)) + answers.append(answer) + if answer.then: + answer.then() + self.join_workers() + self.assertEqual(self.errors, []) + self.assertEqual([(dest.where, bool(buttons)) for dest, buttons in self.posted], [(chat, True)] * 2) + if network == 'discord' and mode == 'http': + self.assertEqual([json.loads(answer.body) for answer in answers], [{'type': 5}] * 2) + self.assertTrue(all(dest.interaction and not dest.shows_typing for dest, _ in self.posted)) + else: + self.assertEqual(len(self.typing), 2) + self.assertTrue(all(dest.shows_typing for dest in self.typing)) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_telegram_api.py b/tests/test_telegram_api.py index 457ae87c..0300e612 100644 --- a/tests/test_telegram_api.py +++ b/tests/test_telegram_api.py @@ -9,7 +9,7 @@ class TelegramWebhook(FinbotCase): def setUp(self): """A Telegram bot with an outgoing webhook configured.""" super().setUp() - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}) + self.env(telegram_api='https://api.telegram.org/botX/') self.env(telegramOutgoingWebhook='https://example.com/telegram') diff --git a/tests/test_yahoo.py b/tests/test_yahoo.py index 1511a4aa..bfda3812 100644 --- a/tests/test_yahoo.py +++ b/tests/test_yahoo.py @@ -264,7 +264,8 @@ def get(url, **kwargs): return Resp(status=404) if '/chart/T3?' in url or '/chart/SPY?' in url else good self.patch(yahoo.net, 'get', get) def slack(text): - """Run a command on Slack, where reports come as text, and return the text.""" + """Run a command on Slack, its report as text (no charts), and return the text.""" + self.prefs(graph=False) self.sent.clear(); self.errors.clear() worker.process_request(inbound.Message('slack', 'C1', '<@U1>', text, BOT, 'U')) self.assertEqual(self.errors, [], text) diff --git a/util/check_config.py b/util/check_config.py index 12c354b9..4cd41a07 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -49,12 +49,18 @@ def warnings(startup_path): return [] -def notes(preferences_path): - """The keys of a valid preferences.conf that do nothing any more, a line each.""" +def notes(preferences_path, startup_path=None): + """The keys of the two files, where valid, that do nothing any more, a line each with what to do instead.""" + lines = [] + if startup_path: + try: + lines += config.retired_notes(config.read_startup_file(startup_path), 'finbot.env') + except config.ConfigError: + pass try: - return config.retired_notes(config.read_preferences(preferences_path) or {}) + return lines + config.retired_notes(config.read_preferences(preferences_path) or {}) except config.ConfigError: - return [] + return lines if __name__ == '__main__': @@ -71,6 +77,6 @@ def notes(preferences_path): args.preferences if Path(args.preferences).is_file() else '(no preferences.conf, defaults)') for line in warnings(args.startup): print('config: warning:', line) - for line in notes(args.preferences): + for line in notes(args.preferences, args.startup): print('config:', line) sys.exit(1 if problems else 0) diff --git a/util/record_outputs.py b/util/record_outputs.py index 3cb2daa7..503c2118 100755 --- a/util/record_outputs.py +++ b/util/record_outputs.py @@ -28,8 +28,7 @@ SCHEDULED = ['trades.py', 'price.py midsession', 'price.py interday', 'price.py premarket', 'price.py close', 'price.py 7', 'cal.py earnings', 'cal.py ex-dividend', 'performance.py 7', 'rating.py', 'shorts.py', 'milestone.py', 'reminder.py'] -WEBHOOKS = {'telegram': 'https://api.telegram.org/botX/', 'slack': 'https://hooks.slack.com/services/T/B/x', - 'discord': 'https://discord.com/api/webhooks/1/abc'} +TELEGRAM_API = 'https://api.telegram.org/botX/' ANSWER = {'ok': True, 'upload_url': 'https://files.slack.com/u', 'file_id': 'F1', 'id': '1'} # good enough for every API @@ -126,14 +125,17 @@ def step(label, run): for data in BUTTONS: label = f'button {data}' if service == 'telegram' else f'{service} button {data}' step(label, pressing(service, 'cb', chat, message, data)) - self.env(webhooks=WEBHOOKS, telegramChatID='1') - for where in ('', ' (slackChannelID)'): # Slack's incoming webhook, then the channel as the bot - if where: - self.env(slackChannelID='C9') - for line in SCHEDULED: - label, module, kwargs = jobs.parse_command(line)[0] - run = getattr(module, 'run', None) or module.lambda_handler # lambda_handler: a commit before run() - step(f'scheduled {line}{where}', lambda: run(**kwargs)) + # the scheduled reports go to each network's home chat as its bot (a commit from before telegram_api had + # webhooks, the incoming ones among them, and one from before the Discord bot no discordChannelID) + fields = {field.name for field in common.dataclasses.fields(common.config.env)} + api = {'telegram_api': TELEGRAM_API} if 'telegram_api' in fields else { + 'webhooks': {'telegram': TELEGRAM_API}} + discord = {'discordChannelID': '78'} if 'discordChannelID' in fields else {} + self.env(telegramChatID='1', slackChannelID='C9', **api, **discord) + for line in SCHEDULED: + label, module, kwargs = jobs.parse_command(line)[0] + run = getattr(module, 'run', None) or module.lambda_handler # lambda_handler: a commit before run() + step(f'scheduled {line}', lambda: run(**kwargs)) Path(output).write_text(json.dumps(steps, indent=1, sort_keys=True, default=str)) result = unittest.TextTestRunner(verbosity=0, stream=io.StringIO()).run(Record()) diff --git a/util/split_env.py b/util/split_env.py index 9033ca96..253119e0 100755 --- a/util/split_env.py +++ b/util/split_env.py @@ -58,6 +58,8 @@ def split(text, path='.env'): notes.append(f'{path} line {number}: {key} dropped ({REMOVED[key]})') elif key == 'STATE_DIRECTORY': notes.append(f'{path} line {number}: {key} is set by systemd, left out') + elif key in config.RETIRED: + notes.append(f'{path} line {number}: {key} dropped: {config.RETIRED[key]}') elif key in config.STARTUP_KEYS: startup.append(f'{key}={value}') elif key in config.PREFERENCE_KEYS: From f3a2cb11b5f2e16917142bcb9ce007aafa388e8f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:12:34 +0000 Subject: [PATCH 07/31] Scheduled reports run in the bot's process, not child processes Each scheduled or manual run started a Python process that imported pandas and matplotlib again; about a hundred a day. Now a command runs in the bot, in a thread of its own, sharing its caches and connections. - The Scheduler's runner calls the report's run() in its thread (jobs.run_in_process); a failure, SystemExit too, is reported as before. - A thread cannot be killed, so JOB_TIMEOUT becomes a soft deadline: a command still running after it is reported once and left to finish, marked running meanwhile so that no second copy starts. Every request goes through lib/net, with its timeouts, so a stuck job ends; a test fails if a module outside lib/net.py calls requests itself or passes timeout=None. - The scheduler is always on: the scheduler key retires (a note at start says to delete it), and a schedule without jobs runs nothing. - Gone: lib/jobrun.py, run_child, stop, exit_reason, the preferences handed to job processes (child_environment, FINBOT_PREFERENCES_PASSED), delivery.cli and guarded, and the twelve top-level scripts that ran a report from cron: running one by hand is `.jobs run`. The checkout's top holds bot.py alone; bot_send_stdin.py is util/send_stdin.py. - Tests that start the bot give it an empty schedule (no_jobs). - util/record_outputs.py loads the shipped schedule, so .jobs lists it. - The README's scheduling notes, the finbot.env template, the flush unit and CLAUDE.md follow. util/record_outputs.py: 247 steps, identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 11 +- README.md | 35 +++--- bot.py | 25 ++-- bot_send_stdin.py | 14 --- cal.py | 5 - cleanup.py | 5 - defaults/finbot.env | 6 +- finbot-flush.service | 2 +- lib/config.py | 18 +-- lib/delivery.py | 22 ---- lib/jobrun.py | 34 ------ lib/jobs.py | 80 ++++--------- lib/reports/__init__.py | 16 +-- lib/reports/about.py | 2 +- milestone.py | 5 - performance.py | 5 - podcasts.py | 5 - price.py | 5 - rating.py | 5 - reminder.py | 5 - sharesight_warm.py | 5 - shorts.py | 5 - tests/common.py | 11 +- tests/test_config.py | 43 ++----- tests/test_edges.py | 9 +- tests/test_finbot.py | 18 ++- tests/test_schedule.py | 31 +---- tests/test_scheduler.py | 234 +++++++++---------------------------- tests/test_sending.py | 10 -- tests/test_telegram_api.py | 5 +- trades.py | 5 - util/record_outputs.py | 6 +- util/send_stdin.py | 16 +++ yahoo_warm.py | 5 - 34 files changed, 190 insertions(+), 518 deletions(-) delete mode 100755 bot_send_stdin.py delete mode 100755 cal.py delete mode 100755 cleanup.py delete mode 100644 lib/jobrun.py delete mode 100755 milestone.py delete mode 100755 performance.py delete mode 100755 podcasts.py delete mode 100755 price.py delete mode 100755 rating.py delete mode 100755 reminder.py delete mode 100755 sharesight_warm.py delete mode 100755 shorts.py delete mode 100755 trades.py create mode 100755 util/send_stdin.py delete mode 100755 yahoo_warm.py diff --git a/CLAUDE.md b/CLAUDE.md index ea1dcdfb..6fc31f2e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,9 +57,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm the typing notice while it works. `process_request(message)` and `process_callback(press)` take inbound's objects. - `lib/reports/`: what finbot says. Scheduled (each has `run(dest=None, **kwargs)`, which posts to the chat that asked or else the default chats, and `job(argv)`, its schedule-line parser): `price performance trades shorts cal rating - milestone reminder podcasts cleanup sharesight_warm yahoo_warm`; the top-level `price.py` and the rest are shims that - run them. Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who - .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). + milestone reminder podcasts cleanup sharesight_warm yahoo_warm`, run in the bot by the scheduler or `.jobs run`. + Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who .pie + .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). - `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or each network's home chat as its bot), `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per network's bot (`TelegramBot`, `SlackBot`, `DiscordBot`), each taking charts and buttons: splitting, captions, charts, buttons, @@ -83,8 +83,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; change one with `with state.edit(name, default) as data:`, never load-then-save. `lib/watchlist.py`: the watchlist. - `lib/config.py`: `config.env` (finbot.env, fixed at start) and `config.prefs` (preferences.conf, a new object on - each reload). `lib/jobs.py` + `lib/jobrun.py`: the scheduler, its shared state under one lock; each command runs in - a child process, stopped after `JOB_TIMEOUT`. + each reload). `lib/jobs.py`: the scheduler, its shared state under one lock; each command runs in the bot's process, + in a thread (`run_in_process`), reported once if still running after `JOB_TIMEOUT` and left to finish (every request + goes through `net`, with its timeouts: a test checks). - `tests/common.py`: `FinbotCase` (offline fixtures). `self.command('.price')` returns what was sent: `('text', lines)` or `('photo', size, caption)`. `wsgi_environ(path, body, **headers)` is a request for `bot.main`. `ProviderCase` runs the real parsers on the answers saved under `doc/` (`sample`), as `Resp` and `serve` return diff --git a/README.md b/README.md index 77cc5da9..0e0afaaa 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ With persistent storage, the default of `past_days=28` suits. This is useful if past_days=28 ``` -Cron trigger: +Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): ``` trades.py ``` @@ -194,13 +194,13 @@ price_percent=5 demote_volatile=True ``` -Cron trigger: +Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): ``` -./price.py [premarket|midsession|interday|close|days (int)] +price.py [premarket|midsession|interday|close|days (int)] ``` -When scheduled (Cron), the mode must be passed as an execution argument: +When scheduled, the mode is its argument: * If `premarket` is passed, it only reports on pre/post market price movements. -* If `midsession` is passed, it only reports for markets currently in session. This is intended to run from Cron to provide mid-session alerts for big price movements of your holdings. For example, it could be run twice per day 12 hours apart, to capture markets in different timezones. +* If `midsession` is passed, it only reports for markets currently in session. This is intended to run on a schedule to provide mid-session alerts for big price movements of your holdings. For example, it could be run twice per day 12 hours apart, to capture markets in different timezones. * If `interday` is passed, it reports current price against the previous market close. * If `close` is passed, it reports daily change in stocks whose market closed in the past 3 hours. * If an integer [i] is passed, it reports current price against [i] days ago. @@ -249,9 +249,9 @@ Config example: past_days=28 ``` -Cron trigger: +Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): ``` -./performance.py 7 +performance.py 7 ``` Interactive trigger: @@ -282,9 +282,9 @@ Interactively, you can also specify a portfolio: future_days=30 ``` -Cron execution: +Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): ``` -./cal.py [earnings|ex-dividend] +cal.py [earnings|ex-dividend] ``` Interactive trigger (earnings): @@ -397,7 +397,7 @@ Interactive trigger: Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run from Cron. Requires persistent storage. ``` -./rating.py +rating.py ``` ### Watchlist @@ -467,7 +467,6 @@ of the chat network keys it needs. | `DATA_DIR` | `var/` in the checkout (`$STATE_DIRECTORY` under systemd `StateDirectory=`) | Where state and cache live | | `ip` | `127.0.0.1` | Address the bot listens on | | `port` | `5000` | Port the bot listens on | -| `scheduler` | `True` | Run the scheduled reports (`schedule.cron`) inside the bot | | `scheduler_catchup` | `5` | Minutes: a run missed while the bot was down still fires if it was due this recently | | `telegramChatID` | | Chat group or channel id that scheduled reports go to on Telegram | | `sharesight_code` | | Sharesight API access | @@ -593,7 +592,7 @@ docker kill -s USR1 # Docker kill -USR1 # anything else ``` The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && -sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, `cleanup.py 0` deletes the same files. +sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, deleting the files in `DATA_DIR/cache` does the same. Two scheduled jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](#scheduling)): `sharesight_warm.py` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm.py` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm.py` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). A data directory with its own `schedule.cron` needs their lines: ``` @@ -624,7 +623,7 @@ hyperlinkProvider=yahoo ``` ## Scheduling -The scheduled reports are started by `bot.py`, each run in its own child process (`python -m lib.jobrun`), so they use the same user, sandbox, environment and data directory as the bot, while heavy work (charts, large reports) cannot hold up incoming chat webhooks. A run that exceeds 5 minutes is stopped. There is no cron to set up, which matters under systemd `DynamicUser=` where only the bot's own user can read the state directory. **This is on by default.** Remove the equivalent entries from any old crontab before the first start, otherwise every job runs twice; or set `scheduler=False` in the environment file to keep using cron. +The scheduled reports are run by `bot.py` itself, each in a thread of its own, so they share the bot's user, sandbox, environment, data directory, caches and connections; charts are drawn one at a time, and a slow report holds up no chat command. There is no cron to set up, which matters under systemd `DynamicUser=` where only the bot's own user can read the state directory. Remove the equivalent entries from any old crontab before the first start, otherwise every job runs twice. The schedulable scripts are `trades.py`, `price.py`, `reminder.py`, `cal.py`, `shorts.py`, `performance.py`, `rating.py`, `milestone.py`, `podcasts.py ""`, `cleanup.py [days]` (deletes cache files older than `days`, default 30), `sharesight_warm.py` and `yahoo_warm.py` (they fill the cache: see [Caching](#caching)). Other jobs, such as a DNS updater or backups, stay in cron or systemd timers. @@ -653,15 +652,11 @@ The bot logs the number of jobs when it starts or reloads, and each job as it ru Behaviour worth knowing: * A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. -* A job still running when it is next due is skipped. A command still running after 5 minutes is stopped and reported, and the rest of its line still runs: each command has 5 minutes of its own. +* A job still running when it is next due is skipped. A command still running after 5 minutes is reported once, and left to finish (a thread cannot be stopped, but every request it makes has a timeout); until it does, it is not started again. Each command of a line has 5 minutes of its own. * Failures are reported to your chat channels, as before. -* With `scheduler=False` nothing is scheduled by the bot, so you can keep using cron or systemd timers and run the scripts yourself. +* A schedule without jobs runs nothing: there is no switch to turn the scheduler off. -**Running a report by hand.** Under `DynamicUser=yes` you cannot simply log in as the bot's user, so run the script as a transient unit with a dynamic user and the same state directory: -``` -sudo systemd-run --pty --wait --collect -p DynamicUser=yes -p StateDirectory=finbot -p EnvironmentFile=/opt/finbot.env /opt/finbot/price.py 7 -``` -The script prints its output on your terminal and sends its report to your chat channels as usual. The transient unit gets its own dynamic user, and systemd may re-own the state directory to it for the run: check `ls -ln /var/lib/private/finbot` afterwards and `systemctl restart finbot` if the bot can no longer write there. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports inside the bot, without any of this. Outside systemd, run a script from the checkout as your own user with the settings loaded into the shell: `set -a; . /opt/finbot.env; set +a; ./price.py 7` (this works while no value in the file contains spaces). +**Running a report by hand** is `.jobs run` in a chat (above): it runs the command in the bot, as the schedule would. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports in the bot too. ## Daemonize (systemd) `finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy `finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of [`finbot.env`](#setup) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start it. `systemctl reload finbot` re-reads `preferences.conf` and the [schedule](#scheduling) without a restart; a change to `finbot.env` needs `systemctl restart finbot`: diff --git a/bot.py b/bot.py index 9a7587b5..1c02276a 100755 --- a/bot.py +++ b/bot.py @@ -230,18 +230,14 @@ def http_server(): print('ERROR config:', line, file=sys.stderr) if os.getuid() == 0: print("Running as superuser. This is not recommended.", file=sys.stderr) - if config.env.scheduler: # read before anything else happens: an impossible time in it stops the start - scheduler = jobs.Scheduler(lambda: config.data_file('schedule.cron')) - try: - if not scheduler.load(): - scheduler = None - except jobs.ImpossibleTime as e: - for line in e.lines: - print('ERROR schedule:', line, file=sys.stderr) - print('ERROR schedule: finbot will not start until every time in the schedule can come', file=sys.stderr) - sys.exit(1) - else: - print('Scheduled reports are off (scheduler=False)', file=sys.stderr) + scheduler = jobs.Scheduler(lambda: config.data_file('schedule.cron')) # without a schedule, it runs nothing + try: # read before anything else happens: an impossible time in it stops the start + scheduler.load() + except jobs.ImpossibleTime as e: + for line in e.lines: + print('ERROR schedule:', line, file=sys.stderr) + print('ERROR schedule: finbot will not start until every time in the schedule can come', file=sys.stderr) + sys.exit(1) render.preload() try: server = http_server() # bound before Telegram is given the URL @@ -265,9 +261,8 @@ def http_server(): except Exception as e: print('ERROR', delivery.error_line(e, 'discord'), file=sys.stderr) sys.exit(1) - if scheduler: - jobs.current = scheduler # for the .jobs chat command - threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs + jobs.current = scheduler # for the .jobs chat command + threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs for listener in pull.listeners(): # the networks without a webhook: finbot fetches their updates threading.Thread(target=listener.run, args=(take,), daemon=True).start() # Startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron diff --git a/bot_send_stdin.py b/bot_send_stdin.py deleted file mode 100755 index 3e9851a2..00000000 --- a/bot_send_stdin.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from lib import delivery - -if os.isatty(sys.stdin.fileno()): - raise RuntimeError("No stdin input detected (not piped or redirected)") - -payload = [line.strip() for line in sys.stdin if line.strip()] - -for dest in delivery.default_chats(): - delivery.deliver(dest, delivery.Report(payload)) - diff --git a/cal.py b/cal.py deleted file mode 100755 index e631d21d..00000000 --- a/cal.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./cal.py [arguments]: runs lib/reports/cal.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/cleanup.py b/cleanup.py deleted file mode 100755 index 530ee071..00000000 --- a/cleanup.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./cleanup.py [arguments]: runs lib/reports/cleanup.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/defaults/finbot.env b/defaults/finbot.env index 910f156b..3f7633e0 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -1,4 +1,4 @@ -# finbot.env: THE STARTUP FILE. Where finbot runs, who it talks to, where it reports, and whether it schedules. +# finbot.env: THE STARTUP FILE. Where finbot runs, who it talks to, and where it reports. # The bot never reads this file. Your launcher loads it into the bot's environment, and a change needs a RESTART: # systemd: EnvironmentFile=/opt/finbot.env (in finbot.service) # docker: --env-file /opt/finbot.env @@ -19,9 +19,7 @@ ip=127.0.0.1 port=5000 -# Run the scheduled reports (schedule.cron) inside the bot, and how many minutes late a run missed while the bot was -# down may still fire -scheduler=True +# How many minutes late a scheduled report missed while the bot was down may still fire scheduler_catchup=5 # Where scheduled reports go on Telegram: the chat group or channel id diff --git a/finbot-flush.service b/finbot-flush.service index d79c3580..d862bc0f 100644 --- a/finbot-flush.service +++ b/finbot-flush.service @@ -3,5 +3,5 @@ Description=Flush finbot's cache (sudo systemctl start finbot-flush) [Service] Type=oneshot -# finbot deletes its cache files on SIGUSR1 and keeps its state; its job processes ignore the signal +# finbot deletes its cache files on SIGUSR1, keeps its state, and fills the cache again with the warm jobs ExecStart=/bin/systemctl kill --signal=USR1 finbot.service diff --git a/lib/config.py b/lib/config.py index b1e00e38..181f8846 100644 --- a/lib/config.py +++ b/lib/config.py @@ -7,7 +7,7 @@ from pathlib import Path # TO CONFIGURE, DO NOT EDIT THIS FILE. There are two config files: -# finbot.env the startup file: DATA_DIR, ip, port, credentials, telegramChatID, scheduler. Never read by the +# finbot.env the startup file: DATA_DIR, ip, port, credentials, the home chats. Never read by the # bot: the launcher loads it into the environment (systemd EnvironmentFile=, docker --env-file, # init.d `set -a; . finbot.env`). A change needs a restart. # preferences.conf how the bot behaves: /preferences.conf, read by the bot itself at start and on every @@ -15,7 +15,7 @@ REPO_DIR = Path(__file__).resolve().parent.parent -STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', 'sharesight_code', +STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler_catchup', 'sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slackBotToken', 'slackAppToken', 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook', 'discordBotToken', 'discordChannelID', 'discordOutgoingWebhook', 'discordPublicKey') @@ -32,9 +32,8 @@ ' line', 'discord_webhook': 'the scheduled reports go to discordChannelID as the bot (discordBotToken); set those, and' ' delete the line', + 'scheduler': 'the scheduler is always on, and a schedule without jobs runs nothing; delete the line', } -# set by the bot for its job processes: the preferences arrive in the environment, do not read the file -PASSED = 'FINBOT_PREFERENCES_PASSED' OLD_ENV = REPO_DIR / '.env' # the single config file of old versions, which python-dotenv loaded; nothing reads it now @@ -193,7 +192,6 @@ class Env: cache_dir: str # data_dir/cache ip: str port: int - scheduler: bool # run the scheduled reports (schedule.cron) inside bot.py scheduler_catchup: int # minutes: a run missed while the bot was down still fires if it was due this recently telegramChatID: str # '' when unset, like every token below telegramBotToken: str @@ -229,7 +227,6 @@ def startup_values(getenv): cache_dir=data_dir + '/cache', ip=r.str('ip', '127.0.0.1'), port=r.int('port', 5000), - scheduler=r.bool('scheduler', True), scheduler_catchup=r.int('scheduler_catchup', 5), telegramChatID=r.str('telegramChatID', ''), telegramBotToken=r.str('telegramBotToken', ''), @@ -332,7 +329,7 @@ def _fail(e): try: env = startup_values(os.environ.get) preferences_path = env.preferences_path - _preferences = {} if os.environ.get(PASSED) else (read_preferences(preferences_path) or {}) + _preferences = read_preferences(preferences_path) or {} prefs = preference_values(_getenv_for(_preferences), _preferences) except ConfigError as e: _fail(e) @@ -387,16 +384,9 @@ def reload(): return changes -def child_environment(): - """For a job process: the preferences this process runs on, so it never sees a file edited since the last reload.""" - return dict(prefs.source, **{PASSED: '1'}) - - def check_startup_environment(): """For bot.py at start. Preferences must come from preferences.conf, not from the environment that finbot.env fills, and a leftover .env from before the split stops the start: its settings would otherwise be silently ignored.""" - if os.environ.get(PASSED): - return problems = [f'{key} is a preference; move it from finbot.env to {PREFERENCES_FILE}' for key in PREFERENCE_KEYS if key in os.environ] if 'volatile_multiplier' in os.environ: # the old name, from before the split: it never worked as a preference diff --git a/lib/delivery.py b/lib/delivery.py index 98af07fa..b567d130 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -100,25 +100,3 @@ def report_error(e, dest=None, context=None): except Exception as reporting_error: print("Failed to report error:", reporting_error, file=sys.stderr) - -def guarded(context, func, *args, exit=True, **kwargs): - """Run a scheduled entry point; on failure report one line to the default channel, then exit non-zero (exit=True, - for the command line) or just return None (exit=False, when the bot runs it in-process). A SystemExit inside counts - as a failure.""" - try: - return func(*args, **kwargs) - except (Exception, SystemExit) as e: - report_error(e, context=context) - if exit: - sys.exit(1) - - -def cli(script, job, handler, argv=None): - """Command line entry point of a report script: parse its arguments with `job(argv) -> (label, kwargs)`, print the - usage on a bad argument, else run the handler guarded.""" - try: - label, kwargs = job(sys.argv[1:] if argv is None else argv) - except ValueError as e: - print("Usage:", script, e, file=sys.stderr) - sys.exit(1) - guarded(label, handler, **kwargs) diff --git a/lib/jobrun.py b/lib/jobrun.py deleted file mode 100644 index 9c07b9ae..00000000 --- a/lib/jobrun.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Runs one scheduled command in its own process: `python -m lib.jobrun lib.reports. ''`. - -The bot starts this for every scheduled or manual run, so a job's work happens in a process of its own. Failures are -reported to the channel and give the exit status REPORTED.""" -import signal -if __name__ == '__main__': - # a reload is for the bot, but `systemctl kill -s HUP` signals every process of the unit - signal.signal(signal.SIGHUP, signal.SIG_IGN) - # likewise a flush (systemctl kill -s USR1); its default action would kill the job - signal.signal(signal.SIGUSR1, signal.SIG_IGN) -import importlib, json, re, sys - -from lib import delivery - -# exit status of a run that failed and said so in the chat; the bot reports any other non-zero status itself -REPORTED = 3 - - -def main(argv): - """Run one scheduled command: argv is the report's module and its JSON keyword arguments. Returns the exit status: 0 - when it ran, REPORTED when it failed and said so in the chat, 2 for bad arguments.""" - if len(argv) != 2 or not re.fullmatch(r'[a-z][a-z_]*(\.[a-z][a-z_]*)*', argv[0]): - print('usage: python -m lib.jobrun ', file=sys.stderr) - return 2 - module = argv[0] - def run(): - """Import the report and run it: an import that fails is reported like any other failure.""" - importlib.import_module(module).run(**json.loads(argv[1])) - return True - return 0 if delivery.guarded(module.rsplit('.', 1)[-1] + '.py', run, exit=False) else REPORTED - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/lib/jobs.py b/lib/jobs.py index 76b8c567..de6f7081 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -2,16 +2,13 @@ The schedule is a file in crontab syntax: five time fields (or an @macro) and then a finbot script with its arguments, e.g. `*/20 * * * Mon-Fri trades.py` or `28 21 * * Sat cal.py earnings; price.py 7`. Pasted crontab lines -(`./price.py > /dev/null`, `{ a; b ;}`) are accepted too. Running in the bot means the jobs share its user, sandbox, -environment and cache.""" +(`./price.py > /dev/null`, `{ a; b ;}`) are accepted too. Each command runs in the bot's process, in a thread of its +own, sharing its user, sandbox, environment, caches and connections.""" import datetime import hmac -import json import math import os import re -import signal -import subprocess import sys import threading import time @@ -24,7 +21,6 @@ from lib import config from lib import state -from lib import jobrun from lib import delivery from lib.reports import cal, cleanup, milestone, performance, podcasts, price, rating, reminder, shorts, trades from lib.reports import sharesight_warm, yahoo_warm @@ -37,7 +33,7 @@ HIDDEN = ('cleanup', 'podcasts', 'sharesight_warm', 'yahoo_warm') WARM = 'sharesight_warm.py; yahoo_warm.py' # the jobs that fill the cache, run at once after a flush too STATE_FILE = 'finbot_jobs.json' # last fire time of every job, so a restart neither repeats a run nor loses one -JOB_TIMEOUT = 5 * 60 # seconds each command may run before it is stopped and reported +JOB_TIMEOUT = 5 * 60 # seconds a command may run before it is reported as late, once (it runs on, see execute) IGNORED_VARIABLES = ('MAILTO', 'SHELL', 'PATH', 'HOME', 'LOGNAME') # what a shell reads as more than a word, outside quotes OPERATOR = re.compile(r'\d*>>?&?\d*|&>>?|;|&&|\|\|?|&|[<`$(){}]') @@ -49,32 +45,9 @@ def checksum(label): return format(zlib.crc32(label.encode()) & 0xffff, '04x') -def run_child(label, module, kwargs): - """Run a command in its own process (python -m lib.jobrun) and return its exit status. The child inherits the - environment, user and output of the bot, so it behaves like the old cron job. One still running after JOB_TIMEOUT - is stopped and TimeoutError raised; any other interruption stops it too.""" - # the preferences this process runs on, not whatever the file says now - env = dict(os.environ, **config.child_environment()) - args = [sys.executable, '-m', 'lib.jobrun', module.__name__, json.dumps(kwargs)] - proc = subprocess.Popen(args, cwd=config.REPO_DIR, env=env) - try: - return proc.wait(timeout=JOB_TIMEOUT) - except subprocess.TimeoutExpired: - stop(proc) - raise TimeoutError(f'still running after {JOB_TIMEOUT // 60} minutes') from None - except BaseException: - stop(proc) - raise - - -def stop(proc): - """End a child process: asked to stop, then killed if it is still there 10 seconds later.""" - proc.terminate() - try: - proc.wait(timeout=10) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() +def run_in_process(label, module, kwargs): + """Run a command here, in the calling thread: its report's run(**kwargs).""" + module.run(**kwargs) def start_thread(target, *args): @@ -83,9 +56,9 @@ def start_thread(target, *args): def warm_now(scheduler=None): - """Run the warm jobs (WARM) now, one after the other, each in its own child process: through the scheduler when it - runs, so that one already running on its schedule is not started twice; else directly, a failure reported in the - chat as the scheduler reports one.""" + """Run the warm jobs (WARM) now, one after the other: through the scheduler, so that one already running on its + schedule is not started twice; without one (a test, a tool), here, a failure reported in the chat as the scheduler + reports one.""" commands = parse_command(WARM) print('cache: filling it again:', ', '.join(label for label, _, _ in commands), file=sys.stderr) if scheduler: @@ -93,22 +66,12 @@ def warm_now(scheduler=None): else: for label, module, kwargs in commands: try: - run_child(label, module, kwargs) + run_in_process(label, module, kwargs) except Exception as e: delivery.report_error(e, context=label) print('cache: warm jobs done', file=sys.stderr) -def exit_reason(status): - """Why a job process ended, when it did not report it itself: killed by a signal, or failed before it could say.""" - if status < 0: - try: - return f'killed by {signal.Signals(-status).name}' - except ValueError: - return f'killed by signal {-status}' - return f'exited with status {status} without saying why' - - def display(label): """A command as shown in chat: 'price.py close' -> 'price close'.""" return re.sub(r'^(\w+)\.py', r'\1', label) @@ -286,7 +249,7 @@ def __init__(self, path, catchup=None, clock=None, sleep=None, spawn=None, zone= self.clock = clock or (lambda: datetime.datetime.now(datetime.timezone.utc)) self.sleep = sleep or time.sleep self.spawn = spawn or start_thread - self.runner = runner or run_child # (label, module, kwargs) -> exit status + self.runner = runner or run_in_process # (label, module, kwargs): runs the command self.zone = zone self.jobs = [] self.base = {} # key -> the time after which the next run is due @@ -387,9 +350,10 @@ def claim(self, label): return True def execute(self, commands): - """Run commands one after another (a job's), each stopped and reported if still running after JOB_TIMEOUT. - One already running (from the line's last run, another line or chat) is skipped. Returns 'done', 'timed out', - 'failed' or 'already running'.""" + """Run commands one after another (a job's), in this thread; a failure is reported. One already running (from + the line's last run, another line or chat) is skipped. One still running after JOB_TIMEOUT is reported as late, + once, and runs on: a thread cannot be stopped, but every request it makes has a timeout, so it ends; until then + it stays marked running, and no second copy starts. Returns 'done', 'failed' or 'already running'.""" outcome = 'done' for label, module, kwargs in commands: if not self.claim(label): @@ -397,19 +361,17 @@ def execute(self, commands): outcome = 'already running' continue print(f'schedule: running {label}', file=sys.stderr) + late = threading.Timer(JOB_TIMEOUT, delivery.report_error, kwargs={'context': label, + 'e': TimeoutError(f'still running after {JOB_TIMEOUT / 60:g} minutes, and left to finish')}) + late.daemon = True + late.start() try: - status = self.runner(label, module, kwargs) - if status: - outcome = 'failed' - if status != jobrun.REPORTED: # it ended without saying why - delivery.report_error(RuntimeError(exit_reason(status)), context=label) - except TimeoutError as e: # run_child stopped it: the rest of the line still runs - delivery.report_error(e, context=label) - outcome = 'timed out' + self.runner(label, module, kwargs) except (Exception, SystemExit) as e: delivery.report_error(e, context=label) outcome = 'failed' finally: + late.cancel() with self.lock: self.busy.discard(label) return outcome diff --git a/lib/reports/__init__.py b/lib/reports/__init__.py index c62a590f..5ce4f3fe 100644 --- a/lib/reports/__init__.py +++ b/lib/reports/__init__.py @@ -1,16 +1,6 @@ """What finbot says. A module per scheduled report: cal, cleanup (of the cache), milestone, performance, podcasts, price, rating, reminder, shorts, trades, and sharesight_warm and yahoo_warm (which fill the cache), each with job(argv), which parses its schedule line, and run(dest=None, **kwargs), which runs it, to the chat that asked (dest) -or else the default chats; the top-level script of the same name runs it from the command line. And the chat-only -reports: about (.help, .jobs), history (.history, .compare), holdings (.holdings, .who, .pie, .watchlist), profile -(.SYMBOL) and rankings (.marketcap, .pe and the rest).""" -import os -import sys - - -def cli(path): - """Run the report a top-level script is named for (price.py: lib/reports/price.py) with the script's arguments.""" - import importlib - from lib import delivery - report = importlib.import_module('lib.reports.' + os.path.basename(path).removesuffix('.py')) - delivery.cli(sys.argv[0], report.job, report.run) +or else the default chats; the scheduler runs them in the bot's process, and `.jobs run` runs one now. And the +chat-only reports: about (.help, .jobs), history (.history, .compare), holdings (.holdings, .who, .pie, .watchlist), +profile (.SYMBOL) and rankings (.marketcap, .pe and the rest).""" diff --git a/lib/reports/about.py b/lib/reports/about.py index 7cbc520e..ef89bab0 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -11,7 +11,7 @@ def prepare_jobs_payload(service, run=None, tappable=False): command, two to a row.""" scheduler = jobs.current if scheduler is None: - return ['Scheduled jobs are off (scheduler=False)'], None + return ['No schedule is running'], None if run: return [run_job_line(scheduler, run)], None commands, system = scheduler.commands(), scheduler.commands(hidden=True) diff --git a/milestone.py b/milestone.py deleted file mode 100755 index 29829cf3..00000000 --- a/milestone.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./milestone.py [arguments]: runs lib/reports/milestone.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/performance.py b/performance.py deleted file mode 100755 index e58e5d5b..00000000 --- a/performance.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./performance.py [arguments]: runs lib/reports/performance.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/podcasts.py b/podcasts.py deleted file mode 100755 index c86a624e..00000000 --- a/podcasts.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./podcasts.py [arguments]: runs lib/reports/podcasts.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/price.py b/price.py deleted file mode 100755 index 4aee8e91..00000000 --- a/price.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./price.py [arguments]: runs lib/reports/price.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/rating.py b/rating.py deleted file mode 100755 index 4a44d44f..00000000 --- a/rating.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./rating.py [arguments]: runs lib/reports/rating.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/reminder.py b/reminder.py deleted file mode 100755 index 09adbc26..00000000 --- a/reminder.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./reminder.py [arguments]: runs lib/reports/reminder.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/sharesight_warm.py b/sharesight_warm.py deleted file mode 100755 index b04e2850..00000000 --- a/sharesight_warm.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./sharesight_warm.py [arguments]: runs lib/reports/sharesight_warm.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/shorts.py b/shorts.py deleted file mode 100755 index 14347057..00000000 --- a/shorts.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./shorts.py [arguments]: runs lib/reports/shorts.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/tests/common.py b/tests/common.py index 098677a5..2b829675 100644 --- a/tests/common.py +++ b/tests/common.py @@ -25,10 +25,15 @@ REAL_DISPATCH = bot.dispatch # FinbotCase notes the worker thread of each update, so that a test can wait for it +def no_jobs(data): + """An empty schedule in a data dir, so that a bot started there runs no scheduled report.""" + Path(data, 'schedule.cron').write_text('# no jobs: a test bot runs none\n') + + def bare_environment(): - """os.environ without finbot's settings (the startup keys, the preferences, the flag that hands a job its - preferences): for a process that must start from the defaults.""" - settings = {*config.STARTUP_KEYS, *config.PREFERENCE_KEYS, config.PASSED} + """os.environ without finbot's settings (the startup keys and the preferences): for a process that must start from + the defaults.""" + settings = {*config.STARTUP_KEYS, *config.PREFERENCE_KEYS} return {k: v for k, v in os.environ.items() if k not in settings} diff --git a/tests/test_config.py b/tests/test_config.py index 7777147c..9f052993 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,5 @@ """The two config files: finbot.env (the startup file, which the launcher loads) and preferences.conf (reloadable).""" -import ast, contextlib, dataclasses, io, os, re, subprocess, sys, unittest +import ast, contextlib, dataclasses, io, os, re, subprocess, sys, types, unittest from pathlib import Path from unittest import mock from tests import common # first: it sets the environment that lib.config reads as it loads @@ -84,7 +84,7 @@ def test_unknown_key_gets_a_hint_and_a_line_number(self): def test_a_startup_key_in_preferences_says_where_it_belongs(self): """A startup key in preferences.conf is sent to finbot.env, without printing its value.""" - errors = self.errors('slackOutgoingWebhook=https://x\nip=0.0.0.0\ntelegramChatID=5\nscheduler=False\n') + errors = self.errors('slackOutgoingWebhook=https://x\nip=0.0.0.0\ntelegramChatID=5\nscheduler_catchup=3\n') self.assertEqual(len(errors), 4) for line in errors: self.assertIn('belongs in finbot.env (a change needs a restart)', line) @@ -311,38 +311,17 @@ def reads_prefs(node): offenders.append(f'{path.name}:{node.lineno} module constant') self.assertEqual(offenders, []) - def test_a_job_process_runs_on_the_snapshot_not_the_edited_file(self): - """A job process gets the bot's preferences in its environment and does not read the file, which may have been - edited since; a script run by hand reads the file.""" - self.write('future_days=45\n') - config.reload() - env = config.child_environment() - self.assertEqual((env['future_days'], env[config.PASSED]), ('45', '1')) - self.write('future_days=99\n') # edited, no reload sent - data = common.temp_dir(self) - Path(data, 'preferences.conf').write_text('future_days=99\n') - show = "from lib import config as c; print(c.prefs.future_days)" - parent = {k: v for k, v in os.environ.items() if k not in config.PREFERENCE_KEYS and k != config.PASSED} - run = lambda **extra: subprocess.run([sys.executable, '-c', show], cwd=REPO, capture_output=True, text=True, - check=True, env={**parent, 'DATA_DIR': data, **extra}).stdout.split()[-1] - self.assertEqual(run(**env), '45') # the parent's values, and the file is not read - self.assertEqual(run(), '99') # a script run by hand reads the file itself - - def test_run_child_hands_the_snapshot_to_the_job(self): - """run_child passes the preferences in force to the job process.""" + def test_a_scheduled_job_runs_on_the_preferences_in_force_not_the_edited_file(self): + """A scheduled job runs in the bot's process, on the preferences of the last reload: a file edited since, and + not reloaded, is not read.""" from lib import jobs - seen = {} - real = jobs.subprocess.Popen - def popen(args, **kw): - """Note the environment the job is started with, and run a no-op instead.""" - seen.update(kw.get('env', {})) - return real([sys.executable, '-c', 'pass']) + seen = [] self.write('future_days=45\n') config.reload() - with mock.patch.object(jobs.subprocess, 'Popen', popen): - from lib.reports import cleanup - self.assertEqual(jobs.run_child('cleanup.py', cleanup, {}), 0) - self.assertEqual((seen['future_days'], seen[config.PASSED]), ('45', '1')) + self.write('future_days=99\n') # edited, no reload sent + module = types.SimpleNamespace(run=lambda: seen.append(config.prefs.future_days)) + jobs.run_in_process('look.py', module, {}) + self.assertEqual(seen, [45]) class Startup(unittest.TestCase): @@ -398,8 +377,6 @@ def test_a_preference_in_the_environment_is_refused_by_the_bot(self): self.assertEqual(len(caught.exception.errors), 2) self.assertIn('future_days is a preference; move it from finbot.env to preferences.conf', '\n'.join(caught.exception.errors)) - with mock.patch.dict(os.environ, {'future_days': '30', config.PASSED: '1'}): - config.check_startup_environment() # a job process gets its preferences this way with mock.patch.dict(os.environ, {'ip': '1.2.3.4', 'telegramChatID': '5'}): config.check_startup_environment() # startup keys are what the environment is for diff --git a/tests/test_edges.py b/tests/test_edges.py index 57d2b1cb..66bbcd99 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -374,7 +374,8 @@ def blocked(self, data): for name in OPTIONAL: (stubs / f'{name}.py').write_text('raise ImportError("blocked by the test")\n') env = bare_environment() - env.update(DATA_DIR=data, port='0', scheduler='False', PYTHONPATH=str(stubs)) + env.update(DATA_DIR=data, port='0', PYTHONPATH=str(stubs)) + no_jobs(data) return env def test_every_module_imports_without_them(self): @@ -414,8 +415,9 @@ def test_sighup_during_startup_is_ignored_and_works_afterwards(self): """A SIGHUP while bot.py starts is ignored, where its default action would kill it; once serving, it reloads.""" skip_if_old_env(self) with tempfile.TemporaryDirectory() as data: + no_jobs(data) env = bare_environment() - env.update(DATA_DIR=data, port='0', scheduler='False') + env.update(DATA_DIR=data, port='0') proc = subprocess.Popen([sys.executable, '-u', 'bot.py'], cwd=ROOT, env=env, stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True) self.addCleanup(proc.wait) @@ -451,8 +453,9 @@ def test_usr1_flushes_the_cache_and_keeps_the_state(self): (cache / name).write_text('{}') Path(data, 'finbot_watchlist.json').write_text('["T1"]') # state, not cache Path(data, 'preferences.conf').write_text('cache=False\n') # the warm jobs a flush starts fetch nothing + no_jobs(data) env = bare_environment() - env.update(DATA_DIR=data, port='0', scheduler='False') + env.update(DATA_DIR=data, port='0') proc = subprocess.Popen([sys.executable, '-u', 'bot.py'], cwd=ROOT, env=env, stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True) self.addCleanup(proc.wait) diff --git a/tests/test_finbot.py b/tests/test_finbot.py index ff4a9e5f..fe13515d 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -1,5 +1,5 @@ """Plumbing: message chunking, timezones, the WSGI handler.""" -import ast, contextlib, datetime, io, json, subprocess, tempfile, unittest, zoneinfo +import ast, contextlib, datetime, io, json, re, subprocess, tempfile, unittest, zoneinfo from pathlib import Path from unittest import mock from tests.common import * @@ -101,6 +101,22 @@ def test_the_bot_and_its_reports_print_only_to_stderr(self): self.assertEqual(offenders, []) +class Timeouts(unittest.TestCase): + def test_every_request_goes_through_net_with_a_timeout(self): + """The scheduled reports run in the bot's threads, which cannot be stopped: every request must end. So no module + but lib/net.py calls requests itself (net gives each call its timeouts), and none passes timeout=None.""" + call = re.compile(r'\brequests\.(?:get|post|put|patch|delete|head|request|Session)\(') + offenders = [] + for path in sorted([*Path(ROOT, 'lib').rglob('*.py'), Path(ROOT, 'bot.py'), *Path(ROOT, 'util').glob('*.py')]): + for number, line in enumerate(path.read_text().splitlines(), 1): + where = f'{path.relative_to(ROOT)}:{number}' + if call.search(line) and path.name != 'net.py': + offenders.append(f'{where} calls requests itself') + if re.search(r'\btimeout\s*=\s*None\b', line): + offenders.append(f'{where} passes timeout=None') + self.assertEqual(offenders, []) + + class OutputComparison(unittest.TestCase): def test_steps_are_compared_by_name_and_a_repeated_one_by_its_count(self): """util/record_outputs.py --diff shows a step that changed, appeared or went; a step run twice is told apart.""" diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 884bdc33..fe59dbaf 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -1,5 +1,5 @@ """The schedule file (lib/jobs.py): its lines parsed, each script's job(argv), and a schedule the bot refuses.""" -import hashlib, io, os, subprocess, sys, tempfile, unittest +import hashlib, os, subprocess, sys, tempfile, unittest from pathlib import Path from unittest import mock from tests.common import * @@ -146,31 +146,10 @@ def test_plain_scripts(self): if name not in ('cal', 'cleanup', 'performance', 'podcasts', 'price'): self.assertEqual(module.job([]), (name + '.py', {})) - def test_cli_prints_usage_and_exits(self): - """A bad argument on the command line prints the usage and exits 1.""" - with mock.patch('sys.stderr', new_callable=io.StringIO) as err, self.assertRaises(SystemExit) as exit_: - delivery.cli('price.py', price.job, lambda **k: None, ['bogus']) - self.assertEqual(exit_.exception.code, 1) - self.assertIn('Usage: price.py [midsession', err.getvalue()) - - def test_cli_runs_the_handler(self): - """Good arguments run the report with what job() made of them.""" - calls = [] - delivery.cli('price.py', price.job, lambda **k: calls.append(k), ['midsession']) - self.assertEqual(calls, [{'mode': 'midsession'}]) - - def test_each_scheduled_report_runs_from_its_top_level_script(self): - """./price.py and the rest run lib/reports/.py, so cron lines and runs by hand keep working.""" - self.assertEqual(sorted(path.stem for path in Path(ROOT).glob('*.py') if path.stem not in ('bot', - 'bot_send_stdin')), - sorted(jobs.SCRIPTS)) - with tempfile.TemporaryDirectory() as data: - run = lambda *args: subprocess.run([sys.executable, *args], cwd=ROOT, capture_output=True, text=True, - timeout=60, env={**os.environ, 'DATA_DIR': data}) - self.assertEqual(run('cleanup.py', '0').returncode, 0) - bad = run('price.py', 'bogus') - self.assertEqual(bad.returncode, 1) - self.assertIn('Usage: price.py [midsession', bad.stderr) + def test_the_checkout_holds_only_the_bot_at_its_top(self): + """The scheduled reports run in the bot (and `.jobs run`), so the checkout's top holds bot.py alone: the scripts + that ran each report from cron are gone.""" + self.assertEqual(sorted(path.name for path in Path(ROOT).glob('*.py')), ['bot.py']) class StartupRefusal(unittest.TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ba3122e6..7abe368d 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,6 +1,6 @@ -"""The in-process scheduler (lib/jobs.py): firing on a fake clock, across the clocks' changes, runs from chat, -and each command in a child process.""" -import contextlib, datetime, io, json, os, signal, subprocess, sys, tempfile, threading, time, unittest +"""The scheduler (lib/jobs.py): firing on a fake clock, across the clocks' changes, runs from chat, and each command +run in the bot's process, past its deadline too.""" +import contextlib, datetime, io, json, os, subprocess, sys, tempfile, threading, time, unittest import zoneinfo from pathlib import Path from unittest import mock @@ -11,12 +11,6 @@ PREFETCH_HISTORY = yahoo.prefetch_history # FinbotCase replaces it; the warm job's test puts it back -def run_in_process(label, module, kwargs): - """The scheduler's runner in these tests: the command's handler, called here (no child process, no deadline).""" - module.run(**kwargs) - return 0 - - def at(day, hour, minute, second=0): """A UTC time in October 2026 (the 13th is a Tuesday).""" return datetime.datetime(2026, 10, day, hour, minute, second, tzinfo=UTC) @@ -141,25 +135,6 @@ def down(): self.assertEqual(self.errors, []) -class SchedulerFlag(unittest.TestCase): - def flag(self, **env): - """config.env.scheduler in a fresh interpreter, given `env`.""" - code = 'from lib import config; print(config.env.scheduler)' - e = {k: v for k, v in os.environ.items() if k != 'scheduler'} - e.update(env) - repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - return subprocess.run([sys.executable, '-c', code], cwd=repo, env=e, capture_output=True, text=True, - check=True).stdout.split()[-1] - - def test_on_by_default_and_off_for_false(self): - """The scheduler is on by default and with any true word, off with any false one.""" - self.assertEqual(self.flag(), 'True') - for value in ('True', 'true', '1', 't'): - self.assertEqual(self.flag(scheduler=value), 'True', value) - for value in ('False', 'false', '0', 'no'): - self.assertEqual(self.flag(scheduler=value), 'False', value) - - class SchedulerBase(FinbotCase): def setUp(self): """A schedule of trades.py every 20 minutes on weekdays, a fake clock; trades and shorts recorded, not run.""" @@ -176,7 +151,7 @@ def setUp(self): def scheduler(self, catchup=5): """A loaded scheduler on the fake clock that runs commands in this process, at once.""" s = jobs.Scheduler(lambda: self.file, catchup=catchup, clock=self.clock, sleep=lambda seconds: None, - spawn=lambda f, *a: f(*a), zone=UTC, runner=run_in_process) + spawn=lambda f, *a: f(*a), zone=UTC) self.assertTrue(s.load()) return s @@ -195,20 +170,19 @@ def test_the_warm_jobs_run_through_the_scheduler(self): self.assertEqual(warmed, ['sharesight', 'yahoo', 'sharesight']) self.assertIn('cache: filling it again: sharesight_warm.py, yahoo_warm.py', log.getvalue()) - def test_without_a_scheduler_each_runs_in_a_child_process(self): - """With the scheduler off, each warm job runs in a child process of its own, a failure reported.""" + def test_without_a_scheduler_each_runs_here(self): + """Without a scheduler (a test, a tool), each warm job runs here, one after the other, a failure reported.""" started = [] - def child(label, module, kwargs): - """Note the child process started; the second one runs out of time.""" + def run(label, module, kwargs): + """Note the run; the first one fails.""" started.append(label) - if label == 'yahoo_warm.py': - raise TimeoutError('still running after 5 minutes') - return 0 - self.patch(jobs, 'run_child', child) + if label == 'sharesight_warm.py': + raise RuntimeError('Sharesight unreachable') + self.patch(jobs, 'run_in_process', run) with contextlib.redirect_stderr(io.StringIO()): jobs.warm_now() self.assertEqual(started, ['sharesight_warm.py', 'yahoo_warm.py']) - self.assertTrue(any('still running after 5 minutes' in e for e in self.errors)) + self.assertEqual(self.errors, ['⚠️ sharesight_warm.py: RuntimeError: Sharesight unreachable']) class SchedulerTests(SchedulerBase): @@ -297,21 +271,40 @@ def quit_(**k): s.execute(s.jobs[0].commands) self.assertEqual(len(self.errors), 1) - def test_a_command_stopped_at_its_deadline_is_reported_and_the_next_still_runs(self): - """A command still running at its deadline (run_child stops it) is reported, and the rest of its line runs.""" + def test_a_command_past_its_deadline_is_reported_once_and_left_to_finish(self): + """A command still running at its deadline is reported as late, once, and runs on to its end (a thread cannot + be stopped); meanwhile it stays marked running, so a second run of it is turned away; then the rest of its line + runs.""" + self.patch(jobs, 'JOB_TIMEOUT', 0.1) self.file.write_text('*/20 * * * * trades.py; shorts.py\n') - def runner(label, module, kwargs): - """trades.py hangs until run_child stops it; the rest run here.""" - if label == 'trades.py': - raise TimeoutError('still running after 5 minutes') - return run_in_process(label, module, kwargs) s = self.scheduler() - s.runner = runner - self.assertEqual(s.execute(s.jobs[0].commands), 'timed out') - self.assertEqual(self.errors, ['⚠️ trades.py: TimeoutError: still running after 5 minutes']) - self.assertEqual(self.ran, [('shorts', {})]) + second = [] + def slow(**k): + """Take three deadlines, and try a second run of this command meanwhile.""" + time.sleep(0.3) + second.append(s.execute(s.jobs[0].commands[:1])) + self.ran.append(('trades', k)) + self.patch(jobs.SCRIPTS['trades'], 'run', slow) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(s.execute(s.jobs[0].commands), 'done') + self.assertEqual(self.errors, ['⚠️ trades.py: TimeoutError: still running after 0.00166667 minutes, and left to' + ' finish']) + self.assertEqual(second, ['already running']) + self.assertEqual(self.ran, [('trades', {}), ('shorts', {})]) self.assertEqual(s.busy, set()) + def test_each_command_of_a_line_has_a_deadline_of_its_own(self): + """The deadline is per command (it once covered the whole line): two that each take most of it are not late.""" + self.patch(jobs, 'JOB_TIMEOUT', 0.4) + self.file.write_text('*/20 * * * * trades.py; shorts.py\n') + self.patch(jobs.SCRIPTS['trades'], 'run', lambda **k: time.sleep(0.25)) + self.patch(jobs.SCRIPTS['shorts'], 'run', lambda **k: time.sleep(0.25)) + s = self.scheduler() + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(s.execute(s.jobs[0].commands), 'done') + time.sleep(0.3) # past where a deadline of the whole line would have struck + self.assertEqual(self.errors, []) + def test_reload_applies_edits_keeps_state_and_survives_a_bad_file(self): """A reload applies edits and keeps unchanged jobs' timing; an unusable file keeps the running schedule.""" s = self.scheduler() @@ -490,13 +483,13 @@ def test_a_scheduled_run_skips_a_command_that_a_manual_run_is_in_the_middle_of(s self.assertEqual(self.s.execute(self.s.jobs[1].commands), 'already running') self.assertEqual(self.ran, [('price', {'mode': 'close'})]) # price.py close ran, trades.py was skipped - def test_run_now_times_out(self): - """A manual run stopped at its deadline says so.""" - def hang(label, module, kwargs): - """Hang until run_child stops it.""" - raise TimeoutError('still running after 5 minutes') - self.s.runner = hang - self.assertEqual(self.s.run_now('8'), ('trades.py', 'timed out')) + def test_a_manual_run_past_its_deadline_finishes(self): + """A manual run past its deadline is reported as late, and says how it went once it has finished.""" + self.patch(jobs, 'JOB_TIMEOUT', 0.05) + self.patch(jobs.SCRIPTS['trades'], 'run', lambda **k: time.sleep(0.2)) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(self.s.run_now('8'), ('trades.py', 'done')) + self.assertEqual(len(self.errors), 1) self.assertEqual(self.s.busy, set()) def test_two_runs_of_one_command_at_once_run_it_once(self): @@ -642,7 +635,7 @@ def test_button_from_an_old_schedule_updates_the_message(self): self.assertEqual(markup, about.prepare_jobs_payload(TELEGRAM, '', tappable=True)[1]) # the current buttons self.patch(jobs, 'current', None) # scheduler off: the list is replaced by the notice and the buttons removed worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|7|0000')) - self.assertEqual(self.edits[-1][2:], ('Scheduled jobs are off (scheduler=False)', [])) # no buttons left + self.assertEqual(self.edits[-1][2:], ('No schedule is running', [])) # no buttons left def test_run_in_chat(self): """.jobs run takes a number or a command's text.""" @@ -661,92 +654,22 @@ def test_run_in_chat_refusals(self): self.assertIn('Usage', self.list_in_chat('.jobs bogus')[0]) self.assertEqual(self.ran, []) - def test_scheduler_off(self): - """With the scheduler off, .jobs says so.""" + def test_no_schedule_running(self): + """Without a schedule running (none was loaded), .jobs says so.""" self.patch(jobs, 'current', None) - self.assertIn('off', self.list_in_chat('.jobs')[0]) - self.assertIn('off', self.list_in_chat('.jobs run 8')[0]) + self.assertEqual(self.list_in_chat('.jobs'), ['No schedule is running']) + self.assertEqual(self.list_in_chat('.jobs run 8'), ['No schedule is running']) def test_job_is_still_a_ticker(self): """.job is a ticker, not .jobs.""" self.assertNotIn('Scheduled', str(self.command('.job'))) -class ChildProcess(FinbotCase): - """Jobs run in their own process, so heavy work cannot hold up the bot's web server.""" - - def sleeper(self, seconds): - """Have run_child start a process that sleeps for `seconds`, kept in self.children.""" - real = jobs.subprocess.Popen - self.children = [] - def popen(args, **kw): - """Start the sleeper instead of the job.""" - self.children.append(real([sys.executable, '-c', f'import time; time.sleep({seconds})'])) - return self.children[-1] - self.patch(jobs.subprocess, 'Popen', popen) - - def test_every_command_can_be_handed_to_a_child(self): - """Every shipped command's arguments survive JSON, the way they reach the child.""" - parsed, _ = jobs.parse_schedule(DEFAULT_SCHEDULE.read_text(), UTC) - for job in parsed: - for label, module, kwargs in job.commands: - self.assertEqual(json.loads(json.dumps(kwargs)), kwargs, label) - - def test_a_real_child_runs_the_handler_and_reports_its_exit_status(self): - """A real child runs the report and exits 0, or REPORTED when it failed and said so.""" - real = jobs.subprocess.Popen - self.patch(jobs.subprocess, 'Popen', - lambda args, **kw: real(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, **kw)) - with tempfile.TemporaryDirectory() as data: - env = {'DATA_DIR': data, 'telegramBotToken': '', 'slack_webhook': '', 'discord_webhook': ''} - with mock.patch.dict(os.environ, env): - self.assertEqual(jobs.run_child('cleanup.py', cleanup, {}), 0) - self.assertEqual(jobs.run_child('cleanup.py 0', cleanup, {'days': 'not a number'}), - jobs.jobrun.REPORTED) - - def test_a_job_process_survives_a_reload_signal(self): - """A job process ignores SIGHUP, which systemctl kill -s HUP sends to every process of the unit.""" - with tempfile.TemporaryDirectory() as scripts, tempfile.TemporaryDirectory() as data: - Path(scripts, 'sleeper.py').write_text( - 'import time\ndef run():\n print("started", flush=True)\n time.sleep(1)\n') - env = {**os.environ, 'PYTHONPATH': scripts, 'DATA_DIR': data, 'telegramBotToken': '', 'slack_webhook': '', - 'discord_webhook': ''} - proc = subprocess.Popen([sys.executable, '-m', 'lib.jobrun', 'sleeper', '{}'], cwd=ROOT, env=env, - stdout=subprocess.PIPE, text=True) - self.addCleanup(proc.kill) - for line in proc.stdout: - if line.strip() == 'started': - break - proc.send_signal(signal.SIGHUP) # what `systemctl kill -s HUP finbot` does to every process of the unit - self.assertEqual(proc.wait(timeout=30), 0) - - def test_a_script_that_cannot_be_imported_is_reported(self): - """A report that cannot be imported is reported like any other failure.""" - with tempfile.TemporaryDirectory() as data: - env = {**os.environ, 'DATA_DIR': data, 'telegramBotToken': '', 'slack_webhook': '', 'discord_webhook': ''} - out = subprocess.run([sys.executable, '-m', 'lib.jobrun', 'nosuchscript', '{}'], cwd=ROOT, env=env, - capture_output=True, text=True) - self.assertEqual(out.returncode, jobs.jobrun.REPORTED, out.stderr) - self.assertIn('⚠️ nosuchscript.py: ModuleNotFoundError', out.stderr) - - def test_scheduler_reports_a_child_that_ended_without_saying_why(self): - """A child killed by a signal or exiting non-zero without a report is reported by the scheduler; one that - reported itself is not reported twice.""" - file = Path(temp_dir(self), 'schedule.cron') - file.write_text('* * * * * trades.py\n') - for status, reason in ((-signal.SIGHUP, 'killed by SIGHUP'), (-signal.SIGKILL, 'killed by SIGKILL'), - (1, 'exited with status 1 without saying why'), - (jobs.jobrun.REPORTED, None)): # the child reported this one itself - with self.subTest(status=status): - self.errors.clear() - s = jobs.Scheduler(lambda: file, zone=UTC, runner=lambda label, module, kwargs: status) - self.assertTrue(s.load()) - self.assertEqual(s.run_now('1'), ('trades.py', 'failed')) - self.assertEqual(self.errors, [f'⚠️ trades.py: RuntimeError: {reason}'] if reason else []) - self.assertEqual(s.busy, set()) +class ReportImports(FinbotCase): + """What importing the reports brings in.""" def test_a_report_script_loads_neither_pandas_nor_the_scheduler(self): - """A job imports only its script; price once pulled in pandas and every script (via charts, reports, jobs).""" + """A report imports only its script: price once pulled in pandas and every script (via charts and jobs).""" code = ('import sys; from lib.reports import price, performance, trades, shorts, cal; ' 'print(sorted(m for m in ("pandas", "lib.jobs", "matplotlib") if m in sys.modules))') with tempfile.TemporaryDirectory() as data: @@ -754,49 +677,6 @@ def test_a_report_script_loads_neither_pandas_nor_the_scheduler(self): env={**os.environ, 'DATA_DIR': data}) self.assertEqual(out.stdout.strip().splitlines()[-1:], ['[]'], out.stderr) - def test_a_child_still_running_at_its_deadline_is_stopped(self): - """A child still running after JOB_TIMEOUT is stopped, rather than left running, and TimeoutError raised.""" - self.patch(jobs, 'JOB_TIMEOUT', 0.3) - self.sleeper(30) - started = time.monotonic() - with self.assertRaises(TimeoutError): - jobs.run_child('cleanup.py', cleanup, {}) - self.assertLess(time.monotonic() - started, 5) - self.assertIsNotNone(self.children[0].poll()) # the child was stopped, not left running - - def test_another_interruption_stops_the_child_too(self): - """An interruption while a child runs (a KeyboardInterrupt, say) stops the child as well.""" - self.sleeper(30) - start = jobs.subprocess.Popen # the sleeper's - def interrupted(args, **kw): - """The sleeper, whose wait for the job is interrupted.""" - child = start(args, **kw) - wait = child.wait - def wait_once(timeout=None): - """Interrupted while waiting for the job; stop()'s own waits go through.""" - if timeout == jobs.JOB_TIMEOUT: - raise KeyboardInterrupt - return wait(timeout) - child.wait = wait_once - return child - self.patch(jobs.subprocess, 'Popen', interrupted) - with self.assertRaises(KeyboardInterrupt): - jobs.run_child('cleanup.py', cleanup, {}) - self.assertIsNotNone(self.children[0].poll()) - - def test_each_command_of_a_line_has_a_deadline_of_its_own(self): - """The deadline is per command (it once covered the whole line): two that each take most of it both finish.""" - self.patch(jobs, 'JOB_TIMEOUT', 0.8) - self.sleeper(0.5) - file = Path(temp_dir(self), 'schedule.cron') - file.write_text('* * * * * trades.py; shorts.py\n') - s = jobs.Scheduler(lambda: file, zone=UTC) - with contextlib.redirect_stderr(io.StringIO()): - self.assertTrue(s.load()) - self.assertEqual(s.execute(s.jobs[0].commands), 'done') - self.assertEqual(len(self.children), 2) - self.assertEqual(self.errors, []) - if __name__ == '__main__': unittest.main() diff --git a/tests/test_sending.py b/tests/test_sending.py index 1ddaabf1..db33ca85 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -84,16 +84,6 @@ def test_report_error_never_raises(self): REAL_REPORT_ERROR(ValueError('boom'), transports.TelegramBot('5')) REAL_REPORT_ERROR(ValueError('boom')) - def test_guarded_reports_and_exits_nonzero(self): - """guarded reports a failure and exits 1; a success returns its value.""" - reported = [] - self.patch(delivery, 'report_error', lambda e, **k: reported.append(e)) - with self.assertRaises(SystemExit) as raised: - delivery.guarded('job', lambda: 1 / 0) - self.assertEqual(raised.exception.code, 1) - self.assertEqual(len(reported), 1) - self.assertEqual(delivery.guarded('job', lambda: 'fine'), 'fine') - class ChartDelivery(FinbotCase): """Charts reach Telegram and Slack replies (uploaded); a report whose chart fails still goes, as text.""" diff --git a/tests/test_telegram_api.py b/tests/test_telegram_api.py index 0300e612..27691312 100644 --- a/tests/test_telegram_api.py +++ b/tests/test_telegram_api.py @@ -77,7 +77,7 @@ def test_importing_never_calls_telegram(self): code = ('import requests\n' 'def refuse(*a, **k): raise SystemExit("network call on import")\n' 'requests.Session.request = refuse\n' - 'from lib import telegram, yahoo, worker, jobrun\n' + 'from lib import telegram, yahoo, worker, jobs\n' 'print("quiet")') with tempfile.TemporaryDirectory() as data: out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True, @@ -90,7 +90,8 @@ def test_the_bot_will_not_start_unregistered_and_never_logs_the_token(self): skip_if_old_env(self) with tempfile.TemporaryDirectory() as data: env = {k: v for k, v in bare_environment().items() if 'proxy' not in k.lower()} - env.update(DATA_DIR=data, port='0', scheduler='False', telegramBotToken='123456:SECRET-token', + no_jobs(data) + env.update(DATA_DIR=data, port='0', telegramBotToken='123456:SECRET-token', telegramOutgoingToken='tok', telegramOutgoingWebhook='https://example.com/telegram', # unreachable, so the registration fails at once and offline diff --git a/trades.py b/trades.py deleted file mode 100755 index d9f3cc9f..00000000 --- a/trades.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./trades.py [arguments]: runs lib/reports/trades.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/util/record_outputs.py b/util/record_outputs.py index 503c2118..ba22947a 100755 --- a/util/record_outputs.py +++ b/util/record_outputs.py @@ -10,7 +10,7 @@ --keep DIR keeps the two recordings; --full shows changed lines whole. The exit status is 0 when the recordings match, 1 when they differ.""" -import argparse, difflib, hashlib, inspect, io, json, os, random, subprocess, sys, tempfile, unittest +import argparse, contextlib, difflib, hashlib, inspect, io, json, os, random, subprocess, sys, tempfile, unittest from pathlib import Path REPO = Path(__file__).resolve().parent.parent @@ -106,6 +106,10 @@ def post(url, method='POST', **kwargs): self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') # Slack's buttons too if 'discordBotToken' in {field.name for field in common.dataclasses.fields(common.config.env)}: self.env(discordBotToken='discord-test') + schedule = jobs.Scheduler(lambda: REPO / 'defaults' / 'schedule.cron', zone=common.UTC) # for .jobs + with contextlib.redirect_stderr(io.StringIO()): + schedule.load() + self.patch(jobs, 'current', schedule) steps = [] def step(label, run): """Run one step and note what it posted, or what it raised.""" diff --git a/util/send_stdin.py b/util/send_stdin.py new file mode 100755 index 00000000..b167a5e3 --- /dev/null +++ b/util/send_stdin.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Post the lines piped in to the home chats, as the scheduled reports go: `echo hello | util/send_stdin.py`.""" +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from lib import delivery + + +if __name__ == '__main__': + if os.isatty(sys.stdin.fileno()): + sys.exit('send_stdin: pipe the lines to post into it, e.g. echo hello | util/send_stdin.py') + lines = [line.strip() for line in sys.stdin if line.strip()] + for dest in delivery.default_chats(): + delivery.deliver(dest, delivery.Report(lines)) diff --git a/yahoo_warm.py b/yahoo_warm.py deleted file mode 100755 index ed868699..00000000 --- a/yahoo_warm.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./yahoo_warm.py [arguments]: runs lib/reports/yahoo_warm.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) From 2c595ad9690c007694b7f04a1dec25fed9b50cbc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:50:58 +0000 Subject: [PATCH 08/31] Jobs per chat, managed with .jobs; schedule.cron migrated once Each chat (a Telegram chat, Slack channel or Discord channel, by its network and id, which a mode switch does not change) has its own jobs, kept in finbot_schedule.json and each posting to its own chat: - .jobs lists the chat's jobs with their next runs (UTC and the timezone preference), then the fixed system jobs (sharesight_warm, yahoo_warm, cleanup), then the subcommands; it has no buttons. - .jobs run / .jobs toggle list them with a button per job that runs it now or switches it on or off (redrawn in place); with a number they act directly. A stale or forged button redraws the list and does nothing else. - .jobs add [CRON_TZ=ZONE] TIME REPORT adds one (alone, it lists the reports and what each takes); .jobs del N deletes one. Up to 30 per chat; a time that never comes, an unknown report or a bad argument is refused, saying why. - Times are in UTC unless a job names its zone; a change counts at once. trades, rating, milestone and podcasts keep a record per chat of what they told it, so the same report scheduled in two chats tells each of them once. As finbot starts, lib/migrate.py gives each home chat, once, the jobs of the old schedule.cron (the data dir's copy, else the shipped one) in the zone they ran in, and copies the reports' shared records to each home chat. With no home chat set it waits. SIGHUP no longer reloads a schedule; report labels lose .py. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 25 +- README.md | 146 ++++---- bot.py | 20 +- defaults/finbot.env | 7 +- defaults/preferences.conf | 2 +- defaults/schedule.cron | 11 +- doc/discord.md | 14 +- doc/slack.md | 11 +- doc/telegram.md | 10 +- finbot.service | 2 +- lib/buttons.py | 33 +- lib/config.py | 8 +- lib/delivery.py | 18 + lib/formats.py | 11 + lib/jobs.py | 531 +++++++++++++-------------- lib/migrate.py | 188 ++++++++++ lib/reports/about.py | 77 ++-- lib/reports/cal.py | 8 +- lib/reports/cleanup.py | 10 +- lib/reports/milestone.py | 40 +- lib/reports/performance.py | 10 +- lib/reports/podcasts.py | 43 ++- lib/reports/price.py | 10 +- lib/reports/rating.py | 55 +-- lib/reports/reminder.py | 11 +- lib/reports/sharesight_warm.py | 13 +- lib/reports/shorts.py | 11 +- lib/reports/trades.py | 36 +- lib/reports/yahoo_warm.py | 13 +- lib/state.py | 19 +- lib/transports.py | 7 + lib/worker.py | 73 +++- tests/common.py | 4 +- tests/test_config.py | 21 +- tests/test_edges.py | 2 +- tests/test_reports.py | 41 ++- tests/test_schedule.py | 329 ++++++++++------- tests/test_scheduler.py | 651 ++++++++++++++++----------------- tests/test_sending.py | 22 +- util/record_outputs.py | 27 +- util/sync_defaults.py | 4 +- 41 files changed, 1499 insertions(+), 1075 deletions(-) create mode 100644 lib/migrate.py diff --git a/CLAUDE.md b/CLAUDE.md index 6fc31f2e..1f8b42b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,13 +55,15 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm percent, a period and a keyword from either end, the words between; a handler raises `Usage` for arguments it cannot take, and gets a ticker with `ticker(args)`. `Request` (`request.reply(lines, chart, buttons)`); each command gets the typing notice while it works. `process_request(message)` and `process_callback(press)` take inbound's objects. -- `lib/reports/`: what finbot says. Scheduled (each has `run(dest=None, **kwargs)`, which posts to the chat that asked - or else the default chats, and `job(argv)`, its schedule-line parser): `price performance trades shorts cal rating - milestone reminder podcasts cleanup sharesight_warm yahoo_warm`, run in the bot by the scheduler or `.jobs run`. +- `lib/reports/`: what finbot says. Scheduled (each has `run(dest=None, **kwargs)`, which posts to the chat that asked, + the job's chat or else the home chats; `job(argv)`, its arguments' parser; and `ARGUMENTS`, what `.jobs add` lists): + `price performance trades shorts cal rating milestone reminder podcasts cleanup sharesight_warm yahoo_warm`. trades, + rating, milestone and podcasts keep a record per chat of what they told it (`state.load_for`, `state.edit_for`), + added to only by a scheduled run (`delivery.interactive(dest)` is false). Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). -- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or each network's home chat - as its bot), `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per network's bot +- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, a chat by its key with + `chat(key)`, or each network's home chat as its bot), `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per network's bot (`TelegramBot`, `SlackBot`, `DiscordBot`), each taking charts and buttons: splitting, captions, charts, buttons, edits, and the typing notice (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop before posting. `DiscordBot`: a slash command's first message replaces its "thinking…" through its @@ -82,10 +84,13 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm `Portfolio` objects: `holdings()`, `tickers()`, `gain(days)` (the portfolio's), `trades(days)`. - `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; change one with `with state.edit(name, default) as data:`, never load-then-save. `lib/watchlist.py`: the watchlist. + A chat's key is its network and id (`Transport.key`, `telegram:-1001234`), the same in both modes. - `lib/config.py`: `config.env` (finbot.env, fixed at start) and `config.prefs` (preferences.conf, a new object on - each reload). `lib/jobs.py`: the scheduler, its shared state under one lock; each command runs in the bot's process, - in a thread (`run_in_process`), reported once if still running after `JOB_TIMEOUT` and left to finish (every request - goes through `net`, with its timeouts: a test checks). + each reload). `lib/jobs.py`: the scheduler: each chat's jobs (`SCHEDULE_FILE`, by chat key; `add`, `delete`, + `toggle`, `run`, which `.jobs` calls through `jobs.current`) and the fixed `SYSTEM` jobs, its shared state under one + lock; each command runs in the bot's process, in a thread (`run_in_process`), reported once if still running after + `JOB_TIMEOUT` and left to finish (every request goes through `net`, with its timeouts: a test checks). + `lib/migrate.py`: once, the old schedule.cron and the reports' shared records become the home chats'. - `tests/common.py`: `FinbotCase` (offline fixtures). `self.command('.price')` returns what was sent: `('text', lines)` or `('photo', size, caption)`. `wsgi_environ(path, body, **headers)` is a request for `bot.main`. `ProviderCase` runs the real parsers on the answers saved under `doc/` (`sample`), as `Resp` and `serve` return @@ -94,8 +99,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - Which test file: commands and scheduled reports `test_reports`; failures, bad input, secrets in logs, signals `test_edges`; webhook requests, Slack events and signatures `test_inbound`; the pull modes `test_pull`; the Discord bot `test_discord`; Yahoo `test_yahoo`; Sharesight `test_sharesight`; posting to each network `test_sending`; - registering Telegram's webhook `test_telegram_api`; the schedule file and job arguments `test_schedule`; the scheduler - and manual runs `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); + registering Telegram's webhook `test_telegram_api`; job lines, job arguments and the migration `test_schedule`; the + scheduler and `.jobs` in a chat `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, and the docstring, stdout and line-width checks `test_finbot`. diff --git a/README.md b/README.md index 0e0afaaa..5f3ed549 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,9 @@ Where: .forward pe [symbol|top|bottom|negative] .history symbol .holdings portfolio -.jobs [run number|command] +.jobs [run|toggle] [number] +.jobs add [CRON_TZ=zone] time report +.jobs del number .marketcap [symbol|top|bottom] .pe [symbol|top|bottom] .peg [symbol|top|bottom|negative] @@ -93,9 +95,9 @@ Where: `!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command, as Discord registers them: `/stock symbol` (or `/profile symbol`) for `.symbol`, `/forwardpe` for `.forward pe`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. -Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying your configured chat networks of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. +Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying each chat that schedules them of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. -The other various reports can also run on a schedule (e.g. daily or weekly), or on demand through the interactive bot. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. Depending on how they're triggered, they will either report to all configured chat networks, or reply to the chat which triggered them. +The other various reports can also run on a schedule (e.g. daily or weekly), or on demand through the interactive bot. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. A scheduled report posts to the chat whose job it is, and a command's reply to the chat that sent it. The interactive bot answers commands in your chats. It fetches its updates from Telegram and Slack itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: * Stock lookup (financials, company profile, price charts) @@ -148,7 +150,7 @@ This report can only be run through the interactive bot. Example usage: ### Trades ![trade update in Slack](img/trades.png?raw=true "Trade update in Slack") -`trades.py` sends recent Sharesight trades to your configured chat services. +The `trades` report sends recent Sharesight trades to each chat that schedules it. By default, this report searches Sharesight for trades from the past 28 days (`past_days`). You can override this in `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Without persistent storage, set `past_days=0` to avoid duplicate trade notifications. In this case, the schedule must run the report exactly once per day, after market close. @@ -159,9 +161,9 @@ With persistent storage, the default of `past_days=28` suits. This is useful if past_days=28 ``` -Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): +Job (see [Scheduling](#scheduling)), such as `.jobs add */20 * * * Mon-Fri trades`: ``` -trades.py +trades ``` Interactive trigger: @@ -184,7 +186,7 @@ You can also specify a portfolio name to get trades for just that portfolio: ![Price chart triggered from Slack](img/finbot_price.png?raw=true "Price chart triggered from Slack") ![Price alert triggered by Cron](img/price.png?raw=true "Price alert triggered by Cron") -`price.py` sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. +The `price` report sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. @@ -194,9 +196,9 @@ price_percent=5 demote_volatile=True ``` -Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): +Job (see [Scheduling](#scheduling)), such as `.jobs add 29 21 * * Mon-Fri price interday`: ``` -price.py [premarket|midsession|interday|close|days (int)] +price premarket|midsession|interday|close|DAYS ``` When scheduled, the mode is its argument: * If `premarket` is passed, it only reports on pre/post market price movements. @@ -242,16 +244,16 @@ Or a specific stock. In this case, the output includes a graph. ### Portfolio performance ![Screenshot of Portfolio performance notification on Slack](img/performance.png?raw=true "Screenshot performance notification on Slack") -`performance.py` reports on Sharesight portfolio performance over a given period. The default is set by `past_days` in the `preferences.conf`, which can be overridden at runtime by specifying an integer as the argument. +The `performance` report covers Sharesight portfolio performance over a given period. The default is set by `past_days` in the `preferences.conf`, which can be overridden at runtime by specifying an integer as the argument. Config example: ``` past_days=28 ``` -Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): +Job (see [Scheduling](#scheduling)), such as `.jobs add 28 21 * * Sat performance 7`: ``` -performance.py 7 +performance [DAYS] ``` Interactive trigger: @@ -273,7 +275,7 @@ Interactively, you can also specify a portfolio: ### Events calendar ![earnings message in Slack](img/earnings.png?raw=true "Earnings message in Slack") -`cal.py` sends upcoming earnings and ex-dividend date alerts. The data is sourced from Yahoo! Finance. It reports on events up to `future_days` into the future. This is set in the `preferences.conf` for triggering the report from Cron, or can be specified as an argument when triggered through the chat bot. +The `cal` report sends upcoming earnings and ex-dividend date alerts. The data is sourced from Yahoo! Finance. It reports on events up to `future_days` into the future. This is set in the `preferences.conf` for a scheduled run, or can be specified as an argument when triggered through the chat bot. **Finance Explainer:** When a company releases its quarterly earnings report, the stock price may undergo a signficant positive or negative movement, depending on whether the company beat or missed market expectations. You may wish to hold off buying more of this stock until after its earnings report, unless you think the stock will beat market expectations. @@ -282,9 +284,9 @@ Interactively, you can also specify a portfolio: future_days=30 ``` -Schedule line (`schedule.cron`, see [Scheduling](#scheduling)): +Job (see [Scheduling](#scheduling)), such as `.jobs add 28 21 * * Sat cal earnings`: ``` -cal.py [earnings|ex-dividend] +cal earnings|ex-dividend ``` Interactive trigger (earnings): @@ -306,7 +308,7 @@ Interactive trigger (ex-dividend): ### Highly shorted stock warnings ![short warnings in Slack](img/shorts.png?raw=true "Short warnings in Slack") -`shorts.py` sends short interest warnings. The data is sourced from Yahoo Finance and Shortman (AU). `shorts_percent` defines the alert threshold for the percentage of a stock's float shorted. This can be specified in the `preferences.conf` for running the report from Cron, or as an argument when triggered through the chat bot. +The `shorts` report sends short interest warnings. The data is sourced from Yahoo Finance and Shortman (AU). `shorts_percent` defines the alert threshold for the percentage of a stock's float shorted. This can be specified in the `preferences.conf` for a scheduled run, or as an argument when triggered through the chat bot. **Finance Explainer:** A high short ratio indicates a stock is exposed to high risks, such as potential banktrupcy. It may also incentivise negative news articles which harm the stock price. If the market is wrong, however, risk tolerant investors may receive windfall gains. This report is intended to alert you to an above-average risk, and prompt you to investigate this stock more closely. ``` @@ -394,10 +396,10 @@ Interactive trigger: ### Consensus rating changes ![Rating change report in Slack](img/finbot_rating.png?raw=true "Rating change report in Slack") -Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run from Cron. Requires persistent storage. +Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run on a schedule, such as `.jobs add 28 21 * * Sat rating` (see [Scheduling](#scheduling)). Requires persistent storage. ``` -rating.py +rating ``` ### Watchlist @@ -468,21 +470,21 @@ of the chat network keys it needs. | `ip` | `127.0.0.1` | Address the bot listens on | | `port` | `5000` | Port the bot listens on | | `scheduler_catchup` | `5` | Minutes: a run missed while the bot was down still fires if it was due this recently | -| `telegramChatID` | | Chat group or channel id that scheduled reports go to on Telegram | +| `telegramChatID` | | Telegram's home chat, a group or channel id: finbot's first start gives it the default jobs, and error reports go there (see [Scheduling](#scheduling)) | | `sharesight_code` | | Sharesight API access | | `sharesight_client_id` | | Sharesight API access | | `sharesight_client_secret` | | Sharesight API access | | `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | | `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | | `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | -| `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`) | +| `slackChannelID` | | Slack's home channel id, as `telegramChatID` (with `slackBotToken`) | | `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | | `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | | `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | | `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | | `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | | `discordBotToken` | | Discord bot token: replies, and commands over the gateway unless `discordOutgoingWebhook` is set | -| `discordChannelID` | | Channel id that scheduled reports go to on Discord as the bot, charts and buttons and all (with `discordBotToken`) | +| `discordChannelID` | | Discord's home channel id, as `telegramChatID` (with `discordBotToken`) | | `discordOutgoingWebhook` | | The Interactions Endpoint URL Discord posts slash commands and button presses to (HTTP mode); empty, the gateway (pull mode) | | `discordPublicKey` | | The app's public key, which checks the signature of each interaction (HTTP mode) | @@ -514,10 +516,10 @@ The preferences, read by the bot itself. A change takes effect on a reload. A tr | `hyperlinkProvider` | `google` | Where ticker links go: `google` or `yahoo` | | `exchange_suffix` | `AX` | Regional: Yahoo's suffix for your exchange (`AX` for the ASX, `NZ`, `L`, `TO` and so on). A bare symbol Yahoo does not know is tried again with it, in every command that takes a symbol (`BHP` becomes `BHP.AX`); empty, a symbol is taken as written | | `country_code` | `AU` | Regional: your country, for the reminder report | -| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report | +| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report, and for each job's next run in `.jobs` (besides UTC) | | `cache` | `True` | Cache Yahoo and Sharesight responses | | `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | -| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm.py` runs, so the holdings never run out | +| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm` runs, so the holdings never run out | | `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | | `debug` | `False` | Extra diagnostics in the log | | `print_headers` | `False` | Log inbound request headers | @@ -540,7 +542,7 @@ util/sync_defaults.py --apply # copy missing files (including a starter prefere util/sync_defaults.py --force # also replace changed files, keeping .bak copies util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory ``` -`reminder.json` holds the reminders that `reminder.py` posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. +`reminder.json` holds the reminders that the `reminder` report posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. @@ -580,11 +582,11 @@ performance_cache_seconds=2400 `performance_cache_seconds` (default 40 minutes) is the cache lifetime of Sharesight performance reports. -Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The scheduled `cleanup.py` removes cache files after 30 days. +Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The weekly system job `cleanup` removes cache files after 30 days. To flush the cache, so that everything is fetched afresh, signal the running bot: it deletes its cache files (Yahoo and Sharesight -answers, charts, the Yahoo cookie) and keeps its state (watchlist, seen trades, ratings, schedule), then runs both warm jobs -(`sharesight_warm.py`, `yahoo_warm.py`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: +answers, charts, the Yahoo cookie) and keeps its state (watchlist, jobs, seen trades, ratings), then runs both warm jobs +(`sharesight_warm`, `yahoo_warm`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: ``` sudo systemctl kill -s USR1 finbot # systemd sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) @@ -594,11 +596,7 @@ kill -USR1 # anything else The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, deleting the files in `DATA_DIR/cache` does the same. -Two scheduled jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](#scheduling)): `sharesight_warm.py` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm.py` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm.py` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). A data directory with its own `schedule.cron` needs their lines: -``` -14,44 * * * * sharesight_warm.py -07 */8 * * * yahoo_warm.py -``` +Two system jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](#scheduling)): `sharesight_warm` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). ### Hyperlinks These `preferences.conf` options relate to hyperlinks: @@ -623,43 +621,61 @@ hyperlinkProvider=yahoo ``` ## Scheduling -The scheduled reports are run by `bot.py` itself, each in a thread of its own, so they share the bot's user, sandbox, environment, data directory, caches and connections; charts are drawn one at a time, and a slow report holds up no chat command. There is no cron to set up, which matters under systemd `DynamicUser=` where only the bot's own user can read the state directory. Remove the equivalent entries from any old crontab before the first start, otherwise every job runs twice. - -The schedulable scripts are `trades.py`, `price.py`, `reminder.py`, `cal.py`, `shorts.py`, `performance.py`, `rating.py`, `milestone.py`, `podcasts.py ""`, `cleanup.py [days]` (deletes cache files older than `days`, default 30), `sharesight_warm.py` and `yahoo_warm.py` (they fill the cache: see [Caching](#caching)). Other jobs, such as a DNS updater or backups, stay in cron or systemd timers. - -The schedule is `defaults/schedule.cron`, written in **cron-compatible syntax**: five time fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists work) or an `@macro` such as `@daily`, then a finbot script and its arguments. Quote an argument that has spaces or special characters (a feed URL with `&`, say). Several commands separated by `;` run one after another. `MAILTO=`, `SHELL=`, `PATH=` and `HOME=` lines are ignored. Anything else a crontab allows (a user column, pipes, other programs) is not supported: such a line is reported at startup and skipped. A time that cannot be or never comes (minute 61, 30 February) is different: finbot refuses to start with it, and a reload with it is refused, leaving the running schedule as it was. Existing crontab lines can be pasted as they are: `> /dev/null` redirects, `{ a; b ;}` groups and a path before the script name are understood. -``` -# Every 20 minutes on weekdays -*/20 * * * Mon-Fri trades.py - -# Weekly, one after the other -28 21 * * Sat cal.py earnings; cal.py ex-dividend; price.py 7; performance.py 7; rating.py - -# New episodes, hourly -01 0-10,23 * * Mon-Fri podcasts.py "https://example.com/podcast.rss" -``` -The default times suit a server set to UTC. Like cron, they follow the server's own time zone; put `CRON_TZ=Australia/Melbourne` on a line of its own above a job to pin another zone. - -**To change a job**, copy the schedule to the data directory, edit the copy, and reload (no restart, so the bot stays up): -``` -util/sync_defaults.py --apply # copies schedule.cron to the data directory if it is not there yet -sudoedit /var/lib/finbot/schedule.cron -sudo systemctl reload finbot # the bot re-reads the file and logs how many jobs it loaded -``` -The bot logs the number of jobs when it starts or reloads, and each job as it runs (`journalctl -u finbot`); a line it cannot use is reported by name. If the edited file has no usable line the previous schedule keeps running. - -**To run a job from chat**, send `.jobs` (also `.cron`, `.crons` or `.schedule`): it lists the scheduled commands, each once and without times, comments or `.py`, and on Telegram and Slack (with its buttons set up) adds a button per command. The system jobs (`cleanup`, `podcasts`, `sharesight_warm` and `yahoo_warm`) follow under _System_, without a number or a button: they run on their schedule only. `.jobs run 2` (or tapping the button) runs that command now, and `.jobs run price.py close` runs it by name; anything not in the schedule is refused, and so is a chain such as `a; b` (run them one at a time). A manual run is exactly a cron run: it reports to all configured chat networks, updates the same state (seen trades, ratings and so on), and does not change the schedule. Anyone who can message the bot can do this; a command that is already running is not started twice. +Each chat has its own jobs: reports that run on a schedule and post to that chat. Anyone in the chat manages them there, +with `.jobs` (also `.job`, `.cron`, `.crons` or `.schedule`): +``` +.jobs this chat's jobs, numbered, each with its next run; then the system jobs +.jobs add the reports a job can run, and what each takes +.jobs add 29 21 * * Mon-Fri price interday +.jobs run the list, with a button per job that runs it now; .jobs run 2 runs job 2 +.jobs toggle the list, with a button per job that switches it on or off; .jobs toggle 2 +.jobs del 2 deletes job 2 +``` + +A job is a time in crontab's syntax, then one or more reports with their arguments: +* **The time** is five fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists + work) or an `@macro` such as `@daily`. A time that cannot be or never comes (minute 61, 30 February) is refused, + saying why. +* **Times are in UTC**, which keeps the ASX's jobs and NYSE's each within an hour of their local times all year. To + follow one market's clocks exactly, a job names its zone first, as crontab writes it: + `.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession`. `.jobs` shows each next run in UTC and in + the `timezone` preference. +* **The reports** are `trades`, `price midsession|interday|premarket|close|DAYS`, `cal earnings|ex-dividend`, + `performance [DAYS]`, `rating`, `milestone`, `shorts`, `reminder` and `podcasts FEED_URL`. Several, separated by `;`, + run one after another: `.jobs add 28 21 * * Sat cal earnings; cal ex-dividend; price 7; performance 7; rating`. + Quote an argument that holds spaces or a `;` (a feed URL, say). +* A chat has 30 jobs at most. + +A job posts to its own chat, whichever way that chat's updates reach finbot: the jobs are kept by network and chat id +(`finbot_schedule.json` in the data directory), so switching a network between pull and HTTP mode keeps them. Each chat +also keeps its own record of what a report has told it (trades, rating changes, record highs, podcast episodes), so the +same report scheduled in two chats tells each of them once. A run from `.jobs run` is a scheduled run: it posts to the +chat and updates the same record. A change counts at once, with no reload. + +**The system jobs**, fixed in `lib/jobs.py` and listed by `.jobs` under _System_, keep the cache full and tidy: +`sharesight_warm` every 30 minutes, `yahoo_warm` every 8 hours (see [Caching](#caching)), and `cleanup 30` weekly, +which deletes cache files older than 30 days. A chat cannot add, switch or run them. + +**From `schedule.cron`**, where the schedule used to be: the first time finbot starts without `finbot_schedule.json`, +it gives each home chat (`telegramChatID`, `slackChannelID`, `discordChannelID`) the jobs of the data directory's +`schedule.cron`, else of the shipped `defaults/schedule.cron`, each in the zone it ran in (the server's), without the +system jobs. It copies the reports' records to each home chat too, so that nothing old is told again. It logs what it +did, and any line it left out, and reads the file no more: delete the data directory's copy afterwards. Behaviour worth knowing: -* A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. -* A job still running when it is next due is skipped. A command still running after 5 minutes is reported once, and left to finish (a thread cannot be stopped, but every request it makes has a timeout); until it does, it is not started again. Each command of a line has 5 minutes of its own. -* Failures are reported to your chat channels, as before. -* A schedule without jobs runs nothing: there is no switch to turn the scheduler off. - -**Running a report by hand** is `.jobs run` in a chat (above): it runs the command in the bot, as the schedule would. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports in the bot too. +* The reports run in `bot.py` itself, each in a thread of its own, sharing the bot's user, sandbox, environment, data + directory, caches and connections; charts are drawn one at a time, and a slow report holds up no chat command. There + is no cron to set up: remove any old crontab entries that ran finbot's scripts, or their reports come twice. +* A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last + `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. A job that is switched off + does not fire, and does not catch up once it is switched on again. +* A job still running when it is next due is skipped. A command still running after 5 minutes is reported once, and + left to finish (a thread cannot be stopped, but every request it makes has a timeout); until it does, it is not + started again. Each command of a job has 5 minutes of its own. +* A failure is reported to the job's chat. The bot logs each job as it runs (`journalctl -u finbot`). ## Daemonize (systemd) -`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy `finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of [`finbot.env`](#setup) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start it. `systemctl reload finbot` re-reads `preferences.conf` and the [schedule](#scheduling) without a restart; a change to `finbot.env` needs `systemctl restart finbot`: +`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy `finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of [`finbot.env`](#setup) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start it. `systemctl reload finbot` re-reads `preferences.conf` without a restart; a change to `finbot.env` needs `systemctl restart finbot`: ``` sudo cp -v finbot.service /etc/systemd/system/ diff --git a/bot.py b/bot.py index 1c02276a..8dc350d1 100755 --- a/bot.py +++ b/bot.py @@ -13,6 +13,7 @@ from lib import inbound from lib import worker from lib import jobs +from lib import migrate from lib import net from lib import pull from lib import telegram @@ -184,8 +185,8 @@ def run(): return lambda *args: threading.Thread(target=run, daemon=True).start() def reload_all(*args): - """SIGHUP: re-read preferences.conf and, if the scheduler is running, schedule.cron. Each reports its own errors; a - bad preferences.conf leaves the running config untouched.""" + """SIGHUP: re-read preferences.conf; a bad one is reported and leaves the running config untouched. (Each chat's + jobs need no reload: a change made with .jobs counts at once.)""" try: changes = config.reload() for change in changes: @@ -194,8 +195,6 @@ def reload_all(*args): file=sys.stderr) except config.ConfigError as e: config.print_errors(e) - if scheduler: - scheduler.reload() def flush_cache(*args): """SIGUSR1 (`systemctl kill -s USR1 finbot`): delete the cache and keep the state (watchlist, seen trades, ratings, @@ -230,14 +229,9 @@ def http_server(): print('ERROR config:', line, file=sys.stderr) if os.getuid() == 0: print("Running as superuser. This is not recommended.", file=sys.stderr) - scheduler = jobs.Scheduler(lambda: config.data_file('schedule.cron')) # without a schedule, it runs nothing - try: # read before anything else happens: an impossible time in it stops the start - scheduler.load() - except jobs.ImpossibleTime as e: - for line in e.lines: - print('ERROR schedule:', line, file=sys.stderr) - print('ERROR schedule: finbot will not start until every time in the schedule can come', file=sys.stderr) - sys.exit(1) + migrate.run() # once: what was kept for all chats becomes the home chats', schedule.cron among it + scheduler = jobs.Scheduler() + scheduler.load() render.preload() try: server = http_server() # bound before Telegram is given the URL @@ -265,7 +259,7 @@ def http_server(): threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs for listener in pull.listeners(): # the networks without a webhook: finbot fetches their updates threading.Thread(target=listener.run, args=(take,), daemon=True).start() - # Startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron + # Startup is done. systemctl reload finbot: pick up an edited preferences.conf signal.signal(signal.SIGHUP, in_thread(reload_all)) signal.signal(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache print('finbot is running: SIGHUP reloads, SIGUSR1 flushes the cache', file=sys.stderr) diff --git a/defaults/finbot.env b/defaults/finbot.env index 3f7633e0..2f9e3b7a 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -22,7 +22,8 @@ port=5000 # How many minutes late a scheduled report missed while the bot was down may still fire scheduler_catchup=5 -# Where scheduled reports go on Telegram: the chat group or channel id +# Telegram's home chat, a group or channel id: finbot's first start gives it the default jobs (.jobs), and errors are +# reported there telegramChatID= # Sharesight API access @@ -33,11 +34,11 @@ sharesight_client_secret= # Chat networks. A Telegram bot token is enough for reports and commands: finbot fetches its updates (pull mode) telegramBotToken= slackBotToken= -# Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken) +# Slack's home channel id, as Telegram's home chat above (needs slackBotToken) slackChannelID= # Slack's app-level token (xapp-) for commands without a server: finbot opens a socket to Slack (Socket Mode) slackAppToken= -# Discord's bot token: reports, and commands over the gateway (pull mode); and the channel id for scheduled reports +# Discord's bot token: reports, and commands over the gateway (pull mode); and its home channel id, as Telegram's discordBotToken= discordChannelID= diff --git a/defaults/preferences.conf b/defaults/preferences.conf index b8b69bed..6d382c7a 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -37,7 +37,7 @@ hyperlinkProvider=google # Regional: where you are. # exchange_suffix: a ticker Yahoo does not know is tried again on your exchange, by Yahoo's suffix for it (AX: BHP is # tried as BHP.AX; NZ, L, TO and so on for others). Empty: a ticker is taken as written. -# country_code and timezone: the reminder report's country and time zone. +# country_code and timezone: the reminder report's country and time zone; .jobs also shows each next run in timezone. exchange_suffix=AX country_code=AU timezone=Australia/Melbourne diff --git a/defaults/schedule.cron b/defaults/schedule.cron index 5020d035..8e340293 100644 --- a/defaults/schedule.cron +++ b/defaults/schedule.cron @@ -1,9 +1,8 @@ -# The scheduled reports, run by the bot (bot.py). Same time syntax as crontab(5): minute hour day-of-month month -# day-of-week, or an @macro such as @daily. The command is a finbot script and its arguments; separate several with ";" -# to run them in turn. -# Times are in the server's own time zone, like cron; add a line "CRON_TZ=Australia/Melbourne" above a job to pin one. -# To change the schedule, copy this file to the data directory (util/sync_defaults.py --apply), edit the copy, then: -# systemctl reload finbot +# NO LONGER READ, BUT ONCE: each chat has its own jobs now, changed with .jobs (add, del, toggle, run). As finbot +# starts the first time without them, it gives every home chat (telegramChatID, slackChannelID, discordChannelID) the +# jobs below (or those of the data directory's copy of this file), each in the zone it ran in, and reads this file no +# more. It goes once every install has done that (lib/migrate.py). The system jobs (sharesight_warm, yahoo_warm, +# cleanup) are not copied: they run on their own schedule, in lib/jobs.py. # Every 20 minutes on weekdays */20 * * * Mon-Fri trades.py diff --git a/doc/discord.md b/doc/discord.md index e675c337..24372294 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -16,10 +16,10 @@ The bot does the same either way. What differs is how it hears from Discord: | Python modules | `discord.py` (Debian: `python3-discord`) | `PyNaCl` and `waitress` (Debian: `python3-nacl`, `python3-waitress`) | | `finbot.env` keys | `discordBotToken`, `discordChannelID` | Those two, plus `discordOutgoingWebhook` and `discordPublicKey` | -In both: the scheduled reports with their charts; period buttons under charts and `.jobs` buttons, which redraw their -message in place; linked tickers, without link previews and without pinging anyone; and direct messages with the bot. -A command in a message shows "finbot is typing…" while it works; a slash command shows Discord's "finbot is -thinking…" until its answer takes its place. +In both: the scheduled reports with their charts; period buttons under charts and the buttons of `.jobs run` and +`.jobs toggle`, which redraw their message in place; linked tickers, without link previews and without pinging anyone; +and direct messages with the bot. A command in a message shows "finbot is typing…" while it works; a slash command +shows Discord's "finbot is thinking…" until its answer takes its place. `discordOutgoingWebhook` chooses the mode: set, HTTP mode; empty, gateway mode. Discord sends the slash commands and button presses to the app's _Interactions Endpoint URL_ while it is set, and over the gateway while it is empty, so @@ -37,8 +37,10 @@ change both, and restart finbot, to switch. 3. Under _OAuth2 > URL Generator_, tick the scopes `bot` and `applications.commands`, then the bot permissions _View Channels_, _Send Messages_, _Attach Files_ and _Read Message History_. Open the URL it makes, and add the bot to your server. -4. In Discord, turn on _User Settings > Advanced > Developer Mode_; then right-click the channel for the scheduled - reports, _Copy Channel ID_, and put it in `finbot.env` as `discordChannelID`. +4. In Discord, turn on _User Settings > Advanced > Developer Mode_; then right-click the channel to be Discord's home + chat, _Copy Channel ID_, and put it in `finbot.env` as `discordChannelID`. finbot's first start gives that channel + the default jobs ([Scheduling](../README.md#scheduling)), and it gets the error reports. Every channel manages its + own jobs with `.jobs`. 5. Install discord.py: `sudo apt install python3-discord` (or `pip install discord.py`). 6. Restart finbot (`sudo systemctl restart finbot`). As it starts, it registers the slash commands, which can take a few minutes to show in Discord the first time, and the log says `discord: gateway (no discordOutgoingWebhook)`. diff --git a/doc/slack.md b/doc/slack.md index 7599f8d6..11b07aac 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -110,9 +110,10 @@ slackChannelID=C0123456789 5. Restart finbot, and invite the bot to the channel: `/invite @finbot`. In Socket Mode the log says `slack: Socket Mode (no slackOutgoingWebhook)`. 6. Send `@finbot help` in the channel (or `.help`, with `message.channels`). -7. For the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: the - _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot posts the scheduled - reports there, as it posts its replies. +7. Set `slackChannelID` to the channel's id (in Slack, click the channel's name: the _Channel ID_, which starts with + C, is at the bottom of _About_), and restart finbot. It is Slack's home chat: finbot's first start gives it the + default jobs ([Scheduling](../README.md#scheduling)), and it gets the error reports. Every channel manages its own + jobs with `.jobs`. A chart is uploaded with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`). With its period buttons, it goes in a message of blocks: the report, the chart (uploaded into no channel, and shown by @@ -122,5 +123,5 @@ drawn and uploaded anew (Slack cannot replace a file). An upload that no message are past, so the bot's files do not pile up. If Slack will not show a chart with its buttons, the chart is uploaded into the channel without them; if an upload fails (a missing scope, say), the report goes out as text. The log says why. -`.jobs` lists the scheduled commands with a button for each, as on Telegram; a button from a list that has since -changed puts the current list in its place. +`.jobs run` and `.jobs toggle` list the channel's jobs with a button for each, as on Telegram; a button from a list +that has since changed puts the current list in its place. diff --git a/doc/telegram.md b/doc/telegram.md index 78482470..66219300 100644 --- a/doc/telegram.md +++ b/doc/telegram.md @@ -14,8 +14,9 @@ Telegram: | `finbot.env` keys | `telegramBotToken`, `telegramChatID` | Those two, plus `telegramOutgoingWebhook` and `telegramOutgoingToken` | Everything else is the same in both: the scheduled reports with their charts, the chat commands (`.price`, `.history`, -`.performance` and the rest, [the list](../README.md#supported-commands)), period buttons under charts and `.jobs` -buttons, "typing…" while a command works, and a keyboard of the common commands in a direct chat with the bot. +`.performance` and the rest, [the list](../README.md#supported-commands)), period buttons under charts and the +buttons of `.jobs run` and `.jobs toggle`, "typing…" while a command works, and a keyboard of the common commands in a +direct chat with the bot. `telegramOutgoingWebhook` chooses the mode: set, HTTP mode; empty, pull mode. A restart switches. As finbot starts in pull mode it deletes the webhook, and in HTTP mode it sets it; the updates sent meanwhile are kept by Telegram and @@ -42,8 +43,9 @@ arrive after the switch. starts). * A group's id changes when Telegram turns it into a supergroup, which some changes to its settings do. Update `telegramChatID` then: finbot does not follow the change. -6. Start finbot (`sudo systemctl restart finbot`). The log says `telegram: long polling for updates`, and the scheduled - reports now arrive in the chat. +6. Start finbot (`sudo systemctl restart finbot`). The log says `telegram: long polling for updates`. The first start + gives this chat, Telegram's home chat, the default jobs ([Scheduling](../README.md#scheduling)), so the scheduled + reports arrive there; it gets the error reports too. Every chat manages its own jobs with `.jobs`. 7. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. ``` diff --git a/finbot.service b/finbot.service index fd74abf0..993302ba 100644 --- a/finbot.service +++ b/finbot.service @@ -6,7 +6,7 @@ Type=simple User=CHANGEME # Or, instead of User=: DynamicUser=yes plus StateDirectory=finbot (state goes to /var/lib/finbot automatically) ExecStart=/home/CHANGEME/finbot/bot.py -# systemctl reload finbot re-reads preferences.conf and schedule.cron without a restart +# systemctl reload finbot re-reads preferences.conf without a restart ExecReload=/bin/kill -HUP $MAINPID # Refuse to start on a mistake in finbot.env or preferences.conf (the bot never reads finbot.env itself, so this is # where its typos are caught). "+" runs it as root. diff --git a/lib/buttons.py b/lib/buttons.py index c93b53ff..c11fe28c 100644 --- a/lib/buttons.py +++ b/lib/buttons.py @@ -1,6 +1,6 @@ """Buttons under a message, what their data says when one is pressed (parse), and what a press is told at once (toast): a chart's period buttons (1D | 7D | 1M | 1Y | Max), whose data carries what its builder needs -(charts.BUILDERS), and a .jobs command's button. Each transport draws Button rows its own way.""" +(charts.BUILDERS), and a job's buttons in .jobs run and .jobs toggle. Each transport draws Button rows its own way.""" import hashlib import typing @@ -21,10 +21,14 @@ class ChartPress(typing.NamedTuple): class JobPress(typing.NamedTuple): - """A .jobs command's button pressed: the command's number in the list and its checksum, both for - jobs.Scheduler.pick to check.""" + """A job's button pressed: the job's number in its chat's list and its checksum, both for jobs.Scheduler.pick to + check, and what to do with it: run it now, or switch it on or off (toggle).""" number: str checksum: str + action: str = 'run' + + +JOB_ACTIONS = {'j': 'run', 'e': 'toggle'} # a job button's kind, its data's first letter, and what it does PERIODS = {'d': ('1D', 1), 'w': ('7D', 7), 'm': ('1M', 30), 'y': ('1Y', 365), 'x': ('Max', None)} @@ -71,18 +75,20 @@ def keyboard(kind, args, active=None): ref = _ref(args) return [[Button(PERIODS[key][0], f"c|{kind}|{key}|{ref}", key == active) for key in BUTTONS[kind]]] -def job_button(number, label, checksum): - """A .jobs command's button: 'N. label', cut to 30 characters, its data the number and the command's checksum.""" - return Button(f"{number}. {label}"[:30], f"j|{number}|{checksum}") +def job_button(number, label, checksum, action='run'): + """A job's button, which runs it now (or with action 'toggle', switches it on or off): 'N. label', cut to 30 + characters, its data the job's number and checksum.""" + kind = next(kind for kind, does in JOB_ACTIONS.items() if does == action) + return Button(f"{number}. {label}"[:30], f"{kind}|{number}|{checksum}") def parse(data): - """What a pressed button's data asks for: a JobPress for a .jobs button (whatever follows 'j|', which pick refuses - unless it is a command's number and checksum), a ChartPress for a period button of a known kind and period, or None - for data of neither (from an older finbot, or forged).""" + """What a pressed button's data asks for: a JobPress for a job's button (whatever follows 'j|' or 'e|', which pick + refuses unless it is a job's number and checksum), a ChartPress for a period button of a known kind and period, or + None for data of neither (from an older finbot, or forged).""" kind, _, rest = data.partition('|') - if kind == 'j': + if kind in JOB_ACTIONS: number, _, checksum = rest.partition('|') - return JobPress(number, checksum) + return JobPress(number, checksum, JOB_ACTIONS[kind]) parts = rest.split('|', 2) if kind == 'c' and len(parts) == 3 and parts[0] in BUTTONS and parts[1] in BUTTONS[parts[0]]: return ChartPress(*parts) @@ -90,8 +96,9 @@ def parse(data): def toast(data): """What a press is told at once, from the button's data alone (the network acknowledges a press before finbot - knows more): 'Loading 1Y…' for a period button, 'Running…' for a .jobs command's, None for data of neither.""" + knows more): 'Loading 1Y…' for a period button, 'Running…' or 'Switching…' for a job's, None for data of + neither.""" pressed = parse(data) if isinstance(pressed, JobPress): - return 'Running…' + return 'Running…' if pressed.action == 'run' else 'Switching…' return f"Loading {PERIODS[pressed.period][0]}…" if pressed else None diff --git a/lib/config.py b/lib/config.py index 181f8846..39d9e44c 100644 --- a/lib/config.py +++ b/lib/config.py @@ -188,23 +188,23 @@ def check(self): @dataclasses.dataclass(frozen=True) class Env: """The startup settings: finbot.env, which the launcher loads into the environment; fixed for the process.""" - data_dir: str # state: watchlist, seen trades, ratings, the schedule's record, preferences.conf + data_dir: str # state: each chat's jobs, the watchlist, seen trades, ratings, preferences.conf cache_dir: str # data_dir/cache ip: str port: int scheduler_catchup: int # minutes: a run missed while the bot was down still fires if it was due this recently - telegramChatID: str # '' when unset, like every token below + telegramChatID: str # Telegram's home chat (its first jobs, the error reports); '' when unset, like the rest below telegramBotToken: str telegramOutgoingWebhook: str telegramOutgoingToken: str slackBotToken: str slackAppToken: str # xapp-: Socket Mode, Slack's pull mode (without slackOutgoingWebhook) - slackChannelID: str # with slackBotToken, scheduled reports go to this channel as the bot + slackChannelID: str # with slackBotToken, Slack's home chat (its first jobs, the error reports) slackOutgoingWebhook: str slackOutgoingToken: str slackSigningSecret: str discordBotToken: str - discordChannelID: str # with discordBotToken, scheduled reports go to this channel as the bot + discordChannelID: str # with discordBotToken, Discord's home chat (its first jobs, the error reports) discordOutgoingWebhook: str # the interactions endpoint URL (http mode); unset, finbot keeps to the gateway discordPublicKey: str # the app's key, which checks the signature of each interaction posted to the endpoint sharesight_auth: dict diff --git a/lib/delivery.py b/lib/delivery.py index b567d130..0a4f873b 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -31,6 +31,23 @@ def reply_to(service, chat, message=None, interaction='', pressed=False): return kind(chat, message, interaction, pressed) if kind else None +def chat(key, scheduled=False): + """The destination of a chat by its key ('telegram:-1001234', Transport.key), as its network's bot; scheduled: for a + scheduled run, whose reports keep their record of what they told it.""" + network, _, where = key.partition(':') + dest = reply_to(network, where) + if dest is None: + raise ValueError(f'no chat {key!r}: no such network') + dest.scheduled = scheduled + return dest + + +def interactive(dest): + """Whether a report answers a command (dest, the chat that asked), not a scheduled run (dest a job's chat, or None + for the home chats): a scheduled run tells each chat only what it has not told it before.""" + return dest is not None and not dest.scheduled + + def default_chats(): """Where a scheduled report goes: each network's home chat (telegramChatID, slackChannelID, discordChannelID), as its bot. Raises RuntimeError when there is nowhere to post.""" @@ -41,6 +58,7 @@ def default_chats(): (env.discordChannelID, env.discordBotToken, transports.DiscordBot)): if chat and token: chats.append(kind(chat)) + chats[-1].scheduled = True if not chats: raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID or discordChannelID,' ' each with its bot token') diff --git a/lib/formats.py b/lib/formats.py index bbb90c62..00b21c1f 100644 --- a/lib/formats.py +++ b/lib/formats.py @@ -1,6 +1,9 @@ """Text for people: company names trimmed, numbers in units, dates and spans in words, signed percents, ordinals.""" import datetime import re +import zoneinfo + +from lib import config TITLE_TRIMS = [ # transform_title: (text, replacement), in order, as each sees what the ones before it made (' - ', ' '), ('Roundhill ETF Trust ', ''), ('Class A Common Stock', ''), @@ -152,3 +155,11 @@ def days_from_human_days(arg): def ordinal(num): """The ordinal suffix of a whole number: 'st' for 1, 'nd' for 22, 'th' for 11 and 13.""" return 'th' if num % 100 in (11, 12, 13) else {1: 'st', 2: 'nd', 3: 'rd'}.get(num % 10, 'th') + + +def zone(): + """The timezone preference as a zone: UTC where it names none.""" + try: + return zoneinfo.ZoneInfo(config.prefs.timezone) + except (ValueError, zoneinfo.ZoneInfoNotFoundError): + return zoneinfo.ZoneInfo('UTC') diff --git a/lib/jobs.py b/lib/jobs.py index de6f7081..4e39e83f 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -1,21 +1,21 @@ -"""The scheduled reports (formerly cron jobs), run by the bot itself. +"""The scheduled reports: each chat's jobs, which its people manage with .jobs (lib/worker.py), kept in SCHEDULE_FILE +and each posting to its own chat; and the system jobs (SYSTEM), fixed here, which fill the cache and clear it out. -The schedule is a file in crontab syntax: five time fields (or an @macro) and then a finbot script with its -arguments, e.g. `*/20 * * * Mon-Fri trades.py` or `28 21 * * Sat cal.py earnings; price.py 7`. Pasted crontab lines -(`./price.py > /dev/null`, `{ a; b ;}`) are accepted too. Each command runs in the bot's process, in a thread of its -own, sharing its user, sandbox, environment, caches and connections.""" +A job is a time in crontab's syntax (five fields, or an @macro such as @daily), in UTC unless it names its own zone +first (CRON_TZ=America/New_York), then one or more reports with their arguments, separated by ";": +`*/20 * * * Mon-Fri trades`, `28 21 * * Sat cal earnings; price 7`. The bot runs each command itself, in a thread of +its own, sharing its user, sandbox, environment, caches and connections.""" import datetime import hmac import math -import os import re +import shlex import sys import threading import time import zlib import zoneinfo from dataclasses import dataclass -from pathlib import Path from croniter import croniter, CroniterBadDateError @@ -28,26 +28,64 @@ SCRIPTS = {'cal': cal, 'cleanup': cleanup, 'milestone': milestone, 'performance': performance, 'podcasts': podcasts, 'price': price, 'rating': rating, 'reminder': reminder, 'shorts': shorts, 'trades': trades, 'sharesight_warm': sharesight_warm, 'yahoo_warm': yahoo_warm} -current = None # the running Scheduler, set by bot.py; the chat commands use it -# scheduled, but not run from chat: .jobs lists them under System -HIDDEN = ('cleanup', 'podcasts', 'sharesight_warm', 'yahoo_warm') -WARM = 'sharesight_warm.py; yahoo_warm.py' # the jobs that fill the cache, run at once after a flush too +# the system jobs, in UTC: the Sharesight portfolios and holdings fetched afresh every 30 minutes (cached 40), the price +# histories of the stocks followed every 8 hours (cached 12), so that no request waits on either; and cache files +# older than 30 days deleted weekly +SYSTEM = (('14,44 * * * *', 'sharesight_warm'), ('7 */8 * * *', 'yahoo_warm'), ('51 18 * * Sat', 'cleanup 30')) +SYSTEM_SCRIPTS = ('sharesight_warm', 'yahoo_warm', 'cleanup') # a chat cannot schedule these +WARM = 'sharesight_warm; yahoo_warm' # the jobs that fill the cache, run at once after a flush too +SCHEDULE_FILE = 'finbot_schedule.json' # {chat key: [{'when', 'reports', 'zone', 'on'}]}: each chat's jobs, in order STATE_FILE = 'finbot_jobs.json' # last fire time of every job, so a restart neither repeats a run nor loses one +MAX_JOBS = 30 # a chat's jobs, at most JOB_TIMEOUT = 5 * 60 # seconds a command may run before it is reported as late, once (it runs on, see execute) -IGNORED_VARIABLES = ('MAILTO', 'SHELL', 'PATH', 'HOME', 'LOGNAME') -# what a shell reads as more than a word, outside quotes -OPERATOR = re.compile(r'\d*>>?&?\d*|&>>?|;|&&|\|\|?|&|[<`$(){}]') -REDIRECT = re.compile(r'\d*>>?&?\d*|&>>?') # > file, >> file, 2>&1, &> file +UTC = zoneinfo.ZoneInfo('UTC') +current = None # the running Scheduler, set by bot.py; the chat commands use it + + +class ScheduleError(ValueError): + """A job that cannot be: said to the chat that asked for it.""" + + +class ImpossibleTime(ScheduleError): + """A time that cannot be (minute 61) or never comes (30 February).""" + + +@dataclass +class Job: + """A job: its time, in its zone; the commands it runs in turn, [(label, report module, run's keyword arguments)]; + the chat it posts to (its key, as delivery.chat takes it; '' for a system job); whether it is on; and its reports + as written.""" + when: str + commands: list + zone: zoneinfo.ZoneInfo + chat: str + on: bool + text: str + @property + def key(self): + """What the job is, for its record of runs and the checksum of its buttons: its chat, time, zone and reports.""" + return f'{self.chat}|{self.when}|{self.zone.key}|{self.text}' -def checksum(label): - """A command's short stand-in in a button (Telegram allows 64 bytes of data): it detects a changed schedule.""" - return format(zlib.crc32(label.encode()) & 0xffff, '04x') + @property + def line(self): + """The job as .jobs add takes it: its zone if not UTC, its time, its reports.""" + zone = '' if self.zone.key == 'UTC' else f'CRON_TZ={self.zone.key} ' + return f'{zone}{self.when} {self.text}' -def run_in_process(label, module, kwargs): - """Run a command here, in the calling thread: its report's run(**kwargs).""" - module.run(**kwargs) +def checksum(text): + """A job's short stand-in in a button (Telegram allows 64 bytes of data): it tells a changed list.""" + return format(zlib.crc32(text.encode()) & 0xffff, '04x') + + +def run_in_process(label, module, kwargs, dest=None): + """Run a command here, in the calling thread: its report's run(**kwargs), posting to dest (a chat's job) or else to + the home chats.""" + if dest is None: + module.run(**kwargs) + else: + module.run(dest, **kwargs) def start_thread(target, *args): @@ -59,7 +97,7 @@ def warm_now(scheduler=None): """Run the warm jobs (WARM) now, one after the other: through the scheduler, so that one already running on its schedule is not started twice; without one (a test, a tool), here, a failure reported in the chat as the scheduler reports one.""" - commands = parse_command(WARM) + commands = parse_reports(WARM, system=True) print('cache: filling it again:', ', '.join(label for label, _, _ in commands), file=sys.stderr) if scheduler: scheduler.execute(commands) @@ -72,210 +110,101 @@ def warm_now(scheduler=None): print('cache: warm jobs done', file=sys.stderr) -def display(label): - """A command as shown in chat: 'price.py close' -> 'price close'.""" - return re.sub(r'^(\w+)\.py', r'\1', label) - - -class ScheduleError(ValueError): - pass - - -class ImpossibleTime(ScheduleError): - """A time that cannot be (minute 61) or never comes (30 February). Other bad lines are reported and skipped, but - this refuses the whole file: finbot does not start with it, and a reload keeps the schedule that is running. `lines` - says where, one per line.""" - def __init__(self, lines): - """lines: where, one message or a list of them.""" - self.lines = [lines] if isinstance(lines, str) else list(lines) - super().__init__('; '.join(self.lines)) - - -@dataclass -class Job: - expr: str - commands: list # [(label, module, kwargs)], run one after another - zone: zoneinfo.ZoneInfo - text: str - key: str - - -def local_zone(): - """The server's own time zone, which is what cron uses. Containers without one are UTC, as under cron.""" +def parse_reports(text, system=False): + """'cal earnings; price 7' -> [(label, report module, run's keyword arguments)]: report names (lib/reports) and + their arguments, quoted where they hold spaces, separated by ";". The system jobs' reports only with system. Raises + ScheduleError, saying what is wrong.""" + lexer = shlex.shlex(text, posix=True, punctuation_chars=';') + lexer.whitespace_split, lexer.commenters = True, '' try: - if os.environ.get('TZ'): - return zoneinfo.ZoneInfo(os.environ['TZ'].lstrip(':')) - link = os.path.realpath('/etc/localtime') - if '/zoneinfo/' in link: - return zoneinfo.ZoneInfo(link.split('/zoneinfo/', 1)[1]) - return zoneinfo.ZoneInfo(Path('/etc/timezone').read_text().strip()) - except (OSError, ValueError, zoneinfo.ZoneInfoNotFoundError): - return zoneinfo.ZoneInfo('UTC') - - -def tokenize(text): - """A command line as ('word', text) and ('op', text) tokens. Quoted text is always a word, whatever it contains.""" - tokens, word, started, i = [], '', False, 0 - def flush(): - """End the word being read, if there is one, and add it to the tokens.""" - nonlocal word, started - if started: - tokens.append(('word', word)) - word, started = '', False - while i < len(text): - c = text[i] - if c.isspace(): - flush() - i += 1 - elif c in '\'"': - end = text.find(c, i + 1) - if end < 0: - raise ScheduleError('No closing quotation') - word, started, i = word + text[i + 1:end], True, end + 1 - elif c == '\\' and i + 1 < len(text): - word, started, i = word + text[i + 1], True, i + 2 - elif c == '#' and not started: # a trailing comment - break - elif m := OPERATOR.match(text, i): - if word.isdigit() and m.group().startswith('>'): # the 2 of 2>&1 belongs to the redirect - word, started = '', False - flush() - tokens.append(('op', m.group())) - i = m.end() - else: - word, started, i = word + c, True, i + 1 - flush() - return tokens - - -def split_commands(tokens): - """Word lists, one per command: split on unquoted ; and &&, redirects and braces dropped, other syntax refused.""" - commands, current, target = [], [], False - for kind, value in tokens: - if target: # the file of a redirect - target = False - if kind == 'word': - continue - if kind == 'word': - current.append(value) - elif value in (';', '&&'): - commands.append(current) - current = [] - elif value in ('{', '}'): + tokens = list(lexer) + except ValueError as e: # an unclosed quote + raise ScheduleError(str(e)) + commands, words = [], [] + for token in tokens + [';']: + if token.strip(';'): + words.append(token) continue - elif REDIRECT.fullmatch(value): - target = not re.search(r'&\d+$', value) # 2>&1 has no file after it, > /dev/null does - else: - raise ScheduleError('only finbot scripts can be scheduled, not shell commands') - commands.append(current) - return [words for words in commands if words] - - -def parse_command(text): - """'~/finbot/price.py midsession > /dev/null' -> [('price.py midsession', price, {...})]. Raises ScheduleError.""" - commands = [] - for words in split_commands(tokenize(text)): - while words and re.fullmatch(r'\w+=\S*', words[0]): - words.pop(0) # FOO=bar prefix if not words: continue - name = os.path.basename(words[0]).removesuffix('.py') - if name not in SCRIPTS: - raise ScheduleError(f"unknown script '{words[0]}' (known: {', '.join(sorted(SCRIPTS))})") + name, arguments, words = words[0], words[1:], [] + if name not in SCRIPTS or (name in SYSTEM_SCRIPTS and not system): + raise ScheduleError(f"no report {name!r}: send .jobs add for the reports and what each takes") try: - label, kwargs = SCRIPTS[name].job(words[1:]) + label, kwargs = SCRIPTS[name].job(arguments) except ValueError as e: - raise ScheduleError(f'{name}.py {e}') + raise ScheduleError(f'{name} takes {e}') commands.append((label, SCRIPTS[name], kwargs)) if not commands: - raise ScheduleError('no command') + raise ScheduleError('no report: send .jobs add for the reports') return commands -def parse_schedule(text, default_zone=None): - """(jobs, errors): errors are (line number, line, ScheduleError) for the lines that were skipped.""" - zone = default_zone or local_zone() - jobs, errors = [], [] - for number, raw in enumerate(text.splitlines(), 1): - line = raw.strip() - if not line or line.startswith('#'): - continue +def check_time(when): + """Raise ImpossibleTime for a time that cannot be (minute 61) or never comes (30 February).""" + if when == '@reboot': + raise ImpossibleTime('@reboot is no time: the jobs start with the bot') + if not croniter.is_valid(when): + raise ImpossibleTime(f"'{when}' is no time: five fields (minute hour day month weekday) or an @macro") + try: + croniter(when, datetime.datetime(2000, 1, 1)).get_next(datetime.datetime) + except CroniterBadDateError: + raise ImpossibleTime(f"'{when}' never comes: no date matches it") + + +def parse_job(line, chat='', on=True, system=False): + """A Job from its line, as .jobs add takes it: [CRON_TZ=Zone] five time fields (or an @macro), then its reports. + Raises ScheduleError (ImpossibleTime for the time), saying what is wrong.""" + line = line.strip() + zone = UTC + if m := re.match(r'CRON_TZ=(\S*)\s*', line): try: - if m := re.fullmatch(r'CRON_TZ=(\S+)', line): - try: - zone = zoneinfo.ZoneInfo(m.group(1)) - except (ValueError, zoneinfo.ZoneInfoNotFoundError): - raise ScheduleError(f'unknown time zone {m.group(1)}') - continue - if m := re.fullmatch(r'(\w+)=.*', line): - if m.group(1) in IGNORED_VARIABLES: - continue - raise ScheduleError(f'{m.group(1)}= is not supported') - if line.startswith('@'): - expr, _, rest = line.partition(' ') - else: - fields = line.split(None, 5) - if len(fields) < 6: - raise ScheduleError('expected five time fields and a command') - expr, rest = ' '.join(fields[:5]), fields[5] - if expr == '@reboot': - raise ScheduleError('@reboot is not supported: the schedule starts with the bot') - if not croniter.is_valid(expr): - raise ImpossibleTime(f"bad time '{expr}'") - try: - croniter(expr, datetime.datetime(2000, 1, 1)).get_next(datetime.datetime) - except CroniterBadDateError: - raise ImpossibleTime(f"'{expr}' never comes: no date matches it") - commands = parse_command(rest) - key = f"{expr}|{zone.key}|{';'.join(label for label, _, _ in commands)}" - jobs.append(Job(expr, commands, zone, line, key)) - except ScheduleError as e: - errors.append((number, line, e)) - return jobs, errors + zone = zoneinfo.ZoneInfo(m.group(1)) + except (ValueError, zoneinfo.ZoneInfoNotFoundError): + raise ScheduleError(f'no time zone {m.group(1)!r} (such as America/New_York)') + line = line[m.end():] + if line.startswith('@'): + when, _, reports = line.partition(' ') + else: + fields = line.split(None, 5) + if len(fields) < 6: + raise ScheduleError('a job is five time fields (minute hour day month weekday), then its reports') + when, reports = ' '.join(fields[:5]), fields[5] + check_time(when) + return Job(when, parse_reports(reports, system), zone, chat, on, reports.strip()) class Scheduler: - """Fires the jobs of a schedule file. `path` is a callable returning the file (the data dir copy wins over the - default, so a new copy is found on reload). `clock`, `sleep` and `spawn` can be replaced, which the tests do. Its - lock covers what the loop, a reload and runs from chat share: the jobs, their bases, the record of runs and what is - running.""" - - def __init__(self, path, catchup=None, clock=None, sleep=None, spawn=None, zone=None, runner=None): - """path() gives the schedule file's path. catchup is how late, in minutes, a missed run may still start (the - scheduler_catchup setting when None); clock, sleep, spawn, zone and runner stand in for the real ones in - tests.""" - self.path = path + """Fires the jobs: each chat's, from SCHEDULE_FILE, and the system jobs. clock, sleep, spawn and runner can be + replaced, which the tests do. Its lock covers what the loop, a change from chat and the runs share: the jobs, their + bases, the record of runs and what is running.""" + + def __init__(self, catchup=None, clock=None, sleep=None, spawn=None, runner=None): + """catchup is how late, in minutes, a missed run may still start (the scheduler_catchup setting when None); + clock, sleep, spawn and runner stand in for the real ones in tests.""" self.catchup = datetime.timedelta(minutes=config.env.scheduler_catchup if catchup is None else catchup) self.clock = clock or (lambda: datetime.datetime.now(datetime.timezone.utc)) self.sleep = sleep or time.sleep self.spawn = spawn or start_thread - self.runner = runner or run_in_process # (label, module, kwargs): runs the command - self.zone = zone + self.runner = runner or run_in_process # (label, module, kwargs, dest): runs the command self.jobs = [] self.base = {} # key -> the time after which the next run is due self.last = {} - self.busy = set() # labels of the commands running now, scheduled or manual + self.busy = set() # (chat, label) of the commands running now, scheduled or manual self.lock = threading.Lock() def load(self): - """Read the schedule file. A line with an impossible time refuses the whole file: ImpossibleTime is raised, - naming every such line, and nothing changes. Other bad lines are reported and skipped; a file with no usable - line keeps the schedule that is running.""" - path = self.path() - try: - jobs, errors = parse_schedule(Path(path).read_text(encoding='utf-8'), self.zone) - except OSError as e: - delivery.report_error(e, context='schedule') - return False - impossible = [f'{path} line {number}: {error}: {line}' for number, line, error in errors - if isinstance(error, ImpossibleTime)] - if impossible: - raise ImpossibleTime(impossible) - for number, line, error in errors: - delivery.report_error(ScheduleError(f'{path} line {number}: {error}: {line}'), context='schedule') - if not jobs and self.jobs: - print('schedule: no usable line in', path, '- keeping the current schedule', file=sys.stderr) - return False + """Read every chat's jobs from SCHEDULE_FILE, besides the system jobs: at start, and after every change from a + chat. A job the file holds that cannot be any more (a report since retired) is reported and left out. A job + kept from before keeps its timing; a new one counts from now, as one does that was off.""" + jobs = [parse_job(f'{when} {reports}', system=True) for when, reports in SYSTEM] + for chat, entries in state.load(SCHEDULE_FILE, {}).items(): + for entry in entries: + try: + zone = '' if entry.get('zone', 'UTC') == 'UTC' else f"CRON_TZ={entry['zone']} " + jobs.append(parse_job(f"{zone}{entry['when']} {entry['reports']}", chat, entry.get('on', True))) + except (ScheduleError, KeyError, TypeError) as e: + delivery.report_error(ScheduleError(f'a job of {chat} left out: {e}: {entry}'), context='schedule') with self.lock: # no run is recorded between reading the record and swapping it in try: last, problem = state.load(STATE_FILE), None @@ -288,21 +217,14 @@ def load(self): self.jobs = jobs if problem: delivery.report_error(ScheduleError(problem), context='schedule') - print(f"schedule: {len(jobs)} job{'' if len(jobs) == 1 else 's'} from {path}", file=sys.stderr) - return True - - def reload(self, *args): - """SIGHUP: read the edited file. One with an impossible time is refused and reported; the schedule stays.""" - try: - self.load() - except ImpossibleTime as e: - problem = f'not reloaded, the running schedule is unchanged: {e}' - delivery.report_error(ScheduleError(problem), context='schedule') + chats = len({job.chat for job in jobs if job.chat}) + print(f"schedule: {len(jobs) - len(SYSTEM)} jobs of {chats} chat{'' if chats == 1 else 's'}, and" + f" {len(SYSTEM)} system jobs", file=sys.stderr) def start_base(self, job, now): """Where to count from at start: a run missed in the catch-up window fires once; with no record, none is.""" last = self.last.get(job.key) - if not last: + if not last or not job.on: return now try: return max(datetime.datetime.fromisoformat(last), now - self.catchup) @@ -314,7 +236,7 @@ def next_fire(self, job, after): put back after: croniter up to 6.0 (the distro packages) mishandles zoneinfo across DST changes, firing daily jobs twice or an hour late. A time the clocks repeat fires once, the first time; one they skip (02:30 as they go forward) fires an hour later by the clock.""" - times = croniter(job.expr, after.astimezone(job.zone).replace(tzinfo=None)) + times = croniter(job.when, after.astimezone(job.zone).replace(tzinfo=None)) while True: # fold 0: the first of a repeated time t = times.get_next(datetime.datetime).replace(tzinfo=job.zone).timestamp() @@ -323,10 +245,14 @@ def next_fire(self, job, after): return datetime.datetime.fromtimestamp(t, job.zone) def due(self, now): - """The jobs to run now, as (job, fire time). Several missed fire times of one job run it once.""" + """The jobs to run now, as (job, fire time). Several missed fire times of one job run it once. A job that is off + never fires, and counts from now: switched on, it does not catch up.""" fired = [] with self.lock: for job in self.jobs: + if not job.on: + self.base[job.key] = now + continue base, latest = self.base[job.key], None while (t := self.next_fire(job, base)) <= now: latest, base = t, t @@ -341,101 +267,136 @@ def due(self, now): print('schedule: cannot save', STATE_FILE, e, file=sys.stderr) return fired - def claim(self, label): - """Mark a command running, unless it already is (scheduled or manual): whether it was claimed.""" + def claim(self, chat, label): + """Mark a chat's command running, unless it already is (scheduled or manual): whether it was claimed.""" with self.lock: - if label in self.busy: + if (chat, label) in self.busy: return False - self.busy.add(label) + self.busy.add((chat, label)) return True - def execute(self, commands): - """Run commands one after another (a job's), in this thread; a failure is reported. One already running (from - the line's last run, another line or chat) is skipped. One still running after JOB_TIMEOUT is reported as late, - once, and runs on: a thread cannot be stopped, but every request it makes has a timeout, so it ends; until then - it stays marked running, and no second copy starts. Returns 'done', 'failed' or 'already running'.""" + def execute(self, commands, chat=''): + """Run commands one after another (a job's), in this thread, posting to the chat whose job it is (its key; '' + for a system job, whose failure goes to the home chats); a failure is reported. One already running for that + chat is skipped. One still running after JOB_TIMEOUT is reported as late, once, and runs on: a thread cannot be + stopped, but every request it makes has a timeout, so it ends; until then it stays marked running, and no + second copy starts. Returns 'done', 'failed' or 'already running'.""" + dest = delivery.chat(chat, scheduled=True) if chat else None outcome = 'done' for label, module, kwargs in commands: - if not self.claim(label): - print(f'schedule: {label} is already running, skipping it', file=sys.stderr) + if not self.claim(chat, label): + print(f'schedule: {label} is already running for {chat or "the system"}, skipping it', file=sys.stderr) outcome = 'already running' continue - print(f'schedule: running {label}', file=sys.stderr) - late = threading.Timer(JOB_TIMEOUT, delivery.report_error, kwargs={'context': label, - 'e': TimeoutError(f'still running after {JOB_TIMEOUT / 60:g} minutes, and left to finish')}) + print(f'schedule: running {label}' + (f' for {chat}' if chat else ''), file=sys.stderr) + late = threading.Timer(JOB_TIMEOUT, delivery.report_error, args=(TimeoutError(f'still running after' + f' {JOB_TIMEOUT / 60:g} minutes, and left to finish'), dest), kwargs={'context': label}) late.daemon = True late.start() try: - self.runner(label, module, kwargs) + self.runner(label, module, kwargs, dest) except (Exception, SystemExit) as e: - delivery.report_error(e, context=label) + delivery.report_error(e, dest, context=label) outcome = 'failed' finally: late.cancel() with self.lock: - self.busy.discard(label) + self.busy.discard((chat, label)) return outcome - def commands(self, hidden=False): - """Every scheduled command offered in chat once, or with hidden=True each of the HIDDEN scripts' (System), - sorted by name with numbers in numeric order: [(label, module, kwargs)]. The time, comments, redirects and the - other commands of a line are not part of it.""" - unique = {} - system = {SCRIPTS[name] for name in HIDDEN} - for job in self.jobs: - for command in job.commands: - if (command[1] in system) == hidden: - unique.setdefault(command[0], command) - natural = lambda label: [int(part) if part.isdigit() else part for part in re.split(r'(\d+)', label.lower())] - return [unique[label] for label in sorted(unique, key=natural)] - - def select(self, text): - """The scheduled command for `.jobs run `: a number in the list, or the command. Raises ScheduleError.""" - text = text.strip() - commands = self.commands() - if text.isdigit(): - if not 1 <= int(text) <= len(commands): - raise ScheduleError(f'no command {text}; send .jobs to list them') - return commands[int(text) - 1] - try: - wanted = parse_command(text) - except ScheduleError as e: - raise ScheduleError(f'{e}; send .jobs to list the commands') - if len(wanted) != 1: - raise ScheduleError('run one command at a time; send .jobs to list them') - for command in commands: - if command[0] == wanted[0][0]: - return command - if any(command[0] == wanted[0][0] for command in self.commands(hidden=True)): - raise ScheduleError(f'{display(wanted[0][0])} is a system job: it runs on its schedule only') - raise ScheduleError(f"'{text}' is not in the schedule; send .jobs to list the commands") - - def pick(self, number, crc): - """The offered command a button stands for. Button data is untrusted input: the number must be plain digits - within the list and the checksum must match that very command, else ScheduleError.""" - commands = self.commands() - if not (number.isascii() and number.isdigit() and 1 <= int(number) <= len(commands)): - raise ScheduleError('no such command') - command = commands[int(number) - 1] - if not hmac.compare_digest(checksum(command[0]), crc): - raise ScheduleError('the schedule changed') - return command - - def run_command(self, command): - """Run one command returned by select() or pick(). Returns (label, outcome).""" - return command[0], self.execute([command]) - - def run_now(self, text): - """`.jobs run`: one scheduled command now, as cron runs it; the schedule stays. Returns (label, outcome).""" - return self.run_command(self.select(text)) + def running(self, chat): + """The labels of the commands running now for a chat (by its key), and for the system.""" + with self.lock: + return {label for where, label in self.busy if where in (chat, '')} + + def chat_jobs(self, chat): + """A chat's jobs, in their order: .jobs numbers them from 1.""" + with self.lock: + return [job for job in self.jobs if job.chat == chat] + + def system_jobs(self): + """The system jobs.""" + with self.lock: + return [job for job in self.jobs if not job.chat] + + def job(self, chat, number): + """A chat's job by its number in .jobs (a string, from a chat). Raises ScheduleError.""" + jobs = self.chat_jobs(chat) + if not (number.isascii() and number.isdigit() and 1 <= int(number) <= len(jobs)): + raise ScheduleError(f'no job {number}: send .jobs for the list' if jobs else 'no jobs here yet: .jobs add') + return jobs[int(number) - 1] + + def pick(self, chat, number, crc): + """The job a button stands for. Button data is untrusted input: the number must be plain digits within this + chat's list, and the checksum must match that very job, else ScheduleError.""" + job = self.job(chat, number) + if not hmac.compare_digest(checksum(job.key), crc): + raise ScheduleError('the jobs have changed') + return job + + def run(self, job): + """Run a job now, as its schedule would: its outcome (see execute).""" + return self.execute(job.commands, job.chat) + + def change(self, chat, edit): + """Change a chat's jobs, as stored: edit(entries) does it to the list (and may raise ScheduleError); then they + are read again, so that the change counts at once. Returns what edit returned.""" + with state.edit(SCHEDULE_FILE, {}) as chats: + entries = chats.setdefault(chat, []) + result = edit(entries) + if not entries: + del chats[chat] + self.load() + return result + + def add(self, chat, line): + """Add a job to a chat's list (a line as parse_job takes it), on: its number. Raises ScheduleError.""" + job = parse_job(line, chat) + def append(entries): + """Append the job, within MAX_JOBS.""" + if len(entries) >= MAX_JOBS: + raise ScheduleError(f'{MAX_JOBS} jobs is the most a chat may have: delete one first') + entries.append({'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': True}) + return len(entries) + return self.change(chat, append) + + def delete(self, job): + """Delete a chat's job (from job() or pick()).""" + self.change(job.chat, lambda entries: entries.pop(self.index(entries, job))) + + def toggle(self, job): + """Switch a chat's job on or off: whether it is on now.""" + def flip(entries): + """Flip the job's switch.""" + entry = entries[self.index(entries, job)] + entry['on'] = not entry.get('on', True) + return entry['on'] + return self.change(job.chat, flip) + + @staticmethod + def index(entries, job): + """Where a job is among its chat's stored entries. Raises ScheduleError when it is gone meanwhile.""" + for i, entry in enumerate(entries): + zone = entry.get('zone', 'UTC') + if (entry.get('when'), entry.get('reports'), zone) == (job.when, job.text, job.zone.key): + return i + raise ScheduleError('that job is gone: send .jobs for the list') + + def next_run(self, job): + """When a job fires next, or None when it is off.""" + if not job.on: + return None + with self.lock: + base = self.base.get(job.key) or self.clock() + return self.next_fire(job, max(base, self.clock())) def tick(self): """Start what is due; returns the seconds until the next fire time.""" now = self.clock() for job, fire_time in self.due(now): - self.spawn(self.execute, job.commands) + self.spawn(self.execute, job.commands, job.chat) with self.lock: - waits = [(self.next_fire(job, self.base[job.key]) - now).total_seconds() for job in self.jobs] + waits = [(self.next_fire(job, self.base[job.key]) - now).total_seconds() for job in self.jobs if job.on] return min(30, max(1, math.ceil(min(waits)))) if waits else 30 def run_forever(self): diff --git a/lib/migrate.py b/lib/migrate.py new file mode 100644 index 00000000..c3586206 --- /dev/null +++ b/lib/migrate.py @@ -0,0 +1,188 @@ +"""Once, as finbot starts: what the bot kept for all chats at once becomes each home chat's (telegramChatID, +slackChannelID, discordChannelID). The old schedule.cron (the data dir's copy, else the shipped one) is copied to every +home chat as its jobs, each in the zone it ran in, and is read no more; and the reports' records of what they told the +chats (trades, ratings, record highs, podcast episodes) are copied to each home chat, so that nothing old is told again. + +This module, and defaults/schedule.cron, go once every install has started with it: they are its only users.""" +import copy +import datetime +import glob +import os +import re +import shlex +import sys +import zoneinfo +from pathlib import Path + +from lib import config +from lib import delivery +from lib import jobs +from lib import state +from lib.reports import milestone, podcasts, rating, trades + +RECORDS = (trades.STATE_FILE, rating.STATE_FILE, milestone.STATE_FILE) # and each podcast feed's seen file +# what a shell reads as more than a word, outside quotes, in a pasted crontab line +OPERATOR = re.compile(r'\d*>>?&?\d*|&>>?|;|&&|\|\|?|&|[<`$(){}]') +REDIRECT = re.compile(r'\d*>>?&?\d*|&>>?') # > file, >> file, 2>&1, &> file + + +def run(): + """Make the home chats' own what was kept for all chats, where that has not been done yet. With no home chat set, + it all waits until there is one.""" + try: + homes = [chat.key for chat in delivery.default_chats()] + except RuntimeError: + return + schedule(homes) + feeds = sorted(os.path.basename(path) for path in glob.glob(state.path(podcasts.SEEN_FILES))) + for name in RECORDS + tuple(feeds): + record(name, homes) + + +def schedule(homes): + """Give every home chat (keys) the jobs of the old schedule.cron, the system jobs aside (they run in code now), + unless the chats' jobs are kept already (jobs.SCHEDULE_FILE). A line that cannot be a job any more is logged and + left out.""" + if os.path.exists(state.path(jobs.SCHEDULE_FILE)): + return + path = config.data_file('schedule.cron') + try: + text = Path(path).read_text(encoding='utf-8') + except OSError: + text = '' + entries = [] + for number, line, zone, words_list in legacy_lines(text, path): + reports = '; '.join(shlex.join([name, *arguments]) for name, arguments in words_list + if name not in jobs.SYSTEM_SCRIPTS) + if not reports: + continue + try: + job = jobs.parse_job(f"{'' if zone == 'UTC' else f'CRON_TZ={zone} '}{line} {reports}") + except jobs.ScheduleError as e: + print(f'schedule: {path} line {number} left out: {e}', file=sys.stderr) + continue + entries.append({'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': True}) + # written even when empty: the file being there says the migration is done + state.save(jobs.SCHEDULE_FILE, {key: copy.deepcopy(entries) for key in homes if entries}) + print(f"schedule: the {len(entries)} jobs of {path} are now each home chat's ({', '.join(homes)});" + f" finbot reads that file no more: each chat's jobs are in {jobs.SCHEDULE_FILE}, changed with .jobs", + file=sys.stderr) + + +def record(name, homes): + """A report's record kept for all chats at once (anything but {'chats': ...}) becomes each home chat's.""" + try: + data = state.load(name) + except (OSError, ValueError) as e: + print(f'state: {name} left as it is, unreadable: {e}', file=sys.stderr) + return + if data is None or (isinstance(data, dict) and 'chats' in data): + return + state.save(name, {'chats': {key: copy.deepcopy(data) for key in homes}}) + print(f"state: {name} is kept per chat now, its record copied to {', '.join(homes)}", file=sys.stderr) + + +def local_zone(): + """The server's own time zone, which the old schedule ran in, as cron does: UTC by that name when it is UTC by + another (Etc/UTC), or when there is none.""" + try: + if os.environ.get('TZ'): + zone = zoneinfo.ZoneInfo(os.environ['TZ'].lstrip(':')) + elif '/zoneinfo/' in (link := os.path.realpath('/etc/localtime')): + zone = zoneinfo.ZoneInfo(link.split('/zoneinfo/', 1)[1]) + else: + zone = zoneinfo.ZoneInfo(Path('/etc/timezone').read_text().strip()) + except (OSError, ValueError, zoneinfo.ZoneInfoNotFoundError): + return jobs.UTC + year = [datetime.datetime(2026, month, 1, tzinfo=zone).utcoffset() for month in (1, 7)] + return jobs.UTC if year == [datetime.timedelta(0)] * 2 else zone + + +def legacy_lines(text, path): + """The jobs of an old schedule.cron (text, from path): (line number, time, zone name, [(report name, arguments)]) + for each line that parses, in the zone set above it (CRON_TZ=) or the server's own; pasted crontab syntax (paths, + .py, redirects, braces, variables) read as cron read it. A line that does not parse is logged.""" + zone = local_zone().key + for number, raw in enumerate(text.splitlines(), 1): + line = raw.strip() + if not line or line.startswith('#'): + continue + if m := re.fullmatch(r'CRON_TZ=(\S+)', line): + zone = m.group(1) + continue + if re.fullmatch(r'\w+=.*', line): # MAILTO=, PATH= and the like + continue + if line.startswith('@'): + when, _, rest = line.partition(' ') + else: + fields = line.split(None, 5) + if len(fields) < 6: + continue + when, rest = ' '.join(fields[:5]), fields[5] + try: + commands = [(os.path.basename(words[0]).removesuffix('.py'), words[1:]) + for words in split_commands(tokenize(rest))] + except ValueError as e: + print(f'schedule: {path} line {number} left out: {e}', file=sys.stderr) + continue + yield number, when, zone, commands + + +def tokenize(text): + """A command line as ('word', text) and ('op', text) tokens. Quoted text is always a word, whatever it contains.""" + tokens, word, started, i = [], '', False, 0 + def flush(): + """End the word being read, if there is one, and add it to the tokens.""" + nonlocal word, started + if started: + tokens.append(('word', word)) + word, started = '', False + while i < len(text): + c = text[i] + if c.isspace(): + flush() + i += 1 + elif c in '\'"': + end = text.find(c, i + 1) + if end < 0: + raise ValueError('no closing quotation') + word, started, i = word + text[i + 1:end], True, end + 1 + elif c == '\\' and i + 1 < len(text): + word, started, i = word + text[i + 1], True, i + 2 + elif c == '#' and not started: # a trailing comment + break + elif m := OPERATOR.match(text, i): + if word.isdigit() and m.group().startswith('>'): # the 2 of 2>&1 belongs to the redirect + word, started = '', False + flush() + tokens.append(('op', m.group())) + i = m.end() + else: + word, started, i = word + c, True, i + 1 + flush() + return tokens + + +def split_commands(tokens): + """Word lists, one per command: split on unquoted ; and &&, redirects and braces dropped, variables before a command + dropped, other shell syntax refused (ValueError).""" + commands, current, target = [], [], False + for kind, value in tokens: + if target: # the file of a redirect + target = False + if kind == 'word': + continue + if kind == 'word': + if current or not re.fullmatch(r'\w+=\S*', value): # FOO=bar before a command + current.append(value) + elif value in (';', '&&'): + commands.append(current) + current = [] + elif value in ('{', '}'): + continue + elif REDIRECT.fullmatch(value): + target = not re.search(r'&\d+$', value) # 2>&1 has no file after it, > /dev/null does + else: + raise ValueError('only finbot scripts can be scheduled, not shell commands') + commands.append(current) + return [words for words in commands if words] diff --git a/lib/reports/about.py b/lib/reports/about.py index ef89bab0..ea6c8dbb 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -1,49 +1,60 @@ -"""What finbot can do: the .help text, and the .jobs listing of the schedule (with running a job from it).""" +"""What finbot can do: the .help text, and the .jobs listing of a chat's jobs and the reports a job can run.""" from lib import buttons from lib import config from lib import formats from lib import jobs -def prepare_jobs_payload(service, run=None, tappable=False): - """`.jobs`: the scheduled commands, each once and numbered, then the system jobs (HIDDEN) under their own heading, - unnumbered: they run on their schedule only. With `run` (a number in that list, or a command's text) that command - run now, as cron would. Returns (payload, buttons): with tappable (the chat takes buttons) a button per numbered - command, two to a row.""" +def prepare_jobs_payload(service, chat, action=None, tappable=False): + """`.jobs`: a chat's jobs (by its key), numbered, each with its line, whether it is on and its next run, in UTC and + in the timezone preference; then the system jobs, under their own heading and unnumbered: they run on their + schedule only; then the subcommands. With action ('run' or 'toggle'), the list of what a tap does, and where the + chat takes buttons (tappable) a button per job that runs it now or switches it on or off, two to a row. Returns + (payload, buttons).""" scheduler = jobs.current if scheduler is None: return ['No schedule is running'], None - if run: - return [run_job_line(scheduler, run)], None - commands, system = scheduler.commands(), scheduler.commands(hidden=True) - if not commands and not system: - return ['No scheduled jobs'], None - shown = lambda label: jobs.display(label) + (' (running)' if label in scheduler.busy else '') - payload = [] - if commands: - payload = [service.bold('Scheduled commands:')] + [f"{number}. {shown(label)}" - for number, (label, _, _) in enumerate(commands, 1)] - if system: - payload += ([''] if payload else []) + [service.bold('System:')] + [shown(label) for label, _, _ in system] - if commands: - payload.append("\nRun one with .jobs run NUMBER" + (' or tap it' if tappable else '')) + own = scheduler.chat_jobs(chat) + running = scheduler.running(chat) + payload = [service.bold('Jobs here:')] + [job_entry(service, scheduler, number, job, running) + for number, job in enumerate(own, 1)] if own else ['No jobs here yet: add one with .jobs add'] + system = scheduler.system_jobs() + payload += ['', service.bold('System:')] + [f"{job.when} {job.text}" + (' (running)' if any( + label in running for label, _, _ in job.commands) else '') for job in system] + if action and own: + payload.append('\n' + ('Tap one to run it now' if action == 'run' else 'Tap one to switch it on or off') + if tappable else f"\nSend .jobs {action} NUMBER") + elif not action: + payload += ['', '.jobs run [NUMBER]: run one now', '.jobs toggle [NUMBER]: switch one on or off', + '.jobs add [CRON_TZ=ZONE] TIME REPORT: add one (.jobs add lists the reports)', + '.jobs del NUMBER: delete one'] markup = None - if tappable and commands: - row = [buttons.job_button(number, jobs.display(label), jobs.checksum(label)) - for number, (label, _, _) in enumerate(commands, 1)] + if action and tappable and own: + row = [buttons.job_button(number, job.text, jobs.checksum(job.key), action) + for number, job in enumerate(own, 1)] markup = [row[i:i + 2] for i in range(0, len(row), 2)] return payload, markup -def run_job_line(scheduler, text): - """Run a scheduled command now and say how it went in one line.""" - try: - return job_line(scheduler, *scheduler.run_now(text)) - except jobs.ScheduleError as e: - return str(e) +def job_entry(service, scheduler, number, job, running=()): + """'1. */20 * * * Mon-Fri trades: on, next Thu 14:20 UTC (Fri 01:20 AEDT)': a job in .jobs.""" + state = 'running' if any(label in running for label, _, _ in job.commands) else 'on' if job.on else 'off' + when = scheduler.next_run(job) + if when is None: + return f"{number}. {job.line}: {state}" + utc, local = when.astimezone(jobs.UTC), when.astimezone(formats.zone()) + shown = f"{utc:%a %H:%M} UTC" + ('' if local.utcoffset() == utc.utcoffset() else f" ({local:%a %H:%M %Z})") + return f"{number}. {job.line}: {state}, next {shown}" -def job_line(scheduler, label, outcome): - """'3. trades.py: done': a command's number in the .jobs list, its name, and how its run went.""" - number = [command[0] for command in scheduler.commands()].index(label) + 1 - return f"{number}. {jobs.display(label)}: {outcome}" +def job_line(number, job, outcome): + """'3. trades: done': a job's number in the .jobs list, its reports, and how its run went.""" + return f"{number}. {job.text}: {outcome}" + +def prepare_reports(service): + """`.jobs add` alone: the reports a job can run, each with what it takes, and examples.""" + names = [name for name in jobs.SCRIPTS if name not in jobs.SYSTEM_SCRIPTS] + return [service.bold('Reports a job can run:')] + [f"{name} {jobs.SCRIPTS[name].ARGUMENTS}".strip() + for name in sorted(names)] + ['', service.bold('For example:'), '.jobs add 29 21 * * Mon-Fri price interday', + '.jobs add 28 21 * * Sat cal earnings; price 7 (several in turn)', + '.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession (UTC unless it says)'] def prepare_help(service, botName, sections): """The .help text: each section's command lines under its heading (worker.help_sections() has them), then the bot's diff --git a/lib/reports/cal.py b/lib/reports/cal.py index 35140379..2fa38f15 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -1,4 +1,4 @@ -""".earnings and .dividend: upcoming earnings and ex-dividend dates (scheduled: cal.py earnings | ex-dividend).""" +""".earnings and .dividend: upcoming earnings and ex-dividend dates (scheduled: cal earnings | ex-dividend).""" import datetime from lib import config @@ -87,10 +87,12 @@ def report(service, events, days, specific_stock, interactive, event): payload = [f"No events found for {specific_stock}" if specific_stock else f"No events found {ahead}"] return payload +ARGUMENTS = 'earnings|ex-dividend' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case ['earnings' | 'ex-dividend' as event]: - return f'cal.py {event}', {'event': event} + return f'cal {event}', {'event': event} case _: - raise ValueError('[earnings|ex-dividend]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/cleanup.py b/lib/reports/cleanup.py index 3ffb74cb..cb561215 100644 --- a/lib/reports/cleanup.py +++ b/lib/reports/cleanup.py @@ -1,4 +1,4 @@ -"""Deletes old cache files (scheduled: cleanup.py [days]); with days=0 it is the cache flush.""" +"""Deletes old cache files (scheduled: cleanup [days]); with days=0 it is the cache flush.""" import os, sys, time from lib import config @@ -23,12 +23,14 @@ def run(*, days=30): print("cleanup: can't remove", entry.path, e, file=sys.stderr) return removed +ARGUMENTS = '[DAYS]' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case []: - return 'cleanup.py', {} + return 'cleanup', {} case [days] if days.isdigit(): - return f'cleanup.py {days}', {'days': int(days)} + return f'cleanup {days}', {'days': int(days)} case _: - raise ValueError('[days]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/milestone.py b/lib/reports/milestone.py index d9bffd17..a3255a38 100644 --- a/lib/reports/milestone.py +++ b/lib/reports/milestone.py @@ -1,4 +1,4 @@ -"""Record highs and lows (all-time) among the stocks followed (scheduled: milestone.py).""" +"""Record highs and lows (all-time) among the stocks followed (scheduled: milestone).""" from lib import state from lib import delivery from lib import formats @@ -10,25 +10,26 @@ # - need to handle stock splits: flush cache if we see a split in the data # https://finance.yahoo.com/quote/TSLA/history?period1=0&period2=1707609600&filter=split -STATE_FILE = 'finbot_milestone.json' # {ticker: {'all_time_high': ..., 'all_time_low': ...}} +STATE_FILE = 'finbot_milestone.json' # each chat's {ticker: {'all_time_high': ..., 'all_time_low': ...}} def run(dest=None): - """Post the stocks followed that set an all-time high or low to the default chats (or dest), and record the new + """Post the stocks followed that set an all-time high or low, against each chat's record, to each of the run's chats + (dest, a chat's job or the chat that asked; else the home chats); a scheduled run records each chat's new records.""" - market_data, highs, lows, changed = gather() + interactive = delivery.interactive(dest) + tickers = watchlist.with_holdings() + market_data = yahoo.fetch(tickers) for chat in delivery.targets(dest): + highs, lows, changed = compare(tickers, market_data, state.load_for(chat.key, STATE_FILE, {})) delivery.deliver(chat, delivery.Report(report(chat.dialect, market_data, highs, lows))) - if changed: - with state.edit(STATE_FILE, {}) as recorded: - recorded.update(changed) + if changed and not interactive: + with state.edit_for(chat.key, STATE_FILE, {}) as recorded: + recorded.update(changed) -def gather(): - """(the quotes, the tickers at a record high, those at a record low, {ticker: its records after this run}) of the - stocks followed: each one's 52-week range against its all-time record, which a first sighting takes from its - history.""" - tickers = watchlist.with_holdings() - market_data = yahoo.fetch(tickers) - records = state.load(STATE_FILE, {}) +def compare(tickers, market_data, records): + """(the tickers at a record high, those at a record low, {ticker: its records after this run}): each one's 52-week + range (market_data, the quotes) against its all-time record in a chat's records, which a first sighting takes from + its history.""" highs, lows, changed = [], [], {} for ticker in tickers: try: @@ -49,7 +50,7 @@ def gather(): highs.append(ticker) if newlow < oldlow: lows.append(ticker) - return market_data, highs, lows, changed + return highs, lows, changed def report(service, market_data, highs, lows): """The records in one network's markup: the record highs, then the record lows, each list sorted.""" @@ -59,6 +60,11 @@ def line(ticker): return formats.sections(highs and [service.bold('Record high:')] + sorted(line(ticker) for ticker in highs), lows and [service.bold('Record low:')] + sorted(line(ticker) for ticker in lows)) +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: milestone.py takes none.""" - return 'milestone.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); milestone takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'milestone', {} diff --git a/lib/reports/performance.py b/lib/reports/performance.py index f45f1c2b..fee2ac6b 100644 --- a/lib/reports/performance.py +++ b/lib/reports/performance.py @@ -1,4 +1,4 @@ -""".performance: each portfolio's return over a period, beside two benchmarks (scheduled: performance.py [days]).""" +""".performance: each portfolio's return over a period, beside two benchmarks (scheduled: performance [days]).""" import sys from lib import config @@ -61,12 +61,14 @@ def run(dest=None, *, past_days=None, user='', portfolio_select=None): markup = buttons.keyboard('f', [portfolio_select or ''], buttons.period_for_days(past_days)) delivery.deliver(chat, delivery.Report(payload, graph, markup)) +ARGUMENTS = '[DAYS]' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case []: - return 'performance.py', {} + return 'performance', {} case [days] if days.isdigit(): - return f'performance.py {days}', {'past_days': int(days)} + return f'performance {days}', {'past_days': int(days)} case _: - raise ValueError('[integer]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/podcasts.py b/lib/reports/podcasts.py index db9b401f..a7f7cc79 100644 --- a/lib/reports/podcasts.py +++ b/lib/reports/podcasts.py @@ -1,4 +1,4 @@ -"""New episodes of a podcast feed (scheduled: podcasts.py "").""" +"""New episodes of a podcast feed (scheduled: podcasts FEED_URL).""" import hashlib import string import xml.etree.ElementTree as ET @@ -14,8 +14,10 @@ } NEWEST = 5 # episodes posted per run at most: the other new ones (on a first run, say) are only recorded as seen +SEEN_FILES = 'finbot_podcast_seen_*.json' # each feed's, as get_seen_filename names them + def get_seen_filename(url): - """The state file, in the data directory, of the episodes of this feed already seen.""" + """The state file, in the data directory, of the episodes of this feed already seen (by each chat).""" return f"finbot_podcast_seen_{hashlib.md5(url.encode()).hexdigest()}.json" def fetch_and_parse_feed(url): @@ -65,42 +67,43 @@ def media_link(item): return None def fetch_new_episodes(feed_url, seen): - """(the NEWEST newest episodes whose guid is not in `seen`, newest first; the guids of all the new ones).""" + """The feed's episodes whose guid is not in `seen`, newest first, each with its guid.""" channel = fetch_and_parse_feed(feed_url).find("channel") if channel is None: raise RuntimeError("No element found") podcast_name = (channel.findtext("title") or "Podcast").strip() - episodes, guids = [], set() + episodes = [] for item in channel.findall("item"): guid = item.findtext("guid") or item.findtext("link") media_url = media_link(item) if not guid or guid in seen or not media_url: continue when = published(item.findtext("pubDate")) - episodes.append({"podcast": podcast_name, "title": (item.findtext("title") or "Untitled").strip(), + episodes.append({"guid": guid, "podcast": podcast_name, "title": (item.findtext("title") or "Untitled").strip(), "url": media_url, "ago": how_long_ago(when), "duration": format_duration(item.findtext("itunes:duration", namespaces=NAMESPACES)), "published": when or datetime.min.replace(tzinfo=timezone.utc)}) - guids.add(guid) episodes.sort(key=lambda episode: episode["published"], reverse=True) - return episodes[:NEWEST], guids + return episodes def run(dest=None, *, feed_url): - """Post the feed's newest episodes not posted before to the default chats (or dest), then record every new one as - seen.""" + """Post to each of the run's chats (dest, a chat's job; else the home chats) the feed's newest episodes not posted + to it before, then record every new one as seen by it.""" seen_file = get_seen_filename(feed_url) - seen = set(state.load(seen_file, [])) + chats = delivery.targets(dest) + seen = {chat.key: set(state.load_for(chat.key, seen_file, [])) for chat in chats} try: - episodes, guids = fetch_new_episodes(feed_url, seen) + episodes = fetch_new_episodes(feed_url, set.intersection(*seen.values())) # every chat has seen those except Exception as e: raise RuntimeError(f"fetching feed {feed_url}: {e}") - if not guids: - return - for chat in delivery.targets(dest): - delivery.deliver(chat, delivery.Report(report(chat.dialect, episodes))) - with state.edit(seen_file, []) as recorded: # last, so that a failure above leaves these episodes for next run - recorded[:] = sorted(guids.union(recorded)) + for chat in chats: + new = [episode for episode in episodes if episode['guid'] not in seen[chat.key]] + if not new: + continue + delivery.deliver(chat, delivery.Report(report(chat.dialect, new[:NEWEST]))) + with state.edit_for(chat.key, seen_file, []) as recorded: # last: a failure above leaves them for next run + recorded[:] = sorted({episode['guid'] for episode in new}.union(recorded)) def report(service, episodes): """The episodes in one network's markup, each linked to its media, with its length and age.""" @@ -112,11 +115,13 @@ def report(service, episodes): payload.append(f"📣 {podcast_name}: {service.link(episode['url'], title)}{duration} ({episode['ago']})") return payload +ARGUMENTS = 'FEED_URL' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case [feed_url]: # the label matches the seen file's name - return 'podcasts.py ' + hashlib.md5(feed_url.encode()).hexdigest()[:8], {'feed_url': feed_url} + return 'podcasts ' + hashlib.md5(feed_url.encode()).hexdigest()[:8], {'feed_url': feed_url} case _: - raise ValueError('""') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/price.py b/lib/reports/price.py index 3a904e2d..bef364a0 100644 --- a/lib/reports/price.py +++ b/lib/reports/price.py @@ -1,4 +1,4 @@ -""".price, .session, .premarket, .postmarket: price moves past a threshold (scheduled: price.py midsession | interday | +""".price, .session, .premarket, .postmarket: price moves past a threshold (scheduled: price midsession | interday | premarket | close | ).""" import dataclasses import datetime @@ -191,12 +191,14 @@ def markup(chat): payload, graph = report(chat.dialect, found, mode, threshold, days, top, user, specific_stock, interactive) delivery.deliver(chat, delivery.Report(payload, graph, markup(chat) if graph else None)) +ARGUMENTS = 'midsession|interday|premarket|close|DAYS' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case ['midsession' | 'interday' | 'premarket' | 'close' as mode]: - return f'price.py {mode}', {'mode': mode} + return f'price {mode}', {'mode': mode} case [days] if days.isdigit(): - return f'price.py {days}', {'mode': 'period', 'threshold': 0, 'days': int(days), 'top': 10} + return f'price {days}', {'mode': 'period', 'threshold': 0, 'days': int(days), 'top': 10} case _: - raise ValueError('[midsession|interday|premarket|close|days (int)]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/rating.py b/lib/reports/rating.py index 2670d64e..a88a7ff5 100644 --- a/lib/reports/rating.py +++ b/lib/reports/rating.py @@ -1,44 +1,52 @@ -"""Analysts' consensus rating changes since the last scheduled run (rating.py); .buy and .sell are in rankings.""" +"""Analysts' consensus rating changes since the last scheduled run (rating); .buy and .sell are in rankings.""" from lib import delivery from lib import state from lib import links from lib import watchlist from lib import yahoo -STATE_FILE = 'finbot_rating.json' # {ticker: [action, index]} as last seen +STATE_FILE = 'finbot_rating.json' # each chat's {ticker: [action, index]} as its last scheduled run saw them def run(dest=None): - """Post the analysts' consensus ratings that changed since the last scheduled run, which records the ratings it saw: - to the chat that asked (dest), else to the default chats.""" - interactive = dest is not None - changes, seen = gather(watchlist.with_holdings()) + """Post the analysts' consensus ratings that changed since a chat's last scheduled run: to the chat that asked + (dest), against its record; or, a scheduled run, to each of its chats (dest, a chat's job; else the home chats), + recording for each the ratings it saw.""" + interactive = delivery.interactive(dest) + market_data, seen = gather(watchlist.with_holdings()) for chat in delivery.targets(dest): + changes = changed(market_data, seen, state.load_for(chat.key, STATE_FILE, {})) delivery.deliver(chat, delivery.Report(report(chat.dialect, changes, interactive))) - if seen and not interactive: # added to what is recorded: a ticker missing from this run keeps its last rating - with state.edit(STATE_FILE, {}) as ratings: - ratings.update(seen) + if seen and not interactive: # added to its record: a ticker missing from this run keeps its last rating + with state.edit_for(chat.key, STATE_FILE, {}) as ratings: + ratings.update(seen) def gather(tickers): - """(the ratings that changed since the last scheduled run, [(ticker, its details, old [action, index], new [action, - index], analysts)]; {ticker: [action, index]} of every rating seen).""" - old = state.load(STATE_FILE, {}) + """(the stocks' details; {ticker: [action, index]} of every rating seen, which counts only with its analysts).""" market_data = yahoo.fetch_details(tickers) - changes, seen = [], {} + seen = {} for ticker in tickers: + details = market_data.get(ticker, {}) try: - action = market_data[ticker]['recommend'] - index = round(float(market_data[ticker]['recommend_index']), 1) - analysts = market_data[ticker]['recommend_analysts'] + action, index = details['recommend'], round(float(details['recommend_index']), 1) except (KeyError, ValueError): continue - seen[ticker] = [action, index] + if 'recommend_analysts' in details: + seen[ticker] = [action, index] + return market_data, seen + +def changed(market_data, seen, old): + """The ratings seen that changed from a chat's record (old, {ticker: [action, index]}): [(ticker, its details, old + (action, index), new (action, index), analysts)].""" + changes = [] + for ticker, (action, index) in seen.items(): try: old_action, old_index = old[ticker] except (KeyError, TypeError, ValueError): continue if old_action and action != old_action: - changes.append((ticker, market_data[ticker], (old_action, old_index), (action, index), analysts)) - return changes, seen + changes.append((ticker, market_data[ticker], (old_action, old_index), (action, index), + market_data[ticker]['recommend_analysts'])) + return changes def report(service, changes, interactive): """The rating changes in one network's markup, sorted: 📈 when the consensus improved (its index fell: 1 is a strong @@ -57,6 +65,11 @@ def report(service, changes, interactive): payload = ["No rating changes found"] return payload +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: rating.py takes none.""" - return 'rating.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); rating takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'rating', {} diff --git a/lib/reports/reminder.py b/lib/reports/reminder.py index dd05ca12..cbbd5513 100644 --- a/lib/reports/reminder.py +++ b/lib/reports/reminder.py @@ -1,4 +1,4 @@ -"""Reminders for the day: the entries of reminder.json for today, and finbot's birthday (scheduled: reminder.py).""" +"""Reminders for the day: the entries of reminder.json for today, and finbot's birthday (scheduled: reminder).""" import datetime import re import zoneinfo @@ -55,6 +55,11 @@ def markup(line, service): return service.italic(text) if italic else text +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: reminder.py takes none.""" - return 'reminder.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); reminder takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'reminder', {} diff --git a/lib/reports/sharesight_warm.py b/lib/reports/sharesight_warm.py index 85606a98..72cdedb8 100644 --- a/lib/reports/sharesight_warm.py +++ b/lib/reports/sharesight_warm.py @@ -1,5 +1,5 @@ -"""Keeps Sharesight's portfolio list and holdings reports fresh in the cache (scheduled: sharesight_warm.py), so that a -request never waits on Sharesight. The price histories have their own, yahoo_warm.py.""" +"""Keeps Sharesight's portfolio list and holdings reports fresh in the cache (scheduled: sharesight_warm), so that a +request never waits on Sharesight. The price histories have their own, yahoo_warm.""" import sys from lib import config, sharesight @@ -17,6 +17,11 @@ def run(): print("sharesight_warm: refresh failed:", e, file=sys.stderr) return 0 +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: sharesight_warm.py takes none.""" - return 'sharesight_warm.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); sharesight_warm takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'sharesight_warm', {} diff --git a/lib/reports/shorts.py b/lib/reports/shorts.py index a9887433..3ac51449 100644 --- a/lib/reports/shorts.py +++ b/lib/reports/shorts.py @@ -1,4 +1,4 @@ -""".shorts: the most shorted stocks followed, from Short Man (ASX) and Yahoo (scheduled: shorts.py).""" +""".shorts: the most shorted stocks followed, from Short Man (ASX) and Yahoo (scheduled: shorts).""" from lib import config from lib import delivery from lib import render @@ -53,6 +53,11 @@ def report(service, tickers, market_data, threshold, specific_stock, interactive payload = [f"🩳 No stocks shorted over {threshold:g}%. Try specifying a number."] return payload, graph +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: shorts.py takes none.""" - return 'shorts.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); shorts takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'shorts', {} diff --git a/lib/reports/trades.py b/lib/reports/trades.py index 5b898dfe..85792d13 100644 --- a/lib/reports/trades.py +++ b/lib/reports/trades.py @@ -1,4 +1,4 @@ -""".trades: Sharesight buys and sells; a scheduled run reports only new ones (scheduled: trades.py).""" +""".trades: Sharesight buys and sells; a scheduled run reports only new ones (scheduled: trades).""" import dataclasses import datetime import sys @@ -12,7 +12,7 @@ from lib import markets from lib import yahoo -STATE_FILE = "finbot_sharesight_trades.json" # the ids of the trades reported +STATE_FILE = "finbot_sharesight_trades.json" # each chat's ids of the trades reported to it @dataclasses.dataclass(frozen=True) @@ -30,22 +30,25 @@ class Trade: def run(dest=None, *, days=None, user='', portfolio_select=None): - """Post the buys and sells of the past `days`, in every portfolio or the one named: to the chat that asked (dest), - else, only those not reported before, to the default chats. Raises sharesight.PortfolioNotFound for a name that is - none of them.""" - interactive = dest is not None + """Post the buys and sells of the past `days`, in every portfolio or the one named: to the chat that asked (dest); + or, a scheduled run, to each of its chats (dest, a chat's job; else the home chats) those it has not told that chat + of before. Raises sharesight.PortfolioNotFound for a name that is none of them.""" + interactive = delivery.interactive(dest) if days is None: days = config.prefs.past_days - trades = gather(days, portfolio_select, set() if interactive else set(state.load(STATE_FILE, []))) - for chat in delivery.targets(dest): - payload = report(chat.dialect, trades, days, interactive) + chats = delivery.targets(dest) + told = {chat.key: set() if interactive else set(state.load_for(chat.key, STATE_FILE, [])) for chat in chats} + trades = gather(days, portfolio_select, set.intersection(*told.values())) # every chat knows those: not fetched + for chat in chats: + new = [trade for trade in trades if trade.id not in told[chat.key]] + payload = report(chat.dialect, new, days, interactive) if interactive and not payload: # no trades, or none of them a buy or a sell when = formats.days_english(days, 'in the past ') payload = [f"{user} No trades {when}."] delivery.deliver(chat, delivery.Report(payload)) - if trades and not interactive: # recorded once the report is out - with state.edit(STATE_FILE, []) as known: - known.extend(sorted({trade.id for trade in trades}.difference(known))) + if new and not interactive: # recorded once the report is out + with state.edit_for(chat.key, STATE_FILE, []) as known: + known.extend(sorted({trade.id for trade in new}.difference(known))) def gather(days, portfolio_select, known): @@ -96,6 +99,11 @@ def line(trade): if trade.date == date] for date in dates)) +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: trades.py takes none.""" - return 'trades.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); trades takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'trades', {} diff --git a/lib/reports/yahoo_warm.py b/lib/reports/yahoo_warm.py index 2a53f912..80682e60 100644 --- a/lib/reports/yahoo_warm.py +++ b/lib/reports/yahoo_warm.py @@ -1,5 +1,5 @@ -"""Keeps the price history of every stock followed fresh in the cache (scheduled: yahoo_warm.py), so that a period list -or a chart never waits on Yahoo. Sharesight's caches have their own, sharesight_warm.py.""" +"""Keeps the price history of every stock followed fresh in the cache (scheduled: yahoo_warm), so that a period list +or a chart never waits on Yahoo. Sharesight's caches have their own, sharesight_warm.""" import sys from lib import config, watchlist, yahoo @@ -20,6 +20,11 @@ def run(): print(f"yahoo_warm: {len(tickers) - len(failed)} of {len(tickers)} price histories fetched", file=sys.stderr) return len(tickers) - len(failed) +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: yahoo_warm.py takes none.""" - return 'yahoo_warm.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); yahoo_warm takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'yahoo_warm', {} diff --git a/lib/state.py b/lib/state.py index d10f32b9..92bf9d72 100644 --- a/lib/state.py +++ b/lib/state.py @@ -1,6 +1,7 @@ -"""State files in DATA_DIR: what must last from one run to the next (the watchlist, plans, seen trades, ratings, record -highs, the schedule's last runs, chart button refs, keyboard versions, seen podcast episodes). Written whole or not at -all, and flushed.""" +"""State files in DATA_DIR: what must last from one run to the next (each chat's jobs, the watchlist, plans, seen +trades, ratings, record highs, the schedule's last runs, chart button refs, keyboard versions, seen podcast episodes). +Written whole or not at all, and flushed. A report's record of what it told each chat is a file of its own, holding one +record per chat ({'chats': {chat key: record}}): see load_for and edit_for.""" import contextlib import copy import json @@ -36,6 +37,18 @@ def _lock(name): return _locks.setdefault(name, threading.Lock()) +def load_for(chat, name, default): + """A chat's record (by its key) in a state file that holds one per chat, or `default` when it has none yet.""" + return copy.deepcopy(load(name, {}).get('chats', {}).get(chat, default)) + + +@contextlib.contextmanager +def edit_for(chat, name, default): + """Change a chat's record (by its key) in a state file that holds one per chat, as edit() changes a file.""" + with edit(name, {}) as data: + yield data.setdefault('chats', {}).setdefault(chat, copy.deepcopy(default)) + + @contextlib.contextmanager def edit(name, default): """Change a state file in place: `with state.edit(name, {}) as data: data[key] = value`. The file is read as the diff --git a/lib/transports.py b/lib/transports.py index 58683106..6e7b560d 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -31,6 +31,7 @@ class Transport: takes_buttons = True # buttons under a message shows_typing = True # a typing notice while a command works TYPING = (0.5, 7, 10) # the notice: first after 0.5 s (a quick answer gets none), then every 7 s, 10 times at most + scheduled = False # posting a scheduled run's report (a chat's job, or the home chats), not a reply to a command _notices = None # the thread sending the notices _quiet = None # set to stop them @@ -47,6 +48,12 @@ def __repr__(self): """: the kind and where, for logs and test failures.""" return f'<{type(self).__name__} {self.where}>' + @property + def key(self): + """The chat's key, the same whichever way its network's updates arrive: 'telegram:-1001234'. A chat's jobs, its + watchlist and what its reports have told it are kept under it.""" + return f'{self.name}:{self.where}' + def start_typing(self): """Show the typing notice while a command works, where the network has one (TYPING says when), until the first message goes out or the command ends (end_typing).""" diff --git a/lib/worker.py b/lib/worker.py index 1dfc3995..2c096b96 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -43,21 +43,27 @@ def reply(self, lines, chart=None, buttons=None): return delivery.deliver(self.dest, delivery.Report(lines, chart, buttons)) def process_job_callback(press, pressed): - """A command button of .jobs (an inbound.Press; pressed, what its data says): run that command now and answer in - the chat. A button from an older list runs nothing: the list is redrawn in place as it is now.""" + """A job's button (an inbound.Press; pressed, what its data says): run the job now and say how it went, or switch + it on or off and redraw the list in place. A button from an older list does neither: the list is redrawn as it is + now.""" service = dialects.of(press.service) chat = delivery.reply_to(service, press.chat_id, interaction=press.interaction, pressed=True) + def redraw(): + """Put the chat's list, as it is now, in place of the pressed one, with the same kind of buttons.""" + payload, markup = about.prepare_jobs_payload(service, chat.key, pressed.action, chat.takes_buttons) + chat.edit_text(press.message_id, '\n'.join(payload), markup or []) try: - if jobs.current is None: # the scheduler is off now - raise jobs.ScheduleError('the scheduler is off') - command = jobs.current.pick(pressed.number, pressed.checksum) # only a command .jobs offers; anything else fails + if jobs.current is None: + raise jobs.ScheduleError('no schedule is running') + job = jobs.current.pick(chat.key, pressed.number, pressed.checksum) # only this chat's job, as listed except jobs.ScheduleError: - payload, markup = about.prepare_jobs_payload(service, '', chat.takes_buttons) # the list now, for the stale one - chat.edit_text(press.message_id, '\n'.join(payload), markup or []) - return + return redraw() try: - line = about.job_line(jobs.current, *jobs.current.run_command(command)) - delivery.deliver(chat, delivery.Report([line])) + if pressed.action == 'toggle': + jobs.current.toggle(job) + redraw() + else: + delivery.deliver(chat, delivery.Report([about.job_line(pressed.number, job, jobs.current.run(job))])) except Exception as e: delivery.report_error(e, chat, context='jobs') @@ -355,11 +361,42 @@ def on_who(request, args): request.reply(holdings.prepare_who(request.service, re.split(r'[.:]', args.words[0].upper())[0])) def on_jobs(request, args): - """.jobs [run NUMBER|COMMAND]: the schedule, or one of its commands run now.""" - if bool(args.keyword) != bool(args.words): # run without a command, or words without run - raise Usage - payload, markup = about.prepare_jobs_payload(request.service, args.rest or None, request.dest.takes_buttons) - request.reply(payload, buttons=markup) + """.jobs: this chat's jobs; .jobs run or toggle [NUMBER]: one run now or switched on or off (without a number, the + list with a button per job that does it); .jobs add [CRON_TZ=ZONE] TIME REPORT: one more (alone, the reports it can + run); .jobs del NUMBER: one fewer.""" + scheduler, chat, service = jobs.current, request.dest.key, request.service + if scheduler is None: + raise answer.Answer('No schedule is running') + try: + if args.keyword is None and not args.words: + request.reply(about.prepare_jobs_payload(service, chat)[0]) + elif args.keyword == 'add': + first, line = (args.text.split(None, 1) + [''])[:2] + if first.lower() != 'add': # the job's line follows add + raise Usage + if not line: + return request.reply(about.prepare_reports(service)) + number = scheduler.add(chat, line) + request.reply([about.job_entry(service, scheduler, number, scheduler.chat_jobs(chat)[number - 1]), + 'Added: .jobs lists them all']) + elif args.keyword in ('run', 'toggle') and not args.words: + payload, markup = about.prepare_jobs_payload(service, chat, args.keyword, request.dest.takes_buttons) + request.reply(payload, buttons=markup) + elif args.keyword in ('run', 'toggle', 'del') and len(args.words) == 1: + number = args.words[0] + job = scheduler.job(chat, number) + if args.keyword == 'run': + request.reply([about.job_line(number, job, scheduler.run(job))]) + elif args.keyword == 'toggle': + scheduler.toggle(job) + request.reply([about.job_entry(service, scheduler, number, scheduler.job(chat, number))]) + else: + scheduler.delete(job) + request.reply([f"Deleted {number}. {job.line}"]) + else: + raise Usage + except jobs.ScheduleError as e: # a job that cannot be, or a number that is none: said, as an answer + raise answer.Answer(str(e)) def on_profile(request, args): """.SYMBOL, or .stock (.profile) SYMBOL: the company's profile.""" @@ -392,6 +429,8 @@ class Command: NEGATIVE = TOP | dict.fromkeys(('neg', 'negative', 'negatives'), 'negative') # the ratios that can be negative CHANGE = {'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), 'delete') # of the watchlist BY = dict.fromkeys(('sector', 'sectors'), 'sector') | dict.fromkeys(('industry', 'industries'), 'industry') # .pie's +JOB_CHANGES = {'run': 'run', 'toggle': 'toggle', 'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), + 'del') # what .jobs does to a chat's jobs BUY_SELL = ('buy', 'sell', *(a + sep + b for a, b in (('buy', 'sell'), ('sell', 'buy')) for sep in ('', '&', '/', '+'))) PROFILE = Command(('stock', 'profile'), on_profile, ('.SYMBOL', '.stock SYMBOL'), COMPANY) # any first word that is no # command's name but may be a ticker calls it too @@ -426,8 +465,8 @@ class Command: Command(('shorts', 'short'), on_shorts, ('.shorts [percent|SYMBOL]',), RISK, ('.shorts',), percent=True), Command(('trades', 'trade'), on_trades, ('.trades [period] [portfolio]',), TRADES, ('.trades',), period=True), Command(('plan',), on_plan, ('.plan [your plan]',), TRADES), - Command(('jobs', 'cron', 'crons', 'schedule'), on_jobs, ('.jobs [run NUMBER|COMMAND]',), JOBS, - keywords={'run': 'run'}), + Command(('jobs', 'job', 'cron', 'crons', 'schedule'), on_jobs, ('.jobs [run|toggle] [NUMBER]', + '.jobs add [CRON_TZ=ZONE] TIME REPORT', '.jobs del NUMBER'), JOBS, keywords=JOB_CHANGES), PROFILE, ] diff --git a/tests/common.py b/tests/common.py index 2b829675..6d0b1ec0 100644 --- a/tests/common.py +++ b/tests/common.py @@ -26,8 +26,8 @@ def no_jobs(data): - """An empty schedule in a data dir, so that a bot started there runs no scheduled report.""" - Path(data, 'schedule.cron').write_text('# no jobs: a test bot runs none\n') + """No chat's jobs in a data dir, so that a bot started there runs no scheduled report (and migrates none).""" + Path(data, 'finbot_schedule.json').write_text('{}') def bare_environment(): diff --git a/tests/test_config.py b/tests/test_config.py index 9f052993..272b95b2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -481,37 +481,24 @@ def restore(self, file): file.write_text('') config.reload() - def test_reload_all_reloads_preferences_and_the_schedule_independently(self): - """SIGHUP reloads preferences, keeping the running ones on an error (logged), and the schedule regardless.""" + def test_reload_all_reloads_preferences_and_keeps_them_on_an_error(self): + """SIGHUP reloads preferences; on an error (logged) the running ones stay, and the bot survives it. The jobs + need no reload: a change made with .jobs counts at once.""" file = Path(common.temp_dir(self), 'preferences.conf') file.write_text('future_days=45\n') - scheduler = mock.Mock() self.addCleanup(self.restore, file) - with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(bot, 'scheduler', scheduler): + with mock.patch.object(config, 'preferences_path', str(file)): err = io.StringIO() with contextlib.redirect_stderr(err): bot.reload_all() self.assertIn('config: future_days 30 -> 45', err.getvalue()) - scheduler.reload.assert_called_once() - # a preferences error does not stop the schedule reload, and the bot survives it file.write_text('futur_days=1\n') err = io.StringIO() with contextlib.redirect_stderr(err): bot.reload_all() self.assertIn('ERROR config:', err.getvalue()) - self.assertEqual(scheduler.reload.call_count, 2) self.assertEqual(config.prefs.future_days, 45) - def test_reload_with_the_scheduler_off_still_reloads_preferences(self): - """With the scheduler off, SIGHUP still reloads preferences.""" - file = Path(common.temp_dir(self), 'preferences.conf') - file.write_text('future_days=11\n') - self.addCleanup(self.restore, file) - with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(bot, 'scheduler', None): - with contextlib.redirect_stderr(io.StringIO()): - bot.reload_all() - self.assertEqual(config.prefs.future_days, 11) - class Tools(unittest.TestCase): def test_split_env(self): diff --git a/tests/test_edges.py b/tests/test_edges.py index 66bbcd99..469eb4a1 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -479,7 +479,7 @@ def test_usr1_flushes_the_cache_and_keeps_the_state(self): if 'cache: warm jobs done' in line: # each in a child process, as the scheduler is off break self.assertIn('cache: flushed 2 files', ''.join(lines)) - self.assertIn('cache: filling it again: sharesight_warm.py, yahoo_warm.py', ''.join(lines)) + self.assertIn('cache: filling it again: sharesight_warm, yahoo_warm', ''.join(lines)) self.assertEqual(sorted(os.listdir(cache)), ['finbot_x.json.abc.tmp', 'matplotlib']) self.assertTrue(Path(data, 'finbot_watchlist.json').exists()) diff --git a/tests/test_reports.py b/tests/test_reports.py index 76e60aac..c0f9a030 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -381,13 +381,33 @@ def test_trades_asks_yahoo_only_for_new_trades(self): def test_a_scheduled_run_keeps_the_trades_another_run_recorded_meanwhile(self): """The seen trades are read again as the run ends, so a trade another run recorded meanwhile is kept.""" - meanwhile = lambda tickers: state.save('finbot_sharesight_trades.json', - [999]) or {t: market_entry(t) for t in tickers} + meanwhile = lambda tickers: state.save(trades.STATE_FILE, + {'chats': {'telegram:1': [999]}}) or {t: market_entry(t) for t in tickers} self.patch(yahoo, 'fetch', meanwhile) # called after the seen trades were read, before the new ones are recorded trades.run() - known = state.load('finbot_sharesight_trades.json') + known = state.load_for('telegram:1', trades.STATE_FILE, []) self.assertEqual((known[0], len(known)), (999, 2)) + def test_a_trade_or_a_rating_change_reaches_each_chat_that_schedules_the_report(self): + """Each chat keeps its own record of what a report told it, so a trade or a rating change reaches every chat + whose job runs the report, each once: the first chat's run used to mark it told for all.""" + told = [] + self.patch(transports.Transport, 'text', lambda dest, lines, markup=None: + told.append((dest.key, ' '.join(lines))) or []) + chats = [delivery.chat(key, scheduled=True) for key in ('telegram:5', 'slack:C9')] + self.env(slackBotToken='xoxb-test') + for chat in chats * 2: + trades.run(chat) + self.assertEqual([key for key, text in told if 'T1' in text], ['telegram:5', 'slack:C9']) + for chat in chats: + rating.run(chat) # the first sighting, recorded + told.clear() + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | ({'recommend': 'sell', + 'recommend_index': 4.5} if t == 'T1' else {})}) + for chat in chats * 2: + rating.run(chat) + self.assertEqual([key for key, text in told if 'Co T1' in text], ['telegram:5', 'slack:C9']) + def test_milestone(self): """The milestone report posts.""" milestone.run() @@ -399,21 +419,23 @@ def test_milestone_records_a_ticker_only_with_its_new_values(self): self.patch(yahoo, 'fetch', lambda ts: {t: {k: v for k, v in market_entry(t).items() if not k.startswith('fiftyTwoWeek')} for t in ts}) milestone.run() - self.assertFalse(state.load('finbot_milestone.json')) + self.assertFalse(state.load_for('telegram:1', milestone.STATE_FILE, {})) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts}) self.sent.clear() milestone.run() self.assertNotIn('Record high', str(self.sent)) - self.assertEqual(state.load('finbot_milestone.json')['T1'], {'all_time_high': 150.0, 'all_time_low': 50.0}) + self.assertEqual(state.load_for('telegram:1', milestone.STATE_FILE, {})['T1'], + {'all_time_high': 150.0, 'all_time_low': 50.0}) def test_a_record_kept_from_ten_years_is_seeded_again_from_the_whole_history(self): """Records kept before they came from the whole history (record_high, from 10 years) are seeded again: a stock above its 10-year high but below its all-time one posted a false record.""" - state.save('finbot_milestone.json', {'T1': {'record_high': 110.0, 'record_low': 90.0}}) + state.save(milestone.STATE_FILE, {'chats': {'telegram:1': {'T1': {'record_high': 110.0, 'record_low': 90.0}}}}) self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) milestone.run() # the 52-week range (120/80) was past the old record, but not the whole history's self.assertNotIn('Co T1 ', str(self.sent)) - self.assertEqual(state.load('finbot_milestone.json')['T1'], {'all_time_high': 150.0, 'all_time_low': 50.0}) + self.assertEqual(state.load_for('telegram:1', milestone.STATE_FILE, {})['T1'], + {'all_time_high': 150.0, 'all_time_low': 50.0}) def test_milestone_records_are_all_time(self): """A new 52-week high below the all-time high is no record. The record used to become the 52-week high after the @@ -429,7 +451,8 @@ def test_milestone_records_are_all_time(self): def test_a_52_week_record_kept_before_is_replaced_by_the_history(self): """A record kept in the old 52-week format is replaced by the history: no false record high.""" - state.save('finbot_milestone.json', {f'T{i}': {'high': 110.0, 'low': 90.0} for i in range(25)}) + state.save(milestone.STATE_FILE, {'chats': {'telegram:1': {f'T{i}': {'high': 110.0, 'low': 90.0} + for i in range(25)}}}) self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) milestone.run() # 120 beats the old 52-week record of 110, but not the history's 150 self.assertNotIn('Record high', str(self.sent)) @@ -454,7 +477,7 @@ def test_rating_keeps_the_last_rating_of_a_ticker_missing_from_a_run(self): def test_an_interactive_rating_records_nothing(self): """A rating report asked for in a chat leaves the record for the scheduled runs alone.""" rating.run(transports.TelegramBot('5')) - self.assertIsNone(state.load('finbot_rating.json')) + self.assertIsNone(state.load(rating.STATE_FILE)) def test_scheduled_reports_reach_each_home_chat_with_its_own_token(self): """Scheduled reports reach Telegram's and Slack's home chats, each with its own bot's credentials: Slack's diff --git a/tests/test_schedule.py b/tests/test_schedule.py index fe59dbaf..fbd49666 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -1,150 +1,129 @@ -"""The schedule file (lib/jobs.py): its lines parsed, each script's job(argv), and a schedule the bot refuses.""" -import hashlib, os, subprocess, sys, tempfile, unittest +"""Jobs as .jobs add takes them (lib/jobs.py): their lines, and each report's job(argv); and the one-time migration +(lib/migrate.py) of the old schedule.cron, and of the reports' records, to the home chats.""" +import contextlib, hashlib, io, os, unittest from pathlib import Path from unittest import mock from tests.common import * -from lib import jobs -from lib.reports import cal, cleanup, performance, podcasts, price - - -class ParseTests(unittest.TestCase): - def test_shipped_schedule_has_no_errors(self): - """defaults/schedule.cron parses without errors, its Saturday line into its five commands; the warm jobs run - within their caches' lifetimes: Sharesight's every 30 minutes, Yahoo's every 8 hours.""" - parsed, errors = jobs.parse_schedule(DEFAULT_SCHEDULE.read_text(), UTC) - self.assertEqual(errors, []) - self.assertEqual(len(parsed), 11) - lines = [(j.expr, [c[0] for c in j.commands]) for j in parsed] - self.assertIn(('14,44 * * * *', ['sharesight_warm.py']), lines) - self.assertIn(('07 */8 * * *', ['yahoo_warm.py']), lines) - weekly = next(j for j in parsed if j.expr == '28 21 * * Sat') - self.assertEqual([label for label, _, _ in weekly.commands], - ['cal.py earnings', 'cal.py ex-dividend', 'price.py 7', 'performance.py 7', 'rating.py']) - - def test_pasted_crontab_lines(self): - """Lines pasted from a crontab work: variables, paths, redirects, braces, comments and @daily.""" - text = ('MAILTO=me\n' - '*/20 * * * Mon-Fri ~/finbot/trades.py > ~/finbot.trades 2>&1\n' - '28 21 * * Sat { ./cal.py earnings; ./price.py 7 ;} &> /dev/null # weekly (UTC)\n' - '@daily /opt/finbot/reminder.py\n') - parsed, errors = jobs.parse_schedule(text, UTC) - self.assertEqual(errors, []) - self.assertEqual([[label for label, _, _ in j.commands] for j in parsed], - [['trades.py'], ['cal.py earnings', 'price.py 7'], ['reminder.py']]) +from lib import jobs, migrate, state +from lib.reports import cal, cleanup, milestone, performance, podcasts, price, rating, trades + +LOCAL_ZONE = migrate.local_zone # the real one: the migration tests put UTC in its place + + +class JobLines(unittest.TestCase): + def test_a_job_is_a_time_then_its_reports(self): + """A job is five time fields, or an @macro, then one or more reports with their arguments, separated by ";"; + it is on, in UTC, and its reports are kept as written.""" + job = jobs.parse_job('28 21 * * Sat cal earnings; cal ex-dividend;price 7 ; performance 7; rating') + self.assertEqual((job.when, job.zone.key, job.on, job.chat), ('28 21 * * Sat', 'UTC', True, '')) + self.assertEqual([label for label, _, _ in job.commands], + ['cal earnings', 'cal ex-dividend', 'price 7', 'performance 7', 'rating']) + self.assertEqual(job.text, 'cal earnings; cal ex-dividend;price 7 ; performance 7; rating') + self.assertEqual(job.line, '28 21 * * Sat ' + job.text) + self.assertEqual(jobs.parse_job('@daily reminder', 'telegram:1').when, '@daily') + + def test_a_job_may_name_its_zone_first(self): + """A job may name its zone, as crontab writes it, to follow that market's clocks; its line keeps it.""" + job = jobs.parse_job('CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession') + self.assertEqual((job.zone.key, job.when, job.text), + ('America/New_York', '15 13 * * Mon-Fri', 'price midsession')) + self.assertEqual(job.line, 'CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession') + for zone in ('Nowhere/Land', '', '../etc/passwd'): + with self.subTest(zone=zone), self.assertRaisesRegex(jobs.ScheduleError, 'no time zone'): + jobs.parse_job(f'CRON_TZ={zone} 0 1 * * * trades') def test_quoted_arguments_may_hold_anything(self): - """A quoted argument may hold shell characters; commands after a ; still split.""" + """An argument in quotes may hold anything, a ";" included; commands after a ";" outside quotes still split.""" url = 'https://example.com/feed?a=1&b=2;c>d $x |y' - line = f'01 0-10,23 * * Mon-Fri podcasts.py "{url}" > ~/out 2>&1 # hourly (UTC)\n' - parsed, errors = jobs.parse_schedule(line, UTC) - self.assertEqual(errors, []) - self.assertEqual(parsed[0].commands[0][2], {'feed_url': url}) - parsed, errors = jobs.parse_schedule(f"5 4 * * * podcasts.py '{url}'; shorts.py\n", UTC) - self.assertEqual([(label[:9], kw) for label, _, kw in parsed[0].commands], - [('podcasts.', {'feed_url': url}), ('shorts.py', {})]) - - def test_unquoted_shell_syntax_is_still_refused(self): - """Pipes, &&, a trailing &, $(), backticks, input redirects, subshells and an unterminated quote are refused.""" - for line in ('0 1 * * * trades.py | tee x', '0 1 * * * trades.py && shorts.py & ', - '0 1 * * * trades.py $(date)', '0 1 * * * trades.py `x`', - '0 1 * * * trades.py < in', '0 1 * * * (trades.py)', '0 1 * * * trades.py "unterminated'): - parsed, errors = jobs.parse_schedule(line + '\n', UTC) - self.assertEqual((parsed, len(errors)), ([], 1), line) - - def test_redirects_and_braces_are_dropped(self): - """Output redirects, braces, paths and trailing comments are dropped from a command.""" - for command in ('trades.py > /dev/null', 'trades.py >/dev/null 2>&1', 'trades.py &> /dev/null', - 'trades.py >> ~/x 2>&1', - '{ trades.py ;} > ~/x', '/opt/finbot/trades.py # note (x)'): - parsed, errors = jobs.parse_schedule(f'0 1 * * * {command}\n', UTC) - self.assertEqual(errors, [], command) - self.assertEqual([label for label, _, _ in parsed[0].commands], ['trades.py'], command) - - def test_bin_run_wrapper_is_no_longer_accepted(self): - """The old bin/run wrapper and a cd before the command are refused.""" - for command in ('bin/run trades.py', 'cd /opt/finbot; trades.py'): - parsed, errors = jobs.parse_schedule(f'0 1 * * * {command}\n', UTC) - self.assertEqual((parsed, len(errors)), ([], 1), command) - - def test_bad_lines_are_skipped_and_named(self): - """Bad lines are skipped and reported by line number; the good ones load.""" - text = ('0 1 * * * ls | wc -l\n0 1 * * * price.py bogus\n0 1 * * * nothere.py\n61 1 * * * trades.py\n0 1 * *\n' - 'FOO=bar\n5 4 * * * shorts.py\n') - parsed, errors = jobs.parse_schedule(text, UTC) - self.assertEqual([j.expr for j in parsed], ['5 4 * * *']) - self.assertEqual([number for number, _, _ in errors], [1, 2, 3, 4, 5, 6]) - - def test_impossible_times_are_marked_and_reboot_is_just_unsupported(self): - """Times that never come (30 February, minute 61, month 13, @bogus) are ImpossibleTime; @reboot is only - unsupported, and 29 February does come.""" - for line in ('0 0 30 2 * trades.py', '0 9 31 4,6,9,11 * trades.py', '61 1 * * * trades.py', - '0 0 * 13 * trades.py', '@bogus trades.py'): - parsed, errors = jobs.parse_schedule(line + '\n', UTC) - self.assertEqual(parsed, [], line) - self.assertIsInstance(errors[0][2], jobs.ImpossibleTime, line) - parsed, errors = jobs.parse_schedule('@reboot trades.py\n0 0 29 2 * trades.py\n', UTC) - self.assertEqual([j.expr for j in parsed], ['0 0 29 2 *']) # a leap day does come - self.assertNotIsInstance(errors[0][2], jobs.ImpossibleTime) - - def test_cron_tz_applies_to_the_lines_below(self): - """CRON_TZ sets the zone of the lines below it; an unknown zone is skipped.""" - text = '1 1 * * * trades.py\nCRON_TZ=Australia/Melbourne\n2 2 * * * trades.py\nCRON_TZ=Nowhere/Land\n' - parsed, _ = jobs.parse_schedule(text, UTC) - self.assertEqual([j.zone.key for j in parsed], ['UTC', 'Australia/Melbourne']) - - def test_local_zone_follows_tz(self): - """The local zone follows TZ.""" - with mock.patch.dict(os.environ, {'TZ': 'Australia/Melbourne'}): - self.assertEqual(jobs.local_zone().key, 'Australia/Melbourne') + for quoted in (f'"{url}"', f"'{url}'"): + job = jobs.parse_job(f'5 4 * * * podcasts {quoted}; shorts') + self.assertEqual([(label.split()[0], kwargs) for label, _, kwargs in job.commands], + [('podcasts', {'feed_url': url}), ('shorts', {})]) + with self.assertRaisesRegex(jobs.ScheduleError, 'No closing quotation'): + jobs.parse_job('5 4 * * * podcasts "https://example.com') + + def test_shell_syntax_is_only_words_that_no_report_takes(self): + """Nothing runs a shell: pipes, redirects, $() and comments are mere words, which no report takes.""" + for line in ('0 1 * * * trades | tee x', '0 1 * * * trades > /dev/null', '0 1 * * * trades $(date)', + '0 1 * * * trades && shorts', '0 1 * * * trades # a note'): + with self.subTest(line=line), self.assertRaisesRegex(jobs.ScheduleError, 'trades takes no arguments'): + jobs.parse_job(line) + + def test_a_time_that_cannot_be_or_never_comes(self): + """A time that cannot be (minute 61, month 13, @bogus) or never comes (30 February) is ImpossibleTime, and so is + @reboot; 29 February does come.""" + for when, why in (('0 0 30 2 *', 'never comes'), ('0 9 31 4,6,9,11 *', 'never comes'), + ('61 1 * * *', 'is no time'), ('0 0 * 13 *', 'is no time'), ('@bogus', 'is no time'), + ('@reboot', 'the jobs start with the bot')): + with self.subTest(when=when), self.assertRaisesRegex(jobs.ImpossibleTime, why): + jobs.parse_job(f'{when} trades') + self.assertEqual(jobs.parse_job('0 0 29 2 * trades').when, '0 0 29 2 *') + + def test_what_is_no_report(self): + """A job runs reports by name: not a script's file, not a system job's (except as one), not a report that is + not there; with arguments it takes, and with at least one report.""" + for line, why in (('0 1 * * * nothere', "no report 'nothere'"), + ('0 1 * * * trades.py', "no report 'trades.py'"), + ('0 1 * * * ./trades', "no report './trades'"), ('0 1 * * * yahoo_warm', "no report 'yahoo_warm'"), + ('0 1 * * * cleanup 30', "no report 'cleanup'"), ('0 1 * * * trades; nothere', "no report 'nothere'"), + ('0 1 * * * price soon', 'price takes midsession|interday|premarket|close|DAYS'), + ('0 1 * * * ;', 'no report: send .jobs add'), ('0 1 * * *', 'a job is five time fields'), + ('trades', 'a job is five time fields')): + with self.subTest(line=line), self.assertRaisesRegex(jobs.ScheduleError, why): + jobs.parse_job(line) + self.assertEqual(jobs.parse_job('51 18 * * Sat cleanup 30', system=True).commands[0][2], {'days': 30}) class JobArguments(unittest.TestCase): def test_price(self): - """price.py takes one mode or a number of days (the top and bottom ten over it); anything else is its usage.""" - self.assertEqual(price.job(['midsession']), ('price.py midsession', {'mode': 'midsession'})) + """price takes one mode or a number of days (the top and bottom ten over it); anything else is its usage.""" + self.assertEqual(price.job(['midsession']), ('price midsession', {'mode': 'midsession'})) self.assertEqual(price.job(['premarket'])[1], {'mode': 'premarket'}) self.assertEqual(price.job(['interday'])[1], {'mode': 'interday'}) self.assertEqual(price.job(['close'])[1], {'mode': 'close'}) - self.assertEqual(price.job(['28']), ('price.py 28', {'mode': 'period', 'threshold': 0, 'days': 28, 'top': 10})) + self.assertEqual(price.job(['28']), ('price 28', {'mode': 'period', 'threshold': 0, 'days': 28, 'top': 10})) for bad in ([], ['soon'], ['7', '8']): - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, price.ARGUMENTS.replace('|', r'\|')): price.job(bad) def test_cal_and_performance(self): - """cal.py takes earnings or ex-dividend; performance.py an optional number of days.""" - self.assertEqual(cal.job(['earnings']), ('cal.py earnings', {'event': 'earnings'})) - self.assertEqual(cal.job(['ex-dividend']), ('cal.py ex-dividend', {'event': 'ex-dividend'})) - self.assertEqual(performance.job([]), ('performance.py', {})) - self.assertEqual(performance.job(['7']), ('performance.py 7', {'past_days': 7})) + """cal takes earnings or ex-dividend; performance an optional number of days.""" + self.assertEqual(cal.job(['earnings']), ('cal earnings', {'event': 'earnings'})) + self.assertEqual(cal.job(['ex-dividend']), ('cal ex-dividend', {'event': 'ex-dividend'})) + self.assertEqual(performance.job([]), ('performance', {})) + self.assertEqual(performance.job(['7']), ('performance 7', {'past_days': 7})) for bad, module in ((['x'], cal), ([], cal), (['x'], performance)): with self.assertRaises(ValueError): module.job(bad) def test_podcasts(self): - """podcasts.py takes one feed URL; its label carries the start of the hash that names its state file.""" + """podcasts takes one feed URL; its label carries the start of the hash that names its seen file.""" url = 'https://example.com/podcast.rss' label, kwargs = podcasts.job([url]) self.assertEqual(kwargs, {'feed_url': url}) - # the label is the start of the state file's hash self.assertIn('finbot_podcast_seen_' + label.split()[1], podcasts.get_seen_filename(url)) - self.assertEqual(label, 'podcasts.py ' + hashlib.md5(url.encode()).hexdigest()[:8]) + self.assertEqual(label, 'podcasts ' + hashlib.md5(url.encode()).hexdigest()[:8]) for bad in ([], [url, url]): with self.assertRaises(ValueError): podcasts.job(bad) def test_cleanup_arguments(self): - """cleanup.py takes an optional number of days.""" - self.assertEqual(cleanup.job([]), ('cleanup.py', {})) - self.assertEqual(cleanup.job(['7']), ('cleanup.py 7', {'days': 7})) + """cleanup takes an optional number of days.""" + self.assertEqual(cleanup.job([]), ('cleanup', {})) + self.assertEqual(cleanup.job(['7']), ('cleanup 7', {'days': 7})) for bad in (['x'], ['1', '2']): with self.assertRaises(ValueError): cleanup.job(bad) - def test_plain_scripts(self): - """The other reports take no arguments.""" + def test_plain_reports(self): + """The other reports take no arguments, and say so: their ARGUMENTS, which .jobs add lists, are empty.""" for name, module in jobs.SCRIPTS.items(): - if name not in ('cal', 'cleanup', 'performance', 'podcasts', 'price'): - self.assertEqual(module.job([]), (name + '.py', {})) + with self.subTest(name=name): + self.assertIsInstance(module.ARGUMENTS, str) + if name not in ('cal', 'cleanup', 'performance', 'podcasts', 'price'): + self.assertEqual((module.job([]), module.ARGUMENTS), ((name, {}), '')) + with self.assertRaisesRegex(ValueError, 'no arguments'): + module.job(['7']) def test_the_checkout_holds_only_the_bot_at_its_top(self): """The scheduled reports run in the bot (and `.jobs run`), so the checkout's top holds bot.py alone: the scripts @@ -152,19 +131,115 @@ def test_the_checkout_holds_only_the_bot_at_its_top(self): self.assertEqual(sorted(path.name for path in Path(ROOT).glob('*.py')), ['bot.py']) -class StartupRefusal(unittest.TestCase): - def test_the_bot_will_not_start_with_an_impossible_time(self): - """bot.py will not start while the schedule holds a time that never comes, and says which.""" - skip_if_old_env(self) - with tempfile.TemporaryDirectory() as data: - Path(data, 'schedule.cron').write_text('*/20 * * * Mon-Fri trades.py\n0 9 31 6 * price.py close\n') - env = bare_environment() - env.update(DATA_DIR=data, port='0') - out = subprocess.run([sys.executable, 'bot.py'], cwd=ROOT, env=env, capture_output=True, text=True, - timeout=120) - self.assertEqual(out.returncode, 1, out.stderr) - self.assertIn(f"ERROR schedule: {data}/schedule.cron line 2: '0 9 31 6 *' never comes", out.stderr) - - -if __name__ == '__main__': - unittest.main() +class Migration(FinbotCase): + """As finbot first starts with jobs per chat, the home chats get what was kept for all chats, once.""" + + def setUp(self): + """Two home chats, Telegram's and Slack's; the old schedule ran in UTC.""" + super().setUp() + self.env(slackBotToken='xoxb-1', slackChannelID='C1') + self.patch(migrate, 'local_zone', lambda: jobs.UTC) + + def migrate(self): + """Run the migration, as finbot does as it starts: what it logs.""" + with contextlib.redirect_stderr(io.StringIO()) as log: + migrate.run() + return log.getvalue() + + def test_the_old_schedule_becomes_each_home_chats_jobs_once(self): + """Each home chat gets the old schedule's jobs (the data dir's copy), pasted crontab syntax read as cron read + it, each in the zone it ran in, without the system jobs; a line that cannot be a job is logged and left out. + Then the old file is read no more.""" + url = 'https://example.com/feed?a=1&b=2' + Path(config.env.data_dir, 'schedule.cron').write_text('MAILTO=me\n' + '*/20 * * * Mon-Fri ~/finbot/trades.py > ~/finbot.trades 2>&1\n' + '28 21 * * Sat { ./cal.py earnings; ./price.py 7 ;} &> /dev/null # weekly\n' + '@daily /opt/finbot/reminder.py\n' + 'CRON_TZ=Australia/Melbourne\n' + '14,44 * * * * sharesight_warm.py\n' + f'01 * * * * podcasts.py "{url}"; cleanup.py 30\n' + '0 1 * * * ls | wc -l\n' + '0 9 31 6 * price.py close\n' + '0 2 * * * bin/run trades.py\n') + log = self.migrate() + jobs_each = [{'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}, + {'when': '28 21 * * Sat', 'reports': 'cal earnings; price 7', 'zone': 'UTC', 'on': True}, + {'when': '@daily', 'reports': 'reminder', 'zone': 'UTC', 'on': True}, + {'when': '01 * * * *', 'reports': f"podcasts '{url}'", 'zone': 'Australia/Melbourne', 'on': True}] + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {'telegram:1': jobs_each, 'slack:C1': jobs_each}) + path = config.env.data_dir + '/schedule.cron' + self.assertEqual([line for line in log.splitlines() if 'left out' in line], [ + f'schedule: {path} line 8 left out: only finbot scripts can be scheduled, not shell commands', + f"schedule: {path} line 9 left out: '0 9 31 6 *' never comes: no date matches it", + f"schedule: {path} line 10 left out: no report 'run': send .jobs add for the reports and what each takes"]) + self.assertIn(f"schedule: the 4 jobs of {path} are now each home chat's (telegram:1, slack:C1)", log) + s = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + s.load() + self.assertEqual([job.line for job in s.chat_jobs('slack:C1')][-1], f"CRON_TZ=Australia/Melbourne 01 * * * *" + f" podcasts '{url}'") + Path(config.env.data_dir, 'schedule.cron').write_text('0 3 * * * shorts.py\n') # read no more + self.assertEqual(self.migrate(), '') + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {'telegram:1': jobs_each, 'slack:C1': jobs_each}) + + def test_the_shipped_schedule_migrates_whole(self): + """Without a copy in the data dir, the home chats get the shipped schedule's jobs, every line of it but the + system jobs', the weekly line's five reports among them.""" + self.assertNotIn('left out', self.migrate()) + entries = state.load(jobs.SCHEDULE_FILE)['telegram:1'] + self.assertEqual(len(entries), 8) + self.assertIn({'when': '28 21 * * Sat', 'reports': 'cal earnings; cal ex-dividend; price 7; performance 7;' + ' rating', 'zone': 'UTC', 'on': True}, entries) + self.assertFalse(any(name in entry['reports'] for entry in entries for name in jobs.SYSTEM_SCRIPTS)) + + def test_an_empty_schedule_migrates_to_no_jobs_and_is_done(self): + """A schedule with no jobs gives the home chats none, and the migration is still done: it is not tried again.""" + Path(config.env.data_dir, 'schedule.cron').write_text('# nothing\n14,44 * * * * sharesight_warm.py\n') + self.migrate() + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {}) + Path(config.env.data_dir, 'schedule.cron').write_text('0 3 * * * shorts.py\n') + self.assertEqual(self.migrate(), '') + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {}) + + def test_with_no_home_chat_it_waits(self): + """With no home chat set there is nowhere to put the jobs and records, so the migration waits for one.""" + self.env(telegramChatID='', slackChannelID='') + state.save(trades.STATE_FILE, ['t1']) + self.assertEqual(self.migrate(), '') + self.assertFalse(os.path.exists(state.path(jobs.SCHEDULE_FILE))) + self.assertEqual(state.load(trades.STATE_FILE), ['t1']) + + def test_the_reports_records_become_each_home_chats_once(self): + """What a report has told the chats (trades, ratings, record highs, each feed's episodes) becomes each home + chat's record, so that nothing old is told again; a record kept per chat already is left as it is, and so is + one that cannot be read (said in the log).""" + feed = podcasts.get_seen_filename('https://example.com/podcast.rss') + old = {trades.STATE_FILE: ['t1', 't2'], rating.STATE_FILE: {'NVDA': ['upgrade', 3]}, + milestone.STATE_FILE: {'NVDA': {'all_time_high': 200}}, feed: ['guid-1']} + for name, record in old.items(): + state.save(name, record) + log = self.migrate() + for name, record in old.items(): + with self.subTest(name=name): + self.assertEqual(state.load(name), {'chats': {'telegram:1': record, 'slack:C1': record}}) + self.assertIn(f'state: {feed} is kept per chat now, its record copied to telegram:1, slack:C1', log) + state.save(trades.STATE_FILE, {'chats': {'telegram:1': ['t3']}}) + Path(state.path(rating.STATE_FILE)).write_text('{"NVDA": ["upgr') + log = self.migrate() + self.assertEqual(state.load(trades.STATE_FILE), {'chats': {'telegram:1': ['t3']}}) + self.assertEqual(Path(state.path(rating.STATE_FILE)).read_text(), '{"NVDA": ["upgr') + self.assertIn(f'state: {rating.STATE_FILE} left as it is, unreadable', log) + + def test_pasted_redirects_braces_and_paths_are_dropped(self): + """Output redirects, braces, paths, a .py and trailing comments are dropped from an old line's command.""" + for command in ('trades.py > /dev/null', 'trades.py >/dev/null 2>&1', 'trades.py &> /dev/null', + 'trades.py >> ~/x 2>&1', '{ trades.py ;} > ~/x', '/opt/finbot/trades.py # note (x)', 'FOO=1 trades'): + with self.subTest(command=command): + lines = migrate.legacy_lines(f'0 1 * * * {command}\n', 'schedule.cron') + self.assertEqual([commands for _, _, _, commands in lines], [[('trades', [])]]) + + def test_the_local_zone_follows_tz(self): + """The old schedule ran in the server's zone, which TZ sets: UTC by that name when it is UTC by another.""" + for tz, zone in (('Australia/Melbourne', 'Australia/Melbourne'), ('Etc/UTC', 'UTC'), (':UTC', 'UTC')): + with self.subTest(tz=tz), mock.patch.dict(os.environ, {'TZ': tz}): + self.assertEqual(LOCAL_ZONE().key, zone) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 7abe368d..67bffb4d 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,12 +1,13 @@ -"""The scheduler (lib/jobs.py): firing on a fake clock, across the clocks' changes, runs from chat, and each command -run in the bot's process, past its deadline too.""" -import contextlib, datetime, io, json, os, subprocess, sys, tempfile, threading, time, unittest +"""The scheduler (lib/jobs.py): each chat's jobs and the system jobs firing on a fake clock, across the clocks' changes; +.jobs in a chat, and its buttons; each command run in the bot's process, past its deadline too.""" +import contextlib, datetime, io, os, subprocess, sys, tempfile, time, unittest import zoneinfo from pathlib import Path from unittest import mock from tests.common import * from lib import jobs from lib.reports import cleanup, podcasts, price, sharesight_warm, yahoo_warm +from tests import test_sending PREFETCH_HISTORY = yahoo.prefetch_history # FinbotCase replaces it; the warm job's test puts it back @@ -135,24 +136,45 @@ def down(): self.assertEqual(self.errors, []) +CHAT, OTHER = 'telegram:55', 'slack:C1' # two chats, by their keys + + +def entry(line, on=True): + """A job as SCHEDULE_FILE stores it, from its line as .jobs add takes it.""" + job = jobs.parse_job(line) + return {'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': on} + + class SchedulerBase(FinbotCase): def setUp(self): - """A schedule of trades.py every 20 minutes on weekdays, a fake clock; trades and shorts recorded, not run.""" + """Chat telegram:55 runs trades every 20 minutes on weekdays; a fake clock; no system jobs (they have tests of + their own); the reports noted with the chat each posts to, not run.""" super().setUp() - tmp = tempfile.TemporaryDirectory() - self.addCleanup(tmp.cleanup) - self.file = Path(tmp.name, 'schedule.cron') - self.file.write_text('*/20 * * * Mon-Fri trades.py\n') + self.patch(jobs, 'SYSTEM', ()) + self.store(CHAT, '*/20 * * * Mon-Fri trades') self.clock = Clock(at(13, 21, 3)) - self.ran = [] - self.patch(jobs.SCRIPTS['trades'], 'run', lambda **k: self.ran.append(('trades', k))) - self.patch(jobs.SCRIPTS['shorts'], 'run', lambda **k: self.ran.append(('shorts', k))) + self.ran, self.dests = [], [] + for name in ('trades', 'shorts', 'price', 'rating', 'reminder'): + self.patch(jobs.SCRIPTS[name], 'run', self.recorder(name)) + + def recorder(self, name): + """A report's run() that notes the run, and the chat it would post to (its key; None for the home chats).""" + def run(dest=None, **kwargs): + """Note the run and where it posts.""" + self.ran.append((name, kwargs)) + self.dests.append(dest.key if dest else None) + return run + + def store(self, chat, *lines, on=True): + """Set a chat's jobs, as .jobs add would have stored them.""" + with state.edit(jobs.SCHEDULE_FILE, {}) as chats: + chats[chat] = [entry(line, on) for line in lines] def scheduler(self, catchup=5): """A loaded scheduler on the fake clock that runs commands in this process, at once.""" - s = jobs.Scheduler(lambda: self.file, catchup=catchup, clock=self.clock, sleep=lambda seconds: None, - spawn=lambda f, *a: f(*a), zone=UTC) - self.assertTrue(s.load()) + s = jobs.Scheduler(catchup=catchup, clock=self.clock, sleep=lambda seconds: None, spawn=lambda f, *a: f(*a)) + with contextlib.redirect_stderr(io.StringIO()): + s.load() return s @@ -165,24 +187,43 @@ def test_the_warm_jobs_run_through_the_scheduler(self): s = self.scheduler() with contextlib.redirect_stderr(io.StringIO()) as log: jobs.warm_now(s) - s.busy.add('yahoo_warm.py') # on its schedule + s.busy.add(('', 'yahoo_warm')) # on its schedule jobs.warm_now(s) self.assertEqual(warmed, ['sharesight', 'yahoo', 'sharesight']) - self.assertIn('cache: filling it again: sharesight_warm.py, yahoo_warm.py', log.getvalue()) + self.assertIn('cache: filling it again: sharesight_warm, yahoo_warm', log.getvalue()) def test_without_a_scheduler_each_runs_here(self): """Without a scheduler (a test, a tool), each warm job runs here, one after the other, a failure reported.""" started = [] - def run(label, module, kwargs): + def run(label, module, kwargs, dest=None): """Note the run; the first one fails.""" started.append(label) - if label == 'sharesight_warm.py': + if label == 'sharesight_warm': raise RuntimeError('Sharesight unreachable') self.patch(jobs, 'run_in_process', run) with contextlib.redirect_stderr(io.StringIO()): jobs.warm_now() - self.assertEqual(started, ['sharesight_warm.py', 'yahoo_warm.py']) - self.assertEqual(self.errors, ['⚠️ sharesight_warm.py: RuntimeError: Sharesight unreachable']) + self.assertEqual(started, ['sharesight_warm', 'yahoo_warm']) + self.assertEqual(self.errors, ['⚠️ sharesight_warm: RuntimeError: Sharesight unreachable']) + + +class SystemJobs(FinbotCase): + def test_the_system_jobs_are_fixed_and_no_chats(self): + """The system jobs run on a schedule of their own, the same for every install: each warm job within its + cache's lifetime (Sharesight's every 30 minutes, cached 40; Yahoo's every 8 hours, cached 12), and cleanup + weekly. A chat can neither add one nor see it among its own jobs.""" + s = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + s.load() + system = {job.text: job.when for job in s.system_jobs()} + self.assertEqual(system, {'sharesight_warm': '14,44 * * * *', 'yahoo_warm': '7 */8 * * *', + 'cleanup 30': '51 18 * * Sat'}) + self.assertLess(30 * 60, config.prefs.performance_cache_seconds) + self.assertLess(8 * 3600, config.prefs.cache_seconds) + self.assertEqual(s.chat_jobs(''), s.system_jobs()) + for name in jobs.SYSTEM_SCRIPTS: + with self.subTest(name=name), self.assertRaisesRegex(jobs.ScheduleError, f"no report '{name}'"): + s.add(CHAT, f'0 1 * * * {name}') class SchedulerTests(SchedulerBase): @@ -201,20 +242,49 @@ def test_fires_on_the_minute_on_weekdays_only(self): self.clock.now = at(17, 21, 20) # a Saturday self.assertEqual(s.due(self.clock.now), []) - def test_tick_runs_the_job_and_returns_the_wait(self): - """tick() runs what is due and returns how long to sleep: up to 30 seconds, less when the next run is nearer.""" + def test_tick_runs_the_job_for_its_chat_and_returns_the_wait(self): + """tick() runs what is due, posting to the job's chat as a scheduled run, and returns how long to sleep: up to + 30 seconds, less when the next run is nearer.""" + dests = [] + self.patch(jobs.SCRIPTS['trades'], 'run', lambda dest=None, **k: dests.append(dest)) s = self.scheduler() self.clock.now = at(13, 21, 20) - wait = s.tick() - self.assertEqual(self.ran, [('trades', {})]) - self.assertEqual(wait, 30) # the next run is 20 minutes away: wake at most every 30 seconds + self.assertEqual(s.tick(), 30) # the next run is 20 minutes away: wake at most every 30 seconds + self.assertEqual([(dest.key, dest.scheduled) for dest in dests], [(CHAT, True)]) self.clock.now = at(13, 21, 39, 50) self.assertEqual(s.tick(), 10) + def test_each_chats_jobs_post_to_that_chat(self): + """Two chats' jobs at the same time each run, each posting to its own chat.""" + self.store(OTHER, '*/20 * * * Mon-Fri trades', '0 22 * * * shorts') + s = self.scheduler() + self.clock.now = at(13, 21, 20) + s.tick() + self.assertEqual(set(zip(self.dests, [name for name, _ in self.ran])), {(CHAT, 'trades'), (OTHER, 'trades')}) + self.ran.clear() + self.dests.clear() + self.clock.now = at(13, 22, 0) + s.tick() + self.assertEqual(set(zip(self.dests, [name for name, _ in self.ran])), {(CHAT, 'trades'), (OTHER, 'trades'), + (OTHER, 'shorts')}) + + def test_a_job_that_is_off_never_fires_nor_catches_up_once_on(self): + """A job that is off does not fire; switched on, it fires at its next time, not for the runs it missed.""" + self.store(CHAT, '*/20 * * * Mon-Fri trades', on=False) + s = self.scheduler() + self.clock.now = at(13, 21, 20) + self.assertEqual(s.due(self.clock.now), []) + self.clock.now = at(13, 21, 30) + s.toggle(s.chat_jobs(CHAT)[0]) + self.assertTrue(s.chat_jobs(CHAT)[0].on) + self.assertEqual(s.due(self.clock.now), []) # 21:20 is not caught up + self.clock.now = at(13, 21, 40) + self.assertEqual([t for _, t in s.due(self.clock.now)], [at(13, 21, 40)]) + def test_a_run_missed_within_the_catch_up_window_fires_once(self): """After a restart, a run missed within the catch-up window fires once; an older one is skipped.""" first = self.scheduler() - first.last = {first.jobs[0].key: at(13, 21, 0).isoformat()} + first.last = {first.chat_jobs(CHAT)[0].key: at(13, 21, 0).isoformat()} state.save(jobs.STATE_FILE, first.last) self.clock.now = at(13, 21, 43) s = self.scheduler() @@ -223,16 +293,13 @@ def test_a_run_missed_within_the_catch_up_window_fires_once(self): def test_a_corrupt_state_file_is_reported_and_the_schedule_starts_afresh(self): """A state file cut off mid-write is reported once, and the schedule runs as usual and writes it afresh.""" - # cut off mid-write - Path(config.env.data_dir, jobs.STATE_FILE).write_text('{"*/20 * * * Mon-Fri|UTC|trades.py": "2026-10-13T21:') + Path(config.env.data_dir, jobs.STATE_FILE).write_text('{"telegram:55|*/20 * * * Mon-Fri|UTC|trades": "2026-1') s = self.scheduler() self.assertEqual((s.last, len(self.errors)), ({}, 1)) self.assertIn('cannot read finbot_jobs.json', self.errors[0]) self.clock.now = at(13, 21, 20) self.assertEqual(len(s.due(self.clock.now)), 1) # runs as usual, and the record is written afresh - self.assertEqual(state.load(jobs.STATE_FILE), {s.jobs[0].key: at(13, 21, 20).isoformat()}) - s.reload() # and a reload is no trouble either - self.assertEqual(len(self.errors), 1) + self.assertEqual(state.load(jobs.STATE_FILE), {s.chat_jobs(CHAT)[0].key: at(13, 21, 20).isoformat()}) def test_no_record_means_nothing_is_missed(self): """With no record of past runs, nothing counts as missed.""" @@ -249,97 +316,73 @@ def test_a_restart_just_after_a_run_does_not_repeat_it(self): again = self.scheduler() self.assertEqual(again.due(self.clock.now), []) - def test_a_failing_job_is_reported_and_the_next_command_still_runs(self): - """A command that fails is reported, and the next command on its line still runs.""" - self.file.write_text('*/20 * * * * trades.py; shorts.py\n') - def boom(**k): + def test_a_failing_job_is_reported_to_its_chat_and_the_next_command_still_runs(self): + """A command that fails is reported to the chat whose job it is, and the next command of the job still runs.""" + self.store(CHAT, '*/20 * * * * trades; shorts') + def boom(dest=None, **k): """Fail.""" raise RuntimeError('boom') self.patch(jobs.SCRIPTS['trades'], 'run', boom) + reported = [] + self.patch(delivery, 'report_error', lambda e, dest=None, context=None: reported.append((context, dest.key))) s = self.scheduler() - s.execute(s.jobs[0].commands) + self.assertEqual(s.execute(s.chat_jobs(CHAT)[0].commands, CHAT), 'failed') + self.assertEqual(reported, [('trades', CHAT)]) self.assertEqual(self.ran, [('shorts', {})]) - self.assertTrue(any('boom' in e for e in self.errors)) def test_system_exit_inside_a_job_is_reported_not_raised(self): - """A SystemExit inside a job is reported, not raised, and the job is marked done.""" - def quit_(**k): - """Exit, as a script calling sys.exit would.""" - raise SystemExit(3) + """A report calling sys.exit is reported like any failure and does not stop the scheduler.""" + def quit_(dest=None, **k): + """Exit, as a report may.""" + raise SystemExit(1) self.patch(jobs.SCRIPTS['trades'], 'run', quit_) s = self.scheduler() - s.execute(s.jobs[0].commands) + s.run(s.chat_jobs(CHAT)[0]) self.assertEqual(len(self.errors), 1) def test_a_command_past_its_deadline_is_reported_once_and_left_to_finish(self): """A command still running at its deadline is reported as late, once, and runs on to its end (a thread cannot - be stopped); meanwhile it stays marked running, so a second run of it is turned away; then the rest of its line + be stopped); meanwhile it stays marked running, so a second run of it is turned away; then the rest of its job runs.""" self.patch(jobs, 'JOB_TIMEOUT', 0.1) - self.file.write_text('*/20 * * * * trades.py; shorts.py\n') + self.store(CHAT, '*/20 * * * * trades; shorts') s = self.scheduler() second = [] - def slow(**k): + def slow(dest=None, **k): """Take three deadlines, and try a second run of this command meanwhile.""" time.sleep(0.3) - second.append(s.execute(s.jobs[0].commands[:1])) + second.append(s.execute(s.chat_jobs(CHAT)[0].commands[:1], CHAT)) self.ran.append(('trades', k)) self.patch(jobs.SCRIPTS['trades'], 'run', slow) with contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(s.execute(s.jobs[0].commands), 'done') - self.assertEqual(self.errors, ['⚠️ trades.py: TimeoutError: still running after 0.00166667 minutes, and left to' + self.assertEqual(s.run(s.chat_jobs(CHAT)[0]), 'done') + self.assertEqual(self.errors, ['⚠️ trades: TimeoutError: still running after 0.00166667 minutes, and left to' ' finish']) self.assertEqual(second, ['already running']) self.assertEqual(self.ran, [('trades', {}), ('shorts', {})]) self.assertEqual(s.busy, set()) - def test_each_command_of_a_line_has_a_deadline_of_its_own(self): + def test_each_command_of_a_job_has_a_deadline_of_its_own(self): """The deadline is per command (it once covered the whole line): two that each take most of it are not late.""" self.patch(jobs, 'JOB_TIMEOUT', 0.4) - self.file.write_text('*/20 * * * * trades.py; shorts.py\n') - self.patch(jobs.SCRIPTS['trades'], 'run', lambda **k: time.sleep(0.25)) - self.patch(jobs.SCRIPTS['shorts'], 'run', lambda **k: time.sleep(0.25)) + self.store(CHAT, '*/20 * * * * trades; shorts') + self.patch(jobs.SCRIPTS['trades'], 'run', lambda dest=None, **k: time.sleep(0.25)) + self.patch(jobs.SCRIPTS['shorts'], 'run', lambda dest=None, **k: time.sleep(0.25)) s = self.scheduler() with contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(s.execute(s.jobs[0].commands), 'done') - time.sleep(0.3) # past where a deadline of the whole line would have struck + self.assertEqual(s.run(s.chat_jobs(CHAT)[0]), 'done') + time.sleep(0.3) # past where a deadline of the whole job would have struck self.assertEqual(self.errors, []) - def test_reload_applies_edits_keeps_state_and_survives_a_bad_file(self): - """A reload applies edits and keeps unchanged jobs' timing; an unusable file keeps the running schedule.""" - s = self.scheduler() - old_key = s.jobs[0].key - base = s.base[old_key] - self.file.write_text('*/20 * * * Mon-Fri trades.py\n5 4 * * * shorts.py\n') - self.assertTrue(s.load()) - self.assertEqual(len(s.jobs), 2) - self.assertEqual(s.base[old_key], base) - self.file.write_text('# nothing usable\nnot a job\n') - self.assertFalse(s.load()) - self.assertEqual(len(s.jobs), 2) - - def test_an_impossible_time_refuses_the_whole_file(self): - """A time that never comes refuses the whole file: a reload says why and the running schedule carries on.""" + def test_a_stored_job_that_cannot_be_is_reported_and_the_rest_run(self): + """A stored job that cannot be any more (a report since retired, say) is reported and left out; the chat's other + jobs load.""" + with state.edit(jobs.SCHEDULE_FILE, {}) as chats: + chats[CHAT].append({'when': '0 1 * * *', 'reports': 'nothere', 'zone': 'UTC', 'on': True}) s = self.scheduler() - self.file.write_text('*/20 * * * Mon-Fri trades.py\n0 9 30 2 * shorts.py\n5 4 * * * nothere.py\n') - with self.assertRaises(jobs.ImpossibleTime) as caught: - s.load() - self.assertEqual(len(caught.exception.lines), 1) - self.assertIn("line 2: '0 9 30 2 *' never comes", caught.exception.lines[0]) - self.assertEqual(self.errors, []) # load reports nothing for a refused file... - s.reload() # ...a reload says why it was refused, and the running schedule carries on - self.assertEqual(len(self.errors), 1) - self.assertIn('not reloaded, the running schedule is unchanged', self.errors[0]) - self.assertEqual([j.text for j in s.jobs], ['*/20 * * * Mon-Fri trades.py']) - self.clock.now = at(13, 21, 20) - self.assertEqual(len(s.due(self.clock.now)), 1) - - def test_load_reports_bad_lines_once_and_runs_the_rest(self): - """A bad line is reported once and the rest of the schedule runs.""" - self.file.write_text('0 1 * * * nothere.py\n*/20 * * * * trades.py\n') - s = self.scheduler() - self.assertEqual(len(s.jobs), 1) + self.assertEqual([job.text for job in s.chat_jobs(CHAT)], ['trades']) self.assertEqual(len(self.errors), 1) + self.assertIn("a job of telegram:55 left out: no report 'nothere'", self.errors[0]) def test_run_forever_survives_an_error(self): """run_forever reports a failed tick and carries on.""" @@ -358,15 +401,15 @@ def sleep(seconds): self.assertTrue(any('tick failed' in e for e in self.errors)) -class DaylightSaving(unittest.TestCase): +class DaylightSaving(FinbotCase): """Melbourne's clocks go back an hour at 03:00 on 5 April 2026 and forward at 02:00 on 4 October 2026. The fire times must be right on any croniter: the distro packages (up to 6.0) got zoneinfo's changes wrong.""" MEL = zoneinfo.ZoneInfo('Australia/Melbourne') def fires(self, expr, start, end): - """When a job fires between start and end, as Melbourne times ('04 09:00 AEDT').""" - job = jobs.parse_schedule(f'{expr} trades.py\n', self.MEL)[0][0] - s = jobs.Scheduler(lambda: None, zone=self.MEL) + """When a Melbourne job fires between start and end, as Melbourne times ('04 09:00 AEDT').""" + job = jobs.parse_job(f'CRON_TZ=Australia/Melbourne {expr} trades') + s = jobs.Scheduler() times, t = [], start while (t := s.next_fire(job, t)) <= end: # by way of UTC: the real local time @@ -404,265 +447,211 @@ def test_starting_in_the_repeated_hour_runs_nothing_twice(self): self.assertEqual(self.fires('30 2 * * *', second_time_round, self.mel(2026, 4, 6, 12)), ['06 02:30 AEST']) def test_the_scheduler_runs_a_daily_job_once_across_the_change(self): - """The scheduler runs a daily Melbourne job once a day across the change to daylight time.""" - file = Path(temp_dir(self), 'schedule.cron') - file.write_text('CRON_TZ=Australia/Melbourne\n0 9 * * * trades.py\n') + """The scheduler runs a chat's daily Melbourne job once a day across the change to daylight time.""" + self.patch(jobs, 'SYSTEM', ()) + with state.edit(jobs.SCHEDULE_FILE, {}) as chats: + chats[CHAT] = [entry('CRON_TZ=Australia/Melbourne 0 9 * * * trades')] clock = Clock(self.mel(2026, 10, 3, 12).astimezone(UTC)) - s = jobs.Scheduler(lambda: file, clock=clock, zone=UTC, runner=lambda *a: 0) - elsewhere = dataclasses.replace(config.env, data_dir=temp_dir(self)) - with mock.patch.object(config, 'env', elsewhere), contextlib.redirect_stderr(io.StringIO()): - self.assertTrue(s.load()) - fired = [] - while clock.now < self.mel(2026, 10, 5, 12): - fired += [t.astimezone(UTC).astimezone(self.MEL).strftime('%d %H:%M') for _, t in s.due(clock.now)] - clock.now += datetime.timedelta(minutes=10) + s = jobs.Scheduler(clock=clock, runner=lambda *a: None) + with contextlib.redirect_stderr(io.StringIO()): + s.load() + fired = [] + while clock.now < self.mel(2026, 10, 5, 12): + fired += [t.astimezone(UTC).astimezone(self.MEL).strftime('%d %H:%M') for _, t in s.due(clock.now)] + clock.now += datetime.timedelta(minutes=10) self.assertEqual(fired, ['04 09:00', '05 09:00']) -class ManualRuns(SchedulerBase): +class JobsInChat(SchedulerBase): + """What a chat does with its jobs: .jobs lists them; run, toggle, add and del; their buttons.""" + def setUp(self): - """A schedule with repeats, comments, braces and the commands that cannot be picked (cleanup, podcasts).""" + """Chat telegram:55 with its one job, the scheduler running, Telegram's toasts and edits noted.""" super().setUp() - self.file.write_text('# a comment\n*/20 * * * Mon-Fri trades.py > /dev/null\n' - '28 21 * * Sat price.py close; trades.py # weekly\n5 4 * * * { price.py close ;}\n6 4 * * * price.py 7\n' - '0 3 * * * performance.py 365; performance.py 7; performance.py 90; performance.py 30; performance.py 180\n' - '0 2 * * * cleanup.py\n1 * * * * podcasts.py "https://example.com/feed"\n') self.s = self.scheduler() self.patch(jobs, 'current', self.s) - self.patch(jobs.SCRIPTS['price'], 'run', lambda **k: self.ran.append(('price', k))) - - def test_commands_are_unique_clean_and_in_order(self): - """The commands offered are each once, without redirects, sorted by name with numbers in numeric order.""" - self.assertEqual([label for label, _, _ in self.s.commands()], - ['performance.py 7', 'performance.py 30', 'performance.py 90', 'performance.py 180', - 'performance.py 365', 'price.py 7', - 'price.py close', 'trades.py']) - - def test_select_by_number_or_command_text(self): - """A command is picked by its number or its text, quotes, paths and redirects aside.""" - self.assertEqual(self.s.select('7')[0], 'price.py close') - self.assertEqual(self.s.select('8')[0], 'trades.py') - self.assertEqual(self.s.select(' "price.py" close > /dev/null')[0], 'price.py close') - self.assertEqual(self.s.select('~/finbot/price.py 7')[0], 'price.py 7') - - def test_select_refuses_what_is_not_offered(self): - """Numbers out of range, cleanup, podcasts, unscheduled commands, shell syntax and two commands are refused.""" - for text in ('0', '9', 'cleanup.py', 'podcasts.py "https://example.com/feed"', 'price.py 99', 'cal.py earnings', - 'nonsense', 'ls | wc', '', 'price.py close; trades.py'): - with self.assertRaises(jobs.ScheduleError, msg=text): - self.s.select(text) - with self.assertRaisesRegex(jobs.ScheduleError, '^cleanup is a system job: it runs on its schedule only$'): - self.s.select('cleanup.py') - - def test_run_now_runs_one_command_and_leaves_the_schedule_alone(self): - """run_now runs one command and leaves the schedule's timing alone.""" - base, last = dict(self.s.base), dict(self.s.last) - self.assertEqual(self.s.run_now('7'), ('price.py close', 'done')) - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) - self.assertEqual((self.s.base, self.s.last), (base, last)) - self.assertEqual(self.s.busy, set()) - - def test_run_now_refuses_while_the_command_is_running_and_reports_failures(self): - """run_now refuses a command that is running, and reports one that fails.""" - self.s.busy.add('trades.py') - self.assertEqual(self.s.run_now('8'), ('trades.py', 'already running')) - self.assertEqual(self.ran, []) - self.s.busy.clear() - def boom(**k): - """Fail.""" - raise RuntimeError('boom') - self.patch(jobs.SCRIPTS['trades'], 'run', boom) - self.assertEqual(self.s.run_now('8'), ('trades.py', 'failed')) - self.assertTrue(any('boom' in e for e in self.errors)) - self.assertEqual(self.s.busy, set()) - - def test_a_scheduled_run_skips_a_command_that_a_manual_run_is_in_the_middle_of(self): - """A scheduled run skips a command still running (from chat, or the line's last run), and runs the rest of its - line.""" - self.s.busy.add('trades.py') - self.assertEqual(self.s.execute(self.s.jobs[1].commands), 'already running') - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) # price.py close ran, trades.py was skipped - - def test_a_manual_run_past_its_deadline_finishes(self): - """A manual run past its deadline is reported as late, and says how it went once it has finished.""" - self.patch(jobs, 'JOB_TIMEOUT', 0.05) - self.patch(jobs.SCRIPTS['trades'], 'run', lambda **k: time.sleep(0.2)) - with contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(self.s.run_now('8'), ('trades.py', 'done')) - self.assertEqual(len(self.errors), 1) - self.assertEqual(self.s.busy, set()) - - def test_two_runs_of_one_command_at_once_run_it_once(self): - """Of two runs of the same command started together, one runs it and the other is told it is running.""" - def slow(**k): - """Take a while, letting the other run start.""" - time.sleep(0.2) - self.ran.append(('trades', k)) - self.patch(jobs.SCRIPTS['trades'], 'run', slow) - outcomes = [] - threads = [threading.Thread(target=lambda: outcomes.append(self.s.run_now('8'))) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - self.assertEqual(sorted(outcomes), [('trades.py', 'already running'), ('trades.py', 'done')]) - self.assertEqual(self.ran, [('trades', {})]) - self.assertEqual(self.s.busy, set()) - - def list_in_chat(self, text='.jobs'): - """The lines .jobs (or `text`) answers with.""" - return self.command(text)[0][1] - - def test_list_in_chat_is_just_the_commands(self): - """.jobs and aliases number the commands, without times, comments or redirects, and mark one that is running; - the system jobs follow under their own heading, without a number.""" - feed = jobs.display(podcasts.job(['https://example.com/feed'])[0]) # podcasts and the feed's hash - for alias in ('.jobs', '.cron', '.crons', '.schedule', '@' + BOT.lstrip('@') + ' jobs'): - text = '\n'.join(map(str, self.list_in_chat(alias))) - for line in ('1. performance 7', '2. performance 30', '5. performance 365', '6. price 7', '7. price close', - '8. trades\n\nSystem:\ncleanup\n' + feed + '\n'): - self.assertIn(line, text, alias) - for noise in ('*/20', '#', 'Mon-Fri', 'next', 'last', '/dev/null', 'weekly', '.py'): - self.assertNotIn(noise, text, alias) - self.s.busy.add('price.py 7') - self.assertIn('6. price 7 (running)', '\n'.join(map(str, self.list_in_chat()))) - - def test_a_schedule_of_system_jobs_alone_lists_them(self): - """A schedule of system jobs alone lists them, with nothing to run or tap.""" - self.file.write_text('0 2 * * * cleanup.py\n') - self.patch(jobs, 'current', self.scheduler()) - self.assertEqual(about.prepare_jobs_payload(TELEGRAM, '', tappable=True), (['System:', 'cleanup'], None)) - - def test_a_chat_that_takes_buttons_gets_one_per_command(self): - """Where buttons work each command gets one, whose data fits in Telegram's 64 bytes, and the list says to tap - it; elsewhere there are none, and it does not.""" - payload, markup = about.prepare_jobs_payload(TELEGRAM, '', tappable=True) - pressable = [b for row in markup for b in row] - self.assertEqual([b.label for b in pressable], ['1. performance 7', '2. performance 30', '3. performance 90', - '4. performance 180', '5. performance 365', '6. price 7', '7. price close', '8. trades']) - for b in pressable: - self.assertLessEqual(len(b.data.encode()), 64) - self.assertTrue(payload[-1].endswith('or tap it')) - payload, markup = about.prepare_jobs_payload(SLACK, '') - self.assertIsNone(markup) - self.assertTrue(payload[-1].endswith('.jobs run NUMBER')) - - def test_slack_gets_the_buttons_in_block_kit_and_a_stale_list_is_replaced(self): - """With the bot token and slackOutgoingWebhook, Slack's .jobs list carries a button per command, two to a row - of Block Kit, and a stale button's press puts the current list in its place.""" - self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') - self.patch(transports.Transport, 'text', REAL_TEXT) - posts = [] - self.patch(transports.net, 'post', lambda url, **kwargs: posts.append((url, kwargs)) or - mock.Mock(status_code=200, json=lambda: {'ok': True, 'ts': '1.5'})) - with contextlib.redirect_stderr(io.StringIO()): - worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.jobs', BOT, 'U')) - url, kwargs = posts[-1] - blocks = json.loads(kwargs['data']['blocks']) - self.assertTrue(url.endswith('/chat.postMessage')) - self.assertEqual([block['type'] for block in blocks], ['section'] + ['actions'] * 4) - self.assertEqual([button['text']['text'] for button in blocks[4]['elements']], ['7. price close', '8. trades']) - with contextlib.redirect_stderr(io.StringIO()): - worker.process_callback(inbound.Press('slack', 't1', 'C1', '1.5', 'j|7|0000')) - url, kwargs = posts[-1] - self.assertTrue(url.endswith('/chat.update')) - self.assertEqual((kwargs['data']['channel'], kwargs['data']['ts']), ('C1', '1.5')) - self.assertEqual(len(json.loads(kwargs['data']['blocks'])), 5) - self.assertEqual(self.ran, []) - - def test_the_chat_command_passes_the_keyboard_on(self): - """.jobs sends its buttons with the list.""" - seen = [] - self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: seen.append(buttons) or []) - worker.process_request(inbound.Message('telegram', '55', '@u', '.jobs', BOT, 'U')) - self.assertEqual(len(seen[0][0]), 2) # two to a row - - def test_button_press_runs_that_command(self): - """Pressing a command's button runs it, says so in a toast, and reports how it went.""" - sent = [] - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: sent.append(text)) - markup = about.prepare_jobs_payload(TELEGRAM, '', tappable=True)[1] - data = next(b.data for row in markup for b in row if b.label.startswith('7.')) - self.sent.clear() - self.press_button(data) - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) - self.assertEqual(self.sent, [('text', ['7. price close: done'])]) - self.assertEqual(sent, ['Running…']) # one acknowledgement, from the data alone: the worker sends none - - def stale_buttons(self): - """Capture the toasts and the in-place edits that refused button presses produce.""" self.toasts, self.edits = [], [] self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: self.toasts.append(text)) self.patch(transports.TelegramBot, 'edit_text', lambda dest, mid, text, buttons=None: self.edits.append((dest.where, mid, text, buttons))) + self.replies = [] + self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: + self.replies.append((dest.key, list(lines), buttons)) or []) - def test_forged_button_data_never_runs_anything_else(self): - """Bad or forged button data runs nothing, refreshes buttons in place; no number picks cleanup or podcasts.""" - self.stale_buttons() - good = jobs.checksum('trades.py') - for data in ('j|-1|' + good, 'j|0|' + good, 'j|9|' + good, 'j|8|0000', 'j|8|', 'j|8', 'j|8|' + good + '|x', - 'j||' + good, - 'j|٨|' + good, 'j| 8|' + good, 'j|8.0|' + good, 'j|trades.py|' + good, 'j|1|' + good, - 'j|8|price.py close', - 'j|8|' + good.upper() + 'x', 'j|99999999999999999999|' + good): - self.sent.clear() - self.toasts.clear(); self.edits.clear() - self.press_button(data) - self.assertEqual(self.ran, [], data) - self.assertEqual(self.toasts, ['Running…'], data) # from the data alone, as the press arrived - self.assertEqual(len(self.edits), 1, data) - self.assertEqual(self.sent, [], data) # no new message, the old one is edited - # cleanup and podcasts are scheduled but cannot be picked, by number or by their checksum - for label in ('cleanup.py', 'podcasts.py'): - for n in range(1, 12): - with self.assertRaises(jobs.ScheduleError): - self.s.pick(str(n), jobs.checksum(label)) - worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', 'j|8|' + good)) # the genuine one still works + def say(self, text, chat='55'): + """Send text in a Telegram chat: the lines of the reply.""" + self.replies.clear() + with contextlib.redirect_stderr(io.StringIO()): + worker.process_request(inbound.Message('telegram', chat, '@u', text, BOT, 'U')) + self.assertEqual(self.errors, [], text) + return self.replies[-1][1] if self.replies else None + + def test_add_list_and_del(self): + """A job is added, listed with its number, line, state and next run (in UTC and Melbourne), and deleted.""" + added = self.say('.jobs add 29 21 * * Mon-Fri price interday') + self.assertEqual(added, ['2. 29 21 * * Mon-Fri price interday: on, next Tue 21:29 UTC (Wed 08:29 AEDT)', + 'Added: .jobs lists them all']) + listed = self.say('.jobs') + self.assertEqual(listed[:3], ['Jobs here:', '1. */20 * * * Mon-Fri trades: on, next Tue 21:20 UTC (Wed' + ' 08:20 AEDT)', '2. 29 21 * * Mon-Fri price interday: on, next Tue 21:29 UTC (Wed 08:29 AEDT)']) + self.assertEqual(listed[-4:], ['.jobs run [NUMBER]: run one now', '.jobs toggle [NUMBER]: switch one on or off', + '.jobs add [CRON_TZ=ZONE] TIME REPORT: add one (.jobs add lists the reports)', + '.jobs del NUMBER: delete one']) + self.assertIsNone(self.replies[-1][2]) # .jobs alone has no buttons + self.assertEqual(self.say('.jobs del 1'), ['Deleted 1. */20 * * * Mon-Fri trades']) + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {CHAT: [entry('29 21 * * Mon-Fri price interday')]}) + self.assertEqual([job.text for job in self.s.chat_jobs(CHAT)], ['price interday']) # at once, no reload + self.assertEqual(self.say('.job del 1'), ['Deleted 1. 29 21 * * Mon-Fri price interday']) + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {}) + self.assertEqual(self.say('.jobs')[0], 'No jobs here yet: add one with .jobs add') + + def test_a_job_names_its_zone_runs_several_reports_and_quotes_its_arguments(self): + """A job is in UTC unless it names its zone; it may run several reports in turn, and an argument in quotes may + hold anything.""" + url = 'https://example.com/feed?a=1&b=2;c' + self.say(f'.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession') + self.say(f'.jobs add 5 4 * * * podcasts "{url}"; shorts') + self.say('.jobs add @daily reminder') + ny, feed, daily = self.s.chat_jobs(CHAT)[1:] + self.assertEqual((ny.zone.key, ny.line), ('America/New_York', 'CRON_TZ=America/New_York 15 13 * * Mon-Fri' + ' price midsession')) + self.assertEqual([(label.split()[0], kwargs) for label, _, kwargs in feed.commands], + [('podcasts', {'feed_url': url}), ('shorts', {})]) + self.assertEqual((daily.when, daily.zone.key), ('@daily', 'UTC')) + self.assertIn('2. CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession: on, next Wed 17:15 UTC', + self.say('.jobs')[2]) + + def test_add_alone_lists_the_reports_a_job_can_run(self): + """.jobs add alone lists the reports with what each takes, the system jobs' aside, and examples.""" + lines = self.say('.jobs add') + self.assertEqual(lines[0], 'Reports a job can run:') + for line in ('cal earnings|ex-dividend', 'performance [DAYS]', 'podcasts FEED_URL', + 'price midsession|interday|premarket|close|DAYS', 'trades'): + self.assertIn(line, lines) + self.assertFalse(any(name in line for line in lines for name in jobs.SYSTEM_SCRIPTS)) + + def test_a_job_that_cannot_be_is_refused_saying_why(self): + """A time that cannot be or never comes, an unknown report, a script's file name, a system job, a bad argument, + an unknown zone or too few fields: refused, saying why, and nothing stored.""" + for line, why in (('61 1 * * * trades', "'61 1 * * *' is no time"), + ('0 0 30 2 * trades', "'0 0 30 2 *' never comes"), ('@reboot trades', '@reboot is no time'), + ('0 1 * * * nothere', "no report 'nothere'"), ('0 1 * * * trades.py', "no report 'trades.py'"), + ('0 1 * * * yahoo_warm', "no report 'yahoo_warm'"), + ('0 1 * * * price bogus', 'price takes midsession|interday|premarket|close|DAYS'), + ('0 1 * * * trades 7', 'trades takes no arguments'), + ('CRON_TZ=Nowhere/Land 0 1 * * * trades', "no time zone 'Nowhere/Land'"), + ('0 1 * *', 'a job is five time fields')): + with self.subTest(line=line): + self.assertIn(why, self.say(f'.jobs add {line}')[0]) + self.assertEqual(len(self.s.chat_jobs(CHAT)), 1) + + def test_a_chat_has_thirty_jobs_at_most(self): + """A chat may have 30 jobs; the 31st is refused.""" + for minute in range(1, 30): + self.s.add(CHAT, f'{minute} 1 * * * trades') + self.assertIn('30 jobs is the most a chat may have', self.say('.jobs add 31 1 * * * trades')[0]) + self.assertEqual(len(self.s.chat_jobs(CHAT)), 30) + + def test_run_a_job_now(self): + """.jobs run N runs job N now, posting to this chat, and says how it went; a number that is none says so.""" + self.assertEqual(self.say('.jobs run 1'), ['1. trades: done']) + self.assertEqual((self.ran, self.dests), ([('trades', {})], [CHAT])) + self.assertEqual(self.say('.jobs run 2'), ['no job 2: send .jobs for the list']) + self.assertEqual(self.say('.jobs run 1', chat='56'), ['no jobs here yet: .jobs add']) # another chat's list self.assertEqual(self.ran, [('trades', {})]) - def test_button_from_an_old_schedule_updates_the_message(self): - """A stale button runs nothing; the message gets the current list and buttons (scheduler off: a notice), which - says it all: the toast is the one any job button gets as it arrives, with no second one from the worker.""" - self.stale_buttons() - self.press_button('j|7|0000') - self.press_button('j|9|0000') - self.assertEqual(self.ran, []) - self.assertEqual(self.sent, []) - self.assertEqual(self.toasts, ['Running…'] * 2) - chat, mid, text, markup = self.edits[0] - self.assertEqual((chat, mid), ('55', '9')) # the message with the old buttons is edited in place - self.assertIn('7. price close', text) - self.assertEqual(markup, about.prepare_jobs_payload(TELEGRAM, '', tappable=True)[1]) # the current buttons - self.patch(jobs, 'current', None) # scheduler off: the list is replaced by the notice and the buttons removed - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|7|0000')) - self.assertEqual(self.edits[-1][2:], ('No schedule is running', [])) # no buttons left - - def test_run_in_chat(self): - """.jobs run takes a number or a command's text.""" - self.assertEqual(self.command('.jobs run 8'), [('text', ['8. trades: done'])]) + def test_toggle_a_job_off_and_on(self): + """.jobs toggle N switches job N off (it no longer fires) and on again.""" + self.assertEqual(self.say('.jobs toggle 1'), ['1. */20 * * * Mon-Fri trades: off']) + self.clock.now = at(13, 21, 20) + self.assertEqual(self.s.due(self.clock.now), []) + self.assertEqual(self.say('.jobs toggle 1')[0][:30], '1. */20 * * * Mon-Fri trades: ') + self.clock.now = at(13, 21, 40) + self.assertEqual(len(self.s.due(self.clock.now)), 1) + + def test_the_run_and_toggle_lists_have_a_button_per_job(self): + """.jobs run and .jobs toggle list the jobs with a button each that runs it now or switches it.""" + self.s.add(CHAT, '0 22 * * * shorts') + for action, kind in (('run', 'j'), ('toggle', 'e')): + with self.subTest(action=action): + self.say(f'.jobs {action}') + lines, markup = self.replies[-1][1:] + tap = 'Tap one to run it now' if action == 'run' else 'Tap one to switch it on or off' + self.assertIn(tap, lines[-1]) + job = self.s.chat_jobs(CHAT)[1] + self.assertEqual(markup[0][1], buttons.Button('2. shorts', f'{kind}|2|{jobs.checksum(job.key)}')) + + def test_a_run_button_runs_its_job_and_a_toggle_button_redraws_the_list(self): + """A run button runs its job and says how it went; a toggle button switches its job and redraws the list in + place, with the buttons still there.""" + job = self.s.chat_jobs(CHAT)[0] + self.press_button(f'j|1|{jobs.checksum(job.key)}') + self.assertEqual((self.toasts, self.ran), (['Running…'], [('trades', {})])) + self.assertEqual(self.replies[-1][1], ['1. trades: done']) + self.press_button(f'e|1|{jobs.checksum(job.key)}') + self.assertEqual(self.toasts[-1], 'Switching…') + self.assertFalse(self.s.chat_jobs(CHAT)[0].on) + chat, message, text, markup = self.edits[-1] + self.assertEqual((chat, message), ('55', '9')) + self.assertIn('1. */20 * * * Mon-Fri trades: off', text) + self.assertEqual(markup[0][0].data, f'e|1|{jobs.checksum(job.key)}') # the key does not hold the switch + + def test_a_stale_or_forged_button_redraws_the_list_and_does_nothing(self): + """A button whose number or checksum is not this chat's job (an older list, another chat's, forged data) runs + and switches nothing: the list is redrawn in its place, with that kind of buttons.""" + self.store(OTHER, '*/20 * * * Mon-Fri trades') + self.s.load() + good = jobs.checksum(self.s.chat_jobs(CHAT)[0].key) + others = jobs.checksum(self.s.chat_jobs(OTHER)[0].key) + for data in ('j|2|' + good, 'j|0|' + good, 'j|1|0000', 'j|1|' + others, 'e|1|0000', 'j|1', 'j|٨|' + good, + 'j| 1|' + good, 'e|99999999999999999999|' + good): + with self.subTest(data=data): + self.edits.clear() + self.press_button(data) + self.assertEqual(self.ran, []) + self.assertTrue(self.s.chat_jobs(CHAT)[0].on) + self.assertEqual(len(self.edits), 1) + self.assertIn('1. */20 * * * Mon-Fri trades: on', self.edits[0][2]) + self.assertEqual(self.edits[0][3][0][0].data[0], data[0]) # the same kind of buttons + self.press_button('j|1|' + good) # the genuine one still works self.assertEqual(self.ran, [('trades', {})]) - self.ran.clear() - self.assertEqual(self.command('.cron run price close')[0][1], ['7. price close: done']) - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) - - def test_run_in_chat_refusals(self): - """.jobs run refuses an unscheduled command, an unknown number and two commands, else gives its usage.""" - self.assertIn('not in the schedule', self.list_in_chat('.jobs run cal.py earnings')[0]) - self.assertIn('no command 99', self.list_in_chat('.jobs run 99')[0]) - self.assertIn('one command at a time', self.list_in_chat('.jobs run price.py close; trades.py')[0]) - self.assertIn('Usage', self.list_in_chat('.jobs run')[0]) - self.assertIn('Usage', self.list_in_chat('.jobs bogus')[0]) - self.assertEqual(self.ran, []) def test_no_schedule_running(self): - """Without a schedule running (none was loaded), .jobs says so.""" + """Without a schedule running, .jobs says so.""" self.patch(jobs, 'current', None) - self.assertEqual(self.list_in_chat('.jobs'), ['No schedule is running']) - self.assertEqual(self.list_in_chat('.jobs run 8'), ['No schedule is running']) - - def test_job_is_still_a_ticker(self): - """.job is a ticker, not .jobs.""" - self.assertNotIn('Scheduled', str(self.command('.job'))) + self.assertEqual(self.say('.jobs'), ['No schedule is running']) + + def test_a_job_is_its_chats_whichever_way_its_updates_arrive(self): + """A chat's jobs are kept under its network and its id, which a network gives the same in both modes: a job + added in http mode is that chat's in pull mode, listed and fired there, and the other way round.""" + updates = test_sending.EveryNetworkAndMode() + updates.updates = 0 + self.patch(inbound.DISCORD, 'bot_id', '99') + self.env(slackBotToken='xoxb-1', discordBotToken='x') + for network, chat in (('telegram', '-1001'), ('slack', 'C1'), ('discord', '77')): + for first, then in (('http', 'pull'), ('pull', 'http')): + with self.subTest(network=network, added=first): + self.env(**test_sending.EveryNetworkAndMode.MODES[(network, first)]) + reader = getattr(inbound, network.upper()) + with contextlib.redirect_stderr(io.StringIO()): + answer = bot.take(reader, updates.update(network, first, chat, '.jobs add 0 23 * * * shorts')) + if answer.then: + answer.then() + self.join_workers() + key = f'{network}:{chat}' + self.assertEqual([job.text for job in self.s.chat_jobs(key)], ['shorts']) + self.env(**test_sending.EveryNetworkAndMode.MODES[(network, then)]) + self.replies.clear() + answer = bot.take(reader, updates.update(network, then, chat, '.jobs')) + if answer.then: + answer.then() + self.join_workers() + self.assertIn('1. 0 23 * * * shorts: on', self.replies[-1][1][1]) + self.assertEqual(self.replies[-1][0], key) + self.dests.clear() + self.s.run(self.s.chat_jobs(key)[0]) + self.assertEqual(self.dests, [key]) # posted to the same chat + self.s.delete(self.s.chat_jobs(key)[0]) class ReportImports(FinbotCase): diff --git a/tests/test_sending.py b/tests/test_sending.py index db33ca85..50ac5242 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -535,8 +535,8 @@ def post(url, **kwargs): class EveryNetworkAndMode(FinbotCase): """One rule for the six networks and modes, in a channel and in a DM: a chart has its period buttons and a .jobs - list its buttons, a command shows the network's notice that finbot is at work, and the reply goes to the chat that - asked.""" + run list its buttons, a command shows the network's notice that finbot is at work, and the reply goes to the chat + that asked.""" MODES = { ('telegram', 'http'): {'telegramOutgoingWebhook': 'https://example.com/telegram'}, ('telegram', 'pull'): {'telegramOutgoingWebhook': ''}, @@ -549,13 +549,15 @@ class EveryNetworkAndMode(FinbotCase): CHATS = {'telegram': ('-1001', '55'), 'slack': ('C1', 'D1'), 'discord': ('77', '88')} # a channel, then a DM def setUp(self): - """A schedule for .jobs to list, and what each command posts noted with its destination.""" + """A job in each chat for .jobs run to list, and what each command posts noted with its destination.""" super().setUp() self.patch(inbound.DISCORD, 'bot_id', '99') - schedule = Path(temp_dir(self), 'schedule.cron') - schedule.write_text('*/20 * * * Mon-Fri trades.py\n') - scheduler = jobs.Scheduler(lambda: schedule) - scheduler.load() + every = [{'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}] + state.save(jobs.SCHEDULE_FILE, {f'{network}:{chat}': every for network, chats in self.CHATS.items() + for chat in chats}) + scheduler = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + scheduler.load() self.patch(jobs, 'current', scheduler) self.posted = [] self.patch(transports.Transport, 'chart', lambda dest, image, caption, buttons=None: @@ -583,8 +585,8 @@ def update(self, network, mode, chat, text): 'data': {'name': name, 'options': options}} def test_every_network_in_both_modes_in_a_channel_and_a_dm(self): - """A chart and a .jobs list each come with their buttons, in the chat that asked; the command showed the notice - that finbot is at work: "typing…" (Telegram, a Discord message), ⏳ on the message (Slack), or Discord's + """A chart and a .jobs run list each come with their buttons, in the chat that asked; the command showed the + notice that finbot is at work: "typing…" (Telegram, a Discord message), ⏳ on the message (Slack), or Discord's "thinking…" for a slash command, which its answer replaces.""" for (network, mode), keys in self.MODES.items(): for chat in self.CHATS[network]: @@ -593,7 +595,7 @@ def test_every_network_in_both_modes_in_a_channel_and_a_dm(self): self.posted.clear(); self.typing.clear() answers = [] with contextlib.redirect_stderr(io.StringIO()): - for text in ('.price', '.jobs'): + for text in ('.price', '.jobs run'): answer = bot.take(getattr(inbound, network.upper()), self.update(network, mode, chat, text)) answers.append(answer) if answer.then: diff --git a/util/record_outputs.py b/util/record_outputs.py index ba22947a..047c9b42 100755 --- a/util/record_outputs.py +++ b/util/record_outputs.py @@ -22,12 +22,14 @@ '.session T1', '.premarket', '.postmarket', '.shorts', '.shorts 30', '.shorts T1', '.performance', '.performance 7d', '.performance 1y', '.performance Nope', '.pie Rob', '.pie Rob sector', '.pie', '.compare T1 T2', '.compare T1 T2 3m', '.compare T1', '.history T1', '.history', '.watchlist', '.watchlist add MSFT', - '.watchlist add MSFT', '.watchlist del MSFT', '.watchlist frob X', '.T1', '.jobs', 'hi @bot', 'noise'] + '.watchlist add MSFT', '.watchlist del MSFT', '.watchlist frob X', '.T1', '.jobs', '.jobs run', '.jobs toggle', + '.jobs add', '.jobs add 0 22 * * * shorts', '.jobs del 4', '.jobs run 1', 'hi @bot', 'noise'] BUTTONS = ['c|h|m|T1', 'c|h|x|T1', 'c|p|w|T1', 'c|c|y|T1,T2', 'c|f|y|', 'c|l|d|3,0', 'c|l|m|0,10', 'c|zz|m|T1', - 'j|1|0000'] -SCHEDULED = ['trades.py', 'price.py midsession', 'price.py interday', 'price.py premarket', 'price.py close', - 'price.py 7', 'cal.py earnings', 'cal.py ex-dividend', 'performance.py 7', 'rating.py', 'shorts.py', 'milestone.py', - 'reminder.py'] + 'j|1|0000', 'e|1|0000'] +JOBS = ['*/20 * * * Mon-Fri trades', '29 21 * * Mon-Fri price interday', # each chat's, for .jobs + '28 21 * * Sat cal earnings; cal ex-dividend; price 7; performance 7; rating'] +SCHEDULED = ['trades', 'price midsession', 'price interday', 'price premarket', 'price close', 'price 7', + 'cal earnings', 'cal ex-dividend', 'performance 7', 'rating', 'shorts', 'milestone', 'reminder'] TELEGRAM_API = 'https://api.telegram.org/botX/' ANSWER = {'ok': True, 'upload_url': 'https://files.slack.com/u', 'file_id': 'F1', 'id': '1'} # good enough for every API @@ -106,7 +108,14 @@ def post(url, method='POST', **kwargs): self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') # Slack's buttons too if 'discordBotToken' in {field.name for field in common.dataclasses.fields(common.config.env)}: self.env(discordBotToken='discord-test') - schedule = jobs.Scheduler(lambda: REPO / 'defaults' / 'schedule.cron', zone=common.UTC) # for .jobs + if hasattr(jobs, 'SCHEDULE_FILE'): # each chat's jobs, since jobs per chat + entries = [{'when': job.when, 'reports': job.text, 'zone': 'UTC', 'on': True} + for job in map(jobs.parse_job, JOBS)] + chats = ('telegram:55', 'slack:C1', 'discord:77') + common.state.save(jobs.SCHEDULE_FILE, {key: entries for key in chats}) + schedule = jobs.Scheduler() + else: # one schedule for all chats, the shipped one + schedule = jobs.Scheduler(lambda: REPO / 'defaults' / 'schedule.cron', zone=common.UTC) with contextlib.redirect_stderr(io.StringIO()): schedule.load() self.patch(jobs, 'current', schedule) @@ -137,7 +146,11 @@ def step(label, run): discord = {'discordChannelID': '78'} if 'discordChannelID' in fields else {} self.env(telegramChatID='1', slackChannelID='C9', **api, **discord) for line in SCHEDULED: - label, module, kwargs = jobs.parse_command(line)[0] + if hasattr(jobs, 'parse_reports'): + label, module, kwargs = jobs.parse_reports(line)[0] + else: # a commit from before jobs per chat, whose reports were scripts + name, _, arguments = line.partition(' ') + label, module, kwargs = jobs.parse_command(f'{name}.py {arguments}'.strip())[0] run = getattr(module, 'run', None) or module.lambda_handler # lambda_handler: a commit before run() step(f'scheduled {line}', lambda: run(**kwargs)) Path(output).write_text(json.dumps(steps, indent=1, sort_keys=True, default=str)) diff --git a/util/sync_defaults.py b/util/sync_defaults.py index 081018cb..50e1f950 100755 --- a/util/sync_defaults.py +++ b/util/sync_defaults.py @@ -21,9 +21,9 @@ def sync(apply=False, force=False, dest=None): for src in sorted((config.REPO_DIR / 'defaults').iterdir()): if not src.is_file(): continue - if src.name == 'finbot.env': + if src.name in ('finbot.env', 'schedule.cron'): # the startup file is not data: it holds credentials, lives wherever the launcher loads it from, and the bot - # never reads it + # never reads it; the old schedule is read only to give the home chats their jobs once (lib/migrate.py) continue name = src.name target = dest_dir / name From a0cb7a4e8750f1240d246bc5a56f4185baabeb3a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:53:41 +0000 Subject: [PATCH 09/31] A chat's scheduled price, shorts or cal job runs as scheduled price, shorts and cal took any destination for a chat command, so a chat's job of them posted "nothing meets the threshold" on every quiet run, skipped the trigger bars and left the earnings report unpinned. They now ask delivery.interactive(dest), as trades, rating and milestone do. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/reports/cal.py | 6 +++--- lib/reports/performance.py | 4 ++-- lib/reports/price.py | 6 +++--- lib/reports/reminder.py | 2 +- lib/reports/shorts.py | 5 +++-- tests/test_reports.py | 17 +++++++++++++++++ 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/reports/cal.py b/lib/reports/cal.py index 2fa38f15..5018ace5 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -18,9 +18,9 @@ def primary_listings(): def run(dest=None, *, days=None, specific_stock=None, event='earnings'): """Post the earnings or ex-dividend dates ahead (`event`) for the tracked stocks or one (Yahoo's symbol): to the - chat that asked (dest), else to the default chats. The scheduled earnings report is pinned where the network can; a - reply is not, or the chat would collect pinned copies.""" - interactive = dest is not None + chat that asked or whose job it is (dest), else to the home chats. The scheduled earnings report is pinned where the + network can; a reply is not, or the chat would collect pinned copies.""" + interactive = delivery.interactive(dest) if days is None: days = config.prefs.future_days events = gather(watchlist.covered(specific_stock), days, specific_stock, event) diff --git a/lib/reports/performance.py b/lib/reports/performance.py index fee2ac6b..13086d5f 100644 --- a/lib/reports/performance.py +++ b/lib/reports/performance.py @@ -49,8 +49,8 @@ def report(service, past_days, portfolio_select=None, performance=None): def run(dest=None, *, past_days=None, user='', portfolio_select=None): """Post each portfolio's performance over past_days (the past_days preference when None), or the one named, beside - the benchmarks: to the chat that asked (dest), else to the default chats, with its chart and period buttons where - they can go.""" + the benchmarks: to the chat that asked or whose job it is (dest), else to the home chats, with its chart and period + buttons where they can go.""" if past_days is None: past_days = config.prefs.past_days performance = gather(past_days, portfolio_select) diff --git a/lib/reports/price.py b/lib/reports/price.py index bef364a0..10a1fb46 100644 --- a/lib/reports/price.py +++ b/lib/reports/price.py @@ -172,9 +172,9 @@ def nothing(mode, threshold, user, ticker, in_session, days=None): return f"{user}, no price movements meet threshold {threshold:g}%" def run(dest=None, *, mode='interday', threshold=None, days=None, top=None, user='', specific_stock=None): - """Post the price report (report() has the modes): to the chat that asked (dest), else to the default chats, with - its chart and period buttons where they can go.""" - interactive = dest is not None + """Post the price report (report() has the modes): to the chat that asked or whose job it is (dest), else to the + home chats, with its chart and period buttons where they can go.""" + interactive = delivery.interactive(dest) if threshold is None: threshold = config.prefs.price_percent found = moves(mode, threshold, days, specific_stock, interactive) diff --git a/lib/reports/reminder.py b/lib/reports/reminder.py index cbbd5513..1c86763f 100644 --- a/lib/reports/reminder.py +++ b/lib/reports/reminder.py @@ -13,7 +13,7 @@ def run(dest=None): - """Post today's reminders to the default chats (or dest), if there are any.""" + """Post today's reminders, if there are any, to the chat whose job it is (dest), else to the home chats.""" today = datetime.datetime.now(zoneinfo.ZoneInfo(config.prefs.timezone)).date() entries = config.data_json("reminder.json", []) for chat in delivery.targets(dest): diff --git a/lib/reports/shorts.py b/lib/reports/shorts.py index 3ac51449..25e23b57 100644 --- a/lib/reports/shorts.py +++ b/lib/reports/shorts.py @@ -8,8 +8,9 @@ def run(dest=None, *, threshold=None, specific_stock=None): """Post the stocks followed whose short interest is over `threshold` percent (the shorts_percent preference when - None), or one stock's (Yahoo's symbol): to the chat that asked (dest), else to the default chats.""" - interactive = dest is not None + None), or one stock's (Yahoo's symbol): to the chat that asked or whose job it is (dest), else to the home + chats.""" + interactive = delivery.interactive(dest) if threshold is None: threshold = config.prefs.shorts_percent tickers, market_data = gather(specific_stock) diff --git a/tests/test_reports.py b/tests/test_reports.py index c0f9a030..16d5f7bc 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -345,6 +345,23 @@ def test_only_the_scheduled_earnings_report_is_pinned(self): cal.run(event='earnings') self.assertEqual(pins, ['1']) + def test_a_chats_job_runs_its_report_as_scheduled_not_as_a_command(self): + """A report a chat's job runs is a scheduled run, as the home chats' is: it says nothing when nothing passes, + where a command's reply says so, and the earnings report is pinned. price, shorts and cal took any chat they + were given for a command.""" + pins = [] + self.patch(transports.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) + job_chat = delivery.chat('telegram:5', scheduled=True) + for module, kwargs in ((price, {'mode': 'premarket', 'threshold': 1000}), (shorts, {'threshold': 1000})): + with self.subTest(report=module.__name__): + self.sent.clear() + module.run(job_chat, **kwargs) + self.assertEqual(self.sent, []) + module.run(transports.TelegramBot('5'), **kwargs) # a command's reply says that nothing passed + self.assertTrue(self.sent) + cal.run(job_chat, event='earnings') + self.assertEqual(pins, ['5']) + def test_earnings_skip_an_adr_whose_primary_listing_yahoo_does_not_know(self): """T1 has no earnings date, so its primary listing is looked up: Yahoo not knowing it crashed the report.""" self.patch(cal, 'primary_listings', lambda: {'T1': 'T1.XX'}) From d5a1a5e46e093000bf6b432eab3a26244d4bb6b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:01:51 +0000 Subject: [PATCH 10/31] Watchlist and plans per chat Each chat now has its own watchlist (finbot_watchlist.json, one list per chat key) and its own plans (finbot_plan.json, {user: plan} per chat): - .watchlist lists and changes this chat's watchlist; every report in a chat, asked for or scheduled (price, shorts, cal, rating, milestone, the rankings, a price list's period buttons), covers that chat's watchlist plus the Sharesight holdings, which every chat shares. - yahoo_warm keeps every chat's watchlist warm (watchlist.all_followed). - .plan records and lists the plans of this chat only. The migration copies the old shared watchlist and plans to each home chat once, as it does the reports' records. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 9 +++-- README.md | 13 +++----- lib/charts.py | 29 ++++++++-------- lib/migrate.py | 12 ++++--- lib/reports/cal.py | 8 ++--- lib/reports/holdings.py | 20 ++++++----- lib/reports/milestone.py | 6 ++-- lib/reports/price.py | 29 ++++++++-------- lib/reports/rankings.py | 37 +++++++++++---------- lib/reports/rating.py | 8 ++--- lib/reports/shorts.py | 16 ++++----- lib/reports/yahoo_warm.py | 14 ++++---- lib/watchlist.py | 70 ++++++++++++++++++++++++--------------- lib/worker.py | 34 ++++++++++--------- tests/common.py | 2 +- tests/test_edges.py | 23 +++++++------ tests/test_inbound.py | 5 ++- tests/test_reports.py | 70 ++++++++++++++++++++++++++++----------- tests/test_schedule.py | 15 +++++---- tests/test_scheduler.py | 12 +++---- tests/test_util.py | 16 ++++++--- 21 files changed, 257 insertions(+), 191 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1f8b42b4..7a58099e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,14 +83,17 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm when Yahoo does not know it bare: every ticker a chat gives goes through it. `sharesight.portfolios(name=None)` gives `Portfolio` objects: `holdings()`, `tickers()`, `gain(days)` (the portfolio's), `trades(days)`. - `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; - change one with `with state.edit(name, default) as data:`, never load-then-save. `lib/watchlist.py`: the watchlist. - A chat's key is its network and id (`Transport.key`, `telegram:-1001234`), the same in both modes. + change one with `with state.edit(name, default) as data:`, never load-then-save; a file of one record per chat with + `state.load_for(chat, name, default)` and `state.edit_for`. A chat's key is its network and id (`Transport.key`, + `telegram:-1001234`), the same in both modes. `lib/watchlist.py`: each chat's watchlist (`tickers(chat)`), and + `with_holdings(chat)`, the stocks a report in that chat covers; a report takes the chat from its destination. - `lib/config.py`: `config.env` (finbot.env, fixed at start) and `config.prefs` (preferences.conf, a new object on each reload). `lib/jobs.py`: the scheduler: each chat's jobs (`SCHEDULE_FILE`, by chat key; `add`, `delete`, `toggle`, `run`, which `.jobs` calls through `jobs.current`) and the fixed `SYSTEM` jobs, its shared state under one lock; each command runs in the bot's process, in a thread (`run_in_process`), reported once if still running after `JOB_TIMEOUT` and left to finish (every request goes through `net`, with its timeouts: a test checks). - `lib/migrate.py`: once, the old schedule.cron and the reports' shared records become the home chats'. + `lib/migrate.py`: once, the old schedule.cron, the watchlist, the plans and the reports' shared records become the + home chats'. - `tests/common.py`: `FinbotCase` (offline fixtures). `self.command('.price')` returns what was sent: `('text', lines)` or `('photo', size, caption)`. `wsgi_environ(path, body, **headers)` is a request for `bot.main`. `ProviderCase` runs the real parsers on the answers saved under `doc/` (`sample`), as `Resp` and `serve` return diff --git a/README.md b/README.md index 5f3ed549..8da22666 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ * Price history graph * Stock lookup with price/valuation related stats and warnings * Company profiles - * Shared watch list + * A watch list per chat * List stocks (holdings + watchlist) by P/E, Forward P/E, PEG, market cap, or Yahoo analyst consensus ### Chat networks @@ -101,7 +101,7 @@ The other various reports can also run on a schedule (e.g. daily or weekly), or The interactive bot answers commands in your chats. It fetches its updates from Telegram and Slack itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: * Stock lookup (financials, company profile, price charts) -* Group maintainable watch list, picked up by the various reports +* A watch list per chat, which anyone in it can change, picked up by the reports there * Listing of portfolios and portfolio performance * Listing of holdings sorted by various attributes * Running the other stock reports on demand @@ -405,14 +405,9 @@ rating ### Watchlist ![Shared watchlist in Slack](img/watchlist.png?raw=true "Shared watchlist in Slack") -Tracks additional securities which are not in your Sharesight holdings. The various reports will include wishlist items as if they are holdings. +Tracks additional securities which are not in your Sharesight holdings. Each chat has its own watchlist, which anyone in the chat can change: the reports in that chat, asked for or scheduled, include its watchlist's stocks as if they were holdings, and no other chat's. The Sharesight holdings are the same in every chat. `yahoo_warm` keeps every chat's watchlist warm in the cache. -It is stored in `var/finbot_watchlist.json` by default. It is a JSON array literal containing Yahoo symbols. Example: -``` -["2454.TW", "3217.TWO", "ASO.AX", "STEM"] -``` - -When the bot is run interactively, the watchlist is dynamic and can be edited by members of a chat group. In this case, there is no need to create/edit the file manually. +The watchlists are kept in `finbot_watchlist.json` in the data directory, by chat (`{"chats": {"telegram:-1001234": ["2454.TW", "ASO.AX", "STEM"]}}`), so there is no need to edit the file. The first start with watchlists per chat gives each home chat the watchlist kept before, and the same goes for the plans that `.plan` records: each chat's `.plan` lists only the plans recorded in it. Interactive trigger: ``` diff --git a/lib/charts.py b/lib/charts.py index 0dbf7781..185bb35f 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -1,18 +1,20 @@ """A chart with period buttons, built by its kind's builder: the command's first view, and again for the period of a -button pressed, so that both look alike.""" +button pressed, so that both look alike. Each builder takes the network's dialect, the chat's key (a price list covers +the stocks that chat follows), the period, its days, then what the buttons carry.""" from lib import answer from lib import buttons from lib import formats from lib import links +from lib import watchlist from lib import yahoo from lib.reports import history, performance, price -def history_chart(service, period, days, ticker): +def history_chart(service, chat, period, days, ticker): """.history: the change over every period, the chosen one marked, and the line of that period (or of it all).""" lines, chart = history.history_payload(ticker, service, days) return '\n'.join(lines), chart -def price_chart(service, period, days, ticker): +def price_chart(service, chat, period, days, ticker): """.price SYMBOL [period]: one stock's change over the period (all its history for Max), from Yahoo, and its line.""" market_data = yahoo.fetch([ticker]) @@ -25,22 +27,22 @@ def price_chart(service, period, days, ticker): caption = f"{formats.get_emoji(percent)} {title} ({link}) {formats.signed_percent(round(percent))}" return caption, chart -def compare_chart(service, period, days, *tickers): +def compare_chart(service, chat, period, days, *tickers): """.compare: the stocks' changes over the period, rebased on one chart.""" caption, chart = history.prepare_compare(service, list(tickers), days) return '\n'.join(caption), chart -def performance_chart(service, period, days, portfolio): +def performance_chart(service, chat, period, days, portfolio): """.performance: each portfolio over the period, or the one named ('' for all).""" payload, chart = performance.report(service, days or buttons.MAX_DAYS, portfolio or None) return '\n'.join(payload), chart -def price_list_chart(service, period, days, threshold, top): - """.price: the moves past `threshold`, or the top and bottom `top` ('0': not a top list), over the period; period - 'd' is today's moves, the command's default view, not a lookback of one day.""" +def price_list_chart(service, chat, period, days, threshold, top): + """.price: the moves past `threshold` among the stocks the chat follows, or the top and bottom `top` ('0': not a + top list), over the period; period 'd' is today's moves, the command's default view, not a lookback of one day.""" span = {'mode': 'interday'} if period == 'd' else {'mode': 'period', 'days': days or buttons.MAX_DAYS} threshold, top = float(threshold), int(top) or None - found = price.moves(threshold=threshold, interactive=True, **span) + found = price.moves(watchlist.with_holdings(chat), threshold=threshold, interactive=True, **span) payload, chart = price.report(service, found, threshold=threshold, interactive=True, top=top, **span) if not chart: raise answer.Answer(*(payload or ['no price data'])) @@ -48,9 +50,10 @@ def price_list_chart(service, period, days, threshold, top): BUILDERS = {'h': history_chart, 'p': price_chart, 'c': compare_chart, 'f': performance_chart, 'l': price_list_chart} -def build(kind, args, service, period=None, days=None): - """(caption, chart spec or None) of a chart kind from the arguments its buttons carry (buttons.keyboard): over a - button period, or over `days` (a command's own span: .compare 3m), or the command's default view.""" +def build(kind, args, service, chat, period=None, days=None): + """(caption, chart spec or None) of a chart kind, in a chat (its key), from the arguments its buttons carry + (buttons.keyboard): over a button period, or over `days` (a command's own span: .compare 3m), or the command's + default view.""" if kind not in BUILDERS: raise ValueError(f"unknown chart kind {kind}") - return BUILDERS[kind](service, period, buttons.PERIODS[period][1] if period else days, *args) + return BUILDERS[kind](service, chat, period, buttons.PERIODS[period][1] if period else days, *args) diff --git a/lib/migrate.py b/lib/migrate.py index c3586206..00ba9cfa 100644 --- a/lib/migrate.py +++ b/lib/migrate.py @@ -1,7 +1,8 @@ """Once, as finbot starts: what the bot kept for all chats at once becomes each home chat's (telegramChatID, slackChannelID, discordChannelID). The old schedule.cron (the data dir's copy, else the shipped one) is copied to every -home chat as its jobs, each in the zone it ran in, and is read no more; and the reports' records of what they told the -chats (trades, ratings, record highs, podcast episodes) are copied to each home chat, so that nothing old is told again. +home chat as its jobs, each in the zone it ran in, and is read no more; the watchlist and the plans are copied to each +home chat; and so are the reports' records of what they told the chats (trades, ratings, record highs, podcast +episodes), so that nothing old is told again. This module, and defaults/schedule.cron, go once every install has started with it: they are its only users.""" import copy @@ -18,9 +19,12 @@ from lib import delivery from lib import jobs from lib import state +from lib import watchlist +from lib import worker from lib.reports import milestone, podcasts, rating, trades -RECORDS = (trades.STATE_FILE, rating.STATE_FILE, milestone.STATE_FILE) # and each podcast feed's seen file +# kept for all chats, each now one record per chat; and each podcast feed's seen file +RECORDS = (watchlist.FILE, worker.PLANS, trades.STATE_FILE, rating.STATE_FILE, milestone.STATE_FILE) # what a shell reads as more than a word, outside quotes, in a pasted crontab line OPERATOR = re.compile(r'\d*>>?&?\d*|&>>?|;|&&|\|\|?|&|[<`$(){}]') REDIRECT = re.compile(r'\d*>>?&?\d*|&>>?') # > file, >> file, 2>&1, &> file @@ -70,7 +74,7 @@ def schedule(homes): def record(name, homes): - """A report's record kept for all chats at once (anything but {'chats': ...}) becomes each home chat's.""" + """A file's record kept for all chats at once (anything but {'chats': ...}) becomes each home chat's.""" try: data = state.load(name) except (OSError, ValueError) as e: diff --git a/lib/reports/cal.py b/lib/reports/cal.py index 5018ace5..a77f12ea 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -17,14 +17,14 @@ def primary_listings(): 'ex-dividend': ('ex_dividend_date', 'Upcoming ex-dividends')} # -> (field, heading) def run(dest=None, *, days=None, specific_stock=None, event='earnings'): - """Post the earnings or ex-dividend dates ahead (`event`) for the tracked stocks or one (Yahoo's symbol): to the - chat that asked or whose job it is (dest), else to the home chats. The scheduled earnings report is pinned where the - network can; a reply is not, or the chat would collect pinned copies.""" + """Post the earnings or ex-dividend dates ahead (`event`) for the stocks a chat follows or one (Yahoo's symbol): to + the chat that asked or whose job it is (dest), else to the home chats. The scheduled earnings report is pinned where + the network can; a reply is not, or the chat would collect pinned copies.""" interactive = delivery.interactive(dest) if days is None: days = config.prefs.future_days - events = gather(watchlist.covered(specific_stock), days, specific_stock, event) for chat in delivery.targets(dest): + events = gather(watchlist.covered(chat.key, specific_stock), days, specific_stock, event) payload = report(chat.dialect, events, days, specific_stock, interactive, event) answers = delivery.deliver(chat, delivery.Report(payload)) if event == 'earnings' and not specific_stock and not interactive and hasattr(chat, 'pin'): diff --git a/lib/reports/holdings.py b/lib/reports/holdings.py index d89c6e6a..6afc7f4f 100644 --- a/lib/reports/holdings.py +++ b/lib/reports/holdings.py @@ -60,12 +60,12 @@ def prepare_pie(portfolio_name, by, service, max_slices=8): subtitle = f"{len(totals)} {noun[0] if len(totals) == 1 else noun[1]}" return payload, render.Chart('pie', title, subtitle, tuple(rows)) -def prepare_watchlist(service, user, action=None, ticker=None): - """The .watchlist reply: the companies watched, or one line on adding ('add') or removing ('delete') a ticker - (Yahoo's symbol, as yahoo.symbol gives it).""" +def prepare_watchlist(service, chat, user, action=None, ticker=None): + """The .watchlist reply in a chat (by its key): the companies it watches, or one line on adding ('add') or removing + ('delete') a ticker (Yahoo's symbol, as yahoo.symbol gives it).""" if action: - return [watchlist_change(service, user, action, ticker)] - watched = watchlist.tickers() + return [watchlist_change(service, chat, user, action, ticker)] + watched = watchlist.tickers(chat) market_data = yahoo.fetch(watched) if watched else {} lines = [links.stock_line(ticker, quote, service) for ticker, quote in sorted(market_data.items(), key=lambda item: item[1]['profile_title'].lower())] @@ -73,15 +73,17 @@ def prepare_watchlist(service, user, action=None, ticker=None): return ['Watchlist is empty. Try ".watchlist add SYMBOL" to create it'] return [f"Hi {user}, I'm currently tracking:"] + lines -def watchlist_change(service, user, action, ticker): - """The reply to adding or removing `ticker` (Yahoo's symbol).""" +def watchlist_change(service, chat, user, action, ticker): + """The reply to adding `ticker` (Yahoo's symbol) to a chat's watchlist, or removing it.""" quote = yahoo.fetch([ticker]).get(ticker) or {} link = links.finance_link(ticker, quote.get('profile_exchange') or ticker, service) described = service.bold(f"{quote['profile_title']} ({link})" if quote.get('profile_title') else link) if action == 'delete': - if not watchlist.remove(ticker): + if not watchlist.remove(chat, ticker): return f"Beep Boop. I could not find {service.bold(ticker)} to remove it" return f"Ok {user}, I deleted {described}" if not quote: return f"Beep Boop. I could not find {service.bold(ticker)} to add it" - return f"Ok {user}, I added {described}" if watchlist.add(ticker) else f"{user}, I'm already tracking {described}" + if not watchlist.add(chat, ticker): + return f"{user}, I'm already tracking {described}" + return f"Ok {user}, I added {described}" diff --git a/lib/reports/milestone.py b/lib/reports/milestone.py index a3255a38..7ad8dc9d 100644 --- a/lib/reports/milestone.py +++ b/lib/reports/milestone.py @@ -13,13 +13,13 @@ STATE_FILE = 'finbot_milestone.json' # each chat's {ticker: {'all_time_high': ..., 'all_time_low': ...}} def run(dest=None): - """Post the stocks followed that set an all-time high or low, against each chat's record, to each of the run's chats + """Post the stocks a chat follows that set an all-time high or low, against its record, to each of the run's chats (dest, a chat's job or the chat that asked; else the home chats); a scheduled run records each chat's new records.""" interactive = delivery.interactive(dest) - tickers = watchlist.with_holdings() - market_data = yahoo.fetch(tickers) for chat in delivery.targets(dest): + tickers = watchlist.with_holdings(chat.key) + market_data = yahoo.fetch(tickers) highs, lows, changed = compare(tickers, market_data, state.load_for(chat.key, STATE_FILE, {})) delivery.deliver(chat, delivery.Report(report(chat.dialect, market_data, highs, lows))) if changed and not interactive: diff --git a/lib/reports/price.py b/lib/reports/price.py index 10a1fb46..f250deed 100644 --- a/lib/reports/price.py +++ b/lib/reports/price.py @@ -15,16 +15,15 @@ from lib import buttons from lib.reports import history -def gather(specific_stock=None, days=None): - """(tickers, market data) for a price report on the stocks followed or one (Yahoo's symbol): Yahoo's quotes, and - over `days` each one's change as percent_change_period, from its Yahoo history (the change its chart shows), the - histories fetched together up front.""" - tickers = watchlist.covered(specific_stock) +def gather(tickers, specific_stock=None, days=None): + """The market data of a price report on tickers (the stocks a chat follows, or the one asked about, Yahoo's + symbol): Yahoo's quotes, and over `days` each one's change as percent_change_period, from its Yahoo history (the + change its chart shows), the histories fetched together up front.""" market_data = yahoo.fetch(tickers) if config.prefs.demote_volatile: # trigger_bar compares market caps, in US dollars yahoo.add_usd_market_caps(market_data) if not days: - return tickers, market_data + return market_data if not specific_stock: yahoo.prefetch_history(list(market_data)) for ticker in list(market_data): @@ -36,7 +35,7 @@ def gather(specific_stock=None, days=None): print(ticker, e, file=sys.stderr) continue market_data[ticker]['percent_change_period'] = changes[days] - return tickers, market_data + return market_data def needs_beta(quote): """Whether trigger_bar must know this ticker's beta: a small cap between US$150M and US$1B, which counts as volatile @@ -98,13 +97,13 @@ def change(mode, quote, now, interactive): return quote.get('percent_change_premarket', quote.get('percent_change_postmarket')) return quote.get('percent_change_period') -def moves(mode='interday', threshold=None, days=None, specific_stock=None, interactive=False): - """The Moves of a price report in a mode of MODES ('period': over `days`): each stock followed whose move is - `threshold` percent or more, or the one stock asked about, whatever its move. A scheduled report keeps them only - when one passes its trigger_bar.""" +def moves(tickers, mode='interday', threshold=None, days=None, specific_stock=None, interactive=False): + """The Moves of a price report on tickers (watchlist.covered: the stocks a chat follows, or the one asked about) in + a mode of MODES ('period': over `days`): each whose move is `threshold` percent or more, or the one stock asked + about, whatever its move. A scheduled report keeps them only when one passes its trigger_bar.""" if threshold is None: threshold = config.prefs.price_percent - _, market_data = gather(specific_stock, days) + market_data = gather(tickers, specific_stock, days) now = datetime.datetime.now() rows, exchanges = [], set() for ticker, quote in market_data.items(): @@ -172,12 +171,11 @@ def nothing(mode, threshold, user, ticker, in_session, days=None): return f"{user}, no price movements meet threshold {threshold:g}%" def run(dest=None, *, mode='interday', threshold=None, days=None, top=None, user='', specific_stock=None): - """Post the price report (report() has the modes): to the chat that asked or whose job it is (dest), else to the - home chats, with its chart and period buttons where they can go.""" + """Post the price report (report() has the modes) on the stocks each chat follows: to the chat that asked or whose + job it is (dest), else to the home chats, with its chart and period buttons where they can go.""" interactive = delivery.interactive(dest) if threshold is None: threshold = config.prefs.price_percent - found = moves(mode, threshold, days, specific_stock, interactive) def markup(chat): """A chart's period buttons, where the chat takes buttons: the daily and N-day lists, but not the intraday ones.""" @@ -188,6 +186,7 @@ def markup(chat): buttons.period_for_days(days, 'd' if not days else None)) return None for chat in delivery.targets(dest): + found = moves(watchlist.covered(chat.key, specific_stock), mode, threshold, days, specific_stock, interactive) payload, graph = report(chat.dialect, found, mode, threshold, days, top, user, specific_stock, interactive) delivery.deliver(chat, delivery.Report(payload, graph, markup(chat) if graph else None)) diff --git a/lib/reports/rankings.py b/lib/reports/rankings.py index 908ca8c9..cca7d5c7 100644 --- a/lib/reports/rankings.py +++ b/lib/reports/rankings.py @@ -21,11 +21,11 @@ def marketcap_line(service, ticker): cap += f" (USD {formats.human_units(data['market_cap_usd'])})" return f"{links.stock_line(ticker, data, service)} mkt cap: {cap}" -def marketcap_list(service, bottom=False, length=15): - """The `length` stocks followed of largest (or, `bottom`, smallest) market cap, compared in US dollars at Yahoo's - exchange rates: a market cap in won or yen is a far bigger number.""" +def marketcap_list(service, chat, bottom=False, length=15): + """The `length` stocks a chat follows (by its key) of largest (or, `bottom`, smallest) market cap, compared in US + dollars at Yahoo's exchange rates: a market cap in won or yen is a far bigger number.""" caps = [] # (market cap in US dollars, line) - for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(watchlist.with_holdings())).items(): + for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(watchlist.with_holdings(chat))).items(): if 'market_cap_usd' not in data: why = "no market cap" if 'market_cap' not in data else f"no exchange rate for {data.get('currency')}" print(ticker, why, file=sys.stderr) @@ -38,13 +38,13 @@ def marketcap_list(service, bottom=False, length=15): heading = f"{'Bottom' if bottom else 'Top'} {length} tracked stocks by market cap (USD)" return [service.bold(heading)] + [line for _, line in caps[:length]] -def prepare_rating_payload(service, action, length=15): - """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by min_analysts or more analysts - (fewer, and the consensus says little: ASX small caps covered by two to six brokers are nearly all a perfect 1.0 - buy), the strongest rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give - first.""" +def prepare_rating_payload(service, chat, action, length=15): + """(payload, chart) of the `length` stocks a chat follows (by its key) rated `action` ('buy' or 'sell') by + min_analysts or more analysts (fewer, and the consensus says little: ASX small caps covered by two to six brokers + are nearly all a perfect 1.0 buy), the strongest rating first (1 is a strong buy, 5 a strong sell); on equal + ratings, the one more analysts give first.""" found = [] # (rating, analysts, line, ticker) - for ticker, data in yahoo.fetch_details(watchlist.with_holdings()).items(): + for ticker, data in yahoo.fetch_details(watchlist.with_holdings(chat)).items(): if 'recommend' not in data or data['recommend_analysts'] < config.prefs.min_analysts: continue recommend = data['recommend'] @@ -85,12 +85,12 @@ def ratio_line(service, ticker, kind): raise answer.Answer(f"{ticker}: no {kind} ratio found") return f"{links.stock_line(ticker, data, service)} {figures}" -def ratio_list(service, kind, mode='top', length=15): - """(payload, chart): the `length` stocks followed by a ratio (kind: pe, forward pe or peg), lowest first (mode top), - highest first (bottom), or the negative ones only, lowest first (negative). A forward P/E is analysts' estimate, so - its list takes only stocks that min_analysts or more analysts cover, as .buy does.""" +def ratio_list(service, chat, kind, mode='top', length=15): + """(payload, chart): the `length` stocks a chat follows (by its key) by a ratio (kind: pe, forward pe or peg), + lowest first (mode top), highest first (bottom), or the negative ones only, lowest first (negative). A forward P/E + is analysts' estimate, so its list takes only stocks that min_analysts or more analysts cover, as .buy does.""" field = RATIOS[kind] - market_data = yahoo.fetch(watchlist.with_holdings()) + market_data = yahoo.fetch(watchlist.with_holdings(chat)) if kind == 'peg': # the PEG is in Yahoo's details market_data |= yahoo.fetch_details(market_data) covered = None # for a forward P/E list: the tickers enough analysts cover @@ -118,10 +118,11 @@ def ratio_list(service, kind, mode='top', length=15): chart = render.Chart('bars', heading, rows=bars, value_format='number') return [service.bold(heading)] + [line for _, _, line in rows], chart -def prepare_beta_payload(service): - """(payload, chart) of the stocks followed with a beta over 1.5 and a market cap under US$2B, highest beta first.""" +def prepare_beta_payload(service, chat): + """(payload, chart) of the stocks a chat follows (by its key) with a beta over 1.5 and a market cap under US$2B, + highest beta first.""" found = [] # (beta, line, chart row) - for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch_details(watchlist.with_holdings())).items(): + for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch_details(watchlist.with_holdings(chat))).items(): if 'beta' not in data: continue beta = round(data['beta'], 1) diff --git a/lib/reports/rating.py b/lib/reports/rating.py index a88a7ff5..fb538cae 100644 --- a/lib/reports/rating.py +++ b/lib/reports/rating.py @@ -8,12 +8,12 @@ STATE_FILE = 'finbot_rating.json' # each chat's {ticker: [action, index]} as its last scheduled run saw them def run(dest=None): - """Post the analysts' consensus ratings that changed since a chat's last scheduled run: to the chat that asked - (dest), against its record; or, a scheduled run, to each of its chats (dest, a chat's job; else the home chats), - recording for each the ratings it saw.""" + """Post the analysts' consensus ratings of the stocks a chat follows that changed since its last scheduled run: to + the chat that asked (dest), against its record; or, a scheduled run, to each of its chats (dest, a chat's job; else + the home chats), recording for each the ratings it saw.""" interactive = delivery.interactive(dest) - market_data, seen = gather(watchlist.with_holdings()) for chat in delivery.targets(dest): + market_data, seen = gather(watchlist.with_holdings(chat.key)) changes = changed(market_data, seen, state.load_for(chat.key, STATE_FILE, {})) delivery.deliver(chat, delivery.Report(report(chat.dialect, changes, interactive))) if seen and not interactive: # added to its record: a ticker missing from this run keeps its last rating diff --git a/lib/reports/shorts.py b/lib/reports/shorts.py index 25e23b57..a18011db 100644 --- a/lib/reports/shorts.py +++ b/lib/reports/shorts.py @@ -7,24 +7,24 @@ from lib import shortman def run(dest=None, *, threshold=None, specific_stock=None): - """Post the stocks followed whose short interest is over `threshold` percent (the shorts_percent preference when - None), or one stock's (Yahoo's symbol): to the chat that asked or whose job it is (dest), else to the home + """Post the stocks a chat follows whose short interest is over `threshold` percent (the shorts_percent preference + when None), or one stock's (Yahoo's symbol): to the chat that asked or whose job it is (dest), else to the home chats.""" interactive = delivery.interactive(dest) if threshold is None: threshold = config.prefs.shorts_percent - tickers, market_data = gather(specific_stock) for chat in delivery.targets(dest): + tickers = watchlist.covered(chat.key, specific_stock) + market_data = gather(tickers, specific_stock) payload, graph = report(chat.dialect, tickers, market_data, threshold, specific_stock, interactive) delivery.deliver(chat, delivery.Report(payload, graph)) -def gather(specific_stock): - """(the stocks followed, or the one, their quotes with short interest): Short Man has the ASX's figures, Yahoo's - details the US's.""" - tickers = watchlist.covered(specific_stock) +def gather(tickers, specific_stock): + """The quotes of tickers (the stocks a chat follows, or the one asked about), with their short interest: Short Man + has the ASX's figures, Yahoo's details the US's.""" market_data = yahoo.fetch(tickers) market_data |= yahoo.fetch_details((ticker for ticker in tickers if '.' not in ticker), strict=bool(specific_stock)) - return tickers, shortman.fetch(market_data) + return shortman.fetch(market_data) def report(service, tickers, market_data, threshold, specific_stock, interactive): """(the report in one network's markup, its chart or None), the most shorted first.""" diff --git a/lib/reports/yahoo_warm.py b/lib/reports/yahoo_warm.py index 80682e60..f3f8cde9 100644 --- a/lib/reports/yahoo_warm.py +++ b/lib/reports/yahoo_warm.py @@ -4,17 +4,17 @@ from lib import config, watchlist, yahoo def run(): - """Fetch afresh the price history of every stock followed (the watchlist, and the holdings where Sharesight is - configured), so each is cached for cache_seconds (12 hours by default) from now: the default schedule runs this - every 8 hours, before any runs out, and a flush at once. With the cache preference off nothing would be kept, so - nothing is fetched. Returns how many it fetched.""" + """Fetch afresh the price history of every stock followed (every chat's watchlist, and the holdings where + Sharesight is configured), so each is cached for cache_seconds (12 hours by default) from now: the system job runs + this every 8 hours, before any runs out, and a flush at once. With the cache preference off nothing would be kept, + so nothing is fetched. Returns how many it fetched.""" if not config.prefs.cache: return 0 - tickers = watchlist.tickers() + tickers = watchlist.all_watched() if config.env.sharesight_auth.get('client_id'): try: - tickers = watchlist.with_holdings() - except Exception as e: # Sharesight unreachable, say: the watchlist's at least + tickers = watchlist.all_followed() + except Exception as e: # Sharesight unreachable, say: the watchlists' at least print("yahoo_warm: no holdings from Sharesight:", e, file=sys.stderr) failed = yahoo.prefetch_history(tickers, seconds=0) print(f"yahoo_warm: {len(tickers) - len(failed)} of {len(tickers)} price histories fetched", file=sys.stderr) diff --git a/lib/watchlist.py b/lib/watchlist.py index 27a47af0..3aee9397 100644 --- a/lib/watchlist.py +++ b/lib/watchlist.py @@ -1,5 +1,6 @@ -"""The watchlist: tickers followed beside the Sharesight holdings, in DATA_DIR/finbot_watchlist.json. Every call reads -the file, so the bot and its scheduled reports (other processes) always see the same list.""" +"""The watchlist: each chat's tickers followed beside the Sharesight holdings, in DATA_DIR/finbot_watchlist.json, by +the chat's key ({'chats': {chat key: [tickers]}}, as state.load_for keeps it). Every call reads the file, so every +thread sees the same lists. The holdings are the same for every chat: a portfolio is not a chat's.""" from lib import config from lib import markets from lib import sharesight @@ -8,48 +9,63 @@ FILE = 'finbot_watchlist.json' -def tickers(): - """The watched tickers, in the order they were added.""" - return state.load(FILE, []) +def tickers(chat): + """A chat's watched tickers (by its key), in the order they were added.""" + return state.load_for(chat, FILE, []) -def watched(typed): - """The watched ticker that a ticker as written names: BHP.AX as written, or BHP on the local exchange (the - exchange_suffix preference) when only BHP.AX is watched; else Yahoo's spelling of what was written. Yahoo is not - asked, so a ticker it no longer knows can still be removed.""" - ticker = markets.transform_to_yahoo(typed.upper()) +def watched(chat, typed): + """The ticker of a chat's watchlist that a ticker as written names: BHP.AX as written, or BHP on the local exchange + (the exchange_suffix preference) when only BHP.AX is watched; else Yahoo's spelling of what was written. Yahoo is + not asked, so a ticker it no longer knows can still be removed.""" + ticker, watching = markets.transform_to_yahoo(typed.upper()), tickers(chat) local = f'{ticker}.{config.prefs.exchange_suffix}' - if ticker not in tickers() and config.prefs.exchange_suffix and '.' not in ticker and local in tickers(): + if ticker not in watching and config.prefs.exchange_suffix and '.' not in ticker and local in watching: return local return ticker -def add(ticker): - """Watch a ticker (check it with find first). False when it was already watched.""" - with state.edit(FILE, []) as watched: - if ticker in watched: +def add(chat, ticker): + """Add a ticker to a chat's watchlist (check it with find first). False when it was there already.""" + with state.edit_for(chat, FILE, []) as watching: + if ticker in watching: return False - watched.append(ticker) + watching.append(ticker) return True -def remove(ticker): - """Stop watching a ticker. False when it was not watched.""" - with state.edit(FILE, []) as watched: - if ticker not in watched: +def remove(chat, ticker): + """Take a ticker off a chat's watchlist. False when it was not on it.""" + with state.edit_for(chat, FILE, []) as watching: + if ticker not in watching: return False - watched.remove(ticker) + watching.remove(ticker) return True -def covered(ticker=None): - """The tickers a report covers: the one asked about, else every stock followed (with_holdings).""" - return [ticker] if ticker else with_holdings() +def covered(chat, ticker=None): + """The tickers a report covers in a chat: the one asked about, else every stock the chat follows (with_holdings).""" + return [ticker] if ticker else with_holdings(chat) -def with_holdings(): - """The watchlist and every Sharesight holding: the tickers the scheduled reports cover, sorted.""" - followed = set(sharesight.tickers()) | set(tickers()) +def with_holdings(chat): + """A chat's watchlist and every Sharesight holding: the tickers its reports cover, sorted.""" + return _followed(tickers(chat)) + + +def all_watched(): + """Every chat's watched tickers, each once, sorted.""" + return sorted({ticker for watching in state.load(FILE, {}).get('chats', {}).values() for ticker in watching}) + + +def all_followed(): + """Every chat's watchlist and every Sharesight holding, sorted: what yahoo_warm keeps warm.""" + return _followed(all_watched()) + + +def _followed(watching): + """The holdings and the tickers watched, sorted; GOOGL left out beside GOOG, its other class of shares.""" + followed = set(sharesight.tickers()) | set(watching) if 'GOOG' in followed and 'GOOGL' in followed: followed.remove("GOOGL") return sorted(followed) diff --git a/lib/worker.py b/lib/worker.py index 2c096b96..81e35e53 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -68,8 +68,8 @@ def redraw(): delivery.report_error(e, chat, context='jobs') def process_callback(press): - """A button pressed (an inbound.Press, which bot.take acknowledged as it arrived): run a scheduled command from - .jobs, or rebuild a chart message for the chosen period and edit it in place. Data of neither is ignored.""" + """A button pressed (an inbound.Press, which bot.take acknowledged as it arrived): a job's (run it, or switch it), + or a chart's, whose message is rebuilt for the chosen period and edited in place. Data of neither is ignored.""" pressed = buttons.parse(press.data) if isinstance(pressed, buttons.JobPress): return process_job_callback(press, pressed) @@ -82,7 +82,7 @@ def process_callback(press): markup = buttons.keyboard(pressed.kind, args, pressed.period) if chat.reshow(press.message_id, markup): # that period as posted lately, where the network keeps it (Slack) return - caption, chart = charts.build(pressed.kind, args, service, period=pressed.period) + caption, chart = charts.build(pressed.kind, args, service, chat.key, period=pressed.period) if chart is None: # too few prices in that period to draw a line raise answer.Answer(f"Too few prices to draw {buttons.PERIODS[pressed.period][0]}") chat.edit_chart(press.message_id, render.draw(chart), caption, markup) @@ -215,14 +215,15 @@ def portfolios(): return 'Portfolios: ' + ', '.join(portfolio.name for portfolio in sharesight.portfolios()) def on_watchlist(request, args): - """.watchlist [add|del SYMBOL]: the watchlist, or a change to it.""" + """.watchlist [add|del SYMBOL]: this chat's watchlist, or a change to it.""" + chat = request.dest.key if args.keyword and len(args.words) == 1: # what is removed is what is watched: Yahoo may know it no longer - symbol = ticker(args) if args.keyword == 'add' else watchlist.watched(args.words[0]) - request.reply(holdings.prepare_watchlist(request.service, request.user, args.keyword, symbol)) + symbol = ticker(args) if args.keyword == 'add' else watchlist.watched(chat, args.words[0]) + request.reply(holdings.prepare_watchlist(request.service, chat, request.user, args.keyword, symbol)) elif args.keyword or args.words: raise Usage else: - request.reply(holdings.prepare_watchlist(request.service, request.user)) + request.reply(holdings.prepare_watchlist(request.service, chat, request.user)) def on_help(request, args): """.help: every command (on Discord as its slash command, which works in both modes); in a Telegram DM the command @@ -283,7 +284,8 @@ def on_marketcap(request, args): if args.words: request.reply([rankings.marketcap_line(request.service, ticker(args))]) else: - request.reply(rankings.marketcap_list(request.service, bottom=args.keyword == 'bottom', length=15)) + request.reply(rankings.marketcap_list(request.service, request.dest.key, bottom=args.keyword == 'bottom', + length=15)) def on_ratio(kind): """The handler of .pe, .forward pe or .peg (kind: pe, forward pe or peg): the top (by default) or bottom 15 by the @@ -293,12 +295,13 @@ def handler(request, args): if args.words: request.reply([rankings.ratio_line(request.service, ticker(args), kind)]) else: - request.reply(*rankings.ratio_list(request.service, kind, args.keyword or 'top', length=15)) + mode = args.keyword or 'top' + request.reply(*rankings.ratio_list(request.service, request.dest.key, kind, mode, length=15)) return handler def on_beta(request, args): """.beta: the volatile small caps among the stocks followed.""" - payload, chart = rankings.prepare_beta_payload(request.service) + payload, chart = rankings.prepare_beta_payload(request.service, request.dest.key) request.reply(payload, chart) def on_pie(request, args): @@ -311,7 +314,7 @@ def on_pie(request, args): def on_rating(request, args): """.buy, .sell or both (.buy & sell, a keyboard button): the analysts' strongest buy or sell ratings, as asked.""" for action in dict.fromkeys(re.findall('buy|sell', f'{args.word} {args.text}'.lower())): # each once, as asked - payload, chart = rankings.prepare_rating_payload(request.service, action, length=15) + payload, chart = rankings.prepare_rating_payload(request.service, request.dest.key, action, length=15) request.reply(payload, chart) def on_compare(request, args): @@ -332,18 +335,18 @@ def on_history(request, args): def chart_reply(request, kind, args, days=None): """The first view of a chart with period buttons, built as a button press builds it (charts.build): its caption, chart and buttons where they go, the period shown marked (Max when days is None).""" - caption, chart = charts.build(kind, args, request.service, days=days) + caption, chart = charts.build(kind, args, request.service, request.dest.key, days=days) markup = None if chart and request.dest.takes_buttons: markup = buttons.keyboard(kind, args, buttons.period_for_days(days) if days else 'x') request.reply(caption.split('\n'), chart, markup) def on_plan(request, args): - """.plan [text]: record the sender's plan, and list everyone's.""" + """.plan [text]: record the sender's plan in this chat, and list everyone's here.""" if args.text and not args.text.startswith('@'): - with state.edit('finbot_plan.json', {}) as plans: + with state.edit_for(request.dest.key, PLANS, {}) as plans: plans[request.user] = args.text - plan = state.load('finbot_plan.json', {}) + plan = state.load_for(request.dest.key, PLANS, {}) if not plan: raise answer.Answer("No plans yet: .plan TEXT records yours") def who(user): @@ -428,6 +431,7 @@ class Command: TOP = {'top': 'top', 'bottom': 'bottom'} # the ends of a ranked list NEGATIVE = TOP | dict.fromkeys(('neg', 'negative', 'negatives'), 'negative') # the ratios that can be negative CHANGE = {'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), 'delete') # of the watchlist +PLANS = 'finbot_plan.json' # each chat's plans, {user: plan}, as state.load_for keeps them BY = dict.fromkeys(('sector', 'sectors'), 'sector') | dict.fromkeys(('industry', 'industries'), 'industry') # .pie's JOB_CHANGES = {'run': 'run', 'toggle': 'toggle', 'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), 'del') # what .jobs does to a chat's jobs diff --git a/tests/common.py b/tests/common.py index 6d0b1ec0..12168efa 100644 --- a/tests/common.py +++ b/tests/common.py @@ -137,7 +137,7 @@ def setUp(self): self.patch(transports.Transport, 'chart', lambda dest, image, caption, markup=None: self.record_chart(dest, image, caption, markup)) self.patch(transports.Transport, 'text', lambda dest, lines, markup=None: self.record_text(dest, lines)) - self.patch(watchlist, 'with_holdings', lambda: TICKERS) + self.patch(watchlist, 'with_holdings', lambda chat: TICKERS) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts}) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t)}) moved = {t: float(i * 4 - 40) for i, t in enumerate(TICKERS)} # each stock's move over any period diff --git a/tests/test_edges.py b/tests/test_edges.py index 469eb4a1..4d91e712 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -72,7 +72,7 @@ def down(*a, **k): def test_no_holdings_and_empty_watchlist(self): """With nothing held and nothing watched, the watchlist, price and earnings commands still answer.""" - self.patch(watchlist, 'with_holdings', lambda: []) + self.patch(watchlist, 'with_holdings', lambda chat: []) for cmd in ('.watchlist', '.price', '.earnings'): with self.subTest(cmd=cmd): self.command(cmd) @@ -83,12 +83,12 @@ def test_add_duplicate_delete_and_unknown_action(self): """Add, add again (said differently), remove, and an unknown action is refused by name.""" add = self.command('.watchlist add MSFT') self.assertTrue(add) - self.assertEqual(state.load('finbot_watchlist.json'), ['MSFT']) + self.assertEqual(state.load('finbot_watchlist.json'), {'chats': {'telegram:55': ['MSFT']}}) again = self.command('.watchlist add MSFT') self.assertNotEqual(add, again) # a duplicate is reported differently from a first add - self.assertEqual(state.load('finbot_watchlist.json'), ['MSFT']) + self.assertEqual(watchlist.tickers('telegram:55'), ['MSFT']) self.command('.watchlist rm MSFT') - self.assertEqual(state.load('finbot_watchlist.json'), []) + self.assertEqual(watchlist.tickers('telegram:55'), []) self.assertEqual(self.command('.watchlist frobnicate MSFT'), [('text', ['Usage: .watchlist [add|del SYMBOL]'])]) def test_a_bare_symbol_yahoo_does_not_know_is_tried_on_the_asx(self): @@ -99,9 +99,9 @@ def test_a_bare_symbol_yahoo_does_not_know_is_tried_on_the_asx(self): self.assertIn('already tracking', str(self.command('.watchlist add BHP'))) # a ticker with a market is not retried self.assertIn('could not find', str(self.command('.watchlist add BHP.L'))) - self.assertEqual(watchlist.tickers(), ['BHP.AX']) + self.assertEqual(watchlist.tickers('telegram:55'), ['BHP.AX']) self.assertIn('I deleted', str(self.command('.watchlist del bhp'))) - self.assertEqual(watchlist.tickers(), []) + self.assertEqual(watchlist.tickers('telegram:55'), []) def test_two_chats_adding_at_once_both_count(self): """Two chats adding at once both count: Yahoo is asked before the file is opened, where both adds used to read @@ -111,21 +111,22 @@ def slow_fetch(tickers): time.sleep(0.01) return {t: market_entry(t) for t in tickers} self.patch(yahoo, 'fetch', slow_fetch) - threads = [threading.Thread(target=holdings.prepare_watchlist, args=(TELEGRAM, '@u', 'add', t)) for t in ('T1', - 'T2')] + threads = [threading.Thread(target=holdings.prepare_watchlist, args=(TELEGRAM, chat, '@u', 'add', ticker)) + for chat, ticker in (('telegram:55', 'T1'), ('telegram:55', 'T2'), ('slack:C1', 'T3'))] for thread in threads: thread.start() for thread in threads: thread.join() - self.assertEqual(sorted(watchlist.tickers()), ['T1', 'T2']) + self.assertEqual(sorted(watchlist.tickers('telegram:55')), ['T1', 'T2']) + self.assertEqual(watchlist.tickers('slack:C1'), ['T3']) def test_removing_a_ticker_yahoo_no_longer_knows(self): """A ticker Yahoo no longer knows is still removed, and said to be; removing it again says it is not there.""" - state.save('finbot_watchlist.json', ['GONE', 'T1']) + state.save('finbot_watchlist.json', {'chats': {'telegram:55': ['GONE', 'T1']}}) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts if t != 'GONE'}) self.assertIn('I deleted', str(self.command('.watchlist rm GONE'))) self.assertIn('could not find', str(self.command('.watchlist rm GONE'))) - self.assertEqual(watchlist.tickers(), ['T1']) + self.assertEqual(watchlist.tickers('telegram:55'), ['T1']) class BadArguments(FinbotCase): diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 350e7e85..133cd55c 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -346,11 +346,10 @@ def test_a_plan_is_shown_as_text(self): self.assertEqual(self.command('.plan')[0][1], reply) # and the same way again self.sent.clear() # inbound has unescaped Slack's text - worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.plan sell ', BOT, 'U')) + worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.plan sell & run', BOT, 'U')) text = '\n'.join(self.sent[0][1]) self.assertIn('*<@U1>*', text) # a Slack mention stays markup - self.assertIn('_sell <dear>_', text) # escaped once, not twice - self.assertIn('_buy <cheap> & hold_', text) + self.assertIn('_sell <dear> & run_', text) # escaped once, not twice def test_a_telegram_name_is_escaped_where_it_arrives(self): """A Telegram first name is escaped as the update arrives, in the handle and the name.""" diff --git a/tests/test_reports.py b/tests/test_reports.py index 16d5f7bc..0e1d6136 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -114,12 +114,12 @@ def test_a_portfolio_command_without_a_portfolio_lists_them(self): def test_prepare_marketcap(self): """The market cap list is built.""" - self.assertTrue(rankings.marketcap_list(TELEGRAM, length=5)) + self.assertTrue(rankings.marketcap_list(TELEGRAM, 'telegram:55', length=5)) def test_prepare_rating(self): """The buy and sell lists are built.""" for action in ('buy', 'sell'): - self.assertTrue(rankings.prepare_rating_payload(TELEGRAM, action, length=5)[0]) + self.assertTrue(rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', action, length=5)[0]) def test_ratings_list_the_strongest_first(self): """1 is a strong buy and 5 a strong sell: .sell used to list the mildest sells first. On equal ratings, more @@ -129,19 +129,19 @@ def test_ratings_list_the_strongest_first(self): 'T7': ('strong_buy', 1.0, 6), 'T4': ('sell', 4.2, 7), 'T5': ('underperform', 4.6, 10), 'T6': ('sell', 4.2, 8), 'T8': ('strong_buy', 1.0, 2)} - self.patch(watchlist, 'with_holdings', lambda: list(ratings)) + self.patch(watchlist, 'with_holdings', lambda chat: list(ratings)) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | dict(zip(('recommend', 'recommend_index', 'recommend_analysts'), ratings[t]))}) def order(action): """(the tickers listed, the tickers charted) of a ratings report, in order.""" - payload, chart = rankings.prepare_rating_payload(TELEGRAM, action) + payload, chart = rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', action) return [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], [label for label, _ in chart.rows] self.assertEqual(order('buy'), (['T2', 'T3', 'T1'], ['T2', 'T3', 'T1'])) self.assertEqual(order('sell'), (['T5', 'T6', 'T4'], ['T5', 'T6', 'T4'])) - self.assertIn('(7+ analysts)', rankings.prepare_rating_payload(TELEGRAM, 'buy')[0][0]) - self.patch(watchlist, 'with_holdings', lambda: ['T7', 'T8']) - self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'buy'), + self.assertIn('(7+ analysts)', rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', 'buy')[0][0]) + self.patch(watchlist, 'with_holdings', lambda chat: ['T7', 'T8']) + self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', 'buy'), (['No tracked stocks rated buy by 7+ analysts'], None)) def test_a_small_cap_quoted_in_won_needs_the_bigger_move_too(self): @@ -151,7 +151,7 @@ def test_a_small_cap_quoted_in_won_needs_the_bigger_move_too(self): 'market_cap': 1e11}) for t in ts}) self.prefs(demote_volatile=True) - quote = price.gather()[1]['T1'] + quote = price.gather(TICKERS)['T1'] self.assertAlmostEqual(quote['market_cap_usd'], 1e11 / 1380.0) # US$72M self.assertEqual(price.trigger_bar('T1', quote, 5, {}), 5 * config.prefs.volatility_multiplier) @@ -160,8 +160,7 @@ def test_a_short_top_list_repeats_none_of_its_best_as_the_worst(self): with none left there is no Bottom list and no gap in the chart; the subtitle says "over the past 3 months".""" for count, bottom, bars in ((12, 2, 13), (8, 0, 8)): with self.subTest(count=count): - self.patch(watchlist, 'with_holdings', lambda: TICKERS[:count]) - found = price.moves('period', 0, 90, interactive=True) + found = price.moves(TICKERS[:count], 'period', 0, 90, interactive=True) payload, chart = price.report(TELEGRAM, found, 'period', 0, 90, top=10, interactive=True) shown = payload[payload.index('Bottom 10') + 1:] if bottom else [] self.assertEqual((len(shown), len(chart.rows)), (bottom, bars)) @@ -184,13 +183,13 @@ def test_the_small_caps_betas_are_fetched_together(self): def test_forward_pe_lists_need_as_many_analysts_as_buy(self): """A forward P/E is analysts' estimate: from few, it says little, as a perfect 1.0 buy from two brokers does.""" analysts = {'T1': 7, 'T2': 6, 'T3': 12, 'T4': 2} - self.patch(watchlist, 'with_holdings', lambda: list(analysts)) + self.patch(watchlist, 'with_holdings', lambda chat: list(analysts)) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) | {'price_to_earnings_forward': -5 if t == 'T3' else 20} for t in ts}) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'recommend_analysts': analysts[t]}}) def listed(mode='top'): """(heading, the tickers listed, the chart) of a forward P/E list.""" - payload, chart = rankings.ratio_list(TELEGRAM, 'forward pe', mode) + payload, chart = rankings.ratio_list(TELEGRAM, 'telegram:55', 'forward pe', mode) return payload[0], [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], chart heading, names, chart = listed() # T2 and T4: too few analysts; T3: negative @@ -200,7 +199,7 @@ def listed(mode='top'): self.assertIn('Co T2', rankings.ratio_line(TELEGRAM, 'T2', 'forward pe')) # one ticker asked for: shown self.prefs(min_analysts=13) # the one setting moves .buy and .forward pe together self.assertEqual(listed()[0], 'No tracked stocks with 13+ analysts have a forward pe ratio') - self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'buy')[0], + self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', 'buy')[0], ['No tracked stocks rated buy by 13+ analysts']) def test_market_caps_compare_in_us_dollars(self): @@ -215,15 +214,15 @@ def fetch(tickers): for t in tickers if t in fx or t in stocks} self.patch(yahoo, 'fetch', fetch) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: fetch([t])) - self.patch(watchlist, 'with_holdings', lambda: list(stocks)) - payload = rankings.marketcap_list(TELEGRAM) + self.patch(watchlist, 'with_holdings', lambda chat: list(stocks)) + payload = rankings.marketcap_list(TELEGRAM, 'telegram:55') self.assertIn('by market cap (USD)', payload[0]) # Z1: no rate, left out self.assertEqual([line.split('Co ')[1].split(' ')[0] for line in payload[1:]], ['U1', 'K1.KS', 'L1.L', 'K2.KS']) self.assertIn('mkt cap: 731.88 B', payload[2]) # 1.01 Q won self.assertIn('mkt cap: KRW 1.01 Q (USD 731.88 B)', rankings.marketcap_line(TELEGRAM, 'K1.KS')) self.assertIn('mkt cap: GBP 62.50 B (USD 79.11 B)', rankings.marketcap_line(TELEGRAM, 'L1.L')) # pence: pounds - beta = rankings.prepare_beta_payload(TELEGRAM)[0] + beta = rankings.prepare_beta_payload(TELEGRAM, 'telegram:55')[0] self.assertIn('Co K2.KS', str(beta)) # 500 billion won is US$362M: a small cap, which it never was unconverted self.assertNotIn('Co K1.KS', str(beta)) @@ -234,7 +233,7 @@ def test_prepare_value(self): ('peg', 'top', 'Top 5 tracked stocks by peg ratio'), ('peg', 'negative', 'negative peg: no data found')): with self.subTest(kind=kind, mode=mode): - payload, chart = rankings.ratio_list(TELEGRAM, kind, mode, length=5) + payload, chart = rankings.ratio_list(TELEGRAM, 'telegram:55', kind, mode, length=5) self.assertIn(heading, payload[0]) def test_one_stocks_ratio_shows_what_yahoo_has(self): @@ -254,11 +253,11 @@ def test_pe_sends_its_chart_as_forward_pe_and_peg_do(self): def test_prepare_beta(self): """The beta report charts a bar per stock listed, and says so when none qualify.""" - payload, chart = rankings.prepare_beta_payload(TELEGRAM) + payload, chart = rankings.prepare_beta_payload(TELEGRAM, 'telegram:55') self.assertIn('Beta over 1.5', payload[0]) self.assertEqual(len(chart.rows), len(payload) - 1) # a bar per stock listed self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'beta': 1.0}}) - self.assertEqual(rankings.prepare_beta_payload(TELEGRAM), + self.assertEqual(rankings.prepare_beta_payload(TELEGRAM, 'telegram:55'), (['No tracked stocks have a beta over 1.5 and mkt cap under US$2B'], None)) def test_prepare_profile(self): @@ -299,7 +298,38 @@ def test_a_profile_warns_of_a_ps_ratio_only_above_15(self): def test_prepare_watchlist(self): """The watchlist is listed.""" - self.assertTrue(holdings.prepare_watchlist(TELEGRAM, '@u')) + self.assertTrue(holdings.prepare_watchlist(TELEGRAM, 'telegram:55', '@u')) + + def test_each_chat_follows_its_own_watchlist_besides_the_holdings(self): + """Each chat has a watchlist of its own: .watchlist lists it, and a report, asked for or scheduled, covers that + chat's watchlist and the holdings, which every chat shares, and no other chat's watchlist.""" + self.patch(watchlist, 'with_holdings', REAL_GET_HOLDINGS) + self.patch(sharesight, 'tickers', lambda: ['T1']) + fetched = [] + self.patch(yahoo, 'fetch', lambda ts: fetched.append(sorted(ts)) or {t: market_entry(t) for t in ts}) + self.command('.watchlist add T2') + self.command('.watchlist add T3', chat='56') + listed = [str(self.command('.watchlist', chat=chat)) for chat in ('55', '56')] + self.assertEqual([('Co T2' in text, 'Co T3' in text) for text in listed], [(True, False), (False, True)]) + for chat, followed in (('55', ['T1', 'T2']), ('56', ['T1', 'T3'])): + with self.subTest(chat=chat): + fetched.clear() + self.command('.price', chat=chat) + self.assertEqual(fetched[0], followed) + fetched.clear() + self.command('.marketcap', chat=chat) + self.assertEqual(fetched[0], followed) + fetched.clear() + shorts.run(delivery.chat('telegram:56', scheduled=True)) + self.assertEqual(fetched[0], ['T1', 'T3']) + + def test_plans_are_kept_per_chat(self): + """.plan records the sender's plan in this chat, and lists this chat's plans, not another chat's.""" + self.command('.plan buy low') + self.assertIn('buy low', str(self.command('.plan'))) + self.assertEqual(self.command('.plan', chat='56'), [('text', ['No plans yet: .plan TEXT records yours'])]) + self.assertNotIn('buy low', str(self.command('.plan sell high', chat='56'))) + self.assertNotIn('sell high', str(self.command('.plan'))) def test_compare_usage(self): """.compare with one ticker gets its usage, from the command table.""" diff --git a/tests/test_schedule.py b/tests/test_schedule.py index fbd49666..4e67246c 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -4,7 +4,7 @@ from pathlib import Path from unittest import mock from tests.common import * -from lib import jobs, migrate, state +from lib import jobs, migrate, state, watchlist, worker from lib.reports import cal, cleanup, milestone, performance, podcasts, price, rating, trades LOCAL_ZONE = migrate.local_zone # the real one: the migration tests put UTC in its place @@ -209,13 +209,14 @@ def test_with_no_home_chat_it_waits(self): self.assertFalse(os.path.exists(state.path(jobs.SCHEDULE_FILE))) self.assertEqual(state.load(trades.STATE_FILE), ['t1']) - def test_the_reports_records_become_each_home_chats_once(self): - """What a report has told the chats (trades, ratings, record highs, each feed's episodes) becomes each home - chat's record, so that nothing old is told again; a record kept per chat already is left as it is, and so is - one that cannot be read (said in the log).""" + def test_the_watchlist_plans_and_records_become_each_home_chats_once(self): + """The watchlist and the plans become each home chat's, and so does what a report has told the chats (trades, + ratings, record highs, each feed's episodes), so that nothing old is told again; a file kept per chat already is + left as it is, and so is one that cannot be read (said in the log).""" feed = podcasts.get_seen_filename('https://example.com/podcast.rss') - old = {trades.STATE_FILE: ['t1', 't2'], rating.STATE_FILE: {'NVDA': ['upgrade', 3]}, - milestone.STATE_FILE: {'NVDA': {'all_time_high': 200}}, feed: ['guid-1']} + old = {watchlist.FILE: ['MSFT', 'BHP.AX'], worker.PLANS: {'@rob': 'buy low'}, trades.STATE_FILE: ['t1', 't2'], + rating.STATE_FILE: {'NVDA': ['upgrade', 3]}, milestone.STATE_FILE: {'NVDA': {'all_time_high': 200}}, + feed: ['guid-1']} for name, record in old.items(): state.save(name, record) log = self.migrate() diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 67bffb4d..66c16d02 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -86,16 +86,16 @@ def test_every_followed_stocks_history_is_fetched_afresh(self): self.assertEqual(cache.read('finbot_yahoo_history_T1.json'), {'history of': 'T1'}) self.assertIn(f'yahoo_warm: {len(TICKERS) - 1} of {len(TICKERS)} price histories fetched', log.getvalue()) - def test_without_sharesight_the_watchlist_is_fetched(self): - """While Sharesight cannot be asked for the holdings, the watchlist's histories are fetched; without Sharesight - configured, it is not asked.""" + def test_without_sharesight_the_watchlists_are_fetched(self): + """While Sharesight cannot be asked for the holdings, every chat's watchlist's histories are fetched; without + Sharesight configured, it is not asked.""" asked = [] def unreachable(): - """Sharesight failing, as watchlist.with_holdings meets it.""" + """Sharesight failing, as watchlist.all_followed meets it.""" asked.append(1) raise RuntimeError('Sharesight auth: down') - self.patch(watchlist, 'with_holdings', unreachable) - self.patch(watchlist, 'tickers', lambda: ['W1']) + self.patch(watchlist, 'all_followed', unreachable) + self.patch(watchlist, 'all_watched', lambda: ['W1']) with contextlib.redirect_stderr(io.StringIO()) as log: self.assertEqual(yahoo_warm.run(), 1) self.assertEqual(cache.read('finbot_yahoo_history_W1.json'), {'history of': 'W1'}) diff --git a/tests/test_util.py b/tests/test_util.py index 00459347..65c43ad5 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -266,12 +266,20 @@ def test_binary_cache_roundtrip(self): self.assertEqual(cache.read_bytes('img.png').read(), b'png-bytes') def test_holdings_and_watchlist_without_a_watchlist_file(self): - """On the first run there is no watchlist file: the holdings alone, GOOGL dropped when GOOG is held.""" + """On the first run there is no watchlist file: the holdings alone, GOOGL dropped when GOOG is held. A chat's + watchlist adds to the holdings in that chat only; yahoo_warm keeps every chat's warm.""" self.patch(sharesight, 'tickers', lambda: ['AAPL', 'GOOG', 'GOOGL']) self.patch(watchlist, 'with_holdings', REAL_GET_HOLDINGS) - self.assertEqual(watchlist.with_holdings(), ['AAPL', 'GOOG']) # GOOGL dropped when GOOG is present - state.save('finbot_watchlist.json', ['MSFT']) - self.assertEqual(watchlist.with_holdings(), ['AAPL', 'GOOG', 'MSFT']) + self.assertEqual(watchlist.with_holdings('telegram:1'), ['AAPL', 'GOOG']) # GOOGL dropped when GOOG is present + self.assertEqual((watchlist.all_watched(), watchlist.all_followed()), ([], ['AAPL', 'GOOG'])) + watchlist.add('telegram:1', 'MSFT') + watchlist.add('slack:C1', 'NVDA') + watchlist.add('slack:C1', 'MSFT') + self.assertEqual(watchlist.with_holdings('telegram:1'), ['AAPL', 'GOOG', 'MSFT']) + self.assertEqual(watchlist.with_holdings('slack:C1'), ['AAPL', 'GOOG', 'MSFT', 'NVDA']) + self.assertEqual(watchlist.with_holdings('discord:7'), ['AAPL', 'GOOG']) + self.assertEqual(watchlist.all_watched(), ['MSFT', 'NVDA']) + self.assertEqual(watchlist.all_followed(), ['AAPL', 'GOOG', 'MSFT', 'NVDA']) if __name__ == '__main__': From d5253dd5d3f5865ec193e58aa26cd5c16256a1b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:03:18 +0000 Subject: [PATCH 11/31] Docs: every bot listens, on every network The README and the guides no longer describe reports without commands: every network's bot answers commands, fetching its updates itself (pull mode) unless its webhook URL is set. "Interactive trigger" becomes "Command", the pull mode names Discord's gateway beside Telegram's and Slack's, and each channel gets its own scheduled reports. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 40 ++++++++++++++++++++-------------------- doc/discord.md | 4 ++-- doc/slack.md | 4 ++-- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 8da22666..03c70f45 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ * Ex-dividend date warnings * Highly shorted stock warnings (AU, US) * Interday and premarket price movements over a defined threshold -* Interactive chat commands (Telegram, Slack & Discord): +* Chat commands (Telegram, Slack & Discord), with no server to host: * All the above reports * Price history graph * Stock lookup with price/valuation related stats and warnings @@ -45,7 +45,7 @@ posting them to your HTTPS server) before the steps. ### Requirements * Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key * A Telegram bot, a Slack app or a Discord bot (see the network's guide, above) -* Commands need no server: finbot fetches its updates from Telegram and Slack (pull mode). In HTTP mode instead, a network posts them to a public web server with a domain name and a matching certificate, which proxies them to the bot (see the [Telegram](doc/telegram.md#http-mode) and [Slack](doc/slack.md#http-mode) guides). +* No server: finbot fetches its updates from each network itself (pull mode: Telegram's long polling, Slack's Socket Mode, Discord's gateway). In HTTP mode instead, a network posts them to a public web server with a domain name and a matching certificate, which proxies them to the bot (see the [Telegram](doc/telegram.md#http-mode), [Slack](doc/slack.md#http-mode) and [Discord](doc/discord.md#http-mode) guides). * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: ``` @@ -97,9 +97,9 @@ Where: Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying each chat that schedules them of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. -The other various reports can also run on a schedule (e.g. daily or weekly), or on demand through the interactive bot. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. A scheduled report posts to the chat whose job it is, and a command's reply to the chat that sent it. +The other various reports can also run on a schedule (e.g. daily or weekly), or on demand with a command. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. A scheduled report posts to the chat whose job it is, and a command's reply to the chat that sent it. -The interactive bot answers commands in your chats. It fetches its updates from Telegram and Slack itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: +The bot answers commands in every chat it is in. It fetches its updates from each network itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: * Stock lookup (financials, company profile, price charts) * A watch list per chat, which anyone in it can change, picked up by the reports there * Listing of portfolios and portfolio performance @@ -139,7 +139,7 @@ The stock lookup returns various stats relevant to a stock's valuation, growth a * 🔼🔼🔼 revenue increased in all of the past three years * 🔻🔺🔻 In the past three years, earnings decreased, increased, then decreased, and was negative for all three years -This report can only be run through the interactive bot. Example usage: +This report is a command only. Example usage: ``` .AAPL ``` @@ -166,7 +166,7 @@ Job (see [Scheduling](#scheduling)), such as `.jobs add */20 * * * Mon-Fri trade trades ``` -Interactive trigger: +Command: ``` .trades [period] ``` @@ -207,7 +207,7 @@ When scheduled, the mode is its argument: * If `close` is passed, it reports daily change in stocks whose market closed in the past 3 hours. * If an integer [i] is passed, it reports current price against [i] days ago. -Interactive trigger (pre-market): +Command (pre-market): ``` .premarket [percent] ``` @@ -215,7 +215,7 @@ Interactive trigger (pre-market): @botname premarket [percent] ``` -Interactive trigger (mid-session): +Command (mid-session): ``` .session [percent] ``` @@ -223,7 +223,7 @@ Interactive trigger (mid-session): @botname session [percent] ``` -Interactive trigger (interday). +Command (interday). ``` .price [percent] ``` @@ -256,7 +256,7 @@ Job (see [Scheduling](#scheduling)), such as `.jobs add 28 21 * * Sat performanc performance [DAYS] ``` -Interactive trigger: +Command: ``` .performance [period] ``` @@ -264,7 +264,7 @@ Interactive trigger: @botname performance [period] ``` -Interactively, you can also specify a portfolio: +As a command, it also takes a portfolio: ``` .performance [period] [[portfolio] ``` @@ -289,7 +289,7 @@ Job (see [Scheduling](#scheduling)), such as `.jobs add 28 21 * * Sat cal earnin cal earnings|ex-dividend ``` -Interactive trigger (earnings): +Command (earnings): ``` .earnings [period] ``` @@ -297,7 +297,7 @@ Interactive trigger (earnings): @botname earnings [period] ``` -Interactive trigger (ex-dividend): +Command (ex-dividend): ``` .dividend [period] ``` @@ -315,7 +315,7 @@ The `shorts` report sends short interest warnings. The data is sourced from Yaho shorts_percent=15 ``` -Interactive trigger: +Command: ``` .shorts [percent] ``` @@ -328,7 +328,7 @@ Interactive trigger: List securities by P/E ratio, forward P/E ratio, or PEG ratio. A forward P/E is an analysts' estimate, so its lists take only stocks covered by at least `min_analysts` analysts (7 by default; see `preferences.conf`). -Interactive trigger: +Command: ``` .pe [SYMBOL] .fpe [SYMBOL|bottom|negative] @@ -346,7 +346,7 @@ Interactive trigger: Shows the long term performance of a security. It creates a 5Y chart, and then lists performance for several periods (5Y 3Y 1Y 6M 3M 1M 1W 1D) -Interactive trigger: +Command: ``` .history ``` @@ -359,7 +359,7 @@ Interactive trigger: Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts (7 by default; see `preferences.conf`) -Interactive trigger: +Command: ``` .buy ``` @@ -372,7 +372,7 @@ Interactive trigger: Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts -Interactive trigger: +Command: ``` .sell ``` @@ -385,7 +385,7 @@ Interactive trigger: List the top (or bottom) stocks by market cap, in US dollars: market caps in other currencies are converted at Yahoo's exchange rates. With a symbol, its market cap in its own currency and in US dollars. -Interactive trigger: +Command: ``` .marketcap [top|bottom|SYMBOL] ``` @@ -409,7 +409,7 @@ Tracks additional securities which are not in your Sharesight holdings. Each cha The watchlists are kept in `finbot_watchlist.json` in the data directory, by chat (`{"chats": {"telegram:-1001234": ["2454.TW", "ASO.AX", "STEM"]}}`), so there is no need to edit the file. The first start with watchlists per chat gives each home chat the watchlist kept before, and the same goes for the plans that `.plan` records: each chat's `.plan` lists only the plans recorded in it. -Interactive trigger: +Command: ``` .watchlist ``` diff --git a/doc/discord.md b/doc/discord.md index 24372294..a039f444 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -1,8 +1,8 @@ # Discord finbot talks to Discord as the bot user of a Discord application that you create in the Developer Portal. The bot -posts the scheduled reports into a channel and answers commands: slash commands (`/price`) in both modes, and messages -(`.price`, `@finbot price`) over the gateway. +posts each channel's scheduled reports there and answers commands: slash commands (`/price`) in both modes, and +messages (`.price`, `@finbot price`) over the gateway. ## Gateway mode or HTTP mode? diff --git a/doc/slack.md b/doc/slack.md index 11b07aac..b64f2a37 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -1,7 +1,7 @@ # Slack -finbot reaches Slack through a Slack app of your own, whose bot posts the scheduled reports into a channel, charts and -all, and answers commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)), with +finbot reaches Slack through a Slack app of your own, whose bot posts each channel's scheduled reports there, charts +and all, and answers commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)), with charts in its replies, ⏳ on a command's message while it works (Slack has no typing notice for bots), and period buttons under a chart that redraw it in place. From 3f9ac259b495ca6f075e3fc7c861272ada533e7a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:09:03 +0000 Subject: [PATCH 12/31] Remove pandas pandas did three small things with dated closes, at 0.8 s and 37 MB to load: - yahoo.price_series now gives (name, exchange, ((date, close), ...)). - .compare crops and rebases those rows with bisect: the common start, the last close on or before it, the rows after, first to last. - render draws a line's dates as datetime.date and its closes as a numpy array, both of which matplotlib takes as they are. Every chart is pixel for pixel what it was: the five goldens and a range of line and compare charts (lengths from 2 days to 10 years, every sign) hash the same at full size, and util/record_outputs.py finds all 268 steps identical. A test fails if any module imports pandas, and another draws every chart with pandas blocked. requirements.txt and the README drop it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 2 +- lib/render.py | 36 +++++++++++++++++++----------------- lib/reports/history.py | 30 +++++++++++++++--------------- lib/yahoo.py | 11 ++++------- requirements.txt | 1 - tests/common.py | 9 ++++----- tests/test_charts.py | 22 +++++++++++++++++++--- tests/test_finbot.py | 11 +++++++++++ tests/test_scheduler.py | 7 ++++--- 9 files changed, 77 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 03c70f45..c7e7a150 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ posting them to your HTTPS server) before the steps. * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: ``` -croniter python-dateutil requests pandas matplotlib numpy +croniter python-dateutil requests matplotlib numpy ``` * For a network in http mode (its webhook URL set), also `waitress` (Debian: `python3-waitress`); for Slack in Socket Mode, `websocket-client` (`python3-websocket`); for Discord's gateway, `discord.py` (`python3-discord`), and for its diff --git a/lib/render.py b/lib/render.py index 85b4c2d2..96d79f54 100644 --- a/lib/render.py +++ b/lib/render.py @@ -1,6 +1,7 @@ """Charts drawn as PNGs with matplotlib, one at a time (_drawn). A report describes its chart as a Chart, which draw() draws only for a chat that takes charts, and once: the image is cached by the spec's hash.""" import dataclasses +import datetime import functools import hashlib import importlib.metadata @@ -121,10 +122,12 @@ def _heading(fig, title, subtitle='', subtitle_color=None): _shrink([fig.text(0.03, 0.910, subtitle, color=subtitle_color or ink2, fontsize=8.5, ha='left', va='center')], room) -def _series(rows): - """(date, close) rows as a pandas Series of the closes, indexed by date.""" - import pandas as pd - return pd.Series([float(close) for _, close in rows], index=pd.to_datetime([day for day, _ in rows])) +def _dated(rows): + """(date, close) rows, each date an ISO date or a datetime.date, as (the dates as datetime.date, the closes as a + numpy array): matplotlib takes both as they are.""" + import numpy as np + dates = [day if isinstance(day, datetime.date) else datetime.date.fromisoformat(day) for day, _ in rows] + return dates, np.array([float(close) for _, close in rows]) def _png(fig, facecolor): """The figure as PNG bytes, rewound (or an upload sends nothing); not bbox_inches='tight': it alters 1280x800.""" @@ -141,8 +144,7 @@ def draw_line(chart): import matplotlib.dates as mdates import numpy as np bg, ink, ink2, grid = PALETTE - series = _series(chart.rows) - x, y = series.index, series.values + x, y = _dated(chart.rows) change = chart.change if chart.change is not None else y[-1] - y[0] color = UP if change > 0 else DOWN if change < 0 else FLAT @@ -189,23 +191,23 @@ def draw_compare(chart): colors = NEITHER_UP_NOR_DOWN fig, _ = _figure(bg) ax = _time_axes(fig, bg, 0.72) - rebased = [] + rebased = [] # (label, dates, % change since the first close) for label, points in chart.rows: - y = _series(points) - rebased.append((label, (y / y.iloc[0] - 1) * 100)) + x, y = _dated(points) + rebased.append((label, x, (y / y[0] - 1) * 100)) ax.axhline(0, color=ink2, linewidth=0.8, alpha=0.6, zorder=1) - for (label, y), color in zip(rebased, colors): - ax.plot(y.index, y.values, color=color, linewidth=1.5, solid_capstyle='round', zorder=3) - lo = min(y.min() for _, y in rebased) - hi = max(y.max() for _, y in rebased) + for (label, x, y), color in zip(rebased, colors): + ax.plot(x, y, color=color, linewidth=1.5, solid_capstyle='round', zorder=3) + lo = min(y.min() for _, _, y in rebased) + hi = max(y.max() for _, _, y in rebased) span = (hi - lo) or 1.0 ax.set_ylim(min(lo, 0) - span * 0.06, max(hi, 0) + span * 0.06) ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:+,.0f}%" if v else "0%")) - _style_time_axes(fig, ax, min(y.index[0] for _, y in rebased), max(y.index[-1] for _, y in rebased)) + _style_time_axes(fig, ax, min(x[0] for _, x, _ in rebased), max(x[-1] for _, x, _ in rebased)) # direct end-labels in the right margin, spread so they never overlap ymin, ymax = ax.get_ylim() - ends = sorted(((y.iloc[-1], i) for i, (_, y) in enumerate(rebased))) + ends = sorted(((y[-1], i) for i, (_, _, y) in enumerate(rebased))) pos = [(v - ymin) / (ymax - ymin) for v, _ in ends] gap = 0.065 for i in range(1, len(pos)): @@ -216,10 +218,10 @@ def draw_compare(chart): for i in range(len(pos) - 2, -1, -1): pos[i] = min(pos[i], pos[i + 1] - gap) for (value, i), p in zip(ends, pos): - label, y = rebased[i] + label, x, _ = rebased[i] color = colors[i] ax.plot([1.015], [p], marker='s', markersize=4, color=color, transform=ax.transAxes, clip_on=False, zorder=4) - ax.annotate(f"{label} {value:+,.1f}%", xy=(y.index[-1], value), xycoords='data', xytext=(1.03, p), + ax.annotate(f"{label} {value:+,.1f}%", xy=(x[-1], value), xycoords='data', xytext=(1.03, p), textcoords='axes fraction', ha='left', va='center', fontsize=7.5, color=ink, annotation_clip=False, arrowprops=dict(arrowstyle='-', color=color, linewidth=0.7, alpha=0.7, shrinkA=0, shrinkB=2, relpos=(0, 0.5))) diff --git a/lib/reports/history.py b/lib/reports/history.py index 9b972f23..6f7f7c8a 100644 --- a/lib/reports/history.py +++ b/lib/reports/history.py @@ -1,4 +1,5 @@ """Price history: .history (one stock over every period) and .compare (two to six stocks, rebased).""" +import bisect import datetime from lib import answer from lib import buttons @@ -10,36 +11,35 @@ def prepare_compare(service, tickers, period_days=None): """(caption lines, chart spec): 2-6 tickers (Yahoo's symbols) as rebased % change over the last `period_days`, else 10 years (as Max is on every chart), from no earlier than the first date every one of them covers.""" - import pandas as pd names, exchanges, closes = {}, {}, {} for ticker, history in zip(tickers, yahoo.fetch_each(yahoo.price_series, tickers)): # downloaded at the same time if isinstance(history, Exception): raise history names[ticker], exchanges[ticker], closes[ticker] = history - limiting = max(closes, key=lambda t: closes[t].index[0]) # ticker with the shortest history - common_start = closes[limiting].index[0] - requested = pd.Timestamp(datetime.datetime.now().date() - datetime.timedelta(days=period_days or 3655)) + limiting = max(closes, key=lambda t: closes[t][0][0]) # ticker with the shortest history + common_start = closes[limiting][0][0] + requested = datetime.datetime.now().date() - datetime.timedelta(days=period_days or 3655) start = max(common_start, requested) note = f" (limited by {limiting} history)" if period_days and requested < common_start else '' cropped = [] for ticker in tickers: - series = closes[ticker] - base = series[series.index <= start] - base_value = base.iloc[-1] if len(base) else series.iloc[0] # last close on/before the start date - series = pd.concat([pd.Series([base_value], index=[start]), series[series.index > start]]) - cropped.append((ticker, series)) - if any(len(series) < 2 for _, series in cropped): + rows = closes[ticker] + before = bisect.bisect_right([day for day, _ in rows], start) # the closes on or before the start date + base_value = rows[before - 1][1] if before else rows[0][1] # the last of them, else the first close + cropped.append((ticker, ((start, base_value), *rows[before:]))) + if any(len(rows) < 2 for _, rows in cropped): raise answer.Answer("not enough price history in that period to compare") - span_days = (datetime.datetime.now().date() - start.date()).days + span_days = (datetime.datetime.now().date() - start).days period_label = formats.days_english(period_days) if period_days and not note else f"{span_days} days" - by_performance = sorted(cropped, key=lambda item: item[1].iloc[-1] / item[1].iloc[0], reverse=True) + growth = lambda rows: rows[-1][1] / rows[0][1] # last close over first + by_performance = sorted(cropped, key=lambda item: growth(item[1]), reverse=True) title = ' vs '.join(formats.short_ticker(t) for t, _ in by_performance) # best first; colours keep input order subtitle = f"% change over {period_label}, since {start:%-d %b %Y}" + note - lines = tuple((formats.short_ticker(t), tuple((f"{day:%Y-%m-%d}", float(close)) for day, close in series.items())) - for t, series in cropped) + lines = tuple((formats.short_ticker(t), tuple((f"{day:%Y-%m-%d}", float(close)) for day, close in rows)) + for t, rows in cropped) chart = render.Chart('compare', title, subtitle, lines) caption = [service.bold(f"{period_label}, since {start:%-d %b %Y}") + note] - results = sorted(((series.iloc[-1] / series.iloc[0] - 1) * 100, t) for t, series in cropped) + results = sorted(((growth(rows) - 1) * 100, t) for t, rows in cropped) for pct, t in reversed(results): link = links.finance_link(t, exchanges[t], service, days=span_days, brief=False) caption.append(f"{formats.get_emoji(pct)} {names[t]} ({link}): {formats.signed_percent(pct, 1)}") diff --git a/lib/yahoo.py b/lib/yahoo.py index c9a0e9c5..6a1b17b8 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -364,18 +364,15 @@ def price_history(ticker, days=None, quote=None): tuple((str(day), float(last)) for day, last in closes)) def price_series(ticker): - """A ticker's daily closes over 10 years, from the same chart as price_history(): (name, exchange, pandas Series - indexed by date). Raises NoData when there is no history.""" - import pandas as pd + """A ticker's daily closes over 10 years, from the same chart as price_history(): (name, exchange, ((date, + close), ...)), oldest first, each date a datetime.date. Raises NoData when there is no history.""" data = fetch_chart_json(ticker) meta = dig(data, 'chart', 'result', 0, 'meta') or {} - closes = [(day, last) for day, _, last in bars(data) if last is not None] + closes = tuple((day, float(last)) for day, _, last in bars(data) if last is not None) if not closes: raise NoData(f"no price history for {ticker}") - series = pd.Series([float(last) for _, last in closes], index=pd.to_datetime([str(day) for day, _ in closes]), - name=ticker) name = formats.transform_title(meta.get('longName') or meta.get('shortName') or ticker) - return name, meta.get('fullExchangeName') or meta.get('exchangeName') or '', series + return name, meta.get('fullExchangeName') or meta.get('exchangeName') or '', closes def fetch_each(get, tickers, workers=8): """get(ticker) for each ticker, `workers` at a time so the waits overlap: [its result or the exception raised].""" diff --git a/requirements.txt b/requirements.txt index 2b6ef139..fe66405a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,5 @@ croniter>=1.3.15 matplotlib>=3.9.2,<4 numpy>=2.1.0,<3 -pandas>=2.2.3,<3 # 3.x not yet tried python-dateutil>=2.9.0 requests>=2.32.3 diff --git a/tests/common.py b/tests/common.py index 12168efa..930bb50c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,9 +1,8 @@ """Shared fixture: points finbot at a temp cache dir and replaces every external service (Telegram, Yahoo, Sharesight, Short Man) with canned data. Nothing here touches the network. ProviderCase runs the real parsers instead, on the API answers saved under doc/ (sample) as a stand-in network returns them (Resp, serve).""" -import collections, dataclasses, io, json, os, shutil, sys, tempfile, time, unittest, zoneinfo +import collections, dataclasses, datetime, io, json, os, shutil, sys, tempfile, time, unittest, zoneinfo from pathlib import Path -import pandas as pd from unittest import mock ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -146,9 +145,9 @@ def setUp(self): ()))) self.patch(yahoo, 'prefetch_history', lambda ts: None) self.patch(yahoo, 'historic_high', lambda t: (100.0, 50.0)) - dates = pd.date_range(end=pd.Timestamp.today().normalize(), periods=400) - self.patch(yahoo, 'price_series', - lambda t: ('Co ' + t, 'NASDAQ', pd.Series(range(100, 500), index=dates, name=t, dtype=float))) + today = datetime.date.today() # 400 days of closes, to today: 100, 101, ... 499 + closes = tuple((today - datetime.timedelta(days=399 - i), float(100 + i)) for i in range(400)) + self.patch(yahoo, 'price_series', lambda t: ('Co ' + t, 'NASDAQ', closes)) self.patch(shortman, 'fetch', lambda md: md) held = [{'id': i, 'instrument': {'code': t, 'market_code': 'NASDAQ'}, 'portfolio': {'name': 'Rob'}, 'capital_gain_percent': str(i * 4 - 40), 'value': '1000'} for i, t in enumerate(TICKERS)] diff --git a/tests/test_charts.py b/tests/test_charts.py index 7f6c2599..85e26aad 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -4,10 +4,9 @@ 3. one golden image per chart type, compared with a tolerance. Refresh after an intended restyle with: UPDATE_GOLDEN=1 python -m unittest tests.test_charts """ -import datetime, os, threading, unittest +import datetime, os, subprocess, sys, tempfile, threading, unittest from unittest import mock import numpy as np -import pandas as pd from PIL import Image from matplotlib.axes import Axes from matplotlib.figure import Figure @@ -47,7 +46,7 @@ def line(rows, title='Apple Inc (AAPL) 120 days', subtitle='164.12 USD ▲ 6.9 def compare_rows(): """Three made-up price series over the same 120 days, as a compare chart's rows.""" - days = [str(day.date()) for day in pd.date_range('2026-01-01', periods=120)] + days = [str(datetime.date(2026, 1, 1) + datetime.timedelta(days=i)) for i in range(120)] return tuple((name, tuple(zip(days, 100 + np.cumsum(np.sin(np.arange(120) / k) + drift)))) for name, k, drift in (('AAA', 7, 0.2), ('BBB', 11, -0.1), ('CCC', 5, 0.05))) @@ -346,6 +345,23 @@ def test_shorts_is_sorted_and_carries_the_threshold(self): self.assertTrue(all(v > chart.threshold for v in values)) +class WithoutPandas(unittest.TestCase): + def test_every_chart_draws_with_pandas_blocked(self): + """Every kind of chart draws in a process that cannot import pandas: the line charts' dates are plain dates and + their closes numpy arrays, which matplotlib takes as they are.""" + code = ('import sys\nfrom tests import test_charts\nfor draw in test_charts.CHARTS.values(): draw()\n' + 'print(sorted(test_charts.CHARTS))') + with tempfile.TemporaryDirectory() as data: + stubs = Path(data, 'stubs') + stubs.mkdir() + (stubs / 'pandas.py').write_text('raise ImportError("blocked by the test")\n') + env = bare_environment() + env.update(DATA_DIR=data, PYTHONPATH=str(stubs)) + out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=env, capture_output=True, text=True, + timeout=120) + self.assertEqual(out.stdout.strip().splitlines()[-1:], [str(sorted(CHARTS))], out.stderr) + + class GoldenImages(unittest.TestCase): """One reference image per chart type. Tolerant on purpose: compares 160x100 thumbnails, so font anti-aliasing and minor matplotlib differences pass, while changed colours, layout, bar sizes or missing text do not.""" diff --git a/tests/test_finbot.py b/tests/test_finbot.py index fe13515d..3498d574 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -90,6 +90,17 @@ def test_every_line_fits_in_120_columns(self): self.assertGreater(len(files), 60) self.assertEqual(long, []) + def test_nothing_imports_pandas(self): + """finbot has no use for pandas, which took 0.8 s and 37 MB to load for three small jobs on dated closes: no + module imports it, not even inside a function.""" + files = [*Path(ROOT).glob('*.py'), *Path(ROOT, 'lib').rglob('*.py'), *Path(ROOT, 'util').glob('*.py'), + *Path(ROOT, 'tests').glob('*.py')] + found = [f'{path.relative_to(ROOT)}:{number}' for path in files + for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1) + if re.match(r'\s*(import|from) pandas\b', line)] + self.assertGreater(len(files), 60) + self.assertEqual(found, []) + def test_the_bot_and_its_reports_print_only_to_stderr(self): """systemd discards stdout (StandardOutput=null), so only util/ tools, run by hand, may print there.""" offenders = [] diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 66c16d02..402e44eb 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -657,10 +657,11 @@ def test_a_job_is_its_chats_whichever_way_its_updates_arrive(self): class ReportImports(FinbotCase): """What importing the reports brings in.""" - def test_a_report_script_loads_neither_pandas_nor_the_scheduler(self): - """A report imports only its script: price once pulled in pandas and every script (via charts and jobs).""" + def test_a_report_loads_neither_matplotlib_nor_the_scheduler(self): + """A report imports only its own module: price once pulled in every report, through charts and jobs, and the + chart code until a chart is drawn.""" code = ('import sys; from lib.reports import price, performance, trades, shorts, cal; ' - 'print(sorted(m for m in ("pandas", "lib.jobs", "matplotlib") if m in sys.modules))') + 'print(sorted(m for m in ("lib.jobs", "matplotlib") if m in sys.modules))') with tempfile.TemporaryDirectory() as data: out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True, env={**os.environ, 'DATA_DIR': data}) From 993e3fe2a13ff12dfc5e74ceffa2b37d2e6f67dc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:19:56 +0000 Subject: [PATCH 13/31] Slash commands on every network The command table gives every network the same slash commands (worker.slash_commands: one per .help line, a command's several lines joined in its usage, /help, and the second names /profile and /job): - Telegram: as finbot starts, in both modes, the bot's command menu is set (setMyCommands, only when Telegram's differs; a failure is only logged). The menu leaves out /profile and /job, which work typed. - Slack: util/slack_manifest.py prints the app's manifest (slash commands, scopes, events, interactivity, the messages tab, Socket Mode on or off, and in HTTP mode the Request URL), which replaces the manual steps of doc/slack.md. A slash command arrives as a form over HTTP or a slash_commands envelope over the socket, both read as the message that says the same (/price 5 7d) in the channel or DM it was sent in. - Discord: /job is registered beside /jobs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 16 +++--- README.md | 6 +-- bot.py | 1 + doc/discord.md | 2 +- doc/slack.md | 101 +++++++++++++++++++----------------- doc/telegram.md | 13 +++-- lib/inbound.py | 41 ++++++++++----- lib/pull.py | 9 ++-- lib/telegram.py | 8 +++ lib/worker.py | 45 +++++++++------- tests/test_inbound.py | 103 +++++++++++++++++++++++++++++++++++-- tests/test_pull.py | 17 ++++++ tests/test_telegram_api.py | 31 ++++++++++- util/slack_manifest.py | 75 +++++++++++++++++++++++++++ 14 files changed, 366 insertions(+), 102 deletions(-) create mode 100755 util/slack_manifest.py diff --git a/CLAUDE.md b/CLAUDE.md index 7a58099e..1b7bb673 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,7 +50,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm or, over the gateway, posted to its callback. `lib/discord.py`: Discord's REST calls (`call`, rate limits waited out) and the slash commands registered at start (`register`, from `worker.slash_commands()`). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, - keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the + keywords)`, which also makes `.help`, the Telegram command keyboard, kept current in DMs, and `slash_commands()`, which + every network gets (Telegram's menu, Slack's manifest from `util/slack_manifest.py`, Discord's registered commands). `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a percent, a period and a keyword from either end, the words between; a handler raises `Usage` for arguments it cannot take, and gets a ticker with `ticker(args)`. `Request` (`request.reply(lines, chart, buttons)`); each command gets @@ -100,12 +101,13 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm them; `temp_dir(self)` is a directory removed after the test, `bare_environment()` the environment without finbot's settings. - Which test file: commands and scheduled reports `test_reports`; failures, bad input, secrets in logs, signals - `test_edges`; webhook requests, Slack events and signatures `test_inbound`; the pull modes `test_pull`; the Discord - bot `test_discord`; Yahoo `test_yahoo`; Sharesight `test_sharesight`; posting to each network `test_sending`; - registering Telegram's webhook `test_telegram_api`; job lines, job arguments and the migration `test_schedule`; the - scheduler and `.jobs` in a chat `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); - markets, formats, links, buttons, cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, - and the docstring, stdout and line-width checks `test_finbot`. + `test_edges`; webhook requests, Slack events, slash commands and signatures, and the Slack manifest `test_inbound`; + the pull modes `test_pull`; the Discord bot `test_discord`; Yahoo `test_yahoo`; Sharesight `test_sharesight`; posting + to each network `test_sending`; Telegram's webhook and command menu `test_telegram_api`; job lines, job arguments and + the migration `test_schedule`; the scheduler and `.jobs` in a chat `test_scheduler`; config `test_config`; charts + `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, cache and state files + `test_util`; data dir `test_datadir`; chunking, git_version, and the docstring, stdout, line-width and no-pandas + checks `test_finbot`. ## Conventions - Read settings as `config.env.` / `config.prefs.` where they are used, never into a default or module diff --git a/README.md b/README.md index c7e7a150..83b04f56 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ What finbot can do on each network. Telegram takes everything; the others take w | Feature | Telegram | Slack | Discord | |---|---|---|---| -| Chat commands | ✓ | ✓ | ✓: slash commands (`/price`), and messages (`.price`) over the gateway | -| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots) | ✓ for a message; "thinking…" for a slash command | +| Chat commands | ✓: messages (`.price`), and slash commands (`/price`) from the command menu | ✓: messages, and slash commands (`/price`) from the app's manifest | ✓: slash commands (`/price`), and messages over the gateway | +| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots); none for a slash command, which has no message | ✓ for a message; "thinking…" for a slash command | | Charts in replies | ✓ | ✓ | ✓ | | Charts in scheduled reports | ✓ | ✓ | ✓ | | Period and job buttons | ✓ | ✓ | ✓ | @@ -93,7 +93,7 @@ Where: .who symbol ``` -`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command, as Discord registers them: `/stock symbol` (or `/profile symbol`) for `.symbol`, `/forwardpe` for `.forward pe`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. +`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command on every network, which completes its name as it is typed: Telegram's command menu, set as finbot starts; Slack's, from the app's manifest (`util/slack_manifest.py`, see the [Slack guide](doc/slack.md#slash-commands)); Discord's, registered as finbot starts. `/stock symbol` (or `/profile symbol`) is `.symbol`, `/forwardpe` is `.forward pe`, and `/job` is `/jobs`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying each chat that schedules them of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. diff --git a/bot.py b/bot.py index 8dc350d1..f81a0460 100755 --- a/bot.py +++ b/bot.py @@ -248,6 +248,7 @@ def http_server(): except Exception as e: print('ERROR', delivery.error_line(e, 'telegram'), file=sys.stderr) sys.exit(1) + telegram.set_commands(worker.slash_commands(aliases=False)) # the menu; /profile and /job work typed threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background if config.env.discordBotToken: try: diff --git a/doc/discord.md b/doc/discord.md index a039f444..4f5184e0 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -95,7 +95,7 @@ discordPublicKey=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcde finbot registers a slash command for each line of `.help`, as it starts, and again only when the list has changed: `/price`, `/dividend` and `/earnings`, `/buy` and `/sell`, `/forwardpe` for `.forward pe`, `/stock` (or `/profile`) for -`.SYMBOL`, and the rest. Each takes its arguments, if any, as one text option, as the message would: `/price` with +`.SYMBOL`, `/jobs` (or `/job`), and the rest. Each takes its arguments, if any, as one text option, as the message would: `/price` with `5 7d` is `.price 5 7d`. They work in the server's channels and in a direct message with the bot. A message holds up to 2,000 characters, so a long report arrives as several messages. A chart goes as an attachment of diff --git a/doc/slack.md b/doc/slack.md index b64f2a37..f3889c50 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -1,9 +1,9 @@ # Slack finbot reaches Slack through a Slack app of your own, whose bot posts each channel's scheduled reports there, charts -and all, and answers commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)), with -charts in its replies, ⏳ on a command's message while it works (Slack has no typing notice for bots), and period -buttons under a chart that redraw it in place. +and all, and answers commands: `/price`, `.price`, `@finbot price` and the rest +([the list](../README.md#supported-commands)), with charts in its replies, ⏳ on a command's message while it works +(Slack has no typing notice for bots), and period buttons under a chart that redraw it in place. ## The bot @@ -13,25 +13,29 @@ The bot does the same either way. What differs is how it hears from Slack: | | Socket Mode (the default) | HTTP mode | |---|---|---| -| How messages and button presses reach finbot | Over a socket that finbot opens to Slack | Slack posts them to your HTTPS server | +| How messages, slash commands and button presses reach finbot | Over a socket that finbot opens to Slack | Slack posts them to your HTTPS server | | Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | | Python module | `websocket-client` (Debian: `python3-websocket`) | `waitress` (Debian: `python3-waitress`) | | `finbot.env` keys | `slackBotToken`, `slackAppToken` | `slackBotToken`, `slackOutgoingWebhook`, `slackSigningSecret` | `slackOutgoingWebhook` chooses the mode: set, HTTP mode; empty, Socket Mode. The app's own _Socket Mode_ setting must agree, as Slack sends to one or the other: to finbot's socket while it is on, to the Request URLs while it is off. To -switch, change both and restart finbot. +switch, change both (paste the manifest for the other mode, which sets the app's side) and restart finbot. ### Socket Mode -1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick - your workspace. -2. Under _Settings > Socket Mode_, turn _Enable Socket Mode_ on. Slack asks for an app-level token: name it (finbot, - say) and _Generate_; it has the `connections:write` scope it needs. Copy the token, which starts with `xapp-`, into - `finbot.env` as `slackAppToken`. It is also under _Basic Information > App-Level Tokens_. -3. Install the socket module: `sudo apt install python3-websocket` (or `pip install websocket-client`). -4. Go on with [the events, the buttons and the bot token](#the-events-the-buttons-and-the-bot-token), below. In Socket - Mode there is no Request URL to enter. +1. Print the app's manifest, which sets up the whole app at once (the slash commands, the scopes, the events, the + buttons, direct messages and Socket Mode): `util/slack_manifest.py`. The bot then hears the messages that mention it + (`@finbot price`), its slash commands and direct messages. With `--every-message`, it hears every message in its + channels instead of the mentions, so that `.price` works there without mentioning it; not both, or it would answer a + mention twice. +2. At https://api.slack.com/apps choose _Create New App > From an app manifest_, pick your workspace, paste the + manifest into the JSON tab, and _Create_. +3. Under _Basic Information > App-Level Tokens_, choose _Generate Token and Scopes_: name it (finbot, say), add the + `connections:write` scope and _Generate_. Copy the token, which starts with `xapp-`, into `finbot.env` as + `slackAppToken`. +4. Install the socket module: `sudo apt install python3-websocket` (or `pip install websocket-client`). +5. Go on with [installing the app](#installing-the-app), below. ``` slackAppToken=xapp-your-app-level-token @@ -41,13 +45,11 @@ slackChannelID=C0123456789 ### HTTP mode -Slack delivers the messages that mention the bot (or every message in a channel, as you choose) to finbot over HTTPS. -finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, -on a domain name with a valid certificate. +Slack delivers the messages, the button presses and the slash commands to finbot over HTTPS. finbot listens on +`http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name +with a valid certificate. -1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick - your workspace. Leave _Settings > Socket Mode_ off. -2. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests +1. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests come from: ``` server { @@ -69,15 +71,19 @@ on a domain name with a valid certificate. `util/nginx_aws_cron` writes `/etc/nginx/aws_subnets`, an `allow` line for each of Amazon's networks, and reloads nginx. Run it once as root before nginx loads this configuration, or put it in `/etc/cron.daily/` to keep the list current. -3. In the app's _Basic Information > App Credentials_, copy the _Signing Secret_ into `finbot.env` as +2. In `finbot.env`, set `slackOutgoingWebhook` to the URL Slack should post to, and restart finbot, so that it can + answer Slack's check of the URL in step 4. +3. Print the app's manifest with that URL, which Slack then posts everything to: + `util/slack_manifest.py --url https://www.example.com:8443/slack` (and `--every-message`, as for + [Socket Mode](#socket-mode)). +4. At https://api.slack.com/apps choose _Create New App > From an app manifest_, pick your workspace, paste the + manifest into the JSON tab, and _Create_. Slack checks the URL, and finbot answers. +5. In the app's _Basic Information > App Credentials_, copy the _Signing Secret_ into `finbot.env` as `slackSigningSecret`. finbot checks Slack's signature on every request with it, and turns away a request signed more than five minutes earlier: a replay, or a server clock that is wrong. * The older _Verification Token_, set as `slackOutgoingToken`, still works in place of the signing secret, but Slack has deprecated it: the token travels in every request, where the signing secret never does. -4. In `finbot.env`, set `slackOutgoingWebhook` to the URL Slack should post to, and restart finbot, so that it can - answer Slack's check of the URL in the next steps. -5. Go on with [the events, the buttons and the bot token](#the-events-the-buttons-and-the-bot-token), below, entering - this URL as the _Request URL_ where it asks for one. +6. Go on with [installing the app](#installing-the-app), below. ``` slackOutgoingWebhook=https://www.example.com:8443/slack @@ -86,35 +92,34 @@ slackBotToken=xoxb-your-bot-token slackChannelID=C0123456789 ``` -### The events, the buttons and the bot token - -1. Under _Event Subscriptions_: - * Turn _Enable Events_ on. In HTTP mode, enter the URL as the _Request URL_: Slack checks it at once, and finbot - answers. - * Under _Subscribe to bot events_, add `app_mention`: the bot sees messages that mention it, such as - `@finbot price`. - * Or add `message.channels` instead, for the bot to see every message in its channels and answer the commands - that start with `.`. Not both: the bot would answer a mention twice. - * For direct messages to the bot, add `message.im` (Slack adds the `im:history` scope it needs), and tick - _App Home > Allow users to send Slash commands and messages from the messages tab_. - * _Save Changes_. -2. Under _Interactivity & Shortcuts_, turn _Interactivity_ on (in HTTP mode, with the same URL as the _Request URL_) - and _Save Changes_. Slack sends finbot the presses of the period buttons under a chart; without it, Slack answers a - press with an error. -3. Under _OAuth & Permissions > Scopes > Bot Token Scopes_, add `chat:write`, `files:write`, `files:read` and - `reactions:write`. The upload of a chart needs both `files` scopes, and the ⏳ on a command that takes a while - `reactions:write`. -4. At the top of _OAuth & Permissions_, _Install to Workspace_ (or _Reinstall to Workspace_ when you changed scopes of - an installed app), and copy the _Bot User OAuth Token_ into `finbot.env` as `slackBotToken`. finbot replies with it; - without it, commands get no reply, and the log says so at startup and for each reply. -5. Restart finbot, and invite the bot to the channel: `/invite @finbot`. In Socket Mode the log says +### Installing the app + +1. Under _Install App_ (or _OAuth & Permissions_), _Install to Workspace_, and copy the _Bot User OAuth Token_ into + `finbot.env` as `slackBotToken`. finbot replies with it; without it, commands get no reply, and the log says so at + startup and for each reply. +2. Restart finbot, and invite the bot to the channel: `/invite @finbot`. In Socket Mode the log says `slack: Socket Mode (no slackOutgoingWebhook)`. -6. Send `@finbot help` in the channel (or `.help`, with `message.channels`). -7. Set `slackChannelID` to the channel's id (in Slack, click the channel's name: the _Channel ID_, which starts with +3. Send `/help` or `@finbot help` in the channel (or `.help`, with `--every-message`), or a direct message to the bot + in its _Messages_ tab. +4. Set `slackChannelID` to the channel's id (in Slack, click the channel's name: the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. It is Slack's home chat: finbot's first start gives it the default jobs ([Scheduling](../README.md#scheduling)), and it gets the error reports. Every channel manages its own jobs with `.jobs`. +An app set up by hand before takes the manifest too: paste it into _Features > App Manifest_ and _Save Changes_; +Slack shows what changes, and asks to reinstall the app when its scopes do. + +### Slash commands + +Each command is a slash command too, with its usage as Slack's hint: `/price`, `/dividend` and `/earnings`, `/buy` +and `/sell`, `/forwardpe` for `.forward pe`, `/stock` (or `/profile`) for `.SYMBOL`, `/jobs` (or `/job`), and the rest. +Slack makes slash commands only from the app's settings, so they come in the manifest: when finbot's commands change, +print it again and paste it into _App Manifest_. + +A slash command's answer goes to the channel or direct message it was sent in, as a message's does; the bot must be in +the channel (`/invite @finbot`). Slack shows no ⏳ while it works: there is no message of a slash command to put it +on. + A chart is uploaded with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`). With its period buttons, it goes in a message of blocks: the report, the chart (uploaded into no channel, and shown by its id) and the buttons. A press shows the chart for that period in the message in place of the last one: the upload of diff --git a/doc/telegram.md b/doc/telegram.md index 66219300..b2363e38 100644 --- a/doc/telegram.md +++ b/doc/telegram.md @@ -13,10 +13,15 @@ Telegram: | Public HTTPS server | Not needed | Needed: a domain name with a valid certificate, on port 443, 80, 88 or 8443 | | `finbot.env` keys | `telegramBotToken`, `telegramChatID` | Those two, plus `telegramOutgoingWebhook` and `telegramOutgoingToken` | -Everything else is the same in both: the scheduled reports with their charts, the chat commands (`.price`, `.history`, -`.performance` and the rest, [the list](../README.md#supported-commands)), period buttons under charts and the -buttons of `.jobs run` and `.jobs toggle`, "typing…" while a command works, and a keyboard of the common commands in a -direct chat with the bot. +Everything else is the same in both: the scheduled reports with their charts, the chat commands (`/price` or `.price`, +`.history`, `.performance` and the rest, [the list](../README.md#supported-commands)), period buttons under charts and +the buttons of `.jobs run` and `.jobs toggle`, "typing…" while a command works, and a keyboard of the common commands in +a direct chat with the bot. + +As finbot starts, it sets the bot's command menu, which Telegram shows when `/` is typed: each command with its usage +(`/forwardpe` for `.forward pe`, `/stock` for `.SYMBOL`). `/profile` and `/job`, the second names of `/stock` and +`/jobs`, are left out of the menu, and work when typed, as every name of a command does. In a group, `/price@finbot_bot` +is the same as `/price`. `telegramOutgoingWebhook` chooses the mode: set, HTTP mode; empty, pull mode. A restart switches. As finbot starts in pull mode it deletes the webhook, and in HTTP mode it sets it; the updates sent meanwhile are kept by Telegram and diff --git a/lib/inbound.py b/lib/inbound.py index 3ab8bef8..05c9a350 100644 --- a/lib/inbound.py +++ b/lib/inbound.py @@ -145,7 +145,7 @@ def acknowledge(self, event): class Slack(Network): - """Slack's Events API (messages) and its interactivity (a button pressed).""" + """Slack's Events API (messages), its interactivity (a button pressed) and its slash commands.""" service = 'slack' BY_PEOPLE = (None, 'thread_broadcast') # message subtypes a person writes; the rest are bot posts, edits, joins... @@ -154,10 +154,10 @@ def update_id(self, update): return update.get('event_id') def parse(self, update): - """A message a person wrote, a button pressed, or the challenge that checks the URL. Links and mentions lose - their markup ( becomes dub.ax, <@QWERTY> @QWERTY), and Slack's escapes their meaning - (& becomes &).""" + """A message a person wrote, a slash command, a button pressed, or the challenge that checks the URL.""" kind = update.get('type') + if 'command' in update and kind is None: + return self.slash(update) if kind == 'url_verification': response = json.dumps({'challenge': update['challenge']}) print("replying with", response, file=sys.stderr) @@ -175,10 +175,23 @@ def parse(self, update): if event['type'] not in ('message', 'app_mention') or 'text' not in event: print("[slack]: unhandled event callback type", event['type'], file=sys.stderr) return UNHANDLED - text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), event['text']) - text = html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > user = '<@' + event['user'] + '>' # a mention: Slack shows the name - return Message(self.service, event['channel'], user, text, '@' + bot_user, user, event.get('ts', '')) + return Message(self.service, event['channel'], user, self.plain(event['text']), '@' + bot_user, user, + event.get('ts', '')) + + def slash(self, update): + """A slash command (/price with 5 7d), as the message that says the same (/price 5 7d), in the channel or DM it + was sent in. There is no message of it, for ⏳ to show on.""" + user = '<@' + str(update.get('user_id') or 'unknown') + '>' + text = self.plain(f"{update['command']} {update.get('text') or ''}".strip()) + return Message(self.service, str(update.get('channel_id') or ''), user, text, '', user) + + @staticmethod + def plain(text): + """Text as written: links and mentions lose their markup ( becomes dub.ax, <@QWERTY> + @QWERTY), and Slack's escapes their meaning (& becomes &).""" + text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), text) + return html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > def press(self, update): """A button pressed under one of the bot's messages (a chart's period buttons), or UNSUPPORTED for one that is @@ -306,8 +319,8 @@ def refuse_unread(self, environ): class SlackWebhook(Webhook): - """Slack posting events and button presses to slackOutgoingWebhook (the app's Events and Interactivity URL), each - request signed with the app's signing secret, and carrying its verification token.""" + """Slack posting events, button presses and slash commands to slackOutgoingWebhook (the app's Request URL for each), + each request signed with the app's signing secret, and carrying its verification token.""" def path(self): """The path of slackOutgoingWebhook.""" @@ -339,9 +352,13 @@ def signature_problem(environ, body, secret): return None if same_secret(signature, expected) else "Slack signature incorrect" def decode(self, body): - """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON.""" - if body.startswith(b'payload='): - body = parse_qs(body.decode('utf-8', 'replace')).get('payload', [''])[0].encode() + """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON, and a + slash command as a form of its fields (command, text, channel_id, user_id and the rest).""" + if not body.lstrip().startswith(b'{'): + fields = parse_qs(body.decode('utf-8', 'replace'), keep_blank_values=True) + if 'command' in fields and 'payload' not in fields: + return {name: values[0] for name, values in fields.items()} + body = fields.get('payload', [''])[0].encode() return super().decode(body) diff --git a/lib/pull.py b/lib/pull.py index 9b5737b2..abf3cdb1 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -142,12 +142,13 @@ def close(self): class SlackSocket(Listener): """Slack's Socket Mode: a WebSocket to the URL that apps.connections.open gives for the app-level token - (slackAppToken). Each event or button press comes in an envelope, acked at once, before anything else (Slack sends - again what is not acked within 3 seconds); inside it is what Slack would post to slackOutgoingWebhook. Slack closes - a socket now and then, saying so first (disconnect), and another is opened at once.""" + (slackAppToken). Each event, button press or slash command comes in an envelope, acked at once, before anything + else (Slack sends again what is not acked within 3 seconds); inside it is what Slack would post to + slackOutgoingWebhook. Slack closes a socket now and then, saying so first (disconnect), and another is opened at + once.""" network = inbound.SLACK PING_SECONDS = 30 # silence this long sends a ping; as long again after it, and the socket is taken for dead - TAKEN = ('events_api', 'interactive') # the envelopes whose payload goes to the reader + TAKEN = ('events_api', 'interactive', 'slash_commands') # the envelopes whose payload goes to the reader def describe(self): """The mode, for the log.""" diff --git a/lib/telegram.py b/lib/telegram.py index 5392687e..c371e2f0 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -48,6 +48,14 @@ def register(): setWebhook() return '@' + getMe()['username'] +def set_commands(commands): + """Make the bot's command menu, in every chat, these slash commands ([(name, description, arguments)], from + worker.slash_commands), each with its usage, where Telegram's differs. A failure is only logged: typed, a command + works without the menu.""" + wanted = [{'command': name, 'description': description[:256]} for name, description, _ in commands] + if try_call('getMyCommands') != wanted and try_call('setMyCommands', json={'commands': wanted}) is not None: + print(f"telegram: {len(wanted)} commands in the menu", file=sys.stderr) + def pinChatMessage(chat_id, message_id): """Pin a message, without the "pinned a message" notice. A failure is only logged: the message is there anyway.""" try_call('pinChatMessage', json={'chat_id': chat_id, 'message_id': message_id, 'disable_notification': True}) diff --git a/lib/worker.py b/lib/worker.py index 81e35e53..18429f42 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -478,35 +478,44 @@ def help_sections(slash=False): """The sections of .help: (heading, the .help lines of its commands, in table order), in SECTIONS order; with slash, each line as its slash command (/forwardpe for .forward pe, /stock SYMBOL for .SYMBOL).""" def shown(command, line): - """The line, or its first slash command.""" - forms = slash_forms(command, line) if slash else [] - return f'/{forms[0][0]} {forms[0][1]}'.strip() if forms else line + """The line, or its slash command.""" + form = slash_form(command, line) if slash else None + return f'/{form[0]} {form[1]}'.strip() if form else line return [(section, list(dict.fromkeys(shown(command, line) for command in COMMANDS if command.section == section for line in command.usage))) for section in SECTIONS] -SLASH_NAME = re.compile(r'[a-z0-9_-]{1,32}') # what a slash command may be named, on every network +SLASH_NAME = re.compile(r'[a-z0-9_]{1,32}') # what a slash command may be named, on every network +SLASH_ALIASES = {'stock': 'profile', 'jobs': 'job'} # slash commands with a second name, which the networks that list +# every slash command list too (Discord, Slack); typed, the second name works on Telegram too, as every name does -def slash_forms(command, line): - """The slash commands of one of a command's .help lines: [(name, the arguments it shows)]. One, named by the - command's name the line starts with where a slash command may be so named, else by the command's first name that it - may (.forward pe is /forwardpe); a line that starts with no name (.SYMBOL) has one under each of them.""" +def slash_form(command, line): + """The slash command of one of a command's .help lines, (name, the arguments it shows): named by the command's name + the line starts with where a slash command may be so named, else by the command's first name that may (.forward pe + is /forwardpe, .SYMBOL /stock SYMBOL); None when none may.""" valid = [name for name in command.names if SLASH_NAME.fullmatch(name)] + if not valid: + return None words = line.removeprefix('.').split() named = next((name for name in command.names if [w.lower() for w in words[:len(name.split())]] == name.split()), None) if named is None: - return [(name, ' '.join(words)) for name in valid] - return [(named if named in valid else valid[0], ' '.join(words[len(named.split()):]))] if valid else [] - -def slash_commands(): - """The slash commands to register, each with its description and the arguments it shows: [(name, description, - arguments)], from the .help lines (slash_forms), and /help.""" - found = {'help': ('help', 'every command, and what it takes', '')} + return valid[0], ' '.join(words) + return (named if named in valid else valid[0]), ' '.join(words[len(named.split()):]) + +def slash_commands(aliases=True): + """The slash commands to register, from the .help lines (slash_form), and /help: [(name, description, the + arguments it shows)], the arguments of a command's several lines joined by " | ". With aliases, the second names of + SLASH_ALIASES too, each after its first.""" + found = {'help': []} # name -> the arguments of its lines for command in COMMANDS: for line in command.usage: - for name, arguments in slash_forms(command, line): - found.setdefault(name, (name, f'{name} {arguments}'.strip(), arguments)) - return list(found.values()) + if form := slash_form(command, line): + name, arguments = form + for each in (name, SLASH_ALIASES.get(name) if aliases else None): + if each and arguments not in found.setdefault(each, []): + found[each].append(arguments) + return [('help', 'every command, and what it takes', '') if name == 'help' else + (name, f"{name} {' | '.join(lines)}".strip(), ' | '.join(lines)) for name, lines in found.items()] def command_keyboard(per_row=3): """Persistent reply keyboard of the commands that have buttons (Telegram DMs), alphabetical, row by row; a press diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 133cd55c..55979c98 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -1,8 +1,9 @@ -"""What reaches the bot: webhook requests turned away or taken, Slack's events, presses and signatures, -and user text escaped where it arrives.""" -import contextlib, hashlib, hmac, io, json, time, unittest +"""What reaches the bot: webhook requests turned away or taken, Slack's events, presses, slash commands and signatures, +the Slack app's manifest, and user text escaped where it arrives.""" +import contextlib, hashlib, hmac, io, json, re, time, unittest from urllib.parse import urlencode from tests.common import * +from util import slack_manifest PRESS = {'type': 'block_actions', 'token': 'stok', 'trigger_id': 't1', 'user': {'id': 'U1', 'username': 'rob'}, @@ -335,6 +336,102 @@ def test_unsigned_wrongly_signed_old_or_altered_requests_are_refused(self): self.assertEqual((self.requests, self.errors), ([], [])) +SLASH = {'token': 'stok', 'team_id': 'T1', 'channel_id': 'C1', 'channel_name': 'stocks', 'user_id': 'U1', + 'user_name': 'rob', 'command': '/price', 'text': '5 & 7d', 'response_url': 'https://hooks.slack.com/r/1', + 'trigger_id': 't9', 'api_app_id': 'A1'} # a Slack slash command, as its form's fields or a socket's payload + + +class SlackSlashCommands(FinbotCase): + """A Slack slash command posted to slackOutgoingWebhook as a form, signed as an event is.""" + + def setUp(self): + """Slack in HTTP mode with a signing secret, and the worker replaced by a list of what it would have run.""" + super().setUp() + self.env(slackOutgoingWebhook='https://example.com/slack', slackSigningSecret='sss') + self.requests = [] + self.patch(worker, 'process_request', self.requests.append) + + def post(self, fields, secret='sss'): + """Post the form of a slash command, signed with secret: (the status, the body).""" + body = urlencode(fields).encode() + timestamp = str(int(time.time())) + mac = hmac.new(secret.encode(), b'v0:' + timestamp.encode() + b':' + body, hashlib.sha256).hexdigest() + env = wsgi_environ('/slack', body, HTTP_X_SLACK_REQUEST_TIMESTAMP=timestamp, HTTP_X_SLACK_SIGNATURE='v0=' + mac) + status = [] + with contextlib.redirect_stderr(io.StringIO()): + answer = b''.join(bot.main(env, lambda s, *a, **k: status.append(s))) + self.join_workers() + return status[-1], answer + + def test_a_slash_command_is_the_message_that_says_the_same(self): + """/price with 5 & 7d is the message /price 5 & 7d, from <@U1>, in the channel or DM it was sent in, which + the reply goes to; Slack gets an empty 200 at once. There is no message of it, so no ⏳.""" + self.assertEqual(self.post(SLASH), ('200 OK', b'')) + self.assertEqual(self.post({**SLASH, 'channel_id': 'D1', 'command': '/job', 'text': ''}), ('200 OK', b'')) + self.assertEqual(self.requests, [inbound.Message('slack', 'C1', '<@U1>', '/price 5 & 7d', '', '<@U1>'), + inbound.Message('slack', 'D1', '<@U1>', '/job', '', '<@U1>')]) + self.assertEqual([worker.read(r.text, r.bot_name)[0].names[0] for r in self.requests], ['price', 'jobs']) + self.assertFalse(delivery.reply_to('slack', 'C1', self.requests[0].message_id).shows_typing) + + def test_a_slash_command_unsigned_or_altered_is_refused(self): + """A slash command signed with another secret is refused, and none reaches the worker.""" + self.assertEqual(self.post(SLASH, secret='wrong')[0], '403 Forbidden') + self.assertEqual(self.requests, []) + + +class SlackManifest(unittest.TestCase): + """util/slack_manifest.py: the Slack app's manifest, with the slash commands of the command table.""" + + def test_socket_mode_or_http_mode(self): + """Without a URL, Socket Mode and no Request URL anywhere; with one, HTTP mode and that URL for the events, the + buttons and every slash command. Either way: the messages tab for DMs, interactivity, and the scopes that the + replies, the slash commands, the charts, ⏳ and each event need.""" + socket = slack_manifest.manifest() + self.assertTrue(socket['settings']['socket_mode_enabled']) + self.assertNotIn('request_url', json.dumps(socket)) + self.assertNotIn('"url"', json.dumps(socket)) + url = 'https://www.example.com:8443/slack' + http = slack_manifest.manifest(url=url) + self.assertFalse(http['settings']['socket_mode_enabled']) + self.assertEqual((http['settings']['event_subscriptions']['request_url'], + http['settings']['interactivity']['request_url']), (url, url)) + self.assertTrue(all(command['url'] == url for command in http['features']['slash_commands'])) + for manifest in (socket, http): + self.assertTrue(manifest['features']['app_home']['messages_tab_enabled']) + self.assertTrue(manifest['settings']['interactivity']['is_enabled']) + self.assertEqual(manifest['settings']['event_subscriptions']['bot_events'], ['app_mention', 'message.im']) + self.assertEqual(manifest['oauth_config']['scopes']['bot'], ['app_mentions:read', 'chat:write', 'commands', + 'files:read', 'files:write', 'im:history', 'reactions:write']) + + def test_every_message_and_the_slash_commands(self): + """With every_message the bot hears every message in its channels (not only mentions, which it would then + answer twice). The slash commands are the command table's, second names included, each named and described + as Slack allows, and each reads as its command.""" + manifest = slack_manifest.manifest(every_message=True) + self.assertEqual(manifest['settings']['event_subscriptions']['bot_events'], + ['message.channels', 'message.groups', 'message.im']) + self.assertIn('channels:history', manifest['oauth_config']['scopes']['bot']) + self.assertNotIn('app_mentions:read', manifest['oauth_config']['scopes']['bot']) + commands = manifest['features']['slash_commands'] + self.assertEqual([c['command'] for c in commands], ['/' + name for name, _, _ in worker.slash_commands()]) + self.assertTrue({'/profile', '/job', '/forwardpe'} <= {c['command'] for c in commands}) + for command in commands: + with self.subTest(command=command['command']): + self.assertRegex(command['command'], r'^/[a-z0-9_]{1,31}$') + self.assertTrue(0 < len(command['description']) <= 2000) + self.assertLessEqual(len(command.get('usage_hint', '')), 1000) + self.assertIsNotNone(worker.read(f"{command['command']} {command.get('usage_hint', '')}", '')[0]) + + def test_the_tool_prints_json_and_refuses_a_plain_http_url(self): + """The tool prints the manifest as JSON, to paste; a URL that is not HTTPS is refused, as Slack would.""" + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(slack_manifest.main(['--name', 'stockbot']), 0) + self.assertEqual(json.loads(out.getvalue())['features']['bot_user']['display_name'], 'stockbot') + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(slack_manifest.main(['--url', 'http://www.example.com/slack']), 2) + self.assertIn('--url must be https://', err.getvalue()) + + class UserTextIsEscaped(FinbotCase): """Replies go out as Telegram HTML: text a user wrote is shown, never read as markup. One stray < in a .plan entry made Telegram refuse that reply and every .plan reply after it.""" diff --git a/tests/test_pull.py b/tests/test_pull.py index fbe14316..c1664e7f 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -238,6 +238,23 @@ def test_a_press_over_the_socket_is_the_press_the_webhook_gets(self): self.assertEqual(presses[0], presses[1]) self.assertEqual([m.text for m in messages], ['.price']) + def test_a_slash_command_over_the_socket_is_the_message_the_webhook_gets(self): + """The same slash command, posted to slackOutgoingWebhook as a form (http mode) or sent over the socket (its + envelope acked first), reaches the worker as the same Message.""" + messages = [] + self.patch(worker, 'process_request', messages.append) + slash = {'token': 'stok', 'channel_id': 'C1', 'user_id': 'U1', 'user_name': 'rob', 'command': '/forwardpe', + 'text': 'top', 'trigger_id': 't9'} + self.env(slackOutgoingWebhook='https://example.com/slack', slackOutgoingToken='stok') + with contextlib.redirect_stderr(io.StringIO()): + bot.main(wsgi_environ('/slack', urlencode(slash).encode()), lambda *a: None) + self.env(slackOutgoingWebhook='') + self.socket({'type': 'hello'}, {'envelope_id': 'e1', 'type': 'slash_commands', 'payload': slash}, + {'type': 'disconnect', 'reason': 'warning'}).poll(bot.take) + self.join_workers() + self.assertEqual(messages, [inbound.Message('slack', 'C1', '<@U1>', '/forwardpe top', '', '<@U1>')] * 2) + self.assertIn(('ack', 'e1'), self.log) + def test_the_websocket_is_imported_only_to_connect_and_read_as_slack_needs(self): """websocket-client is imported when a socket is opened, not before; its frames read as text, '' for a pong, None for a timeout, and a close raises ConnectionError.""" diff --git a/tests/test_telegram_api.py b/tests/test_telegram_api.py index 27691312..eb56bf97 100644 --- a/tests/test_telegram_api.py +++ b/tests/test_telegram_api.py @@ -1,5 +1,6 @@ -"""Telegram's Bot API: the webhook set or deleted once, as bot.py starts.""" -import contextlib, io, os, subprocess, unittest +"""Telegram's Bot API: the webhook set or deleted once, as bot.py starts, and the command menu.""" +import contextlib, io, os, re, subprocess, unittest +from unittest import mock from tests.common import * @@ -85,6 +86,32 @@ def test_importing_never_calls_telegram(self): 'telegramOutgoingWebhook': 'https://example.com/telegram'}) self.assertEqual(out.stdout.split()[-1:], ['quiet'], out.stderr) + def test_the_command_menu_is_set_only_when_it_differs(self): + """The menu is the slash commands, each with its usage, without their second names (/profile, /job: typed, they + work as any name does), set only when Telegram's menu differs; a failure is only logged.""" + calls, menu = [], [] + def call(method, **kwargs): + """Telegram's menu, as getMyCommands and setMyCommands see it.""" + calls.append(method) + if method == 'setMyCommands': + menu[:] = kwargs['json']['commands'] + return True + return list(menu) + self.patch(telegram, 'call', call) + with contextlib.redirect_stderr(io.StringIO()) as log: + telegram.set_commands(worker.slash_commands(aliases=False)) + telegram.set_commands(worker.slash_commands(aliases=False)) + self.assertEqual(calls, ['getMyCommands', 'setMyCommands', 'getMyCommands']) + self.assertIn(f'telegram: {len(menu)} commands in the menu', log.getvalue()) + names = [entry['command'] for entry in menu] + self.assertIn({'command': 'forwardpe', 'description': 'forwardpe [SYMBOL|top|bottom|negative]'}, menu) + self.assertEqual([name in names for name in ('stock', 'jobs', 'profile', 'job')], [True, True, False, False]) + self.assertTrue(all(re.fullmatch(r'[a-z0-9_]{1,32}', name) for name in names)) + self.patch(telegram, 'call', mock.Mock(side_effect=RuntimeError('Telegram getMyCommands: Unauthorized'))) + with contextlib.redirect_stderr(io.StringIO()) as log: + telegram.set_commands(worker.slash_commands(aliases=False)) + self.assertIn('Telegram setMyCommands failed: Telegram getMyCommands: Unauthorized', log.getvalue()) + def test_the_bot_will_not_start_unregistered_and_never_logs_the_token(self): """bot.py exits when the webhook cannot be registered, and never logs the token.""" skip_if_old_env(self) diff --git a/util/slack_manifest.py b/util/slack_manifest.py new file mode 100755 index 00000000..f52604fa --- /dev/null +++ b/util/slack_manifest.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Print the manifest of finbot's Slack app, to paste into the app's settings (_App Manifest_, the JSON tab), which sets +it all up at once: the slash commands, from finbot's own command table; the bot's scopes and events; interactivity, for +the buttons; the messages tab of App Home, for direct messages; and Socket Mode, on or off. + + util/slack_manifest.py Socket Mode: finbot opens a socket to Slack (slackAppToken) + util/slack_manifest.py --url https://example.com/slack HTTP mode: Slack posts to slackOutgoingWebhook, this URL + util/slack_manifest.py --every-message the bot sees every message in its channels, not only the + ones that mention it, so that .price works there + +Slack makes slash commands only in an app's settings (there is no API for them without a configuration token), so this +is the way they get there: run it again, and paste what it prints, when finbot's commands change.""" + +import argparse +import json +import os +import sys +from pathlib import Path + +if __name__ == '__main__': + os.environ['FINBOT_CHECK_ONLY'] = '1' # importing lib.config must not create the data directories +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from lib import worker + +SCOPES = ('chat:write', 'commands', 'files:read', 'files:write', 'reactions:write') # replies, slash commands, charts, ⏳ +EVENT_SCOPES = {'app_mention': 'app_mentions:read', 'message.channels': 'channels:history', + 'message.groups': 'groups:history', 'message.im': 'im:history'} # what each event needs + + +def manifest(name='finbot', url='', every_message=False): + """The manifest, as a dict: for Socket Mode without url; for HTTP mode with it, the Request URL of the events, the + buttons and each slash command. The bot hears the messages that mention it, or with every_message each message in + the channels it is in, and its direct messages either way.""" + events = ['message.channels', 'message.groups'] if every_message else ['app_mention'] + events.append('message.im') + posted = {'url': url} if url else {} + commands = [{'command': '/' + command, 'description': description, 'should_escape': False, + **({'usage_hint': arguments} if arguments else {}), **posted} + for command, description, arguments in worker.slash_commands()] + return { + 'display_information': {'name': name, 'description': 'Stock prices, charts and Sharesight reports', + 'background_color': '#14181f'}, + 'features': { + 'app_home': {'home_tab_enabled': False, 'messages_tab_enabled': True, + 'messages_tab_read_only_enabled': False}, + 'bot_user': {'display_name': name, 'always_online': True}, + 'slash_commands': commands, + }, + 'oauth_config': {'scopes': {'bot': sorted({*SCOPES, *(EVENT_SCOPES[event] for event in events)})}}, + 'settings': { + 'event_subscriptions': {'bot_events': events, **({'request_url': url} if url else {})}, + 'interactivity': {'is_enabled': True, **({'request_url': url} if url else {})}, + 'org_deploy_enabled': False, + 'socket_mode_enabled': not url, + 'token_rotation_enabled': False, + }, + } + + +def main(argv=None): + """Print the manifest the arguments ask for; 2 for a URL that is not HTTPS, which Slack takes no other.""" + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('--url', default='', help='HTTP mode: slackOutgoingWebhook, which Slack posts to') + parser.add_argument('--name', default='finbot', help="the app's and the bot's name (default finbot)") + parser.add_argument('--every-message', action='store_true', help='the bot sees every message in its channels') + args = parser.parse_args(argv) + if args.url and not args.url.startswith('https://'): + print('slack_manifest: --url must be https://: Slack posts to no other', file=sys.stderr) + return 2 + print(json.dumps(manifest(args.name, args.url, args.every_message), indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From 68f1113f36c3bf6f866fcf180fb28b260cea074e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:28:39 +0000 Subject: [PATCH 14/31] Discord autocomplete for slash command arguments, in both modes A slash command whose arguments have words to suggest registers its option with autocomplete. While the arguments are typed, Discord's autocomplete interaction is read as a Suggest and answered at once with worker.suggest's choices (type 8, up to 25, each up to 100 characters): on the interactions endpoint as the response, over the gateway through the interaction's callback (an interaction's answer now says so, inbound.Response.interaction). worker.suggest completes the word being typed from the command table: the keywords and periods (until one is given), the portfolios, and tickers (the chat's watchlist and the holdings, read from the cache alone by sharesight.cached, never a request); for /jobs its subcommands, this chat's job numbers with their lines, the reports after add and a time, and what a report takes. A failure suggests nothing and is logged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 4 +- README.md | 1 + bot.py | 2 + doc/discord.md | 13 ++++- lib/discord.py | 7 ++- lib/inbound.py | 37 +++++++++++-- lib/pull.py | 20 ++++--- lib/sharesight.py | 24 +++++++- lib/telegram.py | 2 +- lib/worker.py | 104 ++++++++++++++++++++++++++++++++--- tests/test_discord.py | 121 +++++++++++++++++++++++++++++++++++++++-- tests/test_inbound.py | 2 +- util/slack_manifest.py | 6 +- 13 files changed, 301 insertions(+), 42 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1b7bb673..0930a494 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,7 +51,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm out) and the slash commands registered at start (`register`, from `worker.slash_commands()`). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, keywords)`, which also makes `.help`, the Telegram command keyboard, kept current in DMs, and `slash_commands()`, which - every network gets (Telegram's menu, Slack's manifest from `util/slack_manifest.py`, Discord's registered commands). `read()` finds the + every network gets (Telegram's menu, Slack's manifest from `util/slack_manifest.py`, Discord's registered commands); + `suggest(chat, text)` completes a slash command's arguments as they are typed (Discord's autocomplete), from the + cache alone. `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a percent, a period and a keyword from either end, the words between; a handler raises `Usage` for arguments it cannot take, and gets a ticker with `ticker(args)`. `Request` (`request.reply(lines, chart, buttons)`); each command gets diff --git a/README.md b/README.md index 83b04f56..7d59e655 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ What finbot can do on each network. Telegram takes everything; the others take w | "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots); none for a slash command, which has no message | ✓ for a message; "thinking…" for a slash command | | Charts in replies | ✓ | ✓ | ✓ | | Charts in scheduled reports | ✓ | ✓ | ✓ | +| Slash commands completed as they are typed | names, from the command menu | names, with their usage | names, and the arguments: keywords, periods, portfolios, tickers, jobs | | Period and job buttons | ✓ | ✓ | ✓ | | No link previews | ✓ | ✓ | ✓ | | Linked tickers | ✓ | ✓ | ✓ | diff --git a/bot.py b/bot.py index f81a0460..0d6ac7fc 100755 --- a/bot.py +++ b/bot.py @@ -129,6 +129,8 @@ def take(network, update): event = network.parse(update) if isinstance(event, inbound.Response): return event + if isinstance(event, inbound.Suggest): # answered here and at once, while the command is typed + return network.suggestions(worker.suggest(f'{event.service}:{event.chat_id}', event.text)) if seen_before(network.service, network.update_id(update)): print(f"[{network.service}]: update {network.update_id(update)} seen already, not run again", file=sys.stderr) return inbound.OK diff --git a/doc/discord.md b/doc/discord.md index 4f5184e0..8603d878 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -95,8 +95,17 @@ discordPublicKey=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcde finbot registers a slash command for each line of `.help`, as it starts, and again only when the list has changed: `/price`, `/dividend` and `/earnings`, `/buy` and `/sell`, `/forwardpe` for `.forward pe`, `/stock` (or `/profile`) for -`.SYMBOL`, `/jobs` (or `/job`), and the rest. Each takes its arguments, if any, as one text option, as the message would: `/price` with -`5 7d` is `.price 5 7d`. They work in the server's channels and in a direct message with the bot. +`.SYMBOL`, `/jobs` (or `/job`), and the rest. Each takes its arguments, if any, as one text option, as the message +would: `/price` with `5 7d` is `.price 5 7d`. They work in the server's channels and in a direct message with the bot. + +While the arguments are typed, Discord shows finbot's suggestions, in both modes, for the word being typed: +* the command's keywords (`top`, `bottom`, `negative`, `sector`, `add`, `del`) and periods (`7d`, `3m`, `ytd`), until + it has one; +* the portfolios, for the commands that take one; +* tickers, for the commands that take one: the channel's watchlist and the holdings. They come from the cache, never + from Sharesight or Yahoo, so the suggestions come at once; the holdings are there once `sharesight_warm` has run; +* for `/jobs`: `run`, `toggle`, `add` and `del`; after `run`, `toggle` or `del` the channel's job numbers, each with + its line; after `add` and a time, the reports, each with what it takes, and then what the report takes. A message holds up to 2,000 characters, so a long report arrives as several messages. A chart goes as an attachment of the message that carries its report, with its period buttons under it. diff --git a/lib/discord.py b/lib/discord.py index 4c6a7891..ea42a787 100644 --- a/lib/discord.py +++ b/lib/discord.py @@ -58,10 +58,11 @@ def reason(response): return response.text[:200] -def command(name, description, arguments): +def command(name, description, arguments, completes=False): """A slash command as Discord takes it: a chat command usable in servers and in a DM with the bot, and the arguments - it takes, if any, as one optional text option that they describe.""" - option = {'type': 3, 'name': OPTION, 'description': arguments[:100], 'required': False} + it takes, if any, as one optional text option that they describe; with completes, Discord asks finbot what to + suggest as they are typed (an autocomplete interaction).""" + option = {'type': 3, 'name': OPTION, 'description': arguments[:100], 'required': False, 'autocomplete': completes} return {'name': name, 'type': 1, 'description': description[:100], 'contexts': [GUILD, BOT_DM], 'integration_types': [0], 'options': [option] if arguments else []} diff --git a/lib/inbound.py b/lib/inbound.py index 05c9a350..35d74d46 100644 --- a/lib/inbound.py +++ b/lib/inbound.py @@ -44,13 +44,26 @@ class Press: interaction: str = '' # Discord's press: its application id and token, to edit the message pressed through +@dataclasses.dataclass +class Suggest: + """A slash command's arguments as they are typed (Discord's autocomplete), to answer at once with what to suggest: + the message they make so far (/price 5 7), which the suggestions complete.""" + service: str + chat_id: str + text: str + interaction: str = '' # Discord's interaction: its application id and token + + @dataclasses.dataclass class Response: """What the network gets back at once: a status and a body; and the work to start once it has them (`then`), when - the network must have this answer before any reply (Discord's deferred answer to an interaction).""" + the network must have this answer before any reply (Discord's deferred answer to an interaction). An interaction's + answer (`interaction`) is the HTTP response on Discord's interactions endpoint, and goes to the interaction's + callback when it came over the gateway.""" status: str = '200 OK' body: bytes = b'' then: object = dataclasses.field(default=None, compare=False) + interaction: bool = False OK = Response() @@ -212,8 +225,9 @@ class Discord(Network): interactions endpoint and the gateway, and the gateway's messages.""" service = 'discord' bot_id = None # the bot's user id, from discord.register() as bot.py starts: a message mentioning it is for the bot - PING, COMMAND, COMPONENT = 1, 2, 3 # interaction types - PONG, THINKING, LATER = 1, 5, 6 # answers: to the check, "thinking…" until the reply, nothing to see (a press) + PING, COMMAND, COMPONENT, AUTOCOMPLETE = 1, 2, 3, 4 # interaction types + PONG, THINKING, LATER, CHOICES = 1, 5, 6, 8 # answers: to the check, "thinking…" until the reply, nothing to see (a + # press), the suggestions for what is being typed def update_id(self, update): """The interaction's id, or the message's.""" @@ -230,10 +244,11 @@ def bot_name(self): def interaction(self, update): """The endpoint's check (a PING, answered at once), a slash command as the message it stands for (/price 7d), - or a button pressed under one of the bot's messages; anything else, UNSUPPORTED.""" + its arguments as they are typed (a Suggest), or a button pressed under one of the bot's messages; anything else, + UNSUPPORTED.""" kind = update.get('type') if kind == self.PING: - return Response(body=json.dumps({'type': self.PONG}).encode()) + return Response(body=json.dumps({'type': self.PONG}).encode(), interaction=True) person = (update.get('member') or {}).get('user') or update.get('user') or {} channel = str(update.get('channel_id') or (update.get('channel') or {}).get('id') or '') through = f"{update.get('application_id')}/{update['token']}" @@ -243,6 +258,10 @@ def interaction(self, update): text = f"/{data['name']} {given}".strip() return Message(self.service, channel, f"<@{person.get('id')}>", text, self.bot_name, self.name(person), interaction=through) + if kind == self.AUTOCOMPLETE and data.get('name'): + options = data.get('options') or [] + typed = next((str(option.get('value', '')) for option in options if option.get('focused')), '') + return Suggest(self.service, channel, f"/{data['name']} {typed}", through) message = update.get('message') or {} if kind == self.COMPONENT and data.get('custom_id') and message.get('id'): return Press(self.service, str(update.get('id')), channel, str(message['id']), data['custom_id'], @@ -271,7 +290,13 @@ def acknowledge(self, event): if not event.interaction: return None answer = self.THINKING if isinstance(event, Message) else self.LATER - return Response(body=json.dumps({'type': answer}).encode()) + return Response(body=json.dumps({'type': answer}).encode(), interaction=True) + + def suggestions(self, choices): + """The answer to an autocomplete interaction: the choices, (name shown, value) pairs, 25 at most.""" + choices = [{'name': name, 'value': value} for name, value in choices[:25]] + return Response(body=json.dumps({'type': self.CHOICES, 'data': {'choices': choices}}).encode(), + interaction=True) class Webhook: diff --git a/lib/pull.py b/lib/pull.py index abf3cdb1..cb46a53a 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -53,20 +53,21 @@ def failed(self, what, e): return wait def hand_on(self, take, update): - """take(network, update), and the answer delivered where the network must have it before any reply. A failure - is reported as bot.main reports one of an authenticated request: the update came from the network, so the - default chats are told.""" + """take(network, update), and its answer delivered where the network must have it: an interaction's, and one + that work waits on. A failure is reported as bot.main reports one of an authenticated request: the update came + from the network, so the default chats are told.""" try: answer = take(self.network, update) - if getattr(answer, 'then', None): + if getattr(answer, 'interaction', False) or getattr(answer, 'then', None): self.answer(update, answer) except Exception as e: delivery.report_error(e, context=self.network.service) def answer(self, update, answer): - """Deliver an answer that the network must have before any reply, then start the work waiting on it. Only - Discord's interactions have one (DiscordGateway).""" - answer.then() + """Deliver an answer that the network must have (only Discord's interactions have one: DiscordGateway), then + start the work waiting on it, if any.""" + if answer.then: + answer.then() class TelegramPoll(Listener): @@ -263,9 +264,10 @@ def receive(self, raw, take): threading.Thread(target=self.hand_on, args=(take, message['d']), daemon=True).start() def answer(self, update, answer): - """Post an interaction's answer to its callback, then start the work waiting on it.""" + """Post an interaction's answer to its callback, then start the work waiting on it, if any.""" discord.call('POST', f"/interactions/{update['id']}/{update['token']}/callback", json=json.loads(answer.body)) - answer.then() + if answer.then: + answer.then() def listeners(): diff --git a/lib/sharesight.py b/lib/sharesight.py index 79960a6f..190e3f89 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -139,13 +139,18 @@ def trades(self, days=None): return [trade | {'portfolio': self.name} for trade in _trades(self.id, days)] +def _chosen(listed): + """The portfolios of Sharesight's list to report on, after include_portfolios and exclude_portfolios.""" + include, exclude = config.prefs.include_portfolios, config.prefs.exclude_portfolios + return [Portfolio(each['name'], each['id']) for each in listed + if str(each['id']) not in exclude and (not include or str(each['id']) in include)] + + def portfolios(select=None, seconds=None): """The portfolios to report on (after include_portfolios and exclude_portfolios), or the one called `select`, in any case; the list is cached for `seconds` (cache_seconds when None). Raises RuntimeError when there are none, and PortfolioNotFound for a name that is none of them.""" - include, exclude = config.prefs.include_portfolios, config.prefs.exclude_portfolios - found = [Portfolio(each['name'], each['id']) for each in _portfolios(seconds) - if str(each['id']) not in exclude and (not include or str(each['id']) in include)] + found = _chosen(_portfolios(seconds)) if not found: raise RuntimeError("No Sharesight portfolios found") if not select: @@ -161,6 +166,19 @@ def tickers(): return sorted({ticker for portfolio in portfolios() for ticker in portfolio.tickers()}) +def cached(): + """(the portfolios' names, the Yahoo tickers they hold, sorted) from what the cache holds, whatever its age, and + never a request: for suggestions while a command is typed, which must come at once. Empty where nothing is + cached.""" + found = _chosen((cache.load(PORTFOLIOS_CACHE) or {}).get('portfolios') or []) + held = set() + for portfolio in found: + report = (cache.load(performance_cache_file(portfolio.id, 0)) or {}).get('report') or {} + held |= {markets.transform_to_yahoo(each['instrument']['code'], each['instrument']['market_code']) + for each in report.get('holdings') or []} + return [portfolio.name for portfolio in found], sorted(held) + + def warm(): """Fetch afresh the portfolio list and each portfolio's holdings report (days=0), so that a request never waits on Sharesight (scheduled: sharesight_warm.py). Returns the portfolios. Raises as the requests do, and then the caches diff --git a/lib/telegram.py b/lib/telegram.py index c371e2f0..5633f727 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -52,7 +52,7 @@ def set_commands(commands): """Make the bot's command menu, in every chat, these slash commands ([(name, description, arguments)], from worker.slash_commands), each with its usage, where Telegram's differs. A failure is only logged: typed, a command works without the menu.""" - wanted = [{'command': name, 'description': description[:256]} for name, description, _ in commands] + wanted = [{'command': command.name, 'description': command.description[:256]} for command in commands] if try_call('getMyCommands') != wanted and try_call('setMyCommands', json={'commands': wanted}) is not None: print(f"telegram: {len(wanted)} commands in the menu", file=sys.stderr) diff --git a/lib/worker.py b/lib/worker.py index 18429f42..8f6d7adc 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -502,20 +502,108 @@ def slash_form(command, line): return valid[0], ' '.join(words) return (named if named in valid else valid[0]), ' '.join(words[len(named.split()):]) +class Slash(typing.NamedTuple): + """A slash command to register: its name, its description, the arguments it shows, and whether Discord asks for + suggestions while they are typed (suggest).""" + name: str + description: str + arguments: str + completes: bool = False + def slash_commands(aliases=True): - """The slash commands to register, from the .help lines (slash_form), and /help: [(name, description, the - arguments it shows)], the arguments of a command's several lines joined by " | ". With aliases, the second names of - SLASH_ALIASES too, each after its first.""" - found = {'help': []} # name -> the arguments of its lines + """The slash commands to register (Slash), from the .help lines (slash_form), and /help; the arguments of a + command's several lines joined by " | ". With aliases, the second names of SLASH_ALIASES too, each after its + first.""" + found = {'help': (None, [])} # name -> (its command, the arguments of its lines) for command in COMMANDS: for line in command.usage: if form := slash_form(command, line): name, arguments = form for each in (name, SLASH_ALIASES.get(name) if aliases else None): - if each and arguments not in found.setdefault(each, []): - found[each].append(arguments) - return [('help', 'every command, and what it takes', '') if name == 'help' else - (name, f"{name} {' | '.join(lines)}".strip(), ' | '.join(lines)) for name, lines in found.items()] + if each and arguments not in found.setdefault(each, (command, []))[1]: + found[each][1].append(arguments) + return [Slash('help', 'every command, and what it takes', '') if name == 'help' else + Slash(name, f"{name} {' | '.join(lines)}".strip(), ' | '.join(lines), completes(command)) + for name, (command, lines) in found.items()] + +PERIOD_WORDS = ('1d', '7d', '1m', '3m', '6m', 'ytd', '1y', '5y', '10y') # suggested where a command takes a period +JOB_WORDS = ('run', 'toggle', 'add', 'del') # what /jobs suggests first + +def completes(command): + """Whether suggest has words to offer for a command's arguments: its keywords, periods, tickers, portfolios, or + those of /jobs.""" + usage = ' '.join(command.usage) + return bool(command.handler is on_jobs or command.keywords or command.period or 'SYMBOL' in usage or + 'portfolio' in usage.lower()) + +def suggest(chat, text): + """What to suggest while a slash command's arguments are typed (Discord's autocomplete), from the message they make + so far (text: /price 5 7) in a chat (its key): up to 25 (name shown, value) pairs, each value the arguments with the + word being typed completed. The words come from the command's entry in the table: its keywords, periods, portfolios + and tickers (the chat's watchlist and the holdings, as cached, never a request: suggestions must come at once); for + /jobs, its subcommands, this chat's job numbers and the reports. Never raises: the failure is logged instead, and + nothing suggested.""" + try: + command, _ = read(text, '') + if command is None: + return [] + given = (text.split(None, 1) + [''])[1] + words = given.split() + typing = words.pop() if words and not given[-1].isspace() else '' + done = ' '.join(words) + if command.handler is on_jobs: + offered = job_words(chat, words) + else: + offered = argument_words(command, chat, read_args(command, '', done)) + found = [(f'{done} {shown}'.strip()[:100], f'{done} {word}'.strip()) for word, shown in offered + if word.lower().startswith(typing.lower())] + return [(shown, value) for shown, value in found if len(value) <= 100][:25] + except Exception as e: + print('suggest:', text, delivery.error_line(e), file=sys.stderr) + return [] + +def argument_words(command, chat, given): + """(word, shown) pairs that a command's arguments may hold next, for suggest, after those given (Args): the + keywords its usage shows and periods, unless it has one already; portfolios; tickers (the chat's watchlist and the + holdings, as cached).""" + usage = ' '.join(command.usage) + words = [word for word in re.findall(r'[\w-]+', usage) + if word in command.keywords and word not in command.names and given.keyword is None] + if command.period and given.days is None: + words += PERIOD_WORDS + if 'SYMBOL' in usage or 'portfolio' in usage.lower(): + names, held = sharesight.cached() + if 'portfolio' in usage.lower(): + words += names + if 'SYMBOL' in usage: + words += sorted(set(watchlist.tickers(chat)) | set(held)) + return [(word, word) for word in dict.fromkeys(words)] + +def job_words(chat, words): + """(word, shown) pairs for /jobs, for suggest, after the words given: first its subcommands; after run, toggle or + del, this chat's job numbers, each shown with its line; after add and a time, the reports, each shown with what it + takes; after a report's name, the words it takes.""" + if not words: + return [(word, word) for word in JOB_WORDS] + action = JOB_CHANGES.get(words[0].lower()) + if action in ('run', 'toggle', 'del') and len(words) == 1: + own = jobs.current.chat_jobs(chat) if jobs.current else [] + return [(str(number), f'{number}: {job.line}') for number, job in enumerate(own, 1)] + if action != 'add': + return [] + line = words[1:][1:] if words[1:2] and words[1].startswith('CRON_TZ=') else words[1:] + time = 1 if line and line[0].startswith('@') else 5 + if len(line) < time: + return [] + after = line[time:] + if not after or after[-1].endswith(';'): # a report's name is next + return [(name, f'{name} {module.ARGUMENTS}'.strip()) for name, module in sorted(jobs.SCRIPTS.items()) + if name not in jobs.SYSTEM_SCRIPTS] + start = max(i for i in range(len(after)) if i == 0 or after[i - 1].endswith(';')) # the report being given + module = jobs.SCRIPTS.get(after[start].rstrip(';')) + if module is None or len(after) - start > 1 or after[start].endswith(';'): + return [] + return [(word, word) for word in re.findall(r'[a-z][a-z-]*', module.ARGUMENTS)] def command_keyboard(per_row=3): """Persistent reply keyboard of the commands that have buttons (Telegram DMs), alphabetical, row by row; a press diff --git a/tests/test_discord.py b/tests/test_discord.py index 94f0dda4..ba24d73f 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -3,7 +3,7 @@ and the gateway.""" import contextlib, io, json, sys, types, unittest from tests.common import * -from lib import discord, pull +from lib import discord, jobs, net, pull try: from nacl.signing import SigningKey @@ -18,6 +18,8 @@ def interaction(kind, **fields): COMMAND = interaction(2, data={'name': 'price', 'options': [{'name': 'arguments', 'type': 3, 'value': '5 7d'}]}) +TYPING = interaction(4, id='902', data={'name': 'price', 'options': [{'name': 'arguments', 'type': 3, 'value': '5 7', + 'focused': True}]}) # /price, its arguments being typed PRESS = interaction(3, id='901', data={'custom_id': 'c|h|y|T1', 'component_type': 2}, message={'id': '333'}) MESSAGE = {'id': '700', 'channel_id': '77', 'author': {'id': '5', 'username': 'rob'}, 'content': '<@99> price 7d'} @@ -127,6 +129,15 @@ def test_a_bad_or_missing_signature_gets_401(self): self.join_workers() self.assertEqual(self.taken, []) + def test_arguments_being_typed_get_their_suggestions_at_once(self): + """While a slash command's arguments are typed, Discord asks what to suggest: the answer is the suggestions + (type 8), at once, and no work starts.""" + status, answer, _ = self.post(TYPING) + self.join_workers() + self.assertEqual((status, json.loads(answer)), ('200 OK', {'type': 8, 'data': {'choices': [ + {'name': '5 7d', 'value': '5 7d'}]}})) + self.assertEqual(self.taken, []) + def test_the_work_starts_once_discord_has_its_answer(self): """A slash command is answered "thinking…" (a press, nothing to see); the work starts only once the server has sent that answer, which it says by closing the body.""" @@ -261,20 +272,20 @@ def test_each_help_line_is_a_slash_command_that_reads_as_its_command(self): """Each .help line has a slash command, named as a slash command may be (.forward pe is /forwardpe), and /help, /stock and /profile besides; each reads as its command, its arguments where it takes any.""" commands = worker.slash_commands() - names = [name for name, _, _ in commands] + names = [name for name, _, _, _ in commands] self.assertEqual(len(names), len(set(names))) for expected in ('help', 'dividend', 'earnings', 'buy', 'sell', 'session', 'premarket', 'forwardpe', 'stock', 'profile', 'jobs'): self.assertIn(expected, names) - for name, description, arguments in commands: + for name, description, arguments, _ in commands: with self.subTest(name=name): self.assertRegex(name, r'^[a-z0-9_-]{1,32}$') self.assertTrue(0 < len(description) <= 100) command, args = worker.read(f'/{name} {arguments}'.strip(), BOT) self.assertIsNotNone(command) self.assertEqual(command.handler, (worker.COMMANDS[0] if name == 'help' else command).handler) - self.assertEqual(dict((n, a) for n, _, a in commands)['price'], '[percent|SYMBOL|top|bottom] [period]') - self.assertEqual(dict((n, a) for n, _, a in commands)['beta'], '') + self.assertEqual(dict((n, a) for n, _, a, _ in commands)['price'], '[percent|SYMBOL|top|bottom] [period]') + self.assertEqual(dict((n, a) for n, _, a, _ in commands)['beta'], '') def test_stock_and_profile_are_the_profile_command(self): """.stock SYMBOL and /profile SYMBOL are .SYMBOL; without a symbol, the usage.""" @@ -282,6 +293,16 @@ def test_stock_and_profile_are_the_profile_command(self): self.assertEqual(self.command('/profile T1'), self.command('.T1')) self.assertEqual(self.command('.stock'), [('text', ['Usage: .SYMBOL', 'Usage: .stock SYMBOL'])]) + def test_a_command_with_words_to_suggest_asks_for_suggestions(self): + """The arguments of a command with words to suggest ask Discord for suggestions as they are typed; /plan's, + free text, do not, and a command without arguments has none.""" + registered = {entry['name']: entry for entry in (discord.command(*each) for each in worker.slash_commands())} + for name, completes in (('price', True), ('stock', True), ('jobs', True), ('performance', True), + ('pie', True), ('plan', False)): + with self.subTest(name=name): + self.assertEqual(registered[name]['options'][0]['autocomplete'], completes) + self.assertEqual(registered['beta']['options'], []) + def test_registering_replaces_the_commands_only_when_they_differ(self): """Registering puts the commands only when Discord's differ from them, and gives the bot's user id.""" wanted = [discord.command('price', 'price [period]', '[period]')] @@ -315,6 +336,88 @@ def test_help_on_discord_lists_the_slash_commands(self): self.assertNotIn('.price, or <@99> price', sent[1]) +class Suggestions(FinbotCase): + """What a slash command suggests while its arguments are typed (Discord's autocomplete): the command table's words, + the chat's watchlist and the cached holdings, this chat's jobs and the reports; never a request.""" + + def setUp(self): + """Two portfolios cached, BHP and NVDA held; MSFT watched in channel 77, TSLA in 88; any request fails.""" + super().setUp() + cache.write(sharesight.PORTFOLIOS_CACHE, {'portfolios': [{'name': 'Rob', 'id': 1}, + {'name': 'Jo Smith', 'id': 2}]}) + held = [{'instrument': {'code': code, 'market_code': market}} for code, market in (('BHP', 'ASX'), + ('NVDA', 'NASDAQ'))] + cache.write(sharesight.performance_cache_file(1, 0), {'report': {'holdings': held}}) + watchlist.add('discord:77', 'MSFT') + watchlist.add('discord:88', 'TSLA') + def no_request(*args, **kwargs): + """A request, which suggestions must never make.""" + raise AssertionError('a suggestion made a request') + for module, name in ((sharesight, '_api'), (yahoo, 'fetch'), (yahoo, 'fetch_detail'), (net, 'post')): + self.patch(module, name, no_request) + + def values(self, text, chat='discord:77'): + """The values suggested for the text typed so far, in a chat.""" + return [value for _, value in worker.suggest(chat, text)] + + def test_keywords_periods_portfolios_and_tickers(self): + """A command suggests its keywords and periods until it has one, its portfolios, and tickers: the chat's + watchlist (not another chat's) and the holdings; the word being typed matches in any case.""" + periods = ['1d', '7d', '1m', '3m', '6m', 'ytd', '1y', '5y', '10y'] + self.assertEqual(self.values('/price '), ['top', 'bottom', *periods, 'BHP.AX', 'MSFT', 'NVDA']) + self.assertEqual(self.values('/price 5 7'), ['5 7d']) + self.assertEqual(self.values('/price B'), ['bottom', 'BHP.AX']) + self.assertEqual(self.values('/price 7d t'), ['7d top']) + self.assertEqual(self.values('/price 7d 1'), []) # a period given: none again + self.assertEqual(self.values('/forwardpe n'), ['negative', 'NVDA']) + self.assertEqual(self.values('/compare MSFT n'), ['MSFT NVDA']) + self.assertEqual(self.values('/stock '), ['BHP.AX', 'MSFT', 'NVDA']) + self.assertEqual(self.values('/stock ', chat='discord:88'), ['BHP.AX', 'NVDA', 'TSLA']) + self.assertEqual(self.values('/performance 7d '), ['7d Rob', '7d Jo Smith']) + self.assertEqual(self.values('/pie '), ['sector', 'Rob', 'Jo Smith']) + self.assertEqual(self.values('/plan buy '), []) + + def test_the_jobs_subcommands_numbers_and_reports(self): + """/jobs suggests its subcommands; after run, toggle or del this chat's job numbers, each shown with its line; + after add and a time (five fields, an @macro, a zone first), the reports a chat can schedule, each shown with + what it takes, and then what the report takes.""" + self.patch(jobs, 'SYSTEM', ()) + state.save(jobs.SCHEDULE_FILE, {'discord:77': [ + {'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}, + {'when': '0 22 * * *', 'reports': 'shorts', 'zone': 'UTC', 'on': False}]}) + scheduler = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + scheduler.load() + self.patch(jobs, 'current', scheduler) + self.assertEqual(self.values('/jobs '), ['run', 'toggle', 'add', 'del']) + self.assertEqual(self.values('/job T'), ['toggle']) + self.assertEqual(worker.suggest('discord:77', '/jobs run '), [('run 1: */20 * * * Mon-Fri trades', 'run 1'), + ('run 2: 0 22 * * * shorts', 'run 2')]) + self.assertEqual(self.values('/jobs del 2'), ['del 2']) + self.assertEqual(self.values('/jobs run ', chat='discord:88'), []) + self.assertEqual(self.values('/jobs add 0 22 * * '), []) + reports = worker.suggest('discord:77', '/jobs add 0 22 * * * ') + self.assertIn(('add 0 22 * * * price midsession|interday|premarket|close|DAYS', 'add 0 22 * * * price'), + reports) + self.assertFalse(any(name in value for _, value in reports for name in jobs.SYSTEM_SCRIPTS)) + self.assertEqual(self.values('/jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price m'), + ['add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession']) + self.assertEqual(self.values('/jobs add @daily cal earnings; s'), ['add @daily cal earnings; shorts']) + self.assertEqual(self.values('/jobs add 0 22 * * * trades '), []) + + def test_at_most_25_each_of_100_characters_and_a_failure_suggests_nothing(self): + """Discord takes 25 suggestions at most, each up to 100 characters; a failure suggests nothing, and is + logged.""" + for ticker in TICKERS: + watchlist.add('discord:77', ticker) + self.assertEqual(len(self.values('/price ')), 25) + self.assertEqual(self.values('/compare ' + 'LONGTICKER ' * 9), []) + self.patch(worker, 'argument_words', mock.Mock(side_effect=RuntimeError('broken'))) + with contextlib.redirect_stderr(io.StringIO()) as log: + self.assertEqual(self.values('/price '), []) + self.assertIn('suggest: /price ', log.getvalue()) + + class FakeClient: """discord.py's Client, enough for the gateway listener: its handlers get the raw messages of the test.""" raw = [] @@ -388,6 +491,14 @@ def take(network, update): self.assertEqual(order, [('work', 1)]) self.assertIn('discord: the gateway failed, trying again in 5 s: Improper token', err.getvalue()) + def test_arguments_being_typed_over_the_gateway_get_their_suggestions_through_the_callback(self): + """Over the gateway, the suggestions for what is being typed go to the interaction's callback.""" + FakeClient.raw = [{'op': 0, 't': 'INTERACTION_CREATE', 'd': TYPING}] + with contextlib.redirect_stderr(io.StringIO()): + pull.DiscordGateway().poll(bot.take) + self.assertEqual(self.calls, [('POST', '/interactions/902/tok/callback', {'json': {'type': 8, 'data': { + 'choices': [{'name': '5 7d', 'value': '5 7d'}]}}})]) + @unittest.skipUnless(SigningKey, 'PyNaCl is not installed') def test_a_press_over_the_gateway_is_the_press_the_endpoint_gets(self): """The same press, posted to the endpoint (http mode) or sent over the gateway (pull mode), reaches the worker diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 55979c98..0bfcb78a 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -413,7 +413,7 @@ def test_every_message_and_the_slash_commands(self): self.assertIn('channels:history', manifest['oauth_config']['scopes']['bot']) self.assertNotIn('app_mentions:read', manifest['oauth_config']['scopes']['bot']) commands = manifest['features']['slash_commands'] - self.assertEqual([c['command'] for c in commands], ['/' + name for name, _, _ in worker.slash_commands()]) + self.assertEqual([c['command'] for c in commands], ['/' + command.name for command in worker.slash_commands()]) self.assertTrue({'/profile', '/job', '/forwardpe'} <= {c['command'] for c in commands}) for command in commands: with self.subTest(command=command['command']): diff --git a/util/slack_manifest.py b/util/slack_manifest.py index f52604fa..4613a508 100755 --- a/util/slack_manifest.py +++ b/util/slack_manifest.py @@ -34,9 +34,9 @@ def manifest(name='finbot', url='', every_message=False): events = ['message.channels', 'message.groups'] if every_message else ['app_mention'] events.append('message.im') posted = {'url': url} if url else {} - commands = [{'command': '/' + command, 'description': description, 'should_escape': False, - **({'usage_hint': arguments} if arguments else {}), **posted} - for command, description, arguments in worker.slash_commands()] + commands = [{'command': '/' + command.name, 'description': command.description, 'should_escape': False, + **({'usage_hint': command.arguments} if command.arguments else {}), **posted} + for command in worker.slash_commands()] return { 'display_information': {'name': name, 'description': 'Stock prices, charts and Sharesight reports', 'background_color': '#14181f'}, From 8336cfc18082828f9bc3d470646f47eefe40c62e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:03:26 +0000 Subject: [PATCH 15/31] Docker: a Dockerfile and a compose file, as a second way to install - Dockerfile: debian:trixie-slim with Debian's python3-* packages, every optional library included; a user of its own (10001), the code root's and compiled at build; each start copies the missing defaults into /data, checks the startup file, and only then starts the bot. - compose.yaml: ../finbot.env and ../finbot-data beside the checkout, the log to journald (FINBOT_LOG_DRIVER to change it), no port unless HTTP mode needs one, and hardened: read-only, no capabilities, no new privileges, 256 pids, 400 MB with no swap, init passing SIGHUP and SIGUSR1 on. - util/docker_setup.sh sets up the host side once (data dir, ownership, the startup file from its template); util/sync_defaults.py gets --create, and a missing destination is an error naming it. - git_version names the commit the image was built from (FINBOT_COMMIT). - CI's docker job builds the image, imports every module in it, starts it as compose does (defaults copied, reload and flush through init), and checks a bad startup file stops it. - doc/docker.md, and a quick start in the README. The runtime half was checked here on a python:3.13-slim image (this environment cannot reach deb.debian.org): compose starts it hardened, and every CI check passes against it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- .dockerignore | 11 ++++ .github/workflows/ci.yml | 82 ++++++++++++++++++++++++- .gitignore | 1 + CLAUDE.md | 10 ++-- Dockerfile | 28 +++++++++ README.md | 22 ++++--- bot.py | 10 +++- compose.yaml | 35 +++++++++++ doc/docker.md | 125 +++++++++++++++++++++++++++++++++++++++ tests/test_datadir.py | 30 ++++++++-- tests/test_finbot.py | 20 +++++-- util/docker_setup.sh | 34 +++++++++++ util/sync_defaults.py | 25 ++++++-- 13 files changed, 405 insertions(+), 28 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 doc/docker.md create mode 100755 util/docker_setup.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6d7a9764 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# what the image does without: history, tools for development, the tests and their samples, the docs +.git +.github +.claude +.venv +var +tests +doc +img +**/__pycache__ +*.pyc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4412729..7b8d4834 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,9 +51,89 @@ jobs: python -m compileall -q *.py lib util python -m unittest discover -s tests -v -b + docker: + # The image built as a Docker host builds it, from Debian's packages: a package renamed or gone fails here before + # production meets it. Then every module imports in it, and the container starts as compose.yaml starts it. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # the base branch is needed to see what a pull request changes + - id: changes + name: Classify the change + # As in the test job: a pull request that only touches docs skips the steps below. + env: + BASE_REF: ${{ github.base_ref }} + run: | + docs_only=false + if [ "$GITHUB_EVENT_NAME" = pull_request ]; then + files=$(git -c core.quotePath=false diff --name-only --no-renames "origin/$BASE_REF...HEAD") + docs='^(README\.md|CLAUDE\.md|doc/[^/]+\.md|img/[^/]+\.png)$' + if [ -n "$files" ] && ! grep -qvE "$docs" <<< "$files"; then + docs_only=true + fi + fi + echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT" + - if: steps.changes.outputs.docs_only != 'true' + name: Build the image + run: docker build --build-arg GIT_COMMIT="$GITHUB_SHA" -t finbot . + - if: steps.changes.outputs.docs_only != 'true' + name: Every module imports in the image, the optional libraries too, and the log names the commit + run: | + docker run --rm -i --read-only --tmpfs /tmp -e DATA_DIR=/tmp/data finbot python3 - <<'EOF' + import importlib, os, pkgutil, bot, lib, lib.reports + for package in (lib, lib.reports): + for module in pkgutil.iter_modules(package.__path__, package.__name__ + '.'): + importlib.import_module(module.name) + import discord, nacl, waitress, websocket + assert bot.git_version() == os.environ['FINBOT_COMMIT'][:7], bot.git_version() + print('every module imports; finbot', bot.git_version()) + EOF + - if: steps.changes.outputs.docs_only != 'true' + name: The container starts as compose starts it, copies the defaults, reloads and flushes + run: | + data="$RUNNER_TEMP/finbot-data" + mkdir "$data" && sudo chown 10001:10001 "$data" + docker run -d --name finbot --init --read-only --tmpfs /tmp --cap-drop ALL \ + --security-opt no-new-privileges:true --pids-limit 256 --memory 400m --memory-swap 400m \ + -e DATA_DIR=/data -v "$data:/data" -v "$PWD/defaults/finbot.env:/run/finbot.env:ro" finbot + wait_for() { # the log saved first: grep -q would cut docker logs off mid-write, which pipefail counts + for _ in $(seq 60); do + docker logs finbot > "$RUNNER_TEMP/log" 2>&1 + grep -q "$1" "$RUNNER_TEMP/log" && return 0 + sleep 1 + done + cat "$RUNNER_TEMP/log" + echo "::error::never logged: $1" + return 1 + } + wait_for 'finbot is running' + test -f "$data/preferences.conf" + docker kill -s HUP finbot + wait_for 'config: reloaded' + docker kill -s USR1 finbot + wait_for 'cache: flushed' + docker rm -f finbot + - if: steps.changes.outputs.docs_only != 'true' + name: A mistake in the startup file stops the container before the bot starts + run: | + printf 'telegramBotTokn=x\n' > "$RUNNER_TEMP/bad.env" + if docker run --rm --read-only --tmpfs /tmp -e DATA_DIR=/tmp \ + -v "$RUNNER_TEMP/bad.env:/run/finbot.env:ro" finbot > "$RUNNER_TEMP/out" 2>&1; then + cat "$RUNNER_TEMP/out" + echo "::error::the container started with a mistake in its startup file" + exit 1 + fi + cat "$RUNNER_TEMP/out" + grep -q 'unknown key telegramBotTokn' "$RUNNER_TEMP/out" + if grep -q 'finbot is running' "$RUNNER_TEMP/out"; then + echo "::error::the bot started with a mistake in its startup file" + exit 1 + fi + notify: # Telegram alert when the weekly run fails. Needs repository secrets TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID. - needs: test + needs: [test, docker] if: failure() && github.event_name == 'schedule' runs-on: ubuntu-24.04 steps: diff --git a/.gitignore b/.gitignore index dea55888..4eab49b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .* !.gitignore !.editorconfig +!.dockerignore !.python-version !.claude !.github diff --git a/CLAUDE.md b/CLAUDE.md index 0930a494..6365925c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,12 +15,14 @@ The work branch named in the session instructions (`claude/...`) is the persiste An open PR takes in every commit pushed to its branch, so a PR opened for one small change can merge far more under its old title (#35 was titled as a CI tweak and merged 14 review fixes). The merge commit on `main` keeps that title for good. - Before each push, check for an open PR from the work branch. If there is one, update its title and description so they cover every commit it now holds, not only the change it was opened for. - Never make such a PR mergeable (resolve its conflict, merge `main` into it, enable auto-merge) while its title or description is out of date. -- To merge: `main` takes only a branch that is up to date with it and green on both CI jobs. Merge `origin/main` into the work branch (a merge commit, never a rebase), run the tests, push, wait for CI on that head, then merge the PR with a merge commit. +- To merge: `main` takes only a branch that is up to date with it and green on every CI job. Merge `origin/main` into the work branch (a merge commit, never a rebase), run the tests, push, wait for CI on that head, then merge the PR with a merge commit. ## Facts -- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, except for - pull requests that change only README.md, CLAUDE.md, `doc/*.md` or `img/*.png` (install, compile and tests are - skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). +- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, and + its `docker` job builds the image (`Dockerfile`, Debian's packages), imports every module in it and starts it as + `compose.yaml` does; all except for pull requests that change only README.md, CLAUDE.md, `doc/*.md` or `img/*.png` + (install, compile, tests and the build are skipped; the allowlist in `.github/workflows/ci.yml` must never include a + path a test reads). This cloud environment cannot reach `deb.debian.org`, so only CI builds the image. - Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); `requirements.txt` is for pip/uv users and CI. The libraries in `requirements-optional.txt` are each imported only inside the code of the mode that needs it, never at module load (a test starts finbot with them all blocked). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..44c7cd2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# finbot in a container, on Debian with Debian's own python3 packages (no pip), as on a Debian host. Every optional +# library is in, so the image runs every network in either mode. The startup file and DATA_DIR stay outside the image: +# see compose.yaml and doc/docker.md. +FROM debian:trixie-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-requests python3-croniter python3-dateutil \ + python3-matplotlib python3-numpy python3-waitress python3-websocket python3-discord python3-nacl \ + ca-certificates tzdata \ + && rm -rf /var/lib/apt/lists/* + +# the bot's user: no home, no shell; the code stays root's, so the bot cannot change it +RUN groupadd --system --gid 10001 finbot \ + && useradd --system --uid 10001 --gid finbot --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin \ + finbot + +WORKDIR /opt/finbot +COPY . . +# compiled now, so that a read-only root filesystem costs nothing at start +RUN python3 -m compileall -q bot.py lib util + +ARG GIT_COMMIT=unknown +# the image has no .git: the log names the commit it was built from +ENV FINBOT_COMMIT=$GIT_COMMIT PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 DATA_DIR=/data HOME=/tmp +USER finbot + +# each start: copy the defaults missing from /data, check the startup file, and only then start the bot +CMD ["sh", "-c", "util/sync_defaults.py --apply && util/check_config.py /run/finbot.env && exec ./bot.py"] diff --git a/README.md b/README.md index 7d59e655..1c0652c9 100644 --- a/README.md +++ b/README.md @@ -424,6 +424,16 @@ Command: @botname watchlist [add|del] [symbol] ``` +## Quick start with Docker +1. `git clone https://github.com/robdevops/finbot /opt/finbot` +2. `sudo /opt/finbot/util/docker_setup.sh` +3. Set `telegramBotToken` and `telegramChatID` in `/opt/finbot.env` (see [the Telegram guide](doc/telegram.md)) +4. `cd /opt/finbot && sudo docker compose up -d --build` +5. `journalctl CONTAINER_NAME=finbot -f` + +[doc/docker.md](doc/docker.md) has the rest: the layout on the host, logs, reload and flush, updates, HTTP mode and the +hardening. To install on the host instead, read on. + ## Installation (Linux) ``` sudo $(which apt dnf yum) install git python3-pip @@ -453,7 +463,7 @@ sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finb ### `finbot.env` **Load `finbot.env` with your launcher:** * **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. -* **Docker:** `--env-file /opt/finbot.env`. Docker keeps quotes literally, so use none. Run `util/check_config.py /opt/finbot.env` first if you want the lint check. +* **Docker:** `compose.yaml` loads it, and checks it before the bot starts (see [doc/docker.md](doc/docker.md)). Write the values without quotes. * **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). * **Scheduled jobs:** they are started by `bot.py` (see [Scheduling](#scheduling)), so they run on the same settings. For a one-off run, see [Running a report by hand](#scheduling). @@ -527,22 +537,20 @@ edit `preferences.conf`, then reload if running. The service log shows what chan **Upgrading from one `.env`?** finbot no longer reads it, and refuses to start while a `.env` is in the checkout, so its settings cannot be silently ignored. Run `util/split_env.py` (or `util/split_env.py /path/to/old.env` for one elsewhere). It reads either format, the old dotenv one (`key = 'value' # comment`) or plain `KEY=value`, writes `finbot.env.new` and `preferences.conf.new` next to the old file, never overwriting, and lists any key it dropped or did not recognise. Put `finbot.env` where your launcher loads it and `preferences.conf` in the data directory, then delete the old file. ### Data directory -All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`), e.g. for a Docker bind mount that survives rebuilds: -``` -docker run -v /srv/finbot:/data -e DATA_DIR=/data --env-file /opt/finbot.env ... -``` +All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`); in Docker it is `/data`, the host's `/opt/finbot-data` (see [doc/docker.md](doc/docker.md)). Reference data (`finbot_adr.json`, `finbot_sws_*.json`, `reminder.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: ``` util/sync_defaults.py # dry run util/sync_defaults.py --apply # copy missing files (including a starter preferences.conf) util/sync_defaults.py --force # also replace changed files, keeping .bak copies util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory +util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 750) if it does not exist yet ``` `reminder.json` holds the reminders that the `reminder` report posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. -In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. +In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. Each start copies the defaults the data directory lacks (`util/sync_defaults.py --apply`). ### Sharesight * Email Sharesight support to get an API key and add the [access details](https://portfolio.sharesight.com/oauth_consumers) to `finbot.env`. Example: @@ -586,7 +594,7 @@ answers, charts, the Yahoo cookie) and keeps its state (watchlist, jobs, seen tr ``` sudo systemctl kill -s USR1 finbot # systemd sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) -docker kill -s USR1 # Docker +sudo docker compose kill -s USR1 finbot # Docker kill -USR1 # anything else ``` The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && diff --git a/bot.py b/bot.py index 0d6ac7fc..eef5e967 100755 --- a/bot.py +++ b/bot.py @@ -155,7 +155,8 @@ def dispatch(event): def git_version(checkout=None): """Short hash of the checked-out commit, read straight from the .git directory: no git binary to run, and no - "dubious ownership" refusal when the service user does not own the checkout. 'unknown' when it cannot be read.""" + "dubious ownership" refusal when the service user does not own the checkout. Without one (a Docker image), the + commit it was built from (FINBOT_COMMIT); 'unknown' when neither can be read.""" try: git = Path(checkout or config.REPO_DIR) / '.git' if git.is_file(): # a linked worktree or submodule: .git is a file saying "gitdir: " @@ -170,9 +171,12 @@ def git_version(checkout=None): else: # the branch is only in packed-refs packed = (common / 'packed-refs').read_text().splitlines() head = next((line.split()[0] for line in packed if line.endswith(' ' + ref)), '') - return head[:7] if re.fullmatch(r'[0-9a-f]{40}|[0-9a-f]{64}', head) else 'unknown' + if re.fullmatch(r'[0-9a-f]{40}|[0-9a-f]{64}', head): + return head[:7] except (OSError, IndexError): - return 'unknown' + pass + built = os.environ.get('FINBOT_COMMIT', '') + return built[:7] if re.fullmatch(r'[0-9a-f]{7,64}', built) else 'unknown' scheduler = None _signal_work = threading.Lock() # one signal's work at a time: a second reload waits for the first diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..42a61935 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,35 @@ +# finbot with Docker Compose (doc/docker.md). The configuration lives beside the checkout, outside the image and the +# container: ../finbot.env, the startup file, and ../finbot-data, DATA_DIR (preferences.conf, state and cache), both +# made by util/docker_setup.sh. Build and start: docker compose up -d --build +services: + finbot: + build: + context: . + args: + GIT_COMMIT: ${GIT_COMMIT:-unknown} # GIT_COMMIT=$(git rev-parse HEAD), for the version in the log + image: finbot + container_name: finbot + restart: unless-stopped + env_file: ../finbot.env + environment: + DATA_DIR: /data + ip: 0.0.0.0 # HTTP mode listens inside the container: publish its port below + volumes: + - ../finbot-data:/data + - ../finbot.env:/run/finbot.env:ro # checked before the bot starts + # ports: # HTTP mode only, on the host's loopback, behind your reverse proxy + # - 127.0.0.1:5000:5000 + logging: + driver: ${FINBOT_LOG_DRIVER:-journald} # json-file where the host has no journald + # hardening: the bot needs none of what these take away + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + pids_limit: 256 + mem_limit: 400m + memswap_limit: 400m # no swap on top + init: true # passes SIGHUP and SIGUSR1 on to the bot, and reaps children diff --git a/doc/docker.md b/doc/docker.md new file mode 100644 index 00000000..20b2e06c --- /dev/null +++ b/doc/docker.md @@ -0,0 +1,125 @@ +# Docker + +finbot runs in a container as it runs on a Debian host: the image is `debian:trixie-slim` with Debian's own `python3-*` +packages, every optional library included, so it runs every network in either mode. The image holds only the code. +The configuration lives on the host, beside the checkout, so a rebuild or an update never touches it. + +## Quick start + +1. `git clone https://github.com/robdevops/finbot /opt/finbot` +2. `sudo /opt/finbot/util/docker_setup.sh` +3. Set `telegramBotToken` and `telegramChatID` in `/opt/finbot.env` (or another network's keys: see the + [Telegram](telegram.md), [Slack](slack.md) and [Discord](discord.md) guides). +4. `cd /opt/finbot && sudo docker compose up -d --build` +5. `journalctl CONTAINER_NAME=finbot -f` + +## The layout on the host + +| Path | What | In the container | +|---|---|---| +| `/opt/finbot` | the checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | +| `/opt/finbot.env` | the startup file: tokens, chat ids, the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | +| `/opt/finbot-data` | `DATA_DIR`: `preferences.conf`, each chat's jobs and watchlist, the records, the cache | `/data` | + +`compose.yaml` finds the other two beside the checkout (`../finbot.env`, `../finbot-data`): clone it elsewhere and +they move with it. The startup file stays out of the image because it holds the secrets, and the data dir because it +must outlast every container. + +## What `util/docker_setup.sh` does + +Run it once, as root; it is safe to run again. It: +1. runs `util/sync_defaults.py --create --apply --dest /opt/finbot-data`, which makes the data dir (mode 750) and copies + the shipped defaults into it: `preferences.conf`, `reminder.json` and the data files. It never replaces a file you + have changed. The script needs only the standard library, so any `python3` on the host runs it. +2. gives the data dir to the container's user: `chown -R 10001:10001 /opt/finbot-data`. +3. copies `defaults/finbot.env` to `/opt/finbot.env` if there is none, owned by `root:10001`, mode 640: root and the + container can read it, nobody else. + +By hand, that is: +``` +sudo python3 /opt/finbot/util/sync_defaults.py --create --apply --dest /opt/finbot-data +sudo chown -R 10001:10001 /opt/finbot-data +sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env +sudo chown root:10001 /opt/finbot.env && sudo chmod 640 /opt/finbot.env +``` + +The startup file is `KEY=value` lines, as for systemd; write the values without quotes. + +## Build and start + +``` +cd /opt/finbot && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build +``` +`GIT_COMMIT` is optional: it names the commit in the log's first line (`Starting finbot 0123456`), as the image has no +`.git` to read it from. + +Each start of the container, in order: +1. `util/sync_defaults.py --apply` copies into `/data` any default it lacks, such as a data file added by an update; +2. `util/check_config.py /run/finbot.env` checks the startup file and `preferences.conf`. A mistake stops the start + here, and the log says which key and why; +3. the bot starts. + +## Logs + +The log goes to the host's journal: +``` +journalctl CONTAINER_NAME=finbot -f +``` +On a host without journald, set `FINBOT_LOG_DRIVER=json-file` (in the environment of `docker compose`, or in a `.env` +file beside `compose.yaml`), and read it with `sudo docker compose logs -f`. + +## Reload, flush, update + +``` +sudo docker compose kill -s HUP finbot # re-read preferences.conf +sudo docker compose kill -s USR1 finbot # flush the cache, and fill it again +cd /opt/finbot && sudo git pull && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build +``` +A change to `/opt/finbot.env` needs a restart: `sudo docker compose up -d`. + +## HTTP mode + +In pull mode, the default, finbot fetches its updates and needs no port. In HTTP mode (a network's webhook URL set), +it listens inside the container on port 5000 of every interface (`ip: 0.0.0.0` in `compose.yaml`). Publish the port +on the host's loopback only, by uncommenting `ports` in `compose.yaml`: +``` + ports: + - 127.0.0.1:5000:5000 +``` +and put your reverse proxy in front of it, as each network's guide describes for a host install. + +## Hardening + +`compose.yaml` takes away everything the bot does not need: + +| Setting | Why | +|---|---| +| `user: "10001:10001"` | the bot runs as a user of its own, never root; the code belongs to root, so the bot cannot change it | +| `read_only: true`, `tmpfs: /tmp` | nothing outside `/data` and `/tmp` can be written; the code is compiled at build, so nothing needs to be | +| `cap_drop: [ALL]` | no Linux capabilities: the bot binds no port below 1024 and changes no owner | +| `security_opt: no-new-privileges:true` | nothing in the container can gain privileges, setuid binaries included | +| `pids_limit: 256` | a runaway can start only so many threads | +| `mem_limit: 400m`, `memswap_limit: 400m` | 400 MB, with no swap on top: the bot starts at about 80 MB and reaches about 100 MB drawing charts | +| `init: true` | a minimal init as process 1, which passes `SIGHUP` and `SIGUSR1` on to the bot and reaps children | +| no `ports` | nothing listens on the host unless you publish a port | + +To give it more memory, raise both `mem_limit` and `memswap_limit`. + +## Moving a host install into Docker + +``` +sudo systemctl disable --now finbot +sudo cp -a /var/lib/finbot/. /opt/finbot-data/ # or /opt/finbot/var/. for the default DATA_DIR +sudo chown -R 10001:10001 /opt/finbot-data +``` +then `util/docker_setup.sh` (it keeps your files) and start as above. The startup file can stay at `/opt/finbot.env`. + +## Troubleshooting + +* **The container stops at once and keeps restarting:** the log says why. `ERROR config: ... unknown key ...` is a + mistake in `/opt/finbot.env` or `preferences.conf`; fix it and `sudo docker compose up -d`. +* **`PermissionError` on `/data`:** the data dir is not the container user's: `sudo chown -R 10001:10001 + /opt/finbot-data`. +* **`/data does not exist: --create makes it`:** the data dir is not mounted: `compose.yaml` mounts + `../finbot-data`; run `util/docker_setup.sh` to make it. +* **No log in the journal:** the host has no journald; use `FINBOT_LOG_DRIVER=json-file` (see [Logs](#logs)). diff --git a/tests/test_datadir.py b/tests/test_datadir.py index 8f3d3e6d..e0e30c5b 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -3,7 +3,7 @@ from pathlib import Path from lib import config from util import sync_defaults -from tests.common import FinbotCase +from tests.common import FinbotCase, ROOT, bare_environment REPO = Path(config.__file__).resolve().parent.parent @@ -105,10 +105,32 @@ def test_sync_defaults(self): self.assertEqual(Path(str(mine) + '.bak').read_text(), '{"mine": 1}') def test_sync_defaults_to_another_destination(self): - """util/sync_defaults.py --dest copies there and leaves the data dir alone.""" + """util/sync_defaults.py --dest copies there and leaves the data dir alone; a destination that does not exist + is an error that names --create, which makes it, mode 750, as it copies.""" other = Path(config.env.data_dir, 'elsewhere') - self.assertIn(('finbot_adr.json', 'would copy'), sync_defaults.sync(dest=other)) + with self.assertRaisesRegex(FileNotFoundError, 'does not exist: --create makes it'): + sync_defaults.sync(apply=True, dest=other) + self.assertIn(('finbot_adr.json', 'would copy'), sync_defaults.sync(dest=other, create=True)) self.assertFalse(other.exists()) # dry run - sync_defaults.sync(apply=True, dest=other) + sync_defaults.sync(apply=True, dest=other, create=True) self.assertTrue((other / 'finbot_adr.json').exists()) + self.assertEqual(other.stat().st_mode & 0o777, 0o750) self.assertFalse(Path(config.env.data_dir, 'finbot_adr.json').exists()) # the data dir is untouched + self.assertTrue(sync_defaults.sync(dest=other)) # there now: no --create needed + + def test_sync_defaults_runs_with_any_python_and_creates_nothing_unasked(self): + """The tool needs only the standard library, so a bare python3 runs it (Docker's setup script does, on the + host); importing lib.config creates no data directory, and a missing destination is refused, naming + --create.""" + with tempfile.TemporaryDirectory() as tmp: + dest = Path(tmp, 'data') + env = {**bare_environment(), 'DATA_DIR': str(Path(tmp, 'never'))} + run = lambda *args: subprocess.run([sys.executable, '-I', 'util/sync_defaults.py', *args], cwd=ROOT, + env=env, capture_output=True, text=True, timeout=60) + refused = run('--apply', '--dest', str(dest)) + self.assertEqual(refused.returncode, 2, refused.stderr) + self.assertIn('does not exist: --create makes it', refused.stderr) + made = run('--create', '--apply', '--dest', str(dest)) + self.assertEqual(made.returncode, 0, made.stderr) + self.assertTrue((dest / 'preferences.conf').exists()) + self.assertFalse(Path(tmp, 'never').exists()) diff --git a/tests/test_finbot.py b/tests/test_finbot.py index 3498d574..2490240b 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -77,11 +77,11 @@ def test_every_function_has_a_docstring(self): self.assertEqual(missing, []) def test_every_line_fits_in_120_columns(self): - """Every line of the code, the tests, the shipped defaults, the units, the tools and CI fits in 120 - columns, tabs counting 4: an edit quotes whole lines. README.md, CLAUDE.md and doc/ are left out (CI runs no - tests for a change to those alone, so no test may read them), and so are data files.""" + """Every line of the code, the tests, the shipped defaults, the units, the tools, CI and the Docker files fits + in 120 columns, tabs counting 4: an edit quotes whole lines. README.md, CLAUDE.md and doc/ are left out (CI runs + no tests for a change to those alone, so no test may read them), and so are data files.""" patterns = ('*.py', '*.service', 'lib/**/*.py', 'tests/*.py', 'util/*', 'defaults/*', '.github/workflows/*.yml', - '.claude/hooks/*') + '.claude/hooks/*', 'Dockerfile', 'compose.yaml', '.dockerignore') files = sorted({path for pattern in patterns for path in Path(ROOT).glob(pattern) if path.is_file() and path.suffix != '.json'}) long = [f'{path.relative_to(ROOT)}:{number}' for path in files @@ -193,6 +193,18 @@ def test_unreadable_is_unknown(self): self.repo(empty, 'garbage') self.assertEqual(bot.git_version(empty), 'unknown') + def test_an_image_without_git_names_the_commit_it_was_built_from(self): + """A Docker image has no .git: the commit it was built from (FINBOT_COMMIT, from the build's GIT_COMMIT) names + the version; a value that is no commit is 'unknown', and a .git still wins.""" + with tempfile.TemporaryDirectory() as empty, tempfile.TemporaryDirectory() as checkout: + self.repo(checkout, self.SHA) + for built, expected in ((self.SHA, '0123456'), ('abcdef1', 'abcdef1'), ('unknown', 'unknown'), + ('', 'unknown'), ('$(id)', 'unknown')): + with self.subTest(built=built), mock.patch.dict(os.environ, {'FINBOT_COMMIT': built}): + self.assertEqual(bot.git_version(empty), expected) + with mock.patch.dict(os.environ, {'FINBOT_COMMIT': 'f' * 40}): + self.assertEqual(bot.git_version(checkout), '0123456') + def test_does_not_run_git(self): """git_version reads .git itself: no git binary is run.""" with mock.patch('subprocess.run', side_effect=AssertionError('git was run')): diff --git a/util/docker_setup.sh b/util/docker_setup.sh new file mode 100755 index 00000000..312d5fdb --- /dev/null +++ b/util/docker_setup.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# The host side of finbot's Docker install, set up once, as root; safe to run again (doc/docker.md). Beside the +# checkout, where compose.yaml looks for them: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the +# container's user (10001); and ../finbot.env from the template if there is none, readable by root and the container's +# group only. Cloned to /opt/finbot, they are /opt/finbot-data and /opt/finbot.env. +set -eu + +repo=$(cd "$(dirname "$0")/.." && pwd) +parent=$(dirname "$repo") +data="$parent/finbot-data" +startup="$parent/finbot.env" + +if [ "$(id -u)" -ne 0 ]; then + echo "docker_setup: run it as root: sudo $0" >&2 + exit 1 +fi + +python3 "$repo/util/sync_defaults.py" --create --apply --dest "$data" +chown -R 10001:10001 "$data" + +if [ ! -e "$startup" ]; then + cp "$repo/defaults/finbot.env" "$startup" + echo "docker_setup: $startup written from the template" +fi +chown root:10001 "$startup" +chmod 640 "$startup" + +cat < {Path(args.dest or config.env.data_dir).resolve()}") - for name, outcome in sync(args.apply or args.force, args.force, args.dest): + try: + results = sync(args.apply or args.force, args.force, args.dest, args.create) + except FileNotFoundError as e: + print('sync_defaults:', e, file=sys.stderr) + sys.exit(2) + for name, outcome in results: label = {'copied': '', 'skipped - destination identical': '', 'skipped - destination differs': '[differs]'}.get(outcome, f'[{outcome}]') print(f"{name} {label}".rstrip()) From aebe38cd7f71ace2743d0ad457b2dae913e60b76 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:10:48 +0000 Subject: [PATCH 16/31] Docs: a short README, and a file per topic in doc/ README.md is now what finbot is, its main features, a quick start (Docker in five lines, then each network's keys in a line) and links. The rest moved to doc/, flat so that CI's docs-only rule still covers it: - commands.md: what each network supports, the commands, the reports and their screenshots; - configuration.md: finbot.env and preferences.conf, the data directory, Sharesight, portfolios, hyperlinks; - scheduling.md and caching.md; - install.md: a Debian host from the packages per mode to systemd, and pip or uv elsewhere (Docker is docker.md). The network guides link to the new files, and CLAUDE.md says where each topic now lives. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 16 +- README.md | 722 ++---------------------------------------- doc/caching.md | 26 ++ doc/commands.md | 389 +++++++++++++++++++++++ doc/configuration.md | 141 +++++++++ doc/discord.md | 2 +- doc/install.md | 72 +++++ doc/scheduling.md | 54 ++++ doc/slack.md | 4 +- doc/telegram.md | 4 +- tests/test_edges.py | 2 +- tests/test_sending.py | 2 +- util/check_config.py | 2 +- 13 files changed, 733 insertions(+), 703 deletions(-) create mode 100644 doc/caching.md create mode 100644 doc/commands.md create mode 100644 doc/configuration.md create mode 100644 doc/install.md create mode 100644 doc/scheduling.md diff --git a/CLAUDE.md b/CLAUDE.md index 6365925c..17da54c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,9 +27,9 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm `requirements.txt` is for pip/uv users and CI. The libraries in `requirements-optional.txt` are each imported only inside the code of the mode that needs it, never at module load (a test starts finbot with them all blocked). - A new preference goes in three places: a `Prefs` field with its default (`lib/config.py`; its type says how it is - read), `defaults/preferences.conf`, and the README's preferences table. The tests catch a key missing from the - template or a default that differs, but not the README (no test may read README.md: CI skips the tests for a - README-only change). + read), `defaults/preferences.conf`, and the preferences table of `doc/configuration.md`. The tests catch a key + missing from the template or a default that differs, but not the docs (no test may read README.md or `doc/*.md`: CI + skips the tests for a change to the docs alone). - If Yahoo and Sharesight are unreachable (the cloud environment's network policy can block them), check behaviour against the offline fixtures in `tests/common.py`. - For a refactor, `util/record_outputs.py` (or `--against COMMIT`) records what every command, button and scheduled @@ -132,11 +132,13 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - A chart renderer in `lib/render.py` takes its `Chart` and is listed in `RENDERERS` by kind; it gets `@_drawn`, which draws one chart at a time (matplotlib's caches are shared by every thread). - A new command: a `Command` in `COMMANDS` (its names, and what its arguments may hold), whose usage, section and - buttons make its `.help` lines and keyboard buttons, and the README command list. + buttons make its `.help` lines and keyboard buttons, and the command list of `doc/commands.md`. ## Saving tokens - Run the tests quietly: `.venv/bin/python -m unittest discover -s tests -b` prints only failures (about 0.4 KB when green). One module: `.venv/bin/python -m unittest tests.test_reports -b`. -- README.md is 36 KB: grep for the section instead of reading it whole. Each network's setup is in its own guide, - `doc/telegram.md`, `doc/slack.md` and `doc/discord.md`: a change to a network's setup or keys goes there. `doc/yahoo/` - and `doc/sharesight/` hold large API samples that the tests replay: inspect them with python, don't read them. +- README.md is short: what finbot is, a quick start, and links. The rest is in `doc/`, a file per topic: `commands.md` + (the commands and reports, 16 KB: grep for the section instead of reading it whole), `configuration.md` (the keys + and preferences), `scheduling.md`, `caching.md`, `install.md` (a Debian host), `docker.md`, and a guide per network, + `telegram.md`, `slack.md` and `discord.md`: a change to a network's setup or keys goes in its guide. `doc/yahoo/` and + `doc/sharesight/` hold large API samples that the tests replay: inspect them with python, don't read them. diff --git a/README.md b/README.md index 1c0652c9..9d9e2039 100644 --- a/README.md +++ b/README.md @@ -1,705 +1,51 @@ # finbot -## Description - -### Features -* Discord, Slack and Telegram support -* [Sharesight](https://www.sharesight.com/au/) API features: - * Trade notifications - * Portfolio performance - * Holdings list -* Yahoo Finance data for Sharesight holdings: - * Analyst consensus changes - * Earnings date reminders - * Ex-dividend date warnings - * Highly shorted stock warnings (AU, US) - * Interday and premarket price movements over a defined threshold -* Chat commands (Telegram, Slack & Discord), with no server to host: - * All the above reports - * Price history graph - * Stock lookup with price/valuation related stats and warnings - * Company profiles - * A watch list per chat - * List stocks (holdings + watchlist) by P/E, Forward P/E, PEG, market cap, or Yahoo analyst consensus - -### Chat networks -What finbot can do on each network. Telegram takes everything; the others take what their APIs allow. - -| Feature | Telegram | Slack | Discord | -|---|---|---|---| -| Chat commands | ✓: messages (`.price`), and slash commands (`/price`) from the command menu | ✓: messages, and slash commands (`/price`) from the app's manifest | ✓: slash commands (`/price`), and messages over the gateway | -| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots); none for a slash command, which has no message | ✓ for a message; "thinking…" for a slash command | -| Charts in replies | ✓ | ✓ | ✓ | -| Charts in scheduled reports | ✓ | ✓ | ✓ | -| Slash commands completed as they are typed | names, from the command menu | names, with their usage | names, and the arguments: keywords, periods, portfolios, tickers, jobs | -| Period and job buttons | ✓ | ✓ | ✓ | -| No link previews | ✓ | ✓ | ✓ | -| Linked tickers | ✓ | ✓ | ✓ | -| Charts updated in place by their buttons | ✓ | ✓ | ✓ | - -**Setting up a network:** [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md). Each guide -covers the whole setup of that network's bot, and compares its two modes (finbot fetching its updates, or the network -posting them to your HTTPS server) before the steps. +finbot is a chat bot for Telegram, Slack and Discord that follows your [Sharesight](https://www.sharesight.com/au/) +portfolios and each chat's watchlist with Yahoo Finance data. It posts trades, price moves, earnings and ex-dividend +dates, analyst rating changes and more on a schedule, and answers commands with reports and charts. + +## Features +* Telegram, Slack and Discord, with no server to host: finbot fetches its updates itself (or, in HTTP mode, a network + posts them to your HTTPS server) +* Sharesight trades as they happen, portfolio performance and holdings +* Price moves past a threshold, earnings and ex-dividend dates, analyst rating changes, short interest, record highs +* Stock lookup, price history charts and comparisons, rankings by P/E, PEG, market cap or analyst rating +* Commands as messages (`.price`) or slash commands (`/price`), completed as they are typed where the network allows +* Each chat's own scheduled jobs and watchlist, managed in the chat (`.jobs`, `.watchlist`) +* Docker or a Debian host, hardened either way ![Screenshot of showing trade notifications on Slack](img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") -### Requirements -* Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key -* A Telegram bot, a Slack app or a Discord bot (see the network's guide, above) -* No server: finbot fetches its updates from each network itself (pull mode: Telegram's long polling, Slack's Socket Mode, Discord's gateway). In HTTP mode instead, a network posts them to a public web server with a domain name and a matching certificate, which proxies them to the bot (see the [Telegram](doc/telegram.md#http-mode), [Slack](doc/slack.md#http-mode) and [Discord](doc/discord.md#http-mode) guides). -* Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) -* Python modules, pinned to versions with 3.13 support in `requirements.txt`: -``` -croniter python-dateutil requests matplotlib numpy -``` -* For a network in http mode (its webhook URL set), also `waitress` (Debian: `python3-waitress`); for Slack in Socket - Mode, `websocket-client` (`python3-websocket`); for Discord's gateway, `discord.py` (`python3-discord`), and for its - http mode, `PyNaCl` (`python3-nacl`). The optional modules are in `requirements-optional.txt`, each imported only by - the mode that needs it. - -### Supported commands - -Where: -* _percent_ is a number, optionally followed by '%' -* _period_ is a number followed by 'd', 'w', 'm' or 'y', or 'ytd' -* _portfolio_ is a Sharesight portfolio name -* _symbol_ is a Yahoo Finance symbol; a bare one Yahoo does not know is tried again on your exchange (`exchange_suffix`: `BHP` may be `BHP.AX`) -``` -.symbol -.stock symbol -.beta -.buy -.sell -.compare symbol symbol [symbol...] [period] -.dividend [period|symbol] -.earnings [period|symbol] -.forward pe [symbol|top|bottom|negative] -.history symbol -.holdings portfolio -.jobs [run|toggle] [number] -.jobs add [CRON_TZ=zone] time report -.jobs del number -.marketcap [symbol|top|bottom] -.pe [symbol|top|bottom] -.peg [symbol|top|bottom|negative] -.performance [period] [portfolio] -.pie portfolio [sector] -.plan [your plan] -.premarket [percent|symbol] -.price [percent|symbol|top|bottom] [period] -.session [percent|symbol] -.shorts [percent|symbol] -.trades [period] [portfolio] -.watchlist [add|del symbol] -.who symbol -``` - -`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command on every network, which completes its name as it is typed: Telegram's command menu, set as finbot starts; Slack's, from the app's manifest (`util/slack_manifest.py`, see the [Slack guide](doc/slack.md#slash-commands)); Discord's, registered as finbot starts. `/stock symbol` (or `/profile symbol`) is `.symbol`, `/forwardpe` is `.forward pe`, and `/job` is `/jobs`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. - -Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying each chat that schedules them of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. - -The other various reports can also run on a schedule (e.g. daily or weekly), or on demand with a command. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. A scheduled report posts to the chat whose job it is, and a command's reply to the chat that sent it. - -The bot answers commands in every chat it is in. It fetches its updates from each network itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: -* Stock lookup (financials, company profile, price charts) -* A watch list per chat, which anyone in it can change, picked up by the reports there -* Listing of portfolios and portfolio performance -* Listing of holdings sorted by various attributes -* Running the other stock reports on demand - -## Reports - -### Stock lookup -![Screenshot showing stock info on Slack](img/stockinfo.png?raw=true "Screenshot showing stock info on Slack") - -The stock lookup returns various stats relevant to a stock's valuation, growth and risk factors. - -#### Emoji legend -* The colored circle next to the symbol at the top indicates the current market state: - * 🔴 market closed - * 🟠 pre/post-market - * 🟢 normal trading -* The warning symbol ⚠️ indicates possible risk factors. This is not intended to replace due diligence, but to non-exhaustively highlight where additional diligence is due. Some risk factors include: - * Negative earnings ⚠️ - * Earnings in decline ⚠️ - * Negative cashflow ⚠️ - * High short interest ⚠️ - * High forward P/E ratio ⚠️ - * Net insider selling ⚠️ - * High debt-to-equity ratio ⚠️ -* The arrow grid indicates the directional change in the company earnings and revenue for each reporting period. A red arrow indicates the absolute earnings was negative: - * 🔼 earnings/revenue increased from the previous period - * 🔽 earnings/revenue decreased from the previous period - * 🔺 earnings increased from the previous period but remained negative - * 🔻 earnings decreased from the previous period and was negative - * ▪️ earnings/revenue matched the previous period - * ❌ data is missing for this period -* Using the example screenshot above, Crowdstrike reported: - * 🔼🔼🔼 revenue increased in all of the past three quarters - * 🔺🔻🔻 In the past three quarters, earnings initially increased, then decreased for two quarters, and was negative in every quarter - * 🔼🔼🔼 revenue increased in all of the past three years - * 🔻🔺🔻 In the past three years, earnings decreased, increased, then decreased, and was negative for all three years - -This report is a command only. Example usage: -``` -.AAPL -``` -``` -@botname AAPL -``` - -### Trades -![trade update in Slack](img/trades.png?raw=true "Trade update in Slack") - -The `trades` report sends recent Sharesight trades to each chat that schedules it. -By default, this report searches Sharesight for trades from the past 28 days (`past_days`). You can override this in `preferences.conf`, or by providing a number as argument when triggered through the chat bot. - -Without persistent storage, set `past_days=0` to avoid duplicate trade notifications. In this case, the schedule must run the report exactly once per day, after market close. - -With persistent storage, the default of `past_days=28` suits. This is useful if Sharesight imports trades with past dates for any reason. Note that the initial run will notify on all historical trades for the `past_days` period. It is recommended to schedule it every 20 minutes. - -``` -past_days=28 -``` - -Job (see [Scheduling](#scheduling)), such as `.jobs add */20 * * * Mon-Fri trades`: -``` -trades -``` - -Command: -``` -.trades [period] -``` -``` -@botname trades [period] -``` - -You can also specify a portfolio name to get trades for just that portfolio: -``` -.trades [period] [portfolio] -``` -``` -@botname trades [period] [portfolio] -``` - -### Price alerts -![Price chart triggered from Slack](img/finbot_price.png?raw=true "Price chart triggered from Slack") -![Price alert triggered by Cron](img/price.png?raw=true "Price alert triggered by Cron") - -The `price` report sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. - -If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. - -Config example: -``` -price_percent=5 -demote_volatile=True -``` - -Job (see [Scheduling](#scheduling)), such as `.jobs add 29 21 * * Mon-Fri price interday`: -``` -price premarket|midsession|interday|close|DAYS -``` -When scheduled, the mode is its argument: -* If `premarket` is passed, it only reports on pre/post market price movements. -* If `midsession` is passed, it only reports for markets currently in session. This is intended to run on a schedule to provide mid-session alerts for big price movements of your holdings. For example, it could be run twice per day 12 hours apart, to capture markets in different timezones. -* If `interday` is passed, it reports current price against the previous market close. -* If `close` is passed, it reports daily change in stocks whose market closed in the past 3 hours. -* If an integer [i] is passed, it reports current price against [i] days ago. - -Command (pre-market): -``` -.premarket [percent] -``` -``` -@botname premarket [percent] -``` - -Command (mid-session): -``` -.session [percent] -``` -``` -@botname session [percent] -``` - -Command (interday). -``` -.price [percent] -``` -``` -@botname price [percent] -``` - -For interday, can also specify the time period. A bare number is the percent; a period carries its unit: `7d`, `2w`, `3m`, `1y` or `ytd`. For example, `.price 3 7d` lists the moves of 3% or more over the past week. -``` -.price [percent] [period] -``` - -Or a specific stock. In this case, the output includes a graph. -``` -.price [symbol] [period] -``` - -### Portfolio performance -![Screenshot of Portfolio performance notification on Slack](img/performance.png?raw=true "Screenshot performance notification on Slack") - -The `performance` report covers Sharesight portfolio performance over a given period. The default is set by `past_days` in the `preferences.conf`, which can be overridden at runtime by specifying an integer as the argument. - -Config example: -``` -past_days=28 -``` - -Job (see [Scheduling](#scheduling)), such as `.jobs add 28 21 * * Sat performance 7`: -``` -performance [DAYS] -``` - -Command: -``` -.performance [period] -``` -``` -@botname performance [period] -``` - -As a command, it also takes a portfolio: -``` -.performance [period] [[portfolio] -``` -``` -@botname performance [period] [portfolio] -``` - -### Events calendar -![earnings message in Slack](img/earnings.png?raw=true "Earnings message in Slack") - -The `cal` report sends upcoming earnings and ex-dividend date alerts. The data is sourced from Yahoo! Finance. It reports on events up to `future_days` into the future. This is set in the `preferences.conf` for a scheduled run, or can be specified as an argument when triggered through the chat bot. - -**Finance Explainer:** When a company releases its quarterly earnings report, the stock price may undergo a signficant positive or negative movement, depending on whether the company beat or missed market expectations. You may wish to hold off buying more of this stock until after its earnings report, unless you think the stock will beat market expectations. - -**Finance Explainer:** When a stock goes ex-dividend, the share price [typically drops](https://www.investopedia.com/articles/stocks/07/ex_dividend.asp) by the amount of the dividend paid. If you buy right before the ex-dividend date, you can expect an unrealised capital loss, plus a tax obligation for the dividend. Thus, you may wish to wait for the ex-dividend date before buying more of this stock. -``` -future_days=30 -``` - -Job (see [Scheduling](#scheduling)), such as `.jobs add 28 21 * * Sat cal earnings`: -``` -cal earnings|ex-dividend -``` - -Command (earnings): -``` -.earnings [period] -``` -``` -@botname earnings [period] -``` - -Command (ex-dividend): -``` -.dividend [period] -``` -``` -@botname dividend [period] -``` - -### Highly shorted stock warnings -![short warnings in Slack](img/shorts.png?raw=true "Short warnings in Slack") - -The `shorts` report sends short interest warnings. The data is sourced from Yahoo Finance and Shortman (AU). `shorts_percent` defines the alert threshold for the percentage of a stock's float shorted. This can be specified in the `preferences.conf` for a scheduled run, or as an argument when triggered through the chat bot. - -**Finance Explainer:** A high short ratio indicates a stock is exposed to high risks, such as potential banktrupcy. It may also incentivise negative news articles which harm the stock price. If the market is wrong, however, risk tolerant investors may receive windfall gains. This report is intended to alert you to an above-average risk, and prompt you to investigate this stock more closely. -``` -shorts_percent=15 -``` - -Command: -``` -.shorts [percent] -``` -``` -@botname shorts [percent] -``` - -### pe / forward pe / peg -![Forward PE report in Slack](img/finbot_fpe.png?raw=true "Forward PE report in Slack") - -List securities by P/E ratio, forward P/E ratio, or PEG ratio. A forward P/E is an analysts' estimate, so its lists take only stocks covered by at least `min_analysts` analysts (7 by default; see `preferences.conf`). - -Command: -``` -.pe [SYMBOL] -.fpe [SYMBOL|bottom|negative] -.peg [SYMBOL|bottom|negative] -``` -``` -@botname pe [SYMBOL|top|bottom] -@botname fpe [SYMBOL|top|bottom|negative] -@botname peg [SYMBOL|top|bottom|negative] -``` - -### History -![Price history report in Slack](img/finbot_history.png?raw=true "Price history report in Slack") - -Shows the long term performance of a security. -It creates a 5Y chart, and then lists performance for several periods (5Y 3Y 1Y 6M 3M 1M 1W 1D) - -Command: -``` -.history -``` -``` -@botname history -``` - -### Buy -![Buy ratings in Slack](img/finbot_buy.png?raw=true "Buy ratings in Slack") - -Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts (7 by default; see `preferences.conf`) - -Command: -``` -.buy -``` -``` -@botname buy -``` - -### Sell -![Rating report in Slack](img/finbot_sell.png?raw=true "Rating report in Slack") - -Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts - -Command: -``` -.sell -``` -``` -@botname sell -``` - -### Marketcap -![Marketcap report in Slack](img/finbot_marketcap.png?raw=true "Marketcap report in Slack") - -List the top (or bottom) stocks by market cap, in US dollars: market caps in other currencies are converted at Yahoo's exchange rates. With a symbol, its market cap in its own currency and in US dollars. - -Command: -``` -.marketcap [top|bottom|SYMBOL] -``` -``` -@botname marketcap [top|bottom|SYMBOL] -``` - -### Consensus rating changes -![Rating change report in Slack](img/finbot_rating.png?raw=true "Rating change report in Slack") - -Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run on a schedule, such as `.jobs add 28 21 * * Sat rating` (see [Scheduling](#scheduling)). Requires persistent storage. - -``` -rating -``` - -### Watchlist -![Shared watchlist in Slack](img/watchlist.png?raw=true "Shared watchlist in Slack") - -Tracks additional securities which are not in your Sharesight holdings. Each chat has its own watchlist, which anyone in the chat can change: the reports in that chat, asked for or scheduled, include its watchlist's stocks as if they were holdings, and no other chat's. The Sharesight holdings are the same in every chat. `yahoo_warm` keeps every chat's watchlist warm in the cache. - -The watchlists are kept in `finbot_watchlist.json` in the data directory, by chat (`{"chats": {"telegram:-1001234": ["2454.TW", "ASO.AX", "STEM"]}}`), so there is no need to edit the file. The first start with watchlists per chat gives each home chat the watchlist kept before, and the same goes for the plans that `.plan` records: each chat's `.plan` lists only the plans recorded in it. - -Command: -``` -.watchlist -``` -``` -.watchlist [add|del] [symbol] -``` -``` -@botname watchlist -``` -``` -@botname watchlist [add|del] [symbol] -``` - -## Quick start with Docker +## Quick start +With Docker: 1. `git clone https://github.com/robdevops/finbot /opt/finbot` 2. `sudo /opt/finbot/util/docker_setup.sh` -3. Set `telegramBotToken` and `telegramChatID` in `/opt/finbot.env` (see [the Telegram guide](doc/telegram.md)) +3. Set the keys of each network you use in `/opt/finbot.env` (below) 4. `cd /opt/finbot && sudo docker compose up -d --build` 5. `journalctl CONTAINER_NAME=finbot -f` -[doc/docker.md](doc/docker.md) has the rest: the layout on the host, logs, reload and flush, updates, HTTP mode and the -hardening. To install on the host instead, read on. - -## Installation (Linux) -``` -sudo $(which apt dnf yum) install git python3-pip -``` -``` -git clone https://github.com/robdevops/finbot.git ~/finbot -``` -``` -cd ~/finbot -``` -``` -pip3 install -r requirements.txt -``` -Or, with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: -``` -uv venv && uv pip install -r requirements.txt -``` - -## Setup -Configuration is split into two files -- finbot.env for startup environment variables, and preferences.conf for values that can be changed at runtime. - -Copy the shipped template: -``` -sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env -``` - -### `finbot.env` -**Load `finbot.env` with your launcher:** -* **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. -* **Docker:** `compose.yaml` loads it, and checks it before the bot starts (see [doc/docker.md](doc/docker.md)). Write the values without quotes. -* **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). -* **Scheduled jobs:** they are started by `bot.py` (see [Scheduling](#scheduling)), so they run on the same settings. For a one-off run, see [Running a report by hand](#scheduling). - -The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which -of the chat network keys it needs. - -| Key | Default | Description | -|---|---|---| -| `DATA_DIR` | `var/` in the checkout (`$STATE_DIRECTORY` under systemd `StateDirectory=`) | Where state and cache live | -| `ip` | `127.0.0.1` | Address the bot listens on | -| `port` | `5000` | Port the bot listens on | -| `scheduler_catchup` | `5` | Minutes: a run missed while the bot was down still fires if it was due this recently | -| `telegramChatID` | | Telegram's home chat, a group or channel id: finbot's first start gives it the default jobs, and error reports go there (see [Scheduling](#scheduling)) | -| `sharesight_code` | | Sharesight API access | -| `sharesight_client_id` | | Sharesight API access | -| `sharesight_client_secret` | | Sharesight API access | -| `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | -| `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | -| `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | -| `slackChannelID` | | Slack's home channel id, as `telegramChatID` (with `slackBotToken`) | -| `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | -| `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | -| `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | -| `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | -| `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | -| `discordBotToken` | | Discord bot token: replies, and commands over the gateway unless `discordOutgoingWebhook` is set | -| `discordChannelID` | | Discord's home channel id, as `telegramChatID` (with `discordBotToken`) | -| `discordOutgoingWebhook` | | The Interactions Endpoint URL Discord posts slash commands and button presses to (HTTP mode); empty, the gateway (pull mode) | -| `discordPublicKey` | | The app's public key, which checks the signature of each interaction (HTTP mode) | - -### `preferences.conf` items - -Copies preferences.conf and metadata files into the data directory (never overwrites) -``` -util/sync_defaults.py [--dest DIR] --apply -``` - -The preferences, read by the bot itself. A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. - -| Key | Default | Description | -|---|---|---| -| `past_days` | `28` | How many days back reports look | -| `future_days` | `30` | How many days ahead the events calendar looks | -| `price_percent` | `5` | Price alert threshold, percent | -| `shorts_percent` | `15` | Short interest alert threshold, percent | -| `min_analysts` | `7` | `.buy`, `.sell` and `.forward pe` list only stocks covered by at least this many analysts; with fewer, a consensus says little (thinly covered small caps are nearly all a perfect buy) | -| `demote_volatile` | `True` | Volatile (small cap, high beta) stocks need a bigger move to alert | -| `demote_leveraged` | `True` | Leveraged (2x, 3x) funds need a bigger move to alert | -| `volatility_multiplier` | `2` | How much bigger a move volatile stocks need | -| `exclude_portfolios` | | Sharesight portfolio IDs to leave out, space separated | -| `include_portfolios` | | Sharesight portfolio IDs to report on, space separated (empty: all that are not excluded) | -| `trades_use_yahoo` | `True` | Use Yahoo for trades; set `False` if Yahoo breaks | -| `graph` | `True` | Send charts with reports | -| `hyperlink` | `True` | Hyperlink tickers in reports | -| `hyperlinkFooter` | `True` | Add a website link footer to stock lookups | -| `hyperlinkProvider` | `google` | Where ticker links go: `google` or `yahoo` | -| `exchange_suffix` | `AX` | Regional: Yahoo's suffix for your exchange (`AX` for the ASX, `NZ`, `L`, `TO` and so on). A bare symbol Yahoo does not know is tried again with it, in every command that takes a symbol (`BHP` becomes `BHP.AX`); empty, a symbol is taken as written | -| `country_code` | `AU` | Regional: your country, for the reminder report | -| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report, and for each job's next run in `.jobs` (besides UTC) | -| `cache` | `True` | Cache Yahoo and Sharesight responses | -| `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | -| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm` runs, so the holdings never run out | -| `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | -| `debug` | `False` | Extra diagnostics in the log | -| `print_headers` | `False` | Log inbound request headers | - -A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. - -edit `preferences.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. - -**Upgrading from one `.env`?** finbot no longer reads it, and refuses to start while a `.env` is in the checkout, so its settings cannot be silently ignored. Run `util/split_env.py` (or `util/split_env.py /path/to/old.env` for one elsewhere). It reads either format, the old dotenv one (`key = 'value' # comment`) or plain `KEY=value`, writes `finbot.env.new` and `preferences.conf.new` next to the old file, never overwriting, and lists any key it dropped or did not recognise. Put `finbot.env` where your launcher loads it and `preferences.conf` in the data directory, then delete the old file. - -### Data directory -All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`); in Docker it is `/data`, the host's `/opt/finbot-data` (see [doc/docker.md](doc/docker.md)). -Reference data (`finbot_adr.json`, `finbot_sws_*.json`, `reminder.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: -``` -util/sync_defaults.py # dry run -util/sync_defaults.py --apply # copy missing files (including a starter preferences.conf) -util/sync_defaults.py --force # also replace changed files, keeping .bak copies -util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory -util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 750) if it does not exist yet -``` -`reminder.json` holds the reminders that the `reminder` report posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. - -With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. - -In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. Each start copies the defaults the data directory lacks (`util/sync_defaults.py --apply`). - -### Sharesight -* Email Sharesight support to get an API key and add the [access details](https://portfolio.sharesight.com/oauth_consumers) to `finbot.env`. Example: -``` -sharesight_code=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f -sharesight_client_id=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f -sharesight_client_secret=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f -``` - -### Chat networks setup -Follow the guide for each network you use: [Telegram](doc/telegram.md), [Slack](doc/slack.md) or -[Discord](doc/discord.md). It covers the bot, its `finbot.env` keys, and for HTTP mode the HTTPS server in front of -finbot. - -### Portfolios -Portfolios are auto-discovered, including other people's portfolios which are shared to you. To exclude specific portfolios, add their IDs to `exclude_portfolios` in the `preferences.conf`: -``` -exclude_portfolios=100003 100004 -``` - -Alternatively, you can include only specific portfolios: -``` -include_portfolios=100001 100002 -``` - -### Caching -Many objects are cached for 12 hours by default (`cache_seconds`). Cache is controlled by the settings below. Trades IDs are stored on disk, but trade queries generally don't result in a cache hit for functional reasons. -``` -cache=True -cache_seconds=43200 -performance_cache_seconds=2400 -``` - -`performance_cache_seconds` (default 40 minutes) is the cache lifetime of Sharesight performance reports. - -Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The weekly system job `cleanup` removes cache files after 30 days. - -To flush the cache, so that everything is fetched afresh, signal the running bot: it deletes its cache files (Yahoo and Sharesight -answers, charts, the Yahoo cookie) and keeps its state (watchlist, jobs, seen trades, ratings), then runs both warm jobs -(`sharesight_warm`, `yahoo_warm`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: -``` -sudo systemctl kill -s USR1 finbot # systemd -sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) -sudo docker compose kill -s USR1 finbot # Docker -kill -USR1 # anything else -``` -The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && -sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, deleting the files in `DATA_DIR/cache` does the same. - -Two system jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](#scheduling)): `sharesight_warm` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). - -### Hyperlinks -These `preferences.conf` options relate to hyperlinks: - -* `hyperlink`, if set to `False`, disables all hyperlinks. Note: it can not disable syntax highlighting and false positives by chat clients that try to detect domains. Defaults to `True`. -``` -hyperlink=True -``` - -* `hyperlinkFooter` appends hyperlinks to the company website and to a Google search for the ticker on simplywall.st for the symbol lookup report. Possible values are `True` or `False`. Defaults to `True`. - -``` -hyperlinkFooter=True -``` - -* `hyperlinkProvider` configures where to send ticker hyperlinks. Available values are `google` and `yahoo`. If set to `google`, it is only used where practical, as Google does not publish earnings date, ex-dividend date, premarket price, or short interest figures. Defaults to `google`. -``` -hyperlinkProvider=google -``` -``` -hyperlinkProvider=yahoo -``` - -## Scheduling -Each chat has its own jobs: reports that run on a schedule and post to that chat. Anyone in the chat manages them there, -with `.jobs` (also `.job`, `.cron`, `.crons` or `.schedule`): -``` -.jobs this chat's jobs, numbered, each with its next run; then the system jobs -.jobs add the reports a job can run, and what each takes -.jobs add 29 21 * * Mon-Fri price interday -.jobs run the list, with a button per job that runs it now; .jobs run 2 runs job 2 -.jobs toggle the list, with a button per job that switches it on or off; .jobs toggle 2 -.jobs del 2 deletes job 2 -``` - -A job is a time in crontab's syntax, then one or more reports with their arguments: -* **The time** is five fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists - work) or an `@macro` such as `@daily`. A time that cannot be or never comes (minute 61, 30 February) is refused, - saying why. -* **Times are in UTC**, which keeps the ASX's jobs and NYSE's each within an hour of their local times all year. To - follow one market's clocks exactly, a job names its zone first, as crontab writes it: - `.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession`. `.jobs` shows each next run in UTC and in - the `timezone` preference. -* **The reports** are `trades`, `price midsession|interday|premarket|close|DAYS`, `cal earnings|ex-dividend`, - `performance [DAYS]`, `rating`, `milestone`, `shorts`, `reminder` and `podcasts FEED_URL`. Several, separated by `;`, - run one after another: `.jobs add 28 21 * * Sat cal earnings; cal ex-dividend; price 7; performance 7; rating`. - Quote an argument that holds spaces or a `;` (a feed URL, say). -* A chat has 30 jobs at most. - -A job posts to its own chat, whichever way that chat's updates reach finbot: the jobs are kept by network and chat id -(`finbot_schedule.json` in the data directory), so switching a network between pull and HTTP mode keeps them. Each chat -also keeps its own record of what a report has told it (trades, rating changes, record highs, podcast episodes), so the -same report scheduled in two chats tells each of them once. A run from `.jobs run` is a scheduled run: it posts to the -chat and updates the same record. A change counts at once, with no reload. - -**The system jobs**, fixed in `lib/jobs.py` and listed by `.jobs` under _System_, keep the cache full and tidy: -`sharesight_warm` every 30 minutes, `yahoo_warm` every 8 hours (see [Caching](#caching)), and `cleanup 30` weekly, -which deletes cache files older than 30 days. A chat cannot add, switch or run them. - -**From `schedule.cron`**, where the schedule used to be: the first time finbot starts without `finbot_schedule.json`, -it gives each home chat (`telegramChatID`, `slackChannelID`, `discordChannelID`) the jobs of the data directory's -`schedule.cron`, else of the shipped `defaults/schedule.cron`, each in the zone it ran in (the server's), without the -system jobs. It copies the reports' records to each home chat too, so that nothing old is told again. It logs what it -did, and any line it left out, and reads the file no more: delete the data directory's copy afterwards. +To install on a Debian host with systemd instead, see [doc/install.md](doc/install.md). -Behaviour worth knowing: -* The reports run in `bot.py` itself, each in a thread of its own, sharing the bot's user, sandbox, environment, data - directory, caches and connections; charts are drawn one at a time, and a slow report holds up no chat command. There - is no cron to set up: remove any old crontab entries that ran finbot's scripts, or their reports come twice. -* A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last - `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. A job that is switched off - does not fire, and does not catch up once it is switched on again. -* A job still running when it is next due is skipped. A command still running after 5 minutes is reported once, and - left to finish (a thread cannot be stopped, but every request it makes has a timeout); until it does, it is not - started again. Each command of a job has 5 minutes of its own. -* A failure is reported to the job's chat. The bot logs each job as it runs (`journalctl -u finbot`). +Each network needs a bot of its own, and finbot fetches its updates with no server (its pull mode): +* **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken` and + `telegramChatID`, restart finbot ([guide](doc/telegram.md)). +* **Slack:** create the app from the manifest that `util/slack_manifest.py` prints, add an app-level token, set + `slackBotToken`, `slackAppToken` and `slackChannelID`, restart finbot ([guide](doc/slack.md)). +* **Discord:** create the application and its bot, turn on its Message Content intent, set `discordBotToken` and + `discordChannelID`, restart finbot ([guide](doc/discord.md)). -## Daemonize (systemd) -`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy `finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of [`finbot.env`](#setup) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start it. `systemctl reload finbot` re-reads `preferences.conf` without a restart; a change to `finbot.env` needs `systemctl restart finbot`: +Then send `.help` in the chat. -``` -sudo cp -v finbot.service /etc/systemd/system/ -``` -``` -sudo sed -i 's/CHANGEME/YOUR LINUX USERNAME/' /etc/systemd/system/finbot.service -``` -``` -sudo systemctl daemon-reload -``` -``` -sudo systemctl enable finbot --now -``` -You can now monitor the bot's stderr with : -``` -journalctl -fu finbot -``` +## Documentation +* [Commands and reports](doc/commands.md), with screenshots +* [Scheduling](doc/scheduling.md): each chat's jobs, and the reports they can run +* [Configuration](doc/configuration.md): `finbot.env` and `preferences.conf`, the data directory, Sharesight +* [Caching](doc/caching.md) +* Installing: [Docker](doc/docker.md) · [a Debian host with systemd](doc/install.md) +* Networks: [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md) ## Suggestions -* Is my code is doing something the hard way? -* Something important is missing from this README? +* Is my code doing something the hard way? +* Is something important missing from the docs? Log an [issue](https://github.com/robdevops/finbot/issues)! diff --git a/doc/caching.md b/doc/caching.md new file mode 100644 index 00000000..72701307 --- /dev/null +++ b/doc/caching.md @@ -0,0 +1,26 @@ +# Caching + +Many objects are cached for 12 hours by default (`cache_seconds`). Cache is controlled by the settings below. Trades IDs are stored on disk, but trade queries generally don't result in a cache hit for functional reasons. +``` +cache=True +cache_seconds=43200 +performance_cache_seconds=2400 +``` + +`performance_cache_seconds` (default 40 minutes) is the cache lifetime of Sharesight performance reports. + +Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The weekly system job `cleanup` removes cache files after 30 days. + +To flush the cache, so that everything is fetched afresh, signal the running bot: it deletes its cache files (Yahoo and Sharesight +answers, charts, the Yahoo cookie) and keeps its state (watchlist, jobs, seen trades, ratings), then runs both warm jobs +(`sharesight_warm`, `yahoo_warm`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: +``` +sudo systemctl kill -s USR1 finbot # systemd +sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) +sudo docker compose kill -s USR1 finbot # Docker +kill -USR1 # anything else +``` +The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && +sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, deleting the files in `DATA_DIR/cache` does the same. + +Two system jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](scheduling.md)): `sharesight_warm` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). diff --git a/doc/commands.md b/doc/commands.md new file mode 100644 index 00000000..85ad9889 --- /dev/null +++ b/doc/commands.md @@ -0,0 +1,389 @@ +# Commands and reports + +What finbot does in a chat: its commands, and the reports that they and each chat's [scheduled jobs](scheduling.md) post. + +## On each network +What finbot can do on each network. Telegram takes everything; the others take what their APIs allow. + +| Feature | Telegram | Slack | Discord | +|---|---|---|---| +| Chat commands | ✓: messages (`.price`), and slash commands (`/price`) from the command menu | ✓: messages, and slash commands (`/price`) from the app's manifest | ✓: slash commands (`/price`), and messages over the gateway | +| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots); none for a slash command, which has no message | ✓ for a message; "thinking…" for a slash command | +| Charts in replies | ✓ | ✓ | ✓ | +| Charts in scheduled reports | ✓ | ✓ | ✓ | +| Slash commands completed as they are typed | names, from the command menu | names, with their usage | names, and the arguments: keywords, periods, portfolios, tickers, jobs | +| Period and job buttons | ✓ | ✓ | ✓ | +| No link previews | ✓ | ✓ | ✓ | +| Linked tickers | ✓ | ✓ | ✓ | +| Charts updated in place by their buttons | ✓ | ✓ | ✓ | + +Each network's guide sets up its bot and compares its two modes: [Telegram](telegram.md), [Slack](slack.md), [Discord](discord.md). + +![Screenshot of showing trade notifications on Slack](../img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") + +## Supported commands + +Where: +* _percent_ is a number, optionally followed by '%' +* _period_ is a number followed by 'd', 'w', 'm' or 'y', or 'ytd' +* _portfolio_ is a Sharesight portfolio name +* _symbol_ is a Yahoo Finance symbol; a bare one Yahoo does not know is tried again on your exchange (`exchange_suffix`: `BHP` may be `BHP.AX`) +``` +.symbol +.stock symbol +.beta +.buy +.sell +.compare symbol symbol [symbol...] [period] +.dividend [period|symbol] +.earnings [period|symbol] +.forward pe [symbol|top|bottom|negative] +.history symbol +.holdings portfolio +.jobs [run|toggle] [number] +.jobs add [CRON_TZ=zone] time report +.jobs del number +.marketcap [symbol|top|bottom] +.pe [symbol|top|bottom] +.peg [symbol|top|bottom|negative] +.performance [period] [portfolio] +.pie portfolio [sector] +.plan [your plan] +.premarket [percent|symbol] +.price [percent|symbol|top|bottom] [period] +.session [percent|symbol] +.shorts [percent|symbol] +.trades [period] [portfolio] +.watchlist [add|del symbol] +.who symbol +``` + +`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command on every network, which completes its name as it is typed: Telegram's command menu, set as finbot starts; Slack's, from the app's manifest (`util/slack_manifest.py`, see the [Slack guide](slack.md#slash-commands)); Discord's, registered as finbot starts. `/stock symbol` (or `/profile symbol`) is `.symbol`, `/forwardpe` is `.forward pe`, and `/job` is `/jobs`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. + +Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](scheduling.md)), and notifying each chat that schedules them of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. + +The other various reports can also run on a schedule (e.g. daily or weekly), or on demand with a command. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. A scheduled report posts to the chat whose job it is, and a command's reply to the chat that sent it. + +The bot answers commands in every chat it is in. It fetches its updates from each network itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: +* Stock lookup (financials, company profile, price charts) +* A watch list per chat, which anyone in it can change, picked up by the reports there +* Listing of portfolios and portfolio performance +* Listing of holdings sorted by various attributes +* Running the other stock reports on demand + +## Reports + +### Stock lookup +![Screenshot showing stock info on Slack](../img/stockinfo.png?raw=true "Screenshot showing stock info on Slack") + +The stock lookup returns various stats relevant to a stock's valuation, growth and risk factors. + +#### Emoji legend +* The colored circle next to the symbol at the top indicates the current market state: + * 🔴 market closed + * 🟠 pre/post-market + * 🟢 normal trading +* The warning symbol ⚠️ indicates possible risk factors. This is not intended to replace due diligence, but to non-exhaustively highlight where additional diligence is due. Some risk factors include: + * Negative earnings ⚠️ + * Earnings in decline ⚠️ + * Negative cashflow ⚠️ + * High short interest ⚠️ + * High forward P/E ratio ⚠️ + * Net insider selling ⚠️ + * High debt-to-equity ratio ⚠️ +* The arrow grid indicates the directional change in the company earnings and revenue for each reporting period. A red arrow indicates the absolute earnings was negative: + * 🔼 earnings/revenue increased from the previous period + * 🔽 earnings/revenue decreased from the previous period + * 🔺 earnings increased from the previous period but remained negative + * 🔻 earnings decreased from the previous period and was negative + * ▪️ earnings/revenue matched the previous period + * ❌ data is missing for this period +* Using the example screenshot above, Crowdstrike reported: + * 🔼🔼🔼 revenue increased in all of the past three quarters + * 🔺🔻🔻 In the past three quarters, earnings initially increased, then decreased for two quarters, and was negative in every quarter + * 🔼🔼🔼 revenue increased in all of the past three years + * 🔻🔺🔻 In the past three years, earnings decreased, increased, then decreased, and was negative for all three years + +This report is a command only. Example usage: +``` +.AAPL +``` +``` +@botname AAPL +``` + +### Trades +![trade update in Slack](../img/trades.png?raw=true "Trade update in Slack") + +The `trades` report sends recent Sharesight trades to each chat that schedules it. +By default, this report searches Sharesight for trades from the past 28 days (`past_days`). You can override this in `preferences.conf`, or by providing a number as argument when triggered through the chat bot. + +Without persistent storage, set `past_days=0` to avoid duplicate trade notifications. In this case, the schedule must run the report exactly once per day, after market close. + +With persistent storage, the default of `past_days=28` suits. This is useful if Sharesight imports trades with past dates for any reason. Note that the initial run will notify on all historical trades for the `past_days` period. It is recommended to schedule it every 20 minutes. + +``` +past_days=28 +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add */20 * * * Mon-Fri trades`: +``` +trades +``` + +Command: +``` +.trades [period] +``` +``` +@botname trades [period] +``` + +You can also specify a portfolio name to get trades for just that portfolio: +``` +.trades [period] [portfolio] +``` +``` +@botname trades [period] [portfolio] +``` + +### Price alerts +![Price chart triggered from Slack](../img/finbot_price.png?raw=true "Price chart triggered from Slack") +![Price alert triggered by Cron](../img/price.png?raw=true "Price alert triggered by Cron") + +The `price` report sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. + +If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. + +Config example: +``` +price_percent=5 +demote_volatile=True +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add 29 21 * * Mon-Fri price interday`: +``` +price premarket|midsession|interday|close|DAYS +``` +When scheduled, the mode is its argument: +* If `premarket` is passed, it only reports on pre/post market price movements. +* If `midsession` is passed, it only reports for markets currently in session. This is intended to run on a schedule to provide mid-session alerts for big price movements of your holdings. For example, it could be run twice per day 12 hours apart, to capture markets in different timezones. +* If `interday` is passed, it reports current price against the previous market close. +* If `close` is passed, it reports daily change in stocks whose market closed in the past 3 hours. +* If an integer [i] is passed, it reports current price against [i] days ago. + +Command (pre-market): +``` +.premarket [percent] +``` +``` +@botname premarket [percent] +``` + +Command (mid-session): +``` +.session [percent] +``` +``` +@botname session [percent] +``` + +Command (interday). +``` +.price [percent] +``` +``` +@botname price [percent] +``` + +For interday, can also specify the time period. A bare number is the percent; a period carries its unit: `7d`, `2w`, `3m`, `1y` or `ytd`. For example, `.price 3 7d` lists the moves of 3% or more over the past week. +``` +.price [percent] [period] +``` + +Or a specific stock. In this case, the output includes a graph. +``` +.price [symbol] [period] +``` + +### Portfolio performance +![Screenshot of Portfolio performance notification on Slack](../img/performance.png?raw=true "Screenshot performance notification on Slack") + +The `performance` report covers Sharesight portfolio performance over a given period. The default is set by `past_days` in the `preferences.conf`, which can be overridden at runtime by specifying an integer as the argument. + +Config example: +``` +past_days=28 +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add 28 21 * * Sat performance 7`: +``` +performance [DAYS] +``` + +Command: +``` +.performance [period] +``` +``` +@botname performance [period] +``` + +As a command, it also takes a portfolio: +``` +.performance [period] [[portfolio] +``` +``` +@botname performance [period] [portfolio] +``` + +### Events calendar +![earnings message in Slack](../img/earnings.png?raw=true "Earnings message in Slack") + +The `cal` report sends upcoming earnings and ex-dividend date alerts. The data is sourced from Yahoo! Finance. It reports on events up to `future_days` into the future. This is set in the `preferences.conf` for a scheduled run, or can be specified as an argument when triggered through the chat bot. + +**Finance Explainer:** When a company releases its quarterly earnings report, the stock price may undergo a signficant positive or negative movement, depending on whether the company beat or missed market expectations. You may wish to hold off buying more of this stock until after its earnings report, unless you think the stock will beat market expectations. + +**Finance Explainer:** When a stock goes ex-dividend, the share price [typically drops](https://www.investopedia.com/articles/stocks/07/ex_dividend.asp) by the amount of the dividend paid. If you buy right before the ex-dividend date, you can expect an unrealised capital loss, plus a tax obligation for the dividend. Thus, you may wish to wait for the ex-dividend date before buying more of this stock. +``` +future_days=30 +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add 28 21 * * Sat cal earnings`: +``` +cal earnings|ex-dividend +``` + +Command (earnings): +``` +.earnings [period] +``` +``` +@botname earnings [period] +``` + +Command (ex-dividend): +``` +.dividend [period] +``` +``` +@botname dividend [period] +``` + +### Highly shorted stock warnings +![short warnings in Slack](../img/shorts.png?raw=true "Short warnings in Slack") + +The `shorts` report sends short interest warnings. The data is sourced from Yahoo Finance and Shortman (AU). `shorts_percent` defines the alert threshold for the percentage of a stock's float shorted. This can be specified in the `preferences.conf` for a scheduled run, or as an argument when triggered through the chat bot. + +**Finance Explainer:** A high short ratio indicates a stock is exposed to high risks, such as potential banktrupcy. It may also incentivise negative news articles which harm the stock price. If the market is wrong, however, risk tolerant investors may receive windfall gains. This report is intended to alert you to an above-average risk, and prompt you to investigate this stock more closely. +``` +shorts_percent=15 +``` + +Command: +``` +.shorts [percent] +``` +``` +@botname shorts [percent] +``` + +### pe / forward pe / peg +![Forward PE report in Slack](../img/finbot_fpe.png?raw=true "Forward PE report in Slack") + +List securities by P/E ratio, forward P/E ratio, or PEG ratio. A forward P/E is an analysts' estimate, so its lists take only stocks covered by at least `min_analysts` analysts (7 by default; see `preferences.conf`). + +Command: +``` +.pe [SYMBOL] +.fpe [SYMBOL|bottom|negative] +.peg [SYMBOL|bottom|negative] +``` +``` +@botname pe [SYMBOL|top|bottom] +@botname fpe [SYMBOL|top|bottom|negative] +@botname peg [SYMBOL|top|bottom|negative] +``` + +### History +![Price history report in Slack](../img/finbot_history.png?raw=true "Price history report in Slack") + +Shows the long term performance of a security. +It creates a 5Y chart, and then lists performance for several periods (5Y 3Y 1Y 6M 3M 1M 1W 1D) + +Command: +``` +.history +``` +``` +@botname history +``` + +### Buy +![Buy ratings in Slack](../img/finbot_buy.png?raw=true "Buy ratings in Slack") + +Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts (7 by default; see `preferences.conf`) + +Command: +``` +.buy +``` +``` +@botname buy +``` + +### Sell +![Rating report in Slack](../img/finbot_sell.png?raw=true "Rating report in Slack") + +Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts + +Command: +``` +.sell +``` +``` +@botname sell +``` + +### Marketcap +![Marketcap report in Slack](../img/finbot_marketcap.png?raw=true "Marketcap report in Slack") + +List the top (or bottom) stocks by market cap, in US dollars: market caps in other currencies are converted at Yahoo's exchange rates. With a symbol, its market cap in its own currency and in US dollars. + +Command: +``` +.marketcap [top|bottom|SYMBOL] +``` +``` +@botname marketcap [top|bottom|SYMBOL] +``` + +### Consensus rating changes +![Rating change report in Slack](../img/finbot_rating.png?raw=true "Rating change report in Slack") + +Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run on a schedule, such as `.jobs add 28 21 * * Sat rating` (see [Scheduling](scheduling.md)). Requires persistent storage. + +``` +rating +``` + +### Watchlist +![Shared watchlist in Slack](../img/watchlist.png?raw=true "Shared watchlist in Slack") + +Tracks additional securities which are not in your Sharesight holdings. Each chat has its own watchlist, which anyone in the chat can change: the reports in that chat, asked for or scheduled, include its watchlist's stocks as if they were holdings, and no other chat's. The Sharesight holdings are the same in every chat. `yahoo_warm` keeps every chat's watchlist warm in the cache. + +The watchlists are kept in `finbot_watchlist.json` in the data directory, by chat (`{"chats": {"telegram:-1001234": ["2454.TW", "ASO.AX", "STEM"]}}`), so there is no need to edit the file. The first start with watchlists per chat gives each home chat the watchlist kept before, and the same goes for the plans that `.plan` records: each chat's `.plan` lists only the plans recorded in it. + +Command: +``` +.watchlist +``` +``` +.watchlist [add|del] [symbol] +``` +``` +@botname watchlist +``` +``` +@botname watchlist [add|del] [symbol] +``` diff --git a/doc/configuration.md b/doc/configuration.md new file mode 100644 index 00000000..fbf2fa8b --- /dev/null +++ b/doc/configuration.md @@ -0,0 +1,141 @@ +# Configuration + +Configuration is split into two files -- finbot.env for startup environment variables, and preferences.conf for values that can be changed at runtime. + +Copy the shipped template: +``` +sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env +``` + +## `finbot.env` +**Load `finbot.env` with your launcher:** +* **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. +* **Docker:** `compose.yaml` loads it, and checks it before the bot starts (see [docker.md](docker.md)). Write the values without quotes. +* **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). +* **Scheduled jobs:** they are run by `bot.py` (see [Scheduling](scheduling.md)), so they run on the same settings. For a one-off run, send `.jobs run` in the chat. + +The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which +of the chat network keys it needs. + +| Key | Default | Description | +|---|---|---| +| `DATA_DIR` | `var/` in the checkout (`$STATE_DIRECTORY` under systemd `StateDirectory=`) | Where state and cache live | +| `ip` | `127.0.0.1` | Address the bot listens on | +| `port` | `5000` | Port the bot listens on | +| `scheduler_catchup` | `5` | Minutes: a run missed while the bot was down still fires if it was due this recently | +| `telegramChatID` | | Telegram's home chat, a group or channel id: finbot's first start gives it the default jobs, and error reports go there (see [Scheduling](scheduling.md)) | +| `sharesight_code` | | Sharesight API access | +| `sharesight_client_id` | | Sharesight API access | +| `sharesight_client_secret` | | Sharesight API access | +| `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | +| `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | +| `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | +| `slackChannelID` | | Slack's home channel id, as `telegramChatID` (with `slackBotToken`) | +| `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | +| `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | +| `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | +| `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | +| `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | +| `discordBotToken` | | Discord bot token: replies, and commands over the gateway unless `discordOutgoingWebhook` is set | +| `discordChannelID` | | Discord's home channel id, as `telegramChatID` (with `discordBotToken`) | +| `discordOutgoingWebhook` | | The Interactions Endpoint URL Discord posts slash commands and button presses to (HTTP mode); empty, the gateway (pull mode) | +| `discordPublicKey` | | The app's public key, which checks the signature of each interaction (HTTP mode) | + +## `preferences.conf` items + +Copies preferences.conf and metadata files into the data directory (never overwrites) +``` +util/sync_defaults.py [--dest DIR] --apply +``` + +The preferences, read by the bot itself. A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. + +| Key | Default | Description | +|---|---|---| +| `past_days` | `28` | How many days back reports look | +| `future_days` | `30` | How many days ahead the events calendar looks | +| `price_percent` | `5` | Price alert threshold, percent | +| `shorts_percent` | `15` | Short interest alert threshold, percent | +| `min_analysts` | `7` | `.buy`, `.sell` and `.forward pe` list only stocks covered by at least this many analysts; with fewer, a consensus says little (thinly covered small caps are nearly all a perfect buy) | +| `demote_volatile` | `True` | Volatile (small cap, high beta) stocks need a bigger move to alert | +| `demote_leveraged` | `True` | Leveraged (2x, 3x) funds need a bigger move to alert | +| `volatility_multiplier` | `2` | How much bigger a move volatile stocks need | +| `exclude_portfolios` | | Sharesight portfolio IDs to leave out, space separated | +| `include_portfolios` | | Sharesight portfolio IDs to report on, space separated (empty: all that are not excluded) | +| `trades_use_yahoo` | `True` | Use Yahoo for trades; set `False` if Yahoo breaks | +| `graph` | `True` | Send charts with reports | +| `hyperlink` | `True` | Hyperlink tickers in reports | +| `hyperlinkFooter` | `True` | Add a website link footer to stock lookups | +| `hyperlinkProvider` | `google` | Where ticker links go: `google` or `yahoo` | +| `exchange_suffix` | `AX` | Regional: Yahoo's suffix for your exchange (`AX` for the ASX, `NZ`, `L`, `TO` and so on). A bare symbol Yahoo does not know is tried again with it, in every command that takes a symbol (`BHP` becomes `BHP.AX`); empty, a symbol is taken as written | +| `country_code` | `AU` | Regional: your country, for the reminder report | +| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report, and for each job's next run in `.jobs` (besides UTC) | +| `cache` | `True` | Cache Yahoo and Sharesight responses | +| `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | +| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm` runs, so the holdings never run out | +| `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | +| `debug` | `False` | Extra diagnostics in the log | +| `print_headers` | `False` | Log inbound request headers | + +A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. + +edit `preferences.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. + +**Upgrading from one `.env`?** finbot no longer reads it, and refuses to start while a `.env` is in the checkout, so its settings cannot be silently ignored. Run `util/split_env.py` (or `util/split_env.py /path/to/old.env` for one elsewhere). It reads either format, the old dotenv one (`key = 'value' # comment`) or plain `KEY=value`, writes `finbot.env.new` and `preferences.conf.new` next to the old file, never overwriting, and lists any key it dropped or did not recognise. Put `finbot.env` where your launcher loads it and `preferences.conf` in the data directory, then delete the old file. + +## Data directory +All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`); in Docker it is `/data`, the host's `/opt/finbot-data` (see [docker.md](docker.md)). +Reference data (`finbot_adr.json`, `finbot_sws_*.json`, `reminder.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: +``` +util/sync_defaults.py # dry run +util/sync_defaults.py --apply # copy missing files (including a starter preferences.conf) +util/sync_defaults.py --force # also replace changed files, keeping .bak copies +util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory +util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 750) if it does not exist yet +``` +`reminder.json` holds the reminders that the `reminder` report posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. + +With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. + +In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. Each start copies the defaults the data directory lacks (`util/sync_defaults.py --apply`). + +## Sharesight +* Email Sharesight support to get an API key and add the [access details](https://portfolio.sharesight.com/oauth_consumers) to `finbot.env`. Example: +``` +sharesight_code=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f +sharesight_client_id=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f +sharesight_client_secret=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f +``` + +## Portfolios +Portfolios are auto-discovered, including other people's portfolios which are shared to you. To exclude specific portfolios, add their IDs to `exclude_portfolios` in the `preferences.conf`: +``` +exclude_portfolios=100003 100004 +``` + +Alternatively, you can include only specific portfolios: +``` +include_portfolios=100001 100002 +``` + +## Hyperlinks +These `preferences.conf` options relate to hyperlinks: + +* `hyperlink`, if set to `False`, disables all hyperlinks. Note: it can not disable syntax highlighting and false positives by chat clients that try to detect domains. Defaults to `True`. +``` +hyperlink=True +``` + +* `hyperlinkFooter` appends hyperlinks to the company website and to a Google search for the ticker on simplywall.st for the symbol lookup report. Possible values are `True` or `False`. Defaults to `True`. + +``` +hyperlinkFooter=True +``` + +* `hyperlinkProvider` configures where to send ticker hyperlinks. Available values are `google` and `yahoo`. If set to `google`, it is only used where practical, as Google does not publish earnings date, ex-dividend date, premarket price, or short interest figures. Defaults to `google`. +``` +hyperlinkProvider=google +``` +``` +hyperlinkProvider=yahoo +``` diff --git a/doc/discord.md b/doc/discord.md index 8603d878..80907301 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -39,7 +39,7 @@ change both, and restart finbot, to switch. server. 4. In Discord, turn on _User Settings > Advanced > Developer Mode_; then right-click the channel to be Discord's home chat, _Copy Channel ID_, and put it in `finbot.env` as `discordChannelID`. finbot's first start gives that channel - the default jobs ([Scheduling](../README.md#scheduling)), and it gets the error reports. Every channel manages its + the default jobs ([Scheduling](scheduling.md)), and it gets the error reports. Every channel manages its own jobs with `.jobs`. 5. Install discord.py: `sudo apt install python3-discord` (or `pip install discord.py`). 6. Restart finbot (`sudo systemctl restart finbot`). As it starts, it registers the slash commands, which can take a diff --git a/doc/install.md b/doc/install.md new file mode 100644 index 00000000..12624d13 --- /dev/null +++ b/doc/install.md @@ -0,0 +1,72 @@ +# Installing on a host + +finbot runs on any Linux host with Python 3.12 or later; production runs Debian's own packages under systemd, as +below. To run it in a container instead, see [docker.md](docker.md). + +## Requirements +* Most features need a Sharesight paid plan, preferably with automatic trade imports, and an API key + ([configuration.md](configuration.md#sharesight)). +* A Telegram bot, a Slack app or a Discord bot: see the network's guide, [Telegram](telegram.md), [Slack](slack.md) or + [Discord](discord.md). +* No server: finbot fetches its updates from each network itself (pull mode: Telegram's long polling, Slack's Socket + Mode, Discord's gateway). In HTTP mode instead, a network posts them to a public web server with a domain name and a + matching certificate, which proxies them to the bot (each guide's _HTTP mode_). +* Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum), and these modules, pinned to versions with + 3.13 support in `requirements.txt`: croniter, python-dateutil, requests, matplotlib and numpy. +* For a network in HTTP mode (its webhook URL set), also waitress; for Slack in Socket Mode, websocket-client; for + Discord's gateway, discord.py, and for its HTTP mode, PyNaCl. These are in `requirements-optional.txt`, each imported + only by the mode that needs it. + +## Debian +1. The packages, and those of the modes you use: + ``` + sudo apt install git python3 python3-requests python3-croniter python3-dateutil python3-matplotlib python3-numpy + sudo apt install python3-waitress # any network in HTTP mode + sudo apt install python3-websocket # Slack's Socket Mode + sudo apt install python3-discord # Discord's gateway + sudo apt install python3-nacl # Discord's HTTP mode + ``` +2. The code: + ``` + git clone https://github.com/robdevops/finbot.git ~/finbot + ``` +3. The startup file, from its template; then set each network's keys from its guide, and Sharesight's: + ``` + sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env + sudoedit /opt/finbot.env + ``` +4. Check it: `~/finbot/util/check_config.py /opt/finbot.env` +5. Run it under systemd, below. The first start makes the data directory (`var/` in the checkout, or + `/var/lib/finbot` with `DynamicUser=`) and gives each home chat its first jobs ([scheduling.md](scheduling.md)). + +[configuration.md](configuration.md) has every key and preference. + +## Other distributions: pip or uv +``` +sudo $(which apt dnf yum) install git python3-pip +git clone https://github.com/robdevops/finbot.git ~/finbot +cd ~/finbot +pip3 install -r requirements.txt -r requirements-optional.txt +``` +Or, with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: +``` +uv venv && uv pip install -r requirements.txt -r requirements-optional.txt +``` +`requirements-optional.txt` can be left out, or cut down to the modes you use. + +## Daemonize (systemd) +`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy +`finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of +[`finbot.env`](configuration.md#finbotenv) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start +it. `systemctl reload finbot` re-reads `preferences.conf` without a restart; a change to `finbot.env` needs +`systemctl restart finbot`: +``` +sudo cp -v ~/finbot/finbot.service /etc/systemd/system/ +sudo sed -i 's/CHANGEME/YOUR LINUX USERNAME/' /etc/systemd/system/finbot.service +sudo systemctl daemon-reload +sudo systemctl enable finbot --now +``` +The bot's log: +``` +journalctl -fu finbot +``` diff --git a/doc/scheduling.md b/doc/scheduling.md new file mode 100644 index 00000000..94ee1421 --- /dev/null +++ b/doc/scheduling.md @@ -0,0 +1,54 @@ +# Scheduling + +Each chat has its own jobs: reports that run on a schedule and post to that chat. Anyone in the chat manages them there, +with `.jobs` (also `.job`, `.cron`, `.crons` or `.schedule`): +``` +.jobs this chat's jobs, numbered, each with its next run; then the system jobs +.jobs add the reports a job can run, and what each takes +.jobs add 29 21 * * Mon-Fri price interday +.jobs run the list, with a button per job that runs it now; .jobs run 2 runs job 2 +.jobs toggle the list, with a button per job that switches it on or off; .jobs toggle 2 +.jobs del 2 deletes job 2 +``` + +A job is a time in crontab's syntax, then one or more reports with their arguments: +* **The time** is five fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists + work) or an `@macro` such as `@daily`. A time that cannot be or never comes (minute 61, 30 February) is refused, + saying why. +* **Times are in UTC**, which keeps the ASX's jobs and NYSE's each within an hour of their local times all year. To + follow one market's clocks exactly, a job names its zone first, as crontab writes it: + `.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession`. `.jobs` shows each next run in UTC and in + the `timezone` preference. +* **The reports** are `trades`, `price midsession|interday|premarket|close|DAYS`, `cal earnings|ex-dividend`, + `performance [DAYS]`, `rating`, `milestone`, `shorts`, `reminder` and `podcasts FEED_URL`. Several, separated by `;`, + run one after another: `.jobs add 28 21 * * Sat cal earnings; cal ex-dividend; price 7; performance 7; rating`. + Quote an argument that holds spaces or a `;` (a feed URL, say). +* A chat has 30 jobs at most. + +A job posts to its own chat, whichever way that chat's updates reach finbot: the jobs are kept by network and chat id +(`finbot_schedule.json` in the data directory), so switching a network between pull and HTTP mode keeps them. Each chat +also keeps its own record of what a report has told it (trades, rating changes, record highs, podcast episodes), so the +same report scheduled in two chats tells each of them once. A run from `.jobs run` is a scheduled run: it posts to the +chat and updates the same record. A change counts at once, with no reload. + +**The system jobs**, fixed in `lib/jobs.py` and listed by `.jobs` under _System_, keep the cache full and tidy: +`sharesight_warm` every 30 minutes, `yahoo_warm` every 8 hours (see [Caching](caching.md)), and `cleanup 30` weekly, +which deletes cache files older than 30 days. A chat cannot add, switch or run them. + +**From `schedule.cron`**, where the schedule used to be: the first time finbot starts without `finbot_schedule.json`, +it gives each home chat (`telegramChatID`, `slackChannelID`, `discordChannelID`) the jobs of the data directory's +`schedule.cron`, else of the shipped `defaults/schedule.cron`, each in the zone it ran in (the server's), without the +system jobs. It copies the reports' records to each home chat too, so that nothing old is told again. It logs what it +did, and any line it left out, and reads the file no more: delete the data directory's copy afterwards. + +Behaviour worth knowing: +* The reports run in `bot.py` itself, each in a thread of its own, sharing the bot's user, sandbox, environment, data + directory, caches and connections; charts are drawn one at a time, and a slow report holds up no chat command. There + is no cron to set up: remove any old crontab entries that ran finbot's scripts, or their reports come twice. +* A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last + `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. A job that is switched off + does not fire, and does not catch up once it is switched on again. +* A job still running when it is next due is skipped. A command still running after 5 minutes is reported once, and + left to finish (a thread cannot be stopped, but every request it makes has a timeout); until it does, it is not + started again. Each command of a job has 5 minutes of its own. +* A failure is reported to the job's chat. The bot logs each job as it runs (`journalctl -u finbot`). diff --git a/doc/slack.md b/doc/slack.md index f3889c50..c3d3dd01 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -2,7 +2,7 @@ finbot reaches Slack through a Slack app of your own, whose bot posts each channel's scheduled reports there, charts and all, and answers commands: `/price`, `.price`, `@finbot price` and the rest -([the list](../README.md#supported-commands)), with charts in its replies, ⏳ on a command's message while it works +([the list](commands.md#supported-commands)), with charts in its replies, ⏳ on a command's message while it works (Slack has no typing notice for bots), and period buttons under a chart that redraw it in place. ## The bot @@ -103,7 +103,7 @@ slackChannelID=C0123456789 in its _Messages_ tab. 4. Set `slackChannelID` to the channel's id (in Slack, click the channel's name: the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. It is Slack's home chat: finbot's first start gives it the - default jobs ([Scheduling](../README.md#scheduling)), and it gets the error reports. Every channel manages its own + default jobs ([Scheduling](scheduling.md)), and it gets the error reports. Every channel manages its own jobs with `.jobs`. An app set up by hand before takes the manifest too: paste it into _Features > App Manifest_ and _Save Changes_; diff --git a/doc/telegram.md b/doc/telegram.md index b2363e38..e74c381a 100644 --- a/doc/telegram.md +++ b/doc/telegram.md @@ -14,7 +14,7 @@ Telegram: | `finbot.env` keys | `telegramBotToken`, `telegramChatID` | Those two, plus `telegramOutgoingWebhook` and `telegramOutgoingToken` | Everything else is the same in both: the scheduled reports with their charts, the chat commands (`/price` or `.price`, -`.history`, `.performance` and the rest, [the list](../README.md#supported-commands)), period buttons under charts and +`.history`, `.performance` and the rest, [the list](commands.md#supported-commands)), period buttons under charts and the buttons of `.jobs run` and `.jobs toggle`, "typing…" while a command works, and a keyboard of the common commands in a direct chat with the bot. @@ -49,7 +49,7 @@ arrive after the switch. * A group's id changes when Telegram turns it into a supergroup, which some changes to its settings do. Update `telegramChatID` then: finbot does not follow the change. 6. Start finbot (`sudo systemctl restart finbot`). The log says `telegram: long polling for updates`. The first start - gives this chat, Telegram's home chat, the default jobs ([Scheduling](../README.md#scheduling)), so the scheduled + gives this chat, Telegram's home chat, the default jobs ([Scheduling](scheduling.md)), so the scheduled reports arrive there; it gets the error reports too. Every chat manages its own jobs with `.jobs`. 7. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. diff --git a/tests/test_edges.py b/tests/test_edges.py index 4d91e712..795b220a 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -237,7 +237,7 @@ def test_keywords_in_any_case(self): self.assertIn('Top 10 performers', str(self.command('.price TOP 7d'))) def test_a_bare_number_on_price_is_a_percent(self): - """As on .session, .premarket and .shorts, and as the README has it; a period carries its unit.""" + """As on .session, .premarket and .shorts, and as doc/commands.md has it; a period carries its unit.""" self.assertIn('3% at close', str(self.command('.price 3'))) # it used to be a 3-day report self.assertIn('Price movement ≥ 3%', str(self.command('.price 3 7d'))) # the 3 used to be dropped self.assertIn('Price movement ≥ 5%', str(self.command('.price 7d'))) diff --git a/tests/test_sending.py b/tests/test_sending.py index 50ac5242..52c15eec 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -192,7 +192,7 @@ def test_scheduled_reports_carry_their_charts_to_every_home_chat(self): self.assertEqual(sorted(set(sent)), [('discord', 'chart'), ('slack', 'chart'), ('telegram', 'chart')]) def test_the_graph_preference_off_sends_no_chart_anywhere(self): - """graph=False means no charts, as the README says; it used to hold back only the price-line charts.""" + """graph=False means no charts, as doc/configuration.md says; it held back only the price-line charts.""" self.prefs(graph=False) self.env(slackBotToken='xoxb-test') diff --git a/util/check_config.py b/util/check_config.py index 4cd41a07..667ffcd7 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -6,7 +6,7 @@ finbot.env (the startup file) is never read by the bot, so this is the only place its typos are caught: an unknown key, a preference key in it, a malformed line or a bad value. preferences.conf defaults to /preferences.conf and is checked the way the bot checks it at start and on reload. Exits 1 and prints every problem if either file is wrong, so a -launcher can refuse to start (see the README).""" +launcher can refuse to start (see doc/install.md and doc/docker.md).""" import argparse import os From 6b8fe3cf6861425f0c3c8369de89d43fb9a15fac Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:27:02 +0000 Subject: [PATCH 17/31] Split the protocol code per network: common, webhook, and the persistent client Each network's code is now in three modules, flat in lib/: - telegram_common.py, slack_common.py, discord_common.py: what both ways in share: the API calls, the reader of the network's updates (READER) and its bot (TelegramBot, SlackBot, DiscordBot); - telegram_webhook.py, slack_webhook.py, discord_webhook.py: the HTTP mode (the path, and how a request proves itself); - telegram_longpolling.py, slack_websocket.py, discord_websocket.py: the pull mode's persistent client. lib/inbound.py, lib/pull.py and lib/transports.py keep only what no network owns: the update types and the Network, Webhook, Listener and Transport bases. lib/telegram.py and lib/discord.py are gone. bot.py puts the networks together: WEBHOOKS and route(), LISTENERS and listeners() (each listener says whether finbot.env wants it), and each network's registration at start. A pure refactor: the code moved verbatim but for the references that the move changes. slack_api is now slack_common.call, like the other two networks' calls; Telegram's register() is split between its two mode modules, and getMe became telegram_common.bot_name(). util/record_outputs.py finds a network's reader in either layout, and records 268 steps identical to HEAD. The tests follow the code to its new modules; CLAUDE.md's code map describes the new layout. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 44 ++-- bot.py | 37 ++- lib/delivery.py | 14 +- lib/discord.py | 89 ------- lib/discord_common.py | 291 ++++++++++++++++++++++ lib/discord_webhook.py | 37 +++ lib/discord_websocket.py | 79 ++++++ lib/inbound.py | 313 +----------------------- lib/pull.py | 237 +----------------- lib/slack_common.py | 342 ++++++++++++++++++++++++++ lib/slack_webhook.py | 60 +++++ lib/slack_websocket.py | 120 +++++++++ lib/telegram.py | 66 ----- lib/telegram_common.py | 212 ++++++++++++++++ lib/telegram_longpolling.py | 48 ++++ lib/telegram_webhook.py | 43 ++++ lib/transports.py | 472 +----------------------------------- lib/worker.py | 4 +- tests/common.py | 10 +- tests/test_discord.py | 81 ++++--- tests/test_edges.py | 19 +- tests/test_finbot.py | 4 +- tests/test_inbound.py | 17 +- tests/test_pull.py | 42 ++-- tests/test_reports.py | 20 +- tests/test_scheduler.py | 8 +- tests/test_sending.py | 82 ++++--- tests/test_telegram_api.py | 43 ++-- util/record_outputs.py | 13 +- 29 files changed, 1494 insertions(+), 1353 deletions(-) delete mode 100644 lib/discord.py create mode 100644 lib/discord_common.py create mode 100644 lib/discord_webhook.py create mode 100644 lib/discord_websocket.py create mode 100644 lib/slack_common.py create mode 100644 lib/slack_webhook.py create mode 100644 lib/slack_websocket.py delete mode 100644 lib/telegram.py create mode 100644 lib/telegram_common.py create mode 100644 lib/telegram_longpolling.py create mode 100644 lib/telegram_webhook.py diff --git a/CLAUDE.md b/CLAUDE.md index 17da54c9..c3243621 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,19 +38,25 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm record real requests. ## Code map (open the one file you need) -- `bot.py`: WSGI entry (route, verify, parse, then a thread of `lib/worker.py` per message or button press), served by - waitress only when some network's webhook is set; plain threads, no gevent; SIGHUP reload and SIGUSR1 cache flush, - each in a thread. `lib/inbound.py`: a class per network that reads its updates - (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back) and acknowledges a press where - the network waits for that (`acknowledge`: Telegram's toast), apart from a class per way they arrive - (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an update - however it arrived, and is the one place a press is acknowledged (the worker never does). `lib/pull.py`: the pull - modes, a listener per network whose webhook URL is not set (`TelegramPoll`: getUpdates; `SlackSocket`: Socket Mode, - over websocket-client; `DiscordGateway`: discord.py's raw events), each in a thread of its own handing its updates to - `bot.take`, waiting longer after each failure in a row (`backoff`). An answer the network must have before any reply - (Discord's deferred interaction) carries the work in its `then`: started once the HTTP answer is sent (`bot.Sent`) - or, over the gateway, posted to its callback. `lib/discord.py`: Discord's REST calls (`call`, rate limits waited - out) and the slash commands registered at start (`register`, from `worker.slash_commands()`). +- `bot.py`, which puts the networks together: the WSGI entry (`route()` by path to a network's `WEBHOOK`, verify, + parse, then a thread of `lib/worker.py` per message or button press), served by waitress only when some network's + webhook is set; the pull listeners that finbot.env asks for (`listeners()`), a thread each; each network registered + at start (Telegram's webhook set or deleted, and its menu; Discord's slash commands); plain threads, no gevent; + SIGHUP reload and SIGUSR1 cache flush, each in a thread. `bot.take(network, update)` takes an update however it + arrived, and is the one place a press is acknowledged (the worker never does). An answer the network must have + before any reply (Discord's deferred interaction) carries the work in its `then`: started once the HTTP answer is + sent (`bot.Sent`) or, over the gateway, posted to its callback. +- Each network in three modules, flat in `lib/`. `_common.py`, what both ways in share: the API calls (`call`, + `try_call`; Discord's wait out its rate limits), the reader of its updates (`READER`: a `Message`, a `Press`, a + `Suggest` or a `Response` given straight back; `acknowledge` where the network waits for that: Telegram's toast, + Discord's interaction answer) and its bot (`TelegramBot`, `SlackBot`, `DiscordBot`). `_webhook.py`, the HTTP + mode: the path and how a request proves itself (`WEBHOOK`). The pull mode's persistent client, handing its updates + to `bot.take`: `telegram_longpolling.py` (`TelegramPoll`: getUpdates), `slack_websocket.py` (`SlackSocket`: Socket + Mode, over websocket-client), `discord_websocket.py` (`DiscordGateway`: discord.py's raw events). Telegram's + `register()`, in each mode's module, sets the webhook or deletes it; Discord's slash commands are registered at start + in both modes (`discord_common.register`, from `worker.slash_commands()`). What no network owns: `lib/inbound.py` + (`Message`, `Press`, `Suggest`, `Response`; the `Network` and `Webhook` bases), `lib/pull.py` (the `Listener` base, + which waits longer after each failure in a row: `backoff`), `lib/transports.py` (the `Transport` base, below). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, keywords)`, which also makes `.help`, the Telegram command keyboard, kept current in DMs, and `slash_commands()`, which every network gets (Telegram's menu, Slack's manifest from `util/slack_manifest.py`, Discord's registered commands); @@ -68,12 +74,12 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). - `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, a chat by its key with - `chat(key)`, or each network's home chat as its bot), `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per network's bot - (`TelegramBot`, `SlackBot`, `DiscordBot`), each taking charts and buttons: splitting, captions, charts, buttons, - edits, and the typing notice (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and - `chart()` stop before posting. `DiscordBot`: a slash command's first message replaces its "thinking…" through its - `interaction` (application id/token), and a pressed message is edited through the press's. `lib/telegram.py`: Bot API - calls (`call`, `try_call`). + `chat(key)`, or each network's home chat as its bot), `deliver(dest, report)`, the one-line error report. + `lib/transports.py`: `Transport`, the base of each network's bot, every one taking charts and buttons: splitting, + captions, charts, buttons, edits, and the typing notice (`start_typing`: after half a second; Slack's ⏳ on the + message), which `text()` and `chart()` stop before posting. `DiscordBot`: a slash command's first message replaces + its "thinking…" through its `interaction` (application id/token), and a pressed message is edited through the + press's. - `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), what a pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. `lib/charts.py`: a chart with period buttons, built by its kind's builder (`build`) for the command's first view diff --git a/bot.py b/bot.py index eef5e967..89eac3a2 100755 --- a/bot.py +++ b/bot.py @@ -9,14 +9,19 @@ from pathlib import Path from lib import config -from lib import discord +from lib import discord_common +from lib import discord_webhook +from lib import discord_websocket from lib import inbound from lib import worker from lib import jobs from lib import migrate from lib import net -from lib import pull -from lib import telegram +from lib import slack_webhook +from lib import slack_websocket +from lib import telegram_common +from lib import telegram_longpolling +from lib import telegram_webhook from lib import delivery from lib import render from lib.reports import cleanup @@ -24,6 +29,8 @@ ROUTED = 'finbot.routed' # a request's WSGI environ: the network whose webhook path it came to (the path is a secret) AUTHENTICATED = 'finbot.authenticated' # set once it has passed that network's check MAX_BODY = 1 << 20 # bytes in a request body, at most: an update from Telegram or Slack takes a few thousand +WEBHOOKS = (telegram_webhook.WEBHOOK, slack_webhook.WEBHOOK, discord_webhook.WEBHOOK) # HTTP: each network posts +LISTENERS = (telegram_longpolling.TelegramPoll, slack_websocket.SlackSocket, discord_websocket.DiscordGateway) # pull _updates_seen = collections.OrderedDict() # (service, update id) of the last updates taken, oldest first _updates_lock = threading.Lock() # updates arrive on several threads: a resent one may race its first copy @@ -42,6 +49,15 @@ def seen_before(service, update_id, keep=1000): _updates_seen.popitem(last=False) return False +def route(path): + """The webhook whose path this is, or None.""" + return next((hook for hook in WEBHOOKS if hook.path() and path == hook.path()), None) + +def listeners(): + """The listeners finbot.env asks for, each network's with its bot token and no webhook URL (Slack's also needs its + app-level token): finbot fetches the updates of those networks itself.""" + return [kind() for kind in LISTENERS if kind.wanted()] + def main(environ, start_response): """WSGI entry point. Never lets an exception escape. A failure handling an authenticated request is reported as one line to the default channel and answered 200, so the service does not retry it into the chat again. Before @@ -83,7 +99,7 @@ def print_headers(): if config.prefs.print_headers: print_headers() path = environ.get('PATH_INFO', '') - source = inbound.route(path) + source = route(path) if source is None: print("Unknown URI", repr(path), file=sys.stderr) return respond(inbound.NOT_FOUND) @@ -217,7 +233,7 @@ def flush_cache(*args): def http_server(): """The HTTP server for the networks that post to finbot (those whose webhook is set), bound to ip and port, or None when none does. waitress is imported only then. Raises OSError when the port cannot be had.""" - if not any(hook.path() for hook in inbound.WEBHOOKS): + if not any(hook.path() for hook in WEBHOOKS): return None import waitress return waitress.create_server(main, host=config.env.ip, port=config.env.port) @@ -249,22 +265,23 @@ def http_server(): else: print('No webhook is set: finbot takes no HTTP requests', file=sys.stderr) if config.env.telegramBotToken: - try: - inbound.TELEGRAM.bot_name = telegram.register() # here only: the job processes import lib.telegram too + try: # here only, never on import: the webhook set (HTTP mode), or deleted for getUpdates (pull mode) + (telegram_webhook if config.env.telegramOutgoingWebhook else telegram_longpolling).register() + telegram_common.READER.bot_name = telegram_common.bot_name() except Exception as e: print('ERROR', delivery.error_line(e, 'telegram'), file=sys.stderr) sys.exit(1) - telegram.set_commands(worker.slash_commands(aliases=False)) # the menu; /profile and /job work typed + telegram_common.set_commands(worker.slash_commands(aliases=False)) # the menu; /profile and /job work typed threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background if config.env.discordBotToken: try: - inbound.DISCORD.bot_id = discord.register(worker.slash_commands()) + discord_common.READER.bot_id = discord_common.register(worker.slash_commands()) except Exception as e: print('ERROR', delivery.error_line(e, 'discord'), file=sys.stderr) sys.exit(1) jobs.current = scheduler # for the .jobs chat command threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs - for listener in pull.listeners(): # the networks without a webhook: finbot fetches their updates + for listener in listeners(): # the networks without a webhook: finbot fetches their updates threading.Thread(target=listener.run, args=(take,), daemon=True).start() # Startup is done. systemctl reload finbot: pick up an edited preferences.conf signal.signal(signal.SIGHUP, in_thread(reload_all)) diff --git a/lib/delivery.py b/lib/delivery.py index 0a4f873b..ebfa4699 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -1,13 +1,16 @@ """Where a report goes and how: Report, the destinations (the chat that asked, or the default chats), deliver(), and the -one-line error report. A destination is a transport (lib/transports.py).""" +one-line error report. A destination is a transport: a network's bot (TelegramBot, SlackBot, DiscordBot).""" import dataclasses import html import sys import traceback from lib import config +from lib import discord_common from lib import net from lib import render +from lib import slack_common +from lib import telegram_common from lib import transports @@ -20,7 +23,8 @@ class Report: buttons: list = None -REPLIES = {'telegram': transports.TelegramBot, 'slack': transports.SlackBot, 'discord': transports.DiscordBot} +REPLIES = {'telegram': telegram_common.TelegramBot, 'slack': slack_common.SlackBot, + 'discord': discord_common.DiscordBot} def reply_to(service, chat, message=None, interaction='', pressed=False): @@ -53,9 +57,9 @@ def default_chats(): its bot. Raises RuntimeError when there is nowhere to post.""" env = config.env chats = [] - for chat, token, kind in ((env.telegramChatID, env.telegramBotToken, transports.TelegramBot), - (env.slackChannelID, env.slackBotToken, transports.SlackBot), - (env.discordChannelID, env.discordBotToken, transports.DiscordBot)): + for chat, token, kind in ((env.telegramChatID, env.telegramBotToken, telegram_common.TelegramBot), + (env.slackChannelID, env.slackBotToken, slack_common.SlackBot), + (env.discordChannelID, env.discordBotToken, discord_common.DiscordBot)): if chat and token: chats.append(kind(chat)) chats[-1].scheduled = True diff --git a/lib/discord.py b/lib/discord.py deleted file mode 100644 index ea42a787..00000000 --- a/lib/discord.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Discord's REST API as the bot (discordBotToken): calls that wait out Discord's rate limits, and the slash commands -registered as finbot starts. The same in both modes: the gateway and the interactions endpoint only bring updates in.""" -import sys -import time - -from lib import config -from lib import net - -API = 'https://discord.com/api/v10' -TRIES = 3 # a call that Discord rate-limits is tried this many times at most -LONGEST_WAIT = 10 # seconds: a rate limit asking for longer is waited this long -SUPPRESS_EMBEDS = 4 # message flag: no link previews -NO_MENTIONS = {'parse': []} # text that looks like @everyone or a mention pings nobody -GUILD, BOT_DM = 0, 1 # where a slash command may be used: in servers, and in a DM with the bot -OPTION = 'arguments' # the name of a slash command's one option: its arguments as text - - -def call(method, path, **kwargs): - """A REST call as the bot: Discord's answer as JSON, or None when it is empty. A 429 waits as long as Discord asks - (LONGEST_WAIT at most) and tries again. Raises RuntimeError with Discord's reason when it refuses, or a requests - error.""" - headers = {'Authorization': 'Bot ' + config.env.discordBotToken, **kwargs.pop('headers', {})} - for attempt in range(1, TRIES + 1): - r = net.request(method, API + path, headers=headers, **kwargs) - if r.status_code != 429 or attempt == TRIES: - break - time.sleep(min(retry_after(r), LONGEST_WAIT)) - if r.status_code >= 400: - raise RuntimeError(f"Discord {method} {path}: {r.status_code} {reason(r)}") - try: - return r.json() - except ValueError: # empty: 204 No Content - return None - - -def try_call(method, path, **kwargs): - """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" - try: - return call(method, path, **kwargs) - except Exception as e: - print(f"Discord {method} failed:", net.redact(e), file=sys.stderr) - - -def retry_after(response): - """The seconds a rate-limited answer asks to wait (its retry_after), or one when it does not say.""" - try: - return float(response.json().get('retry_after')) - except (ValueError, TypeError, AttributeError): - return 1.0 - - -def reason(response): - """Discord's reason for a refusal: its message and code, else the start of the body.""" - try: - answer = response.json() - return f"{answer['message']} ({answer.get('code')})" - except (ValueError, TypeError, KeyError): - return response.text[:200] - - -def command(name, description, arguments, completes=False): - """A slash command as Discord takes it: a chat command usable in servers and in a DM with the bot, and the arguments - it takes, if any, as one optional text option that they describe; with completes, Discord asks finbot what to - suggest as they are typed (an autocomplete interaction).""" - option = {'type': 3, 'name': OPTION, 'description': arguments[:100], 'required': False, 'autocomplete': completes} - return {'name': name, 'type': 1, 'description': description[:100], 'contexts': [GUILD, BOT_DM], - 'integration_types': [0], 'options': [option] if arguments else []} - - -def summary(commands): - """What of a list of slash commands Discord may differ on from ours, in a form to compare: an answer from Discord - carries ids and versions besides.""" - return sorted((c['name'], c.get('description'), sorted(c.get('contexts') or []), - [(o['name'], o.get('description'), o.get('required', False), o.get('autocomplete', False)) - for o in c.get('options') or []]) for c in commands) - - -def register(commands): - """Called once by bot.py as it starts, in both modes: make the application's slash commands these ([(name, - description, arguments)], from worker.slash_commands), replacing them all only when they differ (Discord limits how - many may be made in a day). Returns the bot's user id: a message that mentions it is for the bot. Raises - RuntimeError, or a requests error.""" - application = call('GET', '/applications/@me')['id'] - wanted = [command(*entry) for entry in commands] - path = f'/applications/{application}/commands' - if summary(call('GET', path) or []) != summary(wanted): - call('PUT', path, json=wanted) - print(f"discord: {len(wanted)} slash commands registered", file=sys.stderr) - return str(call('GET', '/users/@me')['id']) diff --git a/lib/discord_common.py b/lib/discord_common.py new file mode 100644 index 00000000..8a016416 --- /dev/null +++ b/lib/discord_common.py @@ -0,0 +1,291 @@ +"""Discord, whichever way its updates arrive (the gateway, discord_websocket; or the interactions endpoint, +discord_webhook): its REST API as the bot (discordBotToken), with calls that wait out Discord's rate limits and the +slash commands registered as finbot starts; the reader of its interactions and messages (Discord, READER); and the bot +that posts into its channels and DMs (DiscordBot).""" +import json +import re +import sys +import time + +import requests + +from lib import config +from lib import dialects +from lib import inbound +from lib import net +from lib import transports + + +API = 'https://discord.com/api/v10' +TRIES = 3 # a call that Discord rate-limits is tried this many times at most +LONGEST_WAIT = 10 # seconds: a rate limit asking for longer is waited this long +SUPPRESS_EMBEDS = 4 # message flag: no link previews +NO_MENTIONS = {'parse': []} # text that looks like @everyone or a mention pings nobody +GUILD, BOT_DM = 0, 1 # where a slash command may be used: in servers, and in a DM with the bot +OPTION = 'arguments' # the name of a slash command's one option: its arguments as text + + +def call(method, path, **kwargs): + """A REST call as the bot: Discord's answer as JSON, or None when it is empty. A 429 waits as long as Discord asks + (LONGEST_WAIT at most) and tries again. Raises RuntimeError with Discord's reason when it refuses, or a requests + error.""" + headers = {'Authorization': 'Bot ' + config.env.discordBotToken, **kwargs.pop('headers', {})} + for attempt in range(1, TRIES + 1): + r = net.request(method, API + path, headers=headers, **kwargs) + if r.status_code != 429 or attempt == TRIES: + break + time.sleep(min(retry_after(r), LONGEST_WAIT)) + if r.status_code >= 400: + raise RuntimeError(f"Discord {method} {path}: {r.status_code} {reason(r)}") + try: + return r.json() + except ValueError: # empty: 204 No Content + return None + + +def try_call(method, path, **kwargs): + """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" + try: + return call(method, path, **kwargs) + except Exception as e: + print(f"Discord {method} failed:", net.redact(e), file=sys.stderr) + + +def retry_after(response): + """The seconds a rate-limited answer asks to wait (its retry_after), or one when it does not say.""" + try: + return float(response.json().get('retry_after')) + except (ValueError, TypeError, AttributeError): + return 1.0 + + +def reason(response): + """Discord's reason for a refusal: its message and code, else the start of the body.""" + try: + answer = response.json() + return f"{answer['message']} ({answer.get('code')})" + except (ValueError, TypeError, KeyError): + return response.text[:200] + + +def command(name, description, arguments, completes=False): + """A slash command as Discord takes it: a chat command usable in servers and in a DM with the bot, and the arguments + it takes, if any, as one optional text option that they describe; with completes, Discord asks finbot what to + suggest as they are typed (an autocomplete interaction).""" + option = {'type': 3, 'name': OPTION, 'description': arguments[:100], 'required': False, 'autocomplete': completes} + return {'name': name, 'type': 1, 'description': description[:100], 'contexts': [GUILD, BOT_DM], + 'integration_types': [0], 'options': [option] if arguments else []} + + +def summary(commands): + """What of a list of slash commands Discord may differ on from ours, in a form to compare: an answer from Discord + carries ids and versions besides.""" + return sorted((c['name'], c.get('description'), sorted(c.get('contexts') or []), + [(o['name'], o.get('description'), o.get('required', False), o.get('autocomplete', False)) + for o in c.get('options') or []]) for c in commands) + + +def register(commands): + """Called once by bot.py as it starts, in both modes: make the application's slash commands these ([(name, + description, arguments)], from worker.slash_commands), replacing them all only when they differ (Discord limits how + many may be made in a day). Returns the bot's user id: a message that mentions it is for the bot. Raises + RuntimeError, or a requests error.""" + application = call('GET', '/applications/@me')['id'] + wanted = [command(*entry) for entry in commands] + path = f'/applications/{application}/commands' + if summary(call('GET', path) or []) != summary(wanted): + call('PUT', path, json=wanted) + print(f"discord: {len(wanted)} slash commands registered", file=sys.stderr) + return str(call('GET', '/users/@me')['id']) + + +class Discord(inbound.Network): + """Discord's interactions (a slash command, a button pressed, the endpoint's check), which read the same from the + interactions endpoint and the gateway, and the gateway's messages.""" + service = 'discord' + bot_id = None # the bot's user id, from register() as bot.py starts: a message mentioning it is for the bot + PING, COMMAND, COMPONENT, AUTOCOMPLETE = 1, 2, 3, 4 # interaction types + PONG, THINKING, LATER, CHOICES = 1, 5, 6, 8 # answers: to the check, "thinking…" until the reply, nothing to see (a + # press), the suggestions for what is being typed + + def update_id(self, update): + """The interaction's id, or the message's.""" + return update.get('id') + + def parse(self, update): + """An interaction (it has a token) or a gateway message.""" + return self.interaction(update) if 'token' in update else self.message(update) + + @property + def bot_name(self): + """The bot as a message names it, @ and its user id (Discord's <@id> mention, read as Slack's are).""" + return f'@{self.bot_id}' if self.bot_id else None + + def interaction(self, update): + """The endpoint's check (a PING, answered at once), a slash command as the message it stands for (/price 7d), + its arguments as they are typed (a Suggest), or a button pressed under one of the bot's messages; anything else, + UNSUPPORTED.""" + kind = update.get('type') + if kind == self.PING: + return inbound.Response(body=json.dumps({'type': self.PONG}).encode(), interaction=True) + person = (update.get('member') or {}).get('user') or update.get('user') or {} + channel = str(update.get('channel_id') or (update.get('channel') or {}).get('id') or '') + through = f"{update.get('application_id')}/{update['token']}" + data = update.get('data') or {} + if kind == self.COMMAND and data.get('name'): + given = ' '.join(str(option.get('value', '')) for option in data.get('options') or []).strip() + text = f"/{data['name']} {given}".strip() + return inbound.Message(self.service, channel, f"<@{person.get('id')}>", text, self.bot_name, + self.name(person), interaction=through) + if kind == self.AUTOCOMPLETE and data.get('name'): + options = data.get('options') or [] + typed = next((str(option.get('value', '')) for option in options if option.get('focused')), '') + return inbound.Suggest(self.service, channel, f"/{data['name']} {typed}", through) + message = update.get('message') or {} + if kind == self.COMPONENT and data.get('custom_id') and message.get('id'): + return inbound.Press(self.service, str(update.get('id')), channel, str(message['id']), data['custom_id'], + '@' + str(person.get('username') or person.get('id') or 'unknown'), through) + return inbound.UNSUPPORTED + + def message(self, update): + """A message a person wrote (MESSAGE_CREATE); a mention <@id> reads as @id, as Slack's do. One by a bot (this + one's replies included) or a webhook is OK: answering a bot could loop.""" + author = update.get('author') or {} + if author.get('bot') or update.get('webhook_id') or not author.get('id') or 'content' not in update: + return inbound.OK + text = re.sub(r'<@!?(\d+)>', r'@\1', update['content']) + return inbound.Message(self.service, str(update.get('channel_id')), f"<@{author['id']}>", text, self.bot_name, + self.name(author), str(update.get('id', ''))) + + @staticmethod + def name(person): + """What to call someone: their display name, else their username.""" + return person.get('global_name') or person.get('username') or 'unknown' + + def acknowledge(self, event): + """An interaction's answer, which Discord must have within 3 seconds and before any reply: "thinking…" for a + slash command, until the reply replaces it; nothing to see for a press, whose message is edited after. A + gateway message needs none.""" + if not event.interaction: + return None + answer = self.THINKING if isinstance(event, inbound.Message) else self.LATER + return inbound.Response(body=json.dumps({'type': answer}).encode(), interaction=True) + + def suggestions(self, choices): + """The answer to an autocomplete interaction: the choices, (name shown, value) pairs, 25 at most.""" + choices = [{'name': name, 'value': value} for name, value in choices[:25]] + return inbound.Response(body=json.dumps({'type': self.CHOICES, 'data': {'choices': choices}}).encode(), + interaction=True) + + +class DiscordBot(transports.Transport): + """A Discord channel or DM, as the bot (discordBotToken): no link previews, no pings, charts attached, buttons as + components, edits in place, the typing notice. A slash command's first message replaces Discord's "thinking…" and + the rest follow it up, through its interaction; a pressed message is edited through the press's.""" + name = 'discord' + dialect = dialects.DISCORD + + def __init__(self, where, message=None, interaction='', pressed=False): + """As Transport; a press has no "thinking…" to replace, so its new messages go into the channel.""" + super().__init__(where, message, interaction, pressed) + self.answered = pressed # the slash command's "thinking…" replaced + + @property + def shows_typing(self): + """The typing notice for a command in a message; a slash command shows Discord's "thinking…" instead.""" + return not self.interaction + + def typing(self): + """POST /typing: "finbot is typing…" for ten seconds, or until it posts.""" + try_call('POST', f'/channels/{self.where}/typing') + + def typing_done(self): + """A slash command over with nothing said: its "thinking…" deleted, or it would show for 15 minutes.""" + if self.interaction and not self.answered: + self.answered = True + try_call('DELETE', f'/webhooks/{self.interaction}/messages/@original') + + @staticmethod + def extras(buttons): + """What a message carries besides its text: no link previews, no pings, and its buttons as components.""" + extra = {'flags': SUPPRESS_EMBEDS, 'allowed_mentions': NO_MENTIONS} + if buttons and not isinstance(buttons, dict): + extra['components'] = components(buttons) + return extra + + def post_text(self, text, buttons=None): + """Post one message (see post).""" + return self.post({'content': text, **self.extras(buttons)}) + + def post_chart(self, image, caption, buttons=None): + """Post a chart, attached to the report as its message (see post).""" + attached = {'content': caption, 'attachments': [{'id': 0, 'filename': 'chart.png'}], **self.extras(buttons)} + return self.post(attached, image) + + def post(self, payload, image=None): + """Post one message: a slash command's first replaces its "thinking…" and the rest follow it up; otherwise it + goes into the channel. Discord's message; MAYBE_SENT when a chart's post timed out; None when it failed + (logged).""" + if self.interaction and not self.answered: + method, path = 'PATCH', f'/webhooks/{self.interaction}/messages/@original' + elif self.interaction and not self.pressed: + method, path = 'POST', f'/webhooks/{self.interaction}' + else: + method, path = 'POST', f'/channels/{self.where}/messages' + try: + answer = call(method, path, **message_body(payload, image)) + except requests.exceptions.ReadTimeout as e: + self.answered = True # it may have arrived + print("Discord post timed out, it may still arrive:", net.redact(e), file=sys.stderr) + return transports.MAYBE_SENT if image is not None else None + except Exception as e: + print("Discord post to", self.where, "failed:", net.redact(e), file=sys.stderr) + return None + self.answered = True + print("OK discord message to", self.where, file=sys.stderr) + return answer or {} + + def message_path(self, message): + """Where a message is edited: through the press's interaction where there is one (a message that answered a + slash command can be edited only so), else in the channel as the bot.""" + if self.interaction: + return f'/webhooks/{self.interaction}/messages/@original' + return f'/channels/{self.where}/messages/{message}' + + def edit_chart(self, message, image, caption, buttons=None): + """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Discord refuses.""" + if hasattr(image, 'seek'): + image.seek(0) + payload = {'content': transports.fit_caption(caption, self.dialect), + 'attachments': [{'id': 0, 'filename': 'chart.png'}], 'components': components(buttons or [])} + call('PATCH', self.message_path(message), **message_body(payload, image)) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + payload = {'content': text} + if buttons is not None: + payload['components'] = components(buttons) + try_call('PATCH', self.message_path(message), json=payload) + + +DISCORD_ROW = 5 # buttons in a row of components, and rows in a message: Discord's limits + + +def components(buttons): + """Rows of buttons.Button as Discord's action rows: DISCORD_ROW buttons to a row and as many rows at most (the + buttons past that are left out), the one shown now blurple and the rest grey.""" + flat = [button for row in buttons for button in row][:DISCORD_ROW * DISCORD_ROW] + return [{'type': 1, 'components': [{'type': 2, 'style': 1 if button.active else 2, 'label': button.label[:80], + 'custom_id': button.data} for button in flat[i:i + DISCORD_ROW]]} for i in range(0, len(flat), DISCORD_ROW)] + + +def message_body(payload, image=None): + """A Discord message's request arguments: its JSON; with a chart, a form of the JSON and the PNG (bytes, so that a + rate-limited call can send it again).""" + if image is None: + return {'json': payload} + data = image.read() if hasattr(image, 'read') else bytes(image) + return {'data': {'payload_json': json.dumps(payload)}, 'files': {'files[0]': ('chart.png', data, 'image/png')}} + + +READER = Discord() diff --git a/lib/discord_webhook.py b/lib/discord_webhook.py new file mode 100644 index 00000000..98c13362 --- /dev/null +++ b/lib/discord_webhook.py @@ -0,0 +1,37 @@ +"""Discord's interactions endpoint, its HTTP mode (discordOutgoingWebhook set): Discord posts each interaction to +finbot's URL, signed with the app's Ed25519 key, for the reader (discord_common.READER), and takes the answer as the +HTTP response.""" +from urllib.parse import urlparse + +from lib import config +from lib import discord_common +from lib import inbound + + +class DiscordInteractions(inbound.Webhook): + """Discord posting interactions to discordOutgoingWebhook (the app's Interactions Endpoint URL), each signed with + the app's Ed25519 key, whose public half is discordPublicKey.""" + REFUSED = inbound.UNAUTHORIZED # Discord checks that a bad signature gets a 401, and drops a URL answering otherwise + + def path(self): + """The path of discordOutgoingWebhook.""" + return urlparse(config.env.discordOutgoingWebhook).path if config.env.discordOutgoingWebhook else '' + + def refuse(self, environ, body, update): + """The request must carry Discord's signature of its timestamp and body under discordPublicKey, checked with + PyNaCl (imported here only: Discord's http mode is the one thing that needs it).""" + signature = environ.get('HTTP_X_SIGNATURE_ED25519', '') + timestamp = environ.get('HTTP_X_SIGNATURE_TIMESTAMP', '') + if not signature or not timestamp: + return "Discord signature not present" + from nacl.exceptions import BadSignatureError + from nacl.signing import VerifyKey + try: + VerifyKey(bytes.fromhex(config.env.discordPublicKey)).verify(timestamp.encode() + body, + bytes.fromhex(signature)) + except (BadSignatureError, ValueError): + return "Discord signature incorrect" + return None + + +WEBHOOK = DiscordInteractions(discord_common.READER) diff --git a/lib/discord_websocket.py b/lib/discord_websocket.py new file mode 100644 index 00000000..fb6a6776 --- /dev/null +++ b/lib/discord_websocket.py @@ -0,0 +1,79 @@ +"""Discord's gateway, its pull mode (no discordOutgoingWebhook), through discord.py: the raw events taken as the +interactions endpoint's, for the reader (discord_common.READER), and an interaction's answer posted to its callback.""" +import asyncio +import json +import threading + +from lib import config +from lib import discord_common +from lib import pull + + +class DiscordGateway(pull.Listener): + """Discord's gateway, through discord.py: imported here only, as Discord's pull mode is the one thing that needs it + (Debian's python3-discord). Its client keeps the connection, reconnecting by itself; finbot takes the raw + MESSAGE_CREATE and INTERACTION_CREATE events, which read as the interactions endpoint's, each in a thread of its + own, off the client's event loop. The gateway takes no answers: an interaction's goes to its callback over REST.""" + network = discord_common.READER + TAKEN = ('MESSAGE_CREATE', 'INTERACTION_CREATE') + + @staticmethod + def wanted(): + """Whether finbot.env asks for it: Discord's bot token, and no discordOutgoingWebhook.""" + return bool(config.env.discordBotToken and not config.env.discordOutgoingWebhook) + + def describe(self): + """The mode, for the log.""" + return 'gateway (no discordOutgoingWebhook)' + + def poll(self, take): + """Run discord.py's client until it stops, which it does only on a failure it does not get over (a token Discord + refuses, or an intent the app is not allowed): then the back-off.""" + try: + asyncio.run(self.run_client(take)) + except Exception as e: + return self.failed('the gateway', e) + return self.failed('the gateway', 'the client stopped') + + async def run_client(self, take): + """Start the client with the bot token, and close it however it ends.""" + client = self.client(take) + async with client: + await client.start(config.env.discordBotToken) + + def client(self, take): + """discord.py's client, with the intents for messages in servers and DMs, and their text (Message Content, which + the app must be allowed in the Developer Portal); its raw events go to receive().""" + import discord as library + library.VoiceClient.warn_nacl = False # finbot has no voice: no warning that PyNaCl is missing + intents = library.Intents.default() + intents.message_content = True + client = library.Client(intents=intents, enable_debug_events=True) + + @client.event + async def on_socket_raw_receive(raw): + """A gateway message, as it arrived.""" + self.receive(raw, take) + + @client.event + async def on_ready(): + """Connected: the failures in a row are over.""" + self.failures = 0 + return client + + def receive(self, raw, take): + """A raw gateway message: an event finbot takes goes to hand_on in a thread of its own, so that the client's + event loop never waits on finbot; the rest is the client's business.""" + try: + message = json.loads(raw) + except (TypeError, ValueError): + return + if message.get('op') == 0 and message.get('t') in self.TAKEN and isinstance(message.get('d'), dict): + threading.Thread(target=self.hand_on, args=(take, message['d']), daemon=True).start() + + def answer(self, update, answer): + """Post an interaction's answer to its callback, then start the work waiting on it, if any.""" + path = f"/interactions/{update['id']}/{update['token']}/callback" + discord_common.call('POST', path, json=json.loads(answer.body)) + if answer.then: + answer.then() diff --git a/lib/inbound.py b/lib/inbound.py index 35d74d46..20147a00 100644 --- a/lib/inbound.py +++ b/lib/inbound.py @@ -1,22 +1,13 @@ -"""What the networks send the bot. Reading an update is apart from how it arrives: a class per network reads its updates -(Telegram, Slack, Discord), and a class per way of arriving takes them in (a webhook: the path the network posts to, -how it proves itself, the body), so that another way (a pull mode) can hand its updates to the same reader. An update -becomes a Message to answer, a Press of a button, or a Response the network gets straight back (a challenge, or nothing -to do). The reader also acknowledges an update where the network waits for that, however it arrived: Telegram's press +"""What the networks send the bot, whichever network and however it arrives. Reading an update is apart from how it +arrives: each network has a reader (a Network: telegram_common, slack_common, discord_common), and each way in hands +its updates to it: a webhook (a Webhook: the path the network posts to, how it proves itself, the body; in +telegram_webhook, slack_webhook, discord_webhook) or a pull mode (lib/pull.py). An update becomes a Message to answer, +a Press of a button, a Suggest to complete, or a Response the network gets straight back (a challenge, or nothing to +do). The reader also acknowledges an update where the network waits for that, however it arrived: Telegram's press with a call, Discord's interaction with the answer it must have before any reply.""" import dataclasses -import hashlib import hmac -import html import json -import re -import sys -import time -from urllib.parse import parse_qs, urlparse - -from lib import buttons -from lib import config -from lib import telegram @dataclasses.dataclass @@ -76,9 +67,6 @@ class Response: TOO_LARGE = Response('413 Payload Too Large', b'

Payload Too Large

') -SIGNATURE_AGE = 300 # seconds: Slack's limit; an older signed request may be a replay - - def same_secret(given, expected): """Whether a secret sent with a request is the one configured, compared in constant time. An empty configured secret proves nothing, so it matches nothing.""" @@ -104,201 +92,6 @@ def acknowledge(self, event): return None -class Telegram(Network): - """Telegram's updates: messages and button presses.""" - service = 'telegram' - bot_name = None # the bot's @username, from telegram.register() as bot.py starts - IGNORED = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', - 'venue', 'poll', 'dice'} # nothing to answer, nothing worth logging - - def update_id(self, update): - """Telegram's update_id.""" - return update.get('update_id') - - def parse(self, update): - """A message (its text, or a photo's caption) or a button press. A first name is escaped as it arrives: - replies are Telegram HTML, and a name is text, not markup.""" - if 'callback_query' in update: # chart period and job buttons - press = update['callback_query'] - message = press.get('message') - if not message or not press.get('data'): - return UNSUPPORTED - sender = press.get('from', {}) - who = '@' + sender['username'] if 'username' in sender else sender.get('first_name', 'unknown') - return Press(self.service, press['id'], str(message['chat']['id']), str(message['message_id']), - press['data'], who) - if 'message' not in update: - return UNSUPPORTED - message = update['message'] - sender = message['from'] - chat_id = str(message['chat']['id']) - first_name = html.escape(sender['first_name'], quote=False) - if 'username' in sender: - user = '@' + sender['username'] - user_name = first_name or user # the handle when the first name is blank - else: - user = user_name = '@' + first_name - if 'text' in message: - text = message['text'] - elif 'photo' in message: - text = message.get('caption', '') - elif self.IGNORED & message.keys(): - return OK - else: - print("[telegram]: unhandled: 'message' without 'text/photo'", file=sys.stderr) - return UNHANDLED - return Message(self.service, chat_id, user, text, self.bot_name, user_name, str(message.get('message_id', ''))) - - def acknowledge(self, event): - """A press: answerCallbackQuery, so that the client's spinner stops and a toast made from the button's data says - what is coming (buttons.toast: 'Loading 1Y…'). A failure is only logged. Nothing goes back with the update.""" - if isinstance(event, Press): - telegram.answerCallbackQuery(event.press_id, buttons.toast(event.data)) - return None - - -class Slack(Network): - """Slack's Events API (messages), its interactivity (a button pressed) and its slash commands.""" - service = 'slack' - BY_PEOPLE = (None, 'thread_broadcast') # message subtypes a person writes; the rest are bot posts, edits, joins... - - def update_id(self, update): - """Slack's event_id (a button press has none).""" - return update.get('event_id') - - def parse(self, update): - """A message a person wrote, a slash command, a button pressed, or the challenge that checks the URL.""" - kind = update.get('type') - if 'command' in update and kind is None: - return self.slash(update) - if kind == 'url_verification': - response = json.dumps({'challenge': update['challenge']}) - print("replying with", response, file=sys.stderr) - return Response(body=response.encode()) - if kind == 'block_actions': - return self.press(update) - if kind != 'event_callback': - print(f"[slack]: unhandled 'type' {kind!r}", file=sys.stderr) - return UNHANDLED - event = update['event'] - bot_user = update['authorizations'][0]['user_id'] # QWERTY - if event.get('bot_id') or event.get('subtype') not in self.BY_PEOPLE or not event.get('user') or \ - event['user'] == bot_user: - return OK # not by a person: answering a bot (this one's replies and webhook reports included) could loop - if event['type'] not in ('message', 'app_mention') or 'text' not in event: - print("[slack]: unhandled event callback type", event['type'], file=sys.stderr) - return UNHANDLED - user = '<@' + event['user'] + '>' # a mention: Slack shows the name - return Message(self.service, event['channel'], user, self.plain(event['text']), '@' + bot_user, user, - event.get('ts', '')) - - def slash(self, update): - """A slash command (/price with 5 7d), as the message that says the same (/price 5 7d), in the channel or DM it - was sent in. There is no message of it, for ⏳ to show on.""" - user = '<@' + str(update.get('user_id') or 'unknown') + '>' - text = self.plain(f"{update['command']} {update.get('text') or ''}".strip()) - return Message(self.service, str(update.get('channel_id') or ''), user, text, '', user) - - @staticmethod - def plain(text): - """Text as written: links and mentions lose their markup ( becomes dub.ax, <@QWERTY> - @QWERTY), and Slack's escapes their meaning (& becomes &).""" - text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), text) - return html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > - - def press(self, update): - """A button pressed under one of the bot's messages (a chart's period buttons), or UNSUPPORTED for one that is - not under a message or carries no value.""" - container = update.get('container') or {} - actions = update.get('actions') or [{}] - value = actions[0].get('value') if isinstance(actions[0], dict) else None - if container.get('type') != 'message' or not container.get('message_ts') or not value: - return UNSUPPORTED - user = update.get('user') or {} - who = '@' + str(user.get('username') or user.get('id') or 'unknown') - return Press(self.service, update.get('trigger_id', ''), container.get('channel_id', ''), - container['message_ts'], value, who) - - -class Discord(Network): - """Discord's interactions (a slash command, a button pressed, the endpoint's check), which read the same from the - interactions endpoint and the gateway, and the gateway's messages.""" - service = 'discord' - bot_id = None # the bot's user id, from discord.register() as bot.py starts: a message mentioning it is for the bot - PING, COMMAND, COMPONENT, AUTOCOMPLETE = 1, 2, 3, 4 # interaction types - PONG, THINKING, LATER, CHOICES = 1, 5, 6, 8 # answers: to the check, "thinking…" until the reply, nothing to see (a - # press), the suggestions for what is being typed - - def update_id(self, update): - """The interaction's id, or the message's.""" - return update.get('id') - - def parse(self, update): - """An interaction (it has a token) or a gateway message.""" - return self.interaction(update) if 'token' in update else self.message(update) - - @property - def bot_name(self): - """The bot as a message names it, @ and its user id (Discord's <@id> mention, read as Slack's are).""" - return f'@{self.bot_id}' if self.bot_id else None - - def interaction(self, update): - """The endpoint's check (a PING, answered at once), a slash command as the message it stands for (/price 7d), - its arguments as they are typed (a Suggest), or a button pressed under one of the bot's messages; anything else, - UNSUPPORTED.""" - kind = update.get('type') - if kind == self.PING: - return Response(body=json.dumps({'type': self.PONG}).encode(), interaction=True) - person = (update.get('member') or {}).get('user') or update.get('user') or {} - channel = str(update.get('channel_id') or (update.get('channel') or {}).get('id') or '') - through = f"{update.get('application_id')}/{update['token']}" - data = update.get('data') or {} - if kind == self.COMMAND and data.get('name'): - given = ' '.join(str(option.get('value', '')) for option in data.get('options') or []).strip() - text = f"/{data['name']} {given}".strip() - return Message(self.service, channel, f"<@{person.get('id')}>", text, self.bot_name, self.name(person), - interaction=through) - if kind == self.AUTOCOMPLETE and data.get('name'): - options = data.get('options') or [] - typed = next((str(option.get('value', '')) for option in options if option.get('focused')), '') - return Suggest(self.service, channel, f"/{data['name']} {typed}", through) - message = update.get('message') or {} - if kind == self.COMPONENT and data.get('custom_id') and message.get('id'): - return Press(self.service, str(update.get('id')), channel, str(message['id']), data['custom_id'], - '@' + str(person.get('username') or person.get('id') or 'unknown'), through) - return UNSUPPORTED - - def message(self, update): - """A message a person wrote (MESSAGE_CREATE); a mention <@id> reads as @id, as Slack's do. One by a bot (this - one's replies included) or a webhook is OK: answering a bot could loop.""" - author = update.get('author') or {} - if author.get('bot') or update.get('webhook_id') or not author.get('id') or 'content' not in update: - return OK - text = re.sub(r'<@!?(\d+)>', r'@\1', update['content']) - return Message(self.service, str(update.get('channel_id')), f"<@{author['id']}>", text, self.bot_name, - self.name(author), str(update.get('id', ''))) - - @staticmethod - def name(person): - """What to call someone: their display name, else their username.""" - return person.get('global_name') or person.get('username') or 'unknown' - - def acknowledge(self, event): - """An interaction's answer, which Discord must have within 3 seconds and before any reply: "thinking…" for a - slash command, until the reply replaces it; nothing to see for a press, whose message is edited after. A - gateway message needs none.""" - if not event.interaction: - return None - answer = self.THINKING if isinstance(event, Message) else self.LATER - return Response(body=json.dumps({'type': answer}).encode(), interaction=True) - - def suggestions(self, choices): - """The answer to an autocomplete interaction: the choices, (name shown, value) pairs, 25 at most.""" - choices = [{'name': name, 'value': value} for name, value in choices[:25]] - return Response(body=json.dumps({'type': self.CHOICES, 'data': {'choices': choices}}).encode(), - interaction=True) - - class Webhook: """Updates a network posts to the bot, for its reader (a Network); subclasses give the path it posts to and how it proves itself.""" @@ -326,97 +119,3 @@ def decode(self, body): def refuse(self, environ, body, update): """Why to refuse the request once read (the body's signature or a secret in it missing or wrong), else None.""" return None - - -class TelegramWebhook(Webhook): - """Telegram posting to telegramOutgoingWebhook, each update with the secret header that setWebhook gave it.""" - - def path(self): - """The path of telegramOutgoingWebhook.""" - return urlparse(config.env.telegramOutgoingWebhook).path if config.env.telegramOutgoingWebhook else '' - - def refuse_unread(self, environ): - """The header must carry telegramOutgoingToken.""" - secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') - if not same_secret(secret, config.env.telegramOutgoingToken): - return f"Telegram authorisation header {'not present' if secret is None else 'is present but incorrect'}" - return None - - -class SlackWebhook(Webhook): - """Slack posting events, button presses and slash commands to slackOutgoingWebhook (the app's Request URL for each), - each request signed with the app's signing secret, and carrying its verification token.""" - - def path(self): - """The path of slackOutgoingWebhook.""" - return urlparse(config.env.slackOutgoingWebhook).path if config.env.slackOutgoingWebhook else '' - - def refuse(self, environ, body, update): - """With slackSigningSecret, the request must carry Slack's signature of its body, made in the last five minutes; - without it, the body must carry slackOutgoingToken (the older way, which Slack has deprecated).""" - if config.env.slackSigningSecret: - return self.signature_problem(environ, body, config.env.slackSigningSecret) - if 'token' not in update: - return "Slack authorisation field not present" - if not same_secret(str(update['token']), config.env.slackOutgoingToken): - return "Slack auth incorrect" - return None - - @staticmethod - def signature_problem(environ, body, secret): - """Why the request's signature fails, or None: X-Slack-Signature must be v0= and the HMAC-SHA256, under the - signing secret, of v0::, and the timestamp within five minutes of now.""" - timestamp = environ.get('HTTP_X_SLACK_REQUEST_TIMESTAMP', '') - signature = environ.get('HTTP_X_SLACK_SIGNATURE') - if signature is None or not (timestamp.isascii() and timestamp.isdigit()): - return "Slack signature not present" - if abs(time.time() - int(timestamp)) > SIGNATURE_AGE: - return "Slack signature too old (a replay, or a clock is wrong)" - signed = b'v0:' + timestamp.encode() + b':' + body - expected = 'v0=' + hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest() - return None if same_secret(signature, expected) else "Slack signature incorrect" - - def decode(self, body): - """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON, and a - slash command as a form of its fields (command, text, channel_id, user_id and the rest).""" - if not body.lstrip().startswith(b'{'): - fields = parse_qs(body.decode('utf-8', 'replace'), keep_blank_values=True) - if 'command' in fields and 'payload' not in fields: - return {name: values[0] for name, values in fields.items()} - body = fields.get('payload', [''])[0].encode() - return super().decode(body) - - -class DiscordInteractions(Webhook): - """Discord posting interactions to discordOutgoingWebhook (the app's Interactions Endpoint URL), each signed with - the app's Ed25519 key, whose public half is discordPublicKey.""" - REFUSED = UNAUTHORIZED # Discord checks that a bad signature gets a 401, and drops a URL that answers otherwise - - def path(self): - """The path of discordOutgoingWebhook.""" - return urlparse(config.env.discordOutgoingWebhook).path if config.env.discordOutgoingWebhook else '' - - def refuse(self, environ, body, update): - """The request must carry Discord's signature of its timestamp and body under discordPublicKey, checked with - PyNaCl (imported here only: Discord's http mode is the one thing that needs it).""" - signature = environ.get('HTTP_X_SIGNATURE_ED25519', '') - timestamp = environ.get('HTTP_X_SIGNATURE_TIMESTAMP', '') - if not signature or not timestamp: - return "Discord signature not present" - from nacl.exceptions import BadSignatureError - from nacl.signing import VerifyKey - try: - VerifyKey(bytes.fromhex(config.env.discordPublicKey)).verify(timestamp.encode() + body, - bytes.fromhex(signature)) - except (BadSignatureError, ValueError): - return "Discord signature incorrect" - return None - - -TELEGRAM, SLACK, DISCORD = Telegram(), Slack(), Discord() # the readers -WEBHOOKS = (TelegramWebhook(TELEGRAM), SlackWebhook(SLACK), DiscordInteractions(DISCORD)) - - -def route(path): - """The webhook whose path this is, or None.""" - return next((hook for hook in WEBHOOKS if hook.path() and path == hook.path()), None) diff --git a/lib/pull.py b/lib/pull.py index cb46a53a..c4c5e273 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -1,20 +1,14 @@ -"""The pull modes: a network that finbot.env gives no webhook URL is asked for its updates instead, by a listener in a -thread of its own, which hands each update to bot.take(network, update) as the webhook would, so that reading it, the -repeat check and acknowledging a press are the same both ways. Each listener keeps going through failures, waiting -longer after each one in a row (backoff). Telegram's long polling, Slack's Socket Mode and Discord's gateway.""" -import asyncio -import json +"""The pull modes, whichever network: a network that finbot.env gives no webhook URL is asked for its updates instead, +by a listener in a thread of its own (Telegram's long polling, Slack's Socket Mode and Discord's gateway: in +telegram_longpolling, slack_websocket and discord_websocket), which hands each update to bot.take(network, update) as +the webhook would, so that reading it, the repeat check and acknowledging a press are the same both ways. Each +listener keeps going through failures, waiting longer after each one in a row (backoff).""" import sys -import threading import time -from lib import config from lib import delivery -from lib import discord -from lib import inbound from lib import net -from lib import telegram -from lib import transports + BACKOFF = (5, 60) # seconds to wait after a failure: the first, doubled after each further one, up to the second @@ -30,6 +24,11 @@ class Listener: network = None # the reader, an inbound.Network failures = 0 # in a row + @staticmethod + def wanted(): + """Whether finbot.env asks for this listener: its network's keys set, and no webhook URL.""" + raise NotImplementedError + def run(self, take): """Fetch and hand on updates for good, waiting as poll() says between rounds.""" print(f"{self.network.service}: {self.describe()}", file=sys.stderr) @@ -68,217 +67,3 @@ def answer(self, update, answer): start the work waiting on it, if any.""" if answer.then: answer.then() - - -class TelegramPoll(Listener): - """Telegram's long polling: getUpdates, which Telegram holds open until there are updates or POLL_SECONDS have - passed, asking for those after the last one taken (offset), which also tells Telegram that one arrived. No webhook - may be set meanwhile: telegram.register() deletes it as finbot starts.""" - network = inbound.TELEGRAM - POLL_SECONDS = 50 # how long Telegram may hold a getUpdates open - READ_SECONDS = 65 # the request's own timeout for the answer: longer than Telegram holds it - offset = None # the update_id after the last one taken - - def describe(self): - """The mode, for the log.""" - return 'long polling for updates (no telegramOutgoingWebhook)' - - def poll(self, take): - """One getUpdates: each update to take, in order; then no wait, or the back-off after a failure (a 409 says a - webhook is set, or another finbot polls with the same token).""" - params = {'timeout': self.POLL_SECONDS, 'allowed_updates': ['message', 'callback_query']} - if self.offset is not None: - params['offset'] = self.offset - try: - updates = telegram.call('getUpdates', json=params, timeout=(net.CONNECT_TIMEOUT, self.READ_SECONDS)) - except Exception as e: - return self.failed('getUpdates', e) - self.failures = 0 - for update in updates: - if isinstance(update.get('update_id'), int): - self.offset = update['update_id'] + 1 # taken, even if handling it fails: it is not fetched again - self.hand_on(take, update) - return 0 - - -class WebSocket: - """A WebSocket connection, through websocket-client: imported here only, as Slack's pull mode is the one thing that - needs it (Debian's python3-websocket).""" - - def __init__(self, url): - """Connect to url, waiting net.CONNECT_TIMEOUT seconds at most for the server.""" - import websocket - self.library = websocket - self.ws = websocket.create_connection(url, timeout=net.CONNECT_TIMEOUT) - - def receive(self, timeout): - """The next message's text; '' for a ping or a pong, which only say the line is alive; None when nothing came - within timeout seconds. Raises ConnectionError when the server closes the socket.""" - self.ws.settimeout(timeout) - try: - opcode, data = self.ws.recv_data(control_frame=True) - except self.library.WebSocketTimeoutException: - return None - if opcode == self.library.ABNF.OPCODE_CLOSE: - raise ConnectionError('the server closed the socket') - if opcode == self.library.ABNF.OPCODE_TEXT: - return data.decode('utf-8') if isinstance(data, bytes) else data - return '' - - def send(self, text): - """Send a text message.""" - self.ws.send(text) - - def ping(self): - """Send a ping, which the server answers with a pong.""" - self.ws.ping() - - def close(self): - """Close the socket; a goodbye that fails is of no matter.""" - try: - self.ws.close(timeout=1) - except Exception: - pass - - -class SlackSocket(Listener): - """Slack's Socket Mode: a WebSocket to the URL that apps.connections.open gives for the app-level token - (slackAppToken). Each event, button press or slash command comes in an envelope, acked at once, before anything - else (Slack sends again what is not acked within 3 seconds); inside it is what Slack would post to - slackOutgoingWebhook. Slack closes a socket now and then, saying so first (disconnect), and another is opened at - once.""" - network = inbound.SLACK - PING_SECONDS = 30 # silence this long sends a ping; as long again after it, and the socket is taken for dead - TAKEN = ('events_api', 'interactive', 'slash_commands') # the envelopes whose payload goes to the reader - - def describe(self): - """The mode, for the log.""" - return 'Socket Mode (no slackOutgoingWebhook)' - - def open(self): - """The URL of a new socket, from apps.connections.open.""" - return transports.slack_api('apps.connections.open', as_app=True)['url'] - - def connect(self, url): - """A WebSocket connected to url.""" - return WebSocket(url) - - def poll(self, take): - """One socket, from opening it to Slack closing it: then another at once; after a failure, the back-off.""" - try: - connection = self.connect(self.open()) - except Exception as e: - return self.failed('opening a socket', e) - try: - reason = self.listen(connection, take) - except Exception as e: - return self.failed('the socket', e) - finally: - connection.close() - print(f"slack: Slack is closing the socket ({reason}), opening another", file=sys.stderr) - return 0 - - def listen(self, connection, take): - """Read envelopes until Slack says it will close the socket, and return its reason: each acked, then its payload - handed on. A ping goes out after PING_SECONDS of silence; as long again with nothing, not even the pong, raises - TimeoutError.""" - pinged = False - while True: - text = connection.receive(self.PING_SECONDS) - if text is None: - if pinged: - raise TimeoutError(f'nothing from Slack for {2 * self.PING_SECONDS} s, even after a ping') - connection.ping() - pinged = True - continue - pinged = False - if not text: # a ping or a pong - continue - envelope = json.loads(text) - if envelope.get('envelope_id'): - connection.send(json.dumps({'envelope_id': envelope['envelope_id']})) - kind = envelope.get('type') - if kind == 'hello': - self.failures = 0 - elif kind == 'disconnect': - return envelope.get('reason') or 'no reason given' - elif kind in self.TAKEN: - self.hand_on(take, envelope.get('payload') or {}) - - -class DiscordGateway(Listener): - """Discord's gateway, through discord.py: imported here only, as Discord's pull mode is the one thing that needs it - (Debian's python3-discord). Its client keeps the connection, reconnecting by itself; finbot takes the raw - MESSAGE_CREATE and INTERACTION_CREATE events, which read as the interactions endpoint's, each in a thread of its - own, off the client's event loop. The gateway takes no answers: an interaction's goes to its callback over REST.""" - network = inbound.DISCORD - TAKEN = ('MESSAGE_CREATE', 'INTERACTION_CREATE') - - def describe(self): - """The mode, for the log.""" - return 'gateway (no discordOutgoingWebhook)' - - def poll(self, take): - """Run discord.py's client until it stops, which it does only on a failure it does not get over (a token Discord - refuses, or an intent the app is not allowed): then the back-off.""" - try: - asyncio.run(self.run_client(take)) - except Exception as e: - return self.failed('the gateway', e) - return self.failed('the gateway', 'the client stopped') - - async def run_client(self, take): - """Start the client with the bot token, and close it however it ends.""" - client = self.client(take) - async with client: - await client.start(config.env.discordBotToken) - - def client(self, take): - """discord.py's client, with the intents for messages in servers and DMs, and their text (Message Content, which - the app must be allowed in the Developer Portal); its raw events go to receive().""" - import discord as library - library.VoiceClient.warn_nacl = False # finbot has no voice: no warning that PyNaCl is missing - intents = library.Intents.default() - intents.message_content = True - client = library.Client(intents=intents, enable_debug_events=True) - - @client.event - async def on_socket_raw_receive(raw): - """A gateway message, as it arrived.""" - self.receive(raw, take) - - @client.event - async def on_ready(): - """Connected: the failures in a row are over.""" - self.failures = 0 - return client - - def receive(self, raw, take): - """A raw gateway message: an event finbot takes goes to hand_on in a thread of its own, so that the client's - event loop never waits on finbot; the rest is the client's business.""" - try: - message = json.loads(raw) - except (TypeError, ValueError): - return - if message.get('op') == 0 and message.get('t') in self.TAKEN and isinstance(message.get('d'), dict): - threading.Thread(target=self.hand_on, args=(take, message['d']), daemon=True).start() - - def answer(self, update, answer): - """Post an interaction's answer to its callback, then start the work waiting on it, if any.""" - discord.call('POST', f"/interactions/{update['id']}/{update['token']}/callback", json=json.loads(answer.body)) - if answer.then: - answer.then() - - -def listeners(): - """The listeners finbot.env asks for: each network with its bot token and no webhook URL (Slack's also needs its - app-level token).""" - env = config.env - found = [] - if env.telegramBotToken and not env.telegramOutgoingWebhook: - found.append(TelegramPoll()) - if env.slackBotToken and env.slackAppToken and not env.slackOutgoingWebhook: - found.append(SlackSocket()) - if env.discordBotToken and not env.discordOutgoingWebhook: - found.append(DiscordGateway()) - return found diff --git a/lib/slack_common.py b/lib/slack_common.py new file mode 100644 index 00000000..4a0e6777 --- /dev/null +++ b/lib/slack_common.py @@ -0,0 +1,342 @@ +"""Slack, whichever way its updates arrive (Socket Mode, slack_websocket; or the Request URL, slack_webhook): the Web +API calls, the reader of its events, button presses and slash commands (Slack, READER), and the bot that posts into +its channels (SlackBot), with the Block Kit of its messages.""" +import html +import json +import re +import sys +import time + +import requests + +from lib import config +from lib import dialects +from lib import inbound +from lib import net +from lib import state +from lib import transports + + +SLACK_CHARTS = 'finbot_slack_charts.json' # Slack's chart messages: the file each shows, and the charts uploaded +SLACK_CHARTS_KEPT = 500 # chart messages noted, newest last: an older one shows its file for good +SLACK_REUSE_SECONDS = 15 * 60 # a period pressed within this long of its chart's upload shows that upload again + + +def call(method, as_app=False, **data): + """POST a Slack Web API method (form-encoded) as the bot, or with the app-level token (as_app, for Socket Mode's + apps.connections.open), and return the answer; an error raises RuntimeError.""" + key = 'slackAppToken' if as_app else 'slackBotToken' + token = getattr(config.env, key) + if not token: + raise RuntimeError(f"Slack {method}: needs {key} in finbot.env") + r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + token}, data=data) + try: + output = r.json() + except ValueError: + output = {} + if not output.get('ok'): + raise RuntimeError(f"Slack {method}: {output.get('error') or r.status_code}") + return output + + +class Slack(inbound.Network): + """Slack's Events API (messages), its interactivity (a button pressed) and its slash commands.""" + service = 'slack' + BY_PEOPLE = (None, 'thread_broadcast') # message subtypes a person writes; the rest are bot posts, edits, joins... + + def update_id(self, update): + """Slack's event_id (a button press has none).""" + return update.get('event_id') + + def parse(self, update): + """A message a person wrote, a slash command, a button pressed, or the challenge that checks the URL.""" + kind = update.get('type') + if 'command' in update and kind is None: + return self.slash(update) + if kind == 'url_verification': + response = json.dumps({'challenge': update['challenge']}) + print("replying with", response, file=sys.stderr) + return inbound.Response(body=response.encode()) + if kind == 'block_actions': + return self.press(update) + if kind != 'event_callback': + print(f"[slack]: unhandled 'type' {kind!r}", file=sys.stderr) + return inbound.UNHANDLED + event = update['event'] + bot_user = update['authorizations'][0]['user_id'] # QWERTY + if event.get('bot_id') or event.get('subtype') not in self.BY_PEOPLE or not event.get('user') or \ + event['user'] == bot_user: # not by a person: answering a bot (this one's replies included) could loop + return inbound.OK + if event['type'] not in ('message', 'app_mention') or 'text' not in event: + print("[slack]: unhandled event callback type", event['type'], file=sys.stderr) + return inbound.UNHANDLED + user = '<@' + event['user'] + '>' # a mention: Slack shows the name + return inbound.Message(self.service, event['channel'], user, self.plain(event['text']), '@' + bot_user, user, + event.get('ts', '')) + + def slash(self, update): + """A slash command (/price with 5 7d), as the message that says the same (/price 5 7d), in the channel or DM it + was sent in. There is no message of it, for ⏳ to show on.""" + user = '<@' + str(update.get('user_id') or 'unknown') + '>' + text = self.plain(f"{update['command']} {update.get('text') or ''}".strip()) + return inbound.Message(self.service, str(update.get('channel_id') or ''), user, text, '', user) + + @staticmethod + def plain(text): + """Text as written: links and mentions lose their markup ( becomes dub.ax, <@QWERTY> + @QWERTY), and Slack's escapes their meaning (& becomes &).""" + text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), text) + return html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > + + def press(self, update): + """A button pressed under one of the bot's messages (a chart's period buttons), or UNSUPPORTED for one that is + not under a message or carries no value.""" + container = update.get('container') or {} + actions = update.get('actions') or [{}] + value = actions[0].get('value') if isinstance(actions[0], dict) else None + if container.get('type') != 'message' or not container.get('message_ts') or not value: + return inbound.UNSUPPORTED + user = update.get('user') or {} + who = '@' + str(user.get('username') or user.get('id') or 'unknown') + return inbound.Press(self.service, update.get('trigger_id', ''), container.get('channel_id', ''), + container['message_ts'], value, who) + + +class SlackBot(transports.Transport): + """A Slack channel, as the bot: chat.postMessage and the file upload, with slackBotToken.""" + name = 'slack' + dialect = dialects.SLACK + WORKING = 'hourglass_flowing_sand' # ⏳: Slack has no typing notice for bots, so a reaction on the message stands in + working = None # True while ⏳ is on the message, False when putting it there failed + + @property + def shows_typing(self): + """⏳ on the message being answered: a scheduled report answers none.""" + return bool(self.message) + + def typing(self): + """Put ⏳ on the message being answered, once (the bot needs the reactions:write scope); a failure is only + logged, and not tried again.""" + if self.working is not None: + return + try: + call('reactions.add', channel=self.where, timestamp=self.message, name=self.WORKING) + self.working = True + except Exception as e: + self.working = False + print("Slack reactions.add failed:", net.redact(e), file=sys.stderr) + + def typing_done(self): + """Take ⏳ off the message again; a failure is only logged.""" + if not self.working: + return + self.working = False + try: + call('reactions.remove', channel=self.where, timestamp=self.message, name=self.WORKING) + except Exception as e: + print("Slack reactions.remove failed:", net.redact(e), file=sys.stderr) + + def post_text(self, text, buttons=None): + """chat.postMessage, without link previews, and with buttons (a Block Kit message) where they work. Slack's + answer, or None when it failed or refused (logged): Slack refuses with HTTP 200 and ok false (not_in_channel, + say), so a refusal is read from the answer.""" + if not config.env.slackBotToken: + print(f"ERROR slack: a reply to channel {self.where} needs slackBotToken in finbot.env; not sent", + file=sys.stderr) + return None + blocks = {} + if buttons and not isinstance(buttons, dict) and self.takes_buttons: + blocks['blocks'] = json.dumps(text_blocks(text, buttons)) + try: + answer = call('chat.postMessage', channel=self.where, text=text, unfurl_links='false', + unfurl_media='false', **blocks) + except Exception as e: + print("Slack chat.postMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) + return None + print("OK slack message to", self.where, file=sys.stderr) + return answer + + def post_chart(self, image, caption, buttons=None): + """Upload the chart into the channel with the report as its message, the way Slack takes files since it retired + files.upload: ask for an upload URL, send the bytes there, then complete the upload into the channel (the bot + needs the files:write and files:read scopes). With period buttons, where they work, it goes in a Block Kit + message instead (post_blocks), or without them if that fails. Slack's answer, MAYBE_SENT, or None if a step + failed.""" + data = image.read() if hasattr(image, 'read') else bytes(image) + if buttons and not isinstance(buttons, dict) and self.takes_buttons: + try: + return self.post_blocks(data, caption, buttons) + except Exception as e: + print("Slack chart with buttons failed, uploading it without them:", net.redact(e), file=sys.stderr) + try: + share = {'files': json.dumps([{'id': self.upload(data)}]), 'channel_id': self.where, + 'initial_comment': caption} + try: + output = call('files.completeUploadExternal', **share) + except requests.exceptions.ReadTimeout as e: + print("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e), + file=sys.stderr) + return transports.MAYBE_SENT + except Exception as e: + print("Slack chart upload failed:", net.redact(e), file=sys.stderr) + return None + print("OK slack chart upload to", self.where, file=sys.stderr) + return output + + def upload(self, data): + """Send a chart's bytes to Slack: the id of the file, not completed yet. Raises RuntimeError on a refusal.""" + ticket = call('files.getUploadURLExternal', filename='chart.png', length=len(data)) + r = net.post(ticket['upload_url'], data=data) + if r.status_code != 200: + raise RuntimeError(f"Slack upload: {r.status_code} {r.text[:200]}") + return ticket['file_id'] + + def unshared(self, data): + """Upload a chart as a file that is in no channel, for an image block to show: its id.""" + file_id = self.upload(data) + call('files.completeUploadExternal', files=json.dumps([{'id': file_id, 'title': 'chart'}])) + return file_id + + def post_blocks(self, data, caption, buttons): + """The chart in a Block Kit message, the report above it and the period buttons under it. Slack's answer, or + MAYBE_SENT when posting the message timed out (it may have arrived); raises RuntimeError when Slack refuses.""" + file_id = self.unshared(data) + blocks = chart_blocks(caption, buttons, file_id) + try: + answer = until_shown(lambda: call('chat.postMessage', channel=self.where, + text=caption[:SECTION_LIMIT], blocks=json.dumps(blocks), unfurl_links='false', unfurl_media='false')) + except requests.exceptions.ReadTimeout as e: + print("Slack chat.postMessage timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) + return transports.MAYBE_SENT + if answer.get('ts'): + self.delete(self.shows(answer['ts'], file_id, caption, buttons)) + return answer + + def reshow(self, message, buttons): + """Show a chart message's period again from its upload of under SLACK_REUSE_SECONDS ago (for this message or + another), with the report it came with: nothing is drawn or uploaded. True when it did; False when there is no + such upload, or Slack refuses it (a file deleted outside finbot, say).""" + view, now = marked(buttons), time.time() + uploads = state.load(SLACK_CHARTS, {}).get('uploads', {}) + recent = [(at, file_id, caption) for file_id, (at, shows, caption) in uploads.items() + if view and shows == view and now - at < SLACK_REUSE_SECONDS] + if not recent: + return False + _, file_id, caption = max(recent) + try: + call('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], + blocks=json.dumps(chart_blocks(caption, buttons, file_id))) + except Exception as e: + print("Slack chat.update with an earlier upload failed, uploading anew:", net.redact(e), file=sys.stderr) + return False + self.delete(self.shows(message, file_id)) + return True + + def edit_chart(self, message, image, caption, buttons=None): + """Show another period in a chart message: the new chart uploaded (a file cannot be replaced) and shown in its + image block, with its caption and buttons. A press for that period within SLACK_REUSE_SECONDS shows this upload + again (reshow). Raises RuntimeError when Slack refuses the update.""" + if hasattr(image, 'seek'): + image.seek(0) + file_id = self.unshared(image.read() if hasattr(image, 'read') else bytes(image)) + blocks = chart_blocks(caption, buttons or [], file_id) + until_shown(lambda: call('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], + blocks=json.dumps(blocks))) + self.delete(self.shows(message, file_id, caption, buttons)) + + def shows(self, message, file_id, caption=None, buttons=None): + """Note that a chart message shows this file now; given its report, that the file was uploaded now for the + period its buttons mark (see reshow). The uploads to delete: those no message shows, a minute past their reuse + (a press reusing one has that minute to show it).""" + now = time.time() + with state.edit(SLACK_CHARTS, {}) as noted: + if set(noted) != {'shown', 'uploads'}: # new, or the {'channel ts': file} of before, whose files stay + noted.clear() + noted.update(shown={}, uploads={}) + shown, uploads = noted['shown'], noted['uploads'] # 'channel ts': file; file: [time, period, report] + if caption is not None: + uploads[file_id] = [now, marked(buttons), caption] + key = f'{self.where} {message}' + shown.pop(key, None) + shown[key] = file_id + for old in list(shown)[:-SLACK_CHARTS_KEPT]: # its message may show the file for good + uploads.pop(shown.pop(old), None) + showing = set(shown.values()) + gone = [old for old, (at, _, _) in uploads.items() + if old not in showing and now - at > SLACK_REUSE_SECONDS + 60] + for old in gone: + del uploads[old] + for upload in uploads.values(): # past reuse, a file shown is kept only to be deleted later + if now - upload[0] >= SLACK_REUSE_SECONDS: + upload[1:] = [None, None] + return gone + + def delete(self, files): + """Delete uploaded charts that no message shows any more; a failure is only logged (the file stays among the + bot's files, in no channel).""" + for file_id in files: + try: + call('files.delete', file=file_id) + except Exception as e: + print("Slack files.delete failed:", net.redact(e), file=sys.stderr) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + data = {'channel': self.where, 'ts': message, 'text': text} + if buttons is not None: + data['blocks'] = json.dumps(text_blocks(text, buttons)) + try: + call('chat.update', **data) + except Exception as e: + print("Slack chat.update failed:", net.redact(e), file=sys.stderr) + + +SECTION_LIMIT = 3000 # characters of text in a Block Kit section + + +def section_blocks(text): + """Block Kit sections of a message's text, as many as its length needs (none for no text).""" + return [{'type': 'section', 'text': {'type': 'mrkdwn', 'text': '\n'.join(part)}} + for part in transports.chunked(text.split('\n'), SECTION_LIMIT) if '\n'.join(part).strip()] + + +def actions_block(row): + """A Block Kit row of buttons, the one shown now marked.""" + return {'type': 'actions', 'elements': [{'type': 'button', 'text': {'type': 'plain_text', 'text': button.label}, + 'value': button.data, 'action_id': button.data, **({'style': 'primary'} if button.active else {})} + for button in row]} + + +def chart_blocks(caption, buttons, file_id): + """Block Kit for a chart message: the report, the chart (an uploaded file, by its id) and the buttons in one + row.""" + row = [button for line in buttons for button in line] + return section_blocks(caption) + [{'type': 'image', 'slack_file': {'id': file_id}, 'alt_text': 'chart'}] + \ + ([actions_block(row)] if row else []) + + +def marked(buttons): + """The data of the button marked as shown now, which names a chart message's period (None when none is).""" + return next((button.data for row in buttons or [] for button in row if button.active), None) + + +def text_blocks(text, buttons): + """Block Kit for a message with buttons: the text, then the buttons, a row of them to a block.""" + return section_blocks(text) + [actions_block(row) for row in buttons if row] + + +def until_shown(post, waits=(1, 2, 4)): + """post() of a message that shows a file just uploaded, tried again after each wait (in seconds) while Slack says + invalid_blocks: a file can take a moment before it can be shown.""" + for wait in waits: + try: + return post() + except RuntimeError as e: + if 'invalid_blocks' not in str(e): + raise + print(f"Slack cannot show the chart yet ({e}), trying again in {wait} s", file=sys.stderr) + time.sleep(wait) + return post() + + +READER = Slack() diff --git a/lib/slack_webhook.py b/lib/slack_webhook.py new file mode 100644 index 00000000..6f4037a6 --- /dev/null +++ b/lib/slack_webhook.py @@ -0,0 +1,60 @@ +"""Slack's Request URL, its HTTP mode (slackOutgoingWebhook set): Slack posts each event, button press and slash +command to finbot's URL, signed with the app's signing secret, for the reader (slack_common.READER).""" +import hashlib +import hmac +import time +from urllib.parse import parse_qs, urlparse + +from lib import config +from lib import inbound +from lib import slack_common + + +SIGNATURE_AGE = 300 # seconds: Slack's limit; an older signed request may be a replay + + +class SlackWebhook(inbound.Webhook): + """Slack posting events, button presses and slash commands to slackOutgoingWebhook (the app's Request URL for each), + each request signed with the app's signing secret, and carrying its verification token.""" + + def path(self): + """The path of slackOutgoingWebhook.""" + return urlparse(config.env.slackOutgoingWebhook).path if config.env.slackOutgoingWebhook else '' + + def refuse(self, environ, body, update): + """With slackSigningSecret, the request must carry Slack's signature of its body, made in the last five minutes; + without it, the body must carry slackOutgoingToken (the older way, which Slack has deprecated).""" + if config.env.slackSigningSecret: + return self.signature_problem(environ, body, config.env.slackSigningSecret) + if 'token' not in update: + return "Slack authorisation field not present" + if not inbound.same_secret(str(update['token']), config.env.slackOutgoingToken): + return "Slack auth incorrect" + return None + + @staticmethod + def signature_problem(environ, body, secret): + """Why the request's signature fails, or None: X-Slack-Signature must be v0= and the HMAC-SHA256, under the + signing secret, of v0::, and the timestamp within five minutes of now.""" + timestamp = environ.get('HTTP_X_SLACK_REQUEST_TIMESTAMP', '') + signature = environ.get('HTTP_X_SLACK_SIGNATURE') + if signature is None or not (timestamp.isascii() and timestamp.isdigit()): + return "Slack signature not present" + if abs(time.time() - int(timestamp)) > SIGNATURE_AGE: + return "Slack signature too old (a replay, or a clock is wrong)" + signed = b'v0:' + timestamp.encode() + b':' + body + expected = 'v0=' + hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest() + return None if inbound.same_secret(signature, expected) else "Slack signature incorrect" + + def decode(self, body): + """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON, and a + slash command as a form of its fields (command, text, channel_id, user_id and the rest).""" + if not body.lstrip().startswith(b'{'): + fields = parse_qs(body.decode('utf-8', 'replace'), keep_blank_values=True) + if 'command' in fields and 'payload' not in fields: + return {name: values[0] for name, values in fields.items()} + body = fields.get('payload', [''])[0].encode() + return super().decode(body) + + +WEBHOOK = SlackWebhook(slack_common.READER) diff --git a/lib/slack_websocket.py b/lib/slack_websocket.py new file mode 100644 index 00000000..01828071 --- /dev/null +++ b/lib/slack_websocket.py @@ -0,0 +1,120 @@ +"""Slack's Socket Mode, its pull mode (no slackOutgoingWebhook): a WebSocket that the app-level token opens, whose +envelopes carry what Slack would post to the Request URL, for the reader (slack_common.READER).""" +import json +import sys + +from lib import config +from lib import net +from lib import pull +from lib import slack_common + + +class WebSocket: + """A WebSocket connection, through websocket-client: imported here only, as Slack's pull mode is the one thing that + needs it (Debian's python3-websocket).""" + + def __init__(self, url): + """Connect to url, waiting net.CONNECT_TIMEOUT seconds at most for the server.""" + import websocket + self.library = websocket + self.ws = websocket.create_connection(url, timeout=net.CONNECT_TIMEOUT) + + def receive(self, timeout): + """The next message's text; '' for a ping or a pong, which only say the line is alive; None when nothing came + within timeout seconds. Raises ConnectionError when the server closes the socket.""" + self.ws.settimeout(timeout) + try: + opcode, data = self.ws.recv_data(control_frame=True) + except self.library.WebSocketTimeoutException: + return None + if opcode == self.library.ABNF.OPCODE_CLOSE: + raise ConnectionError('the server closed the socket') + if opcode == self.library.ABNF.OPCODE_TEXT: + return data.decode('utf-8') if isinstance(data, bytes) else data + return '' + + def send(self, text): + """Send a text message.""" + self.ws.send(text) + + def ping(self): + """Send a ping, which the server answers with a pong.""" + self.ws.ping() + + def close(self): + """Close the socket; a goodbye that fails is of no matter.""" + try: + self.ws.close(timeout=1) + except Exception: + pass + + +class SlackSocket(pull.Listener): + """Slack's Socket Mode: a WebSocket to the URL that apps.connections.open gives for the app-level token + (slackAppToken). Each event, button press or slash command comes in an envelope, acked at once, before anything + else (Slack sends again what is not acked within 3 seconds); inside it is what Slack would post to + slackOutgoingWebhook. Slack closes a socket now and then, saying so first (disconnect), and another is opened at + once.""" + network = slack_common.READER + PING_SECONDS = 30 # silence this long sends a ping; as long again after it, and the socket is taken for dead + TAKEN = ('events_api', 'interactive', 'slash_commands') # the envelopes whose payload goes to the reader + + @staticmethod + def wanted(): + """Whether finbot.env asks for it: Slack's bot token and app-level token, and no slackOutgoingWebhook.""" + env = config.env + return bool(env.slackBotToken and env.slackAppToken and not env.slackOutgoingWebhook) + + def describe(self): + """The mode, for the log.""" + return 'Socket Mode (no slackOutgoingWebhook)' + + def open(self): + """The URL of a new socket, from apps.connections.open.""" + return slack_common.call('apps.connections.open', as_app=True)['url'] + + def connect(self, url): + """A WebSocket connected to url.""" + return WebSocket(url) + + def poll(self, take): + """One socket, from opening it to Slack closing it: then another at once; after a failure, the back-off.""" + try: + connection = self.connect(self.open()) + except Exception as e: + return self.failed('opening a socket', e) + try: + reason = self.listen(connection, take) + except Exception as e: + return self.failed('the socket', e) + finally: + connection.close() + print(f"slack: Slack is closing the socket ({reason}), opening another", file=sys.stderr) + return 0 + + def listen(self, connection, take): + """Read envelopes until Slack says it will close the socket, and return its reason: each acked, then its payload + handed on. A ping goes out after PING_SECONDS of silence; as long again with nothing, not even the pong, raises + TimeoutError.""" + pinged = False + while True: + text = connection.receive(self.PING_SECONDS) + if text is None: + if pinged: + raise TimeoutError(f'nothing from Slack for {2 * self.PING_SECONDS} s, even after a ping') + connection.ping() + pinged = True + continue + pinged = False + if not text: # a ping or a pong + continue + envelope = json.loads(text) + if envelope.get('envelope_id'): + connection.send(json.dumps({'envelope_id': envelope['envelope_id']})) + kind = envelope.get('type') + if kind == 'hello': + self.failures = 0 + elif kind == 'disconnect': + return envelope.get('reason') or 'no reason given' + elif kind in self.TAKEN: + self.hand_on(take, envelope.get('payload') or {}) diff --git a/lib/telegram.py b/lib/telegram.py deleted file mode 100644 index 5633f727..00000000 --- a/lib/telegram.py +++ /dev/null @@ -1,66 +0,0 @@ -import json -import sys -from lib import config -from lib import net - -def call(method, **kwargs): - """POST a Bot API method and return its result. Raises RuntimeError with Telegram's reason when it is not ok.""" - response = net.post(config.env.telegram_api + method, **kwargs) - try: - output = response.json() - except ValueError: - output = {} - if not output.get('ok'): - raise RuntimeError(f"Telegram {method}: {output.get('description') or response.status_code}") - return output['result'] - -def try_call(method, **kwargs): - """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" - try: - return call(method, **kwargs) - except Exception as e: - print(f"Telegram {method} failed:", net.redact(e), file=sys.stderr) - -def setWebhook(): - """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken. The - log does not show the URL: its path is a secret.""" - print("telegram: setting the webhook (telegramOutgoingWebhook)", file=sys.stderr) - params = {'url': config.env.telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), - 'secret_token': config.env.telegramOutgoingToken} - return call('setWebhook', params=params) - -def deleteWebhook(): - """Have Telegram stop posting updates. Those not delivered yet are kept, for the webhook set next or getUpdates.""" - print("telegram: deleting the webhook", file=sys.stderr) - return call('deleteWebhook', json={'drop_pending_updates': False}) - -def getMe(): - """The bot's own account: its username is the bot's name in chats.""" - return call('getMe') - -def register(): - """Called once by bot.py as it starts, never on import (the job processes and the tests import this module too). - With telegramOutgoingWebhook (http mode): delete the webhook and set it again, which wakes up a stale connection on - Telegram's side. Without it (pull mode): delete any webhook, so that getUpdates may take the updates, those not - delivered yet included. Returns the bot's @username. Raises RuntimeError, or a requests error, if a call fails.""" - deleteWebhook() - if config.env.telegramOutgoingWebhook: - setWebhook() - return '@' + getMe()['username'] - -def set_commands(commands): - """Make the bot's command menu, in every chat, these slash commands ([(name, description, arguments)], from - worker.slash_commands), each with its usage, where Telegram's differs. A failure is only logged: typed, a command - works without the menu.""" - wanted = [{'command': command.name, 'description': command.description[:256]} for command in commands] - if try_call('getMyCommands') != wanted and try_call('setMyCommands', json={'commands': wanted}) is not None: - print(f"telegram: {len(wanted)} commands in the menu", file=sys.stderr) - -def pinChatMessage(chat_id, message_id): - """Pin a message, without the "pinned a message" notice. A failure is only logged: the message is there anyway.""" - try_call('pinChatMessage', json={'chat_id': chat_id, 'message_id': message_id, 'disable_notification': True}) - -def answerCallbackQuery(callback_query_id, text=None): - """Acknowledge a button press (the client's spinner stops), `text` as a toast. A failure is only logged.""" - try_call('answerCallbackQuery', json={'callback_query_id': callback_query_id} | ({'text': text} if text else {})) - diff --git a/lib/telegram_common.py b/lib/telegram_common.py new file mode 100644 index 00000000..fddd996d --- /dev/null +++ b/lib/telegram_common.py @@ -0,0 +1,212 @@ +"""Telegram, whichever way its updates arrive (long polling, telegram_longpolling; or the webhook, telegram_webhook): +the Bot API calls, the reader of its updates (Telegram, READER), and the bot that posts into its chats +(TelegramBot).""" +import html +import json +import sys + +import requests + +from lib import buttons +from lib import config +from lib import dialects +from lib import inbound +from lib import net +from lib import transports + + +def call(method, **kwargs): + """POST a Bot API method and return its result. Raises RuntimeError with Telegram's reason when it is not ok.""" + response = net.post(config.env.telegram_api + method, **kwargs) + try: + output = response.json() + except ValueError: + output = {} + if not output.get('ok'): + raise RuntimeError(f"Telegram {method}: {output.get('description') or response.status_code}") + return output['result'] + + +def try_call(method, **kwargs): + """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" + try: + return call(method, **kwargs) + except Exception as e: + print(f"Telegram {method} failed:", net.redact(e), file=sys.stderr) + + +def deleteWebhook(): + """Have Telegram stop posting updates. Those not delivered yet are kept, for the webhook set next or getUpdates.""" + print("telegram: deleting the webhook", file=sys.stderr) + return call('deleteWebhook', json={'drop_pending_updates': False}) + + +def bot_name(): + """The bot's @username, from its own account (getMe): a command in a group may name the bot (/price@finbot).""" + return '@' + call('getMe')['username'] + + +def set_commands(commands): + """Make the bot's command menu, in every chat, these slash commands ([(name, description, arguments)], from + worker.slash_commands), each with its usage, where Telegram's differs. A failure is only logged: typed, a command + works without the menu.""" + wanted = [{'command': command.name, 'description': command.description[:256]} for command in commands] + if try_call('getMyCommands') != wanted and try_call('setMyCommands', json={'commands': wanted}) is not None: + print(f"telegram: {len(wanted)} commands in the menu", file=sys.stderr) + + +def pinChatMessage(chat_id, message_id): + """Pin a message, without the "pinned a message" notice. A failure is only logged: the message is there anyway.""" + try_call('pinChatMessage', json={'chat_id': chat_id, 'message_id': message_id, 'disable_notification': True}) + + +def answerCallbackQuery(callback_query_id, text=None): + """Acknowledge a button press (the client's spinner stops), `text` as a toast. A failure is only logged.""" + try_call('answerCallbackQuery', json={'callback_query_id': callback_query_id} | ({'text': text} if text else {})) + + +class Telegram(inbound.Network): + """Telegram's updates: messages and button presses.""" + service = 'telegram' + bot_name = None # the bot's @username, from bot_name() as bot.py starts + IGNORED = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', + 'venue', 'poll', 'dice'} # nothing to answer, nothing worth logging + + def update_id(self, update): + """Telegram's update_id.""" + return update.get('update_id') + + def parse(self, update): + """A message (its text, or a photo's caption) or a button press. A first name is escaped as it arrives: + replies are Telegram HTML, and a name is text, not markup.""" + if 'callback_query' in update: # chart period and job buttons + press = update['callback_query'] + message = press.get('message') + if not message or not press.get('data'): + return inbound.UNSUPPORTED + sender = press.get('from', {}) + who = '@' + sender['username'] if 'username' in sender else sender.get('first_name', 'unknown') + return inbound.Press(self.service, press['id'], str(message['chat']['id']), str(message['message_id']), + press['data'], who) + if 'message' not in update: + return inbound.UNSUPPORTED + message = update['message'] + sender = message['from'] + chat_id = str(message['chat']['id']) + first_name = html.escape(sender['first_name'], quote=False) + if 'username' in sender: + user = '@' + sender['username'] + user_name = first_name or user # the handle when the first name is blank + else: + user = user_name = '@' + first_name + if 'text' in message: + text = message['text'] + elif 'photo' in message: + text = message.get('caption', '') + elif self.IGNORED & message.keys(): + return inbound.OK + else: + print("[telegram]: unhandled: 'message' without 'text/photo'", file=sys.stderr) + return inbound.UNHANDLED + return inbound.Message(self.service, chat_id, user, text, self.bot_name, user_name, + str(message.get('message_id', ''))) + + def acknowledge(self, event): + """A press: answerCallbackQuery, so that the client's spinner stops and a toast made from the button's data says + what is coming (buttons.toast: 'Loading 1Y…'). A failure is only logged. Nothing goes back with the update.""" + if isinstance(event, inbound.Press): + answerCallbackQuery(event.press_id, buttons.toast(event.data)) + return None + + +class TelegramBot(transports.Transport): + """A Telegram chat via the Bot API: HTML, silent, no link previews; charts, buttons, edits, the typing notice.""" + name = 'telegram' + dialect = dialects.TELEGRAM + + @staticmethod + def markup(buttons): + """Rows of buttons.Button as Telegram's inline keyboard ('● ' marks the active one); a markup of Telegram's own + (the command keyboard) as it is.""" + if isinstance(buttons, dict): + return buttons + return {'inline_keyboard': [[{'text': ('● ' if button.active else '') + button.label, + 'callback_data': button.data} for button in row] for row in buttons]} + + def post_text(self, text, buttons=None): + """sendMessage: Telegram's message, or None when it refused or failed (logged with Telegram's reason, such as + "can't parse entities").""" + payload = {'chat_id': self.where, 'text': text, 'parse_mode': 'HTML', 'disable_web_page_preview': True, + 'disable_notification': True} + if buttons: + payload['reply_markup'] = self.markup(buttons) + try: + message = call('sendMessage', json=payload) + except Exception as e: + print("Telegram sendMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) + return None + print("OK telegram message to", self.where, file=sys.stderr) + return message + + def post_chart(self, image, caption, buttons=None): + """sendPhoto, the report as its caption: Telegram's message, MAYBE_SENT, or None when it refused or failed + (logged with Telegram's reason).""" + data = {'chat_id': self.where, 'caption': caption, 'parse_mode': 'HTML', 'disable_notification': True} + if buttons: + data['reply_markup'] = json.dumps(self.markup(buttons)) + try: + message = call('sendPhoto', data=data, files={'photo': ('image.png', image)}) + except requests.exceptions.ReadTimeout as e: + print("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) + return transports.MAYBE_SENT + except Exception as e: + print("Telegram sendPhoto to", self.where, "failed:", net.redact(e), file=sys.stderr) + return None + print("OK telegram chart to", self.where, file=sys.stderr) + return message + + def edit_chart(self, message, image, caption, buttons=None): + """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Telegram refuses, + unless the message is already so (pressing the active period again is harmless); a request that fails on the way + is only logged.""" + media = {'type': 'photo', 'media': 'attach://photo', 'caption': transports.fit_caption(caption, self.dialect), + 'parse_mode': 'HTML'} + data = {'chat_id': self.where, 'message_id': message, 'media': json.dumps(media)} + if buttons: + data['reply_markup'] = json.dumps(self.markup(buttons)) + if hasattr(image, 'seek'): + image.seek(0) + try: + call('editMessageMedia', data=data, files={'photo': ('image.png', image)}) + except RuntimeError as e: + if 'not modified' not in str(e): + raise + except Exception as e: # it may have arrived even so: no error line for something the chat may already show + print("Telegram editMessageMedia failed:", net.redact(e), file=sys.stderr) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + payload = {'chat_id': self.where, 'message_id': message, 'text': text, 'parse_mode': 'HTML', + 'disable_web_page_preview': True} + if buttons is not None: + payload['reply_markup'] = self.markup(buttons) + try_call('editMessageText', json=payload) + + def typing(self): + """sendChatAction: "typing…" for a few seconds.""" + try_call('sendChatAction', data={'chat_id': self.where, 'action': 'typing'}) + + def pin(self, answers): + """Pin the messages that these answers are (Telegram's), last first, so that the first ends up on top, without + the notification.""" + sent = [] + for answer in answers: + try: + sent.append((answer['chat']['id'], answer['message_id'])) + except (KeyError, TypeError): + print("Unable to pin: unexpected response", file=sys.stderr) + for chat, message in reversed(sent): + pinChatMessage(chat, message) + + +READER = Telegram() diff --git a/lib/telegram_longpolling.py b/lib/telegram_longpolling.py new file mode 100644 index 00000000..b95f8b78 --- /dev/null +++ b/lib/telegram_longpolling.py @@ -0,0 +1,48 @@ +"""Telegram's long polling, its pull mode (no telegramOutgoingWebhook): finbot asks for its updates (getUpdates) again +and again, and hands each to the reader (telegram_common.READER) as the webhook would.""" +from lib import config +from lib import net +from lib import pull +from lib import telegram_common + + +def register(): + """Called once by bot.py as it starts, never on import: delete any webhook, so that getUpdates may take the + updates, those not delivered yet included. Raises RuntimeError, or a requests error, if the call fails.""" + telegram_common.deleteWebhook() + + +class TelegramPoll(pull.Listener): + """Telegram's long polling: getUpdates, which Telegram holds open until there are updates or POLL_SECONDS have + passed, asking for those after the last one taken (offset), which also tells Telegram that one arrived. No webhook + may be set meanwhile: register() deletes it as finbot starts.""" + network = telegram_common.READER + POLL_SECONDS = 50 # how long Telegram may hold a getUpdates open + READ_SECONDS = 65 # the request's own timeout for the answer: longer than Telegram holds it + offset = None # the update_id after the last one taken + + @staticmethod + def wanted(): + """Whether finbot.env asks for it: Telegram's bot token, and no telegramOutgoingWebhook.""" + return bool(config.env.telegramBotToken and not config.env.telegramOutgoingWebhook) + + def describe(self): + """The mode, for the log.""" + return 'long polling for updates (no telegramOutgoingWebhook)' + + def poll(self, take): + """One getUpdates: each update to take, in order; then no wait, or the back-off after a failure (a 409 says a + webhook is set, or another finbot polls with the same token).""" + params = {'timeout': self.POLL_SECONDS, 'allowed_updates': ['message', 'callback_query']} + if self.offset is not None: + params['offset'] = self.offset + try: + updates = telegram_common.call('getUpdates', json=params, timeout=(net.CONNECT_TIMEOUT, self.READ_SECONDS)) + except Exception as e: + return self.failed('getUpdates', e) + self.failures = 0 + for update in updates: + if isinstance(update.get('update_id'), int): + self.offset = update['update_id'] + 1 # taken, even if handling it fails: it is not fetched again + self.hand_on(take, update) + return 0 diff --git a/lib/telegram_webhook.py b/lib/telegram_webhook.py new file mode 100644 index 00000000..4de523ac --- /dev/null +++ b/lib/telegram_webhook.py @@ -0,0 +1,43 @@ +"""Telegram's webhook, its HTTP mode (telegramOutgoingWebhook set): Telegram posts each update to finbot's URL with +the secret header that setWebhook gave it, for the reader (telegram_common.READER).""" +import json +import sys +from urllib.parse import urlparse + +from lib import config +from lib import inbound +from lib import telegram_common + + +def setWebhook(): + """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken. The + log does not show the URL: its path is a secret.""" + print("telegram: setting the webhook (telegramOutgoingWebhook)", file=sys.stderr) + params = {'url': config.env.telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), + 'secret_token': config.env.telegramOutgoingToken} + return telegram_common.call('setWebhook', params=params) + + +def register(): + """Called once by bot.py as it starts, never on import: delete the webhook and set it again, which wakes up a stale + connection on Telegram's side. Raises RuntimeError, or a requests error, if a call fails.""" + telegram_common.deleteWebhook() + setWebhook() + + +class TelegramWebhook(inbound.Webhook): + """Telegram posting to telegramOutgoingWebhook, each update with the secret header that setWebhook gave it.""" + + def path(self): + """The path of telegramOutgoingWebhook.""" + return urlparse(config.env.telegramOutgoingWebhook).path if config.env.telegramOutgoingWebhook else '' + + def refuse_unread(self, environ): + """The header must carry telegramOutgoingToken.""" + secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') + if not inbound.same_secret(secret, config.env.telegramOutgoingToken): + return f"Telegram authorisation header {'not present' if secret is None else 'is present but incorrect'}" + return None + + +WEBHOOK = TelegramWebhook(telegram_common.READER) diff --git a/lib/transports.py b/lib/transports.py index 6e7b560d..69feec4c 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -1,25 +1,13 @@ -"""How a message gets to each network: a class per network's bot, made for one destination (a chat). A transport knows -its dialect, what it can carry (charts, buttons, a typing notice) and the network's size limits; text() and chart() are +"""How a message gets to a network: Transport, the base of each network's bot (TelegramBot, SlackBot and DiscordBot, +in telegram_common, slack_common and discord_common), made for one destination (a chat). A transport knows its +dialect, what it can carry (charts, buttons, a typing notice) and the network's size limits; text() and chart() are the same everywhere, and each network posts one message its own way (post_text, post_chart). Every bot takes buttons: whichever way its network's updates arrive, their presses reach finbot.""" -import json -import sys import threading import time -import requests - -from lib import config -from lib import dialects -from lib import discord -from lib import net -from lib import state -from lib import telegram MAYBE_SENT = object() # a read timeout on the request that posts a chart: it may have arrived, so no text copy follows -SLACK_CHARTS = 'finbot_slack_charts.json' # Slack's chart messages: the file each shows, and the charts uploaded -SLACK_CHARTS_KEPT = 500 # chart messages noted, newest last: an older one shows its file for good -SLACK_REUSE_SECONDS = 15 * 60 # a period pressed within this long of its chart's upload shows that upload again class Transport: @@ -133,460 +121,6 @@ def reshow(self, message, buttons): return False -class TelegramBot(Transport): - """A Telegram chat via the Bot API: HTML, silent, no link previews; charts, buttons, edits, the typing notice.""" - name = 'telegram' - dialect = dialects.TELEGRAM - - @staticmethod - def markup(buttons): - """Rows of buttons.Button as Telegram's inline keyboard ('● ' marks the active one); a markup of Telegram's own - (the command keyboard) as it is.""" - if isinstance(buttons, dict): - return buttons - return {'inline_keyboard': [[{'text': ('● ' if button.active else '') + button.label, - 'callback_data': button.data} for button in row] for row in buttons]} - - def post_text(self, text, buttons=None): - """sendMessage: Telegram's message, or None when it refused or failed (logged with Telegram's reason, such as - "can't parse entities").""" - payload = {'chat_id': self.where, 'text': text, 'parse_mode': 'HTML', 'disable_web_page_preview': True, - 'disable_notification': True} - if buttons: - payload['reply_markup'] = self.markup(buttons) - try: - message = telegram.call('sendMessage', json=payload) - except Exception as e: - print("Telegram sendMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - print("OK telegram message to", self.where, file=sys.stderr) - return message - - def post_chart(self, image, caption, buttons=None): - """sendPhoto, the report as its caption: Telegram's message, MAYBE_SENT, or None when it refused or failed - (logged with Telegram's reason).""" - data = {'chat_id': self.where, 'caption': caption, 'parse_mode': 'HTML', 'disable_notification': True} - if buttons: - data['reply_markup'] = json.dumps(self.markup(buttons)) - try: - message = telegram.call('sendPhoto', data=data, files={'photo': ('image.png', image)}) - except requests.exceptions.ReadTimeout as e: - print("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT - except Exception as e: - print("Telegram sendPhoto to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - print("OK telegram chart to", self.where, file=sys.stderr) - return message - - def edit_chart(self, message, image, caption, buttons=None): - """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Telegram refuses, - unless the message is already so (pressing the active period again is harmless); a request that fails on the way - is only logged.""" - media = {'type': 'photo', 'media': 'attach://photo', 'caption': fit_caption(caption, self.dialect), - 'parse_mode': 'HTML'} - data = {'chat_id': self.where, 'message_id': message, 'media': json.dumps(media)} - if buttons: - data['reply_markup'] = json.dumps(self.markup(buttons)) - if hasattr(image, 'seek'): - image.seek(0) - try: - telegram.call('editMessageMedia', data=data, files={'photo': ('image.png', image)}) - except RuntimeError as e: - if 'not modified' not in str(e): - raise - except Exception as e: # it may have arrived even so: no error line for something the chat may already show - print("Telegram editMessageMedia failed:", net.redact(e), file=sys.stderr) - - def edit_text(self, message, text, buttons=None): - """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" - payload = {'chat_id': self.where, 'message_id': message, 'text': text, 'parse_mode': 'HTML', - 'disable_web_page_preview': True} - if buttons is not None: - payload['reply_markup'] = self.markup(buttons) - telegram.try_call('editMessageText', json=payload) - - def typing(self): - """sendChatAction: "typing…" for a few seconds.""" - telegram.try_call('sendChatAction', data={'chat_id': self.where, 'action': 'typing'}) - - def pin(self, answers): - """Pin the messages that these answers are (Telegram's), last first, so that the first ends up on top, without - the notification.""" - sent = [] - for answer in answers: - try: - sent.append((answer['chat']['id'], answer['message_id'])) - except (KeyError, TypeError): - print("Unable to pin: unexpected response", file=sys.stderr) - for chat, message in reversed(sent): - telegram.pinChatMessage(chat, message) - - -class SlackBot(Transport): - """A Slack channel, as the bot: chat.postMessage and the file upload, with slackBotToken.""" - name = 'slack' - dialect = dialects.SLACK - WORKING = 'hourglass_flowing_sand' # ⏳: Slack has no typing notice for bots, so a reaction on the message stands in - working = None # True while ⏳ is on the message, False when putting it there failed - - @property - def shows_typing(self): - """⏳ on the message being answered: a scheduled report answers none.""" - return bool(self.message) - - def typing(self): - """Put ⏳ on the message being answered, once (the bot needs the reactions:write scope); a failure is only - logged, and not tried again.""" - if self.working is not None: - return - try: - slack_api('reactions.add', channel=self.where, timestamp=self.message, name=self.WORKING) - self.working = True - except Exception as e: - self.working = False - print("Slack reactions.add failed:", net.redact(e), file=sys.stderr) - - def typing_done(self): - """Take ⏳ off the message again; a failure is only logged.""" - if not self.working: - return - self.working = False - try: - slack_api('reactions.remove', channel=self.where, timestamp=self.message, name=self.WORKING) - except Exception as e: - print("Slack reactions.remove failed:", net.redact(e), file=sys.stderr) - - def post_text(self, text, buttons=None): - """chat.postMessage, without link previews, and with buttons (a Block Kit message) where they work. Slack's - answer, or None when it failed or refused (logged): Slack refuses with HTTP 200 and ok false (not_in_channel, - say), so a refusal is read from the answer.""" - if not config.env.slackBotToken: - print(f"ERROR slack: a reply to channel {self.where} needs slackBotToken in finbot.env; not sent", - file=sys.stderr) - return None - blocks = {} - if buttons and not isinstance(buttons, dict) and self.takes_buttons: - blocks['blocks'] = json.dumps(text_blocks(text, buttons)) - try: - answer = slack_api('chat.postMessage', channel=self.where, text=text, unfurl_links='false', - unfurl_media='false', **blocks) - except Exception as e: - print("Slack chat.postMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - print("OK slack message to", self.where, file=sys.stderr) - return answer - - def post_chart(self, image, caption, buttons=None): - """Upload the chart into the channel with the report as its message, the way Slack takes files since it retired - files.upload: ask for an upload URL, send the bytes there, then complete the upload into the channel (the bot - needs the files:write and files:read scopes). With period buttons, where they work, it goes in a Block Kit - message instead (post_blocks), or without them if that fails. Slack's answer, MAYBE_SENT, or None if a step - failed.""" - data = image.read() if hasattr(image, 'read') else bytes(image) - if buttons and not isinstance(buttons, dict) and self.takes_buttons: - try: - return self.post_blocks(data, caption, buttons) - except Exception as e: - print("Slack chart with buttons failed, uploading it without them:", net.redact(e), file=sys.stderr) - try: - share = {'files': json.dumps([{'id': self.upload(data)}]), 'channel_id': self.where, - 'initial_comment': caption} - try: - output = slack_api('files.completeUploadExternal', **share) - except requests.exceptions.ReadTimeout as e: - print("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e), - file=sys.stderr) - return MAYBE_SENT - except Exception as e: - print("Slack chart upload failed:", net.redact(e), file=sys.stderr) - return None - print("OK slack chart upload to", self.where, file=sys.stderr) - return output - - def upload(self, data): - """Send a chart's bytes to Slack: the id of the file, not completed yet. Raises RuntimeError on a refusal.""" - ticket = slack_api('files.getUploadURLExternal', filename='chart.png', length=len(data)) - r = net.post(ticket['upload_url'], data=data) - if r.status_code != 200: - raise RuntimeError(f"Slack upload: {r.status_code} {r.text[:200]}") - return ticket['file_id'] - - def unshared(self, data): - """Upload a chart as a file that is in no channel, for an image block to show: its id.""" - file_id = self.upload(data) - slack_api('files.completeUploadExternal', files=json.dumps([{'id': file_id, 'title': 'chart'}])) - return file_id - - def post_blocks(self, data, caption, buttons): - """The chart in a Block Kit message, the report above it and the period buttons under it. Slack's answer, or - MAYBE_SENT when posting the message timed out (it may have arrived); raises RuntimeError when Slack refuses.""" - file_id = self.unshared(data) - blocks = chart_blocks(caption, buttons, file_id) - try: - answer = until_shown(lambda: slack_api('chat.postMessage', channel=self.where, - text=caption[:SECTION_LIMIT], blocks=json.dumps(blocks), unfurl_links='false', unfurl_media='false')) - except requests.exceptions.ReadTimeout as e: - print("Slack chat.postMessage timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT - if answer.get('ts'): - self.delete(self.shows(answer['ts'], file_id, caption, buttons)) - return answer - - def reshow(self, message, buttons): - """Show a chart message's period again from its upload of under SLACK_REUSE_SECONDS ago (for this message or - another), with the report it came with: nothing is drawn or uploaded. True when it did; False when there is no - such upload, or Slack refuses it (a file deleted outside finbot, say).""" - view, now = marked(buttons), time.time() - uploads = state.load(SLACK_CHARTS, {}).get('uploads', {}) - recent = [(at, file_id, caption) for file_id, (at, shows, caption) in uploads.items() - if view and shows == view and now - at < SLACK_REUSE_SECONDS] - if not recent: - return False - _, file_id, caption = max(recent) - try: - slack_api('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], - blocks=json.dumps(chart_blocks(caption, buttons, file_id))) - except Exception as e: - print("Slack chat.update with an earlier upload failed, uploading anew:", net.redact(e), file=sys.stderr) - return False - self.delete(self.shows(message, file_id)) - return True - - def edit_chart(self, message, image, caption, buttons=None): - """Show another period in a chart message: the new chart uploaded (a file cannot be replaced) and shown in its - image block, with its caption and buttons. A press for that period within SLACK_REUSE_SECONDS shows this upload - again (reshow). Raises RuntimeError when Slack refuses the update.""" - if hasattr(image, 'seek'): - image.seek(0) - file_id = self.unshared(image.read() if hasattr(image, 'read') else bytes(image)) - blocks = chart_blocks(caption, buttons or [], file_id) - until_shown(lambda: slack_api('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], - blocks=json.dumps(blocks))) - self.delete(self.shows(message, file_id, caption, buttons)) - - def shows(self, message, file_id, caption=None, buttons=None): - """Note that a chart message shows this file now; given its report, that the file was uploaded now for the - period its buttons mark (see reshow). The uploads to delete: those no message shows, a minute past their reuse - (a press reusing one has that minute to show it).""" - now = time.time() - with state.edit(SLACK_CHARTS, {}) as noted: - if set(noted) != {'shown', 'uploads'}: # new, or the {'channel ts': file} of before, whose files stay - noted.clear() - noted.update(shown={}, uploads={}) - shown, uploads = noted['shown'], noted['uploads'] # 'channel ts': file; file: [time, period, report] - if caption is not None: - uploads[file_id] = [now, marked(buttons), caption] - key = f'{self.where} {message}' - shown.pop(key, None) - shown[key] = file_id - for old in list(shown)[:-SLACK_CHARTS_KEPT]: # its message may show the file for good - uploads.pop(shown.pop(old), None) - showing = set(shown.values()) - gone = [old for old, (at, _, _) in uploads.items() - if old not in showing and now - at > SLACK_REUSE_SECONDS + 60] - for old in gone: - del uploads[old] - for upload in uploads.values(): # past reuse, a file shown is kept only to be deleted later - if now - upload[0] >= SLACK_REUSE_SECONDS: - upload[1:] = [None, None] - return gone - - def delete(self, files): - """Delete uploaded charts that no message shows any more; a failure is only logged (the file stays among the - bot's files, in no channel).""" - for file_id in files: - try: - slack_api('files.delete', file=file_id) - except Exception as e: - print("Slack files.delete failed:", net.redact(e), file=sys.stderr) - - def edit_text(self, message, text, buttons=None): - """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" - data = {'channel': self.where, 'ts': message, 'text': text} - if buttons is not None: - data['blocks'] = json.dumps(text_blocks(text, buttons)) - try: - slack_api('chat.update', **data) - except Exception as e: - print("Slack chat.update failed:", net.redact(e), file=sys.stderr) - - -class DiscordBot(Transport): - """A Discord channel or DM, as the bot (discordBotToken): no link previews, no pings, charts attached, buttons as - components, edits in place, the typing notice. A slash command's first message replaces Discord's "thinking…" and - the rest follow it up, through its interaction; a pressed message is edited through the press's.""" - name = 'discord' - dialect = dialects.DISCORD - - def __init__(self, where, message=None, interaction='', pressed=False): - """As Transport; a press has no "thinking…" to replace, so its new messages go into the channel.""" - super().__init__(where, message, interaction, pressed) - self.answered = pressed # the slash command's "thinking…" replaced - - @property - def shows_typing(self): - """The typing notice for a command in a message; a slash command shows Discord's "thinking…" instead.""" - return not self.interaction - - def typing(self): - """POST /typing: "finbot is typing…" for ten seconds, or until it posts.""" - discord.try_call('POST', f'/channels/{self.where}/typing') - - def typing_done(self): - """A slash command over with nothing said: its "thinking…" deleted, or it would show for 15 minutes.""" - if self.interaction and not self.answered: - self.answered = True - discord.try_call('DELETE', f'/webhooks/{self.interaction}/messages/@original') - - @staticmethod - def extras(buttons): - """What a message carries besides its text: no link previews, no pings, and its buttons as components.""" - extra = {'flags': discord.SUPPRESS_EMBEDS, 'allowed_mentions': discord.NO_MENTIONS} - if buttons and not isinstance(buttons, dict): - extra['components'] = components(buttons) - return extra - - def post_text(self, text, buttons=None): - """Post one message (see post).""" - return self.post({'content': text, **self.extras(buttons)}) - - def post_chart(self, image, caption, buttons=None): - """Post a chart, attached to the report as its message (see post).""" - attached = {'content': caption, 'attachments': [{'id': 0, 'filename': 'chart.png'}], **self.extras(buttons)} - return self.post(attached, image) - - def post(self, payload, image=None): - """Post one message: a slash command's first replaces its "thinking…" and the rest follow it up; otherwise it - goes into the channel. Discord's message; MAYBE_SENT when a chart's post timed out; None when it failed - (logged).""" - if self.interaction and not self.answered: - method, path = 'PATCH', f'/webhooks/{self.interaction}/messages/@original' - elif self.interaction and not self.pressed: - method, path = 'POST', f'/webhooks/{self.interaction}' - else: - method, path = 'POST', f'/channels/{self.where}/messages' - try: - answer = discord.call(method, path, **message_body(payload, image)) - except requests.exceptions.ReadTimeout as e: - self.answered = True # it may have arrived - print("Discord post timed out, it may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT if image is not None else None - except Exception as e: - print("Discord post to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - self.answered = True - print("OK discord message to", self.where, file=sys.stderr) - return answer or {} - - def message_path(self, message): - """Where a message is edited: through the press's interaction where there is one (a message that answered a - slash command can be edited only so), else in the channel as the bot.""" - if self.interaction: - return f'/webhooks/{self.interaction}/messages/@original' - return f'/channels/{self.where}/messages/{message}' - - def edit_chart(self, message, image, caption, buttons=None): - """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Discord refuses.""" - if hasattr(image, 'seek'): - image.seek(0) - payload = {'content': fit_caption(caption, self.dialect), 'attachments': [{'id': 0, 'filename': 'chart.png'}], - 'components': components(buttons or [])} - discord.call('PATCH', self.message_path(message), **message_body(payload, image)) - - def edit_text(self, message, text, buttons=None): - """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" - payload = {'content': text} - if buttons is not None: - payload['components'] = components(buttons) - discord.try_call('PATCH', self.message_path(message), json=payload) - - -DISCORD_ROW = 5 # buttons in a row of components, and rows in a message: Discord's limits - - -def components(buttons): - """Rows of buttons.Button as Discord's action rows: DISCORD_ROW buttons to a row and as many rows at most (the - buttons past that are left out), the one shown now blurple and the rest grey.""" - flat = [button for row in buttons for button in row][:DISCORD_ROW * DISCORD_ROW] - return [{'type': 1, 'components': [{'type': 2, 'style': 1 if button.active else 2, 'label': button.label[:80], - 'custom_id': button.data} for button in flat[i:i + DISCORD_ROW]]} for i in range(0, len(flat), DISCORD_ROW)] - - -def message_body(payload, image=None): - """A Discord message's request arguments: its JSON; with a chart, a form of the JSON and the PNG (bytes, so that a - rate-limited call can send it again).""" - if image is None: - return {'json': payload} - data = image.read() if hasattr(image, 'read') else bytes(image) - return {'data': {'payload_json': json.dumps(payload)}, 'files': {'files[0]': ('chart.png', data, 'image/png')}} - - -def slack_api(method, as_app=False, **data): - """POST a Slack Web API method (form-encoded) as the bot, or with the app-level token (as_app, for Socket Mode's - apps.connections.open), and return the answer; an error raises RuntimeError.""" - key = 'slackAppToken' if as_app else 'slackBotToken' - token = getattr(config.env, key) - if not token: - raise RuntimeError(f"Slack {method}: needs {key} in finbot.env") - r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + token}, data=data) - try: - output = r.json() - except ValueError: - output = {} - if not output.get('ok'): - raise RuntimeError(f"Slack {method}: {output.get('error') or r.status_code}") - return output - - -SECTION_LIMIT = 3000 # characters of text in a Block Kit section - - -def section_blocks(text): - """Block Kit sections of a message's text, as many as its length needs (none for no text).""" - return [{'type': 'section', 'text': {'type': 'mrkdwn', 'text': '\n'.join(part)}} - for part in chunked(text.split('\n'), SECTION_LIMIT) if '\n'.join(part).strip()] - - -def actions_block(row): - """A Block Kit row of buttons, the one shown now marked.""" - return {'type': 'actions', 'elements': [{'type': 'button', 'text': {'type': 'plain_text', 'text': button.label}, - 'value': button.data, 'action_id': button.data, **({'style': 'primary'} if button.active else {})} - for button in row]} - - -def chart_blocks(caption, buttons, file_id): - """Block Kit for a chart message: the report, the chart (an uploaded file, by its id) and the buttons in one - row.""" - row = [button for line in buttons for button in line] - return section_blocks(caption) + [{'type': 'image', 'slack_file': {'id': file_id}, 'alt_text': 'chart'}] + \ - ([actions_block(row)] if row else []) - - -def marked(buttons): - """The data of the button marked as shown now, which names a chart message's period (None when none is).""" - return next((button.data for row in buttons or [] for button in row if button.active), None) - - -def text_blocks(text, buttons): - """Block Kit for a message with buttons: the text, then the buttons, a row of them to a block.""" - return section_blocks(text) + [actions_block(row) for row in buttons if row] - - -def until_shown(post, waits=(1, 2, 4)): - """post() of a message that shows a file just uploaded, tried again after each wait (in seconds) while Slack says - invalid_blocks: a file can take a moment before it can be shown.""" - for wait in waits: - try: - return post() - except RuntimeError as e: - if 'invalid_blocks' not in str(e): - raise - print(f"Slack cannot show the chart yet ({e}), trying again in {wait} s", file=sys.stderr) - time.sleep(wait) - return post() - - def chunked(lines, limit): """The lines in messages of at most limit characters each, whole lines to a message; a line longer than limit on its own is cut short.""" diff --git a/lib/worker.py b/lib/worker.py index 8f6d7adc..03217f75 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -17,7 +17,7 @@ from lib import charts from lib.reports import about, cal, holdings, performance, price, profile, rankings, shorts, trades from lib import sharesight -from lib import transports +from lib import telegram_common from lib import jobs from lib import watchlist from lib import yahoo @@ -625,7 +625,7 @@ def mark_keyboard(chat_id): def push_keyboard(chat_id): """Telegram can only change a reply keyboard by sending a message, so send a short one carrying it.""" - result = transports.TelegramBot(chat_id).post_text('⌨️ Keyboard updated', command_keyboard()) + result = telegram_common.TelegramBot(chat_id).post_text('⌨️ Keyboard updated', command_keyboard()) if result is not None: mark_keyboard(chat_id) return result diff --git a/tests/common.py b/tests/common.py index 930bb50c..51253e05 100644 --- a/tests/common.py +++ b/tests/common.py @@ -12,8 +12,9 @@ telegramChatID='1') import bot -from lib import answer, buttons, cache, charts, config, delivery, dialects, formats, inbound, links, markets, render -from lib import sharesight, shortman, state, telegram, transports, watchlist, worker, yahoo +from lib import answer, buttons, cache, charts, config, delivery, dialects, formats, inbound, links, markets, net +from lib import render, sharesight, shortman, state, transports, watchlist, worker, yahoo +from lib import discord_common, slack_common, telegram_common # each network's reader and bot from lib.reports import about, cal, history, holdings, milestone, performance, price, profile, rankings, rating from lib.reports import reminder, shorts, trades @@ -51,6 +52,7 @@ def temp_dir(case): (delivery, 'report_error', REAL_REPORT_ERROR)) TICKERS = [f'T{i}' for i in range(25)] TELEGRAM, SLACK, DISCORD = dialects.TELEGRAM, dialects.SLACK, dialects.DISCORD +READERS = {'telegram': telegram_common.READER, 'slack': slack_common.READER, 'discord': discord_common.READER} BOT = '@bot' @@ -103,7 +105,7 @@ def join_workers(self): def press_button(self, data, chat_id=55, message_id=9): """Press a button under one of the bot's Telegram messages, as the update arrives: bot.take acknowledges it (answerCallbackQuery) and hands it to the worker, whose thread this waits for.""" - bot.take(inbound.TELEGRAM, {'callback_query': {'id': 'cb', 'from': {'username': 'u'}, 'data': data, + bot.take(telegram_common.READER, {'callback_query': {'id': 'cb', 'from': {'username': 'u'}, 'data': data, 'message': {'message_id': message_id, 'chat': {'id': chat_id}}}}) self.join_workers() @@ -178,7 +180,7 @@ def record_chart(self, dest, image, caption, markup=None): dest.stop_typing() image.seek(0) self.sent.append(('photo', len(image.read()), caption.split('\n')[0])) - self.markups.append(transports.TelegramBot.markup(markup) if markup else None) + self.markups.append(telegram_common.TelegramBot.markup(markup) if markup else None) return {'ok': True} def no_typing_left(self): diff --git a/tests/test_discord.py b/tests/test_discord.py index ba24d73f..069c44c1 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -3,7 +3,7 @@ and the gateway.""" import contextlib, io, json, sys, types, unittest from tests.common import * -from lib import discord, jobs, net, pull +from lib import discord_websocket, jobs try: from nacl.signing import SigningKey @@ -41,36 +41,36 @@ class Reading(FinbotCase): def setUp(self): """The bot's user id is 99.""" super().setUp() - self.patch(inbound.DISCORD, 'bot_id', '99') + self.patch(discord_common.READER, 'bot_id', '99') def test_a_slash_command_is_the_message_it_stands_for(self): """/price with its arguments is the message '/price 5 7d', answered through its interaction; in a DM the person is the interaction's user.""" expected = inbound.Message('discord', '77', '<@5>', '/price 5 7d', '@99', 'Rob', '', '42/tok') - self.assertEqual(inbound.DISCORD.parse(COMMAND), expected) + self.assertEqual(discord_common.READER.parse(COMMAND), expected) dm = {k: v for k, v in COMMAND.items() if k != 'member'} | {'user': {'id': '5', 'username': 'rob'}} - self.assertEqual(inbound.DISCORD.parse(dm), dataclasses.replace(expected, user_name='rob')) - self.assertEqual(inbound.DISCORD.parse({**COMMAND, 'data': {'name': 'beta'}}).text, '/beta') + self.assertEqual(discord_common.READER.parse(dm), dataclasses.replace(expected, user_name='rob')) + self.assertEqual(discord_common.READER.parse({**COMMAND, 'data': {'name': 'beta'}}).text, '/beta') def test_a_press_is_a_press_of_its_message(self): """A button pressed is a Press of its message, its custom_id the button's data.""" - self.assertEqual(inbound.DISCORD.parse(PRESS), + self.assertEqual(discord_common.READER.parse(PRESS), inbound.Press('discord', '901', '77', '333', 'c|h|y|T1', '@rob', '42/tok')) def test_a_message_by_a_person_reads_with_its_mentions_as_slacks(self): """A gateway message's mentions read as @id, so that '@bot price 7d' calls .price; one by a bot or a webhook (this bot's replies included) is not answered.""" - message = inbound.DISCORD.parse(MESSAGE) + message = discord_common.READER.parse(MESSAGE) self.assertEqual(message, inbound.Message('discord', '77', '<@5>', '@99 price 7d', '@99', 'rob', '700')) command, args = worker.read(message.text, message.bot_name) self.assertEqual((command.names[0], args.days), ('price', 7)) for other in ({**MESSAGE, 'author': {'id': '99', 'bot': True}}, {**MESSAGE, 'webhook_id': '1'}): - self.assertIs(inbound.DISCORD.parse(other), inbound.OK) + self.assertIs(discord_common.READER.parse(other), inbound.OK) def test_the_endpoints_check_and_other_interactions(self): """A PING gets its PONG at once; an interaction of another kind is not taken.""" - self.assertEqual(json.loads(inbound.DISCORD.parse(interaction(1)).body), {'type': 1}) - self.assertIs(inbound.DISCORD.parse(interaction(5)), inbound.UNSUPPORTED) + self.assertEqual(json.loads(discord_common.READER.parse(interaction(1)).body), {'type': 1}) + self.assertIs(discord_common.READER.parse(interaction(5)), inbound.UNSUPPORTED) def test_an_interaction_is_answered_before_its_work_starts(self): """A slash command is answered "thinking…" and a press with nothing to see, and the work waits in the answer @@ -78,13 +78,13 @@ def test_an_interaction_is_answered_before_its_work_starts(self): taken = [] self.patch(worker, 'process_request', taken.append) self.patch(worker, 'process_callback', taken.append) - answer = bot.take(inbound.DISCORD, COMMAND) + answer = bot.take(discord_common.READER, COMMAND) self.assertEqual(json.loads(answer.body), {'type': 5}) self.join_workers() self.assertEqual(taken, []) answer.then() - self.assertEqual(json.loads(bot.take(inbound.DISCORD, PRESS).body), {'type': 6}) - self.assertIs(bot.take(inbound.DISCORD, MESSAGE), inbound.OK) + self.assertEqual(json.loads(bot.take(discord_common.READER, PRESS).body), {'type': 6}) + self.assertIs(bot.take(discord_common.READER, MESSAGE), inbound.OK) self.join_workers() self.assertEqual([event.text if isinstance(event, inbound.Message) else event.data for event in taken], ['/price 5 7d', '@99 price 7d']) @@ -161,9 +161,9 @@ def setUp(self): self.calls, self.answers = [], [] def request(method, url, **kwargs): """Note the call, and give the next answer.""" - self.calls.append((method, url.removeprefix(discord.API), kwargs)) + self.calls.append((method, url.removeprefix(discord_common.API), kwargs)) return self.answers.pop(0) if self.answers else Resp({'id': '1'}) - self.patch(discord.net, 'request', request) + self.patch(net, 'request', request) self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) @@ -172,7 +172,7 @@ def test_a_message_has_no_previews_or_pings_and_its_buttons_in_rows_of_five(self blurple, and past 25 they are left out.""" row = [buttons.Button(f'B{i}', f'c|h|w|{i}', active=i == 0) for i in range(12)] with contextlib.redirect_stderr(io.StringIO()): - transports.DiscordBot('77').text(['hello'], [row[:7], row[7:]]) + discord_common.DiscordBot('77').text(['hello'], [row[:7], row[7:]]) method, path, kwargs = self.calls[0] self.assertEqual((method, path), ('POST', '/channels/77/messages')) payload = kwargs['json'] @@ -180,12 +180,12 @@ def test_a_message_has_no_previews_or_pings_and_its_buttons_in_rows_of_five(self {'parse': []})) self.assertEqual([len(r['components']) for r in payload['components']], [5, 5, 2]) self.assertEqual([b['style'] for b in payload['components'][0]['components']], [1, 2, 2, 2, 2]) - self.assertEqual(len(transports.components([[buttons.Button('x', 'y')] * 30])), 5) + self.assertEqual(len(discord_common.components([[buttons.Button('x', 'y')] * 30])), 5) def test_a_chart_is_attached_to_its_report(self): """A chart goes as an attachment of the message that carries the report and the buttons.""" with contextlib.redirect_stderr(io.StringIO()): - transports.DiscordBot('77').chart(io.BytesIO(b'png'), 'caption', [[buttons.Button('7D', 'c|h|w|T1')]]) + discord_common.DiscordBot('77').chart(io.BytesIO(b'png'), 'caption', [[buttons.Button('7D', 'c|h|w|T1')]]) method, path, kwargs = self.calls[0] payload = json.loads(kwargs['data']['payload_json']) self.assertEqual((method, path, payload['content'], payload['attachments']), ('POST', '/channels/77/messages', @@ -196,7 +196,7 @@ def test_a_chart_is_attached_to_its_report(self): def test_a_slash_commands_answer_replaces_its_thinking_and_the_rest_follow_up(self): """A slash command's first message replaces Discord's "thinking…", through its interaction, and the rest follow it up; there is no typing notice besides.""" - chat = transports.DiscordBot('77', interaction='42/tok') + chat = discord_common.DiscordBot('77', interaction='42/tok') self.assertFalse(chat.shows_typing) with contextlib.redirect_stderr(io.StringIO()): chat.text(['x' * 1500, 'y' * 1500]) @@ -208,7 +208,7 @@ def test_a_slash_commands_answer_replaces_its_thinking_and_the_rest_follow_up(se def test_a_slash_command_that_says_nothing_has_its_thinking_deleted(self): """A slash command over without a word, or one that reads as no command, has its "thinking…" deleted, or it would show for 15 minutes.""" - transports.DiscordBot('77', interaction='42/tok').typing_done() + discord_common.DiscordBot('77', interaction='42/tok').typing_done() worker.process_request(inbound.Message('discord', '77', '<@5>', '/nothing@else', '@99', 'Rob', '', '42/t2')) self.assertEqual([(m, p) for m, p, _ in self.calls], [('DELETE', '/webhooks/42/tok/messages/@original'), ('DELETE', '/webhooks/42/t2/messages/@original')]) @@ -216,7 +216,7 @@ def test_a_slash_command_that_says_nothing_has_its_thinking_deleted(self): def test_a_press_edits_its_message_through_its_interaction(self): """A press edits its message through its own interaction (a message that answered a slash command can be edited only so); a new message about it goes into the channel.""" - chat = transports.DiscordBot('77', interaction='42/tok', pressed=True) + chat = discord_common.DiscordBot('77', interaction='42/tok', pressed=True) chat.edit_chart('333', io.BytesIO(b'png'), 'caption', [[buttons.Button('1Y', 'c|h|y|T1', True)]]) chat.edit_text('333', 'list', []) with contextlib.redirect_stderr(io.StringIO()): @@ -229,7 +229,7 @@ def test_a_press_edits_its_message_through_its_interaction(self): def test_a_command_in_a_message_shows_typing(self): """A command in a message (the gateway's) shows "typing…" in its channel.""" - chat = transports.DiscordBot('77', '700') + chat = discord_common.DiscordBot('77', '700') self.assertTrue(chat.shows_typing) chat.typing() self.assertEqual(self.calls[0][:2], ('POST', '/channels/77/typing')) @@ -238,18 +238,18 @@ def test_rate_limits_are_waited_out_and_refusals_raise(self): """A 429 waits as long as Discord asks, then tries again, three times at most; a refusal raises with Discord's reason; the token in an interaction's path is never logged.""" slept = [] - self.patch(discord.time, 'sleep', slept.append) + self.patch(discord_common.time, 'sleep', slept.append) self.answers += [Resp({'retry_after': 0.25}, 429), Resp({'id': '5'})] - self.assertEqual(discord.call('GET', '/users/@me'), {'id': '5'}) + self.assertEqual(discord_common.call('GET', '/users/@me'), {'id': '5'}) self.assertEqual(slept, [0.25]) self.answers += [Resp({'retry_after': 99}, 429)] * 3 with self.assertRaisesRegex(RuntimeError, '429'): - discord.call('GET', '/users/@me') + discord_common.call('GET', '/users/@me') self.assertEqual(slept, [0.25, 10, 10]) self.answers.append(Resp({'message': 'Missing Access', 'code': 50001}, 403)) err = io.StringIO() with contextlib.redirect_stderr(err): - self.assertIsNone(discord.try_call('PATCH', '/webhooks/42/s3cret-token/messages/@original')) + self.assertIsNone(discord_common.try_call('PATCH', '/webhooks/42/s3cret-token/messages/@original')) self.assertIn('403 Missing Access (50001)', err.getvalue()) self.assertNotIn('s3cret-token', err.getvalue()) @@ -296,7 +296,8 @@ def test_stock_and_profile_are_the_profile_command(self): def test_a_command_with_words_to_suggest_asks_for_suggestions(self): """The arguments of a command with words to suggest ask Discord for suggestions as they are typed; /plan's, free text, do not, and a command without arguments has none.""" - registered = {entry['name']: entry for entry in (discord.command(*each) for each in worker.slash_commands())} + commands = (discord_common.command(*each) for each in worker.slash_commands()) + registered = {entry['name']: entry for entry in commands} for name, completes in (('price', True), ('stock', True), ('jobs', True), ('performance', True), ('pie', True), ('plan', False)): with self.subTest(name=name): @@ -305,25 +306,25 @@ def test_a_command_with_words_to_suggest_asks_for_suggestions(self): def test_registering_replaces_the_commands_only_when_they_differ(self): """Registering puts the commands only when Discord's differ from them, and gives the bot's user id.""" - wanted = [discord.command('price', 'price [period]', '[period]')] + wanted = [discord_common.command('price', 'price [period]', '[period]')] calls = [] def call(method, path, **kwargs): """Answer as Discord would, the commands as registered now.""" calls.append((method, path)) return {'/applications/@me': {'id': '42'}, '/users/@me': {'id': '99'}}.get(path, registered) - self.patch(discord, 'call', call) + self.patch(discord_common, 'call', call) registered = [{**wanted[0], 'id': '1', 'version': '2', 'application_id': '42'}] - self.assertEqual(discord.register([('price', 'price [period]', '[period]')]), '99') + self.assertEqual(discord_common.register([('price', 'price [period]', '[period]')]), '99') self.assertNotIn(('PUT', '/applications/42/commands'), calls) registered = [] with contextlib.redirect_stderr(io.StringIO()): - discord.register([('price', 'price [period]', '[period]')]) + discord_common.register([('price', 'price [period]', '[period]')]) self.assertIn(('PUT', '/applications/42/commands'), calls) def test_help_on_discord_lists_the_slash_commands(self): """.help on Discord lists each command as its slash command, which works in both modes; the messages that also work are given only over the gateway.""" - self.patch(inbound.DISCORD, 'bot_id', '99') + self.patch(discord_common.READER, 'bot_id', '99') sent = [] self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: sent.append(lines) or []) message = inbound.Message('discord', '77', '<@5>', '/help', '@99', 'Rob', '', '42/tok') @@ -461,9 +462,9 @@ def setUp(self): Intents=types.SimpleNamespace(default=lambda: intents)) self.library = library self.patch(sys, 'modules', {**sys.modules, 'discord': library}) - self.patch(pull, 'threading', types.SimpleNamespace(Thread=Now)) # the listener's threads only + self.patch(discord_websocket, 'threading', types.SimpleNamespace(Thread=Now)) # the listener's threads only self.calls = [] - self.patch(discord, 'call', lambda method, path, **kwargs: self.calls.append((method, path, kwargs))) + self.patch(discord_common, 'call', lambda method, path, **kwargs: self.calls.append((method, path, kwargs))) def test_messages_and_interactions_are_taken_and_an_interaction_answered_first(self): """The client asks for the message intents and raw events; MESSAGE_CREATE and INTERACTION_CREATE go to take, the @@ -478,7 +479,7 @@ def take(network, update): return inbound.OK FakeClient.raw = [{'op': 11}, {'op': 0, 't': 'READY', 'd': {}}, {'op': 0, 't': 'MESSAGE_CREATE', 'd': MESSAGE}, {'op': 0, 't': 'INTERACTION_CREATE', 'd': COMMAND}] - listener = pull.DiscordGateway() + listener = discord_websocket.DiscordGateway() listener.failures = 3 err = io.StringIO() with contextlib.redirect_stderr(err): @@ -495,7 +496,7 @@ def test_arguments_being_typed_over_the_gateway_get_their_suggestions_through_th """Over the gateway, the suggestions for what is being typed go to the interaction's callback.""" FakeClient.raw = [{'op': 0, 't': 'INTERACTION_CREATE', 'd': TYPING}] with contextlib.redirect_stderr(io.StringIO()): - pull.DiscordGateway().poll(bot.take) + discord_websocket.DiscordGateway().poll(bot.take) self.assertEqual(self.calls, [('POST', '/interactions/902/tok/callback', {'json': {'type': 8, 'data': { 'choices': [{'name': '5 7d', 'value': '5 7d'}]}}})]) @@ -516,7 +517,7 @@ def test_a_press_over_the_gateway_is_the_press_the_endpoint_gets(self): self.patch(bot, '_updates_seen', collections.OrderedDict()) # a restart in the other mode self.env(discordOutgoingWebhook='') FakeClient.raw = [{'op': 0, 't': 'INTERACTION_CREATE', 'd': PRESS}] - pull.DiscordGateway().poll(bot.take) + discord_websocket.DiscordGateway().poll(bot.take) self.join_workers() self.assertEqual(len(presses), 2) self.assertEqual(presses[0], presses[1]) @@ -526,11 +527,11 @@ def test_discord_is_listened_to_with_its_token_and_no_endpoint(self): """The gateway is used with the bot token and no discordOutgoingWebhook; with the URL (http mode), or without the token, it is not.""" self.env(telegramBotToken='') - self.assertEqual([type(listener) for listener in pull.listeners()], [pull.DiscordGateway]) + self.assertEqual([type(listener) for listener in bot.listeners()], [discord_websocket.DiscordGateway]) self.env(discordOutgoingWebhook='https://example.com/discord') - self.assertEqual(pull.listeners(), []) + self.assertEqual(bot.listeners(), []) self.env(discordOutgoingWebhook='', discordBotToken='') - self.assertEqual(pull.listeners(), []) + self.assertEqual(bot.listeners(), []) if __name__ == '__main__': diff --git a/tests/test_edges.py b/tests/test_edges.py index 795b220a..ded1c569 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -61,8 +61,8 @@ def down(*a, **k): self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) typing = [] - self.patch(transports.TelegramBot, 'post_text', lambda dest, *a, **k: typing.append(dest.is_typing())) - self.patch(transports.TelegramBot, 'post_chart', lambda dest, *a, **k: typing.append(dest.is_typing()) or + self.patch(telegram_common.TelegramBot, 'post_text', lambda dest, *a, **k: typing.append(dest.is_typing())) + self.patch(telegram_common.TelegramBot, 'post_chart', lambda dest, *a, **k: typing.append(dest.is_typing()) or {'ok': True}) for cmd in ('.help', '.price', '.trades 7d', '.performance 7d'): with self.subTest(cmd=cmd), contextlib.redirect_stderr(io.StringIO()): @@ -293,14 +293,15 @@ def test_a_failed_send_logs_no_token(self): def refuse(url, **kwargs): """Fail like requests does, the URL and headers in the message.""" raise ConnectionError(f"Max retries exceeded with url: {url} (headers {kwargs.get('headers')})") - self.patch(transports.net, 'post', refuse) - self.patch(transports.net, 'request', lambda method, url, **kwargs: refuse(url, **kwargs)) + self.patch(net, 'post', refuse) + self.patch(net, 'request', lambda method, url, **kwargs: refuse(url, **kwargs)) token = 'MTAwMDAwMDAwMDAwMDAwMDAwMDA.SECRET.' + 'a' * 38 # a Discord bot token's shape self.env(slackBotToken='xoxb-SECRET', telegram_api=self.SECRET_URL.split('sendMessage')[0], discordBotToken=token) - log = self.stderr_of(lambda: transports.TelegramBot('1').post_text('hi')) - log += self.stderr_of(lambda: transports.SlackBot('C1').post_text('hi')) - log += self.stderr_of(lambda: transports.DiscordBot('77', interaction='42/SECRET-interaction').post_text('hi')) + log = self.stderr_of(lambda: telegram_common.TelegramBot('1').post_text('hi')) + log += self.stderr_of(lambda: slack_common.SlackBot('C1').post_text('hi')) + slash = discord_common.DiscordBot('77', interaction='42/SECRET-interaction') + log += self.stderr_of(lambda: slash.post_text('hi')) self.assertIn('failed', log) for secret in ('SECRET-tok_en', 'xoxb-SECRET', 'SECRET-interaction', token): self.assertNotIn(secret, log) @@ -308,11 +309,11 @@ def refuse(url, **kwargs): def test_the_traceback_of_a_reported_error_is_redacted(self): """The traceback logged with a reported error is redacted.""" self.patch(delivery, 'report_error', REAL_REPORT_ERROR) - self.patch(transports.TelegramBot, 'post_text', lambda *a, **k: None) + self.patch(telegram_common.TelegramBot, 'post_text', lambda *a, **k: None) try: raise ConnectionError('failed: ' + self.SECRET_URL) except ConnectionError as e: - log = self.stderr_of(lambda: delivery.report_error(e, transports.TelegramBot('55'), context='x')) + log = self.stderr_of(lambda: delivery.report_error(e, telegram_common.TelegramBot('55'), context='x')) self.assertIn('Traceback', log) self.assertIn('bot', log) self.assertNotIn('SECRET-tok_en', log) diff --git a/tests/test_finbot.py b/tests/test_finbot.py index 2490240b..3adb024c 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -21,8 +21,8 @@ def post(url, **kwargs): posted.append(kwargs['json']) return Response() - with mock.patch.object(transports.net, 'post', post), mock.patch.object(transports.time, 'sleep', lambda s: 0): - results = REAL_TEXT(transports.TelegramBot('5'), ['x' * 60] * 300, {'keyboard': []}) + with mock.patch.object(net, 'post', post), mock.patch.object(transports.time, 'sleep', lambda s: 0): + results = REAL_TEXT(telegram_common.TelegramBot('5'), ['x' * 60] * 300, {'keyboard': []}) self.assertGreater(len(results), 1) self.assertEqual(len(posted), len(results)) self.assertIn('reply_markup', posted[-1]) diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 0bfcb78a..57c53d9e 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -95,7 +95,7 @@ def test_a_failure_before_authentication_is_only_logged(self): def broken(*a, **k): """Fail while routing.""" raise RuntimeError('boom') - self.patch(inbound, 'urlparse', broken) # fails while routing, before the secret is even looked at + self.patch(bot, 'route', broken) # fails while routing, before the secret is even looked at env = wsgi_environ('/telegram', b'{}', **self.SECRET) status = [] with contextlib.redirect_stderr(io.StringIO()) as err: @@ -147,8 +147,8 @@ def test_an_update_is_taken_however_it_arrived(self): 'event': {'type': 'message', 'channel': 'C1', 'user': 'U1', 'text': '.trades', 'ts': '1.2'}} with contextlib.redirect_stderr(io.StringIO()): for _ in range(2): - self.assertEqual(bot.take(inbound.TELEGRAM, {'update_id': 70, 'message': message}), inbound.OK) - self.assertEqual(bot.take(inbound.SLACK, event), inbound.OK) + self.assertEqual(bot.take(telegram_common.READER, {'update_id': 70, 'message': message}), inbound.OK) + self.assertEqual(bot.take(slack_common.READER, event), inbound.OK) self.join_workers() self.assertEqual([(m.service, m.chat_id, m.text) for m in got], [('telegram', '5', '.price'), ('slack', 'C1', '.trades')]) @@ -172,15 +172,16 @@ def test_a_press_is_acknowledged_once_before_the_worker_gets_it(self): worker has it; a resent copy is neither acknowledged nor run again. Slack's needs no call: the answer to the update is the acknowledgement.""" order = [] - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: order.append(('acknowledged', cid, text))) + self.patch(telegram_common, 'answerCallbackQuery', + lambda cid, text=None: order.append(('acknowledged', cid, text))) self.patch(worker, 'process_callback', lambda press: order.append(('worker', press.service, press.data))) update = {'update_id': 71, 'callback_query': {'id': 'q1', 'from': {'username': 'u'}, 'data': 'c|h|y|T1', 'message': {'message_id': 9, 'chat': {'id': 55}}}} with contextlib.redirect_stderr(io.StringIO()): for _ in range(2): - self.assertEqual(bot.take(inbound.TELEGRAM, update), inbound.OK) + self.assertEqual(bot.take(telegram_common.READER, update), inbound.OK) self.join_workers() - self.assertEqual(bot.take(inbound.SLACK, PRESS), inbound.OK) + self.assertEqual(bot.take(slack_common.READER, PRESS), inbound.OK) self.join_workers() self.assertEqual(order, [('acknowledged', 'q1', 'Loading 1Y…'), ('worker', 'telegram', 'c|h|y|T1'), ('worker', 'slack', 'c|h|w|T1')]) @@ -273,9 +274,9 @@ def test_a_retried_event_runs_once(self): def test_slack_does_not_change_the_telegram_bot_name(self): """A Slack event leaves the Telegram bot's name alone.""" - self.patch(inbound.TELEGRAM, 'bot_name', '@finbot_tg') + self.patch(telegram_common.READER, 'bot_name', '@finbot_tg') self.event(user='U1', text='.price') - self.assertEqual(inbound.TELEGRAM.bot_name, '@finbot_tg') + self.assertEqual(telegram_common.READER.bot_name, '@finbot_tg') class SlackSigning(FinbotCase): diff --git a/tests/test_pull.py b/tests/test_pull.py index c1664e7f..4ba04294 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -3,7 +3,7 @@ import contextlib, io, json, sys, types, unittest from urllib.parse import urlencode from tests.common import * -from lib import pull +from lib import pull, slack_websocket, telegram_longpolling def callback_query(update_id, data='c|h|y|T1'): @@ -20,7 +20,7 @@ class TelegramPolling(FinbotCase): """getUpdates in a loop: each update handed on in order, the next asked for after it, and failures waited out.""" def setUp(self): - """Telegram in pull mode: a bot token and no webhook URL; telegram.call answers from self.answers.""" + """Telegram in pull mode: a bot token and no webhook URL; telegram_common.call answers from self.answers.""" super().setUp() self.env(telegramBotToken='123:abc', telegramOutgoingWebhook='') self.calls, self.answers = [], [] @@ -31,7 +31,7 @@ def call(method, **kwargs): if isinstance(answer, Exception): raise answer return answer - self.patch(telegram, 'call', call) + self.patch(telegram_common, 'call', call) def test_each_update_is_handed_on_in_order_and_the_next_asked_for_after_it(self): """Each update goes to take in order; the next getUpdates asks for those after the last (which also tells @@ -40,10 +40,10 @@ def test_each_update_is_handed_on_in_order_and_the_next_asked_for_after_it(self) message = {'update_id': 10, 'message': {'message_id': 1, 'chat': {'id': 5}, 'from': {'first_name': 'A'}, 'text': '.price'}} self.answers += [[message, callback_query(11)], []] - poll = pull.TelegramPoll() + poll = telegram_longpolling.TelegramPoll() self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) - self.assertEqual(taken, [(inbound.TELEGRAM, message), (inbound.TELEGRAM, callback_query(11))]) + self.assertEqual(taken, [(telegram_common.READER, message), (telegram_common.READER, callback_query(11))]) allowed = ['message', 'callback_query'] self.assertEqual(self.calls, [ ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed}, 'timeout': (5, 65)}), @@ -55,7 +55,7 @@ def test_failures_are_waited_out_longer_each_time_and_logged_without_the_token(s conflict = RuntimeError('Telegram getUpdates: Conflict: terminated by other getUpdates request') down = ConnectionError("Max retries exceeded with url: /bot123:SECRET-token/getUpdates") self.answers += [conflict, down, conflict, conflict, conflict, conflict, [], down] - poll = pull.TelegramPoll() + poll = telegram_longpolling.TelegramPoll() err = io.StringIO() with contextlib.redirect_stderr(err): waits = [poll.poll(lambda *a: None) for _ in range(8)] @@ -73,7 +73,7 @@ def take(network, update): raise KeyError('chat') taken.append(update['update_id']) self.answers += [[callback_query(20), callback_query(21)], []] - poll = pull.TelegramPoll() + poll = telegram_longpolling.TelegramPoll() poll.poll(take) poll.poll(take) self.assertEqual(taken, [21]) @@ -83,7 +83,7 @@ def take(network, update): def test_run_polls_for_good_waiting_as_each_round_says(self): """run() logs the mode, then polls round after round, sleeping what each round returns.""" rounds, slept = iter([0, 5, 0]), [] - poll = pull.TelegramPoll() + poll = telegram_longpolling.TelegramPoll() self.patch(poll, 'poll', lambda take: next(rounds)) def sleep(seconds): """Note the wait; end the loop after three.""" @@ -100,18 +100,18 @@ def sleep(seconds): def test_a_listener_for_each_network_without_a_webhook(self): """Telegram is polled with its bot token and no telegramOutgoingWebhook; with the URL, or without a token, it is not.""" - self.assertEqual([type(listener) for listener in pull.listeners()], [pull.TelegramPoll]) + self.assertEqual([type(listener) for listener in bot.listeners()], [telegram_longpolling.TelegramPoll]) self.env(telegramOutgoingWebhook='https://example.com/telegram') - self.assertEqual(pull.listeners(), []) + self.assertEqual(bot.listeners(), []) self.env(telegramBotToken='', telegramOutgoingWebhook='') - self.assertEqual(pull.listeners(), []) + self.assertEqual(bot.listeners(), []) def test_a_press_polled_for_is_the_press_the_webhook_gets(self): """The same button press, posted to the webhook (http mode) or polled for (pull mode), reaches the worker as the same Press, and is acknowledged the same way.""" presses, toasts = [], [] self.patch(worker, 'process_callback', presses.append) - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) + self.patch(telegram_common, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) body = json.dumps(callback_query(80)).encode() with contextlib.redirect_stderr(io.StringIO()): self.env(telegramOutgoingWebhook='https://example.com/telegram') @@ -120,7 +120,7 @@ def test_a_press_polled_for_is_the_press_the_webhook_gets(self): update = callback_query(81) update['callback_query']['id'] = 'q80' # the same press, as getUpdates gives it self.answers.append([update]) - pull.TelegramPoll().poll(bot.take) + telegram_longpolling.TelegramPoll().poll(bot.take) self.join_workers() self.assertEqual(len(presses), 2) self.assertEqual(presses[0], presses[1]) @@ -170,12 +170,12 @@ def setUp(self): super().setUp() self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') self.log = [] - self.patch(transports, 'slack_api', lambda method, as_app=False, **data: + self.patch(slack_common, 'call', lambda method, as_app=False, **data: self.log.append((method, as_app)) or {'ok': True, 'url': 'wss://example.com/link'}) def socket(self, *messages): """A SlackSocket whose connections give these messages.""" - listener = pull.SlackSocket() + listener = slack_websocket.SlackSocket() self.patch(listener, 'connect', lambda url: self.log.append(('connect', url)) or FakeSocket(messages, self.log)) return listener @@ -206,14 +206,14 @@ def test_a_silent_socket_is_pinged_then_taken_for_dead(self): def test_failures_wait_longer_each_time_and_hello_starts_again(self): """A socket that cannot be opened, or fails, waits longer each time; one that says hello starts again from 5.""" - self.patch(transports, 'slack_api', mock.Mock(side_effect=RuntimeError('Slack apps.connections.open:' + self.patch(slack_common, 'call', mock.Mock(side_effect=RuntimeError('Slack apps.connections.open:' ' invalid_auth'))) - listener = pull.SlackSocket() + listener = slack_websocket.SlackSocket() with contextlib.redirect_stderr(io.StringIO()) as err: self.assertEqual([listener.poll(lambda *a: None) for _ in range(3)], [5, 10, 20]) self.assertIn('slack: opening a socket failed, trying again in 5 s: Slack apps.connections.open:' ' invalid_auth', err.getvalue()) - self.patch(transports, 'slack_api', lambda *a, **k: {'url': 'wss://example.com/link'}) + self.patch(slack_common, 'call', lambda *a, **k: {'url': 'wss://example.com/link'}) self.patch(listener, 'connect', lambda url: FakeSocket([{'type': 'hello'}], self.log)) with contextlib.redirect_stderr(io.StringIO()): self.assertEqual(listener.poll(lambda *a: None), 5) # the socket failed, after a hello @@ -275,7 +275,7 @@ def recv_data(self, control_frame=False): stub = types.SimpleNamespace(create_connection=lambda url, timeout: WS(), WebSocketTimeoutException=Timeout, ABNF=types.SimpleNamespace(OPCODE_TEXT=1, OPCODE_CLOSE=8)) self.patch(sys, 'modules', {**sys.modules, 'websocket': stub}) - socket = pull.WebSocket('wss://example.com/link') + socket = slack_websocket.WebSocket('wss://example.com/link') self.assertEqual([socket.receive(30) for _ in range(3)], ['{"type": "hello"}', '', None]) with self.assertRaises(ConnectionError): socket.receive(30) @@ -284,12 +284,12 @@ def test_slack_is_listened_to_with_both_tokens_and_no_webhook(self): """The socket is opened with the bot and app-level tokens and no slackOutgoingWebhook; with the URL (http mode), or without either token, it is not.""" self.env(telegramBotToken='') - self.assertEqual([type(listener) for listener in pull.listeners()], [pull.SlackSocket]) + self.assertEqual([type(listener) for listener in bot.listeners()], [slack_websocket.SlackSocket]) for changes in ({'slackOutgoingWebhook': 'https://example.com/slack'}, {'slackAppToken': ''}, {'slackBotToken': ''}): with self.subTest(changes=changes): self.env(**changes) - self.assertEqual(pull.listeners(), []) + self.assertEqual(bot.listeners(), []) self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') diff --git a/tests/test_reports.py b/tests/test_reports.py index 0e1d6136..1dc84e1e 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -367,7 +367,7 @@ def test_only_the_scheduled_earnings_report_is_pinned(self): """The scheduled earnings report is pinned on Telegram; a reply to .earnings is not, or the chat would collect pinned copies, and neither is the ex-dividend report.""" pins = [] - self.patch(transports.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) + self.patch(telegram_common.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) self.command('.earnings') self.assertEqual(pins, []) self.env(telegramChatID='1') @@ -380,14 +380,14 @@ def test_a_chats_job_runs_its_report_as_scheduled_not_as_a_command(self): where a command's reply says so, and the earnings report is pinned. price, shorts and cal took any chat they were given for a command.""" pins = [] - self.patch(transports.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) + self.patch(telegram_common.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) job_chat = delivery.chat('telegram:5', scheduled=True) for module, kwargs in ((price, {'mode': 'premarket', 'threshold': 1000}), (shorts, {'threshold': 1000})): with self.subTest(report=module.__name__): self.sent.clear() module.run(job_chat, **kwargs) self.assertEqual(self.sent, []) - module.run(transports.TelegramBot('5'), **kwargs) # a command's reply says that nothing passed + module.run(telegram_common.TelegramBot('5'), **kwargs) # a command's reply says that nothing passed self.assertTrue(self.sent) cal.run(job_chat, event='earnings') self.assertEqual(pins, ['5']) @@ -404,7 +404,7 @@ def test_earnings_skip_an_adr_whose_primary_listing_yahoo_does_not_know(self): def test_trades(self): """.trades run from a chat posts.""" - trades.run(transports.TelegramBot('5')) + trades.run(telegram_common.TelegramBot('5')) self.assertTrue(self.sent) def test_a_trade_is_in_its_market_currency_not_the_brokerage_one(self): @@ -506,7 +506,7 @@ def test_a_52_week_record_kept_before_is_replaced_by_the_history(self): def test_rating(self): """.rating run from a chat posts.""" - rating.run(transports.TelegramBot('5')) + rating.run(telegram_common.TelegramBot('5')) self.assertTrue(self.sent) def test_rating_keeps_the_last_rating_of_a_ticker_missing_from_a_run(self): @@ -523,7 +523,7 @@ def test_rating_keeps_the_last_rating_of_a_ticker_missing_from_a_run(self): def test_an_interactive_rating_records_nothing(self): """A rating report asked for in a chat leaves the record for the scheduled runs alone.""" - rating.run(transports.TelegramBot('5')) + rating.run(telegram_common.TelegramBot('5')) self.assertIsNone(state.load(rating.STATE_FILE)) def test_scheduled_reports_reach_each_home_chat_with_its_own_token(self): @@ -541,7 +541,7 @@ def post(url, **kwargs): data.get('channel') or data.get('channel_id') or data.get('chat_id'))) return mock.Mock(status_code=200, text='OK', json=lambda: {'ok': True, 'upload_url': 'https://files.x/u', 'file_id': 'F1', 'ts': '1.5', 'result': {'message_id': 1, 'chat': {'id': 1}}}) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) for report in (price.run, shorts.run, milestone.run): with self.subTest(report=report): posts.clear() @@ -621,9 +621,9 @@ def test_telegram_charts_have_buttons_in_both_modes(self): def press(self, data): """Press a period button: (the edits made, the toasts shown).""" edits, toasts = [], [] - self.patch(transports.TelegramBot, 'edit_chart', lambda dest, msg, img, caption, buttons=None: - edits.append((caption.split('\n')[0], transports.TelegramBot.markup(buttons)))) - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) + self.patch(telegram_common.TelegramBot, 'edit_chart', lambda dest, msg, img, caption, buttons=None: + edits.append((caption.split('\n')[0], telegram_common.TelegramBot.markup(buttons)))) + self.patch(telegram_common, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) self.press_button(data) return edits, toasts diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 402e44eb..a9c1cc81 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -471,8 +471,8 @@ def setUp(self): self.s = self.scheduler() self.patch(jobs, 'current', self.s) self.toasts, self.edits = [], [] - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: self.toasts.append(text)) - self.patch(transports.TelegramBot, 'edit_text', lambda dest, mid, text, buttons=None: + self.patch(telegram_common, 'answerCallbackQuery', lambda cid, text=None: self.toasts.append(text)) + self.patch(telegram_common.TelegramBot, 'edit_text', lambda dest, mid, text, buttons=None: self.edits.append((dest.where, mid, text, buttons))) self.replies = [] self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: @@ -626,13 +626,13 @@ def test_a_job_is_its_chats_whichever_way_its_updates_arrive(self): added in http mode is that chat's in pull mode, listed and fired there, and the other way round.""" updates = test_sending.EveryNetworkAndMode() updates.updates = 0 - self.patch(inbound.DISCORD, 'bot_id', '99') + self.patch(discord_common.READER, 'bot_id', '99') self.env(slackBotToken='xoxb-1', discordBotToken='x') for network, chat in (('telegram', '-1001'), ('slack', 'C1'), ('discord', '77')): for first, then in (('http', 'pull'), ('pull', 'http')): with self.subTest(network=network, added=first): self.env(**test_sending.EveryNetworkAndMode.MODES[(network, first)]) - reader = getattr(inbound, network.upper()) + reader = READERS[network] with contextlib.redirect_stderr(io.StringIO()): answer = bot.take(reader, updates.update(network, first, chat, '.jobs add 0 23 * * * shorts')) if answer.then: diff --git a/tests/test_sending.py b/tests/test_sending.py index 52c15eec..cacaf78c 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -2,6 +2,7 @@ buttons) and Discord.""" import contextlib, io, json, threading, unittest from unittest import mock +import requests from tests.common import * from lib import jobs @@ -16,8 +17,8 @@ def fake(url, **kwargs): if isinstance(outcome, Exception): raise outcome return outcome - self.patch(transports.net, 'post', fake) - return transports.TelegramBot('5').post_text('hi') + self.patch(net, 'post', fake) + return telegram_common.TelegramBot('5').post_text('hi') def test_success_returns_the_message(self): """A sent message returns the message, as Telegram gives it.""" @@ -41,9 +42,9 @@ def logged(self, outcome): def test_a_chart_goes_without_a_parameter_sendphoto_does_not_take(self): """sendPhoto is sent the chat, caption and markup, and no link-preview flag: a photo has no preview.""" sent = [] - self.patch(transports.net, 'post', lambda url, **kwargs: sent.append((url, kwargs)) or + self.patch(net, 'post', lambda url, **kwargs: sent.append((url, kwargs)) or Resp({'ok': True, 'result': {'message_id': 8}})) - self.assertEqual(transports.TelegramBot('5').post_chart(io.BytesIO(b'png'), 'caption'), {'message_id': 8}) + self.assertEqual(telegram_common.TelegramBot('5').post_chart(io.BytesIO(b'png'), 'caption'), {'message_id': 8}) url, kwargs = sent[0] self.assertTrue(url.endswith('/sendPhoto')) self.assertEqual(kwargs['data'], {'chat_id': '5', 'caption': 'caption', 'parse_mode': 'HTML', @@ -61,7 +62,7 @@ def test_error_reports_are_escaped_for_telegram_and_slack(self): """An error line is escaped for Telegram's HTML and Slack (the same), and left as it is for Discord.""" sent = [] self.patch(transports.Transport, 'text', REAL_TEXT) - for kind in (transports.TelegramBot, transports.SlackBot, transports.DiscordBot): + for kind in (telegram_common.TelegramBot, slack_common.SlackBot, discord_common.DiscordBot): self.patch(kind, 'post_text', lambda dest, text, buttons=None: sent.append((dest.name, text))) self.env(telegramChatID='5', slackChannelID='C9', slackBotToken='xoxb-1', discordChannelID='77', discordBotToken='x') @@ -69,7 +70,7 @@ def test_error_reports_are_escaped_for_telegram_and_slack(self): error = RuntimeError("404 for ZZ at https://x/?a=1&crumb=abc; '<' not supported between ") with contextlib.redirect_stderr(io.StringIO()): REAL_REPORT_ERROR(error, context='.compare ') - REAL_REPORT_ERROR(error, transports.TelegramBot('55'), context='.compare') + REAL_REPORT_ERROR(error, telegram_common.TelegramBot('55'), context='.compare') texts = dict(sent[:3]) self.assertIn('crumb=<redacted>', texts['telegram']) # Telegram's HTML parser rejects a bare self.assertNotIn('<', texts['telegram'] + sent[3][1]) @@ -81,7 +82,7 @@ def test_report_error_never_raises(self): """report_error survives a failed send.""" self.patch(transports.Transport, 'text', mock.Mock(side_effect=RuntimeError('send failed'))) with contextlib.redirect_stderr(io.StringIO()): - REAL_REPORT_ERROR(ValueError('boom'), transports.TelegramBot('5')) + REAL_REPORT_ERROR(ValueError('boom'), telegram_common.TelegramBot('5')) REAL_REPORT_ERROR(ValueError('boom')) @@ -104,7 +105,7 @@ def post(url, **kwargs): if isinstance(outcome, Exception): raise outcome return outcome - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) return calls def upload_steps(self): @@ -116,7 +117,7 @@ def test_every_bot_takes_charts_and_buttons(self): """Every network's bot takes charts and buttons, whichever way its updates arrive: their presses reach finbot, and no setting holds them back.""" self.env(slackBotToken='', slackAppToken='', slackOutgoingWebhook='', telegramOutgoingWebhook='') - for kind in (transports.TelegramBot, transports.SlackBot, transports.DiscordBot): + for kind in (telegram_common.TelegramBot, slack_common.SlackBot, discord_common.DiscordBot): with self.subTest(kind=kind.__name__): self.assertTrue(kind('1').takes_charts and kind('1').takes_buttons) @@ -162,7 +163,7 @@ def test_a_slack_upload_that_fails_goes_out_as_text(self): self.patch(render, 'draw', lambda chart: io.BytesIO(b'png')) good = self.upload_steps() - slow = transports.requests.exceptions.ReadTimeout('slow') + slow = requests.exceptions.ReadTimeout('slow') text = [('text', ['*Heading*', 'line 2'])] for outcomes, expect in (([Resp({'ok': False, 'error': 'missing_scope'})], text), ([ConnectionError('down')], text), @@ -174,7 +175,7 @@ def test_a_slack_upload_that_fails_goes_out_as_text(self): self.slack_answers(*outcomes) self.sent.clear() with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): - delivery.deliver(transports.SlackBot('C1'), delivery.Report(['*Heading*', 'line 2'], CHART)) + delivery.deliver(slack_common.SlackBot('C1'), delivery.Report(['*Heading*', 'line 2'], CHART)) self.assertEqual(self.sent, expect) def test_scheduled_reports_carry_their_charts_to_every_home_chat(self): @@ -198,7 +199,7 @@ def test_the_graph_preference_off_sends_no_chart_anywhere(self): self.env(slackBotToken='xoxb-test') posts = [] - self.patch(transports.net, 'post', lambda url, **k: posts.append(url) or Resp({'ok': True, 'result': True})) + self.patch(net, 'post', lambda url, **k: posts.append(url) or Resp({'ok': True, 'result': True})) for service, chat in (('telegram', '55'), ('slack', 'C1')): for cmd in ('.price', '.price top', '.performance 7d', '.shorts', '.buy', '.peg', '.beta', '.compare T1 T2', '.pie Rob'): @@ -224,13 +225,14 @@ def fake(url, **kwargs): for outcome, expect in ((Resp({'ok': False, 'error_code': 400, 'description': "Bad Request: can't parse entities"}, status=400), text), (Resp(None, status=502), text), (ConnectionError('down'), text), - (transports.requests.exceptions.ReadTimeout('slow'), []), # it may be there + (requests.exceptions.ReadTimeout('slow'), []), # it may be there (Resp({'ok': True, 'result': {'message_id': 1}}), [])): with self.subTest(outcome=outcome): self.sent.clear() - self.patch(transports.net, 'post', post(outcome)) + self.patch(net, 'post', post(outcome)) with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): - delivery.deliver(transports.TelegramBot('5'), delivery.Report(['Heading', 'line 2'], CHART, [[]])) + report = delivery.Report(['Heading', 'line 2'], CHART, [[]]) + delivery.deliver(telegram_common.TelegramBot('5'), report) self.assertEqual(self.sent, expect) @@ -244,7 +246,7 @@ def setUp(self): self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) self.calls = [] - self.patch(transports.discord.net, 'request', lambda method, url, **kwargs: self.calls.append((url, kwargs)) + self.patch(net, 'request', lambda method, url, **kwargs: self.calls.append((url, kwargs)) or Resp({'id': '1'})) def test_a_scheduled_report_carries_its_chart(self): @@ -267,7 +269,7 @@ def test_a_long_report_is_split_by_size(self): """A long report is split into messages of at most 2,000 characters.""" line = 'x' * 150 with contextlib.redirect_stderr(io.StringIO()): - transports.DiscordBot('77').text([line] * 30) + discord_common.DiscordBot('77').text([line] * 30) sizes = [len(kwargs['json']['content']) for _, kwargs in self.calls] self.assertGreater(len(sizes), 2) self.assertTrue(all(size <= 2000 for size in sizes), sizes) # 20 lines a chunk used to make 3,019 characters @@ -286,14 +288,14 @@ def setUp(self): self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) self.posts = [] - self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'ok': True})) + self.patch(net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'ok': True})) def test_link_previews_are_turned_off_in_the_body(self): """Link previews are turned off in the body, where Slack reads them.""" self.env(slackBotToken='xoxb-test') with contextlib.redirect_stderr(io.StringIO()): - transports.SlackBot('C1').post_text('hi') + slack_common.SlackBot('C1').post_text('hi') (_, reply), = self.posts self.assertEqual((reply['data']['unfurl_links'], reply['data']['unfurl_media']), ('false', 'false')) self.assertNotIn('unfurl_links', reply['headers']) # they used to go in the headers, where Slack does not look @@ -301,9 +303,9 @@ def test_link_previews_are_turned_off_in_the_body(self): def test_a_reply_slack_refuses_is_logged_as_failed(self): """Slack refuses with HTTP 200 and ok false (the bot is not in the channel, say): a failure, logged as one.""" self.env(slackBotToken='xoxb-test') - self.patch(transports.net, 'post', lambda url, **kwargs: Resp({'ok': False, 'error': 'not_in_channel'})) + self.patch(net, 'post', lambda url, **kwargs: Resp({'ok': False, 'error': 'not_in_channel'})) with contextlib.redirect_stderr(io.StringIO()) as err: - self.assertIsNone(transports.SlackBot('C1').post_text('hi')) + self.assertIsNone(slack_common.SlackBot('C1').post_text('hi')) self.assertIn('Slack chat.postMessage to C1 failed: Slack chat.postMessage: not_in_channel', err.getvalue()) self.assertNotIn('OK', err.getvalue()) @@ -331,7 +333,7 @@ def post(url, **kwargs): if url.endswith('/reactions.add'): added.set() return Resp({'ok': True}) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) help_text = about.prepare_help self.patch(about, 'prepare_help', lambda *a: (added.wait(5), help_text(*a))[1]) # a command slow enough for ⏳ worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.help', BOT, 'U', '1.2')) @@ -339,14 +341,14 @@ def post(url, **kwargs): ['reactions.add', 'chat.postMessage', 'reactions.remove']) reaction = {'channel': 'C1', 'timestamp': '1.2', 'name': 'hourglass_flowing_sand'} self.assertEqual((self.posts[0][1]['data'], self.posts[2][1]['data']), (reaction, reaction)) - self.assertFalse(transports.SlackBot('C1').shows_typing) # no message to put it on + self.assertFalse(slack_common.SlackBot('C1').shows_typing) # no message to put it on def test_an_hourglass_slack_refuses_is_tried_once(self): """⏳ that Slack refuses (a missing scope) is logged and not tried again, and nothing is taken off.""" self.env(slackBotToken='xoxb-test') refuse = Resp({'ok': False, 'error': 'missing_scope'}) - self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or refuse) - chat = transports.SlackBot('C1', '1.2') + self.patch(net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or refuse) + chat = slack_common.SlackBot('C1', '1.2') with contextlib.redirect_stderr(io.StringIO()) as err: chat.typing() chat.typing() @@ -360,7 +362,7 @@ def test_the_slack_api_says_what_is_missing(self): with self.assertRaisesRegex(RuntimeError, 'Slack files.getUploadURLExternal: needs slackBotToken in finbot.env'): - transports.slack_api('files.getUploadURLExternal', filename='chart.png', length=1) # it was a TypeError + slack_common.call('files.getUploadURLExternal', filename='chart.png', length=1) # it was a TypeError self.assertEqual(self.posts, []) @@ -376,7 +378,7 @@ def setUp(self): self.patch(transports.Transport, 'chart', REAL_CHART) self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') self.posts, self.refusals, self.waits = [], {}, [] - self.patch(transports.time, 'sleep', self.waits.append) + self.patch(slack_common.time, 'sleep', self.waits.append) def post(url, **kwargs): """Record the post and answer it: a refusal set for the method, while there is one left, else success.""" self.posts.append((url, kwargs)) @@ -387,7 +389,7 @@ def post(url, **kwargs): return Resp(None, text='OK - 3') return Resp({'ok': True, 'upload_url': self.UPLOAD_URL, 'file_id': 'F1', 'files': [{'id': 'F1'}], 'ts': '1.5'}) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) def calls(self): """The methods posted, in order (the upload as 'upload').""" @@ -430,14 +432,14 @@ def test_a_press_redraws_the_chart_in_place(self): def uploading(self): """Answer each upload with a new file id, F1 first, as Slack does.""" uploads = iter(f'F{n}' for n in range(1, 10)) - record = transports.net.post + record = net.post def post(url, **kwargs): """Answer as Slack does, with a new file id for each upload.""" answer = record(url, **kwargs) if url.endswith('/files.getUploadURLExternal'): return Resp({'ok': True, 'upload_url': self.UPLOAD_URL, 'file_id': next(uploads)}) return answer - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) def test_a_period_shown_again_within_15_minutes_shows_its_upload_again(self): """A press for a period whose chart was uploaded in the last 15 minutes, for this message or another, shows @@ -455,7 +457,7 @@ def test_a_period_shown_again_within_15_minutes_shows_its_upload_again(self): blocks = self.blocks('chat.update') self.assertEqual(blocks[1]['slack_file'], {'id': 'F1'}) self.assertTrue(blocks[0]['text']['text'].startswith('*≥ 5% at close')) - with state.edit(transports.SLACK_CHARTS, {}) as noted: # 16 minutes on + with state.edit(slack_common.SLACK_CHARTS, {}) as noted: # 16 minutes on for upload in noted['uploads'].values(): upload[0] -= 16 * 60 + 1 worker.process_callback(inbound.Press('slack', 't1', 'C1', '1.5', 'c|l|d|5,0')) # 1D again: F3 @@ -477,10 +479,10 @@ def test_an_upload_slack_will_not_show_again_is_uploaded_anew(self): def test_the_charts_noted_before_uploads_were_reused_are_left_alone(self): """The chart messages noted as before ({'channel ts': file}) are forgotten, their files never deleted.""" - state.save(transports.SLACK_CHARTS, {'C1 1.5': 'F0'}) + state.save(slack_common.SLACK_CHARTS, {'C1 1.5': 'F0'}) worker.process_callback(inbound.Press('slack', 't1', 'C1', '1.5', 'c|l|w|5,0')) self.assertNotIn('files.delete', self.calls()) - self.assertEqual(state.load(transports.SLACK_CHARTS)['shown'], {'C1 1.5': 'F1'}) + self.assertEqual(state.load(slack_common.SLACK_CHARTS)['shown'], {'C1 1.5': 'F1'}) def test_a_chart_not_ready_to_show_is_shown_a_moment_later(self): """Slack refusing the blocks of a chart just uploaded (invalid_blocks) is asked again after a wait.""" @@ -510,21 +512,21 @@ def test_in_socket_mode_a_chart_has_its_buttons_too(self): def test_a_long_report_takes_several_sections(self): """A report longer than a section takes holds is split into sections, and an empty one takes none.""" caption = '\n'.join('x' * 99 for _ in range(70)) # 6,999 characters - blocks = transports.chart_blocks(caption, [], 'F1') + blocks = slack_common.chart_blocks(caption, [], 'F1') self.assertEqual([block['type'] for block in blocks], ['section'] * 3 + ['image']) self.assertTrue(all(len(block['text']['text']) <= 3000 for block in blocks[:3])) - self.assertEqual([block['type'] for block in transports.chart_blocks('', [], 'F1')], ['image']) + self.assertEqual([block['type'] for block in slack_common.chart_blocks('', [], 'F1')], ['image']) def test_a_chart_message_that_times_out_is_not_posted_again(self): """A chart with its buttons whose chat.postMessage times out may have arrived: no plain copy follows.""" - record = transports.net.post + record = net.post def post(url, **kwargs): """Time out on chat.postMessage, as a slow Slack does.""" if url.endswith('/chat.postMessage'): self.posts.append((url, kwargs)) - raise transports.requests.exceptions.ReadTimeout('slow') + raise requests.exceptions.ReadTimeout('slow') return record(url, **kwargs) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) with contextlib.redirect_stderr(io.StringIO()): worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.price', BOT, 'U')) self.assertEqual(self.calls(), ['files.getUploadURLExternal', 'upload', 'files.completeUploadExternal', @@ -551,7 +553,7 @@ class EveryNetworkAndMode(FinbotCase): def setUp(self): """A job in each chat for .jobs run to list, and what each command posts noted with its destination.""" super().setUp() - self.patch(inbound.DISCORD, 'bot_id', '99') + self.patch(discord_common.READER, 'bot_id', '99') every = [{'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}] state.save(jobs.SCHEDULE_FILE, {f'{network}:{chat}': every for network, chats in self.CHATS.items() for chat in chats}) @@ -596,7 +598,7 @@ def test_every_network_in_both_modes_in_a_channel_and_a_dm(self): answers = [] with contextlib.redirect_stderr(io.StringIO()): for text in ('.price', '.jobs run'): - answer = bot.take(getattr(inbound, network.upper()), self.update(network, mode, chat, text)) + answer = bot.take(READERS[network], self.update(network, mode, chat, text)) answers.append(answer) if answer.then: answer.then() diff --git a/tests/test_telegram_api.py b/tests/test_telegram_api.py index eb56bf97..a8e5b0e6 100644 --- a/tests/test_telegram_api.py +++ b/tests/test_telegram_api.py @@ -2,10 +2,11 @@ import contextlib, io, os, re, subprocess, unittest from unittest import mock from tests.common import * +from lib import telegram_longpolling, telegram_webhook class TelegramWebhook(FinbotCase): - """bot.py registers the webhook once as it starts; importing lib.telegram, as each job does, does nothing.""" + """bot.py registers the webhook once as it starts (or deletes it, for long polling); importing does nothing.""" def setUp(self): """A Telegram bot with an outgoing webhook configured.""" @@ -15,33 +16,35 @@ def setUp(self): def test_register_sets_the_webhook_again_or_deletes_it_and_names_the_bot(self): - """register() deletes the webhook, keeping the updates not delivered yet; sets it again with - telegramOutgoingWebhook (http mode), or leaves it unset for getUpdates (pull mode); and returns the bot's @name. - The log never shows the URL: its path is a secret.""" + """register() deletes the webhook, keeping the updates not delivered yet, and sets it again with + telegramOutgoingWebhook (telegram_webhook, http mode), or leaves it unset for getUpdates (telegram_longpolling, + pull mode); bot_name() is the bot's @name. The log never shows the URL: its path is a secret.""" calls = [] def post(url, **kwargs): """Record the method and what it was given, and answer like Telegram.""" calls.append((url.rsplit('/', 1)[1], kwargs.get('params', {}).get('url') or kwargs.get('json'))) return Resp({'ok': True, 'result': {'username': 'finbot'} if url.endswith('getMe') else True}) - self.patch(telegram.net, 'post', post) + self.patch(net, 'post', post) kept = ('deleteWebhook', {'drop_pending_updates': False}) - for hook, expected in (('https://example.com/telegram', [kept, ('setWebhook', 'https://example.com/telegram'), - ('getMe', None)]), ('', [kept, ('getMe', None)])): + for hook, mode, expected in (('https://example.com/telegram', telegram_webhook, + [kept, ('setWebhook', 'https://example.com/telegram'), ('getMe', None)]), + ('', telegram_longpolling, [kept, ('getMe', None)])): with self.subTest(telegramOutgoingWebhook=hook): self.env(telegramOutgoingWebhook=hook) calls.clear() err = io.StringIO() with contextlib.redirect_stderr(err): - self.assertEqual(telegram.register(), '@finbot') + mode.register() + self.assertEqual(telegram_common.bot_name(), '@finbot') self.assertEqual(calls, expected) self.assertNotIn('example.com', err.getvalue()) def test_a_refused_call_raises(self): """A call Telegram refuses raises with its description.""" - self.patch(telegram.net, 'post', + self.patch(net, 'post', lambda url, **k: Resp({'ok': False, 'description': 'Bad Request: bad webhook: HTTPS url must be provided'})) with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(RuntimeError) as caught: - telegram.register() + telegram_webhook.register() self.assertIn('HTTPS url must be provided', str(caught.exception)) def test_edits_toasts_and_typing_go_through_call(self): @@ -58,27 +61,27 @@ def post(url, **kwargs): refused = Resp({'ok': False, 'description': 'Bad Request: message to edit not found'}) unchanged = Resp({'ok': False, 'description': 'Bad Request: message is not modified: specified new message content is the same'}) - chat = transports.TelegramBot('5') + chat = telegram_common.TelegramBot('5') for outcome, raises in ((refused, True), (unchanged, False), (ConnectionError('down'), False), (Resp({'ok': True, 'result': True}), False)): with self.subTest(outcome=outcome): - self.patch(telegram.net, 'post', answer(outcome)) + self.patch(net, 'post', answer(outcome)) with contextlib.redirect_stderr(io.StringIO()): if raises: with self.assertRaisesRegex(RuntimeError, 'not found'): chat.edit_chart('9', io.BytesIO(b'png'), 'caption') else: chat.edit_chart('9', io.BytesIO(b'png'), 'caption') - telegram.answerCallbackQuery('1', 'Loading…') + telegram_common.answerCallbackQuery('1', 'Loading…') chat.edit_text('9', 'text') chat.typing() def test_importing_never_calls_telegram(self): - """Importing the modules a job process uses makes no request.""" + """Importing Telegram's modules, and those they work with, makes no request.""" code = ('import requests\n' 'def refuse(*a, **k): raise SystemExit("network call on import")\n' 'requests.Session.request = refuse\n' - 'from lib import telegram, yahoo, worker, jobs\n' + 'from lib import telegram_common, telegram_longpolling, telegram_webhook, yahoo, worker, jobs\n' 'print("quiet")') with tempfile.TemporaryDirectory() as data: out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True, @@ -97,19 +100,19 @@ def call(method, **kwargs): menu[:] = kwargs['json']['commands'] return True return list(menu) - self.patch(telegram, 'call', call) + self.patch(telegram_common, 'call', call) with contextlib.redirect_stderr(io.StringIO()) as log: - telegram.set_commands(worker.slash_commands(aliases=False)) - telegram.set_commands(worker.slash_commands(aliases=False)) + telegram_common.set_commands(worker.slash_commands(aliases=False)) + telegram_common.set_commands(worker.slash_commands(aliases=False)) self.assertEqual(calls, ['getMyCommands', 'setMyCommands', 'getMyCommands']) self.assertIn(f'telegram: {len(menu)} commands in the menu', log.getvalue()) names = [entry['command'] for entry in menu] self.assertIn({'command': 'forwardpe', 'description': 'forwardpe [SYMBOL|top|bottom|negative]'}, menu) self.assertEqual([name in names for name in ('stock', 'jobs', 'profile', 'job')], [True, True, False, False]) self.assertTrue(all(re.fullmatch(r'[a-z0-9_]{1,32}', name) for name in names)) - self.patch(telegram, 'call', mock.Mock(side_effect=RuntimeError('Telegram getMyCommands: Unauthorized'))) + self.patch(telegram_common, 'call', mock.Mock(side_effect=RuntimeError('Telegram getMyCommands: Unauthorized'))) with contextlib.redirect_stderr(io.StringIO()) as log: - telegram.set_commands(worker.slash_commands(aliases=False)) + telegram_common.set_commands(worker.slash_commands(aliases=False)) self.assertIn('Telegram setMyCommands failed: Telegram getMyCommands: Unauthorized', log.getvalue()) def test_the_bot_will_not_start_unregistered_and_never_logs_the_token(self): diff --git a/util/record_outputs.py b/util/record_outputs.py index 047c9b42..9eacf321 100755 --- a/util/record_outputs.py +++ b/util/record_outputs.py @@ -10,7 +10,8 @@ --keep DIR keeps the two recordings; --full shows changed lines whole. The exit status is 0 when the recordings match, 1 when they differ.""" -import argparse, contextlib, difflib, hashlib, inspect, io, json, os, random, subprocess, sys, tempfile, unittest +import argparse, contextlib, difflib, hashlib, importlib, inspect, io, json, os, random, subprocess, sys, tempfile +import unittest from pathlib import Path REPO = Path(__file__).resolve().parent.parent @@ -64,11 +65,19 @@ def handing(func, kind, *fields): return lambda: func(*fields) return lambda: func(getattr(common.inbound, kind)(*fields)) + def reader(service): + """The tree's reader of a network's updates: READER in lib/_common.py, or in a commit from before the + protocol code was split per network, lib/inbound.py's TELEGRAM, SLACK or DISCORD (None before the readers).""" + try: + return importlib.import_module(f'lib.{service}_common').READER + except ImportError: + return getattr(getattr(common, 'inbound', None), service.upper(), None) + def pressing(service, *fields): """A step that presses a button: acknowledged first where the tree's bot.take does that (Network.acknowledge), then handed to the worker as handing() does.""" run = handing(worker.process_callback, 'Press', service, *fields) - network = getattr(getattr(common, 'inbound', None), service.upper(), None) + network = reader(service) if not hasattr(network, 'acknowledge'): return run def press(): From 834ea8b3e5dc44e62bd9a5ba103e3630c471ac03 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 22:37:39 +0000 Subject: [PATCH 18/31] Follow the current API docs: link previews, typing, User-Agent, webhook secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docs hosts are blocked from this environment, so these come from the docs as quoted by web search (mirrors and library docs) and from what is known of them: - Telegram: link_preview_options {"is_disabled": true} replaces disable_web_page_preview in sendMessage and editMessageText, which Bot API 7.0 deprecated (December 2023). - Telegram: "typing…" shows for 5 seconds at most (sendChatAction), so it is sent again every 4 s, not every 7 s, which left a gap of 2 s each time; Discord's lasts 10 s, so its 7 s stay. - Discord: every REST call sends the User-Agent that Discord asks of a bot, DiscordBot (https://github.com/robdevops/finbot, ); one without may be blocked by Cloudflare. bot.py notes the commit as it starts. - Telegram: a telegramOutgoingToken that is not 1 to 256 of A-Z, a-z, 0-9, _ and - gets a startup warning (and one from check_config): Telegram refuses such a secret, so setting the webhook would fail. util/record_outputs.py: 56 of 268 steps differ, each only in Telegram's preview flag. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 1 + lib/config.py | 3 +++ lib/discord_common.py | 5 ++++- lib/telegram_common.py | 6 ++++-- tests/test_config.py | 4 ++++ tests/test_discord.py | 12 +++++++++++- tests/test_sending.py | 21 +++++++++++++++++++++ 7 files changed, 48 insertions(+), 4 deletions(-) diff --git a/bot.py b/bot.py index 89eac3a2..7cbf6ee9 100755 --- a/bot.py +++ b/bot.py @@ -274,6 +274,7 @@ def http_server(): telegram_common.set_commands(worker.slash_commands(aliases=False)) # the menu; /profile and /job work typed threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background if config.env.discordBotToken: + discord_common.version = git_version() # in each request's User-Agent, as Discord asks try: discord_common.READER.bot_id = discord_common.register(worker.slash_commands()) except Exception as e: diff --git a/lib/config.py b/lib/config.py index 39d9e44c..77340ccc 100644 --- a/lib/config.py +++ b/lib/config.py @@ -444,6 +444,9 @@ def startup_warnings(settings=None): if s.telegramOutgoingWebhook and not s.telegramOutgoingToken: warnings.append('telegramOutgoingWebhook is set but telegramOutgoingToken is not: every Telegram update will be' ' refused') + if s.telegramOutgoingToken and not re.fullmatch(r'[A-Za-z0-9_-]{1,256}', s.telegramOutgoingToken): + warnings.append('telegramOutgoingToken is not 1 to 256 letters, digits, _ or -: Telegram takes no other as the' + " webhook's secret, so setting the webhook will fail") if s.telegramOutgoingToken and not s.telegramOutgoingWebhook: warnings.append('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram' ' updates (pull mode), and the token is not used') diff --git a/lib/discord_common.py b/lib/discord_common.py index 8a016416..179ca6e7 100644 --- a/lib/discord_common.py +++ b/lib/discord_common.py @@ -23,13 +23,16 @@ NO_MENTIONS = {'parse': []} # text that looks like @everyone or a mention pings nobody GUILD, BOT_DM = 0, 1 # where a slash command may be used: in servers, and in a DM with the bot OPTION = 'arguments' # the name of a slash command's one option: its arguments as text +USER_AGENT = 'DiscordBot (https://github.com/robdevops/finbot, {})' # the form Discord asks of every request's agent +version = 'unknown' # finbot's commit, for USER_AGENT: bot.py sets it as it starts def call(method, path, **kwargs): """A REST call as the bot: Discord's answer as JSON, or None when it is empty. A 429 waits as long as Discord asks (LONGEST_WAIT at most) and tries again. Raises RuntimeError with Discord's reason when it refuses, or a requests error.""" - headers = {'Authorization': 'Bot ' + config.env.discordBotToken, **kwargs.pop('headers', {})} + headers = {'Authorization': 'Bot ' + config.env.discordBotToken, 'User-Agent': USER_AGENT.format(version), + **kwargs.pop('headers', {})} for attempt in range(1, TRIES + 1): r = net.request(method, API + path, headers=headers, **kwargs) if r.status_code != 429 or attempt == TRIES: diff --git a/lib/telegram_common.py b/lib/telegram_common.py index fddd996d..d0c33e6b 100644 --- a/lib/telegram_common.py +++ b/lib/telegram_common.py @@ -123,6 +123,8 @@ class TelegramBot(transports.Transport): """A Telegram chat via the Bot API: HTML, silent, no link previews; charts, buttons, edits, the typing notice.""" name = 'telegram' dialect = dialects.TELEGRAM + TYPING = (0.5, 4, 18) # "typing…" shows for 5 s at most (sendChatAction), so it is sent again every 4 s + NO_PREVIEWS = {'is_disabled': True} # link_preview_options, which replaced disable_web_page_preview (Bot API 7.0) @staticmethod def markup(buttons): @@ -136,7 +138,7 @@ def markup(buttons): def post_text(self, text, buttons=None): """sendMessage: Telegram's message, or None when it refused or failed (logged with Telegram's reason, such as "can't parse entities").""" - payload = {'chat_id': self.where, 'text': text, 'parse_mode': 'HTML', 'disable_web_page_preview': True, + payload = {'chat_id': self.where, 'text': text, 'parse_mode': 'HTML', 'link_preview_options': self.NO_PREVIEWS, 'disable_notification': True} if buttons: payload['reply_markup'] = self.markup(buttons) @@ -187,7 +189,7 @@ def edit_chart(self, message, image, caption, buttons=None): def edit_text(self, message, text, buttons=None): """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" payload = {'chat_id': self.where, 'message_id': message, 'text': text, 'parse_mode': 'HTML', - 'disable_web_page_preview': True} + 'link_preview_options': self.NO_PREVIEWS} if buttons is not None: payload['reply_markup'] = self.markup(buttons) try_call('editMessageText', json=payload) diff --git a/tests/test_config.py b/tests/test_config.py index 272b95b2..6e82755a 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -427,11 +427,15 @@ def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(sel ' and the app-level token is not used') not_app = ("slackAppToken does not start with xapp-: it must be an app-level token (the app's Basic" ' Information, App-Level Tokens), not the bot token') + bad_secret = ("telegramOutgoingToken is not 1 to 256 letters, digits, _ or -: Telegram takes no other as the" + " webhook's secret, so setting the webhook will fail") cases = (({}, []), ({'slackBotToken': ''}, [no_reply]), ({'slackOutgoingToken': ''}, [slack_refused]), ({'slackChannelID': 'C9'}, []), ({'slackChannelID': 'C9', 'slackBotToken': ''}, [no_reply, no_bot]), ({'slackOutgoingToken': '', 'slackSigningSecret': 's'}, []), ({'telegramOutgoingToken': ''}, [telegram_refused]), ({'telegramOutgoingWebhook': ''}, [unused]), ({'telegramOutgoingWebhook': '', 'telegramOutgoingToken': ''}, []), + ({'telegramOutgoingToken': 'A-z_09' * 40}, []), ({'telegramOutgoingToken': 'x' * 257}, [bad_secret]), + ({'telegramOutgoingToken': 'pass word!'}, [bad_secret]), ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1'}, []), ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1', 'slackBotToken': ''}, [app_no_reply]), ({'slackAppToken': 'xapp-1'}, [app_unused]), diff --git a/tests/test_discord.py b/tests/test_discord.py index 069c44c1..50b94446 100644 --- a/tests/test_discord.py +++ b/tests/test_discord.py @@ -228,11 +228,21 @@ def test_a_press_edits_its_message_through_its_interaction(self): self.assertEqual(self.calls[1][2]['json'], {'content': 'list', 'components': []}) def test_a_command_in_a_message_shows_typing(self): - """A command in a message (the gateway's) shows "typing…" in its channel.""" + """A command in a message (the gateway's) shows "typing…" in its channel, which lasts 10 seconds (POST + /typing): it is sent again sooner, so that it shows the whole time the command works.""" chat = discord_common.DiscordBot('77', '700') self.assertTrue(chat.shows_typing) chat.typing() self.assertEqual(self.calls[0][:2], ('POST', '/channels/77/typing')) + self.assertLess(chat.TYPING[1], 10) + + def test_each_request_names_finbot_as_discord_asks(self): + """Each request carries the bot token, and the User-Agent that Discord asks of a bot: DiscordBot, its URL and + its version (the commit, which bot.py notes as it starts).""" + self.patch(discord_common, 'version', '0123456') + discord_common.call('GET', '/users/@me') + self.assertEqual(self.calls[0][2]['headers'], {'Authorization': 'Bot x', + 'User-Agent': 'DiscordBot (https://github.com/robdevops/finbot, 0123456)'}) def test_rate_limits_are_waited_out_and_refusals_raise(self): """A 429 waits as long as Discord asks, then tries again, three times at most; a refusal raises with Discord's diff --git a/tests/test_sending.py b/tests/test_sending.py index cacaf78c..658b0d0b 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -39,6 +39,27 @@ def logged(self, outcome): self.post(outcome) return log.getvalue() + def test_text_goes_silent_and_without_link_previews(self): + """A message goes silent and without link previews, and so does an edit: link_preview_options, which replaced + disable_web_page_preview in Bot API 7.0.""" + sent = [] + self.patch(net, 'post', lambda url, **kwargs: sent.append((url.rsplit('/', 1)[1], kwargs['json'])) or + Resp({'ok': True, 'result': {'message_id': 8}})) + chat = telegram_common.TelegramBot('5') + chat.post_text('hi') + chat.edit_text('8', 'edited') + off = {'is_disabled': True} + self.assertEqual(sent, [('sendMessage', {'chat_id': '5', 'text': 'hi', 'parse_mode': 'HTML', + 'link_preview_options': off, 'disable_notification': True}), ('editMessageText', {'chat_id': '5', + 'message_id': '8', 'text': 'edited', 'parse_mode': 'HTML', 'link_preview_options': off})]) + + def test_typing_is_sent_again_before_it_lapses(self): + """"typing…" shows for 5 seconds at most (sendChatAction), so it is sent again sooner, and shows the whole time + a command works, for a minute at least.""" + delay, every, times = telegram_common.TelegramBot.TYPING + self.assertLess(every, 5) + self.assertGreaterEqual(delay + every * times, 60) + def test_a_chart_goes_without_a_parameter_sendphoto_does_not_take(self): """sendPhoto is sent the chat, caption and markup, and no link-preview flag: a photo has no preview.""" sent = [] From 750ecf0a0a8a16671af77e9a69c038b4156c04cf Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 23:28:41 +0000 Subject: [PATCH 19/31] Shorter startup log lines: the keys set, and the preferences file config: startup environment (10 keys set) config: /var/lib/finbot/preferences.conf (14 keys set; SIGHUP to reload) in place of "startup environment provides 10 of 21 startup keys (values hidden)" and "preferences from ... (14 set; reload with SIGHUP)"; one key is "1 key". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/config.py | 8 +++++--- tests/test_config.py | 15 +++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/config.py b/lib/config.py index 77340ccc..4815fb6b 100644 --- a/lib/config.py +++ b/lib/config.py @@ -404,10 +404,12 @@ def check_startup_environment(): def startup_report(): """Lines for the log at start: where the config came from (counts only; check_config names the problems).""" - provided = [key for key in STARTUP_KEYS if key in os.environ] - lines = [f'startup environment provides {len(provided)} of {len(STARTUP_KEYS)} startup keys (values hidden)'] + def keys(count): + """'1 key', '2 keys'.""" + return f"{count} key{'' if count == 1 else 's'}" + lines = [f'startup environment ({keys(sum(key in os.environ for key in STARTUP_KEYS))} set)'] if os.path.isfile(preferences_path): - lines.append(f'preferences from {preferences_path} ({len(prefs.source)} set; reload with SIGHUP)') + lines.append(f'{preferences_path} ({keys(len(prefs.source))} set; SIGHUP to reload)') else: lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, using the defaults; reload will have nothing to' ' re-read') diff --git a/tests/test_config.py b/tests/test_config.py index 6e82755a..31ec09cc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -467,15 +467,22 @@ def test_discord_settings_that_leave_part_of_it_unable_to_work_are_warned_about( self.assertEqual(config.startup_warnings(), warnings) def test_startup_report_is_a_short_count_never_names_or_values(self): - """The startup report counts the startup keys set, and never names them or shows their values.""" + """The startup report counts the startup keys set and the preferences, and never names them or shows their + values.""" with mock.patch.dict(os.environ, {'slackOutgoingWebhook': 'https://secret.example/abc', 'port': '5000'}, clear=True): lines = config.startup_report() - keys = len(config.STARTUP_KEYS) - self.assertEqual(lines[0], f'startup environment provides 2 of {keys} startup keys (values hidden)') - self.assertLessEqual(len('config: ' + lines[0]), 90) # the path in the second line depends on where DATA_DIR is + self.assertEqual(lines[0], 'startup environment (2 keys set)') self.assertNotIn('secret.example', ''.join(lines)) self.assertNotIn('slackOutgoingWebhook', ''.join(lines)) + with mock.patch.dict(os.environ, {'port': '5000'}, clear=True): + self.assertEqual(config.startup_report()[0], 'startup environment (1 key set)') + conf = Path(common.temp_dir(self), 'preferences.conf') + conf.write_text('debug = true\n') + for source, count in (({'debug': 'true'}, '1 key'), ({'debug': 'true', 'graph': 'false'}, '2 keys')): + with self.subTest(source=source), mock.patch.object(config, 'preferences_path', str(conf)), \ + mock.patch.object(config, 'prefs', dataclasses.replace(config.prefs, source=source)): + self.assertEqual(config.startup_report()[1], f'{conf} ({count} set; SIGHUP to reload)') class SigHup(unittest.TestCase): From 478bea97f07c2e40ab2f62cdae1b70ecba9fee60 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 23:31:25 +0000 Subject: [PATCH 20/31] Shorter startup log line without preferences.conf: "reload unavailable" config: no preferences.conf in /var/lib/finbot, reload unavailable in place of "..., using the defaults; reload will have nothing to re-read". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/config.py | 3 +-- tests/test_config.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/config.py b/lib/config.py index 4815fb6b..630008e9 100644 --- a/lib/config.py +++ b/lib/config.py @@ -411,8 +411,7 @@ def keys(count): if os.path.isfile(preferences_path): lines.append(f'{preferences_path} ({keys(len(prefs.source))} set; SIGHUP to reload)') else: - lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, using the defaults; reload will have nothing to' - ' re-read') + lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, reload unavailable') return lines + retired_notes(os.environ, 'finbot.env') + retired_notes(prefs.source) diff --git a/tests/test_config.py b/tests/test_config.py index 31ec09cc..b848175e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -483,6 +483,9 @@ def test_startup_report_is_a_short_count_never_names_or_values(self): with self.subTest(source=source), mock.patch.object(config, 'preferences_path', str(conf)), \ mock.patch.object(config, 'prefs', dataclasses.replace(config.prefs, source=source)): self.assertEqual(config.startup_report()[1], f'{conf} ({count} set; SIGHUP to reload)') + with mock.patch.object(config, 'preferences_path', str(conf.with_name('missing.conf'))): + self.assertEqual(config.startup_report()[1], f'no preferences.conf in {config.env.data_dir}, reload' + ' unavailable') class SigHup(unittest.TestCase): From cc5b7ffd1ac832bccf421a485d45aed4e9ffa716 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 23:45:20 +0000 Subject: [PATCH 21/31] The preferences template sets nothing: every key commented out, never read defaults/preferences.conf keeps its layout, with each key commented out at its built-in default (#past_days=28), and its header says so: remove the # to change one. finbot never reads the template (it never did): without /preferences.conf it runs on the defaults hard-coded in Prefs, and util/sync_defaults.py --apply copies the template there byte for byte, layout and all (a test checks). A key set in the template itself does nothing, so the log says so at start (and check_config, and a reload): "defaults/preferences.conf sets debug, but finbot never reads it: edit /preferences.conf instead", or, when that file does not exist, "run util/sync_defaults.py --apply, then edit ...". doc/configuration.md and CLAUDE.md describe the template. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 3 +- bot.py | 7 +++-- defaults/preferences.conf | 63 ++++++++++++++++++++------------------- doc/configuration.md | 5 ++-- lib/config.py | 31 +++++++++++++++++-- tests/test_config.py | 51 +++++++++++++++++++++++++++---- tests/test_datadir.py | 4 ++- 7 files changed, 120 insertions(+), 44 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c3243621..83b7e889 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm `requirements.txt` is for pip/uv users and CI. The libraries in `requirements-optional.txt` are each imported only inside the code of the mode that needs it, never at module load (a test starts finbot with them all blocked). - A new preference goes in three places: a `Prefs` field with its default (`lib/config.py`; its type says how it is - read), `defaults/preferences.conf`, and the preferences table of `doc/configuration.md`. The tests catch a key + read), `defaults/preferences.conf` (commented out at its default, `#key=value`: the template sets nothing, and finbot + never reads it), and the preferences table of `doc/configuration.md`. The tests catch a key missing from the template or a default that differs, but not the docs (no test may read README.md or `doc/*.md`: CI skips the tests for a change to the docs alone). - If Yahoo and Sharesight are unreachable (the cloud environment's network policy can block them), check behaviour diff --git a/bot.py b/bot.py index 7cbf6ee9..65a1e7b0 100755 --- a/bot.py +++ b/bot.py @@ -207,8 +207,9 @@ def run(): return lambda *args: threading.Thread(target=run, daemon=True).start() def reload_all(*args): - """SIGHUP: re-read preferences.conf; a bad one is reported and leaves the running config untouched. (Each chat's - jobs need no reload: a change made with .jobs counts at once.)""" + """SIGHUP: re-read preferences.conf; a bad one is reported and leaves the running config untouched. A key set in + defaults/preferences.conf, which is never read, is pointed at the file to set it in. (Each chat's jobs need no + reload: a change made with .jobs counts at once.)""" try: changes = config.reload() for change in changes: @@ -217,6 +218,8 @@ def reload_all(*args): file=sys.stderr) except config.ConfigError as e: config.print_errors(e) + if note := config.template_note(config.preferences_path): + print('ERROR config:', note, file=sys.stderr) def flush_cache(*args): """SIGUSR1 (`systemctl kill -s USR1 finbot`): delete the cache and keep the state (watchlist, seen trades, ratings, diff --git a/defaults/preferences.conf b/defaults/preferences.conf index 6d382c7a..19dac7b4 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -1,53 +1,56 @@ -# Preferences for finbot: how the bot behaves. The bot reads this file itself, at start and on every reload. -# Apply an edit without a restart: sudo systemctl reload finbot (docker: docker kill -s HUP ) +# Preferences for finbot: how the bot behaves. This file is only the template, which finbot never reads: the bot +# reads /preferences.conf, at start and on every reload, and without one runs on its built-in defaults. +# util/sync_defaults.py --apply copies this file there. +# Each key is commented out at its built-in default: to change one, remove its # and edit the value. +# Apply an edit without a restart: sudo systemctl reload finbot (docker: docker compose kill -s HUP finbot) # Check an edit first: util/check_config.py /path/to/finbot.env # Flat KEY=value lines, one per line. This is NOT an INI file: no [sections]. No spaces around "=", no quotes, no # trailing comments; put comments on their own line. A key that is not listed here is an error and finbot refuses to # start. -# Each value shown is the built-in default. The startup keys (credentials, ip, port, telegramChatID, scheduler, ...) -# are NOT set here: they live in finbot.env, the startup file, which the launcher loads and the bot never reads. -# Changing them needs a restart. +# The startup keys (credentials, ip, port, telegramChatID, ...) are NOT set here: they live in finbot.env, the startup +# file, which the launcher loads and the bot never reads. Changing them needs a restart. # Reports: how far back and forward to look, and when to alert -past_days=28 -future_days=30 -price_percent=5 -shorts_percent=15 +#past_days=28 +#future_days=30 +#price_percent=5 +#shorts_percent=15 # .buy, .sell and .forward pe list only stocks that at least this many analysts cover: with fewer, a consensus says # little -min_analysts=7 -# price alerts: volatile and leveraged stocks need a bigger move; the multiplier is how much bigger -demote_volatile=True -demote_leveraged=True -volatility_multiplier=2 +#min_analysts=7 +# price alerts: volatile and leveraged stocks need a bigger move; volatility_multiplier is how much bigger for +# volatile ones +#demote_volatile=True +#demote_leveraged=True +#volatility_multiplier=2 # Which portfolios to report on, space separated Sharesight portfolio IDs (empty: all that are not excluded) -exclude_portfolios= -include_portfolios= +#exclude_portfolios= +#include_portfolios= # Data sources: set trades_use_yahoo to False if Yahoo breaks -trades_use_yahoo=True +#trades_use_yahoo=True # Display -graph=True -hyperlink=True -hyperlinkFooter=True -hyperlinkProvider=google +#graph=True +#hyperlink=True +#hyperlinkFooter=True +#hyperlinkProvider=google # Regional: where you are. # exchange_suffix: a ticker Yahoo does not know is tried again on your exchange, by Yahoo's suffix for it (AX: BHP is # tried as BHP.AX; NZ, L, TO and so on for others). Empty: a ticker is taken as written. # country_code and timezone: the reminder report's country and time zone; .jobs also shows each next run in timezone. -exchange_suffix=AX -country_code=AU -timezone=Australia/Melbourne +#exchange_suffix=AX +#country_code=AU +#timezone=Australia/Melbourne # Caching and network, in seconds -cache=True -cache_seconds=43200 -performance_cache_seconds=2400 -http_timeout=18 +#cache=True +#cache_seconds=43200 +#performance_cache_seconds=2400 +#http_timeout=18 # Diagnostics -debug=False -print_headers=False +#debug=False +#print_headers=False diff --git a/doc/configuration.md b/doc/configuration.md index fbf2fa8b..8a893dde 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -43,12 +43,13 @@ of the chat network keys it needs. ## `preferences.conf` items -Copies preferences.conf and metadata files into the data directory (never overwrites) +The preferences, read by the bot itself from `/preferences.conf`; without that file, finbot runs on the built-in defaults below. `defaults/preferences.conf` is only its template: every key in it is commented out at its default, and finbot never reads it. To change a preference, copy the template into the data directory (this never overwrites a file there): ``` util/sync_defaults.py [--dest DIR] --apply ``` +then remove the `#` before the key and edit its value. A key set in `defaults/preferences.conf` itself does nothing: the log at start (and check_config) says so, and where to set it instead. -The preferences, read by the bot itself. A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. +A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. | Key | Default | Description | |---|---|---| diff --git a/lib/config.py b/lib/config.py index 630008e9..10141548 100644 --- a/lib/config.py +++ b/lib/config.py @@ -23,6 +23,7 @@ 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook', 'discordBotToken', 'discordOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' +TEMPLATE = REPO_DIR / 'defaults' / PREFERENCES_FILE # every key commented out at its default; finbot never reads it RETIRED = { # keys of either file that do nothing any more: accepted, so that they never stop a start, with a note at # start saying what to do instead 'alliterate': 'delete the line', @@ -110,6 +111,29 @@ def read_preferences(path): return values +def template_keys(): + """The keys that defaults/preferences.conf sets: its lines that are not commented out, none as shipped. finbot never + reads the template, so a key set there does nothing; an unreadable template sets nothing either.""" + try: + text = read_text(TEMPLATE) or '' + except ConfigError: + return [] + return [line.partition('=')[0].strip() for line in text.splitlines() + if line.strip() and not line.lstrip().startswith('#')] + + +def template_note(preferences_path): + """What to tell whoever set keys in defaults/preferences.conf, which finbot never reads: to set them in + preferences_path instead, made first with util/sync_defaults.py when there is none. None when the template sets + nothing, as shipped.""" + keys = template_keys() + if not keys: + return None + fix = (f'edit {preferences_path} instead' if os.path.isfile(preferences_path) else + f'run util/sync_defaults.py --apply, then edit {preferences_path}') + return f"defaults/preferences.conf sets {', '.join(keys)}, but finbot never reads it: {fix}" + + def read_startup_file(path): """{key: value} of a finbot.env, for checking it offline (the bot never reads this file). Raises ConfigError.""" text = read_text(path) @@ -421,8 +445,9 @@ def retired_notes(values, file=PREFERENCES_FILE): def startup_warnings(settings=None): - """Settings in finbot.env (an Env; the running one by default) that do not stop the start but keep part of the bot - from working, or do nothing, a line each.""" + """Settings in finbot.env (an Env; the running one by default), and keys set in the template + defaults/preferences.conf, that do not stop the start but keep part of the bot from working, or do nothing, a line + each.""" s = settings or env warnings = [] if s.slackBotToken and not (s.slackAppToken or s.slackOutgoingWebhook): @@ -462,4 +487,6 @@ def startup_warnings(settings=None): if s.discordPublicKey and not s.discordOutgoingWebhook: warnings.append('discordPublicKey is set but discordOutgoingWebhook is not: finbot keeps to the Discord gateway' ' (pull mode), and the key is not used') + if note := template_note(s.preferences_path): + warnings.append(note) return warnings diff --git a/tests/test_config.py b/tests/test_config.py index b848175e..b706a4cc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,5 @@ """The two config files: finbot.env (the startup file, which the launcher loads) and preferences.conf (reloadable).""" -import ast, contextlib, dataclasses, io, os, re, subprocess, sys, types, unittest +import ast, contextlib, dataclasses, io, os, re, shutil, subprocess, sys, types, unittest from pathlib import Path from unittest import mock from tests import common # first: it sets the environment that lib.config reads as it loads @@ -42,12 +42,15 @@ def test_unset_preferences_take_their_defaults(self): self.assertEqual(fields(values), fields(config.Prefs())) def test_templates_have_every_key_and_the_real_defaults(self): - """defaults/preferences.conf and finbot.env list every key, change no default, and are valid as shipped.""" - self.assertEqual(sorted(keys_in(REPO / 'defaults' / 'preferences.conf')), sorted(config.PREFERENCE_KEYS)) + """defaults/preferences.conf lists every key, commented out at its built-in default, so that it sets none; + finbot.env lists every key; both are valid as shipped.""" + template = REPO / 'defaults' / 'preferences.conf' + self.assertEqual(sorted(keys_in(template)), sorted(config.PREFERENCE_KEYS)) self.assertEqual(sorted(keys_in(REPO / 'defaults' / 'finbot.env')), sorted(config.STARTUP_KEYS)) - prefs = config.read_preferences(REPO / 'defaults' / 'preferences.conf') - shipped = config.preference_values(config._getenv_for(prefs)) - self.assertEqual(fields(shipped), fields(config.Prefs())) # the template changes nothing + self.assertEqual((config.read_preferences(template), config.template_keys()), ({}, [])) # it sets nothing + shown = dict(re.findall(r'^#([A-Za-z_][A-Za-z0-9_]*)=(.*)$', template.read_text(), re.M)) # #key=value lines + self.assertEqual(sorted(shown), sorted(config.PREFERENCE_KEYS)) + self.assertEqual(fields(config.preference_values(config._getenv_for(shown))), fields(config.Prefs())) startup = config.read_startup_file(REPO / 'defaults' / 'finbot.env') config.startup_values(lambda name, default=None: startup.get(name, default)) # and is valid as it stands @@ -353,6 +356,27 @@ def test_a_broken_preferences_file_refuses_to_start(self): self.assertIn('ERROR config:', out.stderr) self.assertIn(expect, out.stderr) + def test_the_template_is_never_read_and_a_key_set_there_is_pointed_elsewhere(self): + """defaults/preferences.conf is only a template: a key set there changes nothing, and the log at start says to + set it in the data dir's preferences.conf, or to make that file first with util/sync_defaults.py.""" + tree = Path(common.temp_dir(self)) # config.py and a template of its own, which sets a key + (tree / 'lib').mkdir() + shutil.copy(REPO / 'lib' / 'config.py', tree / 'lib') + (tree / 'defaults').mkdir() + (tree / 'defaults' / 'preferences.conf').write_text('# a comment\n#past_days=28\nfuture_days=45\n') + code = 'from lib import config as c; print(c.prefs.future_days); print(*c.startup_warnings(), sep="\\n")' + for made in (False, True): + data = common.temp_dir(self) + if made: + Path(data, 'preferences.conf').write_text('#future_days=30\n') + with self.subTest(preferences_conf=made): + out = subprocess.run([sys.executable, '-c', code], cwd=tree, + env={**common.bare_environment(), 'DATA_DIR': data}, capture_output=True, text=True) + conf = f'{data}/preferences.conf' + fix = f'edit {conf} instead' if made else f'run util/sync_defaults.py --apply, then edit {conf}' + self.assertEqual(out.stdout.split('\n')[:2], ['30', + f'defaults/preferences.conf sets future_days, but finbot never reads it: {fix}'], out.stderr) + def test_an_unreadable_preferences_file_refuses_to_start(self): """A preferences.conf that cannot be read stops the start.""" data = common.temp_dir(self) @@ -513,6 +537,21 @@ def test_reload_all_reloads_preferences_and_keeps_them_on_an_error(self): self.assertIn('ERROR config:', err.getvalue()) self.assertEqual(config.prefs.future_days, 45) + def test_a_reload_points_a_key_set_in_the_template_at_the_file_to_set_it_in(self): + """A key set in defaults/preferences.conf, which finbot never reads, is reported on a reload too, with the file + to set it in.""" + file = Path(common.temp_dir(self), 'preferences.conf') + file.write_text('') + self.addCleanup(self.restore, file) + template = Path(common.temp_dir(self), 'template.conf') + template.write_text('#past_days=28\ndebug=True\n') + with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(config, 'TEMPLATE', template): + with contextlib.redirect_stderr(io.StringIO()) as err: + bot.reload_all() + self.assertIn(f'ERROR config: defaults/preferences.conf sets debug, but finbot never reads it: edit {file}' + ' instead', err.getvalue()) + self.assertFalse(config.prefs.debug) + class Tools(unittest.TestCase): def test_split_env(self): diff --git a/tests/test_datadir.py b/tests/test_datadir.py index e0e30c5b..f740a37c 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -91,7 +91,9 @@ def test_sync_defaults(self): self.assertFalse(Path(config.env.data_dir, 'finbot_adr.json').exists()) # dry run sync_defaults.sync(apply=True) self.assertTrue(Path(config.env.data_dir, 'finbot_adr.json').exists()) - self.assertTrue(Path(config.env.data_dir, 'preferences.conf').exists()) + copied = Path(config.env.data_dir, 'preferences.conf') # the template as it is: every key commented out + self.assertEqual(copied.read_bytes(), (config.REPO_DIR / 'defaults' / 'preferences.conf').read_bytes()) + self.assertEqual(config.read_preferences(copied), {}) # the startup file is never copied into the data dir self.assertFalse(Path(config.env.data_dir, 'finbot.env').exists()) mine = Path(config.env.data_dir, 'finbot_adr.json') From 9015d7a4c8669a6ca37215fca22b7f7c8406b740 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:05:41 +0000 Subject: [PATCH 22/31] Fix .earnings on ETFs, and keep log lines whole when threads log at once - yahoo: an ETF has no analysts' recommendation, so Yahoo sends no recommendationKey, and the converter that turns strong_buy into "strong buy" raised AttributeError on None ('NoneType' object has no attribute 'replace'). That failed .earnings for any watchlist with an ETF in it, and logged that error for each ETF. A missing or null field is now left out before any converter runs, and AttributeError counts as garbage like the others. - bot.py: the bot's stderr writes a whole line at a time (WholeLines). Under PYTHONUNBUFFERED (finbot.service, the image) each piece of a print() is its own write, so threads logging at once mixed their lines ("TTL:cache hit: 34254 s/var/...", and lines with no file name); each thread's pieces now wait for the end of its line. - cache: a hit reads "cache hit: TTL: 34254s". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 33 +++++++++++++++++++++++++++++++++ lib/cache.py | 2 +- lib/yahoo.py | 7 +++++-- tests/test_edges.py | 44 +++++++++++++++++++++++++++++++++++++++++++- tests/test_yahoo.py | 13 +++++++++++++ 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/bot.py b/bot.py index 65a1e7b0..5e31c770 100755 --- a/bot.py +++ b/bot.py @@ -126,6 +126,38 @@ def print_headers(): environ[AUTHENTICATED] = True return respond(take(source.network, update)) +class WholeLines: + """stderr, a whole line at a time. Threads log at once (a report's fetches, say), and print() writes its arguments, + the spaces between them and the newline one by one, so their lines would mix ("TTL:cache hit: 34254 s/var/..."): + each thread's pieces wait in a buffer of its own until its line ends, and the line goes out in one write.""" + def __init__(self, stream): + """stream: where the lines go.""" + self.stream = stream + self.held = threading.local() # each thread's line so far + self.lock = threading.RLock() # re-entrant: a signal handler that logs may interrupt a write + + def write(self, text): + """Hold text until its line ends, then write the whole line; returns its length, as a stream does.""" + lines, newline, rest = (getattr(self.held, 'text', '') + text).rpartition('\n') + self.held.text = rest + if newline: + with self.lock: + self.stream.write(lines + newline) + self.stream.flush() + return len(text) + + def flush(self): + """Write what this thread holds, though its line has not ended.""" + text, self.held.text = getattr(self.held, 'text', ''), '' + with self.lock: + if text: + self.stream.write(text) + self.stream.flush() + + def __getattr__(self, name): + """Anything else as the stream has it: fileno, encoding, isatty.""" + return getattr(self.stream, name) + class Sent(list): """A response body that starts some work once the server has sent it: WSGI calls close() then.""" def __init__(self, body, then): @@ -242,6 +274,7 @@ def http_server(): return waitress.create_server(main, host=config.env.ip, port=config.env.port) if __name__ == '__main__': + sys.stderr = WholeLines(sys.stderr) # threads log at once: each line goes out whole print(f'Starting finbot {git_version()}', file=sys.stderr) try: config.check_startup_environment() diff --git a/lib/cache.py b/lib/cache.py index 7c8bb7ac..8d4f33fa 100644 --- a/lib/cache.py +++ b/lib/cache.py @@ -35,7 +35,7 @@ def _fresh(name, seconds): if written >= seconds: print("cache expired:", where, file=sys.stderr) if config.prefs.debug else None return None - print("cache hit:", where, "TTL:", f"{seconds - written:.0f} s", file=sys.stderr) if config.prefs.debug else None + print(f"cache hit: {where} TTL: {seconds - written:.0f}s", file=sys.stderr) if config.prefs.debug else None return where diff --git a/lib/yahoo.py b/lib/yahoo.py index 6a1b17b8..096256dc 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -111,10 +111,13 @@ def _positive(value): return value def _convert(convert, value): - """convert(value), or None for garbage where a number belongs (left out, like a missing field).""" + """convert(value); None for a field Yahoo leaves out or sends as null (an ETF has no analysts' recommendation), and + for garbage, such as text where a number belongs (left out, like a missing field).""" + if value is None: + return None try: return convert(value) if convert else value - except (TypeError, ValueError, OverflowError): + except (TypeError, ValueError, OverflowError, AttributeError): return None # (key, Yahoo's name, conversion) for the fields of a v7 quote; one Yahoo leaves out, sends as null or garbles is left diff --git a/tests/test_edges.py b/tests/test_edges.py index ded1c569..30f529d0 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -1,6 +1,6 @@ """Failure and no-data scenarios: empty or broken upstream data, bad arguments, unknown portfolios, secrets kept out of logs, and the bot's signals.""" -import contextlib, io, os, signal, subprocess, sys, tempfile, threading, time, unittest +import contextlib, io, os, re, signal, subprocess, sys, tempfile, threading, time, unittest from pathlib import Path from tests.common import * @@ -319,6 +319,48 @@ def test_the_traceback_of_a_reported_error_is_redacted(self): self.assertNotIn('SECRET-tok_en', log) +class Unbuffered(io.StringIO): + """A stream like stderr under PYTHONUNBUFFERED (finbot.service, the image): each write is a write to the journal, + whose system call lets another thread run.""" + + def write(self, text): + """Let another thread run, then write.""" + time.sleep(0) + return super().write(text) + + +class WholeLogLines(unittest.TestCase): + """bot.WholeLines, the bot's stderr: a line goes out whole, whichever threads log at once.""" + + def test_lines_that_threads_print_at_once_stay_whole(self): + """Lines that threads print at the same time come out whole, never mixed ("TTL:cache hit: 34254 s/var/..."): + print() writes its pieces one by one, and a thread's pieces wait for the end of its line.""" + out = Unbuffered() + stream = bot.WholeLines(out) + def log(n): + """Print lines of several pieces each.""" + for i in range(200): + print('cache hit:', f'/var/lib/finbot/cache/{n}.json', 'TTL:', f'{i}s', file=stream) + threads = [threading.Thread(target=log, args=(n,)) for n in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + lines = out.getvalue().splitlines() + self.assertEqual(len(lines), 8 * 200) + self.assertEqual([line for line in lines if not re.fullmatch(r'cache hit: \S+/\d\.json TTL: \d+s', line)], []) + + def test_a_flush_writes_a_line_that_has_not_ended(self): + """A flush writes what the thread holds, though its line has not ended; the rest passes through.""" + out = io.StringIO() + stream = bot.WholeLines(out) + print('partial', end='', file=stream) + self.assertEqual(out.getvalue(), '') + stream.flush() + self.assertEqual(out.getvalue(), 'partial') + self.assertEqual(stream.getvalue(), 'partial') # the stream's own + + class SharedSession(unittest.TestCase): def test_the_session_keeps_no_cookies(self): """Yahoo's cookie goes in a header: a stored one would go to fc.yahoo.com, which then sets no new one.""" diff --git a/tests/test_yahoo.py b/tests/test_yahoo.py index bfda3812..f179b293 100644 --- a/tests/test_yahoo.py +++ b/tests/test_yahoo.py @@ -130,6 +130,19 @@ def test_a_symbol_yahoo_does_not_know_is_empty_and_an_outage_an_error(self): with self.assertRaises(RuntimeError): yahoo.fetch_detail('TSLA') + def test_a_fund_without_an_analysts_recommendation_parses(self): + """An ETF's quoteSummary has no analysts' recommendation (recommendationKey left out, or null), or a garbled + one: it parses without one, where .earnings failed on it ('NoneType' object has no attribute 'replace').""" + for ticker, financial in (('ETPMAG.AX', {}), ('MAGS', {'recommendationKey': None}), ('SMH', { + 'recommendationKey': 5, 'recommendationMean': {'raw': 2.1}, 'numberOfAnalystOpinions': {'raw': 9}})): + with self.subTest(ticker=ticker, financialData=financial): + body = self.combined() + body['quoteSummary']['result'][0]['financialData'] = financial + self.http(yahoo, Resp(body)) + data = yahoo.fetch_detail(ticker)[ticker] + self.assertIn('regularMarketPrice', data) + self.assertEqual([key for key in yahoo.RECOMMENDATION if key in data], []) + def test_missing_names_give_empty(self): """A detail without a name, or without a result, is empty.""" body = self.combined() From 02964381b3acc7054b2fef7aeb7b63f105afec25 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:21:35 +0000 Subject: [PATCH 23/31] Log levels, syslog style: log_level replaces debug; each request logged at INFO - log_level (ERROR, WARN, INFO or DEBUG, any case; INFO by default) sets what reaches the log: that level and those above it. debug is retired: accepted, with a note at start to delete it. - lib/log.py: log.error, log.warn, log.info, log.debug, with print()'s arguments, to stderr. Every log line of the bot has a level now: ERROR for what failed and was lost (a message not sent, a request not answered, a start that cannot go on); WARN for what failed and was got over (a notice not shown, a retry, a fallback, a refused request, a ticker skipped); INFO for what finbot does; DEBUG for cache hits and writes. The lines that debug used to show (requests answered, messages taken, cache misses) are INFO. - Start-up warnings read "WARN config:", not "ERROR config:"; a refused request "refused:", not "Fatal:". - cache: "cache expired" is "cache miss" (INFO), like a file not there. - net: each request that goes out is a line at INFO, its method and URL, without secrets or the scheme, cut to 100 characters ("GET query1.finance.yahoo.com/v8/finance/chart/AAPL?range=1y&crumb="). Telegram's long polling (getUpdates) is left out. record_outputs: 268 steps identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 11 ++++-- bot.py | 72 ++++++++++++++++------------------ defaults/preferences.conf | 5 ++- doc/configuration.md | 2 +- lib/cache.py | 13 +++--- lib/config.py | 15 ++++++- lib/delivery.py | 12 +++--- lib/discord_common.py | 12 +++--- lib/jobs.py | 16 ++++---- lib/log.py | 39 ++++++++++++++++++ lib/migrate.py | 15 ++++--- lib/net.py | 23 ++++++++--- lib/pull.py | 6 +-- lib/reports/cleanup.py | 5 ++- lib/reports/performance.py | 4 +- lib/reports/price.py | 4 +- lib/reports/profile.py | 4 +- lib/reports/rankings.py | 4 +- lib/reports/sharesight_warm.py | 4 +- lib/reports/trades.py | 8 ++-- lib/reports/yahoo_warm.py | 6 +-- lib/sharesight.py | 4 +- lib/shortman.py | 6 +-- lib/slack_common.py | 38 +++++++++--------- lib/slack_websocket.py | 4 +- lib/telegram_common.py | 24 ++++++------ lib/telegram_longpolling.py | 3 +- lib/telegram_webhook.py | 4 +- lib/worker.py | 8 ++-- lib/yahoo.py | 24 ++++++------ tests/test_config.py | 32 +++++++++------ tests/test_edges.py | 63 +++++++++++++++++++++++++++++ tests/test_finbot.py | 5 ++- tests/test_pull.py | 6 +-- 34 files changed, 317 insertions(+), 184 deletions(-) create mode 100644 lib/log.py diff --git a/CLAUDE.md b/CLAUDE.md index 83b7e889..3b58853f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,7 +94,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm `yahoo.symbol(typed)` is Yahoo's symbol for a ticker as written, tried on the local exchange (`exchange_suffix`) when Yahoo does not know it bare: every ticker a chat gives goes through it. `sharesight.portfolios(name=None)` gives `Portfolio` objects: `holdings()`, `tickers()`, `gain(days)` (the portfolio's), `trades(days)`. -- `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; +- `lib/log.py`: the log, by level (`error warn info debug`), as the `log_level` preference lets in; bot.py's `WholeLines` + writes each line whole, whichever threads log at once. `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; change one with `with state.edit(name, default) as data:`, never load-then-save; a file of one record per chat with `state.load_for(chat, name, default)` and `state.edit_for`. A chat's key is its network and id (`Transport.key`, `telegram:-1001234`), the same in both modes. `lib/watchlist.py`: each chat's watchlist (`tickers(chat)`), and @@ -117,7 +118,7 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm to each network `test_sending`; Telegram's webhook and command menu `test_telegram_api`; job lines, job arguments and the migration `test_schedule`; the scheduler and `.jobs` in a chat `test_scheduler`; config `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, cache and state files - `test_util`; data dir `test_datadir`; chunking, git_version, and the docstring, stdout, line-width and no-pandas + `test_util`; data dir `test_datadir`; log levels, request lines and whole lines `test_edges`; chunking, git_version, and the docstring, stdout, line-width and no-pandas checks `test_finbot`. ## Conventions @@ -132,7 +133,11 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - Every function and method has a docstring, tests included (a test checks): one line where it fits, reworded rather than spilling a few words onto a second. Tabs everywhere (`.editorconfig`). Every line stays within 120 columns, tabs counting 4 (an edit quotes whole lines); a test checks all but README.md, CLAUDE.md and `doc/`. -- Log with `print(..., file=sys.stderr)`. systemd discards stdout (`StandardOutput=null`), so the bot and its reports +- Log with `lib/log.py`, at a level, with print()'s arguments: `log.error` (failed and lost: a message not sent), + `log.warn` (failed and got over: a retry, a fallback, a refused request), `log.info` (what finbot does: a message + sent, a job run, a cache miss; `net` logs each request it sends), `log.debug` (cache hits and writes). The + `log_level` preference sets what reaches the log; `lib/config.py` prints its own errors (it loads before + preferences exist). Everything goes to stderr: systemd discards stdout (`StandardOutput=null`), so the bot and its reports never print there (a test checks); only the util/ tools, run by hand, do. - A report gives its chart as a `render.Chart` spec, not a drawing: `deliver()` draws it only for a chat that takes charts, through `render.draw()`, which caches the image by the spec's hash (and the drawing code's). diff --git a/bot.py b/bot.py index 5e31c770..372ed2f9 100755 --- a/bot.py +++ b/bot.py @@ -8,6 +8,7 @@ import collections, dataclasses, json, os, re, sys, threading, traceback from pathlib import Path +from lib import log from lib import config from lib import discord_common from lib import discord_webhook @@ -71,8 +72,8 @@ def main(environ, start_response): status = '200 OK' else: where = environ.get(ROUTED) or repr(environ.get('PATH_INFO')) - print(f"request to {where} failed:", delivery.error_line(e), file=sys.stderr) - print(net.redact(traceback.format_exc()), end='', file=sys.stderr) + log.error(f"request to {where} failed:", delivery.error_line(e)) + log.error(net.redact(traceback.format_exc()), end='') status = '500 Internal Server Error' try: start_response(status, [('Content-type', 'text/plain')], sys.exc_info()) @@ -85,43 +86,41 @@ def handle(environ, start_response): comes from that network (before its body is read, when the proof is a header), read the update (CONTENT_LENGTH bytes, up to MAX_BODY) and take it (take): the network gets its answer at once.""" def respond(response): - """Answer the request: a status line and a body (logged with the debug preference, naming the network, not the - path). Work that must wait for the answer starts once the server has sent it.""" - if config.prefs.debug: - print(f"[http] request for {environ.get(ROUTED) or 'no network'} answered {response.status}", - file=sys.stderr) + """Answer the request: a status line and a body (logged at INFO, naming the network, not the path). Work that + must wait for the answer starts once the server has sent it.""" + log.info(f"[http] request for {environ.get(ROUTED) or 'no network'} answered {response.status}") start_response(response.status, [('Content-type', 'application/json')]) return Sent([response.body], response.then) if response.then else [response.body] - def print_headers(): - """Log the request headers, Telegram's secret header redacted.""" + def print_headers(say=log.info): + """Log the request headers (at INFO, or as say() logs), Telegram's secret header redacted.""" for name, value in sorted(environ.items()): - print((name, '' if name == 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' else value), file=sys.stderr) + say((name, '' if name == 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' else value)) if config.prefs.print_headers: print_headers() path = environ.get('PATH_INFO', '') source = route(path) if source is None: - print("Unknown URI", repr(path), file=sys.stderr) + log.warn("Unknown URI", repr(path)) return respond(inbound.NOT_FOUND) environ[ROUTED] = source.service if why := source.refuse_unread(environ): # strangers are turned away before their body is read - print_headers() - print("Fatal:", why, file=sys.stderr) + print_headers(log.warn) + log.warn("refused:", why) return respond(inbound.FORBIDDEN) length = environ.get('CONTENT_LENGTH') or '0' if not length.isdigit() or int(length) > MAX_BODY: - print(f"[{source.service}]: a body of {length!r} bytes refused unread", file=sys.stderr) + log.warn(f"[{source.service}]: a body of {length!r} bytes refused unread") return respond(inbound.TOO_LARGE if length.isdigit() else inbound.BAD_REQUEST) body = environ['wsgi.input'].read(int(length)) # the body and no more: a server need not end the stream there try: update = source.decode(body) except ValueError: - print(f"[{source.service}]: request body is not a JSON object", file=sys.stderr) + log.warn(f"[{source.service}]: request body is not a JSON object") return respond(inbound.BAD_REQUEST) if why := source.refuse(environ, body, update): shown = {k: '' if k == 'token' else v for k, v in update.items()} # Slack's verification token - print("warning:", why, file=sys.stderr) - print(f"[{source.service}]: refused update", json.dumps(shown, indent=4, default=str), file=sys.stderr) + log.warn("warning:", why) + log.warn(f"[{source.service}]: refused update", json.dumps(shown, indent=4, default=str)) return respond(source.REFUSED) environ[AUTHENTICATED] = True return respond(take(source.network, update)) @@ -180,7 +179,7 @@ def take(network, update): if isinstance(event, inbound.Suggest): # answered here and at once, while the command is typed return network.suggestions(worker.suggest(f'{event.service}:{event.chat_id}', event.text)) if seen_before(network.service, network.update_id(update)): - print(f"[{network.service}]: update {network.update_id(update)} seen already, not run again", file=sys.stderr) + log.info(f"[{network.service}]: update {network.update_id(update)} seen already, not run again") return inbound.OK answer = network.acknowledge(event) if answer is not None: @@ -191,11 +190,10 @@ def take(network, update): def dispatch(event): """Hand a message or a button press to lib/worker, in a thread of its own, which it returns.""" if isinstance(event, inbound.Press): - print(f"[{event.service.capitalize()} button]:", event.user, "chat", event.chat_id, event.data, file=sys.stderr) + log.info(f"[{event.service.capitalize()} button]:", event.user, "chat", event.chat_id, event.data) target = worker.process_callback else: - if config.prefs.debug: - print(f"[{event.service.capitalize()}]:", event.user, event.text, file=sys.stderr) + log.info(f"[{event.service.capitalize()}]:", event.user, event.text) target = worker.process_request thread = threading.Thread(target=target, args=(event,)) thread.start() @@ -245,13 +243,12 @@ def reload_all(*args): try: changes = config.reload() for change in changes: - print('config:', change, file=sys.stderr) - print(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}", - file=sys.stderr) + log.info('config:', change) + log.info(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}") except config.ConfigError as e: config.print_errors(e) if note := config.template_note(config.preferences_path): - print('ERROR config:', note, file=sys.stderr) + log.warn('WARN config:', note) def flush_cache(*args): """SIGUSR1 (`systemctl kill -s USR1 finbot`): delete the cache and keep the state (watchlist, seen trades, ratings, @@ -259,10 +256,9 @@ def flush_cache(*args): for them).""" try: removed = cleanup.run(days=0) - print(f"cache: flushed {removed} file{'' if removed == 1 else 's'} from {config.env.cache_dir}", - file=sys.stderr) + log.info(f"cache: flushed {removed} file{'' if removed == 1 else 's'} from {config.env.cache_dir}") except Exception as e: - print('ERROR', delivery.error_line(e, 'cache flush'), file=sys.stderr) + log.error('ERROR', delivery.error_line(e, 'cache flush')) jobs.start_thread(jobs.warm_now, scheduler) def http_server(): @@ -275,18 +271,18 @@ def http_server(): if __name__ == '__main__': sys.stderr = WholeLines(sys.stderr) # threads log at once: each line goes out whole - print(f'Starting finbot {git_version()}', file=sys.stderr) + log.info(f'Starting finbot {git_version()}') try: config.check_startup_environment() except config.ConfigError as e: config.print_errors(e) sys.exit(1) for line in config.startup_report(): - print('config:', line, file=sys.stderr) + log.info('config:', line) for line in config.startup_warnings(): - print('ERROR config:', line, file=sys.stderr) + log.warn('WARN config:', line) if os.getuid() == 0: - print("Running as superuser. This is not recommended.", file=sys.stderr) + log.warn("Running as superuser. This is not recommended.") migrate.run() # once: what was kept for all chats becomes the home chats', schedule.cron among it scheduler = jobs.Scheduler() scheduler.load() @@ -294,18 +290,18 @@ def http_server(): try: server = http_server() # bound before Telegram is given the URL except OSError as e: - print(e, file=sys.stderr) + log.error(e) sys.exit(1) if server: - print(f'Opening socket on http://{config.env.ip}:{server.effective_port}', file=sys.stderr) + log.info(f'Opening socket on http://{config.env.ip}:{server.effective_port}') else: - print('No webhook is set: finbot takes no HTTP requests', file=sys.stderr) + log.info('No webhook is set: finbot takes no HTTP requests') if config.env.telegramBotToken: try: # here only, never on import: the webhook set (HTTP mode), or deleted for getUpdates (pull mode) (telegram_webhook if config.env.telegramOutgoingWebhook else telegram_longpolling).register() telegram_common.READER.bot_name = telegram_common.bot_name() except Exception as e: - print('ERROR', delivery.error_line(e, 'telegram'), file=sys.stderr) + log.error('ERROR', delivery.error_line(e, 'telegram')) sys.exit(1) telegram_common.set_commands(worker.slash_commands(aliases=False)) # the menu; /profile and /job work typed threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background @@ -314,7 +310,7 @@ def http_server(): try: discord_common.READER.bot_id = discord_common.register(worker.slash_commands()) except Exception as e: - print('ERROR', delivery.error_line(e, 'discord'), file=sys.stderr) + log.error('ERROR', delivery.error_line(e, 'discord')) sys.exit(1) jobs.current = scheduler # for the .jobs chat command threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs @@ -323,7 +319,7 @@ def http_server(): # Startup is done. systemctl reload finbot: pick up an edited preferences.conf signal.signal(signal.SIGHUP, in_thread(reload_all)) signal.signal(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache - print('finbot is running: SIGHUP reloads, SIGUSR1 flushes the cache', file=sys.stderr) + log.info('finbot is running: SIGHUP reloads, SIGUSR1 flushes the cache') if server: server.run() while True: # the threads do the work; this one waits for signals diff --git a/defaults/preferences.conf b/defaults/preferences.conf index 19dac7b4..aaa49751 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -51,6 +51,7 @@ #performance_cache_seconds=2400 #http_timeout=18 -# Diagnostics -#debug=False +# Diagnostics. log_level: what the log shows, that level and those above it: ERROR, WARN, INFO (what finbot does, +# and each request it sends) or DEBUG (cache hits and writes too). +#log_level=INFO #print_headers=False diff --git a/doc/configuration.md b/doc/configuration.md index 8a893dde..0e18e2db 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -75,7 +75,7 @@ A change takes effect on a reload. A true/false preference takes `true`/`false`, | `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | | `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm` runs, so the holdings never run out | | `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | -| `debug` | `False` | Extra diagnostics in the log | +| `log_level` | `INFO` | What the log shows: that level and those above it, syslog style. `ERROR`: what failed and was lost (a message not sent); `WARN`: what failed and was got over (a retry, a fallback, a request refused); `INFO`: what finbot does (each message sent, each job run, each request sent and answered, each cache miss); `DEBUG`: cache hits and writes too. Replaces `debug` | | `print_headers` | `False` | Log inbound request headers | A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. diff --git a/lib/cache.py b/lib/cache.py index 8d4f33fa..86ecf2e5 100644 --- a/lib/cache.py +++ b/lib/cache.py @@ -3,12 +3,12 @@ import io import json import os -import sys import tempfile import time from pathlib import Path from lib import config +from lib import log def path(name): @@ -29,13 +29,10 @@ def _fresh(name, seconds): if seconds is None: seconds = config.prefs.cache_seconds where, written = path(name), age(name) - if written is None: - print("cache miss:", where, file=sys.stderr) if config.prefs.debug else None + if written is None or written >= seconds: + log.info("cache miss:", where) return None - if written >= seconds: - print("cache expired:", where, file=sys.stderr) if config.prefs.debug else None - return None - print(f"cache hit: {where} TTL: {seconds - written:.0f}s", file=sys.stderr) if config.prefs.debug else None + log.debug(f"cache hit: {where} TTL: {seconds - written:.0f}s") return where @@ -85,7 +82,7 @@ def write(name, data): """Write a cache file's JSON, compact: nobody reads it but finbot (a 10-year chart takes a third of the space, and parses faster). See write_file.""" where = path(name) - print("cache write:", where, file=sys.stderr) if config.prefs.debug else None + log.debug("cache write:", where) write_file(where, json.dumps(data, separators=(',', ':')).encode('utf-8')) diff --git a/lib/config.py b/lib/config.py index 10141548..73186649 100644 --- a/lib/config.py +++ b/lib/config.py @@ -34,7 +34,9 @@ 'discord_webhook': 'the scheduled reports go to discordChannelID as the bot (discordBotToken); set those, and' ' delete the line', 'scheduler': 'the scheduler is always on, and a schedule without jobs runs nothing; delete the line', + 'debug': 'log_level sets what the log shows (INFO by default; DEBUG adds cache hits); delete the line', } +LOG_LEVELS = ('ERROR', 'WARN', 'INFO', 'DEBUG') # log_level's values, the most severe first (lib/log.py) OLD_ENV = REPO_DIR / '.env' # the single config file of old versions, which python-dotenv loaded; nothing reads it now @@ -198,6 +200,14 @@ def float(self, name, default): """A number setting.""" return self.number(float, name, default) + def choice(self, name, default, choices): + """One of choices, in any case; anything else is an error, giving default.""" + value = str(self.value(name, default)).strip().upper() + if value in choices: + return value + self.errors.append(f"{name}={self.getenv(name, default)!r} is not one of {', '.join(choices)}") + return default + def words(self, name, default): """A list of words, separated by spaces: a tuple of them.""" value = self.value(name, default) @@ -286,7 +296,6 @@ class Prefs: cache: bool = True cache_seconds: int = 43200 country_code: str = 'AU' - debug: bool = False demote_leveraged: bool = True demote_volatile: bool = True exchange_suffix: str = 'AX' # Yahoo's suffix for the local exchange: a bare ticker Yahoo does not know gets it @@ -298,6 +307,7 @@ class Prefs: hyperlinkFooter: bool = True hyperlinkProvider: str = 'google' include_portfolios: tuple = () # portfolio ids + log_level: str = 'INFO' # what the log shows: one of LOG_LEVELS, and those more severe (lib/log.py) min_analysts: int = 7 # .buy, .sell and .forward pe list only stocks this many analysts cover past_days: int = 28 performance_cache_seconds: int = 2400 # Sharesight performance reports (40 minutes: sharesight_warm runs every 30) @@ -318,7 +328,8 @@ def preference_values(getenv, source=None): r = _Reader(getenv) readers = {bool: r.bool, int: r.int, float: r.float, str: r.str, tuple: r.words, 'exchange_suffix': lambda name, default: r.blankable(name, default).removeprefix('.').upper(), # blank: none - 'hyperlinkProvider': lambda name, default: r.str(name, default).lower()} + 'hyperlinkProvider': lambda name, default: r.str(name, default).lower(), + 'log_level': lambda name, default: r.choice(name, default, LOG_LEVELS)} values = {field.name: readers.get(field.name, readers[field.type])(field.name, field.default) for field in dataclasses.fields(Prefs) if field.compare} r.check() diff --git a/lib/delivery.py b/lib/delivery.py index ebfa4699..793a5351 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -2,9 +2,9 @@ one-line error report. A destination is a transport: a network's bot (TelegramBot, SlackBot, DiscordBot).""" import dataclasses import html -import sys import traceback +from lib import log from lib import config from lib import discord_common from lib import net @@ -88,7 +88,7 @@ def deliver(dest, report): return [] if answer: return [answer] - print("chart not sent, sending the report as text", file=sys.stderr) + log.warn("chart not sent, sending the report as text") return dest.text(caption.split('\n')) @@ -110,15 +110,15 @@ def report_error(e, dest=None, context=None): for the same escapes). Never raises: error reporting must not become a second failure.""" try: line = error_line(e, context) - print(line, file=sys.stderr) + log.error(line) if isinstance(e, BaseException): trace = ''.join(traceback.format_exception(e)) - print(net.redact(trace), end='', file=sys.stderr) # a request's error carries its URL + log.error(net.redact(trace), end='') # a request's error carries its URL for chat in targets(dest): try: chat.text([line if chat.dialect == 'discord' else html.escape(line, quote=False)]) except Exception as send_error: - print("Failed to report error to", chat.name, ":", send_error, file=sys.stderr) + log.error("Failed to report error to", chat.name, ":", send_error) except Exception as reporting_error: - print("Failed to report error:", reporting_error, file=sys.stderr) + log.error("Failed to report error:", reporting_error) diff --git a/lib/discord_common.py b/lib/discord_common.py index 179ca6e7..c249a6f6 100644 --- a/lib/discord_common.py +++ b/lib/discord_common.py @@ -4,11 +4,11 @@ that posts into its channels and DMs (DiscordBot).""" import json import re -import sys import time import requests +from lib import log from lib import config from lib import dialects from lib import inbound @@ -51,7 +51,7 @@ def try_call(method, path, **kwargs): try: return call(method, path, **kwargs) except Exception as e: - print(f"Discord {method} failed:", net.redact(e), file=sys.stderr) + log.warn(f"Discord {method} failed:", net.redact(e)) def retry_after(response): @@ -98,7 +98,7 @@ def register(commands): path = f'/applications/{application}/commands' if summary(call('GET', path) or []) != summary(wanted): call('PUT', path, json=wanted) - print(f"discord: {len(wanted)} slash commands registered", file=sys.stderr) + log.info(f"discord: {len(wanted)} slash commands registered") return str(call('GET', '/users/@me')['id']) @@ -239,13 +239,13 @@ def post(self, payload, image=None): answer = call(method, path, **message_body(payload, image)) except requests.exceptions.ReadTimeout as e: self.answered = True # it may have arrived - print("Discord post timed out, it may still arrive:", net.redact(e), file=sys.stderr) + log.warn("Discord post timed out, it may still arrive:", net.redact(e)) return transports.MAYBE_SENT if image is not None else None except Exception as e: - print("Discord post to", self.where, "failed:", net.redact(e), file=sys.stderr) + log.error("Discord post to", self.where, "failed:", net.redact(e)) return None self.answered = True - print("OK discord message to", self.where, file=sys.stderr) + log.info("OK discord message to", self.where) return answer or {} def message_path(self, message): diff --git a/lib/jobs.py b/lib/jobs.py index 4e39e83f..fe0b8990 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -10,7 +10,6 @@ import math import re import shlex -import sys import threading import time import zlib @@ -19,6 +18,7 @@ from croniter import croniter, CroniterBadDateError +from lib import log from lib import config from lib import state from lib import delivery @@ -98,7 +98,7 @@ def warm_now(scheduler=None): schedule is not started twice; without one (a test, a tool), here, a failure reported in the chat as the scheduler reports one.""" commands = parse_reports(WARM, system=True) - print('cache: filling it again:', ', '.join(label for label, _, _ in commands), file=sys.stderr) + log.info('cache: filling it again:', ', '.join(label for label, _, _ in commands)) if scheduler: scheduler.execute(commands) else: @@ -107,7 +107,7 @@ def warm_now(scheduler=None): run_in_process(label, module, kwargs) except Exception as e: delivery.report_error(e, context=label) - print('cache: warm jobs done', file=sys.stderr) + log.info('cache: warm jobs done') def parse_reports(text, system=False): @@ -218,8 +218,8 @@ def load(self): if problem: delivery.report_error(ScheduleError(problem), context='schedule') chats = len({job.chat for job in jobs if job.chat}) - print(f"schedule: {len(jobs) - len(SYSTEM)} jobs of {chats} chat{'' if chats == 1 else 's'}, and" - f" {len(SYSTEM)} system jobs", file=sys.stderr) + log.info(f"schedule: {len(jobs) - len(SYSTEM)} jobs of {chats} chat{'' if chats == 1 else 's'}, and" + f" {len(SYSTEM)} system jobs") def start_base(self, job, now): """Where to count from at start: a run missed in the catch-up window fires once; with no record, none is.""" @@ -264,7 +264,7 @@ def due(self, now): try: state.save(STATE_FILE, self.last) except OSError as e: - print('schedule: cannot save', STATE_FILE, e, file=sys.stderr) + log.error('schedule: cannot save', STATE_FILE, e) return fired def claim(self, chat, label): @@ -285,10 +285,10 @@ def execute(self, commands, chat=''): outcome = 'done' for label, module, kwargs in commands: if not self.claim(chat, label): - print(f'schedule: {label} is already running for {chat or "the system"}, skipping it', file=sys.stderr) + log.warn(f'schedule: {label} is already running for {chat or "the system"}, skipping it') outcome = 'already running' continue - print(f'schedule: running {label}' + (f' for {chat}' if chat else ''), file=sys.stderr) + log.info(f'schedule: running {label}' + (f' for {chat}' if chat else '')) late = threading.Timer(JOB_TIMEOUT, delivery.report_error, args=(TimeoutError(f'still running after' f' {JOB_TIMEOUT / 60:g} minutes, and left to finish'), dest), kwargs={'context': label}) late.daemon = True diff --git a/lib/log.py b/lib/log.py new file mode 100644 index 00000000..4a44b906 --- /dev/null +++ b/lib/log.py @@ -0,0 +1,39 @@ +"""The log, syslog style: each line has a level, ERROR, WARN, INFO or DEBUG, and goes to stderr (which systemd and +Docker take to the journal) when the log_level preference lets it in: that level and those above it, so ERROR always +does. Each takes print()'s arguments.""" +import sys + +from lib import config + +LEVELS = config.LOG_LEVELS # ERROR WARN INFO DEBUG, the most severe first + + +def enabled(level): + """Whether log_level lets lines of this level into the log.""" + return LEVELS.index(level) <= LEVELS.index(config.prefs.log_level) + + +def write(level, parts, options): + """A line of these parts (and print()'s options) at this level, if log_level lets it in.""" + if enabled(level): + print(*parts, file=sys.stderr, **options) + + +def error(*parts, **options): + """Something failed and was lost: a message not sent, a request not answered, a start that cannot go on.""" + write('ERROR', parts, options) + + +def warn(*parts, **options): + """Something failed and finbot got over it: a notice not shown, a retry, a fallback, a request refused.""" + write('WARN', parts, options) + + +def info(*parts, **options): + """What finbot does: a message sent, a job run, a cache miss, each request it sends and each it answers.""" + write('INFO', parts, options) + + +def debug(*parts, **options): + """The detail of what it does, to find a fault: a cache hit, a cache write.""" + write('DEBUG', parts, options) diff --git a/lib/migrate.py b/lib/migrate.py index 00ba9cfa..8d397184 100644 --- a/lib/migrate.py +++ b/lib/migrate.py @@ -11,10 +11,10 @@ import os import re import shlex -import sys import zoneinfo from pathlib import Path +from lib import log from lib import config from lib import delivery from lib import jobs @@ -63,14 +63,13 @@ def schedule(homes): try: job = jobs.parse_job(f"{'' if zone == 'UTC' else f'CRON_TZ={zone} '}{line} {reports}") except jobs.ScheduleError as e: - print(f'schedule: {path} line {number} left out: {e}', file=sys.stderr) + log.warn(f'schedule: {path} line {number} left out: {e}') continue entries.append({'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': True}) # written even when empty: the file being there says the migration is done state.save(jobs.SCHEDULE_FILE, {key: copy.deepcopy(entries) for key in homes if entries}) - print(f"schedule: the {len(entries)} jobs of {path} are now each home chat's ({', '.join(homes)});" - f" finbot reads that file no more: each chat's jobs are in {jobs.SCHEDULE_FILE}, changed with .jobs", - file=sys.stderr) + log.info(f"schedule: the {len(entries)} jobs of {path} are now each home chat's ({', '.join(homes)});" + f" finbot reads that file no more: each chat's jobs are in {jobs.SCHEDULE_FILE}, changed with .jobs") def record(name, homes): @@ -78,12 +77,12 @@ def record(name, homes): try: data = state.load(name) except (OSError, ValueError) as e: - print(f'state: {name} left as it is, unreadable: {e}', file=sys.stderr) + log.warn(f'state: {name} left as it is, unreadable: {e}') return if data is None or (isinstance(data, dict) and 'chats' in data): return state.save(name, {'chats': {key: copy.deepcopy(data) for key in homes}}) - print(f"state: {name} is kept per chat now, its record copied to {', '.join(homes)}", file=sys.stderr) + log.info(f"state: {name} is kept per chat now, its record copied to {', '.join(homes)}") def local_zone(): @@ -127,7 +126,7 @@ def legacy_lines(text, path): commands = [(os.path.basename(words[0]).removesuffix('.py'), words[1:]) for words in split_commands(tokenize(rest))] except ValueError as e: - print(f'schedule: {path} line {number} left out: {e}', file=sys.stderr) + log.warn(f'schedule: {path} line {number} left out: {e}') continue yield number, when, zone, commands diff --git a/lib/net.py b/lib/net.py index 332860ca..0ddff26b 100644 --- a/lib/net.py +++ b/lib/net.py @@ -9,23 +9,34 @@ import requests from lib import config +from lib import log session = requests.Session() # no domain may set or receive a cookie session.cookies.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[])) CONNECT_TIMEOUT = 5 # seconds to wait for a host to answer the connection: one that is down costs this, not http_timeout +SHOWN = 100 # characters of a request's URL in the log, at most def get(url, **kwargs): - """requests.get through the shared session, which reuses connections; timed out as timeouts() says.""" - return session.get(url, **timeouts(kwargs)) + """requests.get through the shared session, which reuses connections; timed out as timeouts() says; logged.""" + return session.get(url, **timeouts(logged('GET', url, kwargs))) def post(url, **kwargs): - """requests.post through the shared session, which reuses connections; timed out as timeouts() says.""" - return session.post(url, **timeouts(kwargs)) + """requests.post through the shared session, which reuses connections; timed out as timeouts() says; logged.""" + return session.post(url, **timeouts(logged('POST', url, kwargs))) def request(method, url, **kwargs): - """A request of any method through the shared session, which reuses connections; timed out as timeouts() says.""" - return session.request(method, url, **timeouts(kwargs)) + """A request of any method through the shared session, which reuses connections; timed out as timeouts() says; + logged.""" + return session.request(method, url, **timeouts(logged(method, url, kwargs))) + +def logged(method, url, kwargs): + """The request's arguments without `logged`; the request noted at INFO first, unless logged=False (Telegram's long + polling, asked again and again): its method and URL, without secrets or the scheme, cut to SHOWN characters.""" + if kwargs.pop('logged', True): + shown = redact(url).split('://', 1)[-1] + log.info(method, shown if len(shown) <= SHOWN else shown[:SHOWN - 1] + '…') + return kwargs def timeouts(kwargs): """The request's arguments with a timeout, unless it has its own: CONNECT_TIMEOUT to connect, then http_timeout for diff --git a/lib/pull.py b/lib/pull.py index c4c5e273..cae30aab 100644 --- a/lib/pull.py +++ b/lib/pull.py @@ -3,9 +3,9 @@ telegram_longpolling, slack_websocket and discord_websocket), which hands each update to bot.take(network, update) as the webhook would, so that reading it, the repeat check and acknowledging a press are the same both ways. Each listener keeps going through failures, waiting longer after each one in a row (backoff).""" -import sys import time +from lib import log from lib import delivery from lib import net @@ -31,7 +31,7 @@ def wanted(): def run(self, take): """Fetch and hand on updates for good, waiting as poll() says between rounds.""" - print(f"{self.network.service}: {self.describe()}", file=sys.stderr) + log.info(f"{self.network.service}: {self.describe()}") while True: time.sleep(self.poll(take)) @@ -48,7 +48,7 @@ def failed(self, what, e): """Note a failure in a row, log it, and return the wait before trying again.""" self.failures += 1 wait = backoff(self.failures) - print(f"{self.network.service}: {what} failed, trying again in {wait} s:", net.redact(e), file=sys.stderr) + log.warn(f"{self.network.service}: {what} failed, trying again in {wait} s:", net.redact(e)) return wait def hand_on(self, take, update): diff --git a/lib/reports/cleanup.py b/lib/reports/cleanup.py index cb561215..5ab07c21 100644 --- a/lib/reports/cleanup.py +++ b/lib/reports/cleanup.py @@ -1,5 +1,6 @@ """Deletes old cache files (scheduled: cleanup [days]); with days=0 it is the cache flush.""" -import os, sys, time +import os, time +from lib import log from lib import config def run(*, days=30): @@ -20,7 +21,7 @@ def run(*, days=30): os.unlink(entry.path) removed += 1 except OSError as e: - print("cleanup: can't remove", entry.path, e, file=sys.stderr) + log.warn("cleanup: can't remove", entry.path, e) return removed ARGUMENTS = '[DAYS]' # what a scheduled run takes diff --git a/lib/reports/performance.py b/lib/reports/performance.py index 13086d5f..2f05cecd 100644 --- a/lib/reports/performance.py +++ b/lib/reports/performance.py @@ -1,6 +1,6 @@ """.performance: each portfolio's return over a period, beside two benchmarks (scheduled: performance [days]).""" -import sys +from lib import log from lib import config from lib import sharesight from lib import delivery @@ -22,7 +22,7 @@ def benchmark(ticker, market, text, past_days, service): try: percent = yahoo.price_history(ticker, days=past_days)[0][past_days] except Exception as e: # the rest of the report still goes out - print(ticker, e, file=sys.stderr) + log.warn(ticker, e) return f"⚠️ {text}: no price history", None link = links.finance_link(ticker, market, service=service, days=past_days, brief=True, text=text) return f"{formats.get_emoji(percent)} {link} {formats.signed_percent(percent, 2)}", percent diff --git a/lib/reports/price.py b/lib/reports/price.py index f250deed..3dba8f5d 100644 --- a/lib/reports/price.py +++ b/lib/reports/price.py @@ -2,8 +2,8 @@ premarket | close | ).""" import dataclasses import datetime -import sys +from lib import log from lib import config from lib import delivery from lib import formats @@ -32,7 +32,7 @@ def gather(tickers, specific_stock=None, days=None): except RuntimeError as e: # no history: the one stock asked about says so, a list leaves it out if specific_stock: raise - print(ticker, e, file=sys.stderr) + log.warn(ticker, e) continue market_data[ticker]['percent_change_period'] = changes[days] return market_data diff --git a/lib/reports/profile.py b/lib/reports/profile.py index e79c2f15..d321b836 100644 --- a/lib/reports/profile.py +++ b/lib/reports/profile.py @@ -1,7 +1,7 @@ """The .SYMBOL report: what Yahoo and Simply Wall St know about one company.""" from itertools import pairwise import datetime -import sys +from lib import log from lib import config from lib import formats from lib import links @@ -194,7 +194,7 @@ def changes(service, data, ticker, state): try: history, _ = yahoo.price_history(ticker, quote=data) # the percentages only: the profile has no chart except RuntimeError as e: # no chart data: the profile goes out without these lines - print(ticker, e, file=sys.stderr) + log.warn(ticker, e) history = {} for interval in ('5Y', '1Y', '1M', '1D'): if interval in history: diff --git a/lib/reports/rankings.py b/lib/reports/rankings.py index cca7d5c7..13e0c77a 100644 --- a/lib/reports/rankings.py +++ b/lib/reports/rankings.py @@ -1,6 +1,6 @@ """Ranked lists of the stocks followed (.marketcap, .pe, .forward pe and .peg, .beta, .buy and .sell), and one stock's market cap or ratio.""" -import sys +from lib import log from lib import answer from lib import config from lib import watchlist @@ -28,7 +28,7 @@ def marketcap_list(service, chat, bottom=False, length=15): for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(watchlist.with_holdings(chat))).items(): if 'market_cap_usd' not in data: why = "no market cap" if 'market_cap' not in data else f"no exchange rate for {data.get('currency')}" - print(ticker, why, file=sys.stderr) + log.info(ticker, why) continue cap = formats.human_units(data['market_cap_usd']) caps.append((data['market_cap_usd'], f"{links.stock_line(ticker, data, service)} mkt cap: {cap}")) diff --git a/lib/reports/sharesight_warm.py b/lib/reports/sharesight_warm.py index 72cdedb8..6a8b4628 100644 --- a/lib/reports/sharesight_warm.py +++ b/lib/reports/sharesight_warm.py @@ -1,6 +1,6 @@ """Keeps Sharesight's portfolio list and holdings reports fresh in the cache (scheduled: sharesight_warm), so that a request never waits on Sharesight. The price histories have their own, yahoo_warm.""" -import sys +from lib import log from lib import config, sharesight def run(): @@ -14,7 +14,7 @@ def run(): try: return len(sharesight.warm()) except Exception as e: - print("sharesight_warm: refresh failed:", e, file=sys.stderr) + log.warn("sharesight_warm: refresh failed:", e) return 0 ARGUMENTS = '' # what a scheduled run takes: none diff --git a/lib/reports/trades.py b/lib/reports/trades.py index 85792d13..a68e4118 100644 --- a/lib/reports/trades.py +++ b/lib/reports/trades.py @@ -1,8 +1,8 @@ """.trades: Sharesight buys and sells; a scheduled run reports only new ones (scheduled: trades).""" import dataclasses import datetime -import sys +from lib import log from lib import config from lib import state from lib import sharesight @@ -60,10 +60,10 @@ def gather(days, portfolio_select, known): if raw and config.prefs.trades_use_yahoo: tickers = {markets.transform_to_yahoo(trade['symbol'], trade['market']) for trade in raw} try: - print("Fetching tickers from Yahoo: ", tickers, file=sys.stderr) + log.info("Fetching tickers from Yahoo: ", tickers) quotes = yahoo.fetch(tickers) except Exception as e: - print("Warning: could not fetch Yahoo:", e, file=sys.stderr) + log.warn("Warning: could not fetch Yahoo:", e) trades = [] for trade in raw: market = trade['market'] @@ -75,7 +75,7 @@ def gather(days, portfolio_select, known): try: market = quotes[ticker]['profile_exchange'] except Exception as e: - print("Warning: could not get market for", ticker, "from Yahoo:", e, file=sys.stderr) + log.warn("Warning: could not get market for", ticker, "from Yahoo:", e) trades.append(Trade(date=datetime.datetime.strptime(trade['transaction_date'], '%Y-%m-%d').date(), id=int(trade['id']), portfolio=trade['portfolio'], bought=trade['transaction_type'] == 'BUY', value=round(float(trade['price']) * float(trade['quantity'])), diff --git a/lib/reports/yahoo_warm.py b/lib/reports/yahoo_warm.py index f3f8cde9..94793686 100644 --- a/lib/reports/yahoo_warm.py +++ b/lib/reports/yahoo_warm.py @@ -1,6 +1,6 @@ """Keeps the price history of every stock followed fresh in the cache (scheduled: yahoo_warm), so that a period list or a chart never waits on Yahoo. Sharesight's caches have their own, sharesight_warm.""" -import sys +from lib import log from lib import config, watchlist, yahoo def run(): @@ -15,9 +15,9 @@ def run(): try: tickers = watchlist.all_followed() except Exception as e: # Sharesight unreachable, say: the watchlists' at least - print("yahoo_warm: no holdings from Sharesight:", e, file=sys.stderr) + log.warn("yahoo_warm: no holdings from Sharesight:", e) failed = yahoo.prefetch_history(tickers, seconds=0) - print(f"yahoo_warm: {len(tickers) - len(failed)} of {len(tickers)} price histories fetched", file=sys.stderr) + log.info(f"yahoo_warm: {len(tickers) - len(failed)} of {len(tickers)} price histories fetched") return len(tickers) - len(failed) ARGUMENTS = '' # what a scheduled run takes: none diff --git a/lib/sharesight.py b/lib/sharesight.py index 190e3f89..4a4b829e 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -2,10 +2,10 @@ as sent, and requests go one at a time.""" import dataclasses import datetime -import sys import threading import time +from lib import log from lib import answer from lib import cache from lib import config @@ -37,7 +37,7 @@ def _request(path, what, expect, **kwargs): error = data.get('error') if isinstance(data, dict) else None if not (error and 'parallel' in str(error).lower() and attempt < 4): break - print("Sharesight busy:", error, "- retrying", file=sys.stderr) + log.warn("Sharesight busy:", error, "- retrying") time.sleep(1 + attempt) if error: # an OAuth error describes itself too why = ': '.join(str(data[key]) for key in ('error', 'error_description') if data.get(key)) diff --git a/lib/shortman.py b/lib/shortman.py index 407ba905..5fd37a69 100644 --- a/lib/shortman.py +++ b/lib/shortman.py @@ -1,4 +1,4 @@ -import sys +from lib import log from lib import net def fetch(market_data): @@ -7,10 +7,10 @@ def fetch(market_data): try: r = net.get(url) except Exception as e: - print("Failure fetching", url, net.redact(e), file=sys.stderr) + log.warn("Failure fetching", url, net.redact(e)) return market_data # the ASX short interest is missing, the rest of the report is not if r.status_code != 200: - print(r.status_code, "error communicating with", url, file=sys.stderr) + log.warn(r.status_code, "error communicating with", url) return market_data csv = r.content.decode('utf-8') csv = csv.split('\r\n') diff --git a/lib/slack_common.py b/lib/slack_common.py index 4a0e6777..817c94a5 100644 --- a/lib/slack_common.py +++ b/lib/slack_common.py @@ -4,11 +4,11 @@ import html import json import re -import sys import time import requests +from lib import log from lib import config from lib import dialects from lib import inbound @@ -55,12 +55,12 @@ def parse(self, update): return self.slash(update) if kind == 'url_verification': response = json.dumps({'challenge': update['challenge']}) - print("replying with", response, file=sys.stderr) + log.info("replying with", response) return inbound.Response(body=response.encode()) if kind == 'block_actions': return self.press(update) if kind != 'event_callback': - print(f"[slack]: unhandled 'type' {kind!r}", file=sys.stderr) + log.info(f"[slack]: unhandled 'type' {kind!r}") return inbound.UNHANDLED event = update['event'] bot_user = update['authorizations'][0]['user_id'] # QWERTY @@ -68,7 +68,7 @@ def parse(self, update): event['user'] == bot_user: # not by a person: answering a bot (this one's replies included) could loop return inbound.OK if event['type'] not in ('message', 'app_mention') or 'text' not in event: - print("[slack]: unhandled event callback type", event['type'], file=sys.stderr) + log.info("[slack]: unhandled event callback type", event['type']) return inbound.UNHANDLED user = '<@' + event['user'] + '>' # a mention: Slack shows the name return inbound.Message(self.service, event['channel'], user, self.plain(event['text']), '@' + bot_user, user, @@ -124,7 +124,7 @@ def typing(self): self.working = True except Exception as e: self.working = False - print("Slack reactions.add failed:", net.redact(e), file=sys.stderr) + log.warn("Slack reactions.add failed:", net.redact(e)) def typing_done(self): """Take ⏳ off the message again; a failure is only logged.""" @@ -134,15 +134,14 @@ def typing_done(self): try: call('reactions.remove', channel=self.where, timestamp=self.message, name=self.WORKING) except Exception as e: - print("Slack reactions.remove failed:", net.redact(e), file=sys.stderr) + log.warn("Slack reactions.remove failed:", net.redact(e)) def post_text(self, text, buttons=None): """chat.postMessage, without link previews, and with buttons (a Block Kit message) where they work. Slack's answer, or None when it failed or refused (logged): Slack refuses with HTTP 200 and ok false (not_in_channel, say), so a refusal is read from the answer.""" if not config.env.slackBotToken: - print(f"ERROR slack: a reply to channel {self.where} needs slackBotToken in finbot.env; not sent", - file=sys.stderr) + log.error(f"ERROR slack: a reply to channel {self.where} needs slackBotToken in finbot.env; not sent") return None blocks = {} if buttons and not isinstance(buttons, dict) and self.takes_buttons: @@ -151,9 +150,9 @@ def post_text(self, text, buttons=None): answer = call('chat.postMessage', channel=self.where, text=text, unfurl_links='false', unfurl_media='false', **blocks) except Exception as e: - print("Slack chat.postMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) + log.error("Slack chat.postMessage to", self.where, "failed:", net.redact(e)) return None - print("OK slack message to", self.where, file=sys.stderr) + log.info("OK slack message to", self.where) return answer def post_chart(self, image, caption, buttons=None): @@ -167,20 +166,19 @@ def post_chart(self, image, caption, buttons=None): try: return self.post_blocks(data, caption, buttons) except Exception as e: - print("Slack chart with buttons failed, uploading it without them:", net.redact(e), file=sys.stderr) + log.warn("Slack chart with buttons failed, uploading it without them:", net.redact(e)) try: share = {'files': json.dumps([{'id': self.upload(data)}]), 'channel_id': self.where, 'initial_comment': caption} try: output = call('files.completeUploadExternal', **share) except requests.exceptions.ReadTimeout as e: - print("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e), - file=sys.stderr) + log.warn("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e)) return transports.MAYBE_SENT except Exception as e: - print("Slack chart upload failed:", net.redact(e), file=sys.stderr) + log.warn("Slack chart upload failed:", net.redact(e)) return None - print("OK slack chart upload to", self.where, file=sys.stderr) + log.info("OK slack chart upload to", self.where) return output def upload(self, data): @@ -206,7 +204,7 @@ def post_blocks(self, data, caption, buttons): answer = until_shown(lambda: call('chat.postMessage', channel=self.where, text=caption[:SECTION_LIMIT], blocks=json.dumps(blocks), unfurl_links='false', unfurl_media='false')) except requests.exceptions.ReadTimeout as e: - print("Slack chat.postMessage timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) + log.warn("Slack chat.postMessage timed out, the chart may still arrive:", net.redact(e)) return transports.MAYBE_SENT if answer.get('ts'): self.delete(self.shows(answer['ts'], file_id, caption, buttons)) @@ -227,7 +225,7 @@ def reshow(self, message, buttons): call('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], blocks=json.dumps(chart_blocks(caption, buttons, file_id))) except Exception as e: - print("Slack chat.update with an earlier upload failed, uploading anew:", net.redact(e), file=sys.stderr) + log.warn("Slack chat.update with an earlier upload failed, uploading anew:", net.redact(e)) return False self.delete(self.shows(message, file_id)) return True @@ -278,7 +276,7 @@ def delete(self, files): try: call('files.delete', file=file_id) except Exception as e: - print("Slack files.delete failed:", net.redact(e), file=sys.stderr) + log.warn("Slack files.delete failed:", net.redact(e)) def edit_text(self, message, text, buttons=None): """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" @@ -288,7 +286,7 @@ def edit_text(self, message, text, buttons=None): try: call('chat.update', **data) except Exception as e: - print("Slack chat.update failed:", net.redact(e), file=sys.stderr) + log.warn("Slack chat.update failed:", net.redact(e)) SECTION_LIMIT = 3000 # characters of text in a Block Kit section @@ -334,7 +332,7 @@ def until_shown(post, waits=(1, 2, 4)): except RuntimeError as e: if 'invalid_blocks' not in str(e): raise - print(f"Slack cannot show the chart yet ({e}), trying again in {wait} s", file=sys.stderr) + log.info(f"Slack cannot show the chart yet ({e}), trying again in {wait} s") time.sleep(wait) return post() diff --git a/lib/slack_websocket.py b/lib/slack_websocket.py index 01828071..ecf01b0f 100644 --- a/lib/slack_websocket.py +++ b/lib/slack_websocket.py @@ -1,8 +1,8 @@ """Slack's Socket Mode, its pull mode (no slackOutgoingWebhook): a WebSocket that the app-level token opens, whose envelopes carry what Slack would post to the Request URL, for the reader (slack_common.READER).""" import json -import sys +from lib import log from lib import config from lib import net from lib import pull @@ -89,7 +89,7 @@ def poll(self, take): return self.failed('the socket', e) finally: connection.close() - print(f"slack: Slack is closing the socket ({reason}), opening another", file=sys.stderr) + log.info(f"slack: Slack is closing the socket ({reason}), opening another") return 0 def listen(self, connection, take): diff --git a/lib/telegram_common.py b/lib/telegram_common.py index d0c33e6b..8aba0bd8 100644 --- a/lib/telegram_common.py +++ b/lib/telegram_common.py @@ -3,10 +3,10 @@ (TelegramBot).""" import html import json -import sys import requests +from lib import log from lib import buttons from lib import config from lib import dialects @@ -32,12 +32,12 @@ def try_call(method, **kwargs): try: return call(method, **kwargs) except Exception as e: - print(f"Telegram {method} failed:", net.redact(e), file=sys.stderr) + log.warn(f"Telegram {method} failed:", net.redact(e)) def deleteWebhook(): """Have Telegram stop posting updates. Those not delivered yet are kept, for the webhook set next or getUpdates.""" - print("telegram: deleting the webhook", file=sys.stderr) + log.info("telegram: deleting the webhook") return call('deleteWebhook', json={'drop_pending_updates': False}) @@ -52,7 +52,7 @@ def set_commands(commands): works without the menu.""" wanted = [{'command': command.name, 'description': command.description[:256]} for command in commands] if try_call('getMyCommands') != wanted and try_call('setMyCommands', json={'commands': wanted}) is not None: - print(f"telegram: {len(wanted)} commands in the menu", file=sys.stderr) + log.info(f"telegram: {len(wanted)} commands in the menu") def pinChatMessage(chat_id, message_id): @@ -106,7 +106,7 @@ def parse(self, update): elif self.IGNORED & message.keys(): return inbound.OK else: - print("[telegram]: unhandled: 'message' without 'text/photo'", file=sys.stderr) + log.info("[telegram]: unhandled: 'message' without 'text/photo'") return inbound.UNHANDLED return inbound.Message(self.service, chat_id, user, text, self.bot_name, user_name, str(message.get('message_id', ''))) @@ -145,9 +145,9 @@ def post_text(self, text, buttons=None): try: message = call('sendMessage', json=payload) except Exception as e: - print("Telegram sendMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) + log.error("Telegram sendMessage to", self.where, "failed:", net.redact(e)) return None - print("OK telegram message to", self.where, file=sys.stderr) + log.info("OK telegram message to", self.where) return message def post_chart(self, image, caption, buttons=None): @@ -159,12 +159,12 @@ def post_chart(self, image, caption, buttons=None): try: message = call('sendPhoto', data=data, files={'photo': ('image.png', image)}) except requests.exceptions.ReadTimeout as e: - print("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) + log.warn("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e)) return transports.MAYBE_SENT except Exception as e: - print("Telegram sendPhoto to", self.where, "failed:", net.redact(e), file=sys.stderr) + log.warn("Telegram sendPhoto to", self.where, "failed:", net.redact(e)) return None - print("OK telegram chart to", self.where, file=sys.stderr) + log.info("OK telegram chart to", self.where) return message def edit_chart(self, message, image, caption, buttons=None): @@ -184,7 +184,7 @@ def edit_chart(self, message, image, caption, buttons=None): if 'not modified' not in str(e): raise except Exception as e: # it may have arrived even so: no error line for something the chat may already show - print("Telegram editMessageMedia failed:", net.redact(e), file=sys.stderr) + log.warn("Telegram editMessageMedia failed:", net.redact(e)) def edit_text(self, message, text, buttons=None): """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" @@ -206,7 +206,7 @@ def pin(self, answers): try: sent.append((answer['chat']['id'], answer['message_id'])) except (KeyError, TypeError): - print("Unable to pin: unexpected response", file=sys.stderr) + log.warn("Unable to pin: unexpected response") for chat, message in reversed(sent): pinChatMessage(chat, message) diff --git a/lib/telegram_longpolling.py b/lib/telegram_longpolling.py index b95f8b78..a80b8464 100644 --- a/lib/telegram_longpolling.py +++ b/lib/telegram_longpolling.py @@ -37,7 +37,8 @@ def poll(self, take): if self.offset is not None: params['offset'] = self.offset try: - updates = telegram_common.call('getUpdates', json=params, timeout=(net.CONNECT_TIMEOUT, self.READ_SECONDS)) + updates = telegram_common.call('getUpdates', json=params, timeout=(net.CONNECT_TIMEOUT, self.READ_SECONDS), + logged=False) # asked again and again: the log would be nothing else except Exception as e: return self.failed('getUpdates', e) self.failures = 0 diff --git a/lib/telegram_webhook.py b/lib/telegram_webhook.py index 4de523ac..42561a97 100644 --- a/lib/telegram_webhook.py +++ b/lib/telegram_webhook.py @@ -1,9 +1,9 @@ """Telegram's webhook, its HTTP mode (telegramOutgoingWebhook set): Telegram posts each update to finbot's URL with the secret header that setWebhook gave it, for the reader (telegram_common.READER).""" import json -import sys from urllib.parse import urlparse +from lib import log from lib import config from lib import inbound from lib import telegram_common @@ -12,7 +12,7 @@ def setWebhook(): """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken. The log does not show the URL: its path is a secret.""" - print("telegram: setting the webhook (telegramOutgoingWebhook)", file=sys.stderr) + log.info("telegram: setting the webhook (telegramOutgoingWebhook)") params = {'url': config.env.telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), 'secret_token': config.env.telegramOutgoingToken} return telegram_common.call('setWebhook', params=params) diff --git a/lib/worker.py b/lib/worker.py index 03217f75..442688d8 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -5,8 +5,8 @@ import json import math import re -import sys import typing +from lib import log from lib import answer from lib import dialects from lib import state @@ -117,7 +117,7 @@ def process_request(message): try: ensure_keyboard(message.chat_id) # keep DM keyboards current after the reply except Exception as e: - print("keyboard update failed:", e, file=sys.stderr) + log.warn("keyboard update failed:", e) @dataclasses.dataclass class Args: @@ -559,7 +559,7 @@ def suggest(chat, text): if word.lower().startswith(typing.lower())] return [(shown, value) for shown, value in found if len(value) <= 100][:25] except Exception as e: - print('suggest:', text, delivery.error_line(e), file=sys.stderr) + log.warn('suggest:', text, delivery.error_line(e)) return [] def argument_words(command, chat, given): @@ -644,4 +644,4 @@ def refresh_keyboards(): try: push_keyboard(chat_id) except Exception as e: - print("keyboard refresh failed for", chat_id, e, file=sys.stderr) + log.warn("keyboard refresh failed for", chat_id, e) diff --git a/lib/yahoo.py b/lib/yahoo.py index 096256dc..0668eed1 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -5,7 +5,7 @@ import threading import typing import zoneinfo -import sys +from lib import log from lib import answer from lib import config from lib import cache @@ -37,13 +37,13 @@ def credentials(renew=False): try: r = net.get(url, headers={**HEADERS, 'Cookie': cookie}) except Exception as e: - print("Yahoo crumb:", net.redact(e), file=sys.stderr) + log.warn("Yahoo crumb:", net.redact(e)) continue if r.status_code == 200 and r.text.strip(): cache.write(CREDENTIALS, [cookie, r.text.strip()]) - print("Got a new Yahoo cookie and crumb", file=sys.stderr) + log.info("Got a new Yahoo cookie and crumb") return cookie, r.text.strip() - print(r.status_code, "from Yahoo for a crumb", file=sys.stderr) + log.warn(r.status_code, "from Yahoo for a crumb") raise RuntimeError("Yahoo gave no crumb") def _hosts(path): @@ -62,14 +62,14 @@ def _answer(query, what): r = net.get(url, headers={**HEADERS, 'Cookie': cookie}) except Exception as e: refusal = f"Yahoo unreachable for {what}: {net.redact(e)}" - print(refusal, file=sys.stderr) + log.warn(refusal) continue if r.status_code == 200: return r.json() if r.status_code == 404 and not_found(r): return None refusal = f"{r.status_code} error for {what} from Yahoo" - print(refusal + ':', r.text.strip()[:200], file=sys.stderr) + log.warn(refusal + ':', r.text.strip()[:200]) if not refusal.startswith('401 '): # only a refused crumb is worth a new one break raise RuntimeError(refusal) @@ -143,13 +143,13 @@ def fetch(tickers): """v7 quotes for many tickers in one request: {ticker: {...}}. Cached for 5 minutes (DO NOT CACHE LONGER). A ticker Yahoo gives no name or price for is left out. Raises RuntimeError when Yahoo cannot be reached.""" if not tickers: - print("no tickers provided to yahoo.fetch()", file=sys.stderr) + log.info("no tickers provided to yahoo.fetch()") return {} tickers = sorted(set(tickers)) # de-dupe name = "finbot_yahoo_quote_" + hashlib.sha256("_".join(tickers).encode()).hexdigest() + '.json' data = cache.cached(name, 300, lambda: _quote_answer(tickers)) if data is None: - print("Yahoo has no quotes for", ', '.join(tickers), file=sys.stderr) + log.warn("Yahoo has no quotes for", ', '.join(tickers)) return {} # callers iterate the result now = datetime.datetime.now().timestamp() yahoo_output = {} @@ -226,12 +226,12 @@ def fetch_detail(ticker, seconds=None): data = cache.cached("finbot_yahoo_detail_" + ticker + '.json', seconds, lambda: _detail_answer(ticker)) result = dig(data, 'quoteSummary', 'result', 0) if result is None: # Yahoo answers 404 Not Found, or result=null, for a symbol it does not know - print(ticker, "is unknown to Yahoo", file=sys.stderr) + log.warn(ticker, "is unknown to Yahoo") return {} title = next((name for name in (dig(result, 'price', 'longName'), dig(result, 'price', 'shortName')) if isinstance(name, str) and name), None) if not title: # catches some delisted stocks like "DUB" - print(ticker, "has no name at Yahoo", file=sys.stderr) + log.warn(ticker, "has no name at Yahoo") return {} detail = {'profile_title': formats.transform_title(title)} for key, path, *convert in DETAIL_FIELDS: @@ -420,7 +420,7 @@ def fetch_details(tickers, strict=False): if isinstance(detail, Exception): if strict: raise detail - print(ticker, detail, file=sys.stderr) + log.warn(ticker, detail) else: details |= detail return details @@ -452,7 +452,7 @@ def prefetch_history(tickers, workers=8, seconds=None): failed = [] for ticker, result in zip(tickers, fetch_each(lambda t: fetch_chart_json(t, seconds=seconds), tickers, workers)): if isinstance(result, Exception): - print("prefetch failed for", ticker, result, file=sys.stderr) + log.warn("prefetch failed for", ticker, result) failed.append(ticker) return failed diff --git a/tests/test_config.py b/tests/test_config.py index b706a4cc..79ec7701 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -134,6 +134,17 @@ def test_a_blank_value_means_the_default(self): self.assertEqual(values.exchange_suffix, suffix) self.assertEqual(config.preference_values(config._getenv_for({})).exchange_suffix, 'AX') # unset: the default + def test_log_level_is_one_of_syslogs_levels_in_any_case(self): + """log_level takes ERROR, WARN, INFO or DEBUG, in any case, INFO by default; anything else is a mistake that + names the levels.""" + for given, level in (('warn', 'WARN'), ('Debug', 'DEBUG'), ('ERROR', 'ERROR'), ('', 'INFO')): + with self.subTest(log_level=given): + values = config.preference_values(config._getenv_for({'log_level': given})) + self.assertEqual(values.log_level, level) + with self.assertRaises(config.ConfigError) as caught: + config.preference_values(config._getenv_for({'log_level': 'loud'})) + self.assertEqual(caught.exception.errors, ["log_level='loud' is not one of ERROR, WARN, INFO, DEBUG"]) + def test_a_retired_key_is_accepted_and_said_to_do_nothing(self): """A key finbot no longer reads (alliterate in preferences.conf, slack_webhook in finbot.env, say) does not stop a start, as an unknown key would: it is ignored, and the log at start (and check_config) says what to do @@ -229,9 +240,8 @@ def write(self, text): def test_a_change_is_seen_at_once(self): """A reload applies every change, says which, and swaps in a new object, so no request sees half a reload.""" self.write('future_days=45\nprice_percent=7\nshorts_percent=12\nexclude_portfolios=1156263\n' - 'exchange_suffix=nz\nhyperlinkProvider=yahoo\n' - 'hyperlinkFooter=false\ndebug=1\ngraph=0\ndemote_volatile=0\nvolatility_multiplier=3\npast_days=14\n' - 'min_analysts=3\n') + 'exchange_suffix=nz\nhyperlinkProvider=yahoo\nhyperlinkFooter=false\nlog_level=debug\ngraph=0\n' + 'demote_volatile=0\nvolatility_multiplier=3\npast_days=14\nmin_analysts=3\n') before = config.prefs changes = config.reload() self.assertEqual(len(changes), 13) @@ -242,8 +252,8 @@ def test_a_change_is_seen_at_once(self): prefs = config.prefs self.assertEqual((prefs.exclude_portfolios, prefs.exchange_suffix, prefs.hyperlinkProvider, prefs.hyperlinkFooter), (('1156263',), 'NZ', 'yahoo', False)) - self.assertEqual((config.prefs.debug, config.prefs.graph, config.prefs.demote_volatile, - config.prefs.volatility_multiplier), (True, False, False, 3.0)) + self.assertEqual((config.prefs.log_level, config.prefs.graph, config.prefs.demote_volatile, + config.prefs.volatility_multiplier), ('DEBUG', False, False, 3.0)) self.assertEqual(config.reload(), []) # nothing more to change def test_a_value_removed_from_the_file_goes_back_to_its_default(self): @@ -395,7 +405,7 @@ def test_a_bad_startup_value_refuses_to_start(self): def test_a_preference_in_the_environment_is_refused_by_the_bot(self): """A preference in the bot's environment (finbot.env) is refused, but a job process gets its preferences that way, and startup keys belong there.""" - with mock.patch.dict(os.environ, {'future_days': '30', 'debug': '1'}): + with mock.patch.dict(os.environ, {'future_days': '30', 'log_level': 'DEBUG'}): with self.assertRaises(config.ConfigError) as caught: config.check_startup_environment() self.assertEqual(len(caught.exception.errors), 2) @@ -502,8 +512,8 @@ def test_startup_report_is_a_short_count_never_names_or_values(self): with mock.patch.dict(os.environ, {'port': '5000'}, clear=True): self.assertEqual(config.startup_report()[0], 'startup environment (1 key set)') conf = Path(common.temp_dir(self), 'preferences.conf') - conf.write_text('debug = true\n') - for source, count in (({'debug': 'true'}, '1 key'), ({'debug': 'true', 'graph': 'false'}, '2 keys')): + conf.write_text('graph = false\n') + for source, count in (({'graph': 'false'}, '1 key'), ({'log_level': 'DEBUG', 'graph': 'false'}, '2 keys')): with self.subTest(source=source), mock.patch.object(config, 'preferences_path', str(conf)), \ mock.patch.object(config, 'prefs', dataclasses.replace(config.prefs, source=source)): self.assertEqual(config.startup_report()[1], f'{conf} ({count} set; SIGHUP to reload)') @@ -544,13 +554,13 @@ def test_a_reload_points_a_key_set_in_the_template_at_the_file_to_set_it_in(self file.write_text('') self.addCleanup(self.restore, file) template = Path(common.temp_dir(self), 'template.conf') - template.write_text('#past_days=28\ndebug=True\n') + template.write_text('#past_days=28\nlog_level=DEBUG\n') with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(config, 'TEMPLATE', template): with contextlib.redirect_stderr(io.StringIO()) as err: bot.reload_all() - self.assertIn(f'ERROR config: defaults/preferences.conf sets debug, but finbot never reads it: edit {file}' + self.assertIn(f'WARN config: defaults/preferences.conf sets log_level, but finbot never reads it: edit {file}' ' instead', err.getvalue()) - self.assertFalse(config.prefs.debug) + self.assertEqual(config.prefs.log_level, 'INFO') class Tools(unittest.TestCase): diff --git a/tests/test_edges.py b/tests/test_edges.py index 30f529d0..26f52663 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -3,6 +3,7 @@ import contextlib, io, os, re, signal, subprocess, sys, tempfile, threading, time, unittest from pathlib import Path from tests.common import * +from lib import log COMMANDS = ('.price', '.price 7d', '.price top', '.session', '.premarket', '.performance 7d', '.shorts', '.peg', '.buy', @@ -361,6 +362,68 @@ def test_a_flush_writes_a_line_that_has_not_ended(self): self.assertEqual(stream.getvalue(), 'partial') # the stream's own +class LogLevels(FinbotCase): + """log_level: each of syslog's levels lets in its own lines and those of the levels above it.""" + + def logged(self, level, run): + """What run() logs with log_level at this level.""" + self.prefs(log_level=level) + with contextlib.redirect_stderr(io.StringIO()) as err: + run() + return err.getvalue().splitlines() + + def test_each_level_lets_in_its_own_lines_and_those_above_it(self): + """An error always reaches the log; WARN adds the warnings, INFO what finbot does, DEBUG its detail.""" + def each(): + """A line at each level.""" + log.error('e') + log.warn('w') + log.info('i') + log.debug('d') + for level, shown in (('ERROR', 'e'), ('WARN', 'ew'), ('INFO', 'ewi'), ('DEBUG', 'ewid')): + with self.subTest(log_level=level): + self.assertEqual(self.logged(level, each), list(shown)) + + def test_a_cache_miss_is_info_and_a_hit_or_a_write_debug(self): + """A cache miss, of a file not there or too old, is logged at INFO; a hit, and the write after a miss, at + DEBUG.""" + def twice(): + """A miss and its write, then a hit, then a miss of a file too old.""" + cache.cached('finbot_test.json', 60, lambda: {'a': 1}) + cache.cached('finbot_test.json', 60, lambda: {'a': 1}) + cache.cached('finbot_test.json', -1, lambda: {}) + where = cache.path('finbot_test.json') + self.assertEqual(self.logged('INFO', twice), [f'cache miss: {where}', f'cache miss: {where}']) + os.remove(where) + lines = self.logged('DEBUG', twice) + kinds = [line.split(':')[0] for line in lines] + self.assertEqual(kinds, ['cache miss', 'cache write', 'cache hit', 'cache miss']) + self.assertRegex(lines[2], r'^cache hit: \S+finbot_test\.json TTL: \d+s$') + + +class RequestLog(FinbotCase): + """Each request that goes out (a cache miss, or a call that has no cache) is a line at INFO.""" + + def test_each_request_is_a_short_line_without_secrets(self): + """A request logs its method and URL at INFO, without the scheme or a secret, cut to fit; Telegram's long + polling (logged=False) does not, and nothing does below INFO.""" + with contextlib.redirect_stderr(io.StringIO()) as err: + net.get('https://query1.finance.yahoo.com/v8/finance/chart/AAPL?range=1y&crumb=s3cret') + net.post('https://api.telegram.org/bot123:ABC-def/sendMessage', json={}) + net.request('PATCH', 'https://discord.com/api/v10/channels/7/messages/9', json={}) + net.post('https://api.telegram.org/bot123:ABC-def/getUpdates', json={}, logged=False) + net.get('https://example.com/' + 'x' * 200) + lines = err.getvalue().splitlines() + self.assertEqual(lines[:3], ['GET query1.finance.yahoo.com/v8/finance/chart/AAPL?range=1y&crumb=', + 'POST api.telegram.org/bot/sendMessage', 'PATCH discord.com/api/v10/channels/7/messages/9']) + self.assertEqual((len(lines), lines[3][:16], lines[3][-1], len(lines[3])), + (4, 'GET example.com/', '…', len('GET ') + net.SHOWN)) + self.prefs(log_level='WARN') + with contextlib.redirect_stderr(io.StringIO()) as err: + net.get('https://example.com/') + self.assertEqual(err.getvalue(), '') + + class SharedSession(unittest.TestCase): def test_the_session_keeps_no_cookies(self): """Yahoo's cookie goes in a header: a stored one would go to fc.yahoo.com, which then sets no new one.""" diff --git a/tests/test_finbot.py b/tests/test_finbot.py index 3adb024c..baf9d912 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -47,7 +47,8 @@ def test_reminder_runs(self): reminder.run() def test_bot_handler(self): - """A sticker is answered 200 and not logged; git_version matches what git says.""" + """A sticker is answered 200, with nothing logged but the request's line (INFO); git_version matches what git + says.""" def post(message): """Post a Telegram message update to the WSGI handler: (its answer, what it logged).""" body = json.dumps({'update_id': 1, 'message': dict({'message_id': 5, 'chat': {'id': 55, 'type': 'private'}, @@ -57,7 +58,7 @@ def post(message): with contextlib.redirect_stderr(err): out = bot.main(env, lambda *a, **k: None) return out, err.getvalue() - self.assertEqual(post({'sticker': {'file_id': 'x'}}), ([b''], '')) # stickers: silent, not logged + self.assertEqual(post({'sticker': {'file_id': 'x'}}), ([b''], '[http] request for telegram answered 200 OK\n')) self.assertGreaterEqual(len(bot.git_version()), 7) head = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=bot.__file__.rsplit('/', 1)[0], capture_output=True, text=True).stdout.strip() diff --git a/tests/test_pull.py b/tests/test_pull.py index 4ba04294..908a4975 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -45,9 +45,9 @@ def test_each_update_is_handed_on_in_order_and_the_next_asked_for_after_it(self) self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) self.assertEqual(taken, [(telegram_common.READER, message), (telegram_common.READER, callback_query(11))]) allowed = ['message', 'callback_query'] - self.assertEqual(self.calls, [ - ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed}, 'timeout': (5, 65)}), - ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed, 'offset': 12}, 'timeout': (5, 65)})]) + asked = {'timeout': (5, 65), 'logged': False} # not logged: it is asked again and again + self.assertEqual(self.calls, [('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed}, **asked}), + ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed, 'offset': 12}, **asked})]) def test_failures_are_waited_out_longer_each_time_and_logged_without_the_token(self): """A failed getUpdates (a 409 when a webhook is set or another finbot polls, the network down) waits 5 seconds, From 8ca633517f743367da43fae9ef8bd0cf547a4a35 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:29:23 +0000 Subject: [PATCH 24/31] util/flush.py flushes the cache by age, or all of it; finbot-flush.service goes util/flush.py --before 180s the files written more than 180 seconds ago util/flush.py --after 24h the files written in the last 24 hours util/flush.py --after 7d ... 7 days (3m: 3 months, 1y: a year) util/flush.py all every file A time is a number and s, h, d, w, m (months) or y: formats.seconds_from_human, the period words of commands (days_from_human_days) plus seconds and hours. On its own the tool prints its help with these examples. It deletes only finbot's files in DATA_DIR/cache (cleanup.remove, which skips a temp file being written), never the state in DATA_DIR, and the bot can keep running; --data-dir names finbot's data dir when the shell has no DATA_DIR. SIGUSR1 still flushes everything and refills the cache at once. doc/caching.md and doc/docker.md say how. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 3 +- doc/caching.md | 18 +++++----- doc/docker.md | 1 + finbot-flush.service | 7 ---- lib/formats.py | 8 +++++ lib/reports/cleanup.py | 34 ++++++++++++------- tests/test_datadir.py | 66 +++++++++++++++++++++++++++++++++++-- tests/test_util.py | 11 +++++++ util/flush.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 191 insertions(+), 31 deletions(-) delete mode 100644 finbot-flush.service create mode 100755 util/flush.py diff --git a/CLAUDE.md b/CLAUDE.md index 3b58853f..abe74e8e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,7 +90,8 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm goes by. `lib/formats.py`: names, units, dates, percents in words. `lib/links.py`: stock links. `lib/render.py`: every chart, `Chart` specs and `draw()`. - `lib/yahoo.py lib/sharesight.py lib/shortman.py lib/simplywallst.py`: data sources. Yahoo's and Sharesight's answers - are cached in `DATA_DIR/cache` as sent, not parsed, so a parser fix needs no flush (`cache.cached`). + are cached in `DATA_DIR/cache` as sent, not parsed, so a parser fix needs no flush (`cache.cached`); `util/flush.py` + deletes cache files by when they were written (`--before`/`--after TIME`, or `all`), never the state. `yahoo.symbol(typed)` is Yahoo's symbol for a ticker as written, tried on the local exchange (`exchange_suffix`) when Yahoo does not know it bare: every ticker a chat gives goes through it. `sharesight.portfolios(name=None)` gives `Portfolio` objects: `holdings()`, `tickers()`, `gain(days)` (the portfolio's), `trades(days)`. diff --git a/doc/caching.md b/doc/caching.md index 72701307..40d53431 100644 --- a/doc/caching.md +++ b/doc/caching.md @@ -11,16 +11,16 @@ performance_cache_seconds=2400 Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The weekly system job `cleanup` removes cache files after 30 days. -To flush the cache, so that everything is fetched afresh, signal the running bot: it deletes its cache files (Yahoo and Sharesight -answers, charts, the Yahoo cookie) and keeps its state (watchlist, jobs, seen trades, ratings), then runs both warm jobs -(`sharesight_warm`, `yahoo_warm`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: +To flush the cache, so that what it holds is fetched afresh, run `util/flush.py` with the files to delete, by when they were written. With no argument it prints its help: ``` -sudo systemctl kill -s USR1 finbot # systemd -sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) -sudo docker compose kill -s USR1 finbot # Docker -kill -USR1 # anything else +util/flush.py --before 180s # the files written more than 180 seconds ago +util/flush.py --after 24h # the files written in the last 24 hours +util/flush.py --after 7d # ... in the last 7 days (3m: 3 months, 1y: a year) +util/flush.py all # every file +sudo docker compose exec finbot util/flush.py all # in Docker ``` -The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && -sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, deleting the files in `DATA_DIR/cache` does the same. +A time is a number and `s` (seconds), `h` (hours), `d` (days), `w` (weeks), `m` (months) or `y` (years). It deletes only finbot's files in `DATA_DIR/cache` (Yahoo's and Sharesight's answers, the charts, the Yahoo cookie), never the state in `DATA_DIR` (the watchlists, jobs, plans, seen trades and ratings), and the bot can keep running: it fetches a file that is gone afresh. Run it as the user finbot runs as, or as root, with finbot's data dir: `DATA_DIR` as `finbot.env` sets it (`sudo DATA_DIR=/var/lib/finbot util/flush.py all`), `--data-dir /var/lib/finbot`, or else `var/` in the checkout. + +Signalling the bot flushes every file too, and then runs both warm jobs (`sharesight_warm`, `yahoo_warm`) at once to fill the cache again: `sudo systemctl kill -s USR1 finbot`, or `sudo docker compose kill -s USR1 finbot`. The log says `cache: flushed N files`, then `cache: warm jobs done`. Two system jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](scheduling.md)): `sharesight_warm` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). diff --git a/doc/docker.md b/doc/docker.md index 20b2e06c..61521071 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -73,6 +73,7 @@ file beside `compose.yaml`), and read it with `sudo docker compose logs -f`. ``` sudo docker compose kill -s HUP finbot # re-read preferences.conf sudo docker compose kill -s USR1 finbot # flush the cache, and fill it again +sudo docker compose exec finbot util/flush.py --after 24h # flush the files of the last day (see caching.md) cd /opt/finbot && sudo git pull && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build ``` A change to `/opt/finbot.env` needs a restart: `sudo docker compose up -d`. diff --git a/finbot-flush.service b/finbot-flush.service deleted file mode 100644 index d862bc0f..00000000 --- a/finbot-flush.service +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Flush finbot's cache (sudo systemctl start finbot-flush) - -[Service] -Type=oneshot -# finbot deletes its cache files on SIGUSR1, keeps its state, and fills the cache again with the warm jobs -ExecStart=/bin/systemctl kill --signal=USR1 finbot.service diff --git a/lib/formats.py b/lib/formats.py index 00b21c1f..52f58381 100644 --- a/lib/formats.py +++ b/lib/formats.py @@ -152,6 +152,14 @@ def days_from_human_days(arg): pass return int(arg.removesuffix('Y')) * 365 +def seconds_from_human(arg): + """180s, 24h, or any period that days_from_human_days takes (7d, 2w, 3m, 1y, ytd, 7) -> seconds. Raises ValueError + for anything else.""" + for suffix, seconds in (('s', 1), ('h', 3600)): + if arg.lower().endswith(suffix) and arg[:-1].isdigit(): + return int(arg[:-1]) * seconds + return days_from_human_days(arg) * 86400 + def ordinal(num): """The ordinal suffix of a whole number: 'st' for 1, 'nd' for 22, 'th' for 11 and 13.""" return 'th' if num % 100 in (11, 12, 13) else {1: 'st', 2: 'nd', 3: 'rd'}.get(num % 10, 'th') diff --git a/lib/reports/cleanup.py b/lib/reports/cleanup.py index 5ab07c21..615ad4a9 100644 --- a/lib/reports/cleanup.py +++ b/lib/reports/cleanup.py @@ -1,27 +1,37 @@ -"""Deletes old cache files (scheduled: cleanup [days]); with days=0 it is the cache flush.""" +"""Deletes old cache files (scheduled: cleanup [days]); with days=0 it is the cache flush. util/flush.py deletes +them by age too (remove).""" import os, time from lib import log from lib import config def run(*, days=30): """Delete the cache files (finbot_*) that were last written more than `days` days ago (days=0: all of them, a - flush), but not a temp file written in the last minute: that is an atomic write in flight (cache.write_file).""" + flush); see remove.""" + return remove(before=time.time() - days * 86400) + +def remove(before=None, after=None, directory=None): + """Delete the cache files (finbot_*) in directory (the cache dir) last written before the time `before`, or after + the time `after` (epoch seconds; neither: all of them), but not a temp file written in the last minute: that is an + atomic write in flight (cache.write_file). Nothing else in it, and nothing in the data dir. Returns how many.""" now = time.time() - cutoff = now - days * 86400 removed = 0 try: - entries = list(os.scandir(config.env.cache_dir)) + entries = list(os.scandir(directory or config.env.cache_dir)) except FileNotFoundError: return 0 for entry in entries: - if entry.is_file(follow_symlinks=False) and entry.name.startswith('finbot_') and entry.stat().st_mtime < cutoff: - if entry.name.endswith('.tmp') and entry.stat().st_mtime > now - 60: - continue - try: - os.unlink(entry.path) - removed += 1 - except OSError as e: - log.warn("cleanup: can't remove", entry.path, e) + if not (entry.is_file(follow_symlinks=False) and entry.name.startswith('finbot_')): + continue + written = entry.stat().st_mtime + if (before is not None and written >= before) or (after is not None and written <= after): + continue + if entry.name.endswith('.tmp') and written > now - 60: + continue + try: + os.unlink(entry.path) + removed += 1 + except OSError as e: + log.warn("cleanup: can't remove", entry.path, e) return removed ARGUMENTS = '[DAYS]' # what a scheduled run takes diff --git a/tests/test_datadir.py b/tests/test_datadir.py index f740a37c..95701d8c 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -2,8 +2,9 @@ from unittest import mock from pathlib import Path from lib import config -from util import sync_defaults -from tests.common import FinbotCase, ROOT, bare_environment +import contextlib, io +from util import flush, sync_defaults +from tests.common import FinbotCase, ROOT, bare_environment, temp_dir REPO = Path(config.__file__).resolve().parent.parent @@ -136,3 +137,64 @@ def test_sync_defaults_runs_with_any_python_and_creates_nothing_unasked(self): self.assertEqual(made.returncode, 0, made.stderr) self.assertTrue((dest / 'preferences.conf').exists()) self.assertFalse(Path(tmp, 'never').exists()) + + +class Flush(FinbotCase): + """util/flush.py: the cache files written before or after a time ago, or all of them; never the state.""" + + def setUp(self): + """A data dir of this test's own: a state file, and cache files written 60 seconds, 400 seconds and 8 days ago, + with a file in the cache that is not finbot's.""" + super().setUp() + self.data = Path(temp_dir(self)) + self.cache = self.data / 'cache' + self.cache.mkdir() + (self.data / 'finbot_watchlist.json').write_text('{}') + + def make(self): + """The cache files, written that long ago.""" + for name, ago in (('finbot_new.json', 60), ('finbot_old.json', 400), ('finbot_week.json', 8 * 86400), + ('other.txt', 400)): + path = self.cache / name + path.write_text('{}') + os.utime(path, (time.time() - ago,) * 2) + + def flush(self, *args): + """Run the tool on this data dir: (its exit status, what it printed, the cache files left).""" + with contextlib.redirect_stdout(io.StringIO()) as out: + status = flush.main([*args, '--data-dir', str(self.data)]) + return status, out.getvalue().strip(), sorted(path.name for path in self.cache.iterdir()) + + def test_before_after_or_all_and_never_the_state(self): + """--before TIME deletes the cache files written more than TIME ago, --after TIME those written in the last + TIME, all every one of them; nothing but finbot's files in the cache, and nothing in the data dir.""" + for args, left in ((['--before', '180s'], ['finbot_new.json', 'other.txt']), + (['--after', '24h'], ['finbot_week.json', 'other.txt']), + (['--after', '7d'], ['finbot_week.json', 'other.txt']), (['--after', '1y'], ['other.txt']), + (['all'], ['other.txt'])): + with self.subTest(args=args): + self.make() + status, said, files = self.flush(*args) + self.assertEqual((status, files), (0, left)) + self.assertEqual(said, f"flushed {4 - len(left)} files from {self.cache}") + self.assertTrue((self.data / 'finbot_watchlist.json').exists()) + + def test_alone_it_shows_its_help_and_a_time_is_required(self): + """On its own it prints its help, with the examples; a time it does not know, or two of --before, --after and + all, is refused.""" + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(flush.main([]), 2) + for example in ('--before 180s', '--after 24h', '--after 7d', '--after 3m', '--after 1y', 'util/flush.py all'): + self.assertIn(example, out.getvalue()) + for args in (['--after', '7x'], ['all', '--before', '1h'], ['--before', '1h', '--after', '2h'], ['some']): + with self.subTest(args=args), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + flush.main(args) + + def test_it_runs_on_its_own_and_creates_nothing(self): + """As a script, it flushes the data dir it is given and creates no data dir of its own.""" + self.make() + env = {**bare_environment(), 'DATA_DIR': str(self.data / 'never')} + run = subprocess.run([sys.executable, 'util/flush.py', 'all', '--data-dir', str(self.data)], cwd=ROOT, env=env, + capture_output=True, text=True, timeout=60) + self.assertEqual((run.returncode, run.stdout.strip()), (0, f'flushed 3 files from {self.cache}'), run.stderr) + self.assertFalse((self.data / 'never').exists()) diff --git a/tests/test_util.py b/tests/test_util.py index 65c43ad5..8c1b116c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -19,6 +19,17 @@ def test_days_from_human_days_rejects_non_periods(self): with self.subTest(arg=arg), self.assertRaises(ValueError): formats.days_from_human_days(arg) + def test_seconds_from_human(self): + """A time in seconds from 180s or 24h, or any period in days (7d, 2w, 3m, 1y, 7); anything else is a + ValueError.""" + for arg, seconds in (('180s', 180), ('24H', 86400), ('7d', 7 * 86400), ('2w', 14 * 86400), ('3m', 90 * 86400), + ('1y', 365 * 86400), ('7', 7 * 86400), ('0s', 0)): + with self.subTest(arg=arg): + self.assertEqual(formats.seconds_from_human(arg), seconds) + for arg in ('', 's', 'h5', '1.5h', '-5s', '5x', 'AAPL'): + with self.subTest(arg=arg), self.assertRaises(ValueError): + formats.seconds_from_human(arg) + def test_transform_to_yahoo(self): """Sharesight and Google tickers as Yahoo's: suffixes by market, none for US venues, a dash for class shares.""" for args, expected in ((('AAPL',), 'AAPL'), (('FANG', 'ASX'), 'FANG.AX'), (('ARM', 'NASDAQ'), 'ARM'), diff --git a/util/flush.py b/util/flush.py new file mode 100755 index 00000000..37607242 --- /dev/null +++ b/util/flush.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Flush finbot's cache: delete the files in DATA_DIR/cache (Yahoo's and Sharesight's answers, the charts, the Yahoo +cookie) last written before or after a time ago, or all of them, so that what they held is fetched afresh. Never the +state in DATA_DIR itself: the watchlists, jobs, plans and records. The bot may keep running: it fetches a file that is +gone afresh.""" + +import argparse +import os +import sys +import time +from pathlib import Path + +if __name__ == '__main__': + os.environ['FINBOT_CHECK_ONLY'] = '1' # importing lib.config makes no data dir: --data-dir may name another +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from lib import config +from lib import formats +from lib.reports import cleanup + +EXAMPLES = """examples: + util/flush.py --before 180s the files written more than 180 seconds ago + util/flush.py --after 24h the files written in the last 24 hours + util/flush.py --after 7d ... in the last 7 days + util/flush.py --after 3m ... in the last 3 months + util/flush.py --after 1y ... in the last year + util/flush.py all every file + +A TIME is a number and s (seconds), h (hours), d (days), w (weeks), m (months: 30 days) or y (years). +DATA_DIR is finbot's, as finbot.env sets it, or var/ in the checkout; --data-dir names another. In Docker: + sudo docker compose exec finbot util/flush.py all""" + + +def age(text): + """A TIME ago in seconds, for argparse: 180s, 24h, 7d, 2w, 3m or 1y.""" + try: + seconds = formats.seconds_from_human(text) + except ValueError: + seconds = -1 + if seconds < 0: + raise argparse.ArgumentTypeError(f'{text!r} is not a time: a number and s, h, d, w, m or y (180s, 24h, 7d)') + return seconds + + +def parser(): + """The arguments: --before TIME, --after TIME or all; and --data-dir.""" + ap = argparse.ArgumentParser(prog='util/flush.py', description=__doc__, epilog=EXAMPLES, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('all', nargs='?', choices=['all'], metavar='all', help='every cache file') + ap.add_argument('--before', metavar='TIME', type=age, help='the files written more than TIME ago') + ap.add_argument('--after', metavar='TIME', type=age, help='the files written in the last TIME') + ap.add_argument('--data-dir', metavar='DIR', help='the data dir whose cache to flush (default: DATA_DIR, or var/)') + return ap + + +def main(argv=None): + """Flush what the arguments ask for and say how many files went; with none, the help. Returns the exit status.""" + argv = sys.argv[1:] if argv is None else argv + ap = parser() + if not argv: + ap.print_help() + return 2 + args = ap.parse_args(argv) + if [args.all, args.before, args.after].count(None) != 2: + ap.error('give one of --before TIME, --after TIME or all') + cache = os.path.join(args.data_dir, 'cache') if args.data_dir else config.env.cache_dir # as config makes it + now = time.time() + removed = cleanup.remove(before=None if args.before is None else now - args.before, + after=None if args.after is None else now - args.after, directory=cache) + print(f"flushed {removed} file{'' if removed == 1 else 's'} from {cache}") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From dc8cb2a4df7781f77fc20d9b500cd45565b6c580 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:30:28 +0000 Subject: [PATCH 25/31] Cache misses log at DEBUG, with hits and writes At INFO, the request that a miss sends is already a line of its own ("GET query1.finance.yahoo.com/..."), so "cache miss" moves to DEBUG beside "cache hit" and "cache write". The docs, the template's comment and CLAUDE.md say so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 2 +- defaults/preferences.conf | 2 +- doc/configuration.md | 2 +- lib/cache.py | 2 +- lib/log.py | 4 ++-- tests/test_edges.py | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index abe74e8e..8871df9a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -136,7 +136,7 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm tabs counting 4 (an edit quotes whole lines); a test checks all but README.md, CLAUDE.md and `doc/`. - Log with `lib/log.py`, at a level, with print()'s arguments: `log.error` (failed and lost: a message not sent), `log.warn` (failed and got over: a retry, a fallback, a refused request), `log.info` (what finbot does: a message - sent, a job run, a cache miss; `net` logs each request it sends), `log.debug` (cache hits and writes). The + sent, a job run; `net` logs each request it sends), `log.debug` (cache misses, hits and writes). The `log_level` preference sets what reaches the log; `lib/config.py` prints its own errors (it loads before preferences exist). Everything goes to stderr: systemd discards stdout (`StandardOutput=null`), so the bot and its reports never print there (a test checks); only the util/ tools, run by hand, do. diff --git a/defaults/preferences.conf b/defaults/preferences.conf index aaa49751..4440d9e1 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -52,6 +52,6 @@ #http_timeout=18 # Diagnostics. log_level: what the log shows, that level and those above it: ERROR, WARN, INFO (what finbot does, -# and each request it sends) or DEBUG (cache hits and writes too). +# and each request it sends) or DEBUG (each cache miss, hit and write too). #log_level=INFO #print_headers=False diff --git a/doc/configuration.md b/doc/configuration.md index 0e18e2db..e209682e 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -75,7 +75,7 @@ A change takes effect on a reload. A true/false preference takes `true`/`false`, | `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | | `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm` runs, so the holdings never run out | | `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | -| `log_level` | `INFO` | What the log shows: that level and those above it, syslog style. `ERROR`: what failed and was lost (a message not sent); `WARN`: what failed and was got over (a retry, a fallback, a request refused); `INFO`: what finbot does (each message sent, each job run, each request sent and answered, each cache miss); `DEBUG`: cache hits and writes too. Replaces `debug` | +| `log_level` | `INFO` | What the log shows: that level and those above it, syslog style. `ERROR`: what failed and was lost (a message not sent); `WARN`: what failed and was got over (a retry, a fallback, a request refused); `INFO`: what finbot does (each message sent, each job run, each request sent and answered); `DEBUG`: each cache miss, hit and write too. Replaces `debug` | | `print_headers` | `False` | Log inbound request headers | A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. diff --git a/lib/cache.py b/lib/cache.py index 86ecf2e5..280ad7f9 100644 --- a/lib/cache.py +++ b/lib/cache.py @@ -30,7 +30,7 @@ def _fresh(name, seconds): seconds = config.prefs.cache_seconds where, written = path(name), age(name) if written is None or written >= seconds: - log.info("cache miss:", where) + log.debug("cache miss:", where) return None log.debug(f"cache hit: {where} TTL: {seconds - written:.0f}s") return where diff --git a/lib/log.py b/lib/log.py index 4a44b906..46eb9c97 100644 --- a/lib/log.py +++ b/lib/log.py @@ -30,10 +30,10 @@ def warn(*parts, **options): def info(*parts, **options): - """What finbot does: a message sent, a job run, a cache miss, each request it sends and each it answers.""" + """What finbot does: a message sent, a job run, each request it sends (a cache miss's) and each it answers.""" write('INFO', parts, options) def debug(*parts, **options): - """The detail of what it does, to find a fault: a cache hit, a cache write.""" + """The detail of what it does, to find a fault: a cache miss, hit or write.""" write('DEBUG', parts, options) diff --git a/tests/test_edges.py b/tests/test_edges.py index 26f52663..027e1cfb 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -384,16 +384,16 @@ def each(): with self.subTest(log_level=level): self.assertEqual(self.logged(level, each), list(shown)) - def test_a_cache_miss_is_info_and_a_hit_or_a_write_debug(self): - """A cache miss, of a file not there or too old, is logged at INFO; a hit, and the write after a miss, at - DEBUG.""" + def test_a_cache_miss_hit_or_write_is_debug(self): + """A cache miss (of a file not there, or too old), a hit, and the write after a miss are logged at DEBUG only: + at INFO, the request a miss sends is the line.""" def twice(): """A miss and its write, then a hit, then a miss of a file too old.""" cache.cached('finbot_test.json', 60, lambda: {'a': 1}) cache.cached('finbot_test.json', 60, lambda: {'a': 1}) cache.cached('finbot_test.json', -1, lambda: {}) where = cache.path('finbot_test.json') - self.assertEqual(self.logged('INFO', twice), [f'cache miss: {where}', f'cache miss: {where}']) + self.assertEqual(self.logged('INFO', twice), []) os.remove(where) lines = self.logged('DEBUG', twice) kinds = [line.split(':')[0] for line in lines] From 942eb60a5afe7688aada308c4c46f0dbc927e868 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:34:16 +0000 Subject: [PATCH 26/31] Request log lines are cut at 73 bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET query2.finance.yahoo.com/v10/finance/quoteSummary/ARM.L?modules=ca… The whole line, method and URL, is at most 73 bytes, the ellipsis taking 3 in UTF-8 (it was the URL at 100 characters). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/net.py | 8 ++++---- tests/test_edges.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/net.py b/lib/net.py index 0ddff26b..eef3313c 100644 --- a/lib/net.py +++ b/lib/net.py @@ -15,7 +15,7 @@ # no domain may set or receive a cookie session.cookies.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[])) CONNECT_TIMEOUT = 5 # seconds to wait for a host to answer the connection: one that is down costs this, not http_timeout -SHOWN = 100 # characters of a request's URL in the log, at most +SHOWN = 73 # bytes of a request's log line, at most: the method and URL, cut with an ellipsis (3 bytes in UTF-8) def get(url, **kwargs): """requests.get through the shared session, which reuses connections; timed out as timeouts() says; logged.""" @@ -32,10 +32,10 @@ def request(method, url, **kwargs): def logged(method, url, kwargs): """The request's arguments without `logged`; the request noted at INFO first, unless logged=False (Telegram's long - polling, asked again and again): its method and URL, without secrets or the scheme, cut to SHOWN characters.""" + polling, asked again and again): its method and URL, without secrets or the scheme, cut to SHOWN bytes.""" if kwargs.pop('logged', True): - shown = redact(url).split('://', 1)[-1] - log.info(method, shown if len(shown) <= SHOWN else shown[:SHOWN - 1] + '…') + line = f"{method} {redact(url).split('://', 1)[-1]}".encode() + log.info(line.decode() if len(line) <= SHOWN else line[:SHOWN - 3].decode(errors='ignore') + '…') return kwargs def timeouts(kwargs): diff --git a/tests/test_edges.py b/tests/test_edges.py index 027e1cfb..80ba99bd 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -405,19 +405,20 @@ class RequestLog(FinbotCase): """Each request that goes out (a cache miss, or a call that has no cache) is a line at INFO.""" def test_each_request_is_a_short_line_without_secrets(self): - """A request logs its method and URL at INFO, without the scheme or a secret, cut to fit; Telegram's long - polling (logged=False) does not, and nothing does below INFO.""" + """A request logs its method and URL at INFO, without the scheme or a secret, cut to 73 bytes (the ellipsis + takes 3); Telegram's long polling (logged=False) does not, and nothing does below INFO.""" with contextlib.redirect_stderr(io.StringIO()) as err: - net.get('https://query1.finance.yahoo.com/v8/finance/chart/AAPL?range=1y&crumb=s3cret') + net.get('https://query1.finance.yahoo.com/v7/finance/quote?crumb=s3cret') net.post('https://api.telegram.org/bot123:ABC-def/sendMessage', json={}) net.request('PATCH', 'https://discord.com/api/v10/channels/7/messages/9', json={}) net.post('https://api.telegram.org/bot123:ABC-def/getUpdates', json={}, logged=False) - net.get('https://example.com/' + 'x' * 200) + net.get('https://query2.finance.yahoo.com/v10/finance/quoteSummary/ARM.L?modules=calendarEvents,' + 'defaultKeyStatistics,earnings,financialData,price&crumb=s3cret') lines = err.getvalue().splitlines() - self.assertEqual(lines[:3], ['GET query1.finance.yahoo.com/v8/finance/chart/AAPL?range=1y&crumb=', - 'POST api.telegram.org/bot/sendMessage', 'PATCH discord.com/api/v10/channels/7/messages/9']) - self.assertEqual((len(lines), lines[3][:16], lines[3][-1], len(lines[3])), - (4, 'GET example.com/', '…', len('GET ') + net.SHOWN)) + self.assertEqual(lines, ['GET query1.finance.yahoo.com/v7/finance/quote?crumb=', + 'POST api.telegram.org/bot/sendMessage', 'PATCH discord.com/api/v10/channels/7/messages/9', + 'GET query2.finance.yahoo.com/v10/finance/quoteSummary/ARM.L?modules=ca…']) + self.assertEqual(len(lines[3].encode()), 73) self.prefs(log_level='WARN') with contextlib.redirect_stderr(io.StringIO()) as err: net.get('https://example.com/') From aa13089c69786f13cd92ee14d45ca69dab4aec23 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 00:47:54 +0000 Subject: [PATCH 27/31] Every row in the bar chart, shorter names: no fund-trust boilerplate, 30 characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - render: a bar chart keeps every row. Past 40 rows the image grows taller, each row as tall as at 40 and the heading where it was, instead of dropping the middle rows behind a gap: the caption is cut short ("…and 8 more in the chart"), so the chart must show them all. Charts of 40 rows or fewer are pixel-identical. The shorts pictogram, a scene of a fixed size, keeps its 24. - formats.fund_name: a fund's name loses its trust's: "Global X ETF Series OFC Global X China EV ETF" is "Global X China EV ETF", "iShares Trust iShares Core S&P 500 ETF" is "iShares Core S&P 500 ETF"; an earlier ETF goes with the trust's words after it, and OFC anywhere. A company's repeated word ("Johnson & Johnson") stays. - formats.short_name: in a report's list (price, cal, rating, shorts, holdings, and every stock line), a name over 30 characters is cut at a word, with an ellipsis, so that a line fits a phone. The name stays whole elsewhere: the Simply Wall St link and the leveraged-fund check read it. record_outputs: 268 steps identical. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/formats.py | 32 ++++++++++++++++++++++++++++++-- lib/render.py | 41 +++++++++++++++++++++++++++-------------- lib/reports/cal.py | 2 +- lib/reports/holdings.py | 3 ++- lib/reports/price.py | 2 +- lib/reports/rating.py | 3 ++- lib/reports/shorts.py | 3 ++- tests/test_charts.py | 25 ++++++++++++++++--------- tests/test_util.py | 24 ++++++++++++++++++++++++ 9 files changed, 105 insertions(+), 30 deletions(-) diff --git a/lib/formats.py b/lib/formats.py index 52f58381..5394e597 100644 --- a/lib/formats.py +++ b/lib/formats.py @@ -39,6 +39,20 @@ (' Bull', ' '), (' .', ' '), (' ,', ' ') ] +TRUST_WORDS = ('ETF', 'ETFs', 'Series', 'Trust', 'Fund', 'Funds', 'OFC', 'Exchange-Traded', 'Portfolios') # a trust's + +def fund_name(title): + """A fund's own name, without its trust's before it: up to its brand again, where the trust's words come first + ('Global X ETF Series OFC Global X China EV ETF': 'Global X China EV ETF'); an earlier ETF, with the trust's words + after it ('Betashares ETF Trust Global Cybersecurity ETF': 'Betashares Global Cybersecurity ETF'); OFC (a fund + company) anywhere. A company's name ('Johnson & Johnson') stays as it is.""" + words = title.split() + again = next((i for i in range(1, len(words) - 1) + if words[i] == words[0] and any(word in TRUST_WORDS for word in words[1:i])), 0) + title = ' '.join(word for word in words[again:] if word != 'OFC') + trust = '|'.join(re.escape(word) for word in TRUST_WORDS) + return re.sub(rf'\bETFs? (?:(?:{trust}) )*(?=.*\bETF\b)', '', title) + def transform_title(title): """A company or fund name as the reports show it: shorter, without fund-trust boilerplate and legal suffixes.""" title = title.replace(' FPO', '') @@ -55,7 +69,7 @@ def transform_title(title): title = title.title() for suffix in (' &', ' and'): title = title.strip().removesuffix(suffix) - return ' '.join(title.replace(' Usd', ' USD').replace('Ishares', 'iShares').split()) + return fund_name(' '.join(title.replace(' Usd', ' USD').replace('Ishares', 'iShares').split())) def human_units(value, decimal_places=2): """1234567 -> '1.23 M'.""" @@ -74,8 +88,22 @@ def sections(*blocks): lines += block or [] return lines +NAME_LIMIT = 30 # characters of a name in a report's list: a longer line wraps on a phone + +def short_name(name): + """A name for a report's list, cut at a word to at most NAME_LIMIT characters, an ellipsis last: 'Global X Asia + Semiconductor ETF' -> 'Global X Asia Semiconductor…'. The name stays whole everywhere else.""" + if len(name) <= NAME_LIMIT: + return name + cut = name[:NAME_LIMIT - 1] + if ' ' in cut[NAME_LIMIT // 2:]: # a word break past halfway: cut there rather than mid-word + cut = cut[:cut.rindex(' ')] + return cut.rstrip(' ,&-') + '…' + def security_line(flag, title, link): - """'🇦🇺 Title (link)': how a report line names a security (without a leading space where its market has no flag).""" + """'🇦🇺 Title (link)': how a report line names a security (without a leading space where its market has no flag), + the name cut to fit a phone (short_name).""" + title = short_name(title) return f"{flag} {title} ({link})" if flag else f"{title} ({link})" def strip_url(url): diff --git a/lib/render.py b/lib/render.py index 96d79f54..09a8134c 100644 --- a/lib/render.py +++ b/lib/render.py @@ -76,13 +76,22 @@ def alone(*args, **kwargs): return draw(*args, **kwargs) return alone -def _figure(facecolor): - """A 1280x800 figure (Telegram's photo size, so it is never resampled) and its Agg canvas.""" +def _figure(facecolor, height=4.0): + """A 1280x800 figure (Telegram's photo size, so it is never resampled), or one `height` inches tall (200 pixels to + the inch), and its Agg canvas.""" from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=facecolor) + fig = Figure(figsize=(6.4, height), dpi=200, facecolor=facecolor) return fig, FigureCanvasAgg(fig) +def _from_top(y, fig): + """A height in a 1280x800 figure, as a fraction of it, kept as far from the top in a taller figure.""" + return y if fig.get_figheight() == 4.0 else 1 - (1 - y) * 4.0 / fig.get_figheight() + +def _from_bottom(y, fig): + """A height in a 1280x800 figure, as a fraction of it, kept as far from the bottom in a taller figure.""" + return y if fig.get_figheight() == 4.0 else y * 4.0 / fig.get_figheight() + def _time_axes(fig, bg, width): """The axes of a chart over time, `width` of the figure wide: the line chart's, and the compare chart's, whose end labels take the rest.""" @@ -117,10 +126,11 @@ def _heading(fig, title, subtitle='', subtitle_color=None): """A chart's title at the top left, and its subtitle (if any) beneath, each shrunk to fit the width.""" _, ink, ink2, _ = PALETTE room = 0.94 * fig.bbox.width - _shrink([fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center')], room) + _shrink([fig.text(0.03, _from_top(0.957, fig), title, color=ink, fontsize=13, fontweight='bold', ha='left', + va='center')], room) if subtitle: - _shrink([fig.text(0.03, 0.910, subtitle, color=subtitle_color or ink2, fontsize=8.5, ha='left', va='center')], - room) + _shrink([fig.text(0.03, _from_top(0.910, fig), subtitle, color=subtitle_color or ink2, fontsize=8.5, ha='left', + va='center')], room) def _dated(rows): """(date, close) rows, each date an ISO date or a datetime.date, as (the dates as datetime.date, the closes as a @@ -239,21 +249,23 @@ def tinted(bg, color, amount=0.07): def draw_bars(chart): """Horizontal bars that mirror the caption list, first row on top, filling the whole 1280x800 canvas: blue for >= 0, red below, each row's label at the bar's base and its value (as value_format prints it) at the tip. Rows of (label, - value); a (label, None) row is a gap. xlim zooms the value axis (bars then start at its left edge); session - 'midsession' or 'premarket' washes the background green or yellow.""" + value); a (label, None) row is a gap. Past ROWS rows the image grows taller, each row as tall as at ROWS, so that + every row is in it. xlim zooms the value axis (bars then start at its left edge); session 'midsession' or + 'premarket' washes the background green or yellow.""" rows, subtitle, xlim, session = chart.rows, chart.subtitle, chart.xlim, chart.session value_fmt = VALUE_FORMATS[chart.value_format] bg, ink, ink2, _ = PALETTE up, down = BAR_UP, DOWN if session in SESSION_TINTS: bg = tinted(bg, SESSION_TINTS[session]) - fig, canvas = _figure(bg) - top_edge = 0.85 if subtitle else 0.89 + n = len(rows) + fig, canvas = _figure(bg, 4.0 * max(1, n / ROWS)) + top_edge = _from_top(0.85 if subtitle else 0.89, fig) + bottom_edge = _from_bottom(0.02, fig) left = 0.14 if xlim else 0.04 # zoomed charts keep their labels outside the plot - ax = fig.add_axes([left, 0.02, 0.96 - left, top_edge - 0.02], facecolor=bg) + ax = fig.add_axes([left, bottom_edge, 0.96 - left, top_edge - bottom_edge], facecolor=bg) real = [(y, label, v) for y, (label, v) in enumerate(rows) if v is not None] values = [v for _, _, v in real] - n = len(rows) ax.barh([y for y, _, _ in real], values, height=0.72, color=[up if v >= 0 else down for v in values], zorder=3) base = max(xlim[0], 0) if xlim else 0 if not xlim: @@ -270,7 +282,7 @@ def draw_bars(chart): for side in ax.spines.values(): side.set_visible(False) dpi = fig.dpi - axes_px = (top_edge - 0.02) * fig.get_figheight() * dpi + axes_px = (top_edge - bottom_edge) * fig.get_figheight() * dpi bar_px = 0.72 * axes_px / n longest = max([len(str(label)) for _, label, _ in real] + [3]) # text height follows the bar thickness @@ -509,7 +521,8 @@ def _code_version(): CODE_VERSION = _code_version() RENDERERS = {'bars': draw_bars, 'pie': draw_pie, 'shorts': draw_shorts, 'line': draw_line, 'compare': draw_compare} -LIMITS = {'bars': 40, 'shorts': 24} # the rows a chart has room for +ROWS = 40 # bars: the rows a 1280x800 image has room for; more make it taller (draw_bars) +LIMITS = {'shorts': 24} # the rows a chart of a fixed size has room for def trimmed(chart): """The chart with no more rows than it has room for (LIMITS): over that, the first half and the last, a gap diff --git a/lib/reports/cal.py b/lib/reports/cal.py index a77f12ea..c031f899 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -76,7 +76,7 @@ def report(service, events, days, specific_stock, interactive, event): blocks = [] for date in sorted({when.date() for when, *_ in events}): day = f"{date:%b %d} " if specific_stock else '' # one stock's events carry their date - lines = [f"{flag} {day}{title} ({links.yahoo_link(ticker, service)})" + lines = [f"{flag} {day}{formats.short_name(title)} ({links.yahoo_link(ticker, service)})" for when, flag, title, ticker in events if when.date() == date] blocks.append(lines if specific_stock else [service.bold(f"{date:%b %d}")] + lines) payload = sum(blocks, []) if specific_stock else formats.sections(*blocks) diff --git a/lib/reports/holdings.py b/lib/reports/holdings.py index 6afc7f4f..0d2336df 100644 --- a/lib/reports/holdings.py +++ b/lib/reports/holdings.py @@ -11,7 +11,8 @@ def holding_line(holding, service, text=None): """'🇦🇺 Title (CODE)' for a Sharesight holding, CODE (or `text`) linking to the holding on Sharesight.""" link = service.link(f"https://portfolio.sharesight.com/holdings/{holding['holding_id']}/dashboard", holding['code'] if text is None else text) - return f"{markets.flag_from_market(holding['market_code'])} {formats.transform_title(holding['name'])} ({link})" + name = formats.short_name(formats.transform_title(holding['name'])) + return f"{markets.flag_from_market(holding['market_code'])} {name} ({link})" def prepare_holdings_payload(portfolio_name, service): """The holdings of the portfolio called `portfolio_name`. Raises sharesight.PortfolioNotFound.""" diff --git a/lib/reports/price.py b/lib/reports/price.py index 3dba8f5d..d78d38cd 100644 --- a/lib/reports/price.py +++ b/lib/reports/price.py @@ -127,7 +127,7 @@ def report(service, found, mode='interday', threshold=None, days=None, top=None, or more in a mode of MODES ('period': over `days`), or the top and bottom `top`.""" if threshold is None: threshold = config.prefs.price_percent - lines = [f"{formats.get_emoji(percent)} {quote['profile_title']} " + lines = [f"{formats.get_emoji(percent)} {formats.short_name(quote['profile_title'])} " f"({links.finance_link(ticker, quote['profile_exchange'], service, days=days)}) " f"{formats.signed_percent(round(percent))}" for ticker, percent, quote in found.rows] if not lines: diff --git a/lib/reports/rating.py b/lib/reports/rating.py index fb538cae..dc257e46 100644 --- a/lib/reports/rating.py +++ b/lib/reports/rating.py @@ -1,5 +1,6 @@ """Analysts' consensus rating changes since the last scheduled run (rating); .buy and .sell are in rankings.""" from lib import delivery +from lib import formats from lib import state from lib import links from lib import watchlist @@ -57,7 +58,7 @@ def report(service, changes, interactive): ticker_link = links.finance_link(ticker, data['profile_exchange'], service, brief=False) before, after = service.bold(f'{old_index} {old_action}'), service.bold(f'{index} {action}') message = f"{before} to {after} ({analysts} analysts)" - payload.append(f"{emoji} {data['profile_title']} ({ticker_link}) changed from {message}") + payload.append(f"{emoji} {formats.short_name(data['profile_title'])} ({ticker_link}) changed from {message}") payload.sort() if payload: payload.insert(0, service.bold('Consensus rating changes:')) diff --git a/lib/reports/shorts.py b/lib/reports/shorts.py index a18011db..f829ad73 100644 --- a/lib/reports/shorts.py +++ b/lib/reports/shorts.py @@ -1,6 +1,7 @@ """.shorts: the most shorted stocks followed, from Short Man (ASX) and Yahoo (scheduled: shorts).""" from lib import config from lib import delivery +from lib import formats from lib import render from lib import watchlist from lib import yahoo @@ -40,7 +41,7 @@ def report(service, tickers, market_data, threshold, specific_stock, interactive url = 'https://www.shortman.com.au/stock?q=' + code else: url = 'https://finance.yahoo.com/quote/' + ticker + '/key-statistics?p=' + ticker - title = market_data[ticker]['profile_title'] + title = formats.short_name(market_data[ticker]['profile_title']) found.append((short_percent, f"🩳 {title} ({service.link(url, code)}) {round(short_percent)}%", code)) found.sort(key=lambda row: row[0], reverse=True) # largest first, as the chart has them payload, graph = [line for _, line, _ in found], None diff --git a/tests/test_charts.py b/tests/test_charts.py index 85e26aad..df98ac6c 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -85,11 +85,11 @@ def shorts(rows, threshold=15): class ImageSanity(unittest.TestCase): - def check_image(self, buf, name, bg=BG, edge_clean=True): - """Fail unless buf is a 1280x800 PNG that is not blank, starts with the background, and (edge_clean) draws - nothing on its edges, where clipped text or bars would show.""" + def check_image(self, buf, name, bg=BG, edge_clean=True, size=(1280, 800)): + """Fail unless buf is a PNG of this size (1280x800) that is not blank, starts with the background, and + (edge_clean) draws nothing on its edges, where clipped text or bars would show.""" image = decode(buf) - self.assertEqual(image.size, (1280, 800), name) + self.assertEqual(image.size, size, name) pixels = np.asarray(image) self.assertGreater(len({tuple(p) for p in pixels.reshape(-1, 3)[::97]}), 8, f'{name} looks blank') self.assertGreater(pixels.std(), 5, f'{name} looks blank') @@ -117,6 +117,11 @@ def test_edge_case_rows(self): self.check_image(bars(rows, 'Title', 'sub'), name) self.check_image(uncached(render.Chart('bars', 'Title', rows=tuple(rows))), name + '/drawn') + def test_a_long_bar_chart_grows_taller_and_stays_sane(self): + """Past 40 rows a bar chart's image grows taller, each row as tall as at 40, and draws as sanely.""" + rows = [(f'T{i}', (i - 30) * 1.5) for i in range(60)] + self.check_image(bars(rows, 'Title', 'sub'), 'sixty rows', size=(1280, 1200)) + def test_zoomed_axis(self): """A bar chart with a narrow x range (ratings, 1 to 5) draws sane images, wide labels and many rows included.""" self.check_image(bars([('AAA', 1.2), ('BBB', 3.4)], 'Ratings', xlim=(0.9, 4.0)), 'zoomed') @@ -291,11 +296,13 @@ def run_report(self, call): self.assertTrue(captured, 'no chart was drawn') return captured[0] - def test_over_forty_bars_keep_both_ends_with_a_gap(self): - """A bar chart of over 40 rows keeps the first 20 and the last 20, a gap between them; the shorts pictogram, - over 24, the first and last 12, with the same gap (it used to drop the middle without a mark).""" - rows = render.trimmed(render.Chart('bars', 'T', rows=tuple((f'T{i}', i) for i in range(50)))).rows - self.assertEqual((len(rows), rows[20], rows[0][0], rows[-1][0]), (41, ('', None), 'T0', 'T49')) + def test_a_bar_chart_keeps_every_row_and_the_shorts_both_ends(self): + """A bar chart keeps every row, however many: its image grows instead (the caption is cut, so the chart must + show them all); the shorts pictogram, a scene of a fixed size, keeps the first and last 12 of over 24, a gap + between.""" + chart = render.Chart('bars', 'T', rows=tuple((f'T{i}', i) for i in range(50))) + self.assertEqual(render.trimmed(chart).rows, chart.rows) + self.assertEqual(decode(render.draw_bars(chart)).size, (1280, 1000)) rows = render.trimmed(render.Chart('shorts', rows=tuple((f'T{i}', 60 - i) for i in range(30)))).rows self.assertEqual((len(rows), rows[12], rows[11][0], rows[13][0]), (25, ('', None), 'T11', 'T18')) self.assertEqual(render.trimmed(render.Chart('pie', rows=(('a', 1.0),) * 50)).rows, (('a', 1.0),) * 50) diff --git a/tests/test_util.py b/tests/test_util.py index 8c1b116c..d62d91de 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -94,6 +94,30 @@ def test_transform_title(self): with self.subTest(title=title): # a suffix comes off the end only; no run of spaces is left behind self.assertEqual(formats.transform_title(title), expected) + def test_a_name_in_a_list_is_cut_to_fit_a_phone(self): + """A name over 30 characters is cut at a word, with an ellipsis (mid-word where a word break would cut it too + short); a shorter one stays whole.""" + for name, shown in (('Global X Asia Semiconductor ETF', 'Global X Asia Semiconductor…'), + ('Global X China EV ETF', 'Global X China EV ETF'), ('A' * 40, 'A' * 29 + '…'), + ('Short Supercalifragilisticexpialidocious', 'Short Supercalifragilisticexp…')): + with self.subTest(name=name): + self.assertEqual(formats.short_name(name), shown) + self.assertLessEqual(len(formats.short_name(name)), formats.NAME_LIMIT) + + def test_a_funds_name_loses_its_trusts(self): + """A fund's name loses the trust's before it, a repeated brand or ETF, and OFC; a company's repeated word, or a + fund's own Trust, stays.""" + for title, expected in (('Global X ETF Series OFC Global X China EV ETF', 'Global X China EV ETF'), + ('iShares Trust iShares Core S&P 500 ETF', 'iShares Core S&P 500 ETF'), + ('SPDR Series Trust SPDR S&P Biotech ETF', 'SPDR S&P Biotech ETF'), + ('Betashares ETF Trust Global Cybersecurity ETF', 'Betashares Global Cybersecurity ETF'), + ('Global X Uranium ETF ETF', 'Global X Uranium ETF'), + ('VanEck Semiconductor ETF', 'VanEck Semiconductor ETF'), + ('SPDR S&P 500 ETF Trust', 'SPDR S&P 500 ETF Trust'), ('Johnson & Johnson', 'Johnson & Johnson'), + ('Johnson & Johnson Health', 'Johnson & Johnson Health')): + with self.subTest(title=title): + self.assertEqual(formats.transform_title(title), expected) + class Formatting(unittest.TestCase): def test_signed_percent(self): From f11efc0cb617368d8a073d60e75704e5ce4fc35b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 01:07:17 +0000 Subject: [PATCH 28/31] A US listing keeps its own dates; "no market cap" logs at DEBUG - cal: a ticker whose quote lacks the earnings or ex-dividend date takes it from its own details first: what is held is that listing (a US ADR such as ARM or CLBT), so its own date counts. Its home market's listing (ARM.L, CLBT.TA: "unknown to Yahoo" in the log) is asked for only when the US listing has no date at all, as are the listing in finbot_adr.json and the NZX's for an ASX ticker reporting in NZD. - rankings: a stock .marketcap cannot rank (an ETF without a market cap, or a currency without a rate) logs at DEBUG, not INFO. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/reports/cal.py | 34 ++++++++++++++++++---------------- lib/reports/rankings.py | 2 +- tests/test_edges.py | 9 +++++++++ tests/test_reports.py | 21 +++++++++++++++++++-- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/lib/reports/cal.py b/lib/reports/cal.py index c031f899..1761cedc 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -53,22 +53,24 @@ def gather(tickers, days, specific_stock, event): return sorted(found, key=lambda item: item[0]) def primary_listing(ticker, data, field): - """(the primary listing's ticker, its Yahoo details) where an ADR or a second listing without a `field` of its own - finds one, or (None, None): the listing in finbot_adr.json, the NZX for an ASX ticker reporting in NZD, an exchange - suffix for a foreign company's US ticker, else the ticker's own details. It must be the same company, with the - `field`.""" - primary = primary_listings().get(ticker) - if primary is None: - nzd = ticker.endswith('.AX') and data.get('financialCurrency') == 'NZD' - primary = ticker.replace('.AX', '.NZ') if nzd else ticker # fixes GTK.AX and SKO.AX - details = yahoo.fetch_detail(primary).get(primary, {}) - country = details.get('profile_country') # only in the details - if country in markets.YAHOO_COUNTRY and '.' not in primary: - primary = primary + '.' + markets.YAHOO_COUNTRY[country] # fixes ASML and INFY - details = yahoo.fetch_detail(primary).get(primary, {}) - if details.get('profile_title') != data['profile_title'] or field not in details: # another company, or no date - return None, None - return primary, details + """(a listing's ticker, its Yahoo details) with a `field` that a ticker's quote lacks, or (None, None). First the + ticker's own details: what is held is that listing, a US ADR say, so its own date counts, never its home market's + (ARM, not ARM.L). Only when it has none, another listing of the same company: the one in finbot_adr.json, the NZX + for an ASX ticker reporting in NZD (GTK.AX, SKO.AX), or the home market's for a foreign company's US ticker (ASML, + INFY).""" + own = yahoo.fetch_detail(ticker).get(ticker, {}) + listings = [ticker] + if mapped := primary_listings().get(ticker): + listings.append(mapped) + elif ticker.endswith('.AX') and data.get('financialCurrency') == 'NZD': + listings.append(ticker.replace('.AX', '.NZ')) + elif own.get('profile_country') in markets.YAHOO_COUNTRY and '.' not in ticker: # the country: only in the details + listings.append(ticker + '.' + markets.YAHOO_COUNTRY[own['profile_country']]) + for listing in listings: + details = own if listing == ticker else yahoo.fetch_detail(listing).get(listing, {}) + if details.get('profile_title') == data['profile_title'] and field in details: # the same company, with a date + return listing, details + return None, None def report(service, events, days, specific_stock, interactive, event): """The events in one network's markup, grouped by date under a heading (one stock's carry their dates instead).""" diff --git a/lib/reports/rankings.py b/lib/reports/rankings.py index 13e0c77a..f296bfb0 100644 --- a/lib/reports/rankings.py +++ b/lib/reports/rankings.py @@ -28,7 +28,7 @@ def marketcap_list(service, chat, bottom=False, length=15): for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(watchlist.with_holdings(chat))).items(): if 'market_cap_usd' not in data: why = "no market cap" if 'market_cap' not in data else f"no exchange rate for {data.get('currency')}" - log.info(ticker, why) + log.debug(ticker, why) continue cap = formats.human_units(data['market_cap_usd']) caps.append((data['market_cap_usd'], f"{links.stock_line(ticker, data, service)} mkt cap: {cap}")) diff --git a/tests/test_edges.py b/tests/test_edges.py index 80ba99bd..7f394c79 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -401,6 +401,15 @@ def twice(): self.assertRegex(lines[2], r'^cache hit: \S+finbot_test\.json TTL: \d+s$') + def test_a_stock_without_a_market_cap_is_debug(self): + """A stock that .marketcap cannot rank, without a market cap (an ETF), is logged at DEBUG only: nothing went + wrong.""" + self.patch(yahoo, 'fetch', lambda tickers: {'ETF1': {'profile_title': 'A Fund', 'profile_exchange': 'NYSE'}}) + self.patch(yahoo, 'add_usd_market_caps', lambda quotes: quotes) + rank = lambda: rankings.marketcap_list(TELEGRAM, '55') + self.assertEqual((self.logged('INFO', rank), self.logged('DEBUG', rank)), ([], ['ETF1 no market cap'])) + + class RequestLog(FinbotCase): """Each request that goes out (a cache miss, or a call that has no cache) is a line at INFO.""" diff --git a/tests/test_reports.py b/tests/test_reports.py index 1dc84e1e..742f77f9 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -393,15 +393,32 @@ def test_a_chats_job_runs_its_report_as_scheduled_not_as_a_command(self): self.assertEqual(pins, ['5']) def test_earnings_skip_an_adr_whose_primary_listing_yahoo_does_not_know(self): - """T1 has no earnings date, so its primary listing is looked up: Yahoo not knowing it crashed the report.""" + """T1 has no earnings date, even in its details, so its primary listing is looked up: Yahoo not knowing it + crashed the report.""" self.patch(cal, 'primary_listings', lambda: {'T1': 'T1.XX'}) undated = lambda t: {k: v for k, v in market_entry(t).items() if not (t == 'T1' and k == 'earnings_date')} self.patch(yahoo, 'fetch', lambda ts: {t: undated(t) for t in ts}) - self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {} if t == 'T1.XX' else {t: market_entry(t)}) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {} if t == 'T1.XX' else {t: undated(t)}) cal.run(event='earnings') self.assertIn('Co T2 ', str(self.sent)) self.assertNotIn('Co T1 ', str(self.sent)) + def test_a_us_listing_of_a_foreign_company_keeps_its_own_dates(self): + """A foreign company's US listing (an ADR such as ARM, held as it is) whose quote lacks the date takes it from + its own details, never its home market's listing (ARM.L, which Yahoo may not even know); that one is asked + for only when the US listing has no date at all.""" + asked = [] + undated = lambda t: {k: v for k, v in market_entry(t).items() if not (t == 'T1' and k == 'earnings_date')} + self.patch(yahoo, 'fetch', lambda ts: {t: undated(t) for t in ts}) + def details(t, *a, **k): + """T1's details, of a British company, with its date; asked for, noted.""" + asked.append(t) + return {t: {**market_entry(t.split('.')[0]), 'profile_country': 'United Kingdom'}} + self.patch(yahoo, 'fetch_detail', details) + cal.run(event='earnings') + self.assertIn('Co T1 ', str(self.sent)) + self.assertEqual([t for t in asked if '.' in t], []) # no T1.L + def test_trades(self): """.trades run from a chat posts.""" trades.run(telegram_common.TelegramBot('5')) From 94095b10ea4cf5b646d639aec84a2d3f5000aab9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 01:12:08 +0000 Subject: [PATCH 29/31] The same message sent again within a second is taken once Telegram's mobile app can send a command twice (/start in a new chat), each its own update with its own id, so the repeat check by update id lets both through and the chat gets two answers. bot.take now skips a message whose person, chat and text match one taken in the last second (bot.doubled), and logs it at INFO ("@rob_llama sent '/start' again within 1 s, not run again"). Discord's slash commands are left out: each interaction must get its answer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 24 +++++++++++++++++++++-- tests/common.py | 3 +++ tests/test_inbound.py | 44 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/bot.py b/bot.py index 372ed2f9..1b9e1ca1 100755 --- a/bot.py +++ b/bot.py @@ -5,7 +5,7 @@ # ignored until startup is done (the reload handler is installed last): its default action kills the bot signal.signal(signal.SIGHUP, signal.SIG_IGN) signal.signal(signal.SIGUSR1, signal.SIG_IGN) # likewise the cache flush -import collections, dataclasses, json, os, re, sys, threading, traceback +import collections, dataclasses, json, os, re, sys, threading, time, traceback from pathlib import Path from lib import log @@ -34,6 +34,8 @@ LISTENERS = (telegram_longpolling.TelegramPoll, slack_websocket.SlackSocket, discord_websocket.DiscordGateway) # pull _updates_seen = collections.OrderedDict() # (service, update id) of the last updates taken, oldest first _updates_lock = threading.Lock() # updates arrive on several threads: a resent one may race its first copy +DOUBLED = 1 # seconds in which the same message, from the same person in the same chat, is taken once +_said = {} # (service, chat, user, text) of the messages taken in the last DOUBLED seconds: when def seen_before(service, update_id, keep=1000): """Whether this update was taken already: Telegram's update_id, Slack's event_id. Slack resends an event it got no @@ -59,6 +61,20 @@ def listeners(): app-level token): finbot fetches the updates of those networks itself.""" return [kind() for kind in LISTENERS if kind.wanted()] +def doubled(message): + """Whether this message repeats one that the same person sent in the same chat in the last DOUBLED seconds: + Telegram's mobile app can send a command twice (/start in a new chat), each its own update, and one answer is + enough.""" + key = (message.service, message.chat_id, message.user, message.text.strip()) + now = time.monotonic() + with _updates_lock: + for old in [said for said, at in _said.items() if now - at > DOUBLED]: + del _said[old] + if key in _said: + return True + _said[key] = now + return False + def main(environ, start_response): """WSGI entry point. Never lets an exception escape. A failure handling an authenticated request is reported as one line to the default channel and answered 200, so the service does not retry it into the chat again. Before @@ -170,7 +186,8 @@ def close(self): def take(network, update): """An update from a network (an inbound.Network reads it), however it arrived: what it says goes to lib/worker in a - thread of its own (dispatch), unless it is a repeat. It is acknowledged first, the one place that does + thread of its own (dispatch), unless it is a repeat (the same update resent, or the same message sent again at once: + doubled). It is acknowledged first, the one place that does (Network.acknowledge: Telegram's toast for a press). Returns the inbound.Response for the network; when the network must have it before any reply (Discord's interactions), the work waits in its `then`, for whoever delivers it.""" event = network.parse(update) @@ -181,6 +198,9 @@ def take(network, update): if seen_before(network.service, network.update_id(update)): log.info(f"[{network.service}]: update {network.update_id(update)} seen already, not run again") return inbound.OK + if isinstance(event, inbound.Message) and not event.interaction and doubled(event): # each interaction is answered + log.info(f"[{network.service}]: {event.user} sent {event.text!r} again within {DOUBLED} s, not run again") + return inbound.OK answer = network.acknowledge(event) if answer is not None: return dataclasses.replace(answer, then=lambda: dispatch(event)) diff --git a/tests/common.py b/tests/common.py index 51253e05..ac200ee1 100644 --- a/tests/common.py +++ b/tests/common.py @@ -23,6 +23,7 @@ (sharesight, '_portfolios'), (sharesight, '_trades'), (sharesight, '_performance'))} REAL_GET_HOLDINGS = watchlist.with_holdings REAL_DISPATCH = bot.dispatch # FinbotCase notes the worker thread of each update, so that a test can wait for it +REAL_DOUBLED = bot.doubled # FinbotCase takes every message, a repeat sent on purpose too; DoubledMessages puts it back def no_jobs(data): @@ -128,6 +129,8 @@ def setUp(self): self.env(data_dir=tmp.name, cache_dir=tmp.name, telegramBotToken='X', telegram_api='https://api.telegram.org/botX/') self.patch(bot, '_updates_seen', collections.OrderedDict()) # each test starts with no update taken + self.patch(bot, '_said', {}) # nor any message + self.patch(bot, 'doubled', lambda message: False) # a test may send the same message twice on purpose self.workers = [] self.patch(bot, 'dispatch', lambda event: self.workers.append(REAL_DISPATCH(event))) self.sent = [] diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 57c53d9e..68f7e576 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -1,6 +1,6 @@ """What reaches the bot: webhook requests turned away or taken, Slack's events, presses, slash commands and signatures, the Slack app's manifest, and user text escaped where it arrives.""" -import contextlib, hashlib, hmac, io, json, re, time, unittest +import contextlib, hashlib, hmac, io, json, re, time, types, unittest from urllib.parse import urlencode from tests.common import * from util import slack_manifest @@ -187,6 +187,48 @@ def test_a_press_is_acknowledged_once_before_the_worker_gets_it(self): ('worker', 'slack', 'c|h|w|T1')]) +class DoubledMessages(FinbotCase): + """The same message from the same person in the same chat within a second is taken once: Telegram's mobile app + can send a command twice (/start in a new chat), each its own update.""" + + def setUp(self): + """The real check, and a clock of this test's own.""" + super().setUp() + self.patch(bot, 'doubled', REAL_DOUBLED) + self.now = [1000.0] + self.patch(bot, 'time', types.SimpleNamespace(monotonic=lambda: self.now[0])) + + def send(self, update_id, text='/start', user='rob', chat=55): + """Take a Telegram message, as an update of this id arrives.""" + with contextlib.redirect_stderr(io.StringIO()): + bot.take(telegram_common.READER, {'update_id': update_id, 'message': {'message_id': update_id, + 'chat': {'id': chat}, 'from': {'first_name': 'Rob', 'username': user}, 'text': text}}) + self.join_workers() + + def test_a_message_sent_again_within_a_second_is_taken_once(self): + """A second /start within a second is not run again; another person's, another chat's, another text, or one + over a second later is.""" + self.send(1) + self.send(2) + self.assertEqual(len(self.workers), 1) + self.send(3, user='ann') + self.send(4, chat=66) + self.send(5, text='.help') + self.assertEqual(len(self.workers), 4) + self.now[0] += 1.5 + self.send(6) + self.assertEqual(len(self.workers), 5) + + def test_a_discord_slash_command_is_always_answered(self): + """A slash command sent again at once still gets its answer: Discord shows an error for an interaction left + unanswered.""" + command = {'type': 2, 'id': '1', 'token': 't', 'application_id': '9', 'channel_id': '77', + 'member': {'user': {'id': '5', 'username': 'rob'}}, 'data': {'name': 'start'}} + with contextlib.redirect_stderr(io.StringIO()): + answers = [bot.take(discord_common.READER, dict(command, id=str(n))) for n in (1, 2)] + self.assertEqual([json.loads(answer.body) for answer in answers], [{'type': 5}, {'type': 5}]) + + class SlackEvents(FinbotCase): """Slack sends every message in a subscribed channel, the bot's own posts included.""" From db29d2b7da14afb9901f73db2caad1defc729168 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 01:21:18 +0000 Subject: [PATCH 30/31] No home-market suffix for a US listing: no more ARM.L or CLBT.TA The calendar no longer swaps a foreign company's US ticker for its home market's when the US listing has no date: ARM and CLBT are held on US markets (Sharesight says so), so ARM.L and CLBT.TA are never asked for, and a US listing without a date has none. The country -> Yahoo suffix table that made those tickers (markets.YAHOO_COUNTRY) goes with it. The listings named in finbot_adr.json and the NZX's for an ASX ticker reporting in NZD stay, tried only when the ticker's own details have no date. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/markets.py | 43 ------------------------------------------- lib/reports/cal.py | 11 ++++------- tests/test_reports.py | 27 +++++++++++++++------------ 3 files changed, 19 insertions(+), 62 deletions(-) diff --git a/lib/markets.py b/lib/markets.py index b0fee86e..6f191d2b 100644 --- a/lib/markets.py +++ b/lib/markets.py @@ -78,49 +78,6 @@ class Market(typing.NamedTuple): 'Shanghai': 'SHG', 'Shenzhen': 'SHE', 'Thailand': 'BKK', 'Jakarta': 'IDX', 'Kuala Lumpur': 'KLS', } -YAHOO_COUNTRY = { # country -> its Yahoo ticker suffix, for a listing that has none (cal.py) - 'Argentina': 'BA', - 'Austria': 'VI', - 'Australia': 'AX', - 'Belgium': 'BR', - 'Brazil': 'SA', - 'Canada': 'TO', - 'Chile': 'SN', - 'Colombia': 'CL', - 'Czech Republic': 'PR', - 'Denmark': 'CO', - 'Estonia': 'TL', - 'Finland': 'HE', - 'France': 'PA', - 'Germany': 'DE', - 'Hungary': 'BD', - 'Iceland': 'IC', - 'India': 'NS', - 'Indonesia': 'JK', - 'Ireland': 'IR', - 'Israel': 'TA', - 'Italy': 'MI', - 'Latvia': 'RG', - 'Lithuania': 'VS', - 'Mexico': 'MX', - 'Netherlands': 'AS', - 'New Zealand': 'NZ', - 'Norway': 'OL', - 'Portugal': 'LS', - 'Qatar': 'QA', - 'Saudi Arabia': 'SR', - 'South Africa': 'JO', - 'Spain': 'MC', - 'Sweden': 'ST', - 'Switzerland': 'SW', - 'Thailand': 'BK', - 'Turkey': 'IS', - 'United Kingdom': 'L', - 'Venezuela': 'CR' - # markets with numeric tickers purposefully omitted -} - - def code(name): """MARKETS' code for an exchange by any name it goes by: Sharesight's code or Google's, the suffix of its Yahoo tickers, or Yahoo's name for it (each Nasdaq tier has one: NasdaqGS); the name itself when unknown.""" diff --git a/lib/reports/cal.py b/lib/reports/cal.py index 1761cedc..4dad9c4d 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -53,19 +53,16 @@ def gather(tickers, days, specific_stock, event): return sorted(found, key=lambda item: item[0]) def primary_listing(ticker, data, field): - """(a listing's ticker, its Yahoo details) with a `field` that a ticker's quote lacks, or (None, None). First the - ticker's own details: what is held is that listing, a US ADR say, so its own date counts, never its home market's - (ARM, not ARM.L). Only when it has none, another listing of the same company: the one in finbot_adr.json, the NZX - for an ASX ticker reporting in NZD (GTK.AX, SKO.AX), or the home market's for a foreign company's US ticker (ASML, - INFY).""" + """(a listing's ticker, its Yahoo details) with a `field` that a ticker's quote lacks, or (None, None). The ticker's + own details: what is held is that listing, so its own date counts, and a foreign company's US listing (ARM, CLBT) + is never swapped for its home market's (ARM.L, CLBT.TA). Only when it has none, the listing that finbot_adr.json + names, or the NZX's for an ASX ticker reporting in NZD (GTK.AX, SKO.AX).""" own = yahoo.fetch_detail(ticker).get(ticker, {}) listings = [ticker] if mapped := primary_listings().get(ticker): listings.append(mapped) elif ticker.endswith('.AX') and data.get('financialCurrency') == 'NZD': listings.append(ticker.replace('.AX', '.NZ')) - elif own.get('profile_country') in markets.YAHOO_COUNTRY and '.' not in ticker: # the country: only in the details - listings.append(ticker + '.' + markets.YAHOO_COUNTRY[own['profile_country']]) for listing in listings: details = own if listing == ticker else yahoo.fetch_detail(listing).get(listing, {}) if details.get('profile_title') == data['profile_title'] and field in details: # the same company, with a date diff --git a/tests/test_reports.py b/tests/test_reports.py index 742f77f9..72906149 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -404,20 +404,23 @@ def test_earnings_skip_an_adr_whose_primary_listing_yahoo_does_not_know(self): self.assertNotIn('Co T1 ', str(self.sent)) def test_a_us_listing_of_a_foreign_company_keeps_its_own_dates(self): - """A foreign company's US listing (an ADR such as ARM, held as it is) whose quote lacks the date takes it from - its own details, never its home market's listing (ARM.L, which Yahoo may not even know); that one is asked - for only when the US listing has no date at all.""" - asked = [] + """A foreign company's US listing (ARM, CLBT: held on a US market) whose quote lacks the date takes it from its + own details, and is never swapped for its home market's listing (ARM.L, CLBT.TA), even when it has no date at + all: then it has none.""" undated = lambda t: {k: v for k, v in market_entry(t).items() if not (t == 'T1' and k == 'earnings_date')} self.patch(yahoo, 'fetch', lambda ts: {t: undated(t) for t in ts}) - def details(t, *a, **k): - """T1's details, of a British company, with its date; asked for, noted.""" - asked.append(t) - return {t: {**market_entry(t.split('.')[0]), 'profile_country': 'United Kingdom'}} - self.patch(yahoo, 'fetch_detail', details) - cal.run(event='earnings') - self.assertIn('Co T1 ', str(self.sent)) - self.assertEqual([t for t in asked if '.' in t], []) # no T1.L + for own_date, listed in ((True, True), (False, False)): + with self.subTest(own_date=own_date): + asked = [] + def details(t, *a, **k): + """T1's details, of a British company, with its date or without; asked for, noted.""" + asked.append(t) + return {t: {**(market_entry(t) if own_date else undated(t)), 'profile_country': 'United Kingdom'}} + self.patch(yahoo, 'fetch_detail', details) + self.sent.clear() + cal.run(event='earnings') + self.assertEqual('Co T1 ' in str(self.sent), listed) + self.assertEqual([t for t in asked if '.' in t], []) # no T1.L def test_trades(self): """.trades run from a chat posts.""" From 11e4f768a61ab710368abac96c251d07a9debe57 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 01:52:28 +0000 Subject: [PATCH 31/31] Request log lines: "POST from telegram: we answered 200 OK" POST from telegram: we answered 200 OK in place of "[http] request for telegram answered 200 OK". The method is the request's own (REQUEST_METHOD); a request to an unknown path is "from no network" as before. The tests' WSGI requests say POST. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 3 ++- tests/common.py | 3 ++- tests/test_finbot.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bot.py b/bot.py index 1b9e1ca1..0ef8ee33 100755 --- a/bot.py +++ b/bot.py @@ -104,7 +104,8 @@ def handle(environ, start_response): def respond(response): """Answer the request: a status line and a body (logged at INFO, naming the network, not the path). Work that must wait for the answer starts once the server has sent it.""" - log.info(f"[http] request for {environ.get(ROUTED) or 'no network'} answered {response.status}") + network = environ.get(ROUTED) or 'no network' + log.info(f"{environ['REQUEST_METHOD']} from {network}: we answered {response.status}") start_response(response.status, [('Content-type', 'application/json')]) return Sent([response.body], response.then) if response.then else [response.body] def print_headers(say=log.info): diff --git a/tests/common.py b/tests/common.py index ac200ee1..48690d76 100644 --- a/tests/common.py +++ b/tests/common.py @@ -59,7 +59,8 @@ def temp_dir(case): def wsgi_environ(path, body, **headers): """A WSGI environ posting `body` (bytes) to `path`, with these headers (as HTTP_* keys).""" - return {'PATH_INFO': path, 'wsgi.input': io.BytesIO(body), 'CONTENT_LENGTH': str(len(body)), **headers} + return {'REQUEST_METHOD': 'POST', 'PATH_INFO': path, 'wsgi.input': io.BytesIO(body), + 'CONTENT_LENGTH': str(len(body)), **headers} def skip_if_old_env(case): diff --git a/tests/test_finbot.py b/tests/test_finbot.py index baf9d912..516ffe72 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -58,7 +58,7 @@ def post(message): with contextlib.redirect_stderr(err): out = bot.main(env, lambda *a, **k: None) return out, err.getvalue() - self.assertEqual(post({'sticker': {'file_id': 'x'}}), ([b''], '[http] request for telegram answered 200 OK\n')) + self.assertEqual(post({'sticker': {'file_id': 'x'}}), ([b''], 'POST from telegram: we answered 200 OK\n')) self.assertGreaterEqual(len(bot.git_version()), 7) head = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=bot.__file__.rsplit('/', 1)[0], capture_output=True, text=True).stdout.strip()