diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6d7a9764 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# what the image does without: history, tools for development, the tests and their samples, the docs +.git +.github +.claude +.venv +var +tests +doc +img +**/__pycache__ +*.pyc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 080ef50d..7b8d4834 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,13 +47,93 @@ jobs: cache: pip - if: steps.changes.outputs.docs_only != 'true' run: | - pip install -r requirements.txt + pip install -r requirements.txt -r requirements-optional.txt python -m compileall -q *.py lib util python -m unittest discover -s tests -v -b + docker: + # The image built as a Docker host builds it, from Debian's packages: a package renamed or gone fails here before + # production meets it. Then every module imports in it, and the container starts as compose.yaml starts it. + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # the base branch is needed to see what a pull request changes + - id: changes + name: Classify the change + # As in the test job: a pull request that only touches docs skips the steps below. + env: + BASE_REF: ${{ github.base_ref }} + run: | + docs_only=false + if [ "$GITHUB_EVENT_NAME" = pull_request ]; then + files=$(git -c core.quotePath=false diff --name-only --no-renames "origin/$BASE_REF...HEAD") + docs='^(README\.md|CLAUDE\.md|doc/[^/]+\.md|img/[^/]+\.png)$' + if [ -n "$files" ] && ! grep -qvE "$docs" <<< "$files"; then + docs_only=true + fi + fi + echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT" + - if: steps.changes.outputs.docs_only != 'true' + name: Build the image + run: docker build --build-arg GIT_COMMIT="$GITHUB_SHA" -t finbot . + - if: steps.changes.outputs.docs_only != 'true' + name: Every module imports in the image, the optional libraries too, and the log names the commit + run: | + docker run --rm -i --read-only --tmpfs /tmp -e DATA_DIR=/tmp/data finbot python3 - <<'EOF' + import importlib, os, pkgutil, bot, lib, lib.reports + for package in (lib, lib.reports): + for module in pkgutil.iter_modules(package.__path__, package.__name__ + '.'): + importlib.import_module(module.name) + import discord, nacl, waitress, websocket + assert bot.git_version() == os.environ['FINBOT_COMMIT'][:7], bot.git_version() + print('every module imports; finbot', bot.git_version()) + EOF + - if: steps.changes.outputs.docs_only != 'true' + name: The container starts as compose starts it, copies the defaults, reloads and flushes + run: | + data="$RUNNER_TEMP/finbot-data" + mkdir "$data" && sudo chown 10001:10001 "$data" + docker run -d --name finbot --init --read-only --tmpfs /tmp --cap-drop ALL \ + --security-opt no-new-privileges:true --pids-limit 256 --memory 400m --memory-swap 400m \ + -e DATA_DIR=/data -v "$data:/data" -v "$PWD/defaults/finbot.env:/run/finbot.env:ro" finbot + wait_for() { # the log saved first: grep -q would cut docker logs off mid-write, which pipefail counts + for _ in $(seq 60); do + docker logs finbot > "$RUNNER_TEMP/log" 2>&1 + grep -q "$1" "$RUNNER_TEMP/log" && return 0 + sleep 1 + done + cat "$RUNNER_TEMP/log" + echo "::error::never logged: $1" + return 1 + } + wait_for 'finbot is running' + test -f "$data/preferences.conf" + docker kill -s HUP finbot + wait_for 'config: reloaded' + docker kill -s USR1 finbot + wait_for 'cache: flushed' + docker rm -f finbot + - if: steps.changes.outputs.docs_only != 'true' + name: A mistake in the startup file stops the container before the bot starts + run: | + printf 'telegramBotTokn=x\n' > "$RUNNER_TEMP/bad.env" + if docker run --rm --read-only --tmpfs /tmp -e DATA_DIR=/tmp \ + -v "$RUNNER_TEMP/bad.env:/run/finbot.env:ro" finbot > "$RUNNER_TEMP/out" 2>&1; then + cat "$RUNNER_TEMP/out" + echo "::error::the container started with a mistake in its startup file" + exit 1 + fi + cat "$RUNNER_TEMP/out" + grep -q 'unknown key telegramBotTokn' "$RUNNER_TEMP/out" + if grep -q 'finbot is running' "$RUNNER_TEMP/out"; then + echo "::error::the bot started with a mistake in its startup file" + exit 1 + fi + notify: # Telegram alert when the weekly run fails. Needs repository secrets TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID. - needs: test + needs: [test, docker] if: failure() && github.event_name == 'schedule' runs-on: ubuntu-24.04 steps: diff --git a/.gitignore b/.gitignore index dea55888..4eab49b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .* !.gitignore !.editorconfig +!.dockerignore !.python-version !.claude !.github diff --git a/CLAUDE.md b/CLAUDE.md index 597836fa..8871df9a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,18 +15,22 @@ The work branch named in the session instructions (`claude/...`) is the persiste An open PR takes in every commit pushed to its branch, so a PR opened for one small change can merge far more under its old title (#35 was titled as a CI tweak and merged 14 review fixes). The merge commit on `main` keeps that title for good. - Before each push, check for an open PR from the work branch. If there is one, update its title and description so they cover every commit it now holds, not only the change it was opened for. - Never make such a PR mergeable (resolve its conflict, merge `main` into it, enable auto-merge) while its title or description is out of date. -- To merge: `main` takes only a branch that is up to date with it and green on both CI jobs. Merge `origin/main` into the work branch (a merge commit, never a rebase), run the tests, push, wait for CI on that head, then merge the PR with a merge commit. +- To merge: `main` takes only a branch that is up to date with it and green on every CI job. Merge `origin/main` into the work branch (a merge commit, never a rebase), run the tests, push, wait for CI on that head, then merge the PR with a merge commit. ## Facts -- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, except for - pull requests that change only README.md, CLAUDE.md, `doc/*.md` or `img/*.png` (install, compile and tests are - skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). +- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, and + its `docker` job builds the image (`Dockerfile`, Debian's packages), imports every module in it and starts it as + `compose.yaml` does; all except for pull requests that change only README.md, CLAUDE.md, `doc/*.md` or `img/*.png` + (install, compile, tests and the build are skipped; the allowlist in `.github/workflows/ci.yml` must never include a + path a test reads). This cloud environment cannot reach `deb.debian.org`, so only CI builds the image. - Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); - `requirements.txt` is for pip/uv users and CI. + `requirements.txt` is for pip/uv users and CI. The libraries in `requirements-optional.txt` are each imported only + inside the code of the mode that needs it, never at module load (a test starts finbot with them all blocked). - A new preference goes in three places: a `Prefs` field with its default (`lib/config.py`; its type says how it is - read), `defaults/preferences.conf`, and the README's preferences table. The tests catch a key missing from the - template or a default that differs, but not the README (no test may read README.md: CI skips the tests for a - README-only change). + read), `defaults/preferences.conf` (commented out at its default, `#key=value`: the template sets nothing, and finbot + never reads it), and the preferences table of `doc/configuration.md`. The tests catch a key + missing from the template or a default that differs, but not the docs (no test may read README.md or `doc/*.md`: CI + skips the tests for a change to the docs alone). - If Yahoo and Sharesight are unreachable (the cloud environment's network policy can block them), check behaviour against the offline fixtures in `tests/common.py`. - For a refactor, `util/record_outputs.py` (or `--against COMMIT`) records what every command, button and scheduled @@ -35,56 +39,88 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm record real requests. ## Code map (open the one file you need) -- `bot.py`: WSGI entry (route, verify, parse, then a thread of `lib/worker.py` per message or button press); SIGHUP - reload and SIGUSR1 cache flush, each in a thread. `lib/inbound.py`: a class per network that reads its updates - (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back), apart from a class per way they - arrive (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an - update however it arrived. +- `bot.py`, which puts the networks together: the WSGI entry (`route()` by path to a network's `WEBHOOK`, verify, + parse, then a thread of `lib/worker.py` per message or button press), served by waitress only when some network's + webhook is set; the pull listeners that finbot.env asks for (`listeners()`), a thread each; each network registered + at start (Telegram's webhook set or deleted, and its menu; Discord's slash commands); plain threads, no gevent; + SIGHUP reload and SIGUSR1 cache flush, each in a thread. `bot.take(network, update)` takes an update however it + arrived, and is the one place a press is acknowledged (the worker never does). An answer the network must have + before any reply (Discord's deferred interaction) carries the work in its `then`: started once the HTTP answer is + sent (`bot.Sent`) or, over the gateway, posted to its callback. +- Each network in three modules, flat in `lib/`. `_common.py`, what both ways in share: the API calls (`call`, + `try_call`; Discord's wait out its rate limits), the reader of its updates (`READER`: a `Message`, a `Press`, a + `Suggest` or a `Response` given straight back; `acknowledge` where the network waits for that: Telegram's toast, + Discord's interaction answer) and its bot (`TelegramBot`, `SlackBot`, `DiscordBot`). `_webhook.py`, the HTTP + mode: the path and how a request proves itself (`WEBHOOK`). The pull mode's persistent client, handing its updates + to `bot.take`: `telegram_longpolling.py` (`TelegramPoll`: getUpdates), `slack_websocket.py` (`SlackSocket`: Socket + Mode, over websocket-client), `discord_websocket.py` (`DiscordGateway`: discord.py's raw events). Telegram's + `register()`, in each mode's module, sets the webhook or deletes it; Discord's slash commands are registered at start + in both modes (`discord_common.register`, from `worker.slash_commands()`). What no network owns: `lib/inbound.py` + (`Message`, `Press`, `Suggest`, `Response`; the `Network` and `Webhook` bases), `lib/pull.py` (the `Listener` base, + which waits longer after each failure in a row: `backoff`), `lib/transports.py` (the `Transport` base, below). - `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, - keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the + keywords)`, which also makes `.help`, the Telegram command keyboard, kept current in DMs, and `slash_commands()`, which + every network gets (Telegram's menu, Slack's manifest from `util/slack_manifest.py`, Discord's registered commands); + `suggest(chat, text)` completes a slash command's arguments as they are typed (Discord's autocomplete), from the + cache alone. `read()` finds the command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a percent, a period and a keyword from either end, the words between; a handler raises `Usage` for arguments it cannot take, and gets a ticker with `ticker(args)`. `Request` (`request.reply(lines, chart, buttons)`); each command gets the typing notice while it works. `process_request(message)` and `process_callback(press)` take inbound's objects. -- `lib/reports/`: what finbot says. Scheduled (each has `run(dest=None, **kwargs)`, which posts to the chat that asked - or else the default chats, and `job(argv)`, its schedule-line parser): `price performance trades shorts cal rating - milestone reminder podcasts cleanup sharesight_warm yahoo_warm`; the top-level `price.py` and the rest are shims that - run them. Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who - .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). -- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or the default chats), - `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per way of posting (`TelegramBot`, - `SlackBot`, `SlackWebhook`, `DiscordWebhook`): splitting, captions, charts, buttons, edits, and the typing notice - (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop before posting. - `lib/telegram.py`: Bot API calls (`call`, `try_call`). -- `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), and what a - pressed button's data asks for (`parse`); each transport draws them. `lib/charts.py`: a chart with period buttons, - built by its kind's builder (`build`) for the command's first view (`worker.chart_reply`) and for each press. +- `lib/reports/`: what finbot says. Scheduled (each has `run(dest=None, **kwargs)`, which posts to the chat that asked, + the job's chat or else the home chats; `job(argv)`, its arguments' parser; and `ARGUMENTS`, what `.jobs add` lists): + `price performance trades shorts cal rating milestone reminder podcasts cleanup sharesight_warm yahoo_warm`. trades, + rating, milestone and podcasts keep a record per chat of what they told it (`state.load_for`, `state.edit_for`), + added to only by a scheduled run (`delivery.interactive(dest)` is false). + Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who .pie + .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). +- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, a chat by its key with + `chat(key)`, or each network's home chat as its bot), `deliver(dest, report)`, the one-line error report. + `lib/transports.py`: `Transport`, the base of each network's bot, every one taking charts and buttons: splitting, + captions, charts, buttons, edits, and the typing notice (`start_typing`: after half a second; Slack's ⏳ on the + message), which `text()` and `chart()` stop before posting. `DiscordBot`: a slash command's first message replaces + its "thinking…" through its `interaction` (application id/token), and a pressed message is edited through the + press's. +- `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), what a + pressed button's data asks for (`parse`), and the toast it gets at once (`toast`); each transport draws them. + `lib/charts.py`: a chart with period buttons, built by its kind's builder (`build`) for the command's first view + (`worker.chart_reply`) and for each press. - `lib/dialects.py`: each network's markup and message sizes. `lib/markets.py`: one table by market code (flag, currency, Yahoo suffix, Google code; Sharesight's codes, so TSE is Toronto), and `code(name)` for any name an exchange goes by. `lib/formats.py`: names, units, dates, percents in words. `lib/links.py`: stock links. `lib/render.py`: every chart, `Chart` specs and `draw()`. - `lib/yahoo.py lib/sharesight.py lib/shortman.py lib/simplywallst.py`: data sources. Yahoo's and Sharesight's answers - are cached in `DATA_DIR/cache` as sent, not parsed, so a parser fix needs no flush (`cache.cached`). + are cached in `DATA_DIR/cache` as sent, not parsed, so a parser fix needs no flush (`cache.cached`); `util/flush.py` + deletes cache files by when they were written (`--before`/`--after TIME`, or `all`), never the state. `yahoo.symbol(typed)` is Yahoo's symbol for a ticker as written, tried on the local exchange (`exchange_suffix`) when Yahoo does not know it bare: every ticker a chat gives goes through it. `sharesight.portfolios(name=None)` gives `Portfolio` objects: `holdings()`, `tickers()`, `gain(days)` (the portfolio's), `trades(days)`. -- `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; - change one with `with state.edit(name, default) as data:`, never load-then-save. `lib/watchlist.py`: the watchlist. +- `lib/log.py`: the log, by level (`error warn info debug`), as the `log_level` preference lets in; bot.py's `WholeLines` + writes each line whole, whichever threads log at once. `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; + change one with `with state.edit(name, default) as data:`, never load-then-save; a file of one record per chat with + `state.load_for(chat, name, default)` and `state.edit_for`. A chat's key is its network and id (`Transport.key`, + `telegram:-1001234`), the same in both modes. `lib/watchlist.py`: each chat's watchlist (`tickers(chat)`), and + `with_holdings(chat)`, the stocks a report in that chat covers; a report takes the chat from its destination. - `lib/config.py`: `config.env` (finbot.env, fixed at start) and `config.prefs` (preferences.conf, a new object on - each reload). `lib/jobs.py` + `lib/jobrun.py`: the scheduler, its shared state under one lock; each command runs in - a child process, stopped after `JOB_TIMEOUT`. + each reload). `lib/jobs.py`: the scheduler: each chat's jobs (`SCHEDULE_FILE`, by chat key; `add`, `delete`, + `toggle`, `run`, which `.jobs` calls through `jobs.current`) and the fixed `SYSTEM` jobs, its shared state under one + lock; each command runs in the bot's process, in a thread (`run_in_process`), reported once if still running after + `JOB_TIMEOUT` and left to finish (every request goes through `net`, with its timeouts: a test checks). + `lib/migrate.py`: once, the old schedule.cron, the watchlist, the plans and the reports' shared records become the + home chats'. - `tests/common.py`: `FinbotCase` (offline fixtures). `self.command('.price')` returns what was sent: `('text', lines)` or `('photo', size, caption)`. `wsgi_environ(path, body, **headers)` is a request for `bot.main`. `ProviderCase` runs the real parsers on the answers saved under `doc/` (`sample`), as `Resp` and `serve` return them; `temp_dir(self)` is a directory removed after the test, `bare_environment()` the environment without finbot's settings. - Which test file: commands and scheduled reports `test_reports`; failures, bad input, secrets in logs, signals - `test_edges`; webhook requests, Slack events and signatures `test_inbound`; Yahoo `test_yahoo`; Sharesight - `test_sharesight`; posting to each network `test_sending`; registering Telegram's webhook `test_telegram_api`; the - schedule file and job arguments `test_schedule`; the scheduler and manual runs `test_scheduler`; config - `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, - cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, and the docstring, stdout and - line-width checks `test_finbot`. + `test_edges`; webhook requests, Slack events, slash commands and signatures, and the Slack manifest `test_inbound`; + the pull modes `test_pull`; the Discord bot `test_discord`; Yahoo `test_yahoo`; Sharesight `test_sharesight`; posting + to each network `test_sending`; Telegram's webhook and command menu `test_telegram_api`; job lines, job arguments and + the migration `test_schedule`; the scheduler and `.jobs` in a chat `test_scheduler`; config `test_config`; charts + `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, cache and state files + `test_util`; data dir `test_datadir`; log levels, request lines and whole lines `test_edges`; chunking, git_version, and the docstring, stdout, line-width and no-pandas + checks `test_finbot`. ## Conventions - Read settings as `config.env.` / `config.prefs.` where they are used, never into a default or module @@ -98,18 +134,24 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - Every function and method has a docstring, tests included (a test checks): one line where it fits, reworded rather than spilling a few words onto a second. Tabs everywhere (`.editorconfig`). Every line stays within 120 columns, tabs counting 4 (an edit quotes whole lines); a test checks all but README.md, CLAUDE.md and `doc/`. -- Log with `print(..., file=sys.stderr)`. systemd discards stdout (`StandardOutput=null`), so the bot and its reports +- Log with `lib/log.py`, at a level, with print()'s arguments: `log.error` (failed and lost: a message not sent), + `log.warn` (failed and got over: a retry, a fallback, a refused request), `log.info` (what finbot does: a message + sent, a job run; `net` logs each request it sends), `log.debug` (cache misses, hits and writes). The + `log_level` preference sets what reaches the log; `lib/config.py` prints its own errors (it loads before + preferences exist). Everything goes to stderr: systemd discards stdout (`StandardOutput=null`), so the bot and its reports never print there (a test checks); only the util/ tools, run by hand, do. - A report gives its chart as a `render.Chart` spec, not a drawing: `deliver()` draws it only for a chat that takes charts, through `render.draw()`, which caches the image by the spec's hash (and the drawing code's). -- A chart renderer in `lib/render.py` takes its `Chart` and is listed in `RENDERERS` by kind; it gets `@_drawn` (the - bot draws in one native thread) and must not call another `@_drawn` function, or that thread waits on itself. +- A chart renderer in `lib/render.py` takes its `Chart` and is listed in `RENDERERS` by kind; it gets `@_drawn`, which + draws one chart at a time (matplotlib's caches are shared by every thread). - A new command: a `Command` in `COMMANDS` (its names, and what its arguments may hold), whose usage, section and - buttons make its `.help` lines and keyboard buttons, and the README command list. + buttons make its `.help` lines and keyboard buttons, and the command list of `doc/commands.md`. ## Saving tokens - Run the tests quietly: `.venv/bin/python -m unittest discover -s tests -b` prints only failures (about 0.4 KB when green). One module: `.venv/bin/python -m unittest tests.test_reports -b`. -- README.md is 36 KB: grep for the section instead of reading it whole. Each network's setup is in its own guide, - `doc/telegram.md`, `doc/slack.md` and `doc/discord.md`: a change to a network's setup or keys goes there. `doc/yahoo/` - and `doc/sharesight/` hold large API samples that the tests replay: inspect them with python, don't read them. +- README.md is short: what finbot is, a quick start, and links. The rest is in `doc/`, a file per topic: `commands.md` + (the commands and reports, 16 KB: grep for the section instead of reading it whole), `configuration.md` (the keys + and preferences), `scheduling.md`, `caching.md`, `install.md` (a Debian host), `docker.md`, and a guide per network, + `telegram.md`, `slack.md` and `discord.md`: a change to a network's setup or keys goes in its guide. `doc/yahoo/` and + `doc/sharesight/` hold large API samples that the tests replay: inspect them with python, don't read them. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..44c7cd2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# finbot in a container, on Debian with Debian's own python3 packages (no pip), as on a Debian host. Every optional +# library is in, so the image runs every network in either mode. The startup file and DATA_DIR stay outside the image: +# see compose.yaml and doc/docker.md. +FROM debian:trixie-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-requests python3-croniter python3-dateutil \ + python3-matplotlib python3-numpy python3-waitress python3-websocket python3-discord python3-nacl \ + ca-certificates tzdata \ + && rm -rf /var/lib/apt/lists/* + +# the bot's user: no home, no shell; the code stays root's, so the bot cannot change it +RUN groupadd --system --gid 10001 finbot \ + && useradd --system --uid 10001 --gid finbot --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin \ + finbot + +WORKDIR /opt/finbot +COPY . . +# compiled now, so that a read-only root filesystem costs nothing at start +RUN python3 -m compileall -q bot.py lib util + +ARG GIT_COMMIT=unknown +# the image has no .git: the log names the commit it was built from +ENV FINBOT_COMMIT=$GIT_COMMIT PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 DATA_DIR=/data HOME=/tmp +USER finbot + +# each start: copy the defaults missing from /data, check the startup file, and only then start the bot +CMD ["sh", "-c", "util/sync_defaults.py --apply && util/check_config.py /run/finbot.env && exec ./bot.py"] diff --git a/README.md b/README.md index 1a33be11..9d9e2039 100644 --- a/README.md +++ b/README.md @@ -1,682 +1,51 @@ # finbot -## Description - -### Features -* Discord, Slack and Telegram support -* [Sharesight](https://www.sharesight.com/au/) API features: - * Trade notifications - * Portfolio performance - * Holdings list -* Yahoo Finance data for Sharesight holdings: - * Analyst consensus changes - * Earnings date reminders - * Ex-dividend date warnings - * Highly shorted stock warnings (AU, US) - * Interday and premarket price movements over a defined threshold -* Interactive chat commands (Slack & Telegram): - * All the above reports - * Price history graph - * Stock lookup with price/valuation related stats and warnings - * Company profiles - * Shared watch list - * List stocks (holdings + watchlist) by P/E, Forward P/E, PEG, market cap, or Yahoo analyst consensus - -### Chat networks -What finbot can do on each network. Telegram takes everything; the others take what their APIs allow. - -| Feature | Telegram | Slack | Discord | -|---|---|---|---| -| Chat commands | ✓ | ✓ | ✗ (a webhook only posts) | -| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots) | ✗ | -| Charts in replies | ✓ | ✓ with `slackBotToken` | n/a | -| Charts in scheduled reports | ✓ | ✓ with `slackChannelID` and `slackBotToken` (the incoming webhook takes text only) | ✓ | -| Period and job buttons | ✓ | Period buttons, with the interactive bot | ✗ | -| No link previews | ✓ | ✓ | ✓ | -| Linked tickers | ✓ | ✓ | ✓ | -| Charts updated in place by their buttons | ✓ | ✓ | ✗ | - -**Setting up a network:** [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md). Each guide -covers the whole setup of that network, and where there is a choice between reports only and the interactive bot, it -compares the two before the steps. +finbot is a chat bot for Telegram, Slack and Discord that follows your [Sharesight](https://www.sharesight.com/au/) +portfolios and each chat's watchlist with Yahoo Finance data. It posts trades, price moves, earnings and ex-dividend +dates, analyst rating changes and more on a schedule, and answers commands with reports and charts. + +## Features +* Telegram, Slack and Discord, with no server to host: finbot fetches its updates itself (or, in HTTP mode, a network + posts them to your HTTPS server) +* Sharesight trades as they happen, portfolio performance and holdings +* Price moves past a threshold, earnings and ex-dividend dates, analyst rating changes, short interest, record highs +* Stock lookup, price history charts and comparisons, rankings by P/E, PEG, market cap or analyst rating +* Commands as messages (`.price`) or slash commands (`/price`), completed as they are typed where the network allows +* Each chat's own scheduled jobs and watchlist, managed in the chat (`.jobs`, `.watchlist`) +* Docker or a Debian host, hardened either way ![Screenshot of showing trade notifications on Slack](img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") -### Requirements -* Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key -* A Telegram bot, a Slack app or a Discord webhook (see the network's guide, above) -* Interactive commands require a public web server with domain name and matching certificate to proxy chat commands to the bot (see the [Telegram](doc/telegram.md#interactive-bot) and [Slack](doc/slack.md#interactive-bot) guides). -* Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) -* Python modules, pinned to versions with 3.13 support in `requirements.txt`: -``` -croniter python-dateutil requests gevent pandas matplotlib numpy -``` - -### Supported commands - -Where: -* _percent_ is a number, optionally followed by '%' -* _period_ is a number followed by 'd', 'w', 'm' or 'y', or 'ytd' -* _portfolio_ is a Sharesight portfolio name -* _symbol_ is a Yahoo Finance symbol; a bare one Yahoo does not know is tried again on your exchange (`exchange_suffix`: `BHP` may be `BHP.AX`) -``` -.symbol -.beta -.buy -.sell -.compare symbol symbol [symbol...] [period] -.dividend [period|symbol] -.earnings [period|symbol] -.forward pe [symbol|top|bottom|negative] -.history symbol -.holdings portfolio -.jobs [run number|command] -.marketcap [symbol|top|bottom] -.pe [symbol|top|bottom] -.peg [symbol|top|bottom|negative] -.performance [period] [portfolio] -.pie portfolio [sector] -.plan [your plan] -.premarket [percent|symbol] -.price [percent|symbol|top|bottom] [period] -.session [percent|symbol] -.shorts [percent|symbol] -.trades [period] [portfolio] -.watchlist [add|del symbol] -.who symbol -``` - -`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. - -Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying your configured chat networks of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. - -The other various reports can also run on a schedule (e.g. daily or weekly), or on demand through the interactive bot. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. Depending on how they're triggered, they will either report to all configured chat networks, or reply to the chat which triggered them. - -The interactive bot component requires you to host a web service on a domain with a trusted certificate. It subscribes to push updates from native Slack apps / Telegram bots, and reacts to certain regex seen in chat. It provides: -* Stock lookup (financials, company profile, price charts) -* Group maintainable watch list, picked up by the various reports -* Listing of portfolios and portfolio performance -* Listing of holdings sorted by various attributes -* Running the other stock reports on demand - -## Reports - -### Stock lookup -![Screenshot showing stock info on Slack](img/stockinfo.png?raw=true "Screenshot showing stock info on Slack") - -The stock lookup returns various stats relevant to a stock's valuation, growth and risk factors. - -#### Emoji legend -* The colored circle next to the symbol at the top indicates the current market state: - * 🔴 market closed - * 🟠 pre/post-market - * 🟢 normal trading -* The warning symbol ⚠️ indicates possible risk factors. This is not intended to replace due diligence, but to non-exhaustively highlight where additional diligence is due. Some risk factors include: - * Negative earnings ⚠️ - * Earnings in decline ⚠️ - * Negative cashflow ⚠️ - * High short interest ⚠️ - * High forward P/E ratio ⚠️ - * Net insider selling ⚠️ - * High debt-to-equity ratio ⚠️ -* The arrow grid indicates the directional change in the company earnings and revenue for each reporting period. A red arrow indicates the absolute earnings was negative: - * 🔼 earnings/revenue increased from the previous period - * 🔽 earnings/revenue decreased from the previous period - * 🔺 earnings increased from the previous period but remained negative - * 🔻 earnings decreased from the previous period and was negative - * ▪️ earnings/revenue matched the previous period - * ❌ data is missing for this period -* Using the example screenshot above, Crowdstrike reported: - * 🔼🔼🔼 revenue increased in all of the past three quarters - * 🔺🔻🔻 In the past three quarters, earnings initially increased, then decreased for two quarters, and was negative in every quarter - * 🔼🔼🔼 revenue increased in all of the past three years - * 🔻🔺🔻 In the past three years, earnings decreased, increased, then decreased, and was negative for all three years - -This report can only be run through the interactive bot. Example usage: -``` -.AAPL -``` -``` -@botname AAPL -``` - -### Trades -![trade update in Slack](img/trades.png?raw=true "Trade update in Slack") - -`trades.py` sends recent Sharesight trades to your configured chat services. -By default, this report searches Sharesight for trades from the past 28 days (`past_days`). You can override this in `preferences.conf`, or by providing a number as argument when triggered through the chat bot. - -Without persistent storage, set `past_days=0` to avoid duplicate trade notifications. In this case, the schedule must run the report exactly once per day, after market close. - -With persistent storage, the default of `past_days=28` suits. This is useful if Sharesight imports trades with past dates for any reason. Note that the initial run will notify on all historical trades for the `past_days` period. It is recommended to schedule it every 20 minutes. - -``` -past_days=28 -``` - -Cron trigger: -``` -trades.py -``` - -Interactive trigger: -``` -.trades [period] -``` -``` -@botname trades [period] -``` - -You can also specify a portfolio name to get trades for just that portfolio: -``` -.trades [period] [portfolio] -``` -``` -@botname trades [period] [portfolio] -``` - -### Price alerts -![Price chart triggered from Slack](img/finbot_price.png?raw=true "Price chart triggered from Slack") -![Price alert triggered by Cron](img/price.png?raw=true "Price alert triggered by Cron") - -`price.py` sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. - -If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. - -Config example: -``` -price_percent=5 -demote_volatile=True -``` - -Cron trigger: -``` -./price.py [premarket|midsession|interday|close|days (int)] -``` -When scheduled (Cron), the mode must be passed as an execution argument: -* If `premarket` is passed, it only reports on pre/post market price movements. -* If `midsession` is passed, it only reports for markets currently in session. This is intended to run from Cron to provide mid-session alerts for big price movements of your holdings. For example, it could be run twice per day 12 hours apart, to capture markets in different timezones. -* If `interday` is passed, it reports current price against the previous market close. -* If `close` is passed, it reports daily change in stocks whose market closed in the past 3 hours. -* If an integer [i] is passed, it reports current price against [i] days ago. - -Interactive trigger (pre-market): -``` -.premarket [percent] -``` -``` -@botname premarket [percent] -``` - -Interactive trigger (mid-session): -``` -.session [percent] -``` -``` -@botname session [percent] -``` - -Interactive trigger (interday). -``` -.price [percent] -``` -``` -@botname price [percent] -``` - -For interday, can also specify the time period. A bare number is the percent; a period carries its unit: `7d`, `2w`, `3m`, `1y` or `ytd`. For example, `.price 3 7d` lists the moves of 3% or more over the past week. -``` -.price [percent] [period] -``` - -Or a specific stock. In this case, the output includes a graph. -``` -.price [symbol] [period] -``` - -### Portfolio performance -![Screenshot of Portfolio performance notification on Slack](img/performance.png?raw=true "Screenshot performance notification on Slack") - -`performance.py` reports on Sharesight portfolio performance over a given period. The default is set by `past_days` in the `preferences.conf`, which can be overridden at runtime by specifying an integer as the argument. - -Config example: -``` -past_days=28 -``` - -Cron trigger: -``` -./performance.py 7 -``` - -Interactive trigger: -``` -.performance [period] -``` -``` -@botname performance [period] -``` - -Interactively, you can also specify a portfolio: -``` -.performance [period] [[portfolio] -``` -``` -@botname performance [period] [portfolio] -``` - -### Events calendar -![earnings message in Slack](img/earnings.png?raw=true "Earnings message in Slack") - -`cal.py` sends upcoming earnings and ex-dividend date alerts. The data is sourced from Yahoo! Finance. It reports on events up to `future_days` into the future. This is set in the `preferences.conf` for triggering the report from Cron, or can be specified as an argument when triggered through the chat bot. - -**Finance Explainer:** When a company releases its quarterly earnings report, the stock price may undergo a signficant positive or negative movement, depending on whether the company beat or missed market expectations. You may wish to hold off buying more of this stock until after its earnings report, unless you think the stock will beat market expectations. - -**Finance Explainer:** When a stock goes ex-dividend, the share price [typically drops](https://www.investopedia.com/articles/stocks/07/ex_dividend.asp) by the amount of the dividend paid. If you buy right before the ex-dividend date, you can expect an unrealised capital loss, plus a tax obligation for the dividend. Thus, you may wish to wait for the ex-dividend date before buying more of this stock. -``` -future_days=30 -``` - -Cron execution: -``` -./cal.py [earnings|ex-dividend] -``` - -Interactive trigger (earnings): -``` -.earnings [period] -``` -``` -@botname earnings [period] -``` - -Interactive trigger (ex-dividend): -``` -.dividend [period] -``` -``` -@botname dividend [period] -``` - -### Highly shorted stock warnings -![short warnings in Slack](img/shorts.png?raw=true "Short warnings in Slack") - -`shorts.py` sends short interest warnings. The data is sourced from Yahoo Finance and Shortman (AU). `shorts_percent` defines the alert threshold for the percentage of a stock's float shorted. This can be specified in the `preferences.conf` for running the report from Cron, or as an argument when triggered through the chat bot. - -**Finance Explainer:** A high short ratio indicates a stock is exposed to high risks, such as potential banktrupcy. It may also incentivise negative news articles which harm the stock price. If the market is wrong, however, risk tolerant investors may receive windfall gains. This report is intended to alert you to an above-average risk, and prompt you to investigate this stock more closely. -``` -shorts_percent=15 -``` - -Interactive trigger: -``` -.shorts [percent] -``` -``` -@botname shorts [percent] -``` - -### pe / forward pe / peg -![Forward PE report in Slack](img/finbot_fpe.png?raw=true "Forward PE report in Slack") - -List securities by P/E ratio, forward P/E ratio, or PEG ratio. A forward P/E is an analysts' estimate, so its lists take only stocks covered by at least `min_analysts` analysts (7 by default; see `preferences.conf`). - -Interactive trigger: -``` -.pe [SYMBOL] -.fpe [SYMBOL|bottom|negative] -.peg [SYMBOL|bottom|negative] -``` -``` -@botname pe [SYMBOL|top|bottom] -@botname fpe [SYMBOL|top|bottom|negative] -@botname peg [SYMBOL|top|bottom|negative] -``` - -### History -![Price history report in Slack](img/finbot_history.png?raw=true "Price history report in Slack") - -Shows the long term performance of a security. -It creates a 5Y chart, and then lists performance for several periods (5Y 3Y 1Y 6M 3M 1M 1W 1D) - -Interactive trigger: -``` -.history -``` -``` -@botname history -``` - -### Buy -![Buy ratings in Slack](img/finbot_buy.png?raw=true "Buy ratings in Slack") - -Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts (7 by default; see `preferences.conf`) - -Interactive trigger: -``` -.buy -``` -``` -@botname buy -``` - -### Sell -![Rating report in Slack](img/finbot_sell.png?raw=true "Rating report in Slack") - -Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts - -Interactive trigger: -``` -.sell -``` -``` -@botname sell -``` - -### Marketcap -![Marketcap report in Slack](img/finbot_marketcap.png?raw=true "Marketcap report in Slack") - -List the top (or bottom) stocks by market cap, in US dollars: market caps in other currencies are converted at Yahoo's exchange rates. With a symbol, its market cap in its own currency and in US dollars. - -Interactive trigger: -``` -.marketcap [top|bottom|SYMBOL] -``` -``` -@botname marketcap [top|bottom|SYMBOL] -``` - -### Consensus rating changes -![Rating change report in Slack](img/finbot_rating.png?raw=true "Rating change report in Slack") - -Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run from Cron. Requires persistent storage. - -``` -./rating.py -``` - -### Watchlist -![Shared watchlist in Slack](img/watchlist.png?raw=true "Shared watchlist in Slack") - -Tracks additional securities which are not in your Sharesight holdings. The various reports will include wishlist items as if they are holdings. - -It is stored in `var/finbot_watchlist.json` by default. It is a JSON array literal containing Yahoo symbols. Example: -``` -["2454.TW", "3217.TWO", "ASO.AX", "STEM"] -``` - -When the bot is run interactively, the watchlist is dynamic and can be edited by members of a chat group. In this case, there is no need to create/edit the file manually. - -Interactive trigger: -``` -.watchlist -``` -``` -.watchlist [add|del] [symbol] -``` -``` -@botname watchlist -``` -``` -@botname watchlist [add|del] [symbol] -``` - -## Installation (Linux) -``` -sudo $(which apt dnf yum) install git python3-pip -``` -``` -git clone https://github.com/robdevops/finbot.git ~/finbot -``` -``` -cd ~/finbot -``` -``` -pip3 install -r requirements.txt -``` -Or, with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: -``` -uv venv && uv pip install -r requirements.txt -``` - -## Setup -Configuration is split into two files -- finbot.env for startup environment variables, and preferences.conf for values that can be changed at runtime. - -Copy the shipped template: -``` -sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env -``` - -### `finbot.env` -**Load `finbot.env` with your launcher:** -* **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. -* **Docker:** `--env-file /opt/finbot.env`. Docker keeps quotes literally, so use none. Run `util/check_config.py /opt/finbot.env` first if you want the lint check. -* **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). -* **Scheduled jobs:** they are started by `bot.py` (see [Scheduling](#scheduling)), so they run on the same settings. For a one-off run, see [Running a report by hand](#scheduling). - -The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which -of the chat network keys it needs. - -| Key | Default | Description | -|---|---|---| -| `DATA_DIR` | `var/` in the checkout (`$STATE_DIRECTORY` under systemd `StateDirectory=`) | Where state and cache live | -| `ip` | `127.0.0.1` | Address the bot listens on | -| `port` | `5000` | Port the bot listens on | -| `scheduler` | `True` | Run the scheduled reports (`schedule.cron`) inside the bot | -| `scheduler_catchup` | `5` | Minutes: a run missed while the bot was down still fires if it was due this recently | -| `telegramChatID` | | Chat group or channel id that scheduled reports go to on Telegram | -| `sharesight_code` | | Sharesight API access | -| `sharesight_client_id` | | Sharesight API access | -| `sharesight_client_secret` | | Sharesight API access | -| `slack_webhook` | | Slack incoming webhook, for outbound reports | -| `discord_webhook` | | Discord webhook, for outbound reports | -| `telegramBotToken` | | Telegram bot token, for outbound reports | -| `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | -| `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`); `slack_webhook` is then not used | -| `telegramOutgoingWebhook` | | Telegram webhook path the bot accepts (interactive mode) | -| `telegramOutgoingToken` | | Secret token Telegram must present (interactive mode) | -| `slackOutgoingWebhook` | | Slack webhook path the bot accepts (interactive mode) | -| `slackOutgoingToken` | | Token Slack must present (interactive mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | -| `slackSigningSecret` | | Secret Slack signs each request with (interactive mode) | - -### `preferences.conf` items - -Copies preferences.conf and metadata files into the data directory (never overwrites) -``` -util/sync_defaults.py [--dest DIR] --apply -``` - -The preferences, read by the bot itself. A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. - -| Key | Default | Description | -|---|---|---| -| `past_days` | `28` | How many days back reports look | -| `future_days` | `30` | How many days ahead the events calendar looks | -| `price_percent` | `5` | Price alert threshold, percent | -| `shorts_percent` | `15` | Short interest alert threshold, percent | -| `min_analysts` | `7` | `.buy`, `.sell` and `.forward pe` list only stocks covered by at least this many analysts; with fewer, a consensus says little (thinly covered small caps are nearly all a perfect buy) | -| `demote_volatile` | `True` | Volatile (small cap, high beta) stocks need a bigger move to alert | -| `demote_leveraged` | `True` | Leveraged (2x, 3x) funds need a bigger move to alert | -| `volatility_multiplier` | `2` | How much bigger a move volatile stocks need | -| `exclude_portfolios` | | Sharesight portfolio IDs to leave out, space separated | -| `include_portfolios` | | Sharesight portfolio IDs to report on, space separated (empty: all that are not excluded) | -| `trades_use_yahoo` | `True` | Use Yahoo for trades; set `False` if Yahoo breaks | -| `graph` | `True` | Send charts with reports | -| `hyperlink` | `True` | Hyperlink tickers in reports | -| `hyperlinkFooter` | `True` | Add a website link footer to stock lookups | -| `hyperlinkProvider` | `google` | Where ticker links go: `google` or `yahoo` | -| `exchange_suffix` | `AX` | Regional: Yahoo's suffix for your exchange (`AX` for the ASX, `NZ`, `L`, `TO` and so on). A bare symbol Yahoo does not know is tried again with it, in every command that takes a symbol (`BHP` becomes `BHP.AX`); empty, a symbol is taken as written | -| `country_code` | `AU` | Regional: your country, for the reminder report | -| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report | -| `cache` | `True` | Cache Yahoo and Sharesight responses | -| `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | -| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm.py` runs, so the holdings never run out | -| `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | -| `debug` | `False` | Extra diagnostics in the log | -| `print_headers` | `False` | Log inbound request headers | - -A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. - -edit `preferences.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. - -**Upgrading from one `.env`?** finbot no longer reads it, and refuses to start while a `.env` is in the checkout, so its settings cannot be silently ignored. Run `util/split_env.py` (or `util/split_env.py /path/to/old.env` for one elsewhere). It reads either format, the old dotenv one (`key = 'value' # comment`) or plain `KEY=value`, writes `finbot.env.new` and `preferences.conf.new` next to the old file, never overwriting, and lists any key it dropped or did not recognise. Put `finbot.env` where your launcher loads it and `preferences.conf` in the data directory, then delete the old file. - -### Data directory -All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`), e.g. for a Docker bind mount that survives rebuilds: -``` -docker run -v /srv/finbot:/data -e DATA_DIR=/data --env-file /opt/finbot.env ... -``` -Reference data (`finbot_adr.json`, `finbot_sws_*.json`, `reminder.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: -``` -util/sync_defaults.py # dry run -util/sync_defaults.py --apply # copy missing files (including a starter preferences.conf) -util/sync_defaults.py --force # also replace changed files, keeping .bak copies -util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory -``` -`reminder.json` holds the reminders that `reminder.py` posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. - -With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. - -In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. - -### Sharesight -* Email Sharesight support to get an API key and add the [access details](https://portfolio.sharesight.com/oauth_consumers) to `finbot.env`. Example: -``` -sharesight_code=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f -sharesight_client_id=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f -sharesight_client_secret=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f -``` - -### Chat networks setup -Follow the guide for each network you use: [Telegram](doc/telegram.md), [Slack](doc/slack.md) or -[Discord](doc/discord.md). It covers the bot or webhook, its `finbot.env` keys, and for chat commands the HTTPS server in -front of finbot. - -### Portfolios -Portfolios are auto-discovered, including other people's portfolios which are shared to you. To exclude specific portfolios, add their IDs to `exclude_portfolios` in the `preferences.conf`: -``` -exclude_portfolios=100003 100004 -``` - -Alternatively, you can include only specific portfolios: -``` -include_portfolios=100001 100002 -``` - -### Caching -Many objects are cached for 12 hours by default (`cache_seconds`). Cache is controlled by the settings below. Trades IDs are stored on disk, but trade queries generally don't result in a cache hit for functional reasons. -``` -cache=True -cache_seconds=43200 -performance_cache_seconds=2400 -``` - -`performance_cache_seconds` (default 40 minutes) is the cache lifetime of Sharesight performance reports. - -Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The scheduled `cleanup.py` removes cache files after 30 days. - -To flush the cache, so that everything is fetched afresh, signal the running bot: it deletes its cache files (Yahoo and Sharesight -answers, charts, the Yahoo cookie) and keeps its state (watchlist, seen trades, ratings, schedule), then runs both warm jobs -(`sharesight_warm.py`, `yahoo_warm.py`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: -``` -sudo systemctl kill -s USR1 finbot # systemd -sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) -docker kill -s USR1 # Docker -kill -USR1 # anything else -``` -The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && -sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, `cleanup.py 0` deletes the same files. - -Two scheduled jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](#scheduling)): `sharesight_warm.py` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm.py` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm.py` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). A data directory with its own `schedule.cron` needs their lines: -``` -14,44 * * * * sharesight_warm.py -07 */8 * * * yahoo_warm.py -``` - -### Hyperlinks -These `preferences.conf` options relate to hyperlinks: - -* `hyperlink`, if set to `False`, disables all hyperlinks. Note: it can not disable syntax highlighting and false positives by chat clients that try to detect domains. Defaults to `True`. -``` -hyperlink=True -``` - -* `hyperlinkFooter` appends hyperlinks to the company website and to a Google search for the ticker on simplywall.st for the symbol lookup report. Possible values are `True` or `False`. Defaults to `True`. - -``` -hyperlinkFooter=True -``` - -* `hyperlinkProvider` configures where to send ticker hyperlinks. Available values are `google` and `yahoo`. If set to `google`, it is only used where practical, as Google does not publish earnings date, ex-dividend date, premarket price, or short interest figures. Defaults to `google`. -``` -hyperlinkProvider=google -``` -``` -hyperlinkProvider=yahoo -``` - -## Scheduling -The scheduled reports are started by `bot.py`, each run in its own child process (`python -m lib.jobrun`), so they use the same user, sandbox, environment and data directory as the bot, while heavy work (charts, large reports) cannot hold up incoming chat webhooks. A run that exceeds 5 minutes is stopped. There is no cron to set up, which matters under systemd `DynamicUser=` where only the bot's own user can read the state directory. **This is on by default.** Remove the equivalent entries from any old crontab before the first start, otherwise every job runs twice; or set `scheduler=False` in the environment file to keep using cron. - -The schedulable scripts are `trades.py`, `price.py`, `reminder.py`, `cal.py`, `shorts.py`, `performance.py`, `rating.py`, `milestone.py`, `podcasts.py ""`, `cleanup.py [days]` (deletes cache files older than `days`, default 30), `sharesight_warm.py` and `yahoo_warm.py` (they fill the cache: see [Caching](#caching)). Other jobs, such as a DNS updater or backups, stay in cron or systemd timers. - -The schedule is `defaults/schedule.cron`, written in **cron-compatible syntax**: five time fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists work) or an `@macro` such as `@daily`, then a finbot script and its arguments. Quote an argument that has spaces or special characters (a feed URL with `&`, say). Several commands separated by `;` run one after another. `MAILTO=`, `SHELL=`, `PATH=` and `HOME=` lines are ignored. Anything else a crontab allows (a user column, pipes, other programs) is not supported: such a line is reported at startup and skipped. A time that cannot be or never comes (minute 61, 30 February) is different: finbot refuses to start with it, and a reload with it is refused, leaving the running schedule as it was. Existing crontab lines can be pasted as they are: `> /dev/null` redirects, `{ a; b ;}` groups and a path before the script name are understood. -``` -# Every 20 minutes on weekdays -*/20 * * * Mon-Fri trades.py - -# Weekly, one after the other -28 21 * * Sat cal.py earnings; cal.py ex-dividend; price.py 7; performance.py 7; rating.py - -# New episodes, hourly -01 0-10,23 * * Mon-Fri podcasts.py "https://example.com/podcast.rss" -``` -The default times suit a server set to UTC. Like cron, they follow the server's own time zone; put `CRON_TZ=Australia/Melbourne` on a line of its own above a job to pin another zone. - -**To change a job**, copy the schedule to the data directory, edit the copy, and reload (no restart, so the bot stays up): -``` -util/sync_defaults.py --apply # copies schedule.cron to the data directory if it is not there yet -sudoedit /var/lib/finbot/schedule.cron -sudo systemctl reload finbot # the bot re-reads the file and logs how many jobs it loaded -``` -The bot logs the number of jobs when it starts or reloads, and each job as it runs (`journalctl -u finbot`); a line it cannot use is reported by name. If the edited file has no usable line the previous schedule keeps running. - -**To run a job from chat**, send `.jobs` (also `.cron`, `.crons` or `.schedule`): it lists the scheduled commands, each once and without times, comments or `.py`, and on Telegram and Slack (with its buttons set up) adds a button per command. The system jobs (`cleanup`, `podcasts`, `sharesight_warm` and `yahoo_warm`) follow under _System_, without a number or a button: they run on their schedule only. `.jobs run 2` (or tapping the button) runs that command now, and `.jobs run price.py close` runs it by name; anything not in the schedule is refused, and so is a chain such as `a; b` (run them one at a time). A manual run is exactly a cron run: it reports to all configured chat networks, updates the same state (seen trades, ratings and so on), and does not change the schedule. Anyone who can message the bot can do this; a command that is already running is not started twice. - -Behaviour worth knowing: -* A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. -* A job still running when it is next due is skipped. A command still running after 5 minutes is stopped and reported, and the rest of its line still runs: each command has 5 minutes of its own. -* Failures are reported to your chat channels, as before. -* With `scheduler=False` nothing is scheduled by the bot, so you can keep using cron or systemd timers and run the scripts yourself. - -**Running a report by hand.** Under `DynamicUser=yes` you cannot simply log in as the bot's user, so run the script as a transient unit with a dynamic user and the same state directory: -``` -sudo systemd-run --pty --wait --collect -p DynamicUser=yes -p StateDirectory=finbot -p EnvironmentFile=/opt/finbot.env /opt/finbot/price.py 7 -``` -The script prints its output on your terminal and sends its report to your chat channels as usual. The transient unit gets its own dynamic user, and systemd may re-own the state directory to it for the run: check `ls -ln /var/lib/private/finbot` afterwards and `systemctl restart finbot` if the bot can no longer write there. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports inside the bot, without any of this. Outside systemd, run a script from the checkout as your own user with the settings loaded into the shell: `set -a; . /opt/finbot.env; set +a; ./price.py 7` (this works while no value in the file contains spaces). - -## Daemonize (systemd) -`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy `finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of [`finbot.env`](#setup) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start it. `systemctl reload finbot` re-reads `preferences.conf` and the [schedule](#scheduling) without a restart; a change to `finbot.env` needs `systemctl restart finbot`: - -``` -sudo cp -v finbot.service /etc/systemd/system/ -``` -``` -sudo sed -i 's/CHANGEME/YOUR LINUX USERNAME/' /etc/systemd/system/finbot.service -``` -``` -sudo systemctl daemon-reload -``` -``` -sudo systemctl enable finbot --now -``` -You can now monitor the bot's stderr with : -``` -journalctl -fu finbot -``` +## Quick start +With Docker: +1. `git clone https://github.com/robdevops/finbot /opt/finbot` +2. `sudo /opt/finbot/util/docker_setup.sh` +3. Set the keys of each network you use in `/opt/finbot.env` (below) +4. `cd /opt/finbot && sudo docker compose up -d --build` +5. `journalctl CONTAINER_NAME=finbot -f` + +To install on a Debian host with systemd instead, see [doc/install.md](doc/install.md). + +Each network needs a bot of its own, and finbot fetches its updates with no server (its pull mode): +* **Telegram:** create the bot with [@BotFather](https://telegram.me/BotFather), set `telegramBotToken` and + `telegramChatID`, restart finbot ([guide](doc/telegram.md)). +* **Slack:** create the app from the manifest that `util/slack_manifest.py` prints, add an app-level token, set + `slackBotToken`, `slackAppToken` and `slackChannelID`, restart finbot ([guide](doc/slack.md)). +* **Discord:** create the application and its bot, turn on its Message Content intent, set `discordBotToken` and + `discordChannelID`, restart finbot ([guide](doc/discord.md)). + +Then send `.help` in the chat. + +## Documentation +* [Commands and reports](doc/commands.md), with screenshots +* [Scheduling](doc/scheduling.md): each chat's jobs, and the reports they can run +* [Configuration](doc/configuration.md): `finbot.env` and `preferences.conf`, the data directory, Sharesight +* [Caching](doc/caching.md) +* Installing: [Docker](doc/docker.md) · [a Debian host with systemd](doc/install.md) +* Networks: [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md) ## Suggestions -* Is my code is doing something the hard way? -* Something important is missing from this README? +* Is my code doing something the hard way? +* Is something important missing from the docs? Log an [issue](https://github.com/robdevops/finbot/issues)! diff --git a/bot.py b/bot.py index e7f4ab2e..0ef8ee33 100755 --- a/bot.py +++ b/bot.py @@ -5,19 +5,24 @@ # ignored until startup is done (the reload handler is installed last): its default action kills the bot signal.signal(signal.SIGHUP, signal.SIG_IGN) signal.signal(signal.SIGUSR1, signal.SIG_IGN) # likewise the cache flush -import gevent.monkey -gevent.monkey.patch_all() -import collections, json, os, re, sys, traceback +import collections, dataclasses, json, os, re, sys, threading, time, traceback from pathlib import Path -from gevent import pywsgi -import threading +from lib import log from lib import config +from lib import discord_common +from lib import discord_webhook +from lib import discord_websocket from lib import inbound from lib import worker from lib import jobs +from lib import migrate from lib import net -from lib import telegram +from lib import slack_webhook +from lib import slack_websocket +from lib import telegram_common +from lib import telegram_longpolling +from lib import telegram_webhook from lib import delivery from lib import render from lib.reports import cleanup @@ -25,7 +30,12 @@ ROUTED = 'finbot.routed' # a request's WSGI environ: the network whose webhook path it came to (the path is a secret) AUTHENTICATED = 'finbot.authenticated' # set once it has passed that network's check MAX_BODY = 1 << 20 # bytes in a request body, at most: an update from Telegram or Slack takes a few thousand +WEBHOOKS = (telegram_webhook.WEBHOOK, slack_webhook.WEBHOOK, discord_webhook.WEBHOOK) # HTTP: each network posts +LISTENERS = (telegram_longpolling.TelegramPoll, slack_websocket.SlackSocket, discord_websocket.DiscordGateway) # pull _updates_seen = collections.OrderedDict() # (service, update id) of the last updates taken, oldest first +_updates_lock = threading.Lock() # updates arrive on several threads: a resent one may race its first copy +DOUBLED = 1 # seconds in which the same message, from the same person in the same chat, is taken once +_said = {} # (service, chat, user, text) of the messages taken in the last DOUBLED seconds: when def seen_before(service, update_id, keep=1000): """Whether this update was taken already: Telegram's update_id, Slack's event_id. Slack resends an event it got no @@ -34,11 +44,35 @@ def seen_before(service, update_id, keep=1000): if update_id is None: return False key = (service, update_id) - if key in _updates_seen: - return True - _updates_seen[key] = None - if len(_updates_seen) > keep: - _updates_seen.popitem(last=False) + with _updates_lock: + if key in _updates_seen: + return True + _updates_seen[key] = None + if len(_updates_seen) > keep: + _updates_seen.popitem(last=False) + return False + +def route(path): + """The webhook whose path this is, or None.""" + return next((hook for hook in WEBHOOKS if hook.path() and path == hook.path()), None) + +def listeners(): + """The listeners finbot.env asks for, each network's with its bot token and no webhook URL (Slack's also needs its + app-level token): finbot fetches the updates of those networks itself.""" + return [kind() for kind in LISTENERS if kind.wanted()] + +def doubled(message): + """Whether this message repeats one that the same person sent in the same chat in the last DOUBLED seconds: + Telegram's mobile app can send a command twice (/start in a new chat), each its own update, and one answer is + enough.""" + key = (message.service, message.chat_id, message.user, message.text.strip()) + now = time.monotonic() + with _updates_lock: + for old in [said for said, at in _said.items() if now - at > DOUBLED]: + del _said[old] + if key in _said: + return True + _said[key] = now return False def main(environ, start_response): @@ -54,8 +88,8 @@ def main(environ, start_response): status = '200 OK' else: where = environ.get(ROUTED) or repr(environ.get('PATH_INFO')) - print(f"request to {where} failed:", delivery.error_line(e), file=sys.stderr) - print(net.redact(traceback.format_exc()), end='', file=sys.stderr) + log.error(f"request to {where} failed:", delivery.error_line(e)) + log.error(net.redact(traceback.format_exc()), end='') status = '500 Internal Server Error' try: start_response(status, [('Content-type', 'text/plain')], sys.exc_info()) @@ -68,69 +102,128 @@ def handle(environ, start_response): comes from that network (before its body is read, when the proof is a header), read the update (CONTENT_LENGTH bytes, up to MAX_BODY) and take it (take): the network gets its answer at once.""" def respond(response): - """Answer the request: a status line and a body.""" + """Answer the request: a status line and a body (logged at INFO, naming the network, not the path). Work that + must wait for the answer starts once the server has sent it.""" + network = environ.get(ROUTED) or 'no network' + log.info(f"{environ['REQUEST_METHOD']} from {network}: we answered {response.status}") start_response(response.status, [('Content-type', 'application/json')]) - return [response.body] - def print_headers(): - """Log the request headers, Telegram's secret header redacted.""" + return Sent([response.body], response.then) if response.then else [response.body] + def print_headers(say=log.info): + """Log the request headers (at INFO, or as say() logs), Telegram's secret header redacted.""" for name, value in sorted(environ.items()): - print((name, '' if name == 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' else value), file=sys.stderr) + say((name, '' if name == 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' else value)) if config.prefs.print_headers: print_headers() path = environ.get('PATH_INFO', '') - source = inbound.route(path) + source = route(path) if source is None: - print("Unknown URI", repr(path), file=sys.stderr) + log.warn("Unknown URI", repr(path)) return respond(inbound.NOT_FOUND) environ[ROUTED] = source.service if why := source.refuse_unread(environ): # strangers are turned away before their body is read - print_headers() - print("Fatal:", why, file=sys.stderr) + print_headers(log.warn) + log.warn("refused:", why) return respond(inbound.FORBIDDEN) length = environ.get('CONTENT_LENGTH') or '0' if not length.isdigit() or int(length) > MAX_BODY: - print(f"[{source.service}]: a body of {length!r} bytes refused unread", file=sys.stderr) + log.warn(f"[{source.service}]: a body of {length!r} bytes refused unread") return respond(inbound.TOO_LARGE if length.isdigit() else inbound.BAD_REQUEST) body = environ['wsgi.input'].read(int(length)) # the body and no more: a server need not end the stream there try: update = source.decode(body) except ValueError: - print(f"[{source.service}]: request body is not a JSON object", file=sys.stderr) + log.warn(f"[{source.service}]: request body is not a JSON object") return respond(inbound.BAD_REQUEST) if why := source.refuse(environ, body, update): shown = {k: '' if k == 'token' else v for k, v in update.items()} # Slack's verification token - print("warning:", why, file=sys.stderr) - print(f"[{source.service}]: refused update", json.dumps(shown, indent=4, default=str), file=sys.stderr) - return respond(inbound.FORBIDDEN) + log.warn("warning:", why) + log.warn(f"[{source.service}]: refused update", json.dumps(shown, indent=4, default=str)) + return respond(source.REFUSED) environ[AUTHENTICATED] = True return respond(take(source.network, update)) +class WholeLines: + """stderr, a whole line at a time. Threads log at once (a report's fetches, say), and print() writes its arguments, + the spaces between them and the newline one by one, so their lines would mix ("TTL:cache hit: 34254 s/var/..."): + each thread's pieces wait in a buffer of its own until its line ends, and the line goes out in one write.""" + def __init__(self, stream): + """stream: where the lines go.""" + self.stream = stream + self.held = threading.local() # each thread's line so far + self.lock = threading.RLock() # re-entrant: a signal handler that logs may interrupt a write + + def write(self, text): + """Hold text until its line ends, then write the whole line; returns its length, as a stream does.""" + lines, newline, rest = (getattr(self.held, 'text', '') + text).rpartition('\n') + self.held.text = rest + if newline: + with self.lock: + self.stream.write(lines + newline) + self.stream.flush() + return len(text) + + def flush(self): + """Write what this thread holds, though its line has not ended.""" + text, self.held.text = getattr(self.held, 'text', ''), '' + with self.lock: + if text: + self.stream.write(text) + self.stream.flush() + + def __getattr__(self, name): + """Anything else as the stream has it: fileno, encoding, isatty.""" + return getattr(self.stream, name) + +class Sent(list): + """A response body that starts some work once the server has sent it: WSGI calls close() then.""" + def __init__(self, body, then): + """body: the chunks; then: the work.""" + super().__init__(body) + self.then = then + + def close(self): + """The answer is out: start the work.""" + self.then() + def take(network, update): """An update from a network (an inbound.Network reads it), however it arrived: what it says goes to lib/worker in a - thread of its own (dispatch), unless it is a repeat. Returns the inbound.Response for the network.""" + thread of its own (dispatch), unless it is a repeat (the same update resent, or the same message sent again at once: + doubled). It is acknowledged first, the one place that does + (Network.acknowledge: Telegram's toast for a press). Returns the inbound.Response for the network; when the network + must have it before any reply (Discord's interactions), the work waits in its `then`, for whoever delivers it.""" event = network.parse(update) if isinstance(event, inbound.Response): return event + if isinstance(event, inbound.Suggest): # answered here and at once, while the command is typed + return network.suggestions(worker.suggest(f'{event.service}:{event.chat_id}', event.text)) if seen_before(network.service, network.update_id(update)): - print(f"[{network.service}]: update {network.update_id(update)} seen already, not run again", file=sys.stderr) + log.info(f"[{network.service}]: update {network.update_id(update)} seen already, not run again") + return inbound.OK + if isinstance(event, inbound.Message) and not event.interaction and doubled(event): # each interaction is answered + log.info(f"[{network.service}]: {event.user} sent {event.text!r} again within {DOUBLED} s, not run again") return inbound.OK + answer = network.acknowledge(event) + if answer is not None: + return dataclasses.replace(answer, then=lambda: dispatch(event)) dispatch(event) return inbound.OK def dispatch(event): - """Hand a message or a button press to lib/worker, in a thread of its own.""" + """Hand a message or a button press to lib/worker, in a thread of its own, which it returns.""" if isinstance(event, inbound.Press): - print(f"[{event.service.capitalize()} button]:", event.user, "chat", event.chat_id, event.data, file=sys.stderr) + log.info(f"[{event.service.capitalize()} button]:", event.user, "chat", event.chat_id, event.data) target = worker.process_callback else: - if config.prefs.debug: - print(f"[{event.service.capitalize()}]:", event.user, event.text, file=sys.stderr) + log.info(f"[{event.service.capitalize()}]:", event.user, event.text) target = worker.process_request - threading.Thread(target=target, args=(event,)).start() + thread = threading.Thread(target=target, args=(event,)) + thread.start() + return thread def git_version(checkout=None): """Short hash of the checked-out commit, read straight from the .git directory: no git binary to run, and no - "dubious ownership" refusal when the service user does not own the checkout. 'unknown' when it cannot be read.""" + "dubious ownership" refusal when the service user does not own the checkout. Without one (a Docker image), the + commit it was built from (FINBOT_COMMIT); 'unknown' when neither can be read.""" try: git = Path(checkout or config.REPO_DIR) / '.git' if git.is_file(): # a linked worktree or submodule: .git is a file saying "gitdir: " @@ -145,9 +238,12 @@ def git_version(checkout=None): else: # the branch is only in packed-refs packed = (common / 'packed-refs').read_text().splitlines() head = next((line.split()[0] for line in packed if line.endswith(' ' + ref)), '') - return head[:7] if re.fullmatch(r'[0-9a-f]{40}|[0-9a-f]{64}', head) else 'unknown' + if re.fullmatch(r'[0-9a-f]{40}|[0-9a-f]{64}', head): + return head[:7] except (OSError, IndexError): - return 'unknown' + pass + built = os.environ.get('FINBOT_COMMIT', '') + return built[:7] if re.fullmatch(r'[0-9a-f]{7,64}', built) else 'unknown' scheduler = None _signal_work = threading.Lock() # one signal's work at a time: a second reload waits for the first @@ -162,18 +258,18 @@ def run(): return lambda *args: threading.Thread(target=run, daemon=True).start() def reload_all(*args): - """SIGHUP: re-read preferences.conf and, if the scheduler is running, schedule.cron. Each reports its own errors; a - bad preferences.conf leaves the running config untouched.""" + """SIGHUP: re-read preferences.conf; a bad one is reported and leaves the running config untouched. A key set in + defaults/preferences.conf, which is never read, is pointed at the file to set it in. (Each chat's jobs need no + reload: a change made with .jobs counts at once.)""" try: changes = config.reload() for change in changes: - print('config:', change, file=sys.stderr) - print(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}", - file=sys.stderr) + log.info('config:', change) + log.info(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}") except config.ConfigError as e: config.print_errors(e) - if scheduler: - scheduler.reload() + if note := config.template_note(config.preferences_path): + log.warn('WARN config:', note) def flush_cache(*args): """SIGUSR1 (`systemctl kill -s USR1 finbot`): delete the cache and keep the state (watchlist, seen trades, ratings, @@ -181,57 +277,71 @@ def flush_cache(*args): for them).""" try: removed = cleanup.run(days=0) - print(f"cache: flushed {removed} file{'' if removed == 1 else 's'} from {config.env.cache_dir}", - file=sys.stderr) + log.info(f"cache: flushed {removed} file{'' if removed == 1 else 's'} from {config.env.cache_dir}") except Exception as e: - print('ERROR', delivery.error_line(e, 'cache flush'), file=sys.stderr) + log.error('ERROR', delivery.error_line(e, 'cache flush')) jobs.start_thread(jobs.warm_now, scheduler) +def http_server(): + """The HTTP server for the networks that post to finbot (those whose webhook is set), bound to ip and port, or None + when none does. waitress is imported only then. Raises OSError when the port cannot be had.""" + if not any(hook.path() for hook in WEBHOOKS): + return None + import waitress + return waitress.create_server(main, host=config.env.ip, port=config.env.port) + if __name__ == '__main__': - print(f'Starting finbot {git_version()}', file=sys.stderr) + sys.stderr = WholeLines(sys.stderr) # threads log at once: each line goes out whole + log.info(f'Starting finbot {git_version()}') try: config.check_startup_environment() except config.ConfigError as e: config.print_errors(e) sys.exit(1) for line in config.startup_report(): - print('config:', line, file=sys.stderr) + log.info('config:', line) for line in config.startup_warnings(): - print('ERROR config:', line, file=sys.stderr) + log.warn('WARN config:', line) if os.getuid() == 0: - print("Running as superuser. This is not recommended.", file=sys.stderr) - if config.env.scheduler: # read before anything else happens: an impossible time in it stops the start - scheduler = jobs.Scheduler(lambda: config.data_file('schedule.cron')) - try: - if not scheduler.load(): - scheduler = None - except jobs.ImpossibleTime as e: - for line in e.lines: - print('ERROR schedule:', line, file=sys.stderr) - print('ERROR schedule: finbot will not start until every time in the schedule can come', file=sys.stderr) - sys.exit(1) + log.warn("Running as superuser. This is not recommended.") + migrate.run() # once: what was kept for all chats becomes the home chats', schedule.cron among it + scheduler = jobs.Scheduler() + scheduler.load() + render.preload() + try: + server = http_server() # bound before Telegram is given the URL + except OSError as e: + log.error(e) + sys.exit(1) + if server: + log.info(f'Opening socket on http://{config.env.ip}:{server.effective_port}') else: - print('Scheduled reports are off (scheduler=False)', file=sys.stderr) - render.render_off_hub() # charts drawn in a thread of their own: no chat waits while another's chart is drawn - httpd = pywsgi.WSGIServer((config.env.ip, config.env.port), main) - httpd.secure_repr = False if config.prefs.debug else None - print(f'Opening socket on http://{config.env.ip}:{config.env.port}', file=sys.stderr) - if config.env.telegramBotToken and config.env.telegramOutgoingWebhook: - try: - inbound.TELEGRAM.bot_name = telegram.register() # here only: the job processes import lib.telegram too + log.info('No webhook is set: finbot takes no HTTP requests') + if config.env.telegramBotToken: + try: # here only, never on import: the webhook set (HTTP mode), or deleted for getUpdates (pull mode) + (telegram_webhook if config.env.telegramOutgoingWebhook else telegram_longpolling).register() + telegram_common.READER.bot_name = telegram_common.bot_name() except Exception as e: - print('ERROR', delivery.error_line(e, 'telegram webhook'), file=sys.stderr) + log.error('ERROR', delivery.error_line(e, 'telegram')) sys.exit(1) - if config.env.telegramBotToken: + telegram_common.set_commands(worker.slash_commands(aliases=False)) # the menu; /profile and /job work typed threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background - if scheduler: - jobs.current = scheduler # for the .jobs chat command - threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs - # Startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron - gevent.signal_handler(signal.SIGHUP, in_thread(reload_all)) - gevent.signal_handler(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache - try: - httpd.serve_forever() - except OSError as e: - print(e, file=sys.stderr) - sys.exit(1) + if config.env.discordBotToken: + discord_common.version = git_version() # in each request's User-Agent, as Discord asks + try: + discord_common.READER.bot_id = discord_common.register(worker.slash_commands()) + except Exception as e: + log.error('ERROR', delivery.error_line(e, 'discord')) + sys.exit(1) + jobs.current = scheduler # for the .jobs chat command + threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs + for listener in listeners(): # the networks without a webhook: finbot fetches their updates + threading.Thread(target=listener.run, args=(take,), daemon=True).start() + # Startup is done. systemctl reload finbot: pick up an edited preferences.conf + signal.signal(signal.SIGHUP, in_thread(reload_all)) + signal.signal(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache + log.info('finbot is running: SIGHUP reloads, SIGUSR1 flushes the cache') + if server: + server.run() + while True: # the threads do the work; this one waits for signals + signal.pause() diff --git a/bot_send_stdin.py b/bot_send_stdin.py deleted file mode 100755 index 3e9851a2..00000000 --- a/bot_send_stdin.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -from lib import delivery - -if os.isatty(sys.stdin.fileno()): - raise RuntimeError("No stdin input detected (not piped or redirected)") - -payload = [line.strip() for line in sys.stdin if line.strip()] - -for dest in delivery.default_chats(): - delivery.deliver(dest, delivery.Report(payload)) - diff --git a/cal.py b/cal.py deleted file mode 100755 index e631d21d..00000000 --- a/cal.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./cal.py [arguments]: runs lib/reports/cal.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/cleanup.py b/cleanup.py deleted file mode 100755 index 530ee071..00000000 --- a/cleanup.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./cleanup.py [arguments]: runs lib/reports/cleanup.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..42a61935 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,35 @@ +# finbot with Docker Compose (doc/docker.md). The configuration lives beside the checkout, outside the image and the +# container: ../finbot.env, the startup file, and ../finbot-data, DATA_DIR (preferences.conf, state and cache), both +# made by util/docker_setup.sh. Build and start: docker compose up -d --build +services: + finbot: + build: + context: . + args: + GIT_COMMIT: ${GIT_COMMIT:-unknown} # GIT_COMMIT=$(git rev-parse HEAD), for the version in the log + image: finbot + container_name: finbot + restart: unless-stopped + env_file: ../finbot.env + environment: + DATA_DIR: /data + ip: 0.0.0.0 # HTTP mode listens inside the container: publish its port below + volumes: + - ../finbot-data:/data + - ../finbot.env:/run/finbot.env:ro # checked before the bot starts + # ports: # HTTP mode only, on the host's loopback, behind your reverse proxy + # - 127.0.0.1:5000:5000 + logging: + driver: ${FINBOT_LOG_DRIVER:-journald} # json-file where the host has no journald + # hardening: the bot needs none of what these take away + user: "10001:10001" + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + pids_limit: 256 + mem_limit: 400m + memswap_limit: 400m # no swap on top + init: true # passes SIGHUP and SIGUSR1 on to the bot, and reaps children diff --git a/defaults/finbot.env b/defaults/finbot.env index cdbb8783..2f9e3b7a 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -1,4 +1,4 @@ -# finbot.env: THE STARTUP FILE. Where finbot runs, who it talks to, where it reports, and whether it schedules. +# finbot.env: THE STARTUP FILE. Where finbot runs, who it talks to, and where it reports. # The bot never reads this file. Your launcher loads it into the bot's environment, and a change needs a RESTART: # systemd: EnvironmentFile=/opt/finbot.env (in finbot.service) # docker: --env-file /opt/finbot.env @@ -15,16 +15,15 @@ # /var/lib/finbot). # DATA_DIR=/var/lib/finbot -# The listening socket +# The listening socket, for a network in http mode (its webhook URL set below) ip=127.0.0.1 port=5000 -# Run the scheduled reports (schedule.cron) inside the bot, and how many minutes late a run missed while the bot was -# down may still fire -scheduler=True +# How many minutes late a scheduled report missed while the bot was down may still fire scheduler_catchup=5 -# Where scheduled reports go on Telegram: the chat group or channel id +# Telegram's home chat, a group or channel id: finbot's first start gives it the default jobs (.jobs), and errors are +# reported there telegramChatID= # Sharesight API access @@ -32,17 +31,25 @@ sharesight_code= sharesight_client_id= sharesight_client_secret= -# Chat networks: outbound -slack_webhook= -discord_webhook= +# Chat networks. A Telegram bot token is enough for reports and commands: finbot fetches its updates (pull mode) telegramBotToken= slackBotToken= -# Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken; slack_webhook is then unused) +# Slack's home channel id, as Telegram's home chat above (needs slackBotToken) slackChannelID= +# Slack's app-level token (xapp-) for commands without a server: finbot opens a socket to Slack (Socket Mode) +slackAppToken= +# Discord's bot token: reports, and commands over the gateway (pull mode); and its home channel id, as Telegram's +discordBotToken= +discordChannelID= -# Chat networks: inbound (interactive mode) +# Chat networks: inbound over HTTPS (http mode). Telegram: set both to have it post its updates to your server +# instead of finbot fetching them telegramOutgoingWebhook= telegramOutgoingToken= +# Discord: the Interactions Endpoint URL and the app's public key, for slash commands and buttons over HTTPS instead of +# the gateway +discordOutgoingWebhook= +discordPublicKey= slackOutgoingWebhook= slackOutgoingToken= slackSigningSecret= diff --git a/defaults/preferences.conf b/defaults/preferences.conf index b8b69bed..4440d9e1 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -1,53 +1,57 @@ -# Preferences for finbot: how the bot behaves. The bot reads this file itself, at start and on every reload. -# Apply an edit without a restart: sudo systemctl reload finbot (docker: docker kill -s HUP ) +# Preferences for finbot: how the bot behaves. This file is only the template, which finbot never reads: the bot +# reads /preferences.conf, at start and on every reload, and without one runs on its built-in defaults. +# util/sync_defaults.py --apply copies this file there. +# Each key is commented out at its built-in default: to change one, remove its # and edit the value. +# Apply an edit without a restart: sudo systemctl reload finbot (docker: docker compose kill -s HUP finbot) # Check an edit first: util/check_config.py /path/to/finbot.env # Flat KEY=value lines, one per line. This is NOT an INI file: no [sections]. No spaces around "=", no quotes, no # trailing comments; put comments on their own line. A key that is not listed here is an error and finbot refuses to # start. -# Each value shown is the built-in default. The startup keys (credentials, ip, port, telegramChatID, scheduler, ...) -# are NOT set here: they live in finbot.env, the startup file, which the launcher loads and the bot never reads. -# Changing them needs a restart. +# The startup keys (credentials, ip, port, telegramChatID, ...) are NOT set here: they live in finbot.env, the startup +# file, which the launcher loads and the bot never reads. Changing them needs a restart. # Reports: how far back and forward to look, and when to alert -past_days=28 -future_days=30 -price_percent=5 -shorts_percent=15 +#past_days=28 +#future_days=30 +#price_percent=5 +#shorts_percent=15 # .buy, .sell and .forward pe list only stocks that at least this many analysts cover: with fewer, a consensus says # little -min_analysts=7 -# price alerts: volatile and leveraged stocks need a bigger move; the multiplier is how much bigger -demote_volatile=True -demote_leveraged=True -volatility_multiplier=2 +#min_analysts=7 +# price alerts: volatile and leveraged stocks need a bigger move; volatility_multiplier is how much bigger for +# volatile ones +#demote_volatile=True +#demote_leveraged=True +#volatility_multiplier=2 # Which portfolios to report on, space separated Sharesight portfolio IDs (empty: all that are not excluded) -exclude_portfolios= -include_portfolios= +#exclude_portfolios= +#include_portfolios= # Data sources: set trades_use_yahoo to False if Yahoo breaks -trades_use_yahoo=True +#trades_use_yahoo=True # Display -graph=True -hyperlink=True -hyperlinkFooter=True -hyperlinkProvider=google +#graph=True +#hyperlink=True +#hyperlinkFooter=True +#hyperlinkProvider=google # Regional: where you are. # exchange_suffix: a ticker Yahoo does not know is tried again on your exchange, by Yahoo's suffix for it (AX: BHP is # tried as BHP.AX; NZ, L, TO and so on for others). Empty: a ticker is taken as written. -# country_code and timezone: the reminder report's country and time zone. -exchange_suffix=AX -country_code=AU -timezone=Australia/Melbourne +# country_code and timezone: the reminder report's country and time zone; .jobs also shows each next run in timezone. +#exchange_suffix=AX +#country_code=AU +#timezone=Australia/Melbourne # Caching and network, in seconds -cache=True -cache_seconds=43200 -performance_cache_seconds=2400 -http_timeout=18 +#cache=True +#cache_seconds=43200 +#performance_cache_seconds=2400 +#http_timeout=18 -# Diagnostics -debug=False -print_headers=False +# Diagnostics. log_level: what the log shows, that level and those above it: ERROR, WARN, INFO (what finbot does, +# and each request it sends) or DEBUG (each cache miss, hit and write too). +#log_level=INFO +#print_headers=False diff --git a/defaults/schedule.cron b/defaults/schedule.cron index 5020d035..8e340293 100644 --- a/defaults/schedule.cron +++ b/defaults/schedule.cron @@ -1,9 +1,8 @@ -# The scheduled reports, run by the bot (bot.py). Same time syntax as crontab(5): minute hour day-of-month month -# day-of-week, or an @macro such as @daily. The command is a finbot script and its arguments; separate several with ";" -# to run them in turn. -# Times are in the server's own time zone, like cron; add a line "CRON_TZ=Australia/Melbourne" above a job to pin one. -# To change the schedule, copy this file to the data directory (util/sync_defaults.py --apply), edit the copy, then: -# systemctl reload finbot +# NO LONGER READ, BUT ONCE: each chat has its own jobs now, changed with .jobs (add, del, toggle, run). As finbot +# starts the first time without them, it gives every home chat (telegramChatID, slackChannelID, discordChannelID) the +# jobs below (or those of the data directory's copy of this file), each in the zone it ran in, and reads this file no +# more. It goes once every install has done that (lib/migrate.py). The system jobs (sharesight_warm, yahoo_warm, +# cleanup) are not copied: they run on their own schedule, in lib/jobs.py. # Every 20 minutes on weekdays */20 * * * Mon-Fri trades.py diff --git a/doc/caching.md b/doc/caching.md new file mode 100644 index 00000000..40d53431 --- /dev/null +++ b/doc/caching.md @@ -0,0 +1,26 @@ +# Caching + +Many objects are cached for 12 hours by default (`cache_seconds`). Cache is controlled by the settings below. Trades IDs are stored on disk, but trade queries generally don't result in a cache hit for functional reasons. +``` +cache=True +cache_seconds=43200 +performance_cache_seconds=2400 +``` + +`performance_cache_seconds` (default 40 minutes) is the cache lifetime of Sharesight performance reports. + +Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The weekly system job `cleanup` removes cache files after 30 days. + +To flush the cache, so that what it holds is fetched afresh, run `util/flush.py` with the files to delete, by when they were written. With no argument it prints its help: +``` +util/flush.py --before 180s # the files written more than 180 seconds ago +util/flush.py --after 24h # the files written in the last 24 hours +util/flush.py --after 7d # ... in the last 7 days (3m: 3 months, 1y: a year) +util/flush.py all # every file +sudo docker compose exec finbot util/flush.py all # in Docker +``` +A time is a number and `s` (seconds), `h` (hours), `d` (days), `w` (weeks), `m` (months) or `y` (years). It deletes only finbot's files in `DATA_DIR/cache` (Yahoo's and Sharesight's answers, the charts, the Yahoo cookie), never the state in `DATA_DIR` (the watchlists, jobs, plans, seen trades and ratings), and the bot can keep running: it fetches a file that is gone afresh. Run it as the user finbot runs as, or as root, with finbot's data dir: `DATA_DIR` as `finbot.env` sets it (`sudo DATA_DIR=/var/lib/finbot util/flush.py all`), `--data-dir /var/lib/finbot`, or else `var/` in the checkout. + +Signalling the bot flushes every file too, and then runs both warm jobs (`sharesight_warm`, `yahoo_warm`) at once to fill the cache again: `sudo systemctl kill -s USR1 finbot`, or `sudo docker compose kill -s USR1 finbot`. The log says `cache: flushed N files`, then `cache: warm jobs done`. + +Two system jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](scheduling.md)): `sharesight_warm` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). diff --git a/doc/commands.md b/doc/commands.md new file mode 100644 index 00000000..85ad9889 --- /dev/null +++ b/doc/commands.md @@ -0,0 +1,389 @@ +# Commands and reports + +What finbot does in a chat: its commands, and the reports that they and each chat's [scheduled jobs](scheduling.md) post. + +## On each network +What finbot can do on each network. Telegram takes everything; the others take what their APIs allow. + +| Feature | Telegram | Slack | Discord | +|---|---|---|---| +| Chat commands | ✓: messages (`.price`), and slash commands (`/price`) from the command menu | ✓: messages, and slash commands (`/price`) from the app's manifest | ✓: slash commands (`/price`), and messages over the gateway | +| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots); none for a slash command, which has no message | ✓ for a message; "thinking…" for a slash command | +| Charts in replies | ✓ | ✓ | ✓ | +| Charts in scheduled reports | ✓ | ✓ | ✓ | +| Slash commands completed as they are typed | names, from the command menu | names, with their usage | names, and the arguments: keywords, periods, portfolios, tickers, jobs | +| Period and job buttons | ✓ | ✓ | ✓ | +| No link previews | ✓ | ✓ | ✓ | +| Linked tickers | ✓ | ✓ | ✓ | +| Charts updated in place by their buttons | ✓ | ✓ | ✓ | + +Each network's guide sets up its bot and compares its two modes: [Telegram](telegram.md), [Slack](slack.md), [Discord](discord.md). + +![Screenshot of showing trade notifications on Slack](../img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") + +## Supported commands + +Where: +* _percent_ is a number, optionally followed by '%' +* _period_ is a number followed by 'd', 'w', 'm' or 'y', or 'ytd' +* _portfolio_ is a Sharesight portfolio name +* _symbol_ is a Yahoo Finance symbol; a bare one Yahoo does not know is tried again on your exchange (`exchange_suffix`: `BHP` may be `BHP.AX`) +``` +.symbol +.stock symbol +.beta +.buy +.sell +.compare symbol symbol [symbol...] [period] +.dividend [period|symbol] +.earnings [period|symbol] +.forward pe [symbol|top|bottom|negative] +.history symbol +.holdings portfolio +.jobs [run|toggle] [number] +.jobs add [CRON_TZ=zone] time report +.jobs del number +.marketcap [symbol|top|bottom] +.pe [symbol|top|bottom] +.peg [symbol|top|bottom|negative] +.performance [period] [portfolio] +.pie portfolio [sector] +.plan [your plan] +.premarket [percent|symbol] +.price [percent|symbol|top|bottom] [period] +.session [percent|symbol] +.shorts [percent|symbol] +.trades [period] [portfolio] +.watchlist [add|del symbol] +.who symbol +``` + +`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. Each command is also a slash command on every network, which completes its name as it is typed: Telegram's command menu, set as finbot starts; Slack's, from the app's manifest (`util/slack_manifest.py`, see the [Slack guide](slack.md#slash-commands)); Discord's, registered as finbot starts. `/stock symbol` (or `/profile symbol`) is `.symbol`, `/forwardpe` is `.forward pe`, and `/job` is `/jobs`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. + +Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](scheduling.md)), and notifying each chat that schedules them of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. + +The other various reports can also run on a schedule (e.g. daily or weekly), or on demand with a command. They query the Yahoo Finance API for stock data based on current holdings across your Sharesight portfolios, your friends' Sharesight portfolios, plus a custom watch list. A scheduled report posts to the chat whose job it is, and a command's reply to the chat that sent it. + +The bot answers commands in every chat it is in. It fetches its updates from each network itself, with no server to host (pull mode); in HTTP mode instead, a network posts them to a web service you host on a domain with a trusted certificate. It provides: +* Stock lookup (financials, company profile, price charts) +* A watch list per chat, which anyone in it can change, picked up by the reports there +* Listing of portfolios and portfolio performance +* Listing of holdings sorted by various attributes +* Running the other stock reports on demand + +## Reports + +### Stock lookup +![Screenshot showing stock info on Slack](../img/stockinfo.png?raw=true "Screenshot showing stock info on Slack") + +The stock lookup returns various stats relevant to a stock's valuation, growth and risk factors. + +#### Emoji legend +* The colored circle next to the symbol at the top indicates the current market state: + * 🔴 market closed + * 🟠 pre/post-market + * 🟢 normal trading +* The warning symbol ⚠️ indicates possible risk factors. This is not intended to replace due diligence, but to non-exhaustively highlight where additional diligence is due. Some risk factors include: + * Negative earnings ⚠️ + * Earnings in decline ⚠️ + * Negative cashflow ⚠️ + * High short interest ⚠️ + * High forward P/E ratio ⚠️ + * Net insider selling ⚠️ + * High debt-to-equity ratio ⚠️ +* The arrow grid indicates the directional change in the company earnings and revenue for each reporting period. A red arrow indicates the absolute earnings was negative: + * 🔼 earnings/revenue increased from the previous period + * 🔽 earnings/revenue decreased from the previous period + * 🔺 earnings increased from the previous period but remained negative + * 🔻 earnings decreased from the previous period and was negative + * ▪️ earnings/revenue matched the previous period + * ❌ data is missing for this period +* Using the example screenshot above, Crowdstrike reported: + * 🔼🔼🔼 revenue increased in all of the past three quarters + * 🔺🔻🔻 In the past three quarters, earnings initially increased, then decreased for two quarters, and was negative in every quarter + * 🔼🔼🔼 revenue increased in all of the past three years + * 🔻🔺🔻 In the past three years, earnings decreased, increased, then decreased, and was negative for all three years + +This report is a command only. Example usage: +``` +.AAPL +``` +``` +@botname AAPL +``` + +### Trades +![trade update in Slack](../img/trades.png?raw=true "Trade update in Slack") + +The `trades` report sends recent Sharesight trades to each chat that schedules it. +By default, this report searches Sharesight for trades from the past 28 days (`past_days`). You can override this in `preferences.conf`, or by providing a number as argument when triggered through the chat bot. + +Without persistent storage, set `past_days=0` to avoid duplicate trade notifications. In this case, the schedule must run the report exactly once per day, after market close. + +With persistent storage, the default of `past_days=28` suits. This is useful if Sharesight imports trades with past dates for any reason. Note that the initial run will notify on all historical trades for the `past_days` period. It is recommended to schedule it every 20 minutes. + +``` +past_days=28 +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add */20 * * * Mon-Fri trades`: +``` +trades +``` + +Command: +``` +.trades [period] +``` +``` +@botname trades [period] +``` + +You can also specify a portfolio name to get trades for just that portfolio: +``` +.trades [period] [portfolio] +``` +``` +@botname trades [period] [portfolio] +``` + +### Price alerts +![Price chart triggered from Slack](../img/finbot_price.png?raw=true "Price chart triggered from Slack") +![Price alert triggered by Cron](../img/price.png?raw=true "Price alert triggered by Cron") + +The `price` report sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. + +If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. + +Config example: +``` +price_percent=5 +demote_volatile=True +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add 29 21 * * Mon-Fri price interday`: +``` +price premarket|midsession|interday|close|DAYS +``` +When scheduled, the mode is its argument: +* If `premarket` is passed, it only reports on pre/post market price movements. +* If `midsession` is passed, it only reports for markets currently in session. This is intended to run on a schedule to provide mid-session alerts for big price movements of your holdings. For example, it could be run twice per day 12 hours apart, to capture markets in different timezones. +* If `interday` is passed, it reports current price against the previous market close. +* If `close` is passed, it reports daily change in stocks whose market closed in the past 3 hours. +* If an integer [i] is passed, it reports current price against [i] days ago. + +Command (pre-market): +``` +.premarket [percent] +``` +``` +@botname premarket [percent] +``` + +Command (mid-session): +``` +.session [percent] +``` +``` +@botname session [percent] +``` + +Command (interday). +``` +.price [percent] +``` +``` +@botname price [percent] +``` + +For interday, can also specify the time period. A bare number is the percent; a period carries its unit: `7d`, `2w`, `3m`, `1y` or `ytd`. For example, `.price 3 7d` lists the moves of 3% or more over the past week. +``` +.price [percent] [period] +``` + +Or a specific stock. In this case, the output includes a graph. +``` +.price [symbol] [period] +``` + +### Portfolio performance +![Screenshot of Portfolio performance notification on Slack](../img/performance.png?raw=true "Screenshot performance notification on Slack") + +The `performance` report covers Sharesight portfolio performance over a given period. The default is set by `past_days` in the `preferences.conf`, which can be overridden at runtime by specifying an integer as the argument. + +Config example: +``` +past_days=28 +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add 28 21 * * Sat performance 7`: +``` +performance [DAYS] +``` + +Command: +``` +.performance [period] +``` +``` +@botname performance [period] +``` + +As a command, it also takes a portfolio: +``` +.performance [period] [[portfolio] +``` +``` +@botname performance [period] [portfolio] +``` + +### Events calendar +![earnings message in Slack](../img/earnings.png?raw=true "Earnings message in Slack") + +The `cal` report sends upcoming earnings and ex-dividend date alerts. The data is sourced from Yahoo! Finance. It reports on events up to `future_days` into the future. This is set in the `preferences.conf` for a scheduled run, or can be specified as an argument when triggered through the chat bot. + +**Finance Explainer:** When a company releases its quarterly earnings report, the stock price may undergo a signficant positive or negative movement, depending on whether the company beat or missed market expectations. You may wish to hold off buying more of this stock until after its earnings report, unless you think the stock will beat market expectations. + +**Finance Explainer:** When a stock goes ex-dividend, the share price [typically drops](https://www.investopedia.com/articles/stocks/07/ex_dividend.asp) by the amount of the dividend paid. If you buy right before the ex-dividend date, you can expect an unrealised capital loss, plus a tax obligation for the dividend. Thus, you may wish to wait for the ex-dividend date before buying more of this stock. +``` +future_days=30 +``` + +Job (see [Scheduling](scheduling.md)), such as `.jobs add 28 21 * * Sat cal earnings`: +``` +cal earnings|ex-dividend +``` + +Command (earnings): +``` +.earnings [period] +``` +``` +@botname earnings [period] +``` + +Command (ex-dividend): +``` +.dividend [period] +``` +``` +@botname dividend [period] +``` + +### Highly shorted stock warnings +![short warnings in Slack](../img/shorts.png?raw=true "Short warnings in Slack") + +The `shorts` report sends short interest warnings. The data is sourced from Yahoo Finance and Shortman (AU). `shorts_percent` defines the alert threshold for the percentage of a stock's float shorted. This can be specified in the `preferences.conf` for a scheduled run, or as an argument when triggered through the chat bot. + +**Finance Explainer:** A high short ratio indicates a stock is exposed to high risks, such as potential banktrupcy. It may also incentivise negative news articles which harm the stock price. If the market is wrong, however, risk tolerant investors may receive windfall gains. This report is intended to alert you to an above-average risk, and prompt you to investigate this stock more closely. +``` +shorts_percent=15 +``` + +Command: +``` +.shorts [percent] +``` +``` +@botname shorts [percent] +``` + +### pe / forward pe / peg +![Forward PE report in Slack](../img/finbot_fpe.png?raw=true "Forward PE report in Slack") + +List securities by P/E ratio, forward P/E ratio, or PEG ratio. A forward P/E is an analysts' estimate, so its lists take only stocks covered by at least `min_analysts` analysts (7 by default; see `preferences.conf`). + +Command: +``` +.pe [SYMBOL] +.fpe [SYMBOL|bottom|negative] +.peg [SYMBOL|bottom|negative] +``` +``` +@botname pe [SYMBOL|top|bottom] +@botname fpe [SYMBOL|top|bottom|negative] +@botname peg [SYMBOL|top|bottom|negative] +``` + +### History +![Price history report in Slack](../img/finbot_history.png?raw=true "Price history report in Slack") + +Shows the long term performance of a security. +It creates a 5Y chart, and then lists performance for several periods (5Y 3Y 1Y 6M 3M 1M 1W 1D) + +Command: +``` +.history +``` +``` +@botname history +``` + +### Buy +![Buy ratings in Slack](../img/finbot_buy.png?raw=true "Buy ratings in Slack") + +Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts (7 by default; see `preferences.conf`) + +Command: +``` +.buy +``` +``` +@botname buy +``` + +### Sell +![Rating report in Slack](../img/finbot_sell.png?raw=true "Rating report in Slack") + +Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts + +Command: +``` +.sell +``` +``` +@botname sell +``` + +### Marketcap +![Marketcap report in Slack](../img/finbot_marketcap.png?raw=true "Marketcap report in Slack") + +List the top (or bottom) stocks by market cap, in US dollars: market caps in other currencies are converted at Yahoo's exchange rates. With a symbol, its market cap in its own currency and in US dollars. + +Command: +``` +.marketcap [top|bottom|SYMBOL] +``` +``` +@botname marketcap [top|bottom|SYMBOL] +``` + +### Consensus rating changes +![Rating change report in Slack](../img/finbot_rating.png?raw=true "Rating change report in Slack") + +Notifies when Yahoo Finance analyst rating consensus changes category (e.g. from hold to buy). Designed to run on a schedule, such as `.jobs add 28 21 * * Sat rating` (see [Scheduling](scheduling.md)). Requires persistent storage. + +``` +rating +``` + +### Watchlist +![Shared watchlist in Slack](../img/watchlist.png?raw=true "Shared watchlist in Slack") + +Tracks additional securities which are not in your Sharesight holdings. Each chat has its own watchlist, which anyone in the chat can change: the reports in that chat, asked for or scheduled, include its watchlist's stocks as if they were holdings, and no other chat's. The Sharesight holdings are the same in every chat. `yahoo_warm` keeps every chat's watchlist warm in the cache. + +The watchlists are kept in `finbot_watchlist.json` in the data directory, by chat (`{"chats": {"telegram:-1001234": ["2454.TW", "ASO.AX", "STEM"]}}`), so there is no need to edit the file. The first start with watchlists per chat gives each home chat the watchlist kept before, and the same goes for the plans that `.plan` records: each chat's `.plan` lists only the plans recorded in it. + +Command: +``` +.watchlist +``` +``` +.watchlist [add|del] [symbol] +``` +``` +@botname watchlist +``` +``` +@botname watchlist [add|del] [symbol] +``` diff --git a/doc/configuration.md b/doc/configuration.md new file mode 100644 index 00000000..e209682e --- /dev/null +++ b/doc/configuration.md @@ -0,0 +1,142 @@ +# Configuration + +Configuration is split into two files -- finbot.env for startup environment variables, and preferences.conf for values that can be changed at runtime. + +Copy the shipped template: +``` +sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env +``` + +## `finbot.env` +**Load `finbot.env` with your launcher:** +* **systemd:** `EnvironmentFile=/opt/finbot.env` in `finbot.service`, plus `ExecStartPre=+.../util/check_config.py /opt/finbot.env` for pre-flight lint. +* **Docker:** `compose.yaml` loads it, and checks it before the bot starts (see [docker.md](docker.md)). Write the values without quotes. +* **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). +* **Scheduled jobs:** they are run by `bot.py` (see [Scheduling](scheduling.md)), so they run on the same settings. For a one-off run, send `.jobs run` in the chat. + +The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which +of the chat network keys it needs. + +| Key | Default | Description | +|---|---|---| +| `DATA_DIR` | `var/` in the checkout (`$STATE_DIRECTORY` under systemd `StateDirectory=`) | Where state and cache live | +| `ip` | `127.0.0.1` | Address the bot listens on | +| `port` | `5000` | Port the bot listens on | +| `scheduler_catchup` | `5` | Minutes: a run missed while the bot was down still fires if it was due this recently | +| `telegramChatID` | | Telegram's home chat, a group or channel id: finbot's first start gives it the default jobs, and error reports go there (see [Scheduling](scheduling.md)) | +| `sharesight_code` | | Sharesight API access | +| `sharesight_client_id` | | Sharesight API access | +| `sharesight_client_secret` | | Sharesight API access | +| `telegramBotToken` | | Telegram bot token: reports, and commands (finbot fetches its updates unless `telegramOutgoingWebhook` is set) | +| `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | +| `slackAppToken` | | Slack app-level token (`xapp-`): commands over a socket finbot opens to Slack (Socket Mode), when `slackOutgoingWebhook` is empty | +| `slackChannelID` | | Slack's home channel id, as `telegramChatID` (with `slackBotToken`) | +| `telegramOutgoingWebhook` | | The URL Telegram posts its updates to (HTTP mode); empty, finbot fetches them (pull mode) | +| `telegramOutgoingToken` | | Secret token Telegram must present (HTTP mode) | +| `slackOutgoingWebhook` | | The URL Slack posts its events and button presses to (HTTP mode); empty, Socket Mode with `slackAppToken` | +| `slackOutgoingToken` | | Token Slack must present (HTTP mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | +| `slackSigningSecret` | | Secret Slack signs each request with (HTTP mode) | +| `discordBotToken` | | Discord bot token: replies, and commands over the gateway unless `discordOutgoingWebhook` is set | +| `discordChannelID` | | Discord's home channel id, as `telegramChatID` (with `discordBotToken`) | +| `discordOutgoingWebhook` | | The Interactions Endpoint URL Discord posts slash commands and button presses to (HTTP mode); empty, the gateway (pull mode) | +| `discordPublicKey` | | The app's public key, which checks the signature of each interaction (HTTP mode) | + +## `preferences.conf` items + +The preferences, read by the bot itself from `/preferences.conf`; without that file, finbot runs on the built-in defaults below. `defaults/preferences.conf` is only its template: every key in it is commented out at its default, and finbot never reads it. To change a preference, copy the template into the data directory (this never overwrites a file there): +``` +util/sync_defaults.py [--dest DIR] --apply +``` +then remove the `#` before the key and edit its value. A key set in `defaults/preferences.conf` itself does nothing: the log at start (and check_config) says so, and where to set it instead. + +A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. + +| Key | Default | Description | +|---|---|---| +| `past_days` | `28` | How many days back reports look | +| `future_days` | `30` | How many days ahead the events calendar looks | +| `price_percent` | `5` | Price alert threshold, percent | +| `shorts_percent` | `15` | Short interest alert threshold, percent | +| `min_analysts` | `7` | `.buy`, `.sell` and `.forward pe` list only stocks covered by at least this many analysts; with fewer, a consensus says little (thinly covered small caps are nearly all a perfect buy) | +| `demote_volatile` | `True` | Volatile (small cap, high beta) stocks need a bigger move to alert | +| `demote_leveraged` | `True` | Leveraged (2x, 3x) funds need a bigger move to alert | +| `volatility_multiplier` | `2` | How much bigger a move volatile stocks need | +| `exclude_portfolios` | | Sharesight portfolio IDs to leave out, space separated | +| `include_portfolios` | | Sharesight portfolio IDs to report on, space separated (empty: all that are not excluded) | +| `trades_use_yahoo` | `True` | Use Yahoo for trades; set `False` if Yahoo breaks | +| `graph` | `True` | Send charts with reports | +| `hyperlink` | `True` | Hyperlink tickers in reports | +| `hyperlinkFooter` | `True` | Add a website link footer to stock lookups | +| `hyperlinkProvider` | `google` | Where ticker links go: `google` or `yahoo` | +| `exchange_suffix` | `AX` | Regional: Yahoo's suffix for your exchange (`AX` for the ASX, `NZ`, `L`, `TO` and so on). A bare symbol Yahoo does not know is tried again with it, in every command that takes a symbol (`BHP` becomes `BHP.AX`); empty, a symbol is taken as written | +| `country_code` | `AU` | Regional: your country, for the reminder report | +| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report, and for each job's next run in `.jobs` (besides UTC) | +| `cache` | `True` | Cache Yahoo and Sharesight responses | +| `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | +| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm` runs, so the holdings never run out | +| `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | +| `log_level` | `INFO` | What the log shows: that level and those above it, syslog style. `ERROR`: what failed and was lost (a message not sent); `WARN`: what failed and was got over (a retry, a fallback, a request refused); `INFO`: what finbot does (each message sent, each job run, each request sent and answered); `DEBUG`: each cache miss, hit and write too. Replaces `debug` | +| `print_headers` | `False` | Log inbound request headers | + +A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. + +edit `preferences.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. + +**Upgrading from one `.env`?** finbot no longer reads it, and refuses to start while a `.env` is in the checkout, so its settings cannot be silently ignored. Run `util/split_env.py` (or `util/split_env.py /path/to/old.env` for one elsewhere). It reads either format, the old dotenv one (`key = 'value' # comment`) or plain `KEY=value`, writes `finbot.env.new` and `preferences.conf.new` next to the old file, never overwriting, and lists any key it dropped or did not recognise. Put `finbot.env` where your launcher loads it and `preferences.conf` in the data directory, then delete the old file. + +## Data directory +All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`); in Docker it is `/data`, the host's `/opt/finbot-data` (see [docker.md](docker.md)). +Reference data (`finbot_adr.json`, `finbot_sws_*.json`, `reminder.json`) ships in `defaults/`. A file of the same name in the data directory takes precedence. Defaults are never copied or overwritten automatically; to see or pull in updates run: +``` +util/sync_defaults.py # dry run +util/sync_defaults.py --apply # copy missing files (including a starter preferences.conf) +util/sync_defaults.py --force # also replace changed files, keeping .bak copies +util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory +util/sync_defaults.py --dest /srv/finbot --create --apply # and make it (mode 750) if it does not exist yet +``` +`reminder.json` holds the reminders that the `reminder` report posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. + +With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. + +In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. Each start copies the defaults the data directory lacks (`util/sync_defaults.py --apply`). + +## Sharesight +* Email Sharesight support to get an API key and add the [access details](https://portfolio.sharesight.com/oauth_consumers) to `finbot.env`. Example: +``` +sharesight_code=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f +sharesight_client_id=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f +sharesight_client_secret=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f +``` + +## Portfolios +Portfolios are auto-discovered, including other people's portfolios which are shared to you. To exclude specific portfolios, add their IDs to `exclude_portfolios` in the `preferences.conf`: +``` +exclude_portfolios=100003 100004 +``` + +Alternatively, you can include only specific portfolios: +``` +include_portfolios=100001 100002 +``` + +## Hyperlinks +These `preferences.conf` options relate to hyperlinks: + +* `hyperlink`, if set to `False`, disables all hyperlinks. Note: it can not disable syntax highlighting and false positives by chat clients that try to detect domains. Defaults to `True`. +``` +hyperlink=True +``` + +* `hyperlinkFooter` appends hyperlinks to the company website and to a Google search for the ticker on simplywall.st for the symbol lookup report. Possible values are `True` or `False`. Defaults to `True`. + +``` +hyperlinkFooter=True +``` + +* `hyperlinkProvider` configures where to send ticker hyperlinks. Available values are `google` and `yahoo`. If set to `google`, it is only used where practical, as Google does not publish earnings date, ex-dividend date, premarket price, or short interest figures. Defaults to `google`. +``` +hyperlinkProvider=google +``` +``` +hyperlinkProvider=yahoo +``` diff --git a/doc/discord.md b/doc/discord.md index 82db2477..80907301 100644 --- a/doc/discord.md +++ b/doc/discord.md @@ -1,24 +1,111 @@ # Discord -finbot reaches Discord through a webhook, which posts the scheduled reports into a channel. A webhook only posts, so -there is no choice to make: on Discord, finbot does not answer commands. For commands, set up finbot on -[Telegram](telegram.md) or [Slack](slack.md) as well. +finbot talks to Discord as the bot user of a Discord application that you create in the Developer Portal. The bot +posts each channel's scheduled reports there and answers commands: slash commands (`/price`) in both modes, and +messages (`.price`, `@finbot price`) over the gateway. -What the webhook carries: -* the scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest; -* their charts, attached to the messages; -* linked tickers, without link previews, and without pinging anyone. +## Gateway mode or HTTP mode? -## Webhook +The bot does the same either way. What differs is how it hears from Discord: -1. In Discord, open _Server Settings > Integrations > Webhooks_ and choose _New Webhook_. -2. Name it (finbot, say), pick the channel for the reports, and _Copy Webhook URL_. -3. Put the URL in `finbot.env` as `discord_webhook`. -4. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the channel. +| | Gateway mode (the default) | HTTP mode | +|---|---|---| +| How commands and button presses reach finbot | Over Discord's gateway, a connection finbot keeps open | Discord posts them to your HTTPS server | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| Commands | Slash commands (`/price`), and messages (`.price`, `@finbot price`) | Slash commands (`/price`): Discord posts no messages | +| Python modules | `discord.py` (Debian: `python3-discord`) | `PyNaCl` and `waitress` (Debian: `python3-nacl`, `python3-waitress`) | +| `finbot.env` keys | `discordBotToken`, `discordChannelID` | Those two, plus `discordOutgoingWebhook` and `discordPublicKey` | + +In both: the scheduled reports with their charts; period buttons under charts and the buttons of `.jobs run` and +`.jobs toggle`, which redraw their message in place; linked tickers, without link previews and without pinging anyone; +and direct messages with the bot. A command in a message shows "finbot is typing…" while it works; a slash command +shows Discord's "finbot is thinking…" until its answer takes its place. + +`discordOutgoingWebhook` chooses the mode: set, HTTP mode; empty, gateway mode. Discord sends the slash commands and +button presses to the app's _Interactions Endpoint URL_ while it is set, and over the gateway while it is empty, so +change both, and restart finbot, to switch. + +## Gateway mode + +1. In the [Developer Portal](https://discord.com/developers/applications), choose _New Application_ and name it + (finbot, say). +2. Under _Bot_: + * _Reset Token_, and copy the token into `finbot.env` as `discordBotToken`. + * Turn _Message Content Intent_ on, for the bot to read `.price` and `@finbot price` in your channels. Without it, + Discord refuses the connection, and the log says so. + * Turn _Public Bot_ off, so that only you can add the bot to servers. +3. Under _OAuth2 > URL Generator_, tick the scopes `bot` and `applications.commands`, then the bot permissions _View + Channels_, _Send Messages_, _Attach Files_ and _Read Message History_. Open the URL it makes, and add the bot to your + server. +4. In Discord, turn on _User Settings > Advanced > Developer Mode_; then right-click the channel to be Discord's home + chat, _Copy Channel ID_, and put it in `finbot.env` as `discordChannelID`. finbot's first start gives that channel + the default jobs ([Scheduling](scheduling.md)), and it gets the error reports. Every channel manages its + own jobs with `.jobs`. +5. Install discord.py: `sudo apt install python3-discord` (or `pip install discord.py`). +6. Restart finbot (`sudo systemctl restart finbot`). As it starts, it registers the slash commands, which can take a + few minutes to show in Discord the first time, and the log says `discord: gateway (no discordOutgoingWebhook)`. +7. Send `/help` (or `.help`) in the channel, or in a direct message to the bot. ``` -discord_webhook=https://discord.com/api/webhooks/1009998000000000000/aaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbb +discordBotToken=your-bot-token +discordChannelID=1000000000000000000 ``` -A URL ending in `/slack`, from older setups, works as it is: finbot uses Discord's own endpoint either way. A message -holds up to 2,000 characters, so a long report arrives as several messages. +## HTTP mode + +Discord posts the slash commands and button presses to finbot over HTTPS, each signed with the app's key. finbot +listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a +domain name with a valid certificate. The bot shows as offline in Discord, as it keeps no gateway connection. + +1. Set up the bot as for [gateway mode](#gateway-mode), steps 1 to 4. The _Message Content Intent_ is not needed. +2. Install PyNaCl, which checks the signatures, and waitress, the web server: `sudo apt install python3-nacl + python3-waitress` (or `pip install PyNaCl waitress`). +3. Put finbot behind an HTTPS server. An nginx example: + ``` + server { + listen 8443 ssl; + + server_name www.example.com; + ssl_certificate /etc/letsencrypt/live/www.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/www.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location /discord { + proxy_pass http://127.0.0.1:5000/discord; + } + } + ``` + Discord publishes no list of its addresses, so the signature on each request is what keeps strangers out: finbot + refuses a request without a good one. +4. In `finbot.env`, set `discordOutgoingWebhook` to the URL Discord should post to, and `discordPublicKey` to the app's + _General Information > Public Key_. Restart finbot, so that it can answer Discord's check in the next step. +5. In the Developer Portal, set _General Information > Interactions Endpoint URL_ to the same URL, and _Save Changes_. + Discord checks it at once, sending a signed request and one with a bad signature, which finbot refuses as Discord + expects. +6. Send `/help` in the channel, or in a direct message to the bot. + +``` +discordBotToken=your-bot-token +discordChannelID=1000000000000000000 +discordOutgoingWebhook=https://www.example.com:8443/discord +discordPublicKey=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef +``` + +## Slash commands + +finbot registers a slash command for each line of `.help`, as it starts, and again only when the list has changed: +`/price`, `/dividend` and `/earnings`, `/buy` and `/sell`, `/forwardpe` for `.forward pe`, `/stock` (or `/profile`) for +`.SYMBOL`, `/jobs` (or `/job`), and the rest. Each takes its arguments, if any, as one text option, as the message +would: `/price` with `5 7d` is `.price 5 7d`. They work in the server's channels and in a direct message with the bot. + +While the arguments are typed, Discord shows finbot's suggestions, in both modes, for the word being typed: +* the command's keywords (`top`, `bottom`, `negative`, `sector`, `add`, `del`) and periods (`7d`, `3m`, `ytd`), until + it has one; +* the portfolios, for the commands that take one; +* tickers, for the commands that take one: the channel's watchlist and the holdings. They come from the cache, never + from Sharesight or Yahoo, so the suggestions come at once; the holdings are there once `sharesight_warm` has run; +* for `/jobs`: `run`, `toggle`, `add` and `del`; after `run`, `toggle` or `del` the channel's job numbers, each with + its line; after `add` and a time, the reports, each with what it takes, and then what the report takes. + +A message holds up to 2,000 characters, so a long report arrives as several messages. A chart goes as an attachment of +the message that carries its report, with its period buttons under it. diff --git a/doc/docker.md b/doc/docker.md new file mode 100644 index 00000000..61521071 --- /dev/null +++ b/doc/docker.md @@ -0,0 +1,126 @@ +# Docker + +finbot runs in a container as it runs on a Debian host: the image is `debian:trixie-slim` with Debian's own `python3-*` +packages, every optional library included, so it runs every network in either mode. The image holds only the code. +The configuration lives on the host, beside the checkout, so a rebuild or an update never touches it. + +## Quick start + +1. `git clone https://github.com/robdevops/finbot /opt/finbot` +2. `sudo /opt/finbot/util/docker_setup.sh` +3. Set `telegramBotToken` and `telegramChatID` in `/opt/finbot.env` (or another network's keys: see the + [Telegram](telegram.md), [Slack](slack.md) and [Discord](discord.md) guides). +4. `cd /opt/finbot && sudo docker compose up -d --build` +5. `journalctl CONTAINER_NAME=finbot -f` + +## The layout on the host + +| Path | What | In the container | +|---|---|---| +| `/opt/finbot` | the checkout, with `Dockerfile` and `compose.yaml` | the image's `/opt/finbot`, copied at build | +| `/opt/finbot.env` | the startup file: tokens, chat ids, the mode of each network | the environment, and `/run/finbot.env`, read-only, for the check | +| `/opt/finbot-data` | `DATA_DIR`: `preferences.conf`, each chat's jobs and watchlist, the records, the cache | `/data` | + +`compose.yaml` finds the other two beside the checkout (`../finbot.env`, `../finbot-data`): clone it elsewhere and +they move with it. The startup file stays out of the image because it holds the secrets, and the data dir because it +must outlast every container. + +## What `util/docker_setup.sh` does + +Run it once, as root; it is safe to run again. It: +1. runs `util/sync_defaults.py --create --apply --dest /opt/finbot-data`, which makes the data dir (mode 750) and copies + the shipped defaults into it: `preferences.conf`, `reminder.json` and the data files. It never replaces a file you + have changed. The script needs only the standard library, so any `python3` on the host runs it. +2. gives the data dir to the container's user: `chown -R 10001:10001 /opt/finbot-data`. +3. copies `defaults/finbot.env` to `/opt/finbot.env` if there is none, owned by `root:10001`, mode 640: root and the + container can read it, nobody else. + +By hand, that is: +``` +sudo python3 /opt/finbot/util/sync_defaults.py --create --apply --dest /opt/finbot-data +sudo chown -R 10001:10001 /opt/finbot-data +sudo cp /opt/finbot/defaults/finbot.env /opt/finbot.env +sudo chown root:10001 /opt/finbot.env && sudo chmod 640 /opt/finbot.env +``` + +The startup file is `KEY=value` lines, as for systemd; write the values without quotes. + +## Build and start + +``` +cd /opt/finbot && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build +``` +`GIT_COMMIT` is optional: it names the commit in the log's first line (`Starting finbot 0123456`), as the image has no +`.git` to read it from. + +Each start of the container, in order: +1. `util/sync_defaults.py --apply` copies into `/data` any default it lacks, such as a data file added by an update; +2. `util/check_config.py /run/finbot.env` checks the startup file and `preferences.conf`. A mistake stops the start + here, and the log says which key and why; +3. the bot starts. + +## Logs + +The log goes to the host's journal: +``` +journalctl CONTAINER_NAME=finbot -f +``` +On a host without journald, set `FINBOT_LOG_DRIVER=json-file` (in the environment of `docker compose`, or in a `.env` +file beside `compose.yaml`), and read it with `sudo docker compose logs -f`. + +## Reload, flush, update + +``` +sudo docker compose kill -s HUP finbot # re-read preferences.conf +sudo docker compose kill -s USR1 finbot # flush the cache, and fill it again +sudo docker compose exec finbot util/flush.py --after 24h # flush the files of the last day (see caching.md) +cd /opt/finbot && sudo git pull && sudo GIT_COMMIT=$(git rev-parse HEAD) docker compose up -d --build +``` +A change to `/opt/finbot.env` needs a restart: `sudo docker compose up -d`. + +## HTTP mode + +In pull mode, the default, finbot fetches its updates and needs no port. In HTTP mode (a network's webhook URL set), +it listens inside the container on port 5000 of every interface (`ip: 0.0.0.0` in `compose.yaml`). Publish the port +on the host's loopback only, by uncommenting `ports` in `compose.yaml`: +``` + ports: + - 127.0.0.1:5000:5000 +``` +and put your reverse proxy in front of it, as each network's guide describes for a host install. + +## Hardening + +`compose.yaml` takes away everything the bot does not need: + +| Setting | Why | +|---|---| +| `user: "10001:10001"` | the bot runs as a user of its own, never root; the code belongs to root, so the bot cannot change it | +| `read_only: true`, `tmpfs: /tmp` | nothing outside `/data` and `/tmp` can be written; the code is compiled at build, so nothing needs to be | +| `cap_drop: [ALL]` | no Linux capabilities: the bot binds no port below 1024 and changes no owner | +| `security_opt: no-new-privileges:true` | nothing in the container can gain privileges, setuid binaries included | +| `pids_limit: 256` | a runaway can start only so many threads | +| `mem_limit: 400m`, `memswap_limit: 400m` | 400 MB, with no swap on top: the bot starts at about 80 MB and reaches about 100 MB drawing charts | +| `init: true` | a minimal init as process 1, which passes `SIGHUP` and `SIGUSR1` on to the bot and reaps children | +| no `ports` | nothing listens on the host unless you publish a port | + +To give it more memory, raise both `mem_limit` and `memswap_limit`. + +## Moving a host install into Docker + +``` +sudo systemctl disable --now finbot +sudo cp -a /var/lib/finbot/. /opt/finbot-data/ # or /opt/finbot/var/. for the default DATA_DIR +sudo chown -R 10001:10001 /opt/finbot-data +``` +then `util/docker_setup.sh` (it keeps your files) and start as above. The startup file can stay at `/opt/finbot.env`. + +## Troubleshooting + +* **The container stops at once and keeps restarting:** the log says why. `ERROR config: ... unknown key ...` is a + mistake in `/opt/finbot.env` or `preferences.conf`; fix it and `sudo docker compose up -d`. +* **`PermissionError` on `/data`:** the data dir is not the container user's: `sudo chown -R 10001:10001 + /opt/finbot-data`. +* **`/data does not exist: --create makes it`:** the data dir is not mounted: `compose.yaml` mounts + `../finbot-data`; run `util/docker_setup.sh` to make it. +* **No log in the journal:** the host has no journald; use `FINBOT_LOG_DRIVER=json-file` (see [Logs](#logs)). diff --git a/doc/install.md b/doc/install.md new file mode 100644 index 00000000..12624d13 --- /dev/null +++ b/doc/install.md @@ -0,0 +1,72 @@ +# Installing on a host + +finbot runs on any Linux host with Python 3.12 or later; production runs Debian's own packages under systemd, as +below. To run it in a container instead, see [docker.md](docker.md). + +## Requirements +* Most features need a Sharesight paid plan, preferably with automatic trade imports, and an API key + ([configuration.md](configuration.md#sharesight)). +* A Telegram bot, a Slack app or a Discord bot: see the network's guide, [Telegram](telegram.md), [Slack](slack.md) or + [Discord](discord.md). +* No server: finbot fetches its updates from each network itself (pull mode: Telegram's long polling, Slack's Socket + Mode, Discord's gateway). In HTTP mode instead, a network posts them to a public web server with a domain name and a + matching certificate, which proxies them to the bot (each guide's _HTTP mode_). +* Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum), and these modules, pinned to versions with + 3.13 support in `requirements.txt`: croniter, python-dateutil, requests, matplotlib and numpy. +* For a network in HTTP mode (its webhook URL set), also waitress; for Slack in Socket Mode, websocket-client; for + Discord's gateway, discord.py, and for its HTTP mode, PyNaCl. These are in `requirements-optional.txt`, each imported + only by the mode that needs it. + +## Debian +1. The packages, and those of the modes you use: + ``` + sudo apt install git python3 python3-requests python3-croniter python3-dateutil python3-matplotlib python3-numpy + sudo apt install python3-waitress # any network in HTTP mode + sudo apt install python3-websocket # Slack's Socket Mode + sudo apt install python3-discord # Discord's gateway + sudo apt install python3-nacl # Discord's HTTP mode + ``` +2. The code: + ``` + git clone https://github.com/robdevops/finbot.git ~/finbot + ``` +3. The startup file, from its template; then set each network's keys from its guide, and Sharesight's: + ``` + sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finbot.env + sudoedit /opt/finbot.env + ``` +4. Check it: `~/finbot/util/check_config.py /opt/finbot.env` +5. Run it under systemd, below. The first start makes the data directory (`var/` in the checkout, or + `/var/lib/finbot` with `DynamicUser=`) and gives each home chat its first jobs ([scheduling.md](scheduling.md)). + +[configuration.md](configuration.md) has every key and preference. + +## Other distributions: pip or uv +``` +sudo $(which apt dnf yum) install git python3-pip +git clone https://github.com/robdevops/finbot.git ~/finbot +cd ~/finbot +pip3 install -r requirements.txt -r requirements-optional.txt +``` +Or, with [uv](https://docs.astral.sh/uv/), which also fetches the exact Python from `.python-version`: +``` +uv venv && uv pip install -r requirements.txt -r requirements-optional.txt +``` +`requirements-optional.txt` can be left out, or cut down to the modes you use. + +## Daemonize (systemd) +`finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy +`finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of +[`finbot.env`](configuration.md#finbotenv) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start +it. `systemctl reload finbot` re-reads `preferences.conf` without a restart; a change to `finbot.env` needs +`systemctl restart finbot`: +``` +sudo cp -v ~/finbot/finbot.service /etc/systemd/system/ +sudo sed -i 's/CHANGEME/YOUR LINUX USERNAME/' /etc/systemd/system/finbot.service +sudo systemctl daemon-reload +sudo systemctl enable finbot --now +``` +The bot's log: +``` +journalctl -fu finbot +``` diff --git a/doc/scheduling.md b/doc/scheduling.md new file mode 100644 index 00000000..94ee1421 --- /dev/null +++ b/doc/scheduling.md @@ -0,0 +1,54 @@ +# Scheduling + +Each chat has its own jobs: reports that run on a schedule and post to that chat. Anyone in the chat manages them there, +with `.jobs` (also `.job`, `.cron`, `.crons` or `.schedule`): +``` +.jobs this chat's jobs, numbered, each with its next run; then the system jobs +.jobs add the reports a job can run, and what each takes +.jobs add 29 21 * * Mon-Fri price interday +.jobs run the list, with a button per job that runs it now; .jobs run 2 runs job 2 +.jobs toggle the list, with a button per job that switches it on or off; .jobs toggle 2 +.jobs del 2 deletes job 2 +``` + +A job is a time in crontab's syntax, then one or more reports with their arguments: +* **The time** is five fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists + work) or an `@macro` such as `@daily`. A time that cannot be or never comes (minute 61, 30 February) is refused, + saying why. +* **Times are in UTC**, which keeps the ASX's jobs and NYSE's each within an hour of their local times all year. To + follow one market's clocks exactly, a job names its zone first, as crontab writes it: + `.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession`. `.jobs` shows each next run in UTC and in + the `timezone` preference. +* **The reports** are `trades`, `price midsession|interday|premarket|close|DAYS`, `cal earnings|ex-dividend`, + `performance [DAYS]`, `rating`, `milestone`, `shorts`, `reminder` and `podcasts FEED_URL`. Several, separated by `;`, + run one after another: `.jobs add 28 21 * * Sat cal earnings; cal ex-dividend; price 7; performance 7; rating`. + Quote an argument that holds spaces or a `;` (a feed URL, say). +* A chat has 30 jobs at most. + +A job posts to its own chat, whichever way that chat's updates reach finbot: the jobs are kept by network and chat id +(`finbot_schedule.json` in the data directory), so switching a network between pull and HTTP mode keeps them. Each chat +also keeps its own record of what a report has told it (trades, rating changes, record highs, podcast episodes), so the +same report scheduled in two chats tells each of them once. A run from `.jobs run` is a scheduled run: it posts to the +chat and updates the same record. A change counts at once, with no reload. + +**The system jobs**, fixed in `lib/jobs.py` and listed by `.jobs` under _System_, keep the cache full and tidy: +`sharesight_warm` every 30 minutes, `yahoo_warm` every 8 hours (see [Caching](caching.md)), and `cleanup 30` weekly, +which deletes cache files older than 30 days. A chat cannot add, switch or run them. + +**From `schedule.cron`**, where the schedule used to be: the first time finbot starts without `finbot_schedule.json`, +it gives each home chat (`telegramChatID`, `slackChannelID`, `discordChannelID`) the jobs of the data directory's +`schedule.cron`, else of the shipped `defaults/schedule.cron`, each in the zone it ran in (the server's), without the +system jobs. It copies the reports' records to each home chat too, so that nothing old is told again. It logs what it +did, and any line it left out, and reads the file no more: delete the data directory's copy afterwards. + +Behaviour worth knowing: +* The reports run in `bot.py` itself, each in a thread of its own, sharing the bot's user, sandbox, environment, data + directory, caches and connections; charts are drawn one at a time, and a slow report holds up no chat command. There + is no cron to set up: remove any old crontab entries that ran finbot's scripts, or their reports come twice. +* A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last + `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. A job that is switched off + does not fire, and does not catch up once it is switched on again. +* A job still running when it is next due is skipped. A command still running after 5 minutes is reported once, and + left to finish (a thread cannot be stopped, but every request it makes has a timeout); until it does, it is not + started again. Each command of a job has 5 minutes of its own. +* A failure is reported to the job's chat. The bot logs each job as it runs (`journalctl -u finbot`). diff --git a/doc/slack.md b/doc/slack.md index 038d6150..c3d3dd01 100644 --- a/doc/slack.md +++ b/doc/slack.md @@ -1,48 +1,55 @@ # Slack -finbot reaches Slack through a Slack app of your own. The app's incoming webhook posts the scheduled reports into a -channel; the app's bot also answers commands. The bot builds on the webhook, which carries the scheduled reports unless -the bot posts them itself, charts and all (`slackChannelID`). +finbot reaches Slack through a Slack app of your own, whose bot posts each channel's scheduled reports there, charts +and all, and answers commands: `/price`, `.price`, `@finbot price` and the rest +([the list](commands.md#supported-commands)), with charts in its replies, ⏳ on a command's message while it works +(Slack has no typing notice for bots), and period buttons under a chart that redraw it in place. -## Incoming webhook, or the interactive bot? +## The bot -| | Incoming webhook | Interactive bot (with the webhook) | -|---|---|---| -| Scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest | ✓, as text | ✓, as text through the webhook, or with their charts as the bot (`slackChannelID`) | -| Chat commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)) | ✗ | ✓ | -| Charts in the replies to commands | n/a | ✓, uploaded into the channel | -| A sign that finbot is working on a command | ✗ | ✓: ⏳ on the command's message, Slack having no typing notice for bots | -| Period buttons under a chart, which redraw it for that period in place | ✗ | ✓ | -| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | -| `finbot.env` keys | `slack_webhook` | That one, plus `slackOutgoingWebhook`, `slackSigningSecret` and `slackBotToken` (and `slackChannelID`, for charts in the scheduled reports) | +### Socket Mode or HTTP mode? -Start with [Incoming webhook](#incoming-webhook). For commands, go on to [Interactive bot](#interactive-bot) -afterwards, in the same app. +The bot does the same either way. What differs is how it hears from Slack: -## Incoming webhook - -1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick - your workspace. -2. Under _Features > Incoming Webhooks_, turn _Activate Incoming Webhooks_ on, then _Add New Webhook to Workspace_ and - pick the channel for the reports. -3. Copy the webhook URL into `finbot.env` as `slack_webhook`. -4. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the channel. +| | Socket Mode (the default) | HTTP mode | +|---|---|---| +| How messages, slash commands and button presses reach finbot | Over a socket that finbot opens to Slack | Slack posts them to your HTTPS server | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| Python module | `websocket-client` (Debian: `python3-websocket`) | `waitress` (Debian: `python3-waitress`) | +| `finbot.env` keys | `slackBotToken`, `slackAppToken` | `slackBotToken`, `slackOutgoingWebhook`, `slackSigningSecret` | + +`slackOutgoingWebhook` chooses the mode: set, HTTP mode; empty, Socket Mode. The app's own _Socket Mode_ setting must +agree, as Slack sends to one or the other: to finbot's socket while it is on, to the Request URLs while it is off. To +switch, change both (paste the manifest for the other mode, which sets the app's side) and restart finbot. + +### Socket Mode + +1. Print the app's manifest, which sets up the whole app at once (the slash commands, the scopes, the events, the + buttons, direct messages and Socket Mode): `util/slack_manifest.py`. The bot then hears the messages that mention it + (`@finbot price`), its slash commands and direct messages. With `--every-message`, it hears every message in its + channels instead of the mentions, so that `.price` works there without mentioning it; not both, or it would answer a + mention twice. +2. At https://api.slack.com/apps choose _Create New App > From an app manifest_, pick your workspace, paste the + manifest into the JSON tab, and _Create_. +3. Under _Basic Information > App-Level Tokens_, choose _Generate Token and Scopes_: name it (finbot, say), add the + `connections:write` scope and _Generate_. Copy the token, which starts with `xapp-`, into `finbot.env` as + `slackAppToken`. +4. Install the socket module: `sudo apt install python3-websocket` (or `pip install websocket-client`). +5. Go on with [installing the app](#installing-the-app), below. ``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA +slackAppToken=xapp-your-app-level-token +slackBotToken=xoxb-your-bot-token +slackChannelID=C0123456789 ``` -An incoming webhook takes text only, so the scheduled reports come without charts. The [interactive bot](#interactive-bot) -can post them with their charts. +### HTTP mode -## Interactive bot +Slack delivers the messages, the button presses and the slash commands to finbot over HTTPS. finbot listens on +`http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name +with a valid certificate. -Slack delivers the messages that mention the bot (or every message in a channel, as you choose) to finbot over HTTPS. -finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, -on a domain name with a valid certificate. - -1. Set up the [incoming webhook](#incoming-webhook), above. -2. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests +1. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests come from: ``` server { @@ -64,45 +71,55 @@ on a domain name with a valid certificate. `util/nginx_aws_cron` writes `/etc/nginx/aws_subnets`, an `allow` line for each of Amazon's networks, and reloads nginx. Run it once as root before nginx loads this configuration, or put it in `/etc/cron.daily/` to keep the list current. -3. In the app's _Basic Information > App Credentials_, copy the _Signing Secret_ into `finbot.env` as +2. In `finbot.env`, set `slackOutgoingWebhook` to the URL Slack should post to, and restart finbot, so that it can + answer Slack's check of the URL in step 4. +3. Print the app's manifest with that URL, which Slack then posts everything to: + `util/slack_manifest.py --url https://www.example.com:8443/slack` (and `--every-message`, as for + [Socket Mode](#socket-mode)). +4. At https://api.slack.com/apps choose _Create New App > From an app manifest_, pick your workspace, paste the + manifest into the JSON tab, and _Create_. Slack checks the URL, and finbot answers. +5. In the app's _Basic Information > App Credentials_, copy the _Signing Secret_ into `finbot.env` as `slackSigningSecret`. finbot checks Slack's signature on every request with it, and turns away a request signed more than five minutes earlier: a replay, or a server clock that is wrong. * The older _Verification Token_, set as `slackOutgoingToken`, still works in place of the signing secret, but Slack has deprecated it: the token travels in every request, where the signing secret never does. -4. In `finbot.env`, set `slackOutgoingWebhook` to the URL Slack should post to, and restart finbot, so that it can - answer Slack's check of the URL in the next step. -5. Under _Event Subscriptions_: - * Turn _Enable Events_ on, and enter the same URL as the _Request URL_. Slack checks it at once, and finbot answers. - * Under _Subscribe to bot events_, add `app_mention`: the bot sees messages that mention it, such as - `@finbot price`. - * Or add `message.channels` instead, for the bot to see every message in its channels and answer the commands - that start with `.`. Not both: the bot would answer a mention twice. - * For direct messages to the bot, add `message.im`, and tick _App Home > Allow users to send Slash commands and - messages from the messages tab_. - * _Save Changes_. -6. Under _Interactivity & Shortcuts_, turn _Interactivity_ on, enter the same URL as the _Request URL_, and _Save - Changes_. Slack posts there the presses of the period buttons under a chart; without it, Slack answers a press with - an error. -7. Under _OAuth & Permissions > Scopes > Bot Token Scopes_, add `chat:write`, `files:write`, `files:read` and - `reactions:write`. The upload of a chart needs both `files` scopes, and the ⏳ on a command that takes a while - `reactions:write`. -8. At the top of _OAuth & Permissions_, _Install to Workspace_ (or _Reinstall to Workspace_ when you changed scopes of - an installed app), and copy the _Bot User OAuth Token_ into `finbot.env` as `slackBotToken`. finbot replies with it; - without it, commands get no reply, and the log says so at startup and for each reply. -9. Restart finbot, and invite the bot to the channel: `/invite @finbot`. -10. Send `@finbot help` in the channel (or `.help`, with `message.channels`). -11. For charts in the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: - the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot then posts the - scheduled reports there, as it posts its replies, and `slack_webhook` is no longer used. +6. Go on with [installing the app](#installing-the-app), below. ``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA slackOutgoingWebhook=https://www.example.com:8443/slack slackSigningSecret=0123456789abcdef0123456789abcdef slackBotToken=xoxb-your-bot-token slackChannelID=C0123456789 ``` +### Installing the app + +1. Under _Install App_ (or _OAuth & Permissions_), _Install to Workspace_, and copy the _Bot User OAuth Token_ into + `finbot.env` as `slackBotToken`. finbot replies with it; without it, commands get no reply, and the log says so at + startup and for each reply. +2. Restart finbot, and invite the bot to the channel: `/invite @finbot`. In Socket Mode the log says + `slack: Socket Mode (no slackOutgoingWebhook)`. +3. Send `/help` or `@finbot help` in the channel (or `.help`, with `--every-message`), or a direct message to the bot + in its _Messages_ tab. +4. Set `slackChannelID` to the channel's id (in Slack, click the channel's name: the _Channel ID_, which starts with + C, is at the bottom of _About_), and restart finbot. It is Slack's home chat: finbot's first start gives it the + default jobs ([Scheduling](scheduling.md)), and it gets the error reports. Every channel manages its own + jobs with `.jobs`. + +An app set up by hand before takes the manifest too: paste it into _Features > App Manifest_ and _Save Changes_; +Slack shows what changes, and asks to reinstall the app when its scopes do. + +### Slash commands + +Each command is a slash command too, with its usage as Slack's hint: `/price`, `/dividend` and `/earnings`, `/buy` +and `/sell`, `/forwardpe` for `.forward pe`, `/stock` (or `/profile`) for `.SYMBOL`, `/jobs` (or `/job`), and the rest. +Slack makes slash commands only from the app's settings, so they come in the manifest: when finbot's commands change, +print it again and paste it into _App Manifest_. + +A slash command's answer goes to the channel or direct message it was sent in, as a message's does; the bot must be in +the channel (`/invite @finbot`). Slack shows no ⏳ while it works: there is no message of a slash command to put it +on. + A chart is uploaded with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`). With its period buttons, it goes in a message of blocks: the report, the chart (uploaded into no channel, and shown by its id) and the buttons. A press shows the chart for that period in the message in place of the last one: the upload of @@ -111,5 +128,5 @@ drawn and uploaded anew (Slack cannot replace a file). An upload that no message are past, so the bot's files do not pile up. If Slack will not show a chart with its buttons, the chart is uploaded into the channel without them; if an upload fails (a missing scope, say), the report goes out as text. The log says why. -`.jobs` lists the scheduled commands with a button for each, as on Telegram; a button from a list that has since -changed puts the current list in its place. +`.jobs run` and `.jobs toggle` list the channel's jobs with a button for each, as on Telegram; a button from a list +that has since changed puts the current list in its place. diff --git a/doc/telegram.md b/doc/telegram.md index 82520fc2..e74c381a 100644 --- a/doc/telegram.md +++ b/doc/telegram.md @@ -1,57 +1,71 @@ # Telegram -finbot talks to Telegram as a bot user that you create with BotFather. The same bot can post the scheduled reports -only, or also answer commands in your chats: the second builds on the first. +finbot talks to Telegram as a bot user that you create with BotFather. -## Reports only, or the interactive bot? +## Pull mode or HTTP mode? -| | Reports only | Interactive bot | +The bot posts the scheduled reports and answers commands in your chats either way. What differs is how it hears from +Telegram: + +| | Pull mode (the default) | HTTP mode | |---|---|---| -| Scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest | ✓ | ✓ | -| Charts with the reports | ✓ | ✓ | -| Chat commands: `.price`, `.history`, `.performance` and the rest ([the list](../README.md#supported-commands)) | ✗ | ✓ | -| Period buttons under charts, and `.jobs` buttons | ✗ | ✓ | -| "typing…" while a command works | ✗ | ✓ | -| A keyboard of the common commands in a direct chat with the bot | ✗ | ✓ | -| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| How messages and button presses reach finbot | finbot asks Telegram for them (long polling) | Telegram posts them to your HTTPS server | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate, on port 443, 80, 88 or 8443 | | `finbot.env` keys | `telegramBotToken`, `telegramChatID` | Those two, plus `telegramOutgoingWebhook` and `telegramOutgoingToken` | -Start with [Reports only](#reports-only). For commands, go on to [Interactive bot](#interactive-bot) afterwards. +Everything else is the same in both: the scheduled reports with their charts, the chat commands (`/price` or `.price`, +`.history`, `.performance` and the rest, [the list](commands.md#supported-commands)), period buttons under charts and +the buttons of `.jobs run` and `.jobs toggle`, "typing…" while a command works, and a keyboard of the common commands in +a direct chat with the bot. + +As finbot starts, it sets the bot's command menu, which Telegram shows when `/` is typed: each command with its usage +(`/forwardpe` for `.forward pe`, `/stock` for `.SYMBOL`). `/profile` and `/job`, the second names of `/stock` and +`/jobs`, are left out of the menu, and work when typed, as every name of a command does. In a group, `/price@finbot_bot` +is the same as `/price`. -## Reports only +`telegramOutgoingWebhook` chooses the mode: set, HTTP mode; empty, pull mode. A restart switches. As finbot starts in +pull mode it deletes the webhook, and in HTTP mode it sets it; the updates sent meanwhile are kept by Telegram and +arrive after the switch. + +## Pull mode 1. Create the bot: message [@BotFather](https://telegram.me/BotFather), send `/newbot` and answer its questions. It replies with the bot's token: put it in `finbot.env` as `telegramBotToken`. 2. Add the bot to the group or channel the reports should go to. In a channel, make it an administrator, so that it can post. -3. Stop strangers adding your bot to their groups: send BotFather `/setjoingroups`, pick the bot and choose `Disable`. +3. Let the bot read the chat: make it an administrator of the group. A bot that is not an administrator sees only the + commands that start with `/`, and finbot's start with `.`. Alternatively, send BotFather `/setprivacy` and choose + `Disable`. +4. Stop strangers adding your bot to their groups: send BotFather `/setjoingroups`, pick the bot and choose `Disable`. Do this after step 2, as it stops you adding the bot to groups too. -4. Put the chat's id in `finbot.env` as `telegramChatID`. +5. Put the chat's id in `finbot.env` as `telegramChatID`. * A group's id is negative. A channel's or a supergroup's is negative and 13 characters long: prepend `-100` if yours is shorter. - * To find it, send `/start@` in the chat, then open - `https://api.telegram.org/bot/getUpdates` in a browser and look for `"chat":{"id":`. This works - only while the bot has no webhook, so before you set up the interactive bot. + * To find it, stop finbot first (`sudo systemctl stop finbot`): Telegram hands each update to one reader only. Send + `/start@` in the chat, then open `https://api.telegram.org/bot/getUpdates` + in a browser and look for `"chat":{"id":`. If the bot is in HTTP mode, getUpdates refuses while the webhook is + set: open `https://api.telegram.org/bot/deleteWebhook` first (finbot sets it again as it + starts). * A group's id changes when Telegram turns it into a supergroup, which some changes to its settings do. Update `telegramChatID` then: finbot does not follow the change. -5. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the chat. +6. Start finbot (`sudo systemctl restart finbot`). The log says `telegram: long polling for updates`. The first start + gives this chat, Telegram's home chat, the default jobs ([Scheduling](scheduling.md)), so the scheduled + reports arrive there; it gets the error reports too. Every chat manages its own jobs with `.jobs`. +7. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. ``` telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA telegramChatID=-1001000000000 ``` -## Interactive bot +## HTTP mode Telegram delivers the messages in your chats, and each button press, to finbot over HTTPS. finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name with a valid certificate. Telegram posts only to ports 443, 80, 88 and 8443. -1. Set up the bot for [reports only](#reports-only), above. -2. Let the bot read the chat: make it an administrator of the group. A bot that is not an administrator sees only the - commands that start with `/`, and finbot's start with `.`. Alternatively, send BotFather `/setprivacy` and choose - `Disable`. -3. Put finbot behind an HTTPS server. An nginx example, which lets in only Telegram's addresses: +1. Set up the bot as for [pull mode](#pull-mode), above. +2. Put finbot behind an HTTPS server. An nginx example, which lets in only Telegram's addresses: ``` server { listen 8443 ssl; @@ -73,12 +87,12 @@ with a valid certificate. Telegram posts only to ports 443, 80, 88 and 8443. and reloads nginx. Run it once as root before nginx loads this configuration, or put it in `/etc/cron.daily/` to keep the list current. `util/iptables_telegram_cron` goes further, in the firewall: of the traffic to ports 443 and 8443 it lets in only Telegram's networks on 8443 and drops the rest, so nothing else can be served on those ports. -4. In `finbot.env`, set `telegramOutgoingWebhook` to the URL Telegram should post to, and `telegramOutgoingToken` to a +3. In `finbot.env`, set `telegramOutgoingWebhook` to the URL Telegram should post to, and `telegramOutgoingToken` to a password of your choosing: 1 to 256 characters, of letters, digits, `_` and `-`. Telegram sends the password with every update, and finbot turns away a request without it. -5. Restart finbot. As it starts, it registers the URL with Telegram, unregistering it first, which also wakes up a stale - connection. If Telegram refuses, finbot does not start, and the log says why. -6. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. +4. Restart finbot. As it starts, it sets the webhook again, deleting it first, which also wakes up a stale connection. + If Telegram refuses, finbot does not start, and the log says why. +5. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. ``` telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA diff --git a/finbot-flush.service b/finbot-flush.service deleted file mode 100644 index d79c3580..00000000 --- a/finbot-flush.service +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Flush finbot's cache (sudo systemctl start finbot-flush) - -[Service] -Type=oneshot -# finbot deletes its cache files on SIGUSR1 and keeps its state; its job processes ignore the signal -ExecStart=/bin/systemctl kill --signal=USR1 finbot.service diff --git a/finbot.service b/finbot.service index fd74abf0..993302ba 100644 --- a/finbot.service +++ b/finbot.service @@ -6,7 +6,7 @@ Type=simple User=CHANGEME # Or, instead of User=: DynamicUser=yes plus StateDirectory=finbot (state goes to /var/lib/finbot automatically) ExecStart=/home/CHANGEME/finbot/bot.py -# systemctl reload finbot re-reads preferences.conf and schedule.cron without a restart +# systemctl reload finbot re-reads preferences.conf without a restart ExecReload=/bin/kill -HUP $MAINPID # Refuse to start on a mistake in finbot.env or preferences.conf (the bot never reads finbot.env itself, so this is # where its typos are caught). "+" runs it as root. diff --git a/lib/buttons.py b/lib/buttons.py index cf8b0982..c11fe28c 100644 --- a/lib/buttons.py +++ b/lib/buttons.py @@ -1,6 +1,6 @@ -"""Buttons under a message, and what their data says when one is pressed (parse): a chart's period buttons (1D | 7D | -1M | 1Y | Max), whose data carries what its builder needs (charts.BUILDERS), and a .jobs command's button. Each -transport draws Button rows its own way.""" +"""Buttons under a message, what their data says when one is pressed (parse), and what a press is told at once +(toast): a chart's period buttons (1D | 7D | 1M | 1Y | Max), whose data carries what its builder needs +(charts.BUILDERS), and a job's buttons in .jobs run and .jobs toggle. Each transport draws Button rows its own way.""" import hashlib import typing @@ -21,10 +21,14 @@ class ChartPress(typing.NamedTuple): class JobPress(typing.NamedTuple): - """A .jobs command's button pressed: the command's number in the list and its checksum, both for - jobs.Scheduler.pick to check.""" + """A job's button pressed: the job's number in its chat's list and its checksum, both for jobs.Scheduler.pick to + check, and what to do with it: run it now, or switch it on or off (toggle).""" number: str checksum: str + action: str = 'run' + + +JOB_ACTIONS = {'j': 'run', 'e': 'toggle'} # a job button's kind, its data's first letter, and what it does PERIODS = {'d': ('1D', 1), 'w': ('7D', 7), 'm': ('1M', 30), 'y': ('1Y', 365), 'x': ('Max', None)} @@ -71,19 +75,30 @@ def keyboard(kind, args, active=None): ref = _ref(args) return [[Button(PERIODS[key][0], f"c|{kind}|{key}|{ref}", key == active) for key in BUTTONS[kind]]] -def job_button(number, label, checksum): - """A .jobs command's button: 'N. label', cut to 30 characters, its data the number and the command's checksum.""" - return Button(f"{number}. {label}"[:30], f"j|{number}|{checksum}") +def job_button(number, label, checksum, action='run'): + """A job's button, which runs it now (or with action 'toggle', switches it on or off): 'N. label', cut to 30 + characters, its data the job's number and checksum.""" + kind = next(kind for kind, does in JOB_ACTIONS.items() if does == action) + return Button(f"{number}. {label}"[:30], f"{kind}|{number}|{checksum}") def parse(data): - """What a pressed button's data asks for: a JobPress for a .jobs button (whatever follows 'j|', which pick refuses - unless it is a command's number and checksum), a ChartPress for a period button of a known kind and period, or None - for data of neither (from an older finbot, or forged).""" + """What a pressed button's data asks for: a JobPress for a job's button (whatever follows 'j|' or 'e|', which pick + refuses unless it is a job's number and checksum), a ChartPress for a period button of a known kind and period, or + None for data of neither (from an older finbot, or forged).""" kind, _, rest = data.partition('|') - if kind == 'j': + if kind in JOB_ACTIONS: number, _, checksum = rest.partition('|') - return JobPress(number, checksum) + return JobPress(number, checksum, JOB_ACTIONS[kind]) parts = rest.split('|', 2) if kind == 'c' and len(parts) == 3 and parts[0] in BUTTONS and parts[1] in BUTTONS[parts[0]]: return ChartPress(*parts) return None + +def toast(data): + """What a press is told at once, from the button's data alone (the network acknowledges a press before finbot + knows more): 'Loading 1Y…' for a period button, 'Running…' or 'Switching…' for a job's, None for data of + neither.""" + pressed = parse(data) + if isinstance(pressed, JobPress): + return 'Running…' if pressed.action == 'run' else 'Switching…' + return f"Loading {PERIODS[pressed.period][0]}…" if pressed else None diff --git a/lib/cache.py b/lib/cache.py index 7c8bb7ac..280ad7f9 100644 --- a/lib/cache.py +++ b/lib/cache.py @@ -3,12 +3,12 @@ import io import json import os -import sys import tempfile import time from pathlib import Path from lib import config +from lib import log def path(name): @@ -29,13 +29,10 @@ def _fresh(name, seconds): if seconds is None: seconds = config.prefs.cache_seconds where, written = path(name), age(name) - if written is None: - print("cache miss:", where, file=sys.stderr) if config.prefs.debug else None + if written is None or written >= seconds: + log.debug("cache miss:", where) return None - if written >= seconds: - print("cache expired:", where, file=sys.stderr) if config.prefs.debug else None - return None - print("cache hit:", where, "TTL:", f"{seconds - written:.0f} s", file=sys.stderr) if config.prefs.debug else None + log.debug(f"cache hit: {where} TTL: {seconds - written:.0f}s") return where @@ -85,7 +82,7 @@ def write(name, data): """Write a cache file's JSON, compact: nobody reads it but finbot (a 10-year chart takes a third of the space, and parses faster). See write_file.""" where = path(name) - print("cache write:", where, file=sys.stderr) if config.prefs.debug else None + log.debug("cache write:", where) write_file(where, json.dumps(data, separators=(',', ':')).encode('utf-8')) diff --git a/lib/charts.py b/lib/charts.py index 0dbf7781..185bb35f 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -1,18 +1,20 @@ """A chart with period buttons, built by its kind's builder: the command's first view, and again for the period of a -button pressed, so that both look alike.""" +button pressed, so that both look alike. Each builder takes the network's dialect, the chat's key (a price list covers +the stocks that chat follows), the period, its days, then what the buttons carry.""" from lib import answer from lib import buttons from lib import formats from lib import links +from lib import watchlist from lib import yahoo from lib.reports import history, performance, price -def history_chart(service, period, days, ticker): +def history_chart(service, chat, period, days, ticker): """.history: the change over every period, the chosen one marked, and the line of that period (or of it all).""" lines, chart = history.history_payload(ticker, service, days) return '\n'.join(lines), chart -def price_chart(service, period, days, ticker): +def price_chart(service, chat, period, days, ticker): """.price SYMBOL [period]: one stock's change over the period (all its history for Max), from Yahoo, and its line.""" market_data = yahoo.fetch([ticker]) @@ -25,22 +27,22 @@ def price_chart(service, period, days, ticker): caption = f"{formats.get_emoji(percent)} {title} ({link}) {formats.signed_percent(round(percent))}" return caption, chart -def compare_chart(service, period, days, *tickers): +def compare_chart(service, chat, period, days, *tickers): """.compare: the stocks' changes over the period, rebased on one chart.""" caption, chart = history.prepare_compare(service, list(tickers), days) return '\n'.join(caption), chart -def performance_chart(service, period, days, portfolio): +def performance_chart(service, chat, period, days, portfolio): """.performance: each portfolio over the period, or the one named ('' for all).""" payload, chart = performance.report(service, days or buttons.MAX_DAYS, portfolio or None) return '\n'.join(payload), chart -def price_list_chart(service, period, days, threshold, top): - """.price: the moves past `threshold`, or the top and bottom `top` ('0': not a top list), over the period; period - 'd' is today's moves, the command's default view, not a lookback of one day.""" +def price_list_chart(service, chat, period, days, threshold, top): + """.price: the moves past `threshold` among the stocks the chat follows, or the top and bottom `top` ('0': not a + top list), over the period; period 'd' is today's moves, the command's default view, not a lookback of one day.""" span = {'mode': 'interday'} if period == 'd' else {'mode': 'period', 'days': days or buttons.MAX_DAYS} threshold, top = float(threshold), int(top) or None - found = price.moves(threshold=threshold, interactive=True, **span) + found = price.moves(watchlist.with_holdings(chat), threshold=threshold, interactive=True, **span) payload, chart = price.report(service, found, threshold=threshold, interactive=True, top=top, **span) if not chart: raise answer.Answer(*(payload or ['no price data'])) @@ -48,9 +50,10 @@ def price_list_chart(service, period, days, threshold, top): BUILDERS = {'h': history_chart, 'p': price_chart, 'c': compare_chart, 'f': performance_chart, 'l': price_list_chart} -def build(kind, args, service, period=None, days=None): - """(caption, chart spec or None) of a chart kind from the arguments its buttons carry (buttons.keyboard): over a - button period, or over `days` (a command's own span: .compare 3m), or the command's default view.""" +def build(kind, args, service, chat, period=None, days=None): + """(caption, chart spec or None) of a chart kind, in a chat (its key), from the arguments its buttons carry + (buttons.keyboard): over a button period, or over `days` (a command's own span: .compare 3m), or the command's + default view.""" if kind not in BUILDERS: raise ValueError(f"unknown chart kind {kind}") - return BUILDERS[kind](service, period, buttons.PERIODS[period][1] if period else days, *args) + return BUILDERS[kind](service, chat, period, buttons.PERIODS[period][1] if period else days, *args) diff --git a/lib/config.py b/lib/config.py index 7a22a430..73186649 100644 --- a/lib/config.py +++ b/lib/config.py @@ -7,7 +7,7 @@ from pathlib import Path # TO CONFIGURE, DO NOT EDIT THIS FILE. There are two config files: -# finbot.env the startup file: DATA_DIR, ip, port, credentials, telegramChatID, scheduler. Never read by the +# finbot.env the startup file: DATA_DIR, ip, port, credentials, the home chats. Never read by the # bot: the launcher loads it into the environment (systemd EnvironmentFile=, docker --env-file, # init.d `set -a; . finbot.env`). A change needs a restart. # preferences.conf how the bot behaves: /preferences.conf, read by the bot itself at start and on every @@ -15,18 +15,28 @@ REPO_DIR = Path(__file__).resolve().parent.parent -STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', 'sharesight_code', - 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', - 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', - 'telegramOutgoingToken', 'telegramOutgoingWebhook') -SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', - 'discord_webhook', 'slackBotToken', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', - 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed +STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler_catchup', 'sharesight_code', + 'sharesight_client_id', 'sharesight_client_secret', 'slackBotToken', 'slackAppToken', 'slackChannelID', + 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', + 'telegramOutgoingWebhook', 'discordBotToken', 'discordChannelID', 'discordOutgoingWebhook', 'discordPublicKey') +SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slackBotToken', 'slackAppToken', + 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', 'telegramOutgoingToken', + 'telegramOutgoingWebhook', 'discordBotToken', 'discordOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' -RETIRED = ('alliterate', 'chunk_maxlines', 'performance_use_sharesight') # preferences.conf keys that do nothing any -# more: accepted, with a note at start to delete them -# set by the bot for its job processes: the preferences arrive in the environment, do not read the file -PASSED = 'FINBOT_PREFERENCES_PASSED' +TEMPLATE = REPO_DIR / 'defaults' / PREFERENCES_FILE # every key commented out at its default; finbot never reads it +RETIRED = { # keys of either file that do nothing any more: accepted, so that they never stop a start, with a note at + # start saying what to do instead + 'alliterate': 'delete the line', + 'chunk_maxlines': 'delete the line', + 'performance_use_sharesight': 'delete the line (price performance comes from Yahoo)', + 'slack_webhook': 'the scheduled reports go to slackChannelID as the bot (slackBotToken); set those, and delete the' + ' line', + 'discord_webhook': 'the scheduled reports go to discordChannelID as the bot (discordBotToken); set those, and' + ' delete the line', + 'scheduler': 'the scheduler is always on, and a schedule without jobs runs nothing; delete the line', + 'debug': 'log_level sets what the log shows (INFO by default; DEBUG adds cache hits); delete the line', +} +LOG_LEVELS = ('ERROR', 'WARN', 'INFO', 'DEBUG') # log_level's values, the most severe first (lib/log.py) OLD_ENV = REPO_DIR / '.env' # the single config file of old versions, which python-dotenv loaded; nothing reads it now @@ -103,13 +113,36 @@ def read_preferences(path): return values +def template_keys(): + """The keys that defaults/preferences.conf sets: its lines that are not commented out, none as shipped. finbot never + reads the template, so a key set there does nothing; an unreadable template sets nothing either.""" + try: + text = read_text(TEMPLATE) or '' + except ConfigError: + return [] + return [line.partition('=')[0].strip() for line in text.splitlines() + if line.strip() and not line.lstrip().startswith('#')] + + +def template_note(preferences_path): + """What to tell whoever set keys in defaults/preferences.conf, which finbot never reads: to set them in + preferences_path instead, made first with util/sync_defaults.py when there is none. None when the template sets + nothing, as shipped.""" + keys = template_keys() + if not keys: + return None + fix = (f'edit {preferences_path} instead' if os.path.isfile(preferences_path) else + f'run util/sync_defaults.py --apply, then edit {preferences_path}') + return f"defaults/preferences.conf sets {', '.join(keys)}, but finbot never reads it: {fix}" + + def read_startup_file(path): """{key: value} of a finbot.env, for checking it offline (the bot never reads this file). Raises ConfigError.""" text = read_text(path) if text is None: raise ConfigError(f'{path} does not exist') - values, errors = parse_assignments(text, path, - lambda key: key_problem(key, STARTUP_KEYS, PREFERENCE_KEYS, PREFERENCES_FILE)) + values, errors = parse_assignments(text, path, lambda key: None if key in RETIRED else + key_problem(key, STARTUP_KEYS, PREFERENCE_KEYS, PREFERENCES_FILE)) if errors: raise ConfigError(errors) return values @@ -167,6 +200,14 @@ def float(self, name, default): """A number setting.""" return self.number(float, name, default) + def choice(self, name, default, choices): + """One of choices, in any case; anything else is an error, giving default.""" + value = str(self.value(name, default)).strip().upper() + if value in choices: + return value + self.errors.append(f"{name}={self.getenv(name, default)!r} is not one of {', '.join(choices)}") + return default + def words(self, name, default): """A list of words, separated by spaces: a tuple of them.""" value = self.value(name, default) @@ -181,23 +222,27 @@ def check(self): @dataclasses.dataclass(frozen=True) class Env: """The startup settings: finbot.env, which the launcher loads into the environment; fixed for the process.""" - data_dir: str # state: watchlist, seen trades, ratings, the schedule's record, preferences.conf + data_dir: str # state: each chat's jobs, the watchlist, seen trades, ratings, preferences.conf cache_dir: str # data_dir/cache ip: str port: int - scheduler: bool # run the scheduled reports (schedule.cron) inside bot.py scheduler_catchup: int # minutes: a run missed while the bot was down still fires if it was due this recently - telegramChatID: str # '' when unset, like every token below + telegramChatID: str # Telegram's home chat (its first jobs, the error reports); '' when unset, like the rest below telegramBotToken: str telegramOutgoingWebhook: str telegramOutgoingToken: str slackBotToken: str - slackChannelID: str # with slackBotToken, scheduled reports go to this channel as the bot, in place of slack_webhook + slackAppToken: str # xapp-: Socket Mode, Slack's pull mode (without slackOutgoingWebhook) + slackChannelID: str # with slackBotToken, Slack's home chat (its first jobs, the error reports) slackOutgoingWebhook: str slackOutgoingToken: str slackSigningSecret: str + discordBotToken: str + discordChannelID: str # with discordBotToken, Discord's home chat (its first jobs, the error reports) + discordOutgoingWebhook: str # the interactions endpoint URL (http mode); unset, finbot keeps to the gateway + discordPublicKey: str # the app's key, which checks the signature of each interaction posted to the endpoint sharesight_auth: dict - webhooks: dict # service -> outbound URL: Telegram's Bot API base, the Slack and Discord incoming webhooks + telegram_api: str # the Bot API's base URL for telegramBotToken ('' without one): a method's name follows it @property def preferences_path(self): @@ -210,30 +255,27 @@ def startup_values(getenv): r = _Reader(getenv) data_dir = getenv('DATA_DIR') or (getenv('STATE_DIRECTORY') or '').split(':')[0] or 'var' data_dir = str(REPO_DIR / data_dir) # a relative one is in the checkout, wherever finbot starts - hooks = {} - if getenv('slack_webhook'): - hooks['slack'] = getenv('slack_webhook').rstrip('/') - if getenv('discord_webhook'): - # the native endpoint, not the Slack-compatible one - hooks['discord'] = getenv('discord_webhook').rstrip('/').removesuffix('/slack') - if getenv('telegramBotToken'): - hooks['telegram'] = 'https://api.telegram.org/bot' + getenv('telegramBotToken').rstrip('/') + '/' + token = (getenv('telegramBotToken') or '').rstrip('/') env = Env( data_dir=data_dir, cache_dir=data_dir + '/cache', ip=r.str('ip', '127.0.0.1'), port=r.int('port', 5000), - scheduler=r.bool('scheduler', True), scheduler_catchup=r.int('scheduler_catchup', 5), telegramChatID=r.str('telegramChatID', ''), telegramBotToken=r.str('telegramBotToken', ''), telegramOutgoingWebhook=r.str('telegramOutgoingWebhook', ''), telegramOutgoingToken=r.str('telegramOutgoingToken', ''), slackBotToken=r.str('slackBotToken', ''), + slackAppToken=r.str('slackAppToken', ''), slackChannelID=r.str('slackChannelID', ''), slackOutgoingWebhook=r.str('slackOutgoingWebhook', ''), slackOutgoingToken=r.str('slackOutgoingToken', ''), slackSigningSecret=r.str('slackSigningSecret', ''), + discordBotToken=r.str('discordBotToken', ''), + discordChannelID=r.str('discordChannelID', ''), + discordOutgoingWebhook=r.str('discordOutgoingWebhook', ''), + discordPublicKey=r.str('discordPublicKey', ''), sharesight_auth={ "grant_type": 'client_credentials', "code": getenv('sharesight_code'), @@ -241,7 +283,7 @@ def startup_values(getenv): "client_secret": getenv('sharesight_client_secret'), "redirect_uri": 'urn:ietf:wg:oauth:2.0:oob' }, - webhooks=hooks) + telegram_api=f'https://api.telegram.org/bot{token}/' if token else '') r.check() return env @@ -254,7 +296,6 @@ class Prefs: cache: bool = True cache_seconds: int = 43200 country_code: str = 'AU' - debug: bool = False demote_leveraged: bool = True demote_volatile: bool = True exchange_suffix: str = 'AX' # Yahoo's suffix for the local exchange: a bare ticker Yahoo does not know gets it @@ -266,6 +307,7 @@ class Prefs: hyperlinkFooter: bool = True hyperlinkProvider: str = 'google' include_portfolios: tuple = () # portfolio ids + log_level: str = 'INFO' # what the log shows: one of LOG_LEVELS, and those more severe (lib/log.py) min_analysts: int = 7 # .buy, .sell and .forward pe list only stocks this many analysts cover past_days: int = 28 performance_cache_seconds: int = 2400 # Sharesight performance reports (40 minutes: sharesight_warm runs every 30) @@ -286,7 +328,8 @@ def preference_values(getenv, source=None): r = _Reader(getenv) readers = {bool: r.bool, int: r.int, float: r.float, str: r.str, tuple: r.words, 'exchange_suffix': lambda name, default: r.blankable(name, default).removeprefix('.').upper(), # blank: none - 'hyperlinkProvider': lambda name, default: r.str(name, default).lower()} + 'hyperlinkProvider': lambda name, default: r.str(name, default).lower(), + 'log_level': lambda name, default: r.choice(name, default, LOG_LEVELS)} values = {field.name: readers.get(field.name, readers[field.type])(field.name, field.default) for field in dataclasses.fields(Prefs) if field.compare} r.check() @@ -321,7 +364,7 @@ def _fail(e): try: env = startup_values(os.environ.get) preferences_path = env.preferences_path - _preferences = {} if os.environ.get(PASSED) else (read_preferences(preferences_path) or {}) + _preferences = read_preferences(preferences_path) or {} prefs = preference_values(_getenv_for(_preferences), _preferences) except ConfigError as e: _fail(e) @@ -376,16 +419,9 @@ def reload(): return changes -def child_environment(): - """For a job process: the preferences this process runs on, so it never sees a file edited since the last reload.""" - return dict(prefs.source, **{PASSED: '1'}) - - def check_startup_environment(): """For bot.py at start. Preferences must come from preferences.conf, not from the environment that finbot.env fills, and a leftover .env from before the split stops the start: its settings would otherwise be silently ignored.""" - if os.environ.get(PASSED): - return problems = [f'{key} is a preference; move it from finbot.env to {PREFERENCES_FILE}' for key in PREFERENCE_KEYS if key in os.environ] if 'volatile_multiplier' in os.environ: # the old name, from before the split: it never worked as a preference @@ -403,33 +439,65 @@ def check_startup_environment(): def startup_report(): """Lines for the log at start: where the config came from (counts only; check_config names the problems).""" - provided = [key for key in STARTUP_KEYS if key in os.environ] - lines = [f'startup environment provides {len(provided)} of {len(STARTUP_KEYS)} startup keys (values hidden)'] + def keys(count): + """'1 key', '2 keys'.""" + return f"{count} key{'' if count == 1 else 's'}" + lines = [f'startup environment ({keys(sum(key in os.environ for key in STARTUP_KEYS))} set)'] if os.path.isfile(preferences_path): - lines.append(f'preferences from {preferences_path} ({len(prefs.source)} set; reload with SIGHUP)') + lines.append(f'{preferences_path} ({keys(len(prefs.source))} set; SIGHUP to reload)') else: - lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, using the defaults; reload will have nothing to' - ' re-read') - return lines + retired_notes(prefs.source) + lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, reload unavailable') + return lines + retired_notes(os.environ, 'finbot.env') + retired_notes(prefs.source) -def retired_notes(preferences): - """A line for each key of a preferences.conf ({key: value}) that does nothing any more.""" - return [f'{key} in {PREFERENCES_FILE} does nothing any more: delete the line' - for key in RETIRED if key in preferences] +def retired_notes(values, file=PREFERENCES_FILE): + """A line for each key of a file's {key: value} that does nothing any more, saying what to do instead.""" + return [f'{key} in {file} does nothing any more: {instead}' for key, instead in RETIRED.items() if key in values] -def startup_warnings(): - """Settings in finbot.env that do not stop the start but keep part of the bot from working, a line each.""" +def startup_warnings(settings=None): + """Settings in finbot.env (an Env; the running one by default), and keys set in the template + defaults/preferences.conf, that do not stop the start but keep part of the bot from working, or do nothing, a line + each.""" + s = settings or env warnings = [] - if env.slackOutgoingWebhook and not env.slackBotToken: - warnings.append('slackOutgoingWebhook is set but slackBotToken is not: Slack commands will get no reply') - if env.slackChannelID and not env.slackBotToken: + if s.slackBotToken and not (s.slackAppToken or s.slackOutgoingWebhook): + warnings.append('slackBotToken is set but neither slackAppToken nor slackOutgoingWebhook is: finbot hears' + ' nothing from Slack, so its commands and buttons get no answer') + for key in ('slackOutgoingWebhook', 'slackAppToken'): + if getattr(s, key) and not s.slackBotToken: + warnings.append(f'{key} is set but slackBotToken is not: Slack commands will get no reply') + if s.slackAppToken and s.slackOutgoingWebhook: + warnings.append('slackAppToken is set but so is slackOutgoingWebhook: Slack posts its updates there' + ' (http mode), and the app-level token is not used') + if s.slackAppToken and not s.slackAppToken.startswith('xapp-'): + warnings.append("slackAppToken does not start with xapp-: it must be an app-level token (the app's Basic" + ' Information, App-Level Tokens), not the bot token') + if s.slackChannelID and not s.slackBotToken: warnings.append('slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot') - if env.slackOutgoingWebhook and not (env.slackSigningSecret or env.slackOutgoingToken): + if s.slackOutgoingWebhook and not (s.slackSigningSecret or s.slackOutgoingToken): warnings.append('slackOutgoingWebhook is set but neither slackSigningSecret nor slackOutgoingToken is:' ' every Slack request will be refused') - if env.telegramOutgoingWebhook and not env.telegramOutgoingToken: + if s.telegramOutgoingWebhook and not s.telegramOutgoingToken: warnings.append('telegramOutgoingWebhook is set but telegramOutgoingToken is not: every Telegram update will be' ' refused') + if s.telegramOutgoingToken and not re.fullmatch(r'[A-Za-z0-9_-]{1,256}', s.telegramOutgoingToken): + warnings.append('telegramOutgoingToken is not 1 to 256 letters, digits, _ or -: Telegram takes no other as the' + " webhook's secret, so setting the webhook will fail") + if s.telegramOutgoingToken and not s.telegramOutgoingWebhook: + warnings.append('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram' + ' updates (pull mode), and the token is not used') + if s.discordOutgoingWebhook and not s.discordBotToken: + warnings.append('discordOutgoingWebhook is set but discordBotToken is not: Discord commands will get no reply') + if s.discordChannelID and not s.discordBotToken: + warnings.append('discordChannelID is set but discordBotToken is not: scheduled reports cannot go there as the' + ' bot') + if s.discordOutgoingWebhook and not re.fullmatch(r'[0-9a-fA-F]{64}', s.discordPublicKey): + warnings.append("discordOutgoingWebhook is set but discordPublicKey is not the app's public key (64 hex" + ' digits): every Discord interaction will be refused') + if s.discordPublicKey and not s.discordOutgoingWebhook: + warnings.append('discordPublicKey is set but discordOutgoingWebhook is not: finbot keeps to the Discord gateway' + ' (pull mode), and the key is not used') + if note := template_note(s.preferences_path): + warnings.append(note) return warnings diff --git a/lib/delivery.py b/lib/delivery.py index ebdf48c4..793a5351 100644 --- a/lib/delivery.py +++ b/lib/delivery.py @@ -1,13 +1,16 @@ """Where a report goes and how: Report, the destinations (the chat that asked, or the default chats), deliver(), and the -one-line error report. A destination is a transport (lib/transports.py).""" +one-line error report. A destination is a transport: a network's bot (TelegramBot, SlackBot, DiscordBot).""" import dataclasses import html -import sys import traceback +from lib import log from lib import config +from lib import discord_common from lib import net from lib import render +from lib import slack_common +from lib import telegram_common from lib import transports @@ -20,33 +23,49 @@ class Report: buttons: list = None -REPLIES = {'telegram': transports.TelegramBot, 'slack': transports.SlackBot} # a reply into the chat that asked +REPLIES = {'telegram': telegram_common.TelegramBot, 'slack': slack_common.SlackBot, + 'discord': discord_common.DiscordBot} -def reply_to(service, chat, message=None): - """The destination of a reply into a chat (to `message`, the one being answered, if given), or None for a network - that takes none.""" +def reply_to(service, chat, message=None, interaction='', pressed=False): + """The destination of a reply into a chat: to `message`, the one being answered, if given; through `interaction` + where the network answers one (Discord's slash command, or with pressed, its button press). None for a network that + takes no replies.""" kind = REPLIES.get(service) - return kind(chat, message) if kind else None + return kind(chat, message, interaction, pressed) if kind else None + + +def chat(key, scheduled=False): + """The destination of a chat by its key ('telegram:-1001234', Transport.key), as its network's bot; scheduled: for a + scheduled run, whose reports keep their record of what they told it.""" + network, _, where = key.partition(':') + dest = reply_to(network, where) + if dest is None: + raise ValueError(f'no chat {key!r}: no such network') + dest.scheduled = scheduled + return dest + + +def interactive(dest): + """Whether a report answers a command (dest, the chat that asked), not a scheduled run (dest a job's chat, or None + for the home chats): a scheduled run tells each chat only what it has not told it before.""" + return dest is not None and not dest.scheduled def default_chats(): - """Where a scheduled report goes, in this order: Telegram's telegramChatID; Slack's slackChannelID as the bot, with - slackBotToken (so that charts go too), else its incoming webhook; Discord's webhook. Raises RuntimeError when there - is nowhere to post.""" + """Where a scheduled report goes: each network's home chat (telegramChatID, slackChannelID, discordChannelID), as + its bot. Raises RuntimeError when there is nowhere to post.""" env = config.env chats = [] - if env.telegramChatID and 'telegram' in env.webhooks: - chats.append(transports.TelegramBot(env.telegramChatID)) - if env.slackChannelID and env.slackBotToken: - chats.append(transports.SlackBot(env.slackChannelID)) - elif 'slack' in env.webhooks: - chats.append(transports.SlackWebhook(env.webhooks['slack'])) - if 'discord' in env.webhooks: - chats.append(transports.DiscordWebhook(env.webhooks['discord'])) + for chat, token, kind in ((env.telegramChatID, env.telegramBotToken, telegram_common.TelegramBot), + (env.slackChannelID, env.slackBotToken, slack_common.SlackBot), + (env.discordChannelID, env.discordBotToken, discord_common.DiscordBot)): + if chat and token: + chats.append(kind(chat)) + chats[-1].scheduled = True if not chats: - raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID, slack_webhook or' - ' discord_webhook') + raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID or discordChannelID,' + ' each with its bot token') return chats @@ -69,7 +88,7 @@ def deliver(dest, report): return [] if answer: return [answer] - print("chart not sent, sending the report as text", file=sys.stderr) + log.warn("chart not sent, sending the report as text") return dest.text(caption.split('\n')) @@ -91,37 +110,15 @@ def report_error(e, dest=None, context=None): for the same escapes). Never raises: error reporting must not become a second failure.""" try: line = error_line(e, context) - print(line, file=sys.stderr) + log.error(line) if isinstance(e, BaseException): trace = ''.join(traceback.format_exception(e)) - print(net.redact(trace), end='', file=sys.stderr) # a request's error carries its URL + log.error(net.redact(trace), end='') # a request's error carries its URL for chat in targets(dest): try: chat.text([line if chat.dialect == 'discord' else html.escape(line, quote=False)]) except Exception as send_error: - print("Failed to report error to", chat.name, ":", send_error, file=sys.stderr) + log.error("Failed to report error to", chat.name, ":", send_error) except Exception as reporting_error: - print("Failed to report error:", reporting_error, file=sys.stderr) - + log.error("Failed to report error:", reporting_error) -def guarded(context, func, *args, exit=True, **kwargs): - """Run a scheduled entry point; on failure report one line to the default channel, then exit non-zero (exit=True, - for the command line) or just return None (exit=False, when the bot runs it in-process). A SystemExit inside counts - as a failure.""" - try: - return func(*args, **kwargs) - except (Exception, SystemExit) as e: - report_error(e, context=context) - if exit: - sys.exit(1) - - -def cli(script, job, handler, argv=None): - """Command line entry point of a report script: parse its arguments with `job(argv) -> (label, kwargs)`, print the - usage on a bad argument, else run the handler guarded.""" - try: - label, kwargs = job(sys.argv[1:] if argv is None else argv) - except ValueError as e: - print("Usage:", script, e, file=sys.stderr) - sys.exit(1) - guarded(label, handler, **kwargs) diff --git a/lib/discord_common.py b/lib/discord_common.py new file mode 100644 index 00000000..c249a6f6 --- /dev/null +++ b/lib/discord_common.py @@ -0,0 +1,294 @@ +"""Discord, whichever way its updates arrive (the gateway, discord_websocket; or the interactions endpoint, +discord_webhook): its REST API as the bot (discordBotToken), with calls that wait out Discord's rate limits and the +slash commands registered as finbot starts; the reader of its interactions and messages (Discord, READER); and the bot +that posts into its channels and DMs (DiscordBot).""" +import json +import re +import time + +import requests + +from lib import log +from lib import config +from lib import dialects +from lib import inbound +from lib import net +from lib import transports + + +API = 'https://discord.com/api/v10' +TRIES = 3 # a call that Discord rate-limits is tried this many times at most +LONGEST_WAIT = 10 # seconds: a rate limit asking for longer is waited this long +SUPPRESS_EMBEDS = 4 # message flag: no link previews +NO_MENTIONS = {'parse': []} # text that looks like @everyone or a mention pings nobody +GUILD, BOT_DM = 0, 1 # where a slash command may be used: in servers, and in a DM with the bot +OPTION = 'arguments' # the name of a slash command's one option: its arguments as text +USER_AGENT = 'DiscordBot (https://github.com/robdevops/finbot, {})' # the form Discord asks of every request's agent +version = 'unknown' # finbot's commit, for USER_AGENT: bot.py sets it as it starts + + +def call(method, path, **kwargs): + """A REST call as the bot: Discord's answer as JSON, or None when it is empty. A 429 waits as long as Discord asks + (LONGEST_WAIT at most) and tries again. Raises RuntimeError with Discord's reason when it refuses, or a requests + error.""" + headers = {'Authorization': 'Bot ' + config.env.discordBotToken, 'User-Agent': USER_AGENT.format(version), + **kwargs.pop('headers', {})} + for attempt in range(1, TRIES + 1): + r = net.request(method, API + path, headers=headers, **kwargs) + if r.status_code != 429 or attempt == TRIES: + break + time.sleep(min(retry_after(r), LONGEST_WAIT)) + if r.status_code >= 400: + raise RuntimeError(f"Discord {method} {path}: {r.status_code} {reason(r)}") + try: + return r.json() + except ValueError: # empty: 204 No Content + return None + + +def try_call(method, path, **kwargs): + """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" + try: + return call(method, path, **kwargs) + except Exception as e: + log.warn(f"Discord {method} failed:", net.redact(e)) + + +def retry_after(response): + """The seconds a rate-limited answer asks to wait (its retry_after), or one when it does not say.""" + try: + return float(response.json().get('retry_after')) + except (ValueError, TypeError, AttributeError): + return 1.0 + + +def reason(response): + """Discord's reason for a refusal: its message and code, else the start of the body.""" + try: + answer = response.json() + return f"{answer['message']} ({answer.get('code')})" + except (ValueError, TypeError, KeyError): + return response.text[:200] + + +def command(name, description, arguments, completes=False): + """A slash command as Discord takes it: a chat command usable in servers and in a DM with the bot, and the arguments + it takes, if any, as one optional text option that they describe; with completes, Discord asks finbot what to + suggest as they are typed (an autocomplete interaction).""" + option = {'type': 3, 'name': OPTION, 'description': arguments[:100], 'required': False, 'autocomplete': completes} + return {'name': name, 'type': 1, 'description': description[:100], 'contexts': [GUILD, BOT_DM], + 'integration_types': [0], 'options': [option] if arguments else []} + + +def summary(commands): + """What of a list of slash commands Discord may differ on from ours, in a form to compare: an answer from Discord + carries ids and versions besides.""" + return sorted((c['name'], c.get('description'), sorted(c.get('contexts') or []), + [(o['name'], o.get('description'), o.get('required', False), o.get('autocomplete', False)) + for o in c.get('options') or []]) for c in commands) + + +def register(commands): + """Called once by bot.py as it starts, in both modes: make the application's slash commands these ([(name, + description, arguments)], from worker.slash_commands), replacing them all only when they differ (Discord limits how + many may be made in a day). Returns the bot's user id: a message that mentions it is for the bot. Raises + RuntimeError, or a requests error.""" + application = call('GET', '/applications/@me')['id'] + wanted = [command(*entry) for entry in commands] + path = f'/applications/{application}/commands' + if summary(call('GET', path) or []) != summary(wanted): + call('PUT', path, json=wanted) + log.info(f"discord: {len(wanted)} slash commands registered") + return str(call('GET', '/users/@me')['id']) + + +class Discord(inbound.Network): + """Discord's interactions (a slash command, a button pressed, the endpoint's check), which read the same from the + interactions endpoint and the gateway, and the gateway's messages.""" + service = 'discord' + bot_id = None # the bot's user id, from register() as bot.py starts: a message mentioning it is for the bot + PING, COMMAND, COMPONENT, AUTOCOMPLETE = 1, 2, 3, 4 # interaction types + PONG, THINKING, LATER, CHOICES = 1, 5, 6, 8 # answers: to the check, "thinking…" until the reply, nothing to see (a + # press), the suggestions for what is being typed + + def update_id(self, update): + """The interaction's id, or the message's.""" + return update.get('id') + + def parse(self, update): + """An interaction (it has a token) or a gateway message.""" + return self.interaction(update) if 'token' in update else self.message(update) + + @property + def bot_name(self): + """The bot as a message names it, @ and its user id (Discord's <@id> mention, read as Slack's are).""" + return f'@{self.bot_id}' if self.bot_id else None + + def interaction(self, update): + """The endpoint's check (a PING, answered at once), a slash command as the message it stands for (/price 7d), + its arguments as they are typed (a Suggest), or a button pressed under one of the bot's messages; anything else, + UNSUPPORTED.""" + kind = update.get('type') + if kind == self.PING: + return inbound.Response(body=json.dumps({'type': self.PONG}).encode(), interaction=True) + person = (update.get('member') or {}).get('user') or update.get('user') or {} + channel = str(update.get('channel_id') or (update.get('channel') or {}).get('id') or '') + through = f"{update.get('application_id')}/{update['token']}" + data = update.get('data') or {} + if kind == self.COMMAND and data.get('name'): + given = ' '.join(str(option.get('value', '')) for option in data.get('options') or []).strip() + text = f"/{data['name']} {given}".strip() + return inbound.Message(self.service, channel, f"<@{person.get('id')}>", text, self.bot_name, + self.name(person), interaction=through) + if kind == self.AUTOCOMPLETE and data.get('name'): + options = data.get('options') or [] + typed = next((str(option.get('value', '')) for option in options if option.get('focused')), '') + return inbound.Suggest(self.service, channel, f"/{data['name']} {typed}", through) + message = update.get('message') or {} + if kind == self.COMPONENT and data.get('custom_id') and message.get('id'): + return inbound.Press(self.service, str(update.get('id')), channel, str(message['id']), data['custom_id'], + '@' + str(person.get('username') or person.get('id') or 'unknown'), through) + return inbound.UNSUPPORTED + + def message(self, update): + """A message a person wrote (MESSAGE_CREATE); a mention <@id> reads as @id, as Slack's do. One by a bot (this + one's replies included) or a webhook is OK: answering a bot could loop.""" + author = update.get('author') or {} + if author.get('bot') or update.get('webhook_id') or not author.get('id') or 'content' not in update: + return inbound.OK + text = re.sub(r'<@!?(\d+)>', r'@\1', update['content']) + return inbound.Message(self.service, str(update.get('channel_id')), f"<@{author['id']}>", text, self.bot_name, + self.name(author), str(update.get('id', ''))) + + @staticmethod + def name(person): + """What to call someone: their display name, else their username.""" + return person.get('global_name') or person.get('username') or 'unknown' + + def acknowledge(self, event): + """An interaction's answer, which Discord must have within 3 seconds and before any reply: "thinking…" for a + slash command, until the reply replaces it; nothing to see for a press, whose message is edited after. A + gateway message needs none.""" + if not event.interaction: + return None + answer = self.THINKING if isinstance(event, inbound.Message) else self.LATER + return inbound.Response(body=json.dumps({'type': answer}).encode(), interaction=True) + + def suggestions(self, choices): + """The answer to an autocomplete interaction: the choices, (name shown, value) pairs, 25 at most.""" + choices = [{'name': name, 'value': value} for name, value in choices[:25]] + return inbound.Response(body=json.dumps({'type': self.CHOICES, 'data': {'choices': choices}}).encode(), + interaction=True) + + +class DiscordBot(transports.Transport): + """A Discord channel or DM, as the bot (discordBotToken): no link previews, no pings, charts attached, buttons as + components, edits in place, the typing notice. A slash command's first message replaces Discord's "thinking…" and + the rest follow it up, through its interaction; a pressed message is edited through the press's.""" + name = 'discord' + dialect = dialects.DISCORD + + def __init__(self, where, message=None, interaction='', pressed=False): + """As Transport; a press has no "thinking…" to replace, so its new messages go into the channel.""" + super().__init__(where, message, interaction, pressed) + self.answered = pressed # the slash command's "thinking…" replaced + + @property + def shows_typing(self): + """The typing notice for a command in a message; a slash command shows Discord's "thinking…" instead.""" + return not self.interaction + + def typing(self): + """POST /typing: "finbot is typing…" for ten seconds, or until it posts.""" + try_call('POST', f'/channels/{self.where}/typing') + + def typing_done(self): + """A slash command over with nothing said: its "thinking…" deleted, or it would show for 15 minutes.""" + if self.interaction and not self.answered: + self.answered = True + try_call('DELETE', f'/webhooks/{self.interaction}/messages/@original') + + @staticmethod + def extras(buttons): + """What a message carries besides its text: no link previews, no pings, and its buttons as components.""" + extra = {'flags': SUPPRESS_EMBEDS, 'allowed_mentions': NO_MENTIONS} + if buttons and not isinstance(buttons, dict): + extra['components'] = components(buttons) + return extra + + def post_text(self, text, buttons=None): + """Post one message (see post).""" + return self.post({'content': text, **self.extras(buttons)}) + + def post_chart(self, image, caption, buttons=None): + """Post a chart, attached to the report as its message (see post).""" + attached = {'content': caption, 'attachments': [{'id': 0, 'filename': 'chart.png'}], **self.extras(buttons)} + return self.post(attached, image) + + def post(self, payload, image=None): + """Post one message: a slash command's first replaces its "thinking…" and the rest follow it up; otherwise it + goes into the channel. Discord's message; MAYBE_SENT when a chart's post timed out; None when it failed + (logged).""" + if self.interaction and not self.answered: + method, path = 'PATCH', f'/webhooks/{self.interaction}/messages/@original' + elif self.interaction and not self.pressed: + method, path = 'POST', f'/webhooks/{self.interaction}' + else: + method, path = 'POST', f'/channels/{self.where}/messages' + try: + answer = call(method, path, **message_body(payload, image)) + except requests.exceptions.ReadTimeout as e: + self.answered = True # it may have arrived + log.warn("Discord post timed out, it may still arrive:", net.redact(e)) + return transports.MAYBE_SENT if image is not None else None + except Exception as e: + log.error("Discord post to", self.where, "failed:", net.redact(e)) + return None + self.answered = True + log.info("OK discord message to", self.where) + return answer or {} + + def message_path(self, message): + """Where a message is edited: through the press's interaction where there is one (a message that answered a + slash command can be edited only so), else in the channel as the bot.""" + if self.interaction: + return f'/webhooks/{self.interaction}/messages/@original' + return f'/channels/{self.where}/messages/{message}' + + def edit_chart(self, message, image, caption, buttons=None): + """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Discord refuses.""" + if hasattr(image, 'seek'): + image.seek(0) + payload = {'content': transports.fit_caption(caption, self.dialect), + 'attachments': [{'id': 0, 'filename': 'chart.png'}], 'components': components(buttons or [])} + call('PATCH', self.message_path(message), **message_body(payload, image)) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + payload = {'content': text} + if buttons is not None: + payload['components'] = components(buttons) + try_call('PATCH', self.message_path(message), json=payload) + + +DISCORD_ROW = 5 # buttons in a row of components, and rows in a message: Discord's limits + + +def components(buttons): + """Rows of buttons.Button as Discord's action rows: DISCORD_ROW buttons to a row and as many rows at most (the + buttons past that are left out), the one shown now blurple and the rest grey.""" + flat = [button for row in buttons for button in row][:DISCORD_ROW * DISCORD_ROW] + return [{'type': 1, 'components': [{'type': 2, 'style': 1 if button.active else 2, 'label': button.label[:80], + 'custom_id': button.data} for button in flat[i:i + DISCORD_ROW]]} for i in range(0, len(flat), DISCORD_ROW)] + + +def message_body(payload, image=None): + """A Discord message's request arguments: its JSON; with a chart, a form of the JSON and the PNG (bytes, so that a + rate-limited call can send it again).""" + if image is None: + return {'json': payload} + data = image.read() if hasattr(image, 'read') else bytes(image) + return {'data': {'payload_json': json.dumps(payload)}, 'files': {'files[0]': ('chart.png', data, 'image/png')}} + + +READER = Discord() diff --git a/lib/discord_webhook.py b/lib/discord_webhook.py new file mode 100644 index 00000000..98c13362 --- /dev/null +++ b/lib/discord_webhook.py @@ -0,0 +1,37 @@ +"""Discord's interactions endpoint, its HTTP mode (discordOutgoingWebhook set): Discord posts each interaction to +finbot's URL, signed with the app's Ed25519 key, for the reader (discord_common.READER), and takes the answer as the +HTTP response.""" +from urllib.parse import urlparse + +from lib import config +from lib import discord_common +from lib import inbound + + +class DiscordInteractions(inbound.Webhook): + """Discord posting interactions to discordOutgoingWebhook (the app's Interactions Endpoint URL), each signed with + the app's Ed25519 key, whose public half is discordPublicKey.""" + REFUSED = inbound.UNAUTHORIZED # Discord checks that a bad signature gets a 401, and drops a URL answering otherwise + + def path(self): + """The path of discordOutgoingWebhook.""" + return urlparse(config.env.discordOutgoingWebhook).path if config.env.discordOutgoingWebhook else '' + + def refuse(self, environ, body, update): + """The request must carry Discord's signature of its timestamp and body under discordPublicKey, checked with + PyNaCl (imported here only: Discord's http mode is the one thing that needs it).""" + signature = environ.get('HTTP_X_SIGNATURE_ED25519', '') + timestamp = environ.get('HTTP_X_SIGNATURE_TIMESTAMP', '') + if not signature or not timestamp: + return "Discord signature not present" + from nacl.exceptions import BadSignatureError + from nacl.signing import VerifyKey + try: + VerifyKey(bytes.fromhex(config.env.discordPublicKey)).verify(timestamp.encode() + body, + bytes.fromhex(signature)) + except (BadSignatureError, ValueError): + return "Discord signature incorrect" + return None + + +WEBHOOK = DiscordInteractions(discord_common.READER) diff --git a/lib/discord_websocket.py b/lib/discord_websocket.py new file mode 100644 index 00000000..fb6a6776 --- /dev/null +++ b/lib/discord_websocket.py @@ -0,0 +1,79 @@ +"""Discord's gateway, its pull mode (no discordOutgoingWebhook), through discord.py: the raw events taken as the +interactions endpoint's, for the reader (discord_common.READER), and an interaction's answer posted to its callback.""" +import asyncio +import json +import threading + +from lib import config +from lib import discord_common +from lib import pull + + +class DiscordGateway(pull.Listener): + """Discord's gateway, through discord.py: imported here only, as Discord's pull mode is the one thing that needs it + (Debian's python3-discord). Its client keeps the connection, reconnecting by itself; finbot takes the raw + MESSAGE_CREATE and INTERACTION_CREATE events, which read as the interactions endpoint's, each in a thread of its + own, off the client's event loop. The gateway takes no answers: an interaction's goes to its callback over REST.""" + network = discord_common.READER + TAKEN = ('MESSAGE_CREATE', 'INTERACTION_CREATE') + + @staticmethod + def wanted(): + """Whether finbot.env asks for it: Discord's bot token, and no discordOutgoingWebhook.""" + return bool(config.env.discordBotToken and not config.env.discordOutgoingWebhook) + + def describe(self): + """The mode, for the log.""" + return 'gateway (no discordOutgoingWebhook)' + + def poll(self, take): + """Run discord.py's client until it stops, which it does only on a failure it does not get over (a token Discord + refuses, or an intent the app is not allowed): then the back-off.""" + try: + asyncio.run(self.run_client(take)) + except Exception as e: + return self.failed('the gateway', e) + return self.failed('the gateway', 'the client stopped') + + async def run_client(self, take): + """Start the client with the bot token, and close it however it ends.""" + client = self.client(take) + async with client: + await client.start(config.env.discordBotToken) + + def client(self, take): + """discord.py's client, with the intents for messages in servers and DMs, and their text (Message Content, which + the app must be allowed in the Developer Portal); its raw events go to receive().""" + import discord as library + library.VoiceClient.warn_nacl = False # finbot has no voice: no warning that PyNaCl is missing + intents = library.Intents.default() + intents.message_content = True + client = library.Client(intents=intents, enable_debug_events=True) + + @client.event + async def on_socket_raw_receive(raw): + """A gateway message, as it arrived.""" + self.receive(raw, take) + + @client.event + async def on_ready(): + """Connected: the failures in a row are over.""" + self.failures = 0 + return client + + def receive(self, raw, take): + """A raw gateway message: an event finbot takes goes to hand_on in a thread of its own, so that the client's + event loop never waits on finbot; the rest is the client's business.""" + try: + message = json.loads(raw) + except (TypeError, ValueError): + return + if message.get('op') == 0 and message.get('t') in self.TAKEN and isinstance(message.get('d'), dict): + threading.Thread(target=self.hand_on, args=(take, message['d']), daemon=True).start() + + def answer(self, update, answer): + """Post an interaction's answer to its callback, then start the work waiting on it, if any.""" + path = f"/interactions/{update['id']}/{update['token']}/callback" + discord_common.call('POST', path, json=json.loads(answer.body)) + if answer.then: + answer.then() diff --git a/lib/formats.py b/lib/formats.py index bbb90c62..5394e597 100644 --- a/lib/formats.py +++ b/lib/formats.py @@ -1,6 +1,9 @@ """Text for people: company names trimmed, numbers in units, dates and spans in words, signed percents, ordinals.""" import datetime import re +import zoneinfo + +from lib import config TITLE_TRIMS = [ # transform_title: (text, replacement), in order, as each sees what the ones before it made (' - ', ' '), ('Roundhill ETF Trust ', ''), ('Class A Common Stock', ''), @@ -36,6 +39,20 @@ (' Bull', ' '), (' .', ' '), (' ,', ' ') ] +TRUST_WORDS = ('ETF', 'ETFs', 'Series', 'Trust', 'Fund', 'Funds', 'OFC', 'Exchange-Traded', 'Portfolios') # a trust's + +def fund_name(title): + """A fund's own name, without its trust's before it: up to its brand again, where the trust's words come first + ('Global X ETF Series OFC Global X China EV ETF': 'Global X China EV ETF'); an earlier ETF, with the trust's words + after it ('Betashares ETF Trust Global Cybersecurity ETF': 'Betashares Global Cybersecurity ETF'); OFC (a fund + company) anywhere. A company's name ('Johnson & Johnson') stays as it is.""" + words = title.split() + again = next((i for i in range(1, len(words) - 1) + if words[i] == words[0] and any(word in TRUST_WORDS for word in words[1:i])), 0) + title = ' '.join(word for word in words[again:] if word != 'OFC') + trust = '|'.join(re.escape(word) for word in TRUST_WORDS) + return re.sub(rf'\bETFs? (?:(?:{trust}) )*(?=.*\bETF\b)', '', title) + def transform_title(title): """A company or fund name as the reports show it: shorter, without fund-trust boilerplate and legal suffixes.""" title = title.replace(' FPO', '') @@ -52,7 +69,7 @@ def transform_title(title): title = title.title() for suffix in (' &', ' and'): title = title.strip().removesuffix(suffix) - return ' '.join(title.replace(' Usd', ' USD').replace('Ishares', 'iShares').split()) + return fund_name(' '.join(title.replace(' Usd', ' USD').replace('Ishares', 'iShares').split())) def human_units(value, decimal_places=2): """1234567 -> '1.23 M'.""" @@ -71,8 +88,22 @@ def sections(*blocks): lines += block or [] return lines +NAME_LIMIT = 30 # characters of a name in a report's list: a longer line wraps on a phone + +def short_name(name): + """A name for a report's list, cut at a word to at most NAME_LIMIT characters, an ellipsis last: 'Global X Asia + Semiconductor ETF' -> 'Global X Asia Semiconductor…'. The name stays whole everywhere else.""" + if len(name) <= NAME_LIMIT: + return name + cut = name[:NAME_LIMIT - 1] + if ' ' in cut[NAME_LIMIT // 2:]: # a word break past halfway: cut there rather than mid-word + cut = cut[:cut.rindex(' ')] + return cut.rstrip(' ,&-') + '…' + def security_line(flag, title, link): - """'🇦🇺 Title (link)': how a report line names a security (without a leading space where its market has no flag).""" + """'🇦🇺 Title (link)': how a report line names a security (without a leading space where its market has no flag), + the name cut to fit a phone (short_name).""" + title = short_name(title) return f"{flag} {title} ({link})" if flag else f"{title} ({link})" def strip_url(url): @@ -149,6 +180,22 @@ def days_from_human_days(arg): pass return int(arg.removesuffix('Y')) * 365 +def seconds_from_human(arg): + """180s, 24h, or any period that days_from_human_days takes (7d, 2w, 3m, 1y, ytd, 7) -> seconds. Raises ValueError + for anything else.""" + for suffix, seconds in (('s', 1), ('h', 3600)): + if arg.lower().endswith(suffix) and arg[:-1].isdigit(): + return int(arg[:-1]) * seconds + return days_from_human_days(arg) * 86400 + def ordinal(num): """The ordinal suffix of a whole number: 'st' for 1, 'nd' for 22, 'th' for 11 and 13.""" return 'th' if num % 100 in (11, 12, 13) else {1: 'st', 2: 'nd', 3: 'rd'}.get(num % 10, 'th') + + +def zone(): + """The timezone preference as a zone: UTC where it names none.""" + try: + return zoneinfo.ZoneInfo(config.prefs.timezone) + except (ValueError, zoneinfo.ZoneInfoNotFoundError): + return zoneinfo.ZoneInfo('UTC') diff --git a/lib/inbound.py b/lib/inbound.py index 65f7adb5..20147a00 100644 --- a/lib/inbound.py +++ b/lib/inbound.py @@ -1,19 +1,13 @@ -"""What the networks send the bot. Reading an update is apart from how it arrives: a class per network reads its updates -(Telegram, Slack), and a class per way of arriving takes them in (a webhook: the path the network posts to, how it -proves itself, the body), so that another way (a pull mode) can hand its updates to the same reader. An update becomes -a Message to answer, a Press of a button, or a Response the network gets straight back (a challenge, or nothing to -do).""" +"""What the networks send the bot, whichever network and however it arrives. Reading an update is apart from how it +arrives: each network has a reader (a Network: telegram_common, slack_common, discord_common), and each way in hands +its updates to it: a webhook (a Webhook: the path the network posts to, how it proves itself, the body; in +telegram_webhook, slack_webhook, discord_webhook) or a pull mode (lib/pull.py). An update becomes a Message to answer, +a Press of a button, a Suggest to complete, or a Response the network gets straight back (a challenge, or nothing to +do). The reader also acknowledges an update where the network waits for that, however it arrived: Telegram's press +with a call, Discord's interaction with the answer it must have before any reply.""" import dataclasses -import hashlib import hmac -import html import json -import re -import sys -import time -from urllib.parse import parse_qs, urlparse - -from lib import config @dataclasses.dataclass @@ -26,6 +20,7 @@ class Message: bot_name: str # the bot's @name, which can start a command user_name: str # what to call them message_id: str = '' # the message itself: Slack shows ⏳ on it while finbot works + interaction: str = '' # Discord's slash command: its application id and token, to answer through @dataclasses.dataclass @@ -37,13 +32,29 @@ class Press: message_id: str data: str user: str = '' # who pressed it, for the log + interaction: str = '' # Discord's press: its application id and token, to edit the message pressed through + + +@dataclasses.dataclass +class Suggest: + """A slash command's arguments as they are typed (Discord's autocomplete), to answer at once with what to suggest: + the message they make so far (/price 5 7), which the suggestions complete.""" + service: str + chat_id: str + text: str + interaction: str = '' # Discord's interaction: its application id and token @dataclasses.dataclass class Response: - """What the network gets back at once: a status and a body.""" + """What the network gets back at once: a status and a body; and the work to start once it has them (`then`), when + the network must have this answer before any reply (Discord's deferred answer to an interaction). An interaction's + answer (`interaction`) is the HTTP response on Discord's interactions endpoint, and goes to the interaction's + callback when it came over the gateway.""" status: str = '200 OK' body: bytes = b'' + then: object = dataclasses.field(default=None, compare=False) + interaction: bool = False OK = Response() @@ -51,13 +62,11 @@ class Response: UNHANDLED = Response(body=b'

Unhandled

') # a kind it might, which this code does not know BAD_REQUEST = Response('400 Bad Request', b'

Bad Request

') FORBIDDEN = Response('403 Forbidden', b'

Unauthorized

') +UNAUTHORIZED = Response('401 Unauthorized', b'

Unauthorized

') # what Discord expects for a bad signature NOT_FOUND = Response('404 Not Found', b'

404

') TOO_LARGE = Response('413 Payload Too Large', b'

Payload Too Large

') -SIGNATURE_AGE = 300 # seconds: Slack's limit; an older signed request may be a replay - - def same_secret(given, expected): """Whether a secret sent with a request is the one configured, compared in constant time. An empty configured secret proves nothing, so it matches nothing.""" @@ -76,106 +85,17 @@ def parse(self, update): """What the update (a dict) says: a Message, a Press, or a Response to give straight back.""" raise NotImplementedError - -class Telegram(Network): - """Telegram's updates: messages and button presses.""" - service = 'telegram' - bot_name = None # the bot's @username, from telegram.register() as bot.py starts - IGNORED = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', - 'venue', 'poll', 'dice'} # nothing to answer, nothing worth logging - - def update_id(self, update): - """Telegram's update_id.""" - return update.get('update_id') - - def parse(self, update): - """A message (its text, or a photo's caption) or a button press. A first name is escaped as it arrives: - replies are Telegram HTML, and a name is text, not markup.""" - if 'callback_query' in update: # chart period and job buttons - press = update['callback_query'] - message = press.get('message') - if not message or not press.get('data'): - return UNSUPPORTED - sender = press.get('from', {}) - who = '@' + sender['username'] if 'username' in sender else sender.get('first_name', 'unknown') - return Press(self.service, press['id'], str(message['chat']['id']), str(message['message_id']), - press['data'], who) - if 'message' not in update: - return UNSUPPORTED - message = update['message'] - sender = message['from'] - chat_id = str(message['chat']['id']) - first_name = html.escape(sender['first_name'], quote=False) - if 'username' in sender: - user = '@' + sender['username'] - user_name = first_name or user # the handle when the first name is blank - else: - user = user_name = '@' + first_name - if 'text' in message: - text = message['text'] - elif 'photo' in message: - text = message.get('caption', '') - elif self.IGNORED & message.keys(): - return OK - else: - print("[telegram]: unhandled: 'message' without 'text/photo'", file=sys.stderr) - return UNHANDLED - return Message(self.service, chat_id, user, text, self.bot_name, user_name, str(message.get('message_id', ''))) - - -class Slack(Network): - """Slack's Events API (messages) and its interactivity (a button pressed).""" - service = 'slack' - BY_PEOPLE = (None, 'thread_broadcast') # message subtypes a person writes; the rest are bot posts, edits, joins... - - def update_id(self, update): - """Slack's event_id (a button press has none).""" - return update.get('event_id') - - def parse(self, update): - """A message a person wrote, a button pressed, or the challenge that checks the URL. Links and mentions lose - their markup ( becomes dub.ax, <@QWERTY> @QWERTY), and Slack's escapes their meaning - (& becomes &).""" - kind = update.get('type') - if kind == 'url_verification': - response = json.dumps({'challenge': update['challenge']}) - print("replying with", response, file=sys.stderr) - return Response(body=response.encode()) - if kind == 'block_actions': - return self.press(update) - if kind != 'event_callback': - print(f"[slack]: unhandled 'type' {kind!r}", file=sys.stderr) - return UNHANDLED - event = update['event'] - bot_user = update['authorizations'][0]['user_id'] # QWERTY - if event.get('bot_id') or event.get('subtype') not in self.BY_PEOPLE or not event.get('user') or \ - event['user'] == bot_user: - return OK # not by a person: answering a bot (this one's replies and webhook reports included) could loop - if event['type'] not in ('message', 'app_mention') or 'text' not in event: - print("[slack]: unhandled event callback type", event['type'], file=sys.stderr) - return UNHANDLED - text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), event['text']) - text = html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > - user = '<@' + event['user'] + '>' # a mention: Slack shows the name - return Message(self.service, event['channel'], user, text, '@' + bot_user, user, event.get('ts', '')) - - def press(self, update): - """A button pressed under one of the bot's messages (a chart's period buttons), or UNSUPPORTED for one that is - not under a message or carries no value.""" - container = update.get('container') or {} - actions = update.get('actions') or [{}] - value = actions[0].get('value') if isinstance(actions[0], dict) else None - if container.get('type') != 'message' or not container.get('message_ts') or not value: - return UNSUPPORTED - user = update.get('user') or {} - who = '@' + str(user.get('username') or user.get('id') or 'unknown') - return Press(self.service, update.get('trigger_id', ''), container.get('channel_id', ''), - container['message_ts'], value, who) + def acknowledge(self, event): + """Tell the network an update was taken (a Message or a Press), where it waits for that: None when the answer to + the update says nothing more, or the Response the network must have before any reply. Here None: taking the + update is the acknowledgement (Slack's is the 200, or the socket's ack).""" + return None class Webhook: """Updates a network posts to the bot, for its reader (a Network); subclasses give the path it posts to and how it proves itself.""" + REFUSED = FORBIDDEN # the answer to a request that does not prove itself def __init__(self, network): """network: the reader of its updates.""" self.network = network @@ -199,67 +119,3 @@ def decode(self, body): def refuse(self, environ, body, update): """Why to refuse the request once read (the body's signature or a secret in it missing or wrong), else None.""" return None - - -class TelegramWebhook(Webhook): - """Telegram posting to telegramOutgoingWebhook, each update with the secret header that setWebhook gave it.""" - - def path(self): - """The path of telegramOutgoingWebhook.""" - return urlparse(config.env.telegramOutgoingWebhook).path if config.env.telegramOutgoingWebhook else '' - - def refuse_unread(self, environ): - """The header must carry telegramOutgoingToken.""" - secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') - if not same_secret(secret, config.env.telegramOutgoingToken): - return f"Telegram authorisation header {'not present' if secret is None else 'is present but incorrect'}" - return None - - -class SlackWebhook(Webhook): - """Slack posting events and button presses to slackOutgoingWebhook (the app's Events and Interactivity URL), each - request signed with the app's signing secret, and carrying its verification token.""" - - def path(self): - """The path of slackOutgoingWebhook.""" - return urlparse(config.env.slackOutgoingWebhook).path if config.env.slackOutgoingWebhook else '' - - def refuse(self, environ, body, update): - """With slackSigningSecret, the request must carry Slack's signature of its body, made in the last five minutes; - without it, the body must carry slackOutgoingToken (the older way, which Slack has deprecated).""" - if config.env.slackSigningSecret: - return self.signature_problem(environ, body, config.env.slackSigningSecret) - if 'token' not in update: - return "Slack authorisation field not present" - if not same_secret(str(update['token']), config.env.slackOutgoingToken): - return "Slack auth incorrect" - return None - - @staticmethod - def signature_problem(environ, body, secret): - """Why the request's signature fails, or None: X-Slack-Signature must be v0= and the HMAC-SHA256, under the - signing secret, of v0::, and the timestamp within five minutes of now.""" - timestamp = environ.get('HTTP_X_SLACK_REQUEST_TIMESTAMP', '') - signature = environ.get('HTTP_X_SLACK_SIGNATURE') - if signature is None or not (timestamp.isascii() and timestamp.isdigit()): - return "Slack signature not present" - if abs(time.time() - int(timestamp)) > SIGNATURE_AGE: - return "Slack signature too old (a replay, or a clock is wrong)" - signed = b'v0:' + timestamp.encode() + b':' + body - expected = 'v0=' + hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest() - return None if same_secret(signature, expected) else "Slack signature incorrect" - - def decode(self, body): - """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON.""" - if body.startswith(b'payload='): - body = parse_qs(body.decode('utf-8', 'replace')).get('payload', [''])[0].encode() - return super().decode(body) - - -TELEGRAM, SLACK = Telegram(), Slack() # the readers -WEBHOOKS = (TelegramWebhook(TELEGRAM), SlackWebhook(SLACK)) - - -def route(path): - """The webhook whose path this is, or None.""" - return next((hook for hook in WEBHOOKS if hook.path() and path == hook.path()), None) diff --git a/lib/jobrun.py b/lib/jobrun.py deleted file mode 100644 index adcea41f..00000000 --- a/lib/jobrun.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Runs one scheduled command in its own process: `python -m lib.jobrun lib.reports. ''`. - -The bot starts this for every scheduled or manual run, so a job's CPU-heavy work (charts, pandas) happens off the -bot's event loop and cannot hold up incoming webhooks. Failures are reported to the channel and give the exit status -REPORTED.""" -import signal -if __name__ == '__main__': - # a reload is for the bot, but `systemctl kill -s HUP` signals every process of the unit - signal.signal(signal.SIGHUP, signal.SIG_IGN) - # likewise a flush (systemctl kill -s USR1); its default action would kill the job - signal.signal(signal.SIGUSR1, signal.SIG_IGN) -import importlib, json, re, sys - -from lib import delivery - -# exit status of a run that failed and said so in the chat; the bot reports any other non-zero status itself -REPORTED = 3 - - -def main(argv): - """Run one scheduled command: argv is the report's module and its JSON keyword arguments. Returns the exit status: 0 - when it ran, REPORTED when it failed and said so in the chat, 2 for bad arguments.""" - if len(argv) != 2 or not re.fullmatch(r'[a-z][a-z_]*(\.[a-z][a-z_]*)*', argv[0]): - print('usage: python -m lib.jobrun ', file=sys.stderr) - return 2 - module = argv[0] - def run(): - """Import the report and run it: an import that fails is reported like any other failure.""" - importlib.import_module(module).run(**json.loads(argv[1])) - return True - return 0 if delivery.guarded(module.rsplit('.', 1)[-1] + '.py', run, exit=False) else REPORTED - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/lib/jobs.py b/lib/jobs.py index e6db402d..fe0b8990 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -1,30 +1,26 @@ -"""The scheduled reports (formerly cron jobs), run by the bot itself. +"""The scheduled reports: each chat's jobs, which its people manage with .jobs (lib/worker.py), kept in SCHEDULE_FILE +and each posting to its own chat; and the system jobs (SYSTEM), fixed here, which fill the cache and clear it out. -The schedule is a file in crontab syntax: five time fields (or an @macro) and then a finbot script with its -arguments, e.g. `*/20 * * * Mon-Fri trades.py` or `28 21 * * Sat cal.py earnings; price.py 7`. Pasted crontab lines -(`./price.py > /dev/null`, `{ a; b ;}`) are accepted too. Running in the bot means the jobs share its user, sandbox, -environment and cache.""" +A job is a time in crontab's syntax (five fields, or an @macro such as @daily), in UTC unless it names its own zone +first (CRON_TZ=America/New_York), then one or more reports with their arguments, separated by ";": +`*/20 * * * Mon-Fri trades`, `28 21 * * Sat cal earnings; price 7`. The bot runs each command itself, in a thread of +its own, sharing its user, sandbox, environment, caches and connections.""" import datetime import hmac -import json import math -import os import re -import signal -import subprocess -import sys +import shlex import threading import time import zlib import zoneinfo from dataclasses import dataclass -from pathlib import Path from croniter import croniter, CroniterBadDateError +from lib import log from lib import config from lib import state -from lib import jobrun from lib import delivery from lib.reports import cal, cleanup, milestone, performance, podcasts, price, rating, reminder, shorts, trades from lib.reports import sharesight_warm, yahoo_warm @@ -32,287 +28,183 @@ SCRIPTS = {'cal': cal, 'cleanup': cleanup, 'milestone': milestone, 'performance': performance, 'podcasts': podcasts, 'price': price, 'rating': rating, 'reminder': reminder, 'shorts': shorts, 'trades': trades, 'sharesight_warm': sharesight_warm, 'yahoo_warm': yahoo_warm} -current = None # the running Scheduler, set by bot.py; the chat commands use it -# scheduled, but not run from chat: .jobs lists them under System -HIDDEN = ('cleanup', 'podcasts', 'sharesight_warm', 'yahoo_warm') -WARM = 'sharesight_warm.py; yahoo_warm.py' # the jobs that fill the cache, run at once after a flush too +# the system jobs, in UTC: the Sharesight portfolios and holdings fetched afresh every 30 minutes (cached 40), the price +# histories of the stocks followed every 8 hours (cached 12), so that no request waits on either; and cache files +# older than 30 days deleted weekly +SYSTEM = (('14,44 * * * *', 'sharesight_warm'), ('7 */8 * * *', 'yahoo_warm'), ('51 18 * * Sat', 'cleanup 30')) +SYSTEM_SCRIPTS = ('sharesight_warm', 'yahoo_warm', 'cleanup') # a chat cannot schedule these +WARM = 'sharesight_warm; yahoo_warm' # the jobs that fill the cache, run at once after a flush too +SCHEDULE_FILE = 'finbot_schedule.json' # {chat key: [{'when', 'reports', 'zone', 'on'}]}: each chat's jobs, in order STATE_FILE = 'finbot_jobs.json' # last fire time of every job, so a restart neither repeats a run nor loses one -JOB_TIMEOUT = 5 * 60 # seconds each command may run before it is stopped and reported -IGNORED_VARIABLES = ('MAILTO', 'SHELL', 'PATH', 'HOME', 'LOGNAME') -# what a shell reads as more than a word, outside quotes -OPERATOR = re.compile(r'\d*>>?&?\d*|&>>?|;|&&|\|\|?|&|[<`$(){}]') -REDIRECT = re.compile(r'\d*>>?&?\d*|&>>?') # > file, >> file, 2>&1, &> file - - -def checksum(label): - """A command's short stand-in in a button (Telegram allows 64 bytes of data): it detects a changed schedule.""" - return format(zlib.crc32(label.encode()) & 0xffff, '04x') - - -def run_child(label, module, kwargs): - """Run a command in its own process (python -m lib.jobrun) and return its exit status. The child inherits the - environment, user and output of the bot, so it behaves like the old cron job. One still running after JOB_TIMEOUT - is stopped and TimeoutError raised; any other interruption stops it too.""" - # the preferences this process runs on, not whatever the file says now - env = dict(os.environ, **config.child_environment()) - args = [sys.executable, '-m', 'lib.jobrun', module.__name__, json.dumps(kwargs)] - proc = subprocess.Popen(args, cwd=config.REPO_DIR, env=env) - try: - return proc.wait(timeout=JOB_TIMEOUT) - except subprocess.TimeoutExpired: - stop(proc) - raise TimeoutError(f'still running after {JOB_TIMEOUT // 60} minutes') from None - except BaseException: - stop(proc) - raise - - -def stop(proc): - """End a child process: asked to stop, then killed if it is still there 10 seconds later.""" - proc.terminate() - try: - proc.wait(timeout=10) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() +MAX_JOBS = 30 # a chat's jobs, at most +JOB_TIMEOUT = 5 * 60 # seconds a command may run before it is reported as late, once (it runs on, see execute) +UTC = zoneinfo.ZoneInfo('UTC') +current = None # the running Scheduler, set by bot.py; the chat commands use it -def start_thread(target, *args): - """Run target(*args) in a daemon thread of its own (a greenlet, under gevent's monkey patching).""" - threading.Thread(target=target, args=args, daemon=True).start() +class ScheduleError(ValueError): + """A job that cannot be: said to the chat that asked for it.""" -def warm_now(scheduler=None): - """Run the warm jobs (WARM) now, one after the other, each in its own child process: through the scheduler when it - runs, so that one already running on its schedule is not started twice; else directly, a failure reported in the - chat as the scheduler reports one.""" - commands = parse_command(WARM) - print('cache: filling it again:', ', '.join(label for label, _, _ in commands), file=sys.stderr) - if scheduler: - scheduler.execute(commands) - else: - for label, module, kwargs in commands: - try: - run_child(label, module, kwargs) - except Exception as e: - delivery.report_error(e, context=label) - print('cache: warm jobs done', file=sys.stderr) +class ImpossibleTime(ScheduleError): + """A time that cannot be (minute 61) or never comes (30 February).""" -def exit_reason(status): - """Why a job process ended, when it did not report it itself: killed by a signal, or failed before it could say.""" - if status < 0: - try: - return f'killed by {signal.Signals(-status).name}' - except ValueError: - return f'killed by signal {-status}' - return f'exited with status {status} without saying why' +@dataclass +class Job: + """A job: its time, in its zone; the commands it runs in turn, [(label, report module, run's keyword arguments)]; + the chat it posts to (its key, as delivery.chat takes it; '' for a system job); whether it is on; and its reports + as written.""" + when: str + commands: list + zone: zoneinfo.ZoneInfo + chat: str + on: bool + text: str + @property + def key(self): + """What the job is, for its record of runs and the checksum of its buttons: its chat, time, zone and reports.""" + return f'{self.chat}|{self.when}|{self.zone.key}|{self.text}' -def display(label): - """A command as shown in chat: 'price.py close' -> 'price close'.""" - return re.sub(r'^(\w+)\.py', r'\1', label) + @property + def line(self): + """The job as .jobs add takes it: its zone if not UTC, its time, its reports.""" + zone = '' if self.zone.key == 'UTC' else f'CRON_TZ={self.zone.key} ' + return f'{zone}{self.when} {self.text}' -class ScheduleError(ValueError): - pass +def checksum(text): + """A job's short stand-in in a button (Telegram allows 64 bytes of data): it tells a changed list.""" + return format(zlib.crc32(text.encode()) & 0xffff, '04x') -class ImpossibleTime(ScheduleError): - """A time that cannot be (minute 61) or never comes (30 February). Other bad lines are reported and skipped, but - this refuses the whole file: finbot does not start with it, and a reload keeps the schedule that is running. `lines` - says where, one per line.""" - def __init__(self, lines): - """lines: where, one message or a list of them.""" - self.lines = [lines] if isinstance(lines, str) else list(lines) - super().__init__('; '.join(self.lines)) +def run_in_process(label, module, kwargs, dest=None): + """Run a command here, in the calling thread: its report's run(**kwargs), posting to dest (a chat's job) or else to + the home chats.""" + if dest is None: + module.run(**kwargs) + else: + module.run(dest, **kwargs) -@dataclass -class Job: - expr: str - commands: list # [(label, module, kwargs)], run one after another - zone: zoneinfo.ZoneInfo - text: str - key: str +def start_thread(target, *args): + """Run target(*args) in a daemon thread of its own.""" + threading.Thread(target=target, args=args, daemon=True).start() -def local_zone(): - """The server's own time zone, which is what cron uses. Containers without one are UTC, as under cron.""" +def warm_now(scheduler=None): + """Run the warm jobs (WARM) now, one after the other: through the scheduler, so that one already running on its + schedule is not started twice; without one (a test, a tool), here, a failure reported in the chat as the scheduler + reports one.""" + commands = parse_reports(WARM, system=True) + log.info('cache: filling it again:', ', '.join(label for label, _, _ in commands)) + if scheduler: + scheduler.execute(commands) + else: + for label, module, kwargs in commands: + try: + run_in_process(label, module, kwargs) + except Exception as e: + delivery.report_error(e, context=label) + log.info('cache: warm jobs done') + + +def parse_reports(text, system=False): + """'cal earnings; price 7' -> [(label, report module, run's keyword arguments)]: report names (lib/reports) and + their arguments, quoted where they hold spaces, separated by ";". The system jobs' reports only with system. Raises + ScheduleError, saying what is wrong.""" + lexer = shlex.shlex(text, posix=True, punctuation_chars=';') + lexer.whitespace_split, lexer.commenters = True, '' try: - if os.environ.get('TZ'): - return zoneinfo.ZoneInfo(os.environ['TZ'].lstrip(':')) - link = os.path.realpath('/etc/localtime') - if '/zoneinfo/' in link: - return zoneinfo.ZoneInfo(link.split('/zoneinfo/', 1)[1]) - return zoneinfo.ZoneInfo(Path('/etc/timezone').read_text().strip()) - except (OSError, ValueError, zoneinfo.ZoneInfoNotFoundError): - return zoneinfo.ZoneInfo('UTC') - - -def tokenize(text): - """A command line as ('word', text) and ('op', text) tokens. Quoted text is always a word, whatever it contains.""" - tokens, word, started, i = [], '', False, 0 - def flush(): - """End the word being read, if there is one, and add it to the tokens.""" - nonlocal word, started - if started: - tokens.append(('word', word)) - word, started = '', False - while i < len(text): - c = text[i] - if c.isspace(): - flush() - i += 1 - elif c in '\'"': - end = text.find(c, i + 1) - if end < 0: - raise ScheduleError('No closing quotation') - word, started, i = word + text[i + 1:end], True, end + 1 - elif c == '\\' and i + 1 < len(text): - word, started, i = word + text[i + 1], True, i + 2 - elif c == '#' and not started: # a trailing comment - break - elif m := OPERATOR.match(text, i): - if word.isdigit() and m.group().startswith('>'): # the 2 of 2>&1 belongs to the redirect - word, started = '', False - flush() - tokens.append(('op', m.group())) - i = m.end() - else: - word, started, i = word + c, True, i + 1 - flush() - return tokens - - -def split_commands(tokens): - """Word lists, one per command: split on unquoted ; and &&, redirects and braces dropped, other syntax refused.""" - commands, current, target = [], [], False - for kind, value in tokens: - if target: # the file of a redirect - target = False - if kind == 'word': - continue - if kind == 'word': - current.append(value) - elif value in (';', '&&'): - commands.append(current) - current = [] - elif value in ('{', '}'): + tokens = list(lexer) + except ValueError as e: # an unclosed quote + raise ScheduleError(str(e)) + commands, words = [], [] + for token in tokens + [';']: + if token.strip(';'): + words.append(token) continue - elif REDIRECT.fullmatch(value): - target = not re.search(r'&\d+$', value) # 2>&1 has no file after it, > /dev/null does - else: - raise ScheduleError('only finbot scripts can be scheduled, not shell commands') - commands.append(current) - return [words for words in commands if words] - - -def parse_command(text): - """'~/finbot/price.py midsession > /dev/null' -> [('price.py midsession', price, {...})]. Raises ScheduleError.""" - commands = [] - for words in split_commands(tokenize(text)): - while words and re.fullmatch(r'\w+=\S*', words[0]): - words.pop(0) # FOO=bar prefix if not words: continue - name = os.path.basename(words[0]).removesuffix('.py') - if name not in SCRIPTS: - raise ScheduleError(f"unknown script '{words[0]}' (known: {', '.join(sorted(SCRIPTS))})") + name, arguments, words = words[0], words[1:], [] + if name not in SCRIPTS or (name in SYSTEM_SCRIPTS and not system): + raise ScheduleError(f"no report {name!r}: send .jobs add for the reports and what each takes") try: - label, kwargs = SCRIPTS[name].job(words[1:]) + label, kwargs = SCRIPTS[name].job(arguments) except ValueError as e: - raise ScheduleError(f'{name}.py {e}') + raise ScheduleError(f'{name} takes {e}') commands.append((label, SCRIPTS[name], kwargs)) if not commands: - raise ScheduleError('no command') + raise ScheduleError('no report: send .jobs add for the reports') return commands -def parse_schedule(text, default_zone=None): - """(jobs, errors): errors are (line number, line, ScheduleError) for the lines that were skipped.""" - zone = default_zone or local_zone() - jobs, errors = [], [] - for number, raw in enumerate(text.splitlines(), 1): - line = raw.strip() - if not line or line.startswith('#'): - continue +def check_time(when): + """Raise ImpossibleTime for a time that cannot be (minute 61) or never comes (30 February).""" + if when == '@reboot': + raise ImpossibleTime('@reboot is no time: the jobs start with the bot') + if not croniter.is_valid(when): + raise ImpossibleTime(f"'{when}' is no time: five fields (minute hour day month weekday) or an @macro") + try: + croniter(when, datetime.datetime(2000, 1, 1)).get_next(datetime.datetime) + except CroniterBadDateError: + raise ImpossibleTime(f"'{when}' never comes: no date matches it") + + +def parse_job(line, chat='', on=True, system=False): + """A Job from its line, as .jobs add takes it: [CRON_TZ=Zone] five time fields (or an @macro), then its reports. + Raises ScheduleError (ImpossibleTime for the time), saying what is wrong.""" + line = line.strip() + zone = UTC + if m := re.match(r'CRON_TZ=(\S*)\s*', line): try: - if m := re.fullmatch(r'CRON_TZ=(\S+)', line): - try: - zone = zoneinfo.ZoneInfo(m.group(1)) - except (ValueError, zoneinfo.ZoneInfoNotFoundError): - raise ScheduleError(f'unknown time zone {m.group(1)}') - continue - if m := re.fullmatch(r'(\w+)=.*', line): - if m.group(1) in IGNORED_VARIABLES: - continue - raise ScheduleError(f'{m.group(1)}= is not supported') - if line.startswith('@'): - expr, _, rest = line.partition(' ') - else: - fields = line.split(None, 5) - if len(fields) < 6: - raise ScheduleError('expected five time fields and a command') - expr, rest = ' '.join(fields[:5]), fields[5] - if expr == '@reboot': - raise ScheduleError('@reboot is not supported: the schedule starts with the bot') - if not croniter.is_valid(expr): - raise ImpossibleTime(f"bad time '{expr}'") - try: - croniter(expr, datetime.datetime(2000, 1, 1)).get_next(datetime.datetime) - except CroniterBadDateError: - raise ImpossibleTime(f"'{expr}' never comes: no date matches it") - commands = parse_command(rest) - key = f"{expr}|{zone.key}|{';'.join(label for label, _, _ in commands)}" - jobs.append(Job(expr, commands, zone, line, key)) - except ScheduleError as e: - errors.append((number, line, e)) - return jobs, errors + zone = zoneinfo.ZoneInfo(m.group(1)) + except (ValueError, zoneinfo.ZoneInfoNotFoundError): + raise ScheduleError(f'no time zone {m.group(1)!r} (such as America/New_York)') + line = line[m.end():] + if line.startswith('@'): + when, _, reports = line.partition(' ') + else: + fields = line.split(None, 5) + if len(fields) < 6: + raise ScheduleError('a job is five time fields (minute hour day month weekday), then its reports') + when, reports = ' '.join(fields[:5]), fields[5] + check_time(when) + return Job(when, parse_reports(reports, system), zone, chat, on, reports.strip()) class Scheduler: - """Fires the jobs of a schedule file. `path` is a callable returning the file (the data dir copy wins over the - default, so a new copy is found on reload). `clock`, `sleep` and `spawn` can be replaced, which the tests do. Its - lock covers what the loop, a reload and runs from chat share: the jobs, their bases, the record of runs and what is - running.""" - - def __init__(self, path, catchup=None, clock=None, sleep=None, spawn=None, zone=None, runner=None): - """path() gives the schedule file's path. catchup is how late, in minutes, a missed run may still start (the - scheduler_catchup setting when None); clock, sleep, spawn, zone and runner stand in for the real ones in - tests.""" - self.path = path + """Fires the jobs: each chat's, from SCHEDULE_FILE, and the system jobs. clock, sleep, spawn and runner can be + replaced, which the tests do. Its lock covers what the loop, a change from chat and the runs share: the jobs, their + bases, the record of runs and what is running.""" + + def __init__(self, catchup=None, clock=None, sleep=None, spawn=None, runner=None): + """catchup is how late, in minutes, a missed run may still start (the scheduler_catchup setting when None); + clock, sleep, spawn and runner stand in for the real ones in tests.""" self.catchup = datetime.timedelta(minutes=config.env.scheduler_catchup if catchup is None else catchup) self.clock = clock or (lambda: datetime.datetime.now(datetime.timezone.utc)) self.sleep = sleep or time.sleep self.spawn = spawn or start_thread - self.runner = runner or run_child # (label, module, kwargs) -> exit status - self.zone = zone + self.runner = runner or run_in_process # (label, module, kwargs, dest): runs the command self.jobs = [] self.base = {} # key -> the time after which the next run is due self.last = {} - self.busy = set() # labels of the commands running now, scheduled or manual + self.busy = set() # (chat, label) of the commands running now, scheduled or manual self.lock = threading.Lock() def load(self): - """Read the schedule file. A line with an impossible time refuses the whole file: ImpossibleTime is raised, - naming every such line, and nothing changes. Other bad lines are reported and skipped; a file with no usable - line keeps the schedule that is running.""" - path = self.path() - try: - jobs, errors = parse_schedule(Path(path).read_text(encoding='utf-8'), self.zone) - except OSError as e: - delivery.report_error(e, context='schedule') - return False - impossible = [f'{path} line {number}: {error}: {line}' for number, line, error in errors - if isinstance(error, ImpossibleTime)] - if impossible: - raise ImpossibleTime(impossible) - for number, line, error in errors: - delivery.report_error(ScheduleError(f'{path} line {number}: {error}: {line}'), context='schedule') - if not jobs and self.jobs: - print('schedule: no usable line in', path, '- keeping the current schedule', file=sys.stderr) - return False + """Read every chat's jobs from SCHEDULE_FILE, besides the system jobs: at start, and after every change from a + chat. A job the file holds that cannot be any more (a report since retired) is reported and left out. A job + kept from before keeps its timing; a new one counts from now, as one does that was off.""" + jobs = [parse_job(f'{when} {reports}', system=True) for when, reports in SYSTEM] + for chat, entries in state.load(SCHEDULE_FILE, {}).items(): + for entry in entries: + try: + zone = '' if entry.get('zone', 'UTC') == 'UTC' else f"CRON_TZ={entry['zone']} " + jobs.append(parse_job(f"{zone}{entry['when']} {entry['reports']}", chat, entry.get('on', True))) + except (ScheduleError, KeyError, TypeError) as e: + delivery.report_error(ScheduleError(f'a job of {chat} left out: {e}: {entry}'), context='schedule') with self.lock: # no run is recorded between reading the record and swapping it in try: last, problem = state.load(STATE_FILE), None @@ -325,21 +217,14 @@ def load(self): self.jobs = jobs if problem: delivery.report_error(ScheduleError(problem), context='schedule') - print(f"schedule: {len(jobs)} job{'' if len(jobs) == 1 else 's'} from {path}", file=sys.stderr) - return True - - def reload(self, *args): - """SIGHUP: read the edited file. One with an impossible time is refused and reported; the schedule stays.""" - try: - self.load() - except ImpossibleTime as e: - problem = f'not reloaded, the running schedule is unchanged: {e}' - delivery.report_error(ScheduleError(problem), context='schedule') + chats = len({job.chat for job in jobs if job.chat}) + log.info(f"schedule: {len(jobs) - len(SYSTEM)} jobs of {chats} chat{'' if chats == 1 else 's'}, and" + f" {len(SYSTEM)} system jobs") def start_base(self, job, now): """Where to count from at start: a run missed in the catch-up window fires once; with no record, none is.""" last = self.last.get(job.key) - if not last: + if not last or not job.on: return now try: return max(datetime.datetime.fromisoformat(last), now - self.catchup) @@ -351,7 +236,7 @@ def next_fire(self, job, after): put back after: croniter up to 6.0 (the distro packages) mishandles zoneinfo across DST changes, firing daily jobs twice or an hour late. A time the clocks repeat fires once, the first time; one they skip (02:30 as they go forward) fires an hour later by the clock.""" - times = croniter(job.expr, after.astimezone(job.zone).replace(tzinfo=None)) + times = croniter(job.when, after.astimezone(job.zone).replace(tzinfo=None)) while True: # fold 0: the first of a repeated time t = times.get_next(datetime.datetime).replace(tzinfo=job.zone).timestamp() @@ -360,10 +245,14 @@ def next_fire(self, job, after): return datetime.datetime.fromtimestamp(t, job.zone) def due(self, now): - """The jobs to run now, as (job, fire time). Several missed fire times of one job run it once.""" + """The jobs to run now, as (job, fire time). Several missed fire times of one job run it once. A job that is off + never fires, and counts from now: switched on, it does not catch up.""" fired = [] with self.lock: for job in self.jobs: + if not job.on: + self.base[job.key] = now + continue base, latest = self.base[job.key], None while (t := self.next_fire(job, base)) <= now: latest, base = t, t @@ -375,105 +264,139 @@ def due(self, now): try: state.save(STATE_FILE, self.last) except OSError as e: - print('schedule: cannot save', STATE_FILE, e, file=sys.stderr) + log.error('schedule: cannot save', STATE_FILE, e) return fired - def claim(self, label): - """Mark a command running, unless it already is (scheduled or manual): whether it was claimed.""" + def claim(self, chat, label): + """Mark a chat's command running, unless it already is (scheduled or manual): whether it was claimed.""" with self.lock: - if label in self.busy: + if (chat, label) in self.busy: return False - self.busy.add(label) + self.busy.add((chat, label)) return True - def execute(self, commands): - """Run commands one after another (a job's), each stopped and reported if still running after JOB_TIMEOUT. - One already running (from the line's last run, another line or chat) is skipped. Returns 'done', 'timed out', - 'failed' or 'already running'.""" + def execute(self, commands, chat=''): + """Run commands one after another (a job's), in this thread, posting to the chat whose job it is (its key; '' + for a system job, whose failure goes to the home chats); a failure is reported. One already running for that + chat is skipped. One still running after JOB_TIMEOUT is reported as late, once, and runs on: a thread cannot be + stopped, but every request it makes has a timeout, so it ends; until then it stays marked running, and no + second copy starts. Returns 'done', 'failed' or 'already running'.""" + dest = delivery.chat(chat, scheduled=True) if chat else None outcome = 'done' for label, module, kwargs in commands: - if not self.claim(label): - print(f'schedule: {label} is already running, skipping it', file=sys.stderr) + if not self.claim(chat, label): + log.warn(f'schedule: {label} is already running for {chat or "the system"}, skipping it') outcome = 'already running' continue - print(f'schedule: running {label}', file=sys.stderr) + log.info(f'schedule: running {label}' + (f' for {chat}' if chat else '')) + late = threading.Timer(JOB_TIMEOUT, delivery.report_error, args=(TimeoutError(f'still running after' + f' {JOB_TIMEOUT / 60:g} minutes, and left to finish'), dest), kwargs={'context': label}) + late.daemon = True + late.start() try: - status = self.runner(label, module, kwargs) - if status: - outcome = 'failed' - if status != jobrun.REPORTED: # it ended without saying why - delivery.report_error(RuntimeError(exit_reason(status)), context=label) - except TimeoutError as e: # run_child stopped it: the rest of the line still runs - delivery.report_error(e, context=label) - outcome = 'timed out' + self.runner(label, module, kwargs, dest) except (Exception, SystemExit) as e: - delivery.report_error(e, context=label) + delivery.report_error(e, dest, context=label) outcome = 'failed' finally: + late.cancel() with self.lock: - self.busy.discard(label) + self.busy.discard((chat, label)) return outcome - def commands(self, hidden=False): - """Every scheduled command offered in chat once, or with hidden=True each of the HIDDEN scripts' (System), - sorted by name with numbers in numeric order: [(label, module, kwargs)]. The time, comments, redirects and the - other commands of a line are not part of it.""" - unique = {} - system = {SCRIPTS[name] for name in HIDDEN} - for job in self.jobs: - for command in job.commands: - if (command[1] in system) == hidden: - unique.setdefault(command[0], command) - natural = lambda label: [int(part) if part.isdigit() else part for part in re.split(r'(\d+)', label.lower())] - return [unique[label] for label in sorted(unique, key=natural)] - - def select(self, text): - """The scheduled command for `.jobs run `: a number in the list, or the command. Raises ScheduleError.""" - text = text.strip() - commands = self.commands() - if text.isdigit(): - if not 1 <= int(text) <= len(commands): - raise ScheduleError(f'no command {text}; send .jobs to list them') - return commands[int(text) - 1] - try: - wanted = parse_command(text) - except ScheduleError as e: - raise ScheduleError(f'{e}; send .jobs to list the commands') - if len(wanted) != 1: - raise ScheduleError('run one command at a time; send .jobs to list them') - for command in commands: - if command[0] == wanted[0][0]: - return command - if any(command[0] == wanted[0][0] for command in self.commands(hidden=True)): - raise ScheduleError(f'{display(wanted[0][0])} is a system job: it runs on its schedule only') - raise ScheduleError(f"'{text}' is not in the schedule; send .jobs to list the commands") - - def pick(self, number, crc): - """The offered command a button stands for. Button data is untrusted input: the number must be plain digits - within the list and the checksum must match that very command, else ScheduleError.""" - commands = self.commands() - if not (number.isascii() and number.isdigit() and 1 <= int(number) <= len(commands)): - raise ScheduleError('no such command') - command = commands[int(number) - 1] - if not hmac.compare_digest(checksum(command[0]), crc): - raise ScheduleError('the schedule changed') - return command - - def run_command(self, command): - """Run one command returned by select() or pick(). Returns (label, outcome).""" - return command[0], self.execute([command]) - - def run_now(self, text): - """`.jobs run`: one scheduled command now, as cron runs it; the schedule stays. Returns (label, outcome).""" - return self.run_command(self.select(text)) + def running(self, chat): + """The labels of the commands running now for a chat (by its key), and for the system.""" + with self.lock: + return {label for where, label in self.busy if where in (chat, '')} + + def chat_jobs(self, chat): + """A chat's jobs, in their order: .jobs numbers them from 1.""" + with self.lock: + return [job for job in self.jobs if job.chat == chat] + + def system_jobs(self): + """The system jobs.""" + with self.lock: + return [job for job in self.jobs if not job.chat] + + def job(self, chat, number): + """A chat's job by its number in .jobs (a string, from a chat). Raises ScheduleError.""" + jobs = self.chat_jobs(chat) + if not (number.isascii() and number.isdigit() and 1 <= int(number) <= len(jobs)): + raise ScheduleError(f'no job {number}: send .jobs for the list' if jobs else 'no jobs here yet: .jobs add') + return jobs[int(number) - 1] + + def pick(self, chat, number, crc): + """The job a button stands for. Button data is untrusted input: the number must be plain digits within this + chat's list, and the checksum must match that very job, else ScheduleError.""" + job = self.job(chat, number) + if not hmac.compare_digest(checksum(job.key), crc): + raise ScheduleError('the jobs have changed') + return job + + def run(self, job): + """Run a job now, as its schedule would: its outcome (see execute).""" + return self.execute(job.commands, job.chat) + + def change(self, chat, edit): + """Change a chat's jobs, as stored: edit(entries) does it to the list (and may raise ScheduleError); then they + are read again, so that the change counts at once. Returns what edit returned.""" + with state.edit(SCHEDULE_FILE, {}) as chats: + entries = chats.setdefault(chat, []) + result = edit(entries) + if not entries: + del chats[chat] + self.load() + return result + + def add(self, chat, line): + """Add a job to a chat's list (a line as parse_job takes it), on: its number. Raises ScheduleError.""" + job = parse_job(line, chat) + def append(entries): + """Append the job, within MAX_JOBS.""" + if len(entries) >= MAX_JOBS: + raise ScheduleError(f'{MAX_JOBS} jobs is the most a chat may have: delete one first') + entries.append({'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': True}) + return len(entries) + return self.change(chat, append) + + def delete(self, job): + """Delete a chat's job (from job() or pick()).""" + self.change(job.chat, lambda entries: entries.pop(self.index(entries, job))) + + def toggle(self, job): + """Switch a chat's job on or off: whether it is on now.""" + def flip(entries): + """Flip the job's switch.""" + entry = entries[self.index(entries, job)] + entry['on'] = not entry.get('on', True) + return entry['on'] + return self.change(job.chat, flip) + + @staticmethod + def index(entries, job): + """Where a job is among its chat's stored entries. Raises ScheduleError when it is gone meanwhile.""" + for i, entry in enumerate(entries): + zone = entry.get('zone', 'UTC') + if (entry.get('when'), entry.get('reports'), zone) == (job.when, job.text, job.zone.key): + return i + raise ScheduleError('that job is gone: send .jobs for the list') + + def next_run(self, job): + """When a job fires next, or None when it is off.""" + if not job.on: + return None + with self.lock: + base = self.base.get(job.key) or self.clock() + return self.next_fire(job, max(base, self.clock())) def tick(self): """Start what is due; returns the seconds until the next fire time.""" now = self.clock() for job, fire_time in self.due(now): - self.spawn(self.execute, job.commands) + self.spawn(self.execute, job.commands, job.chat) with self.lock: - waits = [(self.next_fire(job, self.base[job.key]) - now).total_seconds() for job in self.jobs] + waits = [(self.next_fire(job, self.base[job.key]) - now).total_seconds() for job in self.jobs if job.on] return min(30, max(1, math.ceil(min(waits)))) if waits else 30 def run_forever(self): diff --git a/lib/log.py b/lib/log.py new file mode 100644 index 00000000..46eb9c97 --- /dev/null +++ b/lib/log.py @@ -0,0 +1,39 @@ +"""The log, syslog style: each line has a level, ERROR, WARN, INFO or DEBUG, and goes to stderr (which systemd and +Docker take to the journal) when the log_level preference lets it in: that level and those above it, so ERROR always +does. Each takes print()'s arguments.""" +import sys + +from lib import config + +LEVELS = config.LOG_LEVELS # ERROR WARN INFO DEBUG, the most severe first + + +def enabled(level): + """Whether log_level lets lines of this level into the log.""" + return LEVELS.index(level) <= LEVELS.index(config.prefs.log_level) + + +def write(level, parts, options): + """A line of these parts (and print()'s options) at this level, if log_level lets it in.""" + if enabled(level): + print(*parts, file=sys.stderr, **options) + + +def error(*parts, **options): + """Something failed and was lost: a message not sent, a request not answered, a start that cannot go on.""" + write('ERROR', parts, options) + + +def warn(*parts, **options): + """Something failed and finbot got over it: a notice not shown, a retry, a fallback, a request refused.""" + write('WARN', parts, options) + + +def info(*parts, **options): + """What finbot does: a message sent, a job run, each request it sends (a cache miss's) and each it answers.""" + write('INFO', parts, options) + + +def debug(*parts, **options): + """The detail of what it does, to find a fault: a cache miss, hit or write.""" + write('DEBUG', parts, options) diff --git a/lib/markets.py b/lib/markets.py index b0fee86e..6f191d2b 100644 --- a/lib/markets.py +++ b/lib/markets.py @@ -78,49 +78,6 @@ class Market(typing.NamedTuple): 'Shanghai': 'SHG', 'Shenzhen': 'SHE', 'Thailand': 'BKK', 'Jakarta': 'IDX', 'Kuala Lumpur': 'KLS', } -YAHOO_COUNTRY = { # country -> its Yahoo ticker suffix, for a listing that has none (cal.py) - 'Argentina': 'BA', - 'Austria': 'VI', - 'Australia': 'AX', - 'Belgium': 'BR', - 'Brazil': 'SA', - 'Canada': 'TO', - 'Chile': 'SN', - 'Colombia': 'CL', - 'Czech Republic': 'PR', - 'Denmark': 'CO', - 'Estonia': 'TL', - 'Finland': 'HE', - 'France': 'PA', - 'Germany': 'DE', - 'Hungary': 'BD', - 'Iceland': 'IC', - 'India': 'NS', - 'Indonesia': 'JK', - 'Ireland': 'IR', - 'Israel': 'TA', - 'Italy': 'MI', - 'Latvia': 'RG', - 'Lithuania': 'VS', - 'Mexico': 'MX', - 'Netherlands': 'AS', - 'New Zealand': 'NZ', - 'Norway': 'OL', - 'Portugal': 'LS', - 'Qatar': 'QA', - 'Saudi Arabia': 'SR', - 'South Africa': 'JO', - 'Spain': 'MC', - 'Sweden': 'ST', - 'Switzerland': 'SW', - 'Thailand': 'BK', - 'Turkey': 'IS', - 'United Kingdom': 'L', - 'Venezuela': 'CR' - # markets with numeric tickers purposefully omitted -} - - def code(name): """MARKETS' code for an exchange by any name it goes by: Sharesight's code or Google's, the suffix of its Yahoo tickers, or Yahoo's name for it (each Nasdaq tier has one: NasdaqGS); the name itself when unknown.""" diff --git a/lib/migrate.py b/lib/migrate.py new file mode 100644 index 00000000..8d397184 --- /dev/null +++ b/lib/migrate.py @@ -0,0 +1,191 @@ +"""Once, as finbot starts: what the bot kept for all chats at once becomes each home chat's (telegramChatID, +slackChannelID, discordChannelID). The old schedule.cron (the data dir's copy, else the shipped one) is copied to every +home chat as its jobs, each in the zone it ran in, and is read no more; the watchlist and the plans are copied to each +home chat; and so are the reports' records of what they told the chats (trades, ratings, record highs, podcast +episodes), so that nothing old is told again. + +This module, and defaults/schedule.cron, go once every install has started with it: they are its only users.""" +import copy +import datetime +import glob +import os +import re +import shlex +import zoneinfo +from pathlib import Path + +from lib import log +from lib import config +from lib import delivery +from lib import jobs +from lib import state +from lib import watchlist +from lib import worker +from lib.reports import milestone, podcasts, rating, trades + +# kept for all chats, each now one record per chat; and each podcast feed's seen file +RECORDS = (watchlist.FILE, worker.PLANS, trades.STATE_FILE, rating.STATE_FILE, milestone.STATE_FILE) +# what a shell reads as more than a word, outside quotes, in a pasted crontab line +OPERATOR = re.compile(r'\d*>>?&?\d*|&>>?|;|&&|\|\|?|&|[<`$(){}]') +REDIRECT = re.compile(r'\d*>>?&?\d*|&>>?') # > file, >> file, 2>&1, &> file + + +def run(): + """Make the home chats' own what was kept for all chats, where that has not been done yet. With no home chat set, + it all waits until there is one.""" + try: + homes = [chat.key for chat in delivery.default_chats()] + except RuntimeError: + return + schedule(homes) + feeds = sorted(os.path.basename(path) for path in glob.glob(state.path(podcasts.SEEN_FILES))) + for name in RECORDS + tuple(feeds): + record(name, homes) + + +def schedule(homes): + """Give every home chat (keys) the jobs of the old schedule.cron, the system jobs aside (they run in code now), + unless the chats' jobs are kept already (jobs.SCHEDULE_FILE). A line that cannot be a job any more is logged and + left out.""" + if os.path.exists(state.path(jobs.SCHEDULE_FILE)): + return + path = config.data_file('schedule.cron') + try: + text = Path(path).read_text(encoding='utf-8') + except OSError: + text = '' + entries = [] + for number, line, zone, words_list in legacy_lines(text, path): + reports = '; '.join(shlex.join([name, *arguments]) for name, arguments in words_list + if name not in jobs.SYSTEM_SCRIPTS) + if not reports: + continue + try: + job = jobs.parse_job(f"{'' if zone == 'UTC' else f'CRON_TZ={zone} '}{line} {reports}") + except jobs.ScheduleError as e: + log.warn(f'schedule: {path} line {number} left out: {e}') + continue + entries.append({'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': True}) + # written even when empty: the file being there says the migration is done + state.save(jobs.SCHEDULE_FILE, {key: copy.deepcopy(entries) for key in homes if entries}) + log.info(f"schedule: the {len(entries)} jobs of {path} are now each home chat's ({', '.join(homes)});" + f" finbot reads that file no more: each chat's jobs are in {jobs.SCHEDULE_FILE}, changed with .jobs") + + +def record(name, homes): + """A file's record kept for all chats at once (anything but {'chats': ...}) becomes each home chat's.""" + try: + data = state.load(name) + except (OSError, ValueError) as e: + log.warn(f'state: {name} left as it is, unreadable: {e}') + return + if data is None or (isinstance(data, dict) and 'chats' in data): + return + state.save(name, {'chats': {key: copy.deepcopy(data) for key in homes}}) + log.info(f"state: {name} is kept per chat now, its record copied to {', '.join(homes)}") + + +def local_zone(): + """The server's own time zone, which the old schedule ran in, as cron does: UTC by that name when it is UTC by + another (Etc/UTC), or when there is none.""" + try: + if os.environ.get('TZ'): + zone = zoneinfo.ZoneInfo(os.environ['TZ'].lstrip(':')) + elif '/zoneinfo/' in (link := os.path.realpath('/etc/localtime')): + zone = zoneinfo.ZoneInfo(link.split('/zoneinfo/', 1)[1]) + else: + zone = zoneinfo.ZoneInfo(Path('/etc/timezone').read_text().strip()) + except (OSError, ValueError, zoneinfo.ZoneInfoNotFoundError): + return jobs.UTC + year = [datetime.datetime(2026, month, 1, tzinfo=zone).utcoffset() for month in (1, 7)] + return jobs.UTC if year == [datetime.timedelta(0)] * 2 else zone + + +def legacy_lines(text, path): + """The jobs of an old schedule.cron (text, from path): (line number, time, zone name, [(report name, arguments)]) + for each line that parses, in the zone set above it (CRON_TZ=) or the server's own; pasted crontab syntax (paths, + .py, redirects, braces, variables) read as cron read it. A line that does not parse is logged.""" + zone = local_zone().key + for number, raw in enumerate(text.splitlines(), 1): + line = raw.strip() + if not line or line.startswith('#'): + continue + if m := re.fullmatch(r'CRON_TZ=(\S+)', line): + zone = m.group(1) + continue + if re.fullmatch(r'\w+=.*', line): # MAILTO=, PATH= and the like + continue + if line.startswith('@'): + when, _, rest = line.partition(' ') + else: + fields = line.split(None, 5) + if len(fields) < 6: + continue + when, rest = ' '.join(fields[:5]), fields[5] + try: + commands = [(os.path.basename(words[0]).removesuffix('.py'), words[1:]) + for words in split_commands(tokenize(rest))] + except ValueError as e: + log.warn(f'schedule: {path} line {number} left out: {e}') + continue + yield number, when, zone, commands + + +def tokenize(text): + """A command line as ('word', text) and ('op', text) tokens. Quoted text is always a word, whatever it contains.""" + tokens, word, started, i = [], '', False, 0 + def flush(): + """End the word being read, if there is one, and add it to the tokens.""" + nonlocal word, started + if started: + tokens.append(('word', word)) + word, started = '', False + while i < len(text): + c = text[i] + if c.isspace(): + flush() + i += 1 + elif c in '\'"': + end = text.find(c, i + 1) + if end < 0: + raise ValueError('no closing quotation') + word, started, i = word + text[i + 1:end], True, end + 1 + elif c == '\\' and i + 1 < len(text): + word, started, i = word + text[i + 1], True, i + 2 + elif c == '#' and not started: # a trailing comment + break + elif m := OPERATOR.match(text, i): + if word.isdigit() and m.group().startswith('>'): # the 2 of 2>&1 belongs to the redirect + word, started = '', False + flush() + tokens.append(('op', m.group())) + i = m.end() + else: + word, started, i = word + c, True, i + 1 + flush() + return tokens + + +def split_commands(tokens): + """Word lists, one per command: split on unquoted ; and &&, redirects and braces dropped, variables before a command + dropped, other shell syntax refused (ValueError).""" + commands, current, target = [], [], False + for kind, value in tokens: + if target: # the file of a redirect + target = False + if kind == 'word': + continue + if kind == 'word': + if current or not re.fullmatch(r'\w+=\S*', value): # FOO=bar before a command + current.append(value) + elif value in (';', '&&'): + commands.append(current) + current = [] + elif value in ('{', '}'): + continue + elif REDIRECT.fullmatch(value): + target = not re.search(r'&\d+$', value) # 2>&1 has no file after it, > /dev/null does + else: + raise ValueError('only finbot scripts can be scheduled, not shell commands') + commands.append(current) + return [words for words in commands if words] diff --git a/lib/net.py b/lib/net.py index c5e59fa7..eef3313c 100644 --- a/lib/net.py +++ b/lib/net.py @@ -9,19 +9,34 @@ import requests from lib import config +from lib import log session = requests.Session() # no domain may set or receive a cookie session.cookies.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[])) CONNECT_TIMEOUT = 5 # seconds to wait for a host to answer the connection: one that is down costs this, not http_timeout +SHOWN = 73 # bytes of a request's log line, at most: the method and URL, cut with an ellipsis (3 bytes in UTF-8) def get(url, **kwargs): - """requests.get through the shared session, which reuses connections; timed out as timeouts() says.""" - return session.get(url, **timeouts(kwargs)) + """requests.get through the shared session, which reuses connections; timed out as timeouts() says; logged.""" + return session.get(url, **timeouts(logged('GET', url, kwargs))) def post(url, **kwargs): - """requests.post through the shared session, which reuses connections; timed out as timeouts() says.""" - return session.post(url, **timeouts(kwargs)) + """requests.post through the shared session, which reuses connections; timed out as timeouts() says; logged.""" + return session.post(url, **timeouts(logged('POST', url, kwargs))) + +def request(method, url, **kwargs): + """A request of any method through the shared session, which reuses connections; timed out as timeouts() says; + logged.""" + return session.request(method, url, **timeouts(logged(method, url, kwargs))) + +def logged(method, url, kwargs): + """The request's arguments without `logged`; the request noted at INFO first, unless logged=False (Telegram's long + polling, asked again and again): its method and URL, without secrets or the scheme, cut to SHOWN bytes.""" + if kwargs.pop('logged', True): + line = f"{method} {redact(url).split('://', 1)[-1]}".encode() + log.info(line.decode() if len(line) <= SHOWN else line[:SHOWN - 3].decode(errors='ignore') + '…') + return kwargs def timeouts(kwargs): """The request's arguments with a timeout, unless it has its own: CONNECT_TIMEOUT to connect, then http_timeout for @@ -29,9 +44,13 @@ def timeouts(kwargs): return {'timeout': (CONNECT_TIMEOUT, config.prefs.http_timeout), **kwargs} def redact(text): - """text (or an exception) without the secrets that request URLs, headers and error messages carry: Telegram bot - tokens, Slack and Discord webhook paths, bearer tokens, and token, crumb, key, secret and auth parameters.""" + """text (or an exception) without the secrets that request URLs, headers and error messages carry: Telegram and + Slack tokens, Slack and Discord webhook paths, Discord's bot and interaction tokens, bearer tokens, and token, + crumb, key, secret and auth parameters.""" text = re.sub(r'bot\d+:[\w-]+', 'bot', str(text)) + text = re.sub(r'\bx(?:ox[a-z]|app)-[\w-]+', '', text) # Slack's tokens text = re.sub(r'(/services/|/api/webhooks/)[^\s\'"?&]+', r'\1', text) + text = re.sub(r'(/(?:webhooks|interactions)/\d+/)[^\s\'"?&/]+', r'\1', text) # Discord's API + text = re.sub(r'\b[\w-]{23,28}\.[\w-]{6,7}\.[\w-]{27,}', '', text) # a Discord bot token text = re.sub(r'(?i)(bearer\s+)[\w.~+/=-]+', r'\1', text) return re.sub(r'(?i)(token|crumb|auth\w*|key|secret)=[^&\s\'"]+', r'\1=', text) diff --git a/lib/pull.py b/lib/pull.py new file mode 100644 index 00000000..cae30aab --- /dev/null +++ b/lib/pull.py @@ -0,0 +1,69 @@ +"""The pull modes, whichever network: a network that finbot.env gives no webhook URL is asked for its updates instead, +by a listener in a thread of its own (Telegram's long polling, Slack's Socket Mode and Discord's gateway: in +telegram_longpolling, slack_websocket and discord_websocket), which hands each update to bot.take(network, update) as +the webhook would, so that reading it, the repeat check and acknowledging a press are the same both ways. Each +listener keeps going through failures, waiting longer after each one in a row (backoff).""" +import time + +from lib import log +from lib import delivery +from lib import net + + +BACKOFF = (5, 60) # seconds to wait after a failure: the first, doubled after each further one, up to the second + + +def backoff(failures): + """The seconds to wait after this many failures in a row.""" + first, most = BACKOFF + return min(most, first * 2 ** (failures - 1)) + + +class Listener: + """Updates fetched from a network, for its reader; run() hands them to take() until the process ends.""" + network = None # the reader, an inbound.Network + failures = 0 # in a row + + @staticmethod + def wanted(): + """Whether finbot.env asks for this listener: its network's keys set, and no webhook URL.""" + raise NotImplementedError + + def run(self, take): + """Fetch and hand on updates for good, waiting as poll() says between rounds.""" + log.info(f"{self.network.service}: {self.describe()}") + while True: + time.sleep(self.poll(take)) + + def describe(self): + """How it listens, for the log as it starts.""" + return 'listening' + + def poll(self, take): + """One round: fetch what there is, hand each update to take(network, update), and return the seconds to wait + before the next round.""" + raise NotImplementedError + + def failed(self, what, e): + """Note a failure in a row, log it, and return the wait before trying again.""" + self.failures += 1 + wait = backoff(self.failures) + log.warn(f"{self.network.service}: {what} failed, trying again in {wait} s:", net.redact(e)) + return wait + + def hand_on(self, take, update): + """take(network, update), and its answer delivered where the network must have it: an interaction's, and one + that work waits on. A failure is reported as bot.main reports one of an authenticated request: the update came + from the network, so the default chats are told.""" + try: + answer = take(self.network, update) + if getattr(answer, 'interaction', False) or getattr(answer, 'then', None): + self.answer(update, answer) + except Exception as e: + delivery.report_error(e, context=self.network.service) + + def answer(self, update, answer): + """Deliver an answer that the network must have (only Discord's interactions have one: DiscordGateway), then + start the work waiting on it, if any.""" + if answer.then: + answer.then() diff --git a/lib/render.py b/lib/render.py index c74e44cc..09a8134c 100644 --- a/lib/render.py +++ b/lib/render.py @@ -1,12 +1,13 @@ -"""Charts drawn as PNGs with matplotlib: the bot draws them in one native thread, off the gevent hub (render_off_hub). A -report describes its chart as a Chart, which draw() draws only for a chat that takes charts, and once: the image is -cached by the spec's hash.""" +"""Charts drawn as PNGs with matplotlib, one at a time (_drawn). A report describes its chart as a Chart, which draw() +draws only for a chat that takes charts, and once: the image is cached by the spec's hash.""" import dataclasses +import datetime import functools import hashlib import importlib.metadata import io import json +import threading from pathlib import Path from lib import config # first: sets MPLCONFIGDIR before matplotlib is imported (lazily, by the chart functions) from lib import cache @@ -57,39 +58,40 @@ def style_date_axis(ax, start, end, ink): UP, DOWN, FLAT = '#34d399', '#fb7185', PALETTE[2] # a rise, a fall, neither: a price line's colour by its change BAR_UP = CATEGORIES[0] # bars rise in blue; they fall in DOWN -_render_pool = None # set by render_off_hub(): the bot draws its charts in one native thread +_drawing = threading.RLock() # one chart at a time: matplotlib's caches are shared by every thread -def render_off_hub(): - """For the bot: draw charts in one native thread from now on, so that the gevent hub keeps serving every other chat - while one is drawn (the shorts pictogram takes over a second; in the hub that froze everything). One thread, because - several at once deadlock matplotlib under gevent's patched locks. matplotlib is imported here, at start, instead of - by the first chart. Job processes and the tests draw in place.""" - global _render_pool +def preload(): + """For the bot, at start: import matplotlib now, instead of when the first chart is asked for.""" import importlib - from gevent.threadpool import ThreadPool for module in ('matplotlib.backends.backend_agg', 'matplotlib.dates', 'matplotlib.figure', 'matplotlib.patches', 'matplotlib.path', 'matplotlib.ticker'): importlib.import_module(module) - _render_pool = ThreadPool(1) def _drawn(draw): - """A chart function that draws in the render pool when there is one. Only for a function that calls none of the - others: the pool's one thread waiting on itself would never finish.""" + """A chart function that draws while no other thread does (it may call another such function).""" @functools.wraps(draw) - def in_pool(*args, **kwargs): - """Draw in the render thread when the bot has one (render_off_hub), else here.""" - if _render_pool is None: + def alone(*args, **kwargs): + """Draw, holding the drawing lock.""" + with _drawing: return draw(*args, **kwargs) - return _render_pool.apply(draw, args, kwargs) - return in_pool + return alone -def _figure(facecolor): - """A 1280x800 figure (Telegram's photo size, so it is never resampled) and its Agg canvas.""" +def _figure(facecolor, height=4.0): + """A 1280x800 figure (Telegram's photo size, so it is never resampled), or one `height` inches tall (200 pixels to + the inch), and its Agg canvas.""" from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=facecolor) + fig = Figure(figsize=(6.4, height), dpi=200, facecolor=facecolor) return fig, FigureCanvasAgg(fig) +def _from_top(y, fig): + """A height in a 1280x800 figure, as a fraction of it, kept as far from the top in a taller figure.""" + return y if fig.get_figheight() == 4.0 else 1 - (1 - y) * 4.0 / fig.get_figheight() + +def _from_bottom(y, fig): + """A height in a 1280x800 figure, as a fraction of it, kept as far from the bottom in a taller figure.""" + return y if fig.get_figheight() == 4.0 else y * 4.0 / fig.get_figheight() + def _time_axes(fig, bg, width): """The axes of a chart over time, `width` of the figure wide: the line chart's, and the compare chart's, whose end labels take the rest.""" @@ -124,15 +126,18 @@ def _heading(fig, title, subtitle='', subtitle_color=None): """A chart's title at the top left, and its subtitle (if any) beneath, each shrunk to fit the width.""" _, ink, ink2, _ = PALETTE room = 0.94 * fig.bbox.width - _shrink([fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center')], room) + _shrink([fig.text(0.03, _from_top(0.957, fig), title, color=ink, fontsize=13, fontweight='bold', ha='left', + va='center')], room) if subtitle: - _shrink([fig.text(0.03, 0.910, subtitle, color=subtitle_color or ink2, fontsize=8.5, ha='left', va='center')], - room) + _shrink([fig.text(0.03, _from_top(0.910, fig), subtitle, color=subtitle_color or ink2, fontsize=8.5, ha='left', + va='center')], room) -def _series(rows): - """(date, close) rows as a pandas Series of the closes, indexed by date.""" - import pandas as pd - return pd.Series([float(close) for _, close in rows], index=pd.to_datetime([day for day, _ in rows])) +def _dated(rows): + """(date, close) rows, each date an ISO date or a datetime.date, as (the dates as datetime.date, the closes as a + numpy array): matplotlib takes both as they are.""" + import numpy as np + dates = [day if isinstance(day, datetime.date) else datetime.date.fromisoformat(day) for day, _ in rows] + return dates, np.array([float(close) for _, close in rows]) def _png(fig, facecolor): """The figure as PNG bytes, rewound (or an upload sends nothing); not bbox_inches='tight': it alters 1280x800.""" @@ -149,8 +154,7 @@ def draw_line(chart): import matplotlib.dates as mdates import numpy as np bg, ink, ink2, grid = PALETTE - series = _series(chart.rows) - x, y = series.index, series.values + x, y = _dated(chart.rows) change = chart.change if chart.change is not None else y[-1] - y[0] color = UP if change > 0 else DOWN if change < 0 else FLAT @@ -197,23 +201,23 @@ def draw_compare(chart): colors = NEITHER_UP_NOR_DOWN fig, _ = _figure(bg) ax = _time_axes(fig, bg, 0.72) - rebased = [] + rebased = [] # (label, dates, % change since the first close) for label, points in chart.rows: - y = _series(points) - rebased.append((label, (y / y.iloc[0] - 1) * 100)) + x, y = _dated(points) + rebased.append((label, x, (y / y[0] - 1) * 100)) ax.axhline(0, color=ink2, linewidth=0.8, alpha=0.6, zorder=1) - for (label, y), color in zip(rebased, colors): - ax.plot(y.index, y.values, color=color, linewidth=1.5, solid_capstyle='round', zorder=3) - lo = min(y.min() for _, y in rebased) - hi = max(y.max() for _, y in rebased) + for (label, x, y), color in zip(rebased, colors): + ax.plot(x, y, color=color, linewidth=1.5, solid_capstyle='round', zorder=3) + lo = min(y.min() for _, _, y in rebased) + hi = max(y.max() for _, _, y in rebased) span = (hi - lo) or 1.0 ax.set_ylim(min(lo, 0) - span * 0.06, max(hi, 0) + span * 0.06) ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:+,.0f}%" if v else "0%")) - _style_time_axes(fig, ax, min(y.index[0] for _, y in rebased), max(y.index[-1] for _, y in rebased)) + _style_time_axes(fig, ax, min(x[0] for _, x, _ in rebased), max(x[-1] for _, x, _ in rebased)) # direct end-labels in the right margin, spread so they never overlap ymin, ymax = ax.get_ylim() - ends = sorted(((y.iloc[-1], i) for i, (_, y) in enumerate(rebased))) + ends = sorted(((y[-1], i) for i, (_, _, y) in enumerate(rebased))) pos = [(v - ymin) / (ymax - ymin) for v, _ in ends] gap = 0.065 for i in range(1, len(pos)): @@ -224,10 +228,10 @@ def draw_compare(chart): for i in range(len(pos) - 2, -1, -1): pos[i] = min(pos[i], pos[i + 1] - gap) for (value, i), p in zip(ends, pos): - label, y = rebased[i] + label, x, _ = rebased[i] color = colors[i] ax.plot([1.015], [p], marker='s', markersize=4, color=color, transform=ax.transAxes, clip_on=False, zorder=4) - ax.annotate(f"{label} {value:+,.1f}%", xy=(y.index[-1], value), xycoords='data', xytext=(1.03, p), + ax.annotate(f"{label} {value:+,.1f}%", xy=(x[-1], value), xycoords='data', xytext=(1.03, p), textcoords='axes fraction', ha='left', va='center', fontsize=7.5, color=ink, annotation_clip=False, arrowprops=dict(arrowstyle='-', color=color, linewidth=0.7, alpha=0.7, shrinkA=0, shrinkB=2, relpos=(0, 0.5))) @@ -245,21 +249,23 @@ def tinted(bg, color, amount=0.07): def draw_bars(chart): """Horizontal bars that mirror the caption list, first row on top, filling the whole 1280x800 canvas: blue for >= 0, red below, each row's label at the bar's base and its value (as value_format prints it) at the tip. Rows of (label, - value); a (label, None) row is a gap. xlim zooms the value axis (bars then start at its left edge); session - 'midsession' or 'premarket' washes the background green or yellow.""" + value); a (label, None) row is a gap. Past ROWS rows the image grows taller, each row as tall as at ROWS, so that + every row is in it. xlim zooms the value axis (bars then start at its left edge); session 'midsession' or + 'premarket' washes the background green or yellow.""" rows, subtitle, xlim, session = chart.rows, chart.subtitle, chart.xlim, chart.session value_fmt = VALUE_FORMATS[chart.value_format] bg, ink, ink2, _ = PALETTE up, down = BAR_UP, DOWN if session in SESSION_TINTS: bg = tinted(bg, SESSION_TINTS[session]) - fig, canvas = _figure(bg) - top_edge = 0.85 if subtitle else 0.89 + n = len(rows) + fig, canvas = _figure(bg, 4.0 * max(1, n / ROWS)) + top_edge = _from_top(0.85 if subtitle else 0.89, fig) + bottom_edge = _from_bottom(0.02, fig) left = 0.14 if xlim else 0.04 # zoomed charts keep their labels outside the plot - ax = fig.add_axes([left, 0.02, 0.96 - left, top_edge - 0.02], facecolor=bg) + ax = fig.add_axes([left, bottom_edge, 0.96 - left, top_edge - bottom_edge], facecolor=bg) real = [(y, label, v) for y, (label, v) in enumerate(rows) if v is not None] values = [v for _, _, v in real] - n = len(rows) ax.barh([y for y, _, _ in real], values, height=0.72, color=[up if v >= 0 else down for v in values], zorder=3) base = max(xlim[0], 0) if xlim else 0 if not xlim: @@ -276,7 +282,7 @@ def draw_bars(chart): for side in ax.spines.values(): side.set_visible(False) dpi = fig.dpi - axes_px = (top_edge - 0.02) * fig.get_figheight() * dpi + axes_px = (top_edge - bottom_edge) * fig.get_figheight() * dpi bar_px = 0.72 * axes_px / n longest = max([len(str(label)) for _, label, _ in real] + [3]) # text height follows the bar thickness @@ -515,7 +521,8 @@ def _code_version(): CODE_VERSION = _code_version() RENDERERS = {'bars': draw_bars, 'pie': draw_pie, 'shorts': draw_shorts, 'line': draw_line, 'compare': draw_compare} -LIMITS = {'bars': 40, 'shorts': 24} # the rows a chart has room for +ROWS = 40 # bars: the rows a 1280x800 image has room for; more make it taller (draw_bars) +LIMITS = {'shorts': 24} # the rows a chart of a fixed size has room for def trimmed(chart): """The chart with no more rows than it has room for (LIMITS): over that, the first half and the last, a gap diff --git a/lib/reports/__init__.py b/lib/reports/__init__.py index c62a590f..5ce4f3fe 100644 --- a/lib/reports/__init__.py +++ b/lib/reports/__init__.py @@ -1,16 +1,6 @@ """What finbot says. A module per scheduled report: cal, cleanup (of the cache), milestone, performance, podcasts, price, rating, reminder, shorts, trades, and sharesight_warm and yahoo_warm (which fill the cache), each with job(argv), which parses its schedule line, and run(dest=None, **kwargs), which runs it, to the chat that asked (dest) -or else the default chats; the top-level script of the same name runs it from the command line. And the chat-only -reports: about (.help, .jobs), history (.history, .compare), holdings (.holdings, .who, .pie, .watchlist), profile -(.SYMBOL) and rankings (.marketcap, .pe and the rest).""" -import os -import sys - - -def cli(path): - """Run the report a top-level script is named for (price.py: lib/reports/price.py) with the script's arguments.""" - import importlib - from lib import delivery - report = importlib.import_module('lib.reports.' + os.path.basename(path).removesuffix('.py')) - delivery.cli(sys.argv[0], report.job, report.run) +or else the default chats; the scheduler runs them in the bot's process, and `.jobs run` runs one now. And the +chat-only reports: about (.help, .jobs), history (.history, .compare), holdings (.holdings, .who, .pie, .watchlist), +profile (.SYMBOL) and rankings (.marketcap, .pe and the rest).""" diff --git a/lib/reports/about.py b/lib/reports/about.py index b794220f..ea6c8dbb 100644 --- a/lib/reports/about.py +++ b/lib/reports/about.py @@ -1,53 +1,69 @@ -"""What finbot can do: the .help text, and the .jobs listing of the schedule (with running a job from it).""" +"""What finbot can do: the .help text, and the .jobs listing of a chat's jobs and the reports a job can run.""" from lib import buttons +from lib import config from lib import formats from lib import jobs -def prepare_jobs_payload(service, run=None, tappable=False): - """`.jobs`: the scheduled commands, each once and numbered, then the system jobs (HIDDEN) under their own heading, - unnumbered: they run on their schedule only. With `run` (a number in that list, or a command's text) that command - run now, as cron would. Returns (payload, buttons): with tappable (the chat takes buttons) a button per numbered - command, two to a row.""" +def prepare_jobs_payload(service, chat, action=None, tappable=False): + """`.jobs`: a chat's jobs (by its key), numbered, each with its line, whether it is on and its next run, in UTC and + in the timezone preference; then the system jobs, under their own heading and unnumbered: they run on their + schedule only; then the subcommands. With action ('run' or 'toggle'), the list of what a tap does, and where the + chat takes buttons (tappable) a button per job that runs it now or switches it on or off, two to a row. Returns + (payload, buttons).""" scheduler = jobs.current if scheduler is None: - return ['Scheduled jobs are off (scheduler=False)'], None - if run: - return [run_job_line(scheduler, run)], None - commands, system = scheduler.commands(), scheduler.commands(hidden=True) - if not commands and not system: - return ['No scheduled jobs'], None - shown = lambda label: jobs.display(label) + (' (running)' if label in scheduler.busy else '') - payload = [] - if commands: - payload = [service.bold('Scheduled commands:')] + [f"{number}. {shown(label)}" - for number, (label, _, _) in enumerate(commands, 1)] - if system: - payload += ([''] if payload else []) + [service.bold('System:')] + [shown(label) for label, _, _ in system] - if commands: - payload.append("\nRun one with .jobs run NUMBER" + (' or tap it' if tappable else '')) + return ['No schedule is running'], None + own = scheduler.chat_jobs(chat) + running = scheduler.running(chat) + payload = [service.bold('Jobs here:')] + [job_entry(service, scheduler, number, job, running) + for number, job in enumerate(own, 1)] if own else ['No jobs here yet: add one with .jobs add'] + system = scheduler.system_jobs() + payload += ['', service.bold('System:')] + [f"{job.when} {job.text}" + (' (running)' if any( + label in running for label, _, _ in job.commands) else '') for job in system] + if action and own: + payload.append('\n' + ('Tap one to run it now' if action == 'run' else 'Tap one to switch it on or off') + if tappable else f"\nSend .jobs {action} NUMBER") + elif not action: + payload += ['', '.jobs run [NUMBER]: run one now', '.jobs toggle [NUMBER]: switch one on or off', + '.jobs add [CRON_TZ=ZONE] TIME REPORT: add one (.jobs add lists the reports)', + '.jobs del NUMBER: delete one'] markup = None - if tappable and commands: - row = [buttons.job_button(number, jobs.display(label), jobs.checksum(label)) - for number, (label, _, _) in enumerate(commands, 1)] + if action and tappable and own: + row = [buttons.job_button(number, job.text, jobs.checksum(job.key), action) + for number, job in enumerate(own, 1)] markup = [row[i:i + 2] for i in range(0, len(row), 2)] return payload, markup -def run_job_line(scheduler, text): - """Run a scheduled command now and say how it went in one line.""" - try: - return job_line(scheduler, *scheduler.run_now(text)) - except jobs.ScheduleError as e: - return str(e) +def job_entry(service, scheduler, number, job, running=()): + """'1. */20 * * * Mon-Fri trades: on, next Thu 14:20 UTC (Fri 01:20 AEDT)': a job in .jobs.""" + state = 'running' if any(label in running for label, _, _ in job.commands) else 'on' if job.on else 'off' + when = scheduler.next_run(job) + if when is None: + return f"{number}. {job.line}: {state}" + utc, local = when.astimezone(jobs.UTC), when.astimezone(formats.zone()) + shown = f"{utc:%a %H:%M} UTC" + ('' if local.utcoffset() == utc.utcoffset() else f" ({local:%a %H:%M %Z})") + return f"{number}. {job.line}: {state}, next {shown}" -def job_line(scheduler, label, outcome): - """'3. trades.py: done': a command's number in the .jobs list, its name, and how its run went.""" - number = [command[0] for command in scheduler.commands()].index(label) + 1 - return f"{number}. {jobs.display(label)}: {outcome}" +def job_line(number, job, outcome): + """'3. trades: done': a job's number in the .jobs list, its reports, and how its run went.""" + return f"{number}. {job.text}: {outcome}" + +def prepare_reports(service): + """`.jobs add` alone: the reports a job can run, each with what it takes, and examples.""" + names = [name for name in jobs.SCRIPTS if name not in jobs.SYSTEM_SCRIPTS] + return [service.bold('Reports a job can run:')] + [f"{name} {jobs.SCRIPTS[name].ARGUMENTS}".strip() + for name in sorted(names)] + ['', service.bold('For example:'), '.jobs add 29 21 * * Mon-Fri price interday', + '.jobs add 28 21 * * Sat cal earnings; price 7 (several in turn)', + '.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession (UTC unless it says)'] def prepare_help(service, botName, sections): """The .help text: each section's command lines under its heading (worker.help_sections() has them), then the bot's - @name as another way to start a command. botName is the bot's @name.""" - example = '<' + botName + '> trades' if service == 'slack' else botName + ' price' + @name as another way to start a command. botName is the bot's @name. On Discord the lines are slash commands, and + the other ways, which are messages, work only over the gateway.""" + if service == 'discord': + example = None if config.env.discordOutgoingWebhook else f'.price, or <{botName}> price' + else: + example = '<' + botName + '> trades' if service == 'slack' else botName + ' price' return formats.sections(*([service.bold(heading + ":")] + lines for heading, lines in sections), - [service.bold("Alternative syntax:"), example], + [service.bold("Alternative syntax:"), example] if example else [], [f"Full help on {service.link('https://github.com/robdevops/finbot', 'GitHub')}"]) diff --git a/lib/reports/cal.py b/lib/reports/cal.py index 35140379..4dad9c4d 100644 --- a/lib/reports/cal.py +++ b/lib/reports/cal.py @@ -1,4 +1,4 @@ -""".earnings and .dividend: upcoming earnings and ex-dividend dates (scheduled: cal.py earnings | ex-dividend).""" +""".earnings and .dividend: upcoming earnings and ex-dividend dates (scheduled: cal earnings | ex-dividend).""" import datetime from lib import config @@ -17,14 +17,14 @@ def primary_listings(): 'ex-dividend': ('ex_dividend_date', 'Upcoming ex-dividends')} # -> (field, heading) def run(dest=None, *, days=None, specific_stock=None, event='earnings'): - """Post the earnings or ex-dividend dates ahead (`event`) for the tracked stocks or one (Yahoo's symbol): to the - chat that asked (dest), else to the default chats. The scheduled earnings report is pinned where the network can; a - reply is not, or the chat would collect pinned copies.""" - interactive = dest is not None + """Post the earnings or ex-dividend dates ahead (`event`) for the stocks a chat follows or one (Yahoo's symbol): to + the chat that asked or whose job it is (dest), else to the home chats. The scheduled earnings report is pinned where + the network can; a reply is not, or the chat would collect pinned copies.""" + interactive = delivery.interactive(dest) if days is None: days = config.prefs.future_days - events = gather(watchlist.covered(specific_stock), days, specific_stock, event) for chat in delivery.targets(dest): + events = gather(watchlist.covered(chat.key, specific_stock), days, specific_stock, event) payload = report(chat.dialect, events, days, specific_stock, interactive, event) answers = delivery.deliver(chat, delivery.Report(payload)) if event == 'earnings' and not specific_stock and not interactive and hasattr(chat, 'pin'): @@ -53,22 +53,21 @@ def gather(tickers, days, specific_stock, event): return sorted(found, key=lambda item: item[0]) def primary_listing(ticker, data, field): - """(the primary listing's ticker, its Yahoo details) where an ADR or a second listing without a `field` of its own - finds one, or (None, None): the listing in finbot_adr.json, the NZX for an ASX ticker reporting in NZD, an exchange - suffix for a foreign company's US ticker, else the ticker's own details. It must be the same company, with the - `field`.""" - primary = primary_listings().get(ticker) - if primary is None: - nzd = ticker.endswith('.AX') and data.get('financialCurrency') == 'NZD' - primary = ticker.replace('.AX', '.NZ') if nzd else ticker # fixes GTK.AX and SKO.AX - details = yahoo.fetch_detail(primary).get(primary, {}) - country = details.get('profile_country') # only in the details - if country in markets.YAHOO_COUNTRY and '.' not in primary: - primary = primary + '.' + markets.YAHOO_COUNTRY[country] # fixes ASML and INFY - details = yahoo.fetch_detail(primary).get(primary, {}) - if details.get('profile_title') != data['profile_title'] or field not in details: # another company, or no date - return None, None - return primary, details + """(a listing's ticker, its Yahoo details) with a `field` that a ticker's quote lacks, or (None, None). The ticker's + own details: what is held is that listing, so its own date counts, and a foreign company's US listing (ARM, CLBT) + is never swapped for its home market's (ARM.L, CLBT.TA). Only when it has none, the listing that finbot_adr.json + names, or the NZX's for an ASX ticker reporting in NZD (GTK.AX, SKO.AX).""" + own = yahoo.fetch_detail(ticker).get(ticker, {}) + listings = [ticker] + if mapped := primary_listings().get(ticker): + listings.append(mapped) + elif ticker.endswith('.AX') and data.get('financialCurrency') == 'NZD': + listings.append(ticker.replace('.AX', '.NZ')) + for listing in listings: + details = own if listing == ticker else yahoo.fetch_detail(listing).get(listing, {}) + if details.get('profile_title') == data['profile_title'] and field in details: # the same company, with a date + return listing, details + return None, None def report(service, events, days, specific_stock, interactive, event): """The events in one network's markup, grouped by date under a heading (one stock's carry their dates instead).""" @@ -76,7 +75,7 @@ def report(service, events, days, specific_stock, interactive, event): blocks = [] for date in sorted({when.date() for when, *_ in events}): day = f"{date:%b %d} " if specific_stock else '' # one stock's events carry their date - lines = [f"{flag} {day}{title} ({links.yahoo_link(ticker, service)})" + lines = [f"{flag} {day}{formats.short_name(title)} ({links.yahoo_link(ticker, service)})" for when, flag, title, ticker in events if when.date() == date] blocks.append(lines if specific_stock else [service.bold(f"{date:%b %d}")] + lines) payload = sum(blocks, []) if specific_stock else formats.sections(*blocks) @@ -87,10 +86,12 @@ def report(service, events, days, specific_stock, interactive, event): payload = [f"No events found for {specific_stock}" if specific_stock else f"No events found {ahead}"] return payload +ARGUMENTS = 'earnings|ex-dividend' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case ['earnings' | 'ex-dividend' as event]: - return f'cal.py {event}', {'event': event} + return f'cal {event}', {'event': event} case _: - raise ValueError('[earnings|ex-dividend]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/cleanup.py b/lib/reports/cleanup.py index 3ffb74cb..615ad4a9 100644 --- a/lib/reports/cleanup.py +++ b/lib/reports/cleanup.py @@ -1,34 +1,47 @@ -"""Deletes old cache files (scheduled: cleanup.py [days]); with days=0 it is the cache flush.""" -import os, sys, time +"""Deletes old cache files (scheduled: cleanup [days]); with days=0 it is the cache flush. util/flush.py deletes +them by age too (remove).""" +import os, time +from lib import log from lib import config def run(*, days=30): """Delete the cache files (finbot_*) that were last written more than `days` days ago (days=0: all of them, a - flush), but not a temp file written in the last minute: that is an atomic write in flight (cache.write_file).""" + flush); see remove.""" + return remove(before=time.time() - days * 86400) + +def remove(before=None, after=None, directory=None): + """Delete the cache files (finbot_*) in directory (the cache dir) last written before the time `before`, or after + the time `after` (epoch seconds; neither: all of them), but not a temp file written in the last minute: that is an + atomic write in flight (cache.write_file). Nothing else in it, and nothing in the data dir. Returns how many.""" now = time.time() - cutoff = now - days * 86400 removed = 0 try: - entries = list(os.scandir(config.env.cache_dir)) + entries = list(os.scandir(directory or config.env.cache_dir)) except FileNotFoundError: return 0 for entry in entries: - if entry.is_file(follow_symlinks=False) and entry.name.startswith('finbot_') and entry.stat().st_mtime < cutoff: - if entry.name.endswith('.tmp') and entry.stat().st_mtime > now - 60: - continue - try: - os.unlink(entry.path) - removed += 1 - except OSError as e: - print("cleanup: can't remove", entry.path, e, file=sys.stderr) + if not (entry.is_file(follow_symlinks=False) and entry.name.startswith('finbot_')): + continue + written = entry.stat().st_mtime + if (before is not None and written >= before) or (after is not None and written <= after): + continue + if entry.name.endswith('.tmp') and written > now - 60: + continue + try: + os.unlink(entry.path) + removed += 1 + except OSError as e: + log.warn("cleanup: can't remove", entry.path, e) return removed +ARGUMENTS = '[DAYS]' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case []: - return 'cleanup.py', {} + return 'cleanup', {} case [days] if days.isdigit(): - return f'cleanup.py {days}', {'days': int(days)} + return f'cleanup {days}', {'days': int(days)} case _: - raise ValueError('[days]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/history.py b/lib/reports/history.py index 9b972f23..6f7f7c8a 100644 --- a/lib/reports/history.py +++ b/lib/reports/history.py @@ -1,4 +1,5 @@ """Price history: .history (one stock over every period) and .compare (two to six stocks, rebased).""" +import bisect import datetime from lib import answer from lib import buttons @@ -10,36 +11,35 @@ def prepare_compare(service, tickers, period_days=None): """(caption lines, chart spec): 2-6 tickers (Yahoo's symbols) as rebased % change over the last `period_days`, else 10 years (as Max is on every chart), from no earlier than the first date every one of them covers.""" - import pandas as pd names, exchanges, closes = {}, {}, {} for ticker, history in zip(tickers, yahoo.fetch_each(yahoo.price_series, tickers)): # downloaded at the same time if isinstance(history, Exception): raise history names[ticker], exchanges[ticker], closes[ticker] = history - limiting = max(closes, key=lambda t: closes[t].index[0]) # ticker with the shortest history - common_start = closes[limiting].index[0] - requested = pd.Timestamp(datetime.datetime.now().date() - datetime.timedelta(days=period_days or 3655)) + limiting = max(closes, key=lambda t: closes[t][0][0]) # ticker with the shortest history + common_start = closes[limiting][0][0] + requested = datetime.datetime.now().date() - datetime.timedelta(days=period_days or 3655) start = max(common_start, requested) note = f" (limited by {limiting} history)" if period_days and requested < common_start else '' cropped = [] for ticker in tickers: - series = closes[ticker] - base = series[series.index <= start] - base_value = base.iloc[-1] if len(base) else series.iloc[0] # last close on/before the start date - series = pd.concat([pd.Series([base_value], index=[start]), series[series.index > start]]) - cropped.append((ticker, series)) - if any(len(series) < 2 for _, series in cropped): + rows = closes[ticker] + before = bisect.bisect_right([day for day, _ in rows], start) # the closes on or before the start date + base_value = rows[before - 1][1] if before else rows[0][1] # the last of them, else the first close + cropped.append((ticker, ((start, base_value), *rows[before:]))) + if any(len(rows) < 2 for _, rows in cropped): raise answer.Answer("not enough price history in that period to compare") - span_days = (datetime.datetime.now().date() - start.date()).days + span_days = (datetime.datetime.now().date() - start).days period_label = formats.days_english(period_days) if period_days and not note else f"{span_days} days" - by_performance = sorted(cropped, key=lambda item: item[1].iloc[-1] / item[1].iloc[0], reverse=True) + growth = lambda rows: rows[-1][1] / rows[0][1] # last close over first + by_performance = sorted(cropped, key=lambda item: growth(item[1]), reverse=True) title = ' vs '.join(formats.short_ticker(t) for t, _ in by_performance) # best first; colours keep input order subtitle = f"% change over {period_label}, since {start:%-d %b %Y}" + note - lines = tuple((formats.short_ticker(t), tuple((f"{day:%Y-%m-%d}", float(close)) for day, close in series.items())) - for t, series in cropped) + lines = tuple((formats.short_ticker(t), tuple((f"{day:%Y-%m-%d}", float(close)) for day, close in rows)) + for t, rows in cropped) chart = render.Chart('compare', title, subtitle, lines) caption = [service.bold(f"{period_label}, since {start:%-d %b %Y}") + note] - results = sorted(((series.iloc[-1] / series.iloc[0] - 1) * 100, t) for t, series in cropped) + results = sorted(((growth(rows) - 1) * 100, t) for t, rows in cropped) for pct, t in reversed(results): link = links.finance_link(t, exchanges[t], service, days=span_days, brief=False) caption.append(f"{formats.get_emoji(pct)} {names[t]} ({link}): {formats.signed_percent(pct, 1)}") diff --git a/lib/reports/holdings.py b/lib/reports/holdings.py index d89c6e6a..0d2336df 100644 --- a/lib/reports/holdings.py +++ b/lib/reports/holdings.py @@ -11,7 +11,8 @@ def holding_line(holding, service, text=None): """'🇦🇺 Title (CODE)' for a Sharesight holding, CODE (or `text`) linking to the holding on Sharesight.""" link = service.link(f"https://portfolio.sharesight.com/holdings/{holding['holding_id']}/dashboard", holding['code'] if text is None else text) - return f"{markets.flag_from_market(holding['market_code'])} {formats.transform_title(holding['name'])} ({link})" + name = formats.short_name(formats.transform_title(holding['name'])) + return f"{markets.flag_from_market(holding['market_code'])} {name} ({link})" def prepare_holdings_payload(portfolio_name, service): """The holdings of the portfolio called `portfolio_name`. Raises sharesight.PortfolioNotFound.""" @@ -60,12 +61,12 @@ def prepare_pie(portfolio_name, by, service, max_slices=8): subtitle = f"{len(totals)} {noun[0] if len(totals) == 1 else noun[1]}" return payload, render.Chart('pie', title, subtitle, tuple(rows)) -def prepare_watchlist(service, user, action=None, ticker=None): - """The .watchlist reply: the companies watched, or one line on adding ('add') or removing ('delete') a ticker - (Yahoo's symbol, as yahoo.symbol gives it).""" +def prepare_watchlist(service, chat, user, action=None, ticker=None): + """The .watchlist reply in a chat (by its key): the companies it watches, or one line on adding ('add') or removing + ('delete') a ticker (Yahoo's symbol, as yahoo.symbol gives it).""" if action: - return [watchlist_change(service, user, action, ticker)] - watched = watchlist.tickers() + return [watchlist_change(service, chat, user, action, ticker)] + watched = watchlist.tickers(chat) market_data = yahoo.fetch(watched) if watched else {} lines = [links.stock_line(ticker, quote, service) for ticker, quote in sorted(market_data.items(), key=lambda item: item[1]['profile_title'].lower())] @@ -73,15 +74,17 @@ def prepare_watchlist(service, user, action=None, ticker=None): return ['Watchlist is empty. Try ".watchlist add SYMBOL" to create it'] return [f"Hi {user}, I'm currently tracking:"] + lines -def watchlist_change(service, user, action, ticker): - """The reply to adding or removing `ticker` (Yahoo's symbol).""" +def watchlist_change(service, chat, user, action, ticker): + """The reply to adding `ticker` (Yahoo's symbol) to a chat's watchlist, or removing it.""" quote = yahoo.fetch([ticker]).get(ticker) or {} link = links.finance_link(ticker, quote.get('profile_exchange') or ticker, service) described = service.bold(f"{quote['profile_title']} ({link})" if quote.get('profile_title') else link) if action == 'delete': - if not watchlist.remove(ticker): + if not watchlist.remove(chat, ticker): return f"Beep Boop. I could not find {service.bold(ticker)} to remove it" return f"Ok {user}, I deleted {described}" if not quote: return f"Beep Boop. I could not find {service.bold(ticker)} to add it" - return f"Ok {user}, I added {described}" if watchlist.add(ticker) else f"{user}, I'm already tracking {described}" + if not watchlist.add(chat, ticker): + return f"{user}, I'm already tracking {described}" + return f"Ok {user}, I added {described}" diff --git a/lib/reports/milestone.py b/lib/reports/milestone.py index d9bffd17..7ad8dc9d 100644 --- a/lib/reports/milestone.py +++ b/lib/reports/milestone.py @@ -1,4 +1,4 @@ -"""Record highs and lows (all-time) among the stocks followed (scheduled: milestone.py).""" +"""Record highs and lows (all-time) among the stocks followed (scheduled: milestone).""" from lib import state from lib import delivery from lib import formats @@ -10,25 +10,26 @@ # - need to handle stock splits: flush cache if we see a split in the data # https://finance.yahoo.com/quote/TSLA/history?period1=0&period2=1707609600&filter=split -STATE_FILE = 'finbot_milestone.json' # {ticker: {'all_time_high': ..., 'all_time_low': ...}} +STATE_FILE = 'finbot_milestone.json' # each chat's {ticker: {'all_time_high': ..., 'all_time_low': ...}} def run(dest=None): - """Post the stocks followed that set an all-time high or low to the default chats (or dest), and record the new + """Post the stocks a chat follows that set an all-time high or low, against its record, to each of the run's chats + (dest, a chat's job or the chat that asked; else the home chats); a scheduled run records each chat's new records.""" - market_data, highs, lows, changed = gather() + interactive = delivery.interactive(dest) for chat in delivery.targets(dest): + tickers = watchlist.with_holdings(chat.key) + market_data = yahoo.fetch(tickers) + highs, lows, changed = compare(tickers, market_data, state.load_for(chat.key, STATE_FILE, {})) delivery.deliver(chat, delivery.Report(report(chat.dialect, market_data, highs, lows))) - if changed: - with state.edit(STATE_FILE, {}) as recorded: - recorded.update(changed) + if changed and not interactive: + with state.edit_for(chat.key, STATE_FILE, {}) as recorded: + recorded.update(changed) -def gather(): - """(the quotes, the tickers at a record high, those at a record low, {ticker: its records after this run}) of the - stocks followed: each one's 52-week range against its all-time record, which a first sighting takes from its - history.""" - tickers = watchlist.with_holdings() - market_data = yahoo.fetch(tickers) - records = state.load(STATE_FILE, {}) +def compare(tickers, market_data, records): + """(the tickers at a record high, those at a record low, {ticker: its records after this run}): each one's 52-week + range (market_data, the quotes) against its all-time record in a chat's records, which a first sighting takes from + its history.""" highs, lows, changed = [], [], {} for ticker in tickers: try: @@ -49,7 +50,7 @@ def gather(): highs.append(ticker) if newlow < oldlow: lows.append(ticker) - return market_data, highs, lows, changed + return highs, lows, changed def report(service, market_data, highs, lows): """The records in one network's markup: the record highs, then the record lows, each list sorted.""" @@ -59,6 +60,11 @@ def line(ticker): return formats.sections(highs and [service.bold('Record high:')] + sorted(line(ticker) for ticker in highs), lows and [service.bold('Record low:')] + sorted(line(ticker) for ticker in lows)) +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: milestone.py takes none.""" - return 'milestone.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); milestone takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'milestone', {} diff --git a/lib/reports/performance.py b/lib/reports/performance.py index f45f1c2b..2f05cecd 100644 --- a/lib/reports/performance.py +++ b/lib/reports/performance.py @@ -1,6 +1,6 @@ -""".performance: each portfolio's return over a period, beside two benchmarks (scheduled: performance.py [days]).""" -import sys +""".performance: each portfolio's return over a period, beside two benchmarks (scheduled: performance [days]).""" +from lib import log from lib import config from lib import sharesight from lib import delivery @@ -22,7 +22,7 @@ def benchmark(ticker, market, text, past_days, service): try: percent = yahoo.price_history(ticker, days=past_days)[0][past_days] except Exception as e: # the rest of the report still goes out - print(ticker, e, file=sys.stderr) + log.warn(ticker, e) return f"⚠️ {text}: no price history", None link = links.finance_link(ticker, market, service=service, days=past_days, brief=True, text=text) return f"{formats.get_emoji(percent)} {link} {formats.signed_percent(percent, 2)}", percent @@ -49,8 +49,8 @@ def report(service, past_days, portfolio_select=None, performance=None): def run(dest=None, *, past_days=None, user='', portfolio_select=None): """Post each portfolio's performance over past_days (the past_days preference when None), or the one named, beside - the benchmarks: to the chat that asked (dest), else to the default chats, with its chart and period buttons where - they can go.""" + the benchmarks: to the chat that asked or whose job it is (dest), else to the home chats, with its chart and period + buttons where they can go.""" if past_days is None: past_days = config.prefs.past_days performance = gather(past_days, portfolio_select) @@ -61,12 +61,14 @@ def run(dest=None, *, past_days=None, user='', portfolio_select=None): markup = buttons.keyboard('f', [portfolio_select or ''], buttons.period_for_days(past_days)) delivery.deliver(chat, delivery.Report(payload, graph, markup)) +ARGUMENTS = '[DAYS]' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case []: - return 'performance.py', {} + return 'performance', {} case [days] if days.isdigit(): - return f'performance.py {days}', {'past_days': int(days)} + return f'performance {days}', {'past_days': int(days)} case _: - raise ValueError('[integer]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/podcasts.py b/lib/reports/podcasts.py index db9b401f..a7f7cc79 100644 --- a/lib/reports/podcasts.py +++ b/lib/reports/podcasts.py @@ -1,4 +1,4 @@ -"""New episodes of a podcast feed (scheduled: podcasts.py "").""" +"""New episodes of a podcast feed (scheduled: podcasts FEED_URL).""" import hashlib import string import xml.etree.ElementTree as ET @@ -14,8 +14,10 @@ } NEWEST = 5 # episodes posted per run at most: the other new ones (on a first run, say) are only recorded as seen +SEEN_FILES = 'finbot_podcast_seen_*.json' # each feed's, as get_seen_filename names them + def get_seen_filename(url): - """The state file, in the data directory, of the episodes of this feed already seen.""" + """The state file, in the data directory, of the episodes of this feed already seen (by each chat).""" return f"finbot_podcast_seen_{hashlib.md5(url.encode()).hexdigest()}.json" def fetch_and_parse_feed(url): @@ -65,42 +67,43 @@ def media_link(item): return None def fetch_new_episodes(feed_url, seen): - """(the NEWEST newest episodes whose guid is not in `seen`, newest first; the guids of all the new ones).""" + """The feed's episodes whose guid is not in `seen`, newest first, each with its guid.""" channel = fetch_and_parse_feed(feed_url).find("channel") if channel is None: raise RuntimeError("No element found") podcast_name = (channel.findtext("title") or "Podcast").strip() - episodes, guids = [], set() + episodes = [] for item in channel.findall("item"): guid = item.findtext("guid") or item.findtext("link") media_url = media_link(item) if not guid or guid in seen or not media_url: continue when = published(item.findtext("pubDate")) - episodes.append({"podcast": podcast_name, "title": (item.findtext("title") or "Untitled").strip(), + episodes.append({"guid": guid, "podcast": podcast_name, "title": (item.findtext("title") or "Untitled").strip(), "url": media_url, "ago": how_long_ago(when), "duration": format_duration(item.findtext("itunes:duration", namespaces=NAMESPACES)), "published": when or datetime.min.replace(tzinfo=timezone.utc)}) - guids.add(guid) episodes.sort(key=lambda episode: episode["published"], reverse=True) - return episodes[:NEWEST], guids + return episodes def run(dest=None, *, feed_url): - """Post the feed's newest episodes not posted before to the default chats (or dest), then record every new one as - seen.""" + """Post to each of the run's chats (dest, a chat's job; else the home chats) the feed's newest episodes not posted + to it before, then record every new one as seen by it.""" seen_file = get_seen_filename(feed_url) - seen = set(state.load(seen_file, [])) + chats = delivery.targets(dest) + seen = {chat.key: set(state.load_for(chat.key, seen_file, [])) for chat in chats} try: - episodes, guids = fetch_new_episodes(feed_url, seen) + episodes = fetch_new_episodes(feed_url, set.intersection(*seen.values())) # every chat has seen those except Exception as e: raise RuntimeError(f"fetching feed {feed_url}: {e}") - if not guids: - return - for chat in delivery.targets(dest): - delivery.deliver(chat, delivery.Report(report(chat.dialect, episodes))) - with state.edit(seen_file, []) as recorded: # last, so that a failure above leaves these episodes for next run - recorded[:] = sorted(guids.union(recorded)) + for chat in chats: + new = [episode for episode in episodes if episode['guid'] not in seen[chat.key]] + if not new: + continue + delivery.deliver(chat, delivery.Report(report(chat.dialect, new[:NEWEST]))) + with state.edit_for(chat.key, seen_file, []) as recorded: # last: a failure above leaves them for next run + recorded[:] = sorted({episode['guid'] for episode in new}.union(recorded)) def report(service, episodes): """The episodes in one network's markup, each linked to its media, with its length and age.""" @@ -112,11 +115,13 @@ def report(service, episodes): payload.append(f"📣 {podcast_name}: {service.link(episode['url'], title)}{duration} ({episode['ago']})") return payload +ARGUMENTS = 'FEED_URL' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case [feed_url]: # the label matches the seen file's name - return 'podcasts.py ' + hashlib.md5(feed_url.encode()).hexdigest()[:8], {'feed_url': feed_url} + return 'podcasts ' + hashlib.md5(feed_url.encode()).hexdigest()[:8], {'feed_url': feed_url} case _: - raise ValueError('""') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/price.py b/lib/reports/price.py index 3a904e2d..d78d38cd 100644 --- a/lib/reports/price.py +++ b/lib/reports/price.py @@ -1,9 +1,9 @@ -""".price, .session, .premarket, .postmarket: price moves past a threshold (scheduled: price.py midsession | interday | +""".price, .session, .premarket, .postmarket: price moves past a threshold (scheduled: price midsession | interday | premarket | close | ).""" import dataclasses import datetime -import sys +from lib import log from lib import config from lib import delivery from lib import formats @@ -15,16 +15,15 @@ from lib import buttons from lib.reports import history -def gather(specific_stock=None, days=None): - """(tickers, market data) for a price report on the stocks followed or one (Yahoo's symbol): Yahoo's quotes, and - over `days` each one's change as percent_change_period, from its Yahoo history (the change its chart shows), the - histories fetched together up front.""" - tickers = watchlist.covered(specific_stock) +def gather(tickers, specific_stock=None, days=None): + """The market data of a price report on tickers (the stocks a chat follows, or the one asked about, Yahoo's + symbol): Yahoo's quotes, and over `days` each one's change as percent_change_period, from its Yahoo history (the + change its chart shows), the histories fetched together up front.""" market_data = yahoo.fetch(tickers) if config.prefs.demote_volatile: # trigger_bar compares market caps, in US dollars yahoo.add_usd_market_caps(market_data) if not days: - return tickers, market_data + return market_data if not specific_stock: yahoo.prefetch_history(list(market_data)) for ticker in list(market_data): @@ -33,10 +32,10 @@ def gather(specific_stock=None, days=None): except RuntimeError as e: # no history: the one stock asked about says so, a list leaves it out if specific_stock: raise - print(ticker, e, file=sys.stderr) + log.warn(ticker, e) continue market_data[ticker]['percent_change_period'] = changes[days] - return tickers, market_data + return market_data def needs_beta(quote): """Whether trigger_bar must know this ticker's beta: a small cap between US$150M and US$1B, which counts as volatile @@ -98,13 +97,13 @@ def change(mode, quote, now, interactive): return quote.get('percent_change_premarket', quote.get('percent_change_postmarket')) return quote.get('percent_change_period') -def moves(mode='interday', threshold=None, days=None, specific_stock=None, interactive=False): - """The Moves of a price report in a mode of MODES ('period': over `days`): each stock followed whose move is - `threshold` percent or more, or the one stock asked about, whatever its move. A scheduled report keeps them only - when one passes its trigger_bar.""" +def moves(tickers, mode='interday', threshold=None, days=None, specific_stock=None, interactive=False): + """The Moves of a price report on tickers (watchlist.covered: the stocks a chat follows, or the one asked about) in + a mode of MODES ('period': over `days`): each whose move is `threshold` percent or more, or the one stock asked + about, whatever its move. A scheduled report keeps them only when one passes its trigger_bar.""" if threshold is None: threshold = config.prefs.price_percent - _, market_data = gather(specific_stock, days) + market_data = gather(tickers, specific_stock, days) now = datetime.datetime.now() rows, exchanges = [], set() for ticker, quote in market_data.items(): @@ -128,7 +127,7 @@ def report(service, found, mode='interday', threshold=None, days=None, top=None, or more in a mode of MODES ('period': over `days`), or the top and bottom `top`.""" if threshold is None: threshold = config.prefs.price_percent - lines = [f"{formats.get_emoji(percent)} {quote['profile_title']} " + lines = [f"{formats.get_emoji(percent)} {formats.short_name(quote['profile_title'])} " f"({links.finance_link(ticker, quote['profile_exchange'], service, days=days)}) " f"{formats.signed_percent(round(percent))}" for ticker, percent, quote in found.rows] if not lines: @@ -172,12 +171,11 @@ def nothing(mode, threshold, user, ticker, in_session, days=None): return f"{user}, no price movements meet threshold {threshold:g}%" def run(dest=None, *, mode='interday', threshold=None, days=None, top=None, user='', specific_stock=None): - """Post the price report (report() has the modes): to the chat that asked (dest), else to the default chats, with - its chart and period buttons where they can go.""" - interactive = dest is not None + """Post the price report (report() has the modes) on the stocks each chat follows: to the chat that asked or whose + job it is (dest), else to the home chats, with its chart and period buttons where they can go.""" + interactive = delivery.interactive(dest) if threshold is None: threshold = config.prefs.price_percent - found = moves(mode, threshold, days, specific_stock, interactive) def markup(chat): """A chart's period buttons, where the chat takes buttons: the daily and N-day lists, but not the intraday ones.""" @@ -188,15 +186,18 @@ def markup(chat): buttons.period_for_days(days, 'd' if not days else None)) return None for chat in delivery.targets(dest): + found = moves(watchlist.covered(chat.key, specific_stock), mode, threshold, days, specific_stock, interactive) payload, graph = report(chat.dialect, found, mode, threshold, days, top, user, specific_stock, interactive) delivery.deliver(chat, delivery.Report(payload, graph, markup(chat) if graph else None)) +ARGUMENTS = 'midsession|interday|premarket|close|DAYS' # what a scheduled run takes + def job(argv): """A scheduled run's arguments: (label, run's keyword arguments); a bad one raises ValueError with the usage.""" match argv: case ['midsession' | 'interday' | 'premarket' | 'close' as mode]: - return f'price.py {mode}', {'mode': mode} + return f'price {mode}', {'mode': mode} case [days] if days.isdigit(): - return f'price.py {days}', {'mode': 'period', 'threshold': 0, 'days': int(days), 'top': 10} + return f'price {days}', {'mode': 'period', 'threshold': 0, 'days': int(days), 'top': 10} case _: - raise ValueError('[midsession|interday|premarket|close|days (int)]') + raise ValueError(ARGUMENTS) diff --git a/lib/reports/profile.py b/lib/reports/profile.py index e79c2f15..d321b836 100644 --- a/lib/reports/profile.py +++ b/lib/reports/profile.py @@ -1,7 +1,7 @@ """The .SYMBOL report: what Yahoo and Simply Wall St know about one company.""" from itertools import pairwise import datetime -import sys +from lib import log from lib import config from lib import formats from lib import links @@ -194,7 +194,7 @@ def changes(service, data, ticker, state): try: history, _ = yahoo.price_history(ticker, quote=data) # the percentages only: the profile has no chart except RuntimeError as e: # no chart data: the profile goes out without these lines - print(ticker, e, file=sys.stderr) + log.warn(ticker, e) history = {} for interval in ('5Y', '1Y', '1M', '1D'): if interval in history: diff --git a/lib/reports/rankings.py b/lib/reports/rankings.py index 908ca8c9..f296bfb0 100644 --- a/lib/reports/rankings.py +++ b/lib/reports/rankings.py @@ -1,6 +1,6 @@ """Ranked lists of the stocks followed (.marketcap, .pe, .forward pe and .peg, .beta, .buy and .sell), and one stock's market cap or ratio.""" -import sys +from lib import log from lib import answer from lib import config from lib import watchlist @@ -21,14 +21,14 @@ def marketcap_line(service, ticker): cap += f" (USD {formats.human_units(data['market_cap_usd'])})" return f"{links.stock_line(ticker, data, service)} mkt cap: {cap}" -def marketcap_list(service, bottom=False, length=15): - """The `length` stocks followed of largest (or, `bottom`, smallest) market cap, compared in US dollars at Yahoo's - exchange rates: a market cap in won or yen is a far bigger number.""" +def marketcap_list(service, chat, bottom=False, length=15): + """The `length` stocks a chat follows (by its key) of largest (or, `bottom`, smallest) market cap, compared in US + dollars at Yahoo's exchange rates: a market cap in won or yen is a far bigger number.""" caps = [] # (market cap in US dollars, line) - for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(watchlist.with_holdings())).items(): + for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(watchlist.with_holdings(chat))).items(): if 'market_cap_usd' not in data: why = "no market cap" if 'market_cap' not in data else f"no exchange rate for {data.get('currency')}" - print(ticker, why, file=sys.stderr) + log.debug(ticker, why) continue cap = formats.human_units(data['market_cap_usd']) caps.append((data['market_cap_usd'], f"{links.stock_line(ticker, data, service)} mkt cap: {cap}")) @@ -38,13 +38,13 @@ def marketcap_list(service, bottom=False, length=15): heading = f"{'Bottom' if bottom else 'Top'} {length} tracked stocks by market cap (USD)" return [service.bold(heading)] + [line for _, line in caps[:length]] -def prepare_rating_payload(service, action, length=15): - """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by min_analysts or more analysts - (fewer, and the consensus says little: ASX small caps covered by two to six brokers are nearly all a perfect 1.0 - buy), the strongest rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give - first.""" +def prepare_rating_payload(service, chat, action, length=15): + """(payload, chart) of the `length` stocks a chat follows (by its key) rated `action` ('buy' or 'sell') by + min_analysts or more analysts (fewer, and the consensus says little: ASX small caps covered by two to six brokers + are nearly all a perfect 1.0 buy), the strongest rating first (1 is a strong buy, 5 a strong sell); on equal + ratings, the one more analysts give first.""" found = [] # (rating, analysts, line, ticker) - for ticker, data in yahoo.fetch_details(watchlist.with_holdings()).items(): + for ticker, data in yahoo.fetch_details(watchlist.with_holdings(chat)).items(): if 'recommend' not in data or data['recommend_analysts'] < config.prefs.min_analysts: continue recommend = data['recommend'] @@ -85,12 +85,12 @@ def ratio_line(service, ticker, kind): raise answer.Answer(f"{ticker}: no {kind} ratio found") return f"{links.stock_line(ticker, data, service)} {figures}" -def ratio_list(service, kind, mode='top', length=15): - """(payload, chart): the `length` stocks followed by a ratio (kind: pe, forward pe or peg), lowest first (mode top), - highest first (bottom), or the negative ones only, lowest first (negative). A forward P/E is analysts' estimate, so - its list takes only stocks that min_analysts or more analysts cover, as .buy does.""" +def ratio_list(service, chat, kind, mode='top', length=15): + """(payload, chart): the `length` stocks a chat follows (by its key) by a ratio (kind: pe, forward pe or peg), + lowest first (mode top), highest first (bottom), or the negative ones only, lowest first (negative). A forward P/E + is analysts' estimate, so its list takes only stocks that min_analysts or more analysts cover, as .buy does.""" field = RATIOS[kind] - market_data = yahoo.fetch(watchlist.with_holdings()) + market_data = yahoo.fetch(watchlist.with_holdings(chat)) if kind == 'peg': # the PEG is in Yahoo's details market_data |= yahoo.fetch_details(market_data) covered = None # for a forward P/E list: the tickers enough analysts cover @@ -118,10 +118,11 @@ def ratio_list(service, kind, mode='top', length=15): chart = render.Chart('bars', heading, rows=bars, value_format='number') return [service.bold(heading)] + [line for _, _, line in rows], chart -def prepare_beta_payload(service): - """(payload, chart) of the stocks followed with a beta over 1.5 and a market cap under US$2B, highest beta first.""" +def prepare_beta_payload(service, chat): + """(payload, chart) of the stocks a chat follows (by its key) with a beta over 1.5 and a market cap under US$2B, + highest beta first.""" found = [] # (beta, line, chart row) - for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch_details(watchlist.with_holdings())).items(): + for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch_details(watchlist.with_holdings(chat))).items(): if 'beta' not in data: continue beta = round(data['beta'], 1) diff --git a/lib/reports/rating.py b/lib/reports/rating.py index 2670d64e..dc257e46 100644 --- a/lib/reports/rating.py +++ b/lib/reports/rating.py @@ -1,44 +1,53 @@ -"""Analysts' consensus rating changes since the last scheduled run (rating.py); .buy and .sell are in rankings.""" +"""Analysts' consensus rating changes since the last scheduled run (rating); .buy and .sell are in rankings.""" from lib import delivery +from lib import formats from lib import state from lib import links from lib import watchlist from lib import yahoo -STATE_FILE = 'finbot_rating.json' # {ticker: [action, index]} as last seen +STATE_FILE = 'finbot_rating.json' # each chat's {ticker: [action, index]} as its last scheduled run saw them def run(dest=None): - """Post the analysts' consensus ratings that changed since the last scheduled run, which records the ratings it saw: - to the chat that asked (dest), else to the default chats.""" - interactive = dest is not None - changes, seen = gather(watchlist.with_holdings()) + """Post the analysts' consensus ratings of the stocks a chat follows that changed since its last scheduled run: to + the chat that asked (dest), against its record; or, a scheduled run, to each of its chats (dest, a chat's job; else + the home chats), recording for each the ratings it saw.""" + interactive = delivery.interactive(dest) for chat in delivery.targets(dest): + market_data, seen = gather(watchlist.with_holdings(chat.key)) + changes = changed(market_data, seen, state.load_for(chat.key, STATE_FILE, {})) delivery.deliver(chat, delivery.Report(report(chat.dialect, changes, interactive))) - if seen and not interactive: # added to what is recorded: a ticker missing from this run keeps its last rating - with state.edit(STATE_FILE, {}) as ratings: - ratings.update(seen) + if seen and not interactive: # added to its record: a ticker missing from this run keeps its last rating + with state.edit_for(chat.key, STATE_FILE, {}) as ratings: + ratings.update(seen) def gather(tickers): - """(the ratings that changed since the last scheduled run, [(ticker, its details, old [action, index], new [action, - index], analysts)]; {ticker: [action, index]} of every rating seen).""" - old = state.load(STATE_FILE, {}) + """(the stocks' details; {ticker: [action, index]} of every rating seen, which counts only with its analysts).""" market_data = yahoo.fetch_details(tickers) - changes, seen = [], {} + seen = {} for ticker in tickers: + details = market_data.get(ticker, {}) try: - action = market_data[ticker]['recommend'] - index = round(float(market_data[ticker]['recommend_index']), 1) - analysts = market_data[ticker]['recommend_analysts'] + action, index = details['recommend'], round(float(details['recommend_index']), 1) except (KeyError, ValueError): continue - seen[ticker] = [action, index] + if 'recommend_analysts' in details: + seen[ticker] = [action, index] + return market_data, seen + +def changed(market_data, seen, old): + """The ratings seen that changed from a chat's record (old, {ticker: [action, index]}): [(ticker, its details, old + (action, index), new (action, index), analysts)].""" + changes = [] + for ticker, (action, index) in seen.items(): try: old_action, old_index = old[ticker] except (KeyError, TypeError, ValueError): continue if old_action and action != old_action: - changes.append((ticker, market_data[ticker], (old_action, old_index), (action, index), analysts)) - return changes, seen + changes.append((ticker, market_data[ticker], (old_action, old_index), (action, index), + market_data[ticker]['recommend_analysts'])) + return changes def report(service, changes, interactive): """The rating changes in one network's markup, sorted: 📈 when the consensus improved (its index fell: 1 is a strong @@ -49,7 +58,7 @@ def report(service, changes, interactive): ticker_link = links.finance_link(ticker, data['profile_exchange'], service, brief=False) before, after = service.bold(f'{old_index} {old_action}'), service.bold(f'{index} {action}') message = f"{before} to {after} ({analysts} analysts)" - payload.append(f"{emoji} {data['profile_title']} ({ticker_link}) changed from {message}") + payload.append(f"{emoji} {formats.short_name(data['profile_title'])} ({ticker_link}) changed from {message}") payload.sort() if payload: payload.insert(0, service.bold('Consensus rating changes:')) @@ -57,6 +66,11 @@ def report(service, changes, interactive): payload = ["No rating changes found"] return payload +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: rating.py takes none.""" - return 'rating.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); rating takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'rating', {} diff --git a/lib/reports/reminder.py b/lib/reports/reminder.py index dd05ca12..1c86763f 100644 --- a/lib/reports/reminder.py +++ b/lib/reports/reminder.py @@ -1,4 +1,4 @@ -"""Reminders for the day: the entries of reminder.json for today, and finbot's birthday (scheduled: reminder.py).""" +"""Reminders for the day: the entries of reminder.json for today, and finbot's birthday (scheduled: reminder).""" import datetime import re import zoneinfo @@ -13,7 +13,7 @@ def run(dest=None): - """Post today's reminders to the default chats (or dest), if there are any.""" + """Post today's reminders, if there are any, to the chat whose job it is (dest), else to the home chats.""" today = datetime.datetime.now(zoneinfo.ZoneInfo(config.prefs.timezone)).date() entries = config.data_json("reminder.json", []) for chat in delivery.targets(dest): @@ -55,6 +55,11 @@ def markup(line, service): return service.italic(text) if italic else text +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: reminder.py takes none.""" - return 'reminder.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); reminder takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'reminder', {} diff --git a/lib/reports/sharesight_warm.py b/lib/reports/sharesight_warm.py index 85606a98..6a8b4628 100644 --- a/lib/reports/sharesight_warm.py +++ b/lib/reports/sharesight_warm.py @@ -1,6 +1,6 @@ -"""Keeps Sharesight's portfolio list and holdings reports fresh in the cache (scheduled: sharesight_warm.py), so that a -request never waits on Sharesight. The price histories have their own, yahoo_warm.py.""" -import sys +"""Keeps Sharesight's portfolio list and holdings reports fresh in the cache (scheduled: sharesight_warm), so that a +request never waits on Sharesight. The price histories have their own, yahoo_warm.""" +from lib import log from lib import config, sharesight def run(): @@ -14,9 +14,14 @@ def run(): try: return len(sharesight.warm()) except Exception as e: - print("sharesight_warm: refresh failed:", e, file=sys.stderr) + log.warn("sharesight_warm: refresh failed:", e) return 0 +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: sharesight_warm.py takes none.""" - return 'sharesight_warm.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); sharesight_warm takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'sharesight_warm', {} diff --git a/lib/reports/shorts.py b/lib/reports/shorts.py index a9887433..f829ad73 100644 --- a/lib/reports/shorts.py +++ b/lib/reports/shorts.py @@ -1,29 +1,31 @@ -""".shorts: the most shorted stocks followed, from Short Man (ASX) and Yahoo (scheduled: shorts.py).""" +""".shorts: the most shorted stocks followed, from Short Man (ASX) and Yahoo (scheduled: shorts).""" from lib import config from lib import delivery +from lib import formats from lib import render from lib import watchlist from lib import yahoo from lib import shortman def run(dest=None, *, threshold=None, specific_stock=None): - """Post the stocks followed whose short interest is over `threshold` percent (the shorts_percent preference when - None), or one stock's (Yahoo's symbol): to the chat that asked (dest), else to the default chats.""" - interactive = dest is not None + """Post the stocks a chat follows whose short interest is over `threshold` percent (the shorts_percent preference + when None), or one stock's (Yahoo's symbol): to the chat that asked or whose job it is (dest), else to the home + chats.""" + interactive = delivery.interactive(dest) if threshold is None: threshold = config.prefs.shorts_percent - tickers, market_data = gather(specific_stock) for chat in delivery.targets(dest): + tickers = watchlist.covered(chat.key, specific_stock) + market_data = gather(tickers, specific_stock) payload, graph = report(chat.dialect, tickers, market_data, threshold, specific_stock, interactive) delivery.deliver(chat, delivery.Report(payload, graph)) -def gather(specific_stock): - """(the stocks followed, or the one, their quotes with short interest): Short Man has the ASX's figures, Yahoo's - details the US's.""" - tickers = watchlist.covered(specific_stock) +def gather(tickers, specific_stock): + """The quotes of tickers (the stocks a chat follows, or the one asked about), with their short interest: Short Man + has the ASX's figures, Yahoo's details the US's.""" market_data = yahoo.fetch(tickers) market_data |= yahoo.fetch_details((ticker for ticker in tickers if '.' not in ticker), strict=bool(specific_stock)) - return tickers, shortman.fetch(market_data) + return shortman.fetch(market_data) def report(service, tickers, market_data, threshold, specific_stock, interactive): """(the report in one network's markup, its chart or None), the most shorted first.""" @@ -39,7 +41,7 @@ def report(service, tickers, market_data, threshold, specific_stock, interactive url = 'https://www.shortman.com.au/stock?q=' + code else: url = 'https://finance.yahoo.com/quote/' + ticker + '/key-statistics?p=' + ticker - title = market_data[ticker]['profile_title'] + title = formats.short_name(market_data[ticker]['profile_title']) found.append((short_percent, f"🩳 {title} ({service.link(url, code)}) {round(short_percent)}%", code)) found.sort(key=lambda row: row[0], reverse=True) # largest first, as the chart has them payload, graph = [line for _, line, _ in found], None @@ -53,6 +55,11 @@ def report(service, tickers, market_data, threshold, specific_stock, interactive payload = [f"🩳 No stocks shorted over {threshold:g}%. Try specifying a number."] return payload, graph +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: shorts.py takes none.""" - return 'shorts.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); shorts takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'shorts', {} diff --git a/lib/reports/trades.py b/lib/reports/trades.py index 5b898dfe..a68e4118 100644 --- a/lib/reports/trades.py +++ b/lib/reports/trades.py @@ -1,8 +1,8 @@ -""".trades: Sharesight buys and sells; a scheduled run reports only new ones (scheduled: trades.py).""" +""".trades: Sharesight buys and sells; a scheduled run reports only new ones (scheduled: trades).""" import dataclasses import datetime -import sys +from lib import log from lib import config from lib import state from lib import sharesight @@ -12,7 +12,7 @@ from lib import markets from lib import yahoo -STATE_FILE = "finbot_sharesight_trades.json" # the ids of the trades reported +STATE_FILE = "finbot_sharesight_trades.json" # each chat's ids of the trades reported to it @dataclasses.dataclass(frozen=True) @@ -30,22 +30,25 @@ class Trade: def run(dest=None, *, days=None, user='', portfolio_select=None): - """Post the buys and sells of the past `days`, in every portfolio or the one named: to the chat that asked (dest), - else, only those not reported before, to the default chats. Raises sharesight.PortfolioNotFound for a name that is - none of them.""" - interactive = dest is not None + """Post the buys and sells of the past `days`, in every portfolio or the one named: to the chat that asked (dest); + or, a scheduled run, to each of its chats (dest, a chat's job; else the home chats) those it has not told that chat + of before. Raises sharesight.PortfolioNotFound for a name that is none of them.""" + interactive = delivery.interactive(dest) if days is None: days = config.prefs.past_days - trades = gather(days, portfolio_select, set() if interactive else set(state.load(STATE_FILE, []))) - for chat in delivery.targets(dest): - payload = report(chat.dialect, trades, days, interactive) + chats = delivery.targets(dest) + told = {chat.key: set() if interactive else set(state.load_for(chat.key, STATE_FILE, [])) for chat in chats} + trades = gather(days, portfolio_select, set.intersection(*told.values())) # every chat knows those: not fetched + for chat in chats: + new = [trade for trade in trades if trade.id not in told[chat.key]] + payload = report(chat.dialect, new, days, interactive) if interactive and not payload: # no trades, or none of them a buy or a sell when = formats.days_english(days, 'in the past ') payload = [f"{user} No trades {when}."] delivery.deliver(chat, delivery.Report(payload)) - if trades and not interactive: # recorded once the report is out - with state.edit(STATE_FILE, []) as known: - known.extend(sorted({trade.id for trade in trades}.difference(known))) + if new and not interactive: # recorded once the report is out + with state.edit_for(chat.key, STATE_FILE, []) as known: + known.extend(sorted({trade.id for trade in new}.difference(known))) def gather(days, portfolio_select, known): @@ -57,10 +60,10 @@ def gather(days, portfolio_select, known): if raw and config.prefs.trades_use_yahoo: tickers = {markets.transform_to_yahoo(trade['symbol'], trade['market']) for trade in raw} try: - print("Fetching tickers from Yahoo: ", tickers, file=sys.stderr) + log.info("Fetching tickers from Yahoo: ", tickers) quotes = yahoo.fetch(tickers) except Exception as e: - print("Warning: could not fetch Yahoo:", e, file=sys.stderr) + log.warn("Warning: could not fetch Yahoo:", e) trades = [] for trade in raw: market = trade['market'] @@ -72,7 +75,7 @@ def gather(days, portfolio_select, known): try: market = quotes[ticker]['profile_exchange'] except Exception as e: - print("Warning: could not get market for", ticker, "from Yahoo:", e, file=sys.stderr) + log.warn("Warning: could not get market for", ticker, "from Yahoo:", e) trades.append(Trade(date=datetime.datetime.strptime(trade['transaction_date'], '%Y-%m-%d').date(), id=int(trade['id']), portfolio=trade['portfolio'], bought=trade['transaction_type'] == 'BUY', value=round(float(trade['price']) * float(trade['quantity'])), @@ -96,6 +99,11 @@ def line(trade): if trade.date == date] for date in dates)) +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: trades.py takes none.""" - return 'trades.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); trades takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'trades', {} diff --git a/lib/reports/yahoo_warm.py b/lib/reports/yahoo_warm.py index 2a53f912..94793686 100644 --- a/lib/reports/yahoo_warm.py +++ b/lib/reports/yahoo_warm.py @@ -1,25 +1,30 @@ -"""Keeps the price history of every stock followed fresh in the cache (scheduled: yahoo_warm.py), so that a period list -or a chart never waits on Yahoo. Sharesight's caches have their own, sharesight_warm.py.""" -import sys +"""Keeps the price history of every stock followed fresh in the cache (scheduled: yahoo_warm), so that a period list +or a chart never waits on Yahoo. Sharesight's caches have their own, sharesight_warm.""" +from lib import log from lib import config, watchlist, yahoo def run(): - """Fetch afresh the price history of every stock followed (the watchlist, and the holdings where Sharesight is - configured), so each is cached for cache_seconds (12 hours by default) from now: the default schedule runs this - every 8 hours, before any runs out, and a flush at once. With the cache preference off nothing would be kept, so - nothing is fetched. Returns how many it fetched.""" + """Fetch afresh the price history of every stock followed (every chat's watchlist, and the holdings where + Sharesight is configured), so each is cached for cache_seconds (12 hours by default) from now: the system job runs + this every 8 hours, before any runs out, and a flush at once. With the cache preference off nothing would be kept, + so nothing is fetched. Returns how many it fetched.""" if not config.prefs.cache: return 0 - tickers = watchlist.tickers() + tickers = watchlist.all_watched() if config.env.sharesight_auth.get('client_id'): try: - tickers = watchlist.with_holdings() - except Exception as e: # Sharesight unreachable, say: the watchlist's at least - print("yahoo_warm: no holdings from Sharesight:", e, file=sys.stderr) + tickers = watchlist.all_followed() + except Exception as e: # Sharesight unreachable, say: the watchlists' at least + log.warn("yahoo_warm: no holdings from Sharesight:", e) failed = yahoo.prefetch_history(tickers, seconds=0) - print(f"yahoo_warm: {len(tickers) - len(failed)} of {len(tickers)} price histories fetched", file=sys.stderr) + log.info(f"yahoo_warm: {len(tickers) - len(failed)} of {len(tickers)} price histories fetched") return len(tickers) - len(failed) +ARGUMENTS = '' # what a scheduled run takes: none + def job(argv): - """Arguments of a scheduled run: yahoo_warm.py takes none.""" - return 'yahoo_warm.py', {} + """A scheduled run's arguments: (label, run's keyword arguments); yahoo_warm takes none, and an argument raises + ValueError.""" + if argv: + raise ValueError('no arguments') + return 'yahoo_warm', {} diff --git a/lib/sharesight.py b/lib/sharesight.py index 79960a6f..4a4b829e 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -2,10 +2,10 @@ as sent, and requests go one at a time.""" import dataclasses import datetime -import sys import threading import time +from lib import log from lib import answer from lib import cache from lib import config @@ -37,7 +37,7 @@ def _request(path, what, expect, **kwargs): error = data.get('error') if isinstance(data, dict) else None if not (error and 'parallel' in str(error).lower() and attempt < 4): break - print("Sharesight busy:", error, "- retrying", file=sys.stderr) + log.warn("Sharesight busy:", error, "- retrying") time.sleep(1 + attempt) if error: # an OAuth error describes itself too why = ': '.join(str(data[key]) for key in ('error', 'error_description') if data.get(key)) @@ -139,13 +139,18 @@ def trades(self, days=None): return [trade | {'portfolio': self.name} for trade in _trades(self.id, days)] +def _chosen(listed): + """The portfolios of Sharesight's list to report on, after include_portfolios and exclude_portfolios.""" + include, exclude = config.prefs.include_portfolios, config.prefs.exclude_portfolios + return [Portfolio(each['name'], each['id']) for each in listed + if str(each['id']) not in exclude and (not include or str(each['id']) in include)] + + def portfolios(select=None, seconds=None): """The portfolios to report on (after include_portfolios and exclude_portfolios), or the one called `select`, in any case; the list is cached for `seconds` (cache_seconds when None). Raises RuntimeError when there are none, and PortfolioNotFound for a name that is none of them.""" - include, exclude = config.prefs.include_portfolios, config.prefs.exclude_portfolios - found = [Portfolio(each['name'], each['id']) for each in _portfolios(seconds) - if str(each['id']) not in exclude and (not include or str(each['id']) in include)] + found = _chosen(_portfolios(seconds)) if not found: raise RuntimeError("No Sharesight portfolios found") if not select: @@ -161,6 +166,19 @@ def tickers(): return sorted({ticker for portfolio in portfolios() for ticker in portfolio.tickers()}) +def cached(): + """(the portfolios' names, the Yahoo tickers they hold, sorted) from what the cache holds, whatever its age, and + never a request: for suggestions while a command is typed, which must come at once. Empty where nothing is + cached.""" + found = _chosen((cache.load(PORTFOLIOS_CACHE) or {}).get('portfolios') or []) + held = set() + for portfolio in found: + report = (cache.load(performance_cache_file(portfolio.id, 0)) or {}).get('report') or {} + held |= {markets.transform_to_yahoo(each['instrument']['code'], each['instrument']['market_code']) + for each in report.get('holdings') or []} + return [portfolio.name for portfolio in found], sorted(held) + + def warm(): """Fetch afresh the portfolio list and each portfolio's holdings report (days=0), so that a request never waits on Sharesight (scheduled: sharesight_warm.py). Returns the portfolios. Raises as the requests do, and then the caches diff --git a/lib/shortman.py b/lib/shortman.py index 407ba905..5fd37a69 100644 --- a/lib/shortman.py +++ b/lib/shortman.py @@ -1,4 +1,4 @@ -import sys +from lib import log from lib import net def fetch(market_data): @@ -7,10 +7,10 @@ def fetch(market_data): try: r = net.get(url) except Exception as e: - print("Failure fetching", url, net.redact(e), file=sys.stderr) + log.warn("Failure fetching", url, net.redact(e)) return market_data # the ASX short interest is missing, the rest of the report is not if r.status_code != 200: - print(r.status_code, "error communicating with", url, file=sys.stderr) + log.warn(r.status_code, "error communicating with", url) return market_data csv = r.content.decode('utf-8') csv = csv.split('\r\n') diff --git a/lib/slack_common.py b/lib/slack_common.py new file mode 100644 index 00000000..817c94a5 --- /dev/null +++ b/lib/slack_common.py @@ -0,0 +1,340 @@ +"""Slack, whichever way its updates arrive (Socket Mode, slack_websocket; or the Request URL, slack_webhook): the Web +API calls, the reader of its events, button presses and slash commands (Slack, READER), and the bot that posts into +its channels (SlackBot), with the Block Kit of its messages.""" +import html +import json +import re +import time + +import requests + +from lib import log +from lib import config +from lib import dialects +from lib import inbound +from lib import net +from lib import state +from lib import transports + + +SLACK_CHARTS = 'finbot_slack_charts.json' # Slack's chart messages: the file each shows, and the charts uploaded +SLACK_CHARTS_KEPT = 500 # chart messages noted, newest last: an older one shows its file for good +SLACK_REUSE_SECONDS = 15 * 60 # a period pressed within this long of its chart's upload shows that upload again + + +def call(method, as_app=False, **data): + """POST a Slack Web API method (form-encoded) as the bot, or with the app-level token (as_app, for Socket Mode's + apps.connections.open), and return the answer; an error raises RuntimeError.""" + key = 'slackAppToken' if as_app else 'slackBotToken' + token = getattr(config.env, key) + if not token: + raise RuntimeError(f"Slack {method}: needs {key} in finbot.env") + r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + token}, data=data) + try: + output = r.json() + except ValueError: + output = {} + if not output.get('ok'): + raise RuntimeError(f"Slack {method}: {output.get('error') or r.status_code}") + return output + + +class Slack(inbound.Network): + """Slack's Events API (messages), its interactivity (a button pressed) and its slash commands.""" + service = 'slack' + BY_PEOPLE = (None, 'thread_broadcast') # message subtypes a person writes; the rest are bot posts, edits, joins... + + def update_id(self, update): + """Slack's event_id (a button press has none).""" + return update.get('event_id') + + def parse(self, update): + """A message a person wrote, a slash command, a button pressed, or the challenge that checks the URL.""" + kind = update.get('type') + if 'command' in update and kind is None: + return self.slash(update) + if kind == 'url_verification': + response = json.dumps({'challenge': update['challenge']}) + log.info("replying with", response) + return inbound.Response(body=response.encode()) + if kind == 'block_actions': + return self.press(update) + if kind != 'event_callback': + log.info(f"[slack]: unhandled 'type' {kind!r}") + return inbound.UNHANDLED + event = update['event'] + bot_user = update['authorizations'][0]['user_id'] # QWERTY + if event.get('bot_id') or event.get('subtype') not in self.BY_PEOPLE or not event.get('user') or \ + event['user'] == bot_user: # not by a person: answering a bot (this one's replies included) could loop + return inbound.OK + if event['type'] not in ('message', 'app_mention') or 'text' not in event: + log.info("[slack]: unhandled event callback type", event['type']) + return inbound.UNHANDLED + user = '<@' + event['user'] + '>' # a mention: Slack shows the name + return inbound.Message(self.service, event['channel'], user, self.plain(event['text']), '@' + bot_user, user, + event.get('ts', '')) + + def slash(self, update): + """A slash command (/price with 5 7d), as the message that says the same (/price 5 7d), in the channel or DM it + was sent in. There is no message of it, for ⏳ to show on.""" + user = '<@' + str(update.get('user_id') or 'unknown') + '>' + text = self.plain(f"{update['command']} {update.get('text') or ''}".strip()) + return inbound.Message(self.service, str(update.get('channel_id') or ''), user, text, '', user) + + @staticmethod + def plain(text): + """Text as written: links and mentions lose their markup ( becomes dub.ax, <@QWERTY> + @QWERTY), and Slack's escapes their meaning (& becomes &).""" + text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), text) + return html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > + + def press(self, update): + """A button pressed under one of the bot's messages (a chart's period buttons), or UNSUPPORTED for one that is + not under a message or carries no value.""" + container = update.get('container') or {} + actions = update.get('actions') or [{}] + value = actions[0].get('value') if isinstance(actions[0], dict) else None + if container.get('type') != 'message' or not container.get('message_ts') or not value: + return inbound.UNSUPPORTED + user = update.get('user') or {} + who = '@' + str(user.get('username') or user.get('id') or 'unknown') + return inbound.Press(self.service, update.get('trigger_id', ''), container.get('channel_id', ''), + container['message_ts'], value, who) + + +class SlackBot(transports.Transport): + """A Slack channel, as the bot: chat.postMessage and the file upload, with slackBotToken.""" + name = 'slack' + dialect = dialects.SLACK + WORKING = 'hourglass_flowing_sand' # ⏳: Slack has no typing notice for bots, so a reaction on the message stands in + working = None # True while ⏳ is on the message, False when putting it there failed + + @property + def shows_typing(self): + """⏳ on the message being answered: a scheduled report answers none.""" + return bool(self.message) + + def typing(self): + """Put ⏳ on the message being answered, once (the bot needs the reactions:write scope); a failure is only + logged, and not tried again.""" + if self.working is not None: + return + try: + call('reactions.add', channel=self.where, timestamp=self.message, name=self.WORKING) + self.working = True + except Exception as e: + self.working = False + log.warn("Slack reactions.add failed:", net.redact(e)) + + def typing_done(self): + """Take ⏳ off the message again; a failure is only logged.""" + if not self.working: + return + self.working = False + try: + call('reactions.remove', channel=self.where, timestamp=self.message, name=self.WORKING) + except Exception as e: + log.warn("Slack reactions.remove failed:", net.redact(e)) + + def post_text(self, text, buttons=None): + """chat.postMessage, without link previews, and with buttons (a Block Kit message) where they work. Slack's + answer, or None when it failed or refused (logged): Slack refuses with HTTP 200 and ok false (not_in_channel, + say), so a refusal is read from the answer.""" + if not config.env.slackBotToken: + log.error(f"ERROR slack: a reply to channel {self.where} needs slackBotToken in finbot.env; not sent") + return None + blocks = {} + if buttons and not isinstance(buttons, dict) and self.takes_buttons: + blocks['blocks'] = json.dumps(text_blocks(text, buttons)) + try: + answer = call('chat.postMessage', channel=self.where, text=text, unfurl_links='false', + unfurl_media='false', **blocks) + except Exception as e: + log.error("Slack chat.postMessage to", self.where, "failed:", net.redact(e)) + return None + log.info("OK slack message to", self.where) + return answer + + def post_chart(self, image, caption, buttons=None): + """Upload the chart into the channel with the report as its message, the way Slack takes files since it retired + files.upload: ask for an upload URL, send the bytes there, then complete the upload into the channel (the bot + needs the files:write and files:read scopes). With period buttons, where they work, it goes in a Block Kit + message instead (post_blocks), or without them if that fails. Slack's answer, MAYBE_SENT, or None if a step + failed.""" + data = image.read() if hasattr(image, 'read') else bytes(image) + if buttons and not isinstance(buttons, dict) and self.takes_buttons: + try: + return self.post_blocks(data, caption, buttons) + except Exception as e: + log.warn("Slack chart with buttons failed, uploading it without them:", net.redact(e)) + try: + share = {'files': json.dumps([{'id': self.upload(data)}]), 'channel_id': self.where, + 'initial_comment': caption} + try: + output = call('files.completeUploadExternal', **share) + except requests.exceptions.ReadTimeout as e: + log.warn("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e)) + return transports.MAYBE_SENT + except Exception as e: + log.warn("Slack chart upload failed:", net.redact(e)) + return None + log.info("OK slack chart upload to", self.where) + return output + + def upload(self, data): + """Send a chart's bytes to Slack: the id of the file, not completed yet. Raises RuntimeError on a refusal.""" + ticket = call('files.getUploadURLExternal', filename='chart.png', length=len(data)) + r = net.post(ticket['upload_url'], data=data) + if r.status_code != 200: + raise RuntimeError(f"Slack upload: {r.status_code} {r.text[:200]}") + return ticket['file_id'] + + def unshared(self, data): + """Upload a chart as a file that is in no channel, for an image block to show: its id.""" + file_id = self.upload(data) + call('files.completeUploadExternal', files=json.dumps([{'id': file_id, 'title': 'chart'}])) + return file_id + + def post_blocks(self, data, caption, buttons): + """The chart in a Block Kit message, the report above it and the period buttons under it. Slack's answer, or + MAYBE_SENT when posting the message timed out (it may have arrived); raises RuntimeError when Slack refuses.""" + file_id = self.unshared(data) + blocks = chart_blocks(caption, buttons, file_id) + try: + answer = until_shown(lambda: call('chat.postMessage', channel=self.where, + text=caption[:SECTION_LIMIT], blocks=json.dumps(blocks), unfurl_links='false', unfurl_media='false')) + except requests.exceptions.ReadTimeout as e: + log.warn("Slack chat.postMessage timed out, the chart may still arrive:", net.redact(e)) + return transports.MAYBE_SENT + if answer.get('ts'): + self.delete(self.shows(answer['ts'], file_id, caption, buttons)) + return answer + + def reshow(self, message, buttons): + """Show a chart message's period again from its upload of under SLACK_REUSE_SECONDS ago (for this message or + another), with the report it came with: nothing is drawn or uploaded. True when it did; False when there is no + such upload, or Slack refuses it (a file deleted outside finbot, say).""" + view, now = marked(buttons), time.time() + uploads = state.load(SLACK_CHARTS, {}).get('uploads', {}) + recent = [(at, file_id, caption) for file_id, (at, shows, caption) in uploads.items() + if view and shows == view and now - at < SLACK_REUSE_SECONDS] + if not recent: + return False + _, file_id, caption = max(recent) + try: + call('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], + blocks=json.dumps(chart_blocks(caption, buttons, file_id))) + except Exception as e: + log.warn("Slack chat.update with an earlier upload failed, uploading anew:", net.redact(e)) + return False + self.delete(self.shows(message, file_id)) + return True + + def edit_chart(self, message, image, caption, buttons=None): + """Show another period in a chart message: the new chart uploaded (a file cannot be replaced) and shown in its + image block, with its caption and buttons. A press for that period within SLACK_REUSE_SECONDS shows this upload + again (reshow). Raises RuntimeError when Slack refuses the update.""" + if hasattr(image, 'seek'): + image.seek(0) + file_id = self.unshared(image.read() if hasattr(image, 'read') else bytes(image)) + blocks = chart_blocks(caption, buttons or [], file_id) + until_shown(lambda: call('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], + blocks=json.dumps(blocks))) + self.delete(self.shows(message, file_id, caption, buttons)) + + def shows(self, message, file_id, caption=None, buttons=None): + """Note that a chart message shows this file now; given its report, that the file was uploaded now for the + period its buttons mark (see reshow). The uploads to delete: those no message shows, a minute past their reuse + (a press reusing one has that minute to show it).""" + now = time.time() + with state.edit(SLACK_CHARTS, {}) as noted: + if set(noted) != {'shown', 'uploads'}: # new, or the {'channel ts': file} of before, whose files stay + noted.clear() + noted.update(shown={}, uploads={}) + shown, uploads = noted['shown'], noted['uploads'] # 'channel ts': file; file: [time, period, report] + if caption is not None: + uploads[file_id] = [now, marked(buttons), caption] + key = f'{self.where} {message}' + shown.pop(key, None) + shown[key] = file_id + for old in list(shown)[:-SLACK_CHARTS_KEPT]: # its message may show the file for good + uploads.pop(shown.pop(old), None) + showing = set(shown.values()) + gone = [old for old, (at, _, _) in uploads.items() + if old not in showing and now - at > SLACK_REUSE_SECONDS + 60] + for old in gone: + del uploads[old] + for upload in uploads.values(): # past reuse, a file shown is kept only to be deleted later + if now - upload[0] >= SLACK_REUSE_SECONDS: + upload[1:] = [None, None] + return gone + + def delete(self, files): + """Delete uploaded charts that no message shows any more; a failure is only logged (the file stays among the + bot's files, in no channel).""" + for file_id in files: + try: + call('files.delete', file=file_id) + except Exception as e: + log.warn("Slack files.delete failed:", net.redact(e)) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + data = {'channel': self.where, 'ts': message, 'text': text} + if buttons is not None: + data['blocks'] = json.dumps(text_blocks(text, buttons)) + try: + call('chat.update', **data) + except Exception as e: + log.warn("Slack chat.update failed:", net.redact(e)) + + +SECTION_LIMIT = 3000 # characters of text in a Block Kit section + + +def section_blocks(text): + """Block Kit sections of a message's text, as many as its length needs (none for no text).""" + return [{'type': 'section', 'text': {'type': 'mrkdwn', 'text': '\n'.join(part)}} + for part in transports.chunked(text.split('\n'), SECTION_LIMIT) if '\n'.join(part).strip()] + + +def actions_block(row): + """A Block Kit row of buttons, the one shown now marked.""" + return {'type': 'actions', 'elements': [{'type': 'button', 'text': {'type': 'plain_text', 'text': button.label}, + 'value': button.data, 'action_id': button.data, **({'style': 'primary'} if button.active else {})} + for button in row]} + + +def chart_blocks(caption, buttons, file_id): + """Block Kit for a chart message: the report, the chart (an uploaded file, by its id) and the buttons in one + row.""" + row = [button for line in buttons for button in line] + return section_blocks(caption) + [{'type': 'image', 'slack_file': {'id': file_id}, 'alt_text': 'chart'}] + \ + ([actions_block(row)] if row else []) + + +def marked(buttons): + """The data of the button marked as shown now, which names a chart message's period (None when none is).""" + return next((button.data for row in buttons or [] for button in row if button.active), None) + + +def text_blocks(text, buttons): + """Block Kit for a message with buttons: the text, then the buttons, a row of them to a block.""" + return section_blocks(text) + [actions_block(row) for row in buttons if row] + + +def until_shown(post, waits=(1, 2, 4)): + """post() of a message that shows a file just uploaded, tried again after each wait (in seconds) while Slack says + invalid_blocks: a file can take a moment before it can be shown.""" + for wait in waits: + try: + return post() + except RuntimeError as e: + if 'invalid_blocks' not in str(e): + raise + log.info(f"Slack cannot show the chart yet ({e}), trying again in {wait} s") + time.sleep(wait) + return post() + + +READER = Slack() diff --git a/lib/slack_webhook.py b/lib/slack_webhook.py new file mode 100644 index 00000000..6f4037a6 --- /dev/null +++ b/lib/slack_webhook.py @@ -0,0 +1,60 @@ +"""Slack's Request URL, its HTTP mode (slackOutgoingWebhook set): Slack posts each event, button press and slash +command to finbot's URL, signed with the app's signing secret, for the reader (slack_common.READER).""" +import hashlib +import hmac +import time +from urllib.parse import parse_qs, urlparse + +from lib import config +from lib import inbound +from lib import slack_common + + +SIGNATURE_AGE = 300 # seconds: Slack's limit; an older signed request may be a replay + + +class SlackWebhook(inbound.Webhook): + """Slack posting events, button presses and slash commands to slackOutgoingWebhook (the app's Request URL for each), + each request signed with the app's signing secret, and carrying its verification token.""" + + def path(self): + """The path of slackOutgoingWebhook.""" + return urlparse(config.env.slackOutgoingWebhook).path if config.env.slackOutgoingWebhook else '' + + def refuse(self, environ, body, update): + """With slackSigningSecret, the request must carry Slack's signature of its body, made in the last five minutes; + without it, the body must carry slackOutgoingToken (the older way, which Slack has deprecated).""" + if config.env.slackSigningSecret: + return self.signature_problem(environ, body, config.env.slackSigningSecret) + if 'token' not in update: + return "Slack authorisation field not present" + if not inbound.same_secret(str(update['token']), config.env.slackOutgoingToken): + return "Slack auth incorrect" + return None + + @staticmethod + def signature_problem(environ, body, secret): + """Why the request's signature fails, or None: X-Slack-Signature must be v0= and the HMAC-SHA256, under the + signing secret, of v0::, and the timestamp within five minutes of now.""" + timestamp = environ.get('HTTP_X_SLACK_REQUEST_TIMESTAMP', '') + signature = environ.get('HTTP_X_SLACK_SIGNATURE') + if signature is None or not (timestamp.isascii() and timestamp.isdigit()): + return "Slack signature not present" + if abs(time.time() - int(timestamp)) > SIGNATURE_AGE: + return "Slack signature too old (a replay, or a clock is wrong)" + signed = b'v0:' + timestamp.encode() + b':' + body + expected = 'v0=' + hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest() + return None if inbound.same_secret(signature, expected) else "Slack signature incorrect" + + def decode(self, body): + """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON, and a + slash command as a form of its fields (command, text, channel_id, user_id and the rest).""" + if not body.lstrip().startswith(b'{'): + fields = parse_qs(body.decode('utf-8', 'replace'), keep_blank_values=True) + if 'command' in fields and 'payload' not in fields: + return {name: values[0] for name, values in fields.items()} + body = fields.get('payload', [''])[0].encode() + return super().decode(body) + + +WEBHOOK = SlackWebhook(slack_common.READER) diff --git a/lib/slack_websocket.py b/lib/slack_websocket.py new file mode 100644 index 00000000..ecf01b0f --- /dev/null +++ b/lib/slack_websocket.py @@ -0,0 +1,120 @@ +"""Slack's Socket Mode, its pull mode (no slackOutgoingWebhook): a WebSocket that the app-level token opens, whose +envelopes carry what Slack would post to the Request URL, for the reader (slack_common.READER).""" +import json + +from lib import log +from lib import config +from lib import net +from lib import pull +from lib import slack_common + + +class WebSocket: + """A WebSocket connection, through websocket-client: imported here only, as Slack's pull mode is the one thing that + needs it (Debian's python3-websocket).""" + + def __init__(self, url): + """Connect to url, waiting net.CONNECT_TIMEOUT seconds at most for the server.""" + import websocket + self.library = websocket + self.ws = websocket.create_connection(url, timeout=net.CONNECT_TIMEOUT) + + def receive(self, timeout): + """The next message's text; '' for a ping or a pong, which only say the line is alive; None when nothing came + within timeout seconds. Raises ConnectionError when the server closes the socket.""" + self.ws.settimeout(timeout) + try: + opcode, data = self.ws.recv_data(control_frame=True) + except self.library.WebSocketTimeoutException: + return None + if opcode == self.library.ABNF.OPCODE_CLOSE: + raise ConnectionError('the server closed the socket') + if opcode == self.library.ABNF.OPCODE_TEXT: + return data.decode('utf-8') if isinstance(data, bytes) else data + return '' + + def send(self, text): + """Send a text message.""" + self.ws.send(text) + + def ping(self): + """Send a ping, which the server answers with a pong.""" + self.ws.ping() + + def close(self): + """Close the socket; a goodbye that fails is of no matter.""" + try: + self.ws.close(timeout=1) + except Exception: + pass + + +class SlackSocket(pull.Listener): + """Slack's Socket Mode: a WebSocket to the URL that apps.connections.open gives for the app-level token + (slackAppToken). Each event, button press or slash command comes in an envelope, acked at once, before anything + else (Slack sends again what is not acked within 3 seconds); inside it is what Slack would post to + slackOutgoingWebhook. Slack closes a socket now and then, saying so first (disconnect), and another is opened at + once.""" + network = slack_common.READER + PING_SECONDS = 30 # silence this long sends a ping; as long again after it, and the socket is taken for dead + TAKEN = ('events_api', 'interactive', 'slash_commands') # the envelopes whose payload goes to the reader + + @staticmethod + def wanted(): + """Whether finbot.env asks for it: Slack's bot token and app-level token, and no slackOutgoingWebhook.""" + env = config.env + return bool(env.slackBotToken and env.slackAppToken and not env.slackOutgoingWebhook) + + def describe(self): + """The mode, for the log.""" + return 'Socket Mode (no slackOutgoingWebhook)' + + def open(self): + """The URL of a new socket, from apps.connections.open.""" + return slack_common.call('apps.connections.open', as_app=True)['url'] + + def connect(self, url): + """A WebSocket connected to url.""" + return WebSocket(url) + + def poll(self, take): + """One socket, from opening it to Slack closing it: then another at once; after a failure, the back-off.""" + try: + connection = self.connect(self.open()) + except Exception as e: + return self.failed('opening a socket', e) + try: + reason = self.listen(connection, take) + except Exception as e: + return self.failed('the socket', e) + finally: + connection.close() + log.info(f"slack: Slack is closing the socket ({reason}), opening another") + return 0 + + def listen(self, connection, take): + """Read envelopes until Slack says it will close the socket, and return its reason: each acked, then its payload + handed on. A ping goes out after PING_SECONDS of silence; as long again with nothing, not even the pong, raises + TimeoutError.""" + pinged = False + while True: + text = connection.receive(self.PING_SECONDS) + if text is None: + if pinged: + raise TimeoutError(f'nothing from Slack for {2 * self.PING_SECONDS} s, even after a ping') + connection.ping() + pinged = True + continue + pinged = False + if not text: # a ping or a pong + continue + envelope = json.loads(text) + if envelope.get('envelope_id'): + connection.send(json.dumps({'envelope_id': envelope['envelope_id']})) + kind = envelope.get('type') + if kind == 'hello': + self.failures = 0 + elif kind == 'disconnect': + return envelope.get('reason') or 'no reason given' + elif kind in self.TAKEN: + self.hand_on(take, envelope.get('payload') or {}) diff --git a/lib/state.py b/lib/state.py index d10f32b9..92bf9d72 100644 --- a/lib/state.py +++ b/lib/state.py @@ -1,6 +1,7 @@ -"""State files in DATA_DIR: what must last from one run to the next (the watchlist, plans, seen trades, ratings, record -highs, the schedule's last runs, chart button refs, keyboard versions, seen podcast episodes). Written whole or not at -all, and flushed.""" +"""State files in DATA_DIR: what must last from one run to the next (each chat's jobs, the watchlist, plans, seen +trades, ratings, record highs, the schedule's last runs, chart button refs, keyboard versions, seen podcast episodes). +Written whole or not at all, and flushed. A report's record of what it told each chat is a file of its own, holding one +record per chat ({'chats': {chat key: record}}): see load_for and edit_for.""" import contextlib import copy import json @@ -36,6 +37,18 @@ def _lock(name): return _locks.setdefault(name, threading.Lock()) +def load_for(chat, name, default): + """A chat's record (by its key) in a state file that holds one per chat, or `default` when it has none yet.""" + return copy.deepcopy(load(name, {}).get('chats', {}).get(chat, default)) + + +@contextlib.contextmanager +def edit_for(chat, name, default): + """Change a chat's record (by its key) in a state file that holds one per chat, as edit() changes a file.""" + with edit(name, {}) as data: + yield data.setdefault('chats', {}).setdefault(chat, copy.deepcopy(default)) + + @contextlib.contextmanager def edit(name, default): """Change a state file in place: `with state.edit(name, {}) as data: data[key] = value`. The file is read as the diff --git a/lib/telegram.py b/lib/telegram.py deleted file mode 100644 index 6920f751..00000000 --- a/lib/telegram.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -import sys -from lib import config -from lib import net - -def call(method, **kwargs): - """POST a Bot API method and return its result. Raises RuntimeError with Telegram's reason when it is not ok.""" - response = net.post(config.env.webhooks['telegram'] + method, **kwargs) - try: - output = response.json() - except ValueError: - output = {} - if not output.get('ok'): - raise RuntimeError(f"Telegram {method}: {output.get('description') or response.status_code}") - return output['result'] - -def try_call(method, **kwargs): - """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" - try: - return call(method, **kwargs) - except Exception as e: - print(f"Telegram {method} failed:", net.redact(e), file=sys.stderr) - -def setWebhook(): - """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken.""" - print("registering", config.env.telegramOutgoingWebhook, file=sys.stderr) - params = {'url': config.env.telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), - 'secret_token': config.env.telegramOutgoingToken} - return call('setWebhook', params=params) - -def delWebhook(): - """Have Telegram stop posting updates.""" - print("deregistering", config.env.telegramOutgoingWebhook, file=sys.stderr) - return call('setWebhook', params={'url': ''}) # unsubscribe - -def getMe(): - """The bot's own account: its username is the bot's name in chats.""" - return call('getMe') - -def register(): - """Called once by bot.py as it starts, never on import (the job processes and the tests import this module too): - unregister and register the webhook again, which wakes up a stale connection on Telegram's side, then return the - bot's @username. Raises RuntimeError, or a requests error, if any of the three calls fails.""" - delWebhook() - setWebhook() - return '@' + getMe()['username'] - -def pinChatMessage(chat_id, message_id): - """Pin a message, without the "pinned a message" notice. A failure is only logged: the message is there anyway.""" - try_call('pinChatMessage', json={'chat_id': chat_id, 'message_id': message_id, 'disable_notification': True}) - -def answerCallbackQuery(callback_query_id, text=None): - """Acknowledge a button press (the client's spinner stops), `text` as a toast. A failure is only logged.""" - try_call('answerCallbackQuery', json={'callback_query_id': callback_query_id} | ({'text': text} if text else {})) - diff --git a/lib/telegram_common.py b/lib/telegram_common.py new file mode 100644 index 00000000..8aba0bd8 --- /dev/null +++ b/lib/telegram_common.py @@ -0,0 +1,214 @@ +"""Telegram, whichever way its updates arrive (long polling, telegram_longpolling; or the webhook, telegram_webhook): +the Bot API calls, the reader of its updates (Telegram, READER), and the bot that posts into its chats +(TelegramBot).""" +import html +import json + +import requests + +from lib import log +from lib import buttons +from lib import config +from lib import dialects +from lib import inbound +from lib import net +from lib import transports + + +def call(method, **kwargs): + """POST a Bot API method and return its result. Raises RuntimeError with Telegram's reason when it is not ok.""" + response = net.post(config.env.telegram_api + method, **kwargs) + try: + output = response.json() + except ValueError: + output = {} + if not output.get('ok'): + raise RuntimeError(f"Telegram {method}: {output.get('description') or response.status_code}") + return output['result'] + + +def try_call(method, **kwargs): + """call(), where a failure must not stop what the bot is doing: it is only logged, and None returned.""" + try: + return call(method, **kwargs) + except Exception as e: + log.warn(f"Telegram {method} failed:", net.redact(e)) + + +def deleteWebhook(): + """Have Telegram stop posting updates. Those not delivered yet are kept, for the webhook set next or getUpdates.""" + log.info("telegram: deleting the webhook") + return call('deleteWebhook', json={'drop_pending_updates': False}) + + +def bot_name(): + """The bot's @username, from its own account (getMe): a command in a group may name the bot (/price@finbot).""" + return '@' + call('getMe')['username'] + + +def set_commands(commands): + """Make the bot's command menu, in every chat, these slash commands ([(name, description, arguments)], from + worker.slash_commands), each with its usage, where Telegram's differs. A failure is only logged: typed, a command + works without the menu.""" + wanted = [{'command': command.name, 'description': command.description[:256]} for command in commands] + if try_call('getMyCommands') != wanted and try_call('setMyCommands', json={'commands': wanted}) is not None: + log.info(f"telegram: {len(wanted)} commands in the menu") + + +def pinChatMessage(chat_id, message_id): + """Pin a message, without the "pinned a message" notice. A failure is only logged: the message is there anyway.""" + try_call('pinChatMessage', json={'chat_id': chat_id, 'message_id': message_id, 'disable_notification': True}) + + +def answerCallbackQuery(callback_query_id, text=None): + """Acknowledge a button press (the client's spinner stops), `text` as a toast. A failure is only logged.""" + try_call('answerCallbackQuery', json={'callback_query_id': callback_query_id} | ({'text': text} if text else {})) + + +class Telegram(inbound.Network): + """Telegram's updates: messages and button presses.""" + service = 'telegram' + bot_name = None # the bot's @username, from bot_name() as bot.py starts + IGNORED = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', + 'venue', 'poll', 'dice'} # nothing to answer, nothing worth logging + + def update_id(self, update): + """Telegram's update_id.""" + return update.get('update_id') + + def parse(self, update): + """A message (its text, or a photo's caption) or a button press. A first name is escaped as it arrives: + replies are Telegram HTML, and a name is text, not markup.""" + if 'callback_query' in update: # chart period and job buttons + press = update['callback_query'] + message = press.get('message') + if not message or not press.get('data'): + return inbound.UNSUPPORTED + sender = press.get('from', {}) + who = '@' + sender['username'] if 'username' in sender else sender.get('first_name', 'unknown') + return inbound.Press(self.service, press['id'], str(message['chat']['id']), str(message['message_id']), + press['data'], who) + if 'message' not in update: + return inbound.UNSUPPORTED + message = update['message'] + sender = message['from'] + chat_id = str(message['chat']['id']) + first_name = html.escape(sender['first_name'], quote=False) + if 'username' in sender: + user = '@' + sender['username'] + user_name = first_name or user # the handle when the first name is blank + else: + user = user_name = '@' + first_name + if 'text' in message: + text = message['text'] + elif 'photo' in message: + text = message.get('caption', '') + elif self.IGNORED & message.keys(): + return inbound.OK + else: + log.info("[telegram]: unhandled: 'message' without 'text/photo'") + return inbound.UNHANDLED + return inbound.Message(self.service, chat_id, user, text, self.bot_name, user_name, + str(message.get('message_id', ''))) + + def acknowledge(self, event): + """A press: answerCallbackQuery, so that the client's spinner stops and a toast made from the button's data says + what is coming (buttons.toast: 'Loading 1Y…'). A failure is only logged. Nothing goes back with the update.""" + if isinstance(event, inbound.Press): + answerCallbackQuery(event.press_id, buttons.toast(event.data)) + return None + + +class TelegramBot(transports.Transport): + """A Telegram chat via the Bot API: HTML, silent, no link previews; charts, buttons, edits, the typing notice.""" + name = 'telegram' + dialect = dialects.TELEGRAM + TYPING = (0.5, 4, 18) # "typing…" shows for 5 s at most (sendChatAction), so it is sent again every 4 s + NO_PREVIEWS = {'is_disabled': True} # link_preview_options, which replaced disable_web_page_preview (Bot API 7.0) + + @staticmethod + def markup(buttons): + """Rows of buttons.Button as Telegram's inline keyboard ('● ' marks the active one); a markup of Telegram's own + (the command keyboard) as it is.""" + if isinstance(buttons, dict): + return buttons + return {'inline_keyboard': [[{'text': ('● ' if button.active else '') + button.label, + 'callback_data': button.data} for button in row] for row in buttons]} + + def post_text(self, text, buttons=None): + """sendMessage: Telegram's message, or None when it refused or failed (logged with Telegram's reason, such as + "can't parse entities").""" + payload = {'chat_id': self.where, 'text': text, 'parse_mode': 'HTML', 'link_preview_options': self.NO_PREVIEWS, + 'disable_notification': True} + if buttons: + payload['reply_markup'] = self.markup(buttons) + try: + message = call('sendMessage', json=payload) + except Exception as e: + log.error("Telegram sendMessage to", self.where, "failed:", net.redact(e)) + return None + log.info("OK telegram message to", self.where) + return message + + def post_chart(self, image, caption, buttons=None): + """sendPhoto, the report as its caption: Telegram's message, MAYBE_SENT, or None when it refused or failed + (logged with Telegram's reason).""" + data = {'chat_id': self.where, 'caption': caption, 'parse_mode': 'HTML', 'disable_notification': True} + if buttons: + data['reply_markup'] = json.dumps(self.markup(buttons)) + try: + message = call('sendPhoto', data=data, files={'photo': ('image.png', image)}) + except requests.exceptions.ReadTimeout as e: + log.warn("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e)) + return transports.MAYBE_SENT + except Exception as e: + log.warn("Telegram sendPhoto to", self.where, "failed:", net.redact(e)) + return None + log.info("OK telegram chart to", self.where) + return message + + def edit_chart(self, message, image, caption, buttons=None): + """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Telegram refuses, + unless the message is already so (pressing the active period again is harmless); a request that fails on the way + is only logged.""" + media = {'type': 'photo', 'media': 'attach://photo', 'caption': transports.fit_caption(caption, self.dialect), + 'parse_mode': 'HTML'} + data = {'chat_id': self.where, 'message_id': message, 'media': json.dumps(media)} + if buttons: + data['reply_markup'] = json.dumps(self.markup(buttons)) + if hasattr(image, 'seek'): + image.seek(0) + try: + call('editMessageMedia', data=data, files={'photo': ('image.png', image)}) + except RuntimeError as e: + if 'not modified' not in str(e): + raise + except Exception as e: # it may have arrived even so: no error line for something the chat may already show + log.warn("Telegram editMessageMedia failed:", net.redact(e)) + + def edit_text(self, message, text, buttons=None): + """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" + payload = {'chat_id': self.where, 'message_id': message, 'text': text, 'parse_mode': 'HTML', + 'link_preview_options': self.NO_PREVIEWS} + if buttons is not None: + payload['reply_markup'] = self.markup(buttons) + try_call('editMessageText', json=payload) + + def typing(self): + """sendChatAction: "typing…" for a few seconds.""" + try_call('sendChatAction', data={'chat_id': self.where, 'action': 'typing'}) + + def pin(self, answers): + """Pin the messages that these answers are (Telegram's), last first, so that the first ends up on top, without + the notification.""" + sent = [] + for answer in answers: + try: + sent.append((answer['chat']['id'], answer['message_id'])) + except (KeyError, TypeError): + log.warn("Unable to pin: unexpected response") + for chat, message in reversed(sent): + pinChatMessage(chat, message) + + +READER = Telegram() diff --git a/lib/telegram_longpolling.py b/lib/telegram_longpolling.py new file mode 100644 index 00000000..a80b8464 --- /dev/null +++ b/lib/telegram_longpolling.py @@ -0,0 +1,49 @@ +"""Telegram's long polling, its pull mode (no telegramOutgoingWebhook): finbot asks for its updates (getUpdates) again +and again, and hands each to the reader (telegram_common.READER) as the webhook would.""" +from lib import config +from lib import net +from lib import pull +from lib import telegram_common + + +def register(): + """Called once by bot.py as it starts, never on import: delete any webhook, so that getUpdates may take the + updates, those not delivered yet included. Raises RuntimeError, or a requests error, if the call fails.""" + telegram_common.deleteWebhook() + + +class TelegramPoll(pull.Listener): + """Telegram's long polling: getUpdates, which Telegram holds open until there are updates or POLL_SECONDS have + passed, asking for those after the last one taken (offset), which also tells Telegram that one arrived. No webhook + may be set meanwhile: register() deletes it as finbot starts.""" + network = telegram_common.READER + POLL_SECONDS = 50 # how long Telegram may hold a getUpdates open + READ_SECONDS = 65 # the request's own timeout for the answer: longer than Telegram holds it + offset = None # the update_id after the last one taken + + @staticmethod + def wanted(): + """Whether finbot.env asks for it: Telegram's bot token, and no telegramOutgoingWebhook.""" + return bool(config.env.telegramBotToken and not config.env.telegramOutgoingWebhook) + + def describe(self): + """The mode, for the log.""" + return 'long polling for updates (no telegramOutgoingWebhook)' + + def poll(self, take): + """One getUpdates: each update to take, in order; then no wait, or the back-off after a failure (a 409 says a + webhook is set, or another finbot polls with the same token).""" + params = {'timeout': self.POLL_SECONDS, 'allowed_updates': ['message', 'callback_query']} + if self.offset is not None: + params['offset'] = self.offset + try: + updates = telegram_common.call('getUpdates', json=params, timeout=(net.CONNECT_TIMEOUT, self.READ_SECONDS), + logged=False) # asked again and again: the log would be nothing else + except Exception as e: + return self.failed('getUpdates', e) + self.failures = 0 + for update in updates: + if isinstance(update.get('update_id'), int): + self.offset = update['update_id'] + 1 # taken, even if handling it fails: it is not fetched again + self.hand_on(take, update) + return 0 diff --git a/lib/telegram_webhook.py b/lib/telegram_webhook.py new file mode 100644 index 00000000..42561a97 --- /dev/null +++ b/lib/telegram_webhook.py @@ -0,0 +1,43 @@ +"""Telegram's webhook, its HTTP mode (telegramOutgoingWebhook set): Telegram posts each update to finbot's URL with +the secret header that setWebhook gave it, for the reader (telegram_common.READER).""" +import json +from urllib.parse import urlparse + +from lib import log +from lib import config +from lib import inbound +from lib import telegram_common + + +def setWebhook(): + """Have Telegram post messages and button presses to telegramOutgoingWebhook, signed with telegramOutgoingToken. The + log does not show the URL: its path is a secret.""" + log.info("telegram: setting the webhook (telegramOutgoingWebhook)") + params = {'url': config.env.telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), + 'secret_token': config.env.telegramOutgoingToken} + return telegram_common.call('setWebhook', params=params) + + +def register(): + """Called once by bot.py as it starts, never on import: delete the webhook and set it again, which wakes up a stale + connection on Telegram's side. Raises RuntimeError, or a requests error, if a call fails.""" + telegram_common.deleteWebhook() + setWebhook() + + +class TelegramWebhook(inbound.Webhook): + """Telegram posting to telegramOutgoingWebhook, each update with the secret header that setWebhook gave it.""" + + def path(self): + """The path of telegramOutgoingWebhook.""" + return urlparse(config.env.telegramOutgoingWebhook).path if config.env.telegramOutgoingWebhook else '' + + def refuse_unread(self, environ): + """The header must carry telegramOutgoingToken.""" + secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') + if not inbound.same_secret(secret, config.env.telegramOutgoingToken): + return f"Telegram authorisation header {'not present' if secret is None else 'is present but incorrect'}" + return None + + +WEBHOOK = TelegramWebhook(telegram_common.READER) diff --git a/lib/transports.py b/lib/transports.py index a4e05cde..69feec4c 100644 --- a/lib/transports.py +++ b/lib/transports.py @@ -1,26 +1,13 @@ -"""How a message gets to each network: a class per way of posting, made for one destination (a chat, or an incoming -webhook). A transport knows its dialect, what it can carry (charts, buttons, a typing notice) and the network's size -limits; text() and chart() are the same everywhere, and each network posts one message its own way (post_text, -post_chart).""" -import json -import sys +"""How a message gets to a network: Transport, the base of each network's bot (TelegramBot, SlackBot and DiscordBot, +in telegram_common, slack_common and discord_common), made for one destination (a chat). A transport knows its +dialect, what it can carry (charts, buttons, a typing notice) and the network's size limits; text() and chart() are +the same everywhere, and each network posts one message its own way (post_text, post_chart). Every bot takes buttons: +whichever way its network's updates arrive, their presses reach finbot.""" import threading import time -import requests - -from lib import config -from lib import dialects -from lib import net -from lib import state -from lib import telegram MAYBE_SENT = object() # a read timeout on the request that posts a chart: it may have arrived, so no text copy follows -SUPPRESS_EMBEDS = 4 # Discord message flag: no link previews -SLACK_CHARTS = 'finbot_slack_charts.json' # Slack's chart messages: the file each shows, and the charts uploaded -SLACK_CHARTS_KEPT = 500 # chart messages noted, newest last: an older one shows its file for good -SLACK_REUSE_SECONDS = 15 * 60 # a period pressed within this long of its chart's upload shows that upload again -NO_MENTIONS = {'parse': []} # Discord: text that looks like @everyone or a mention pings nobody class Transport: @@ -28,22 +15,33 @@ class Transport: typing notice going while a command works.""" name = '' # the network, in logs dialect = None - takes_charts = False # a chart, with the report as its caption - takes_buttons = False # buttons under a message - shows_typing = False # a typing notice while a command works + takes_charts = True # a chart, with the report as its caption + takes_buttons = True # buttons under a message + shows_typing = True # a typing notice while a command works TYPING = (0.5, 7, 10) # the notice: first after 0.5 s (a quick answer gets none), then every 7 s, 10 times at most + scheduled = False # posting a scheduled run's report (a chat's job, or the home chats), not a reply to a command _notices = None # the thread sending the notices _quiet = None # set to stop them - def __init__(self, where, message=None): - """where: the chat id, or the incoming webhook's URL; message: the id of the message being answered, if any.""" + def __init__(self, where, message=None, interaction='', pressed=False): + """where: the chat id; message: the id of the message being answered, if any; + interaction: what the network answers through, where it has one (Discord's application id and token), for a + slash command, or for a button press (pressed).""" self.where = where self.message = message + self.interaction = interaction + self.pressed = pressed def __repr__(self): """: the kind and where, for logs and test failures.""" return f'<{type(self).__name__} {self.where}>' + @property + def key(self): + """The chat's key, the same whichever way its network's updates arrive: 'telegram:-1001234'. A chat's jobs, its + watchlist and what its reports have told it are kept under it.""" + return f'{self.name}:{self.where}' + def start_typing(self): """Show the typing notice while a command works, where the network has one (TYPING says when), until the first message goes out or the command ends (end_typing).""" @@ -117,440 +115,11 @@ def typing(self): def typing_done(self): """Take the typing notice off, where it lasts until taken off (Slack's ⏳).""" - def acknowledge(self, press, text=None): - """Answer a button press, where the network waits for that (Telegram, with `text` as a toast).""" - def reshow(self, message, buttons): """Show a chart message's period (the button marked) again as it was posted lately, where the network keeps that: True when it did. Here never: each period is drawn and sent anew.""" return False - def _post_json(self, url, payload, headers=None): - """POST a JSON message: the answer ({} when it is not JSON), or None when the network refused it or failed.""" - try: - r = net.post(url, headers={'Content-type': 'application/json', **(headers or {})}, json=payload) - except Exception as e: - print("Failure sending to", self.name + ":", net.redact(e), file=sys.stderr) - return None - if r.status_code != 200: - print(r.status_code, "error outbound to", self.name, file=sys.stderr) - return None - print(r.status_code, "OK outbound to", self.name, file=sys.stderr) - try: - return r.json() - except ValueError: - return {} - - -class TelegramBot(Transport): - """A Telegram chat via the Bot API: HTML, silent, no link previews; charts, buttons, edits, the typing notice.""" - name = 'telegram' - dialect = dialects.TELEGRAM - takes_charts = shows_typing = True - - @property - def takes_buttons(self): - """Buttons only where their presses reach finbot: with telegramOutgoingWebhook, Telegram posts them to it.""" - return bool(config.env.telegramOutgoingWebhook) - - @staticmethod - def markup(buttons): - """Rows of buttons.Button as Telegram's inline keyboard ('● ' marks the active one); a markup of Telegram's own - (the command keyboard) as it is.""" - if isinstance(buttons, dict): - return buttons - return {'inline_keyboard': [[{'text': ('● ' if button.active else '') + button.label, - 'callback_data': button.data} for button in row] for row in buttons]} - - def post_text(self, text, buttons=None): - """sendMessage: Telegram's message, or None when it refused or failed (logged with Telegram's reason, such as - "can't parse entities").""" - payload = {'chat_id': self.where, 'text': text, 'parse_mode': 'HTML', 'disable_web_page_preview': True, - 'disable_notification': True} - if buttons: - payload['reply_markup'] = self.markup(buttons) - try: - message = telegram.call('sendMessage', json=payload) - except Exception as e: - print("Telegram sendMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - print("OK telegram message to", self.where, file=sys.stderr) - return message - - def post_chart(self, image, caption, buttons=None): - """sendPhoto, the report as its caption: Telegram's message, MAYBE_SENT, or None when it refused or failed - (logged with Telegram's reason).""" - data = {'chat_id': self.where, 'caption': caption, 'parse_mode': 'HTML', 'disable_notification': True} - if buttons: - data['reply_markup'] = json.dumps(self.markup(buttons)) - try: - message = telegram.call('sendPhoto', data=data, files={'photo': ('image.png', image)}) - except requests.exceptions.ReadTimeout as e: - print("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT - except Exception as e: - print("Telegram sendPhoto to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - print("OK telegram chart to", self.where, file=sys.stderr) - return message - - def edit_chart(self, message, image, caption, buttons=None): - """Replace a chart message's image, caption and buttons in place. Raises RuntimeError when Telegram refuses, - unless the message is already so (pressing the active period again is harmless); a request that fails on the way - is only logged.""" - media = {'type': 'photo', 'media': 'attach://photo', 'caption': fit_caption(caption, self.dialect), - 'parse_mode': 'HTML'} - data = {'chat_id': self.where, 'message_id': message, 'media': json.dumps(media)} - if buttons: - data['reply_markup'] = json.dumps(self.markup(buttons)) - if hasattr(image, 'seek'): - image.seek(0) - try: - telegram.call('editMessageMedia', data=data, files={'photo': ('image.png', image)}) - except RuntimeError as e: - if 'not modified' not in str(e): - raise - except Exception as e: # it may have arrived even so: no error line for something the chat may already show - print("Telegram editMessageMedia failed:", net.redact(e), file=sys.stderr) - - def edit_text(self, message, text, buttons=None): - """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" - payload = {'chat_id': self.where, 'message_id': message, 'text': text, 'parse_mode': 'HTML', - 'disable_web_page_preview': True} - if buttons is not None: - payload['reply_markup'] = self.markup(buttons) - telegram.try_call('editMessageText', json=payload) - - def typing(self): - """sendChatAction: "typing…" for a few seconds.""" - telegram.try_call('sendChatAction', data={'chat_id': self.where, 'action': 'typing'}) - - def acknowledge(self, press, text=None): - """answerCallbackQuery: the client's spinner stops, and `text` shows as a toast.""" - telegram.answerCallbackQuery(press, text) - - def pin(self, answers): - """Pin the messages that these answers are (Telegram's), last first, so that the first ends up on top, without - the notification.""" - sent = [] - for answer in answers: - try: - sent.append((answer['chat']['id'], answer['message_id'])) - except (KeyError, TypeError): - print("Unable to pin: unexpected response", file=sys.stderr) - for chat, message in reversed(sent): - telegram.pinChatMessage(chat, message) - - -class SlackBot(Transport): - """A Slack channel, as the bot: chat.postMessage and the file upload, with slackBotToken.""" - name = 'slack' - dialect = dialects.SLACK - WORKING = 'hourglass_flowing_sand' # ⏳: Slack has no typing notice for bots, so a reaction on the message stands in - working = None # True while ⏳ is on the message, False when putting it there failed - - @property - def takes_charts(self): - """Charts are uploaded with the bot token, so only with one.""" - return bool(config.env.slackBotToken) - - @property - def shows_typing(self): - """⏳ on the message being answered, with the bot token.""" - return bool(self.message and config.env.slackBotToken) - - @property - def takes_buttons(self): - """Buttons only where their presses reach finbot: with the bot token and slackOutgoingWebhook, the URL that the - app's interactivity posts to as its events do.""" - return bool(config.env.slackBotToken and config.env.slackOutgoingWebhook) - - def typing(self): - """Put ⏳ on the message being answered, once (the bot needs the reactions:write scope); a failure is only - logged, and not tried again.""" - if self.working is not None: - return - try: - slack_api('reactions.add', channel=self.where, timestamp=self.message, name=self.WORKING) - self.working = True - except Exception as e: - self.working = False - print("Slack reactions.add failed:", net.redact(e), file=sys.stderr) - - def typing_done(self): - """Take ⏳ off the message again; a failure is only logged.""" - if not self.working: - return - self.working = False - try: - slack_api('reactions.remove', channel=self.where, timestamp=self.message, name=self.WORKING) - except Exception as e: - print("Slack reactions.remove failed:", net.redact(e), file=sys.stderr) - - def post_text(self, text, buttons=None): - """chat.postMessage, without link previews, and with buttons (a Block Kit message) where they work. Slack's - answer, or None when it failed or refused (logged): Slack refuses with HTTP 200 and ok false (not_in_channel, - say), so a refusal is read from the answer.""" - if not config.env.slackBotToken: - print(f"ERROR slack: a reply to channel {self.where} needs slackBotToken in finbot.env; not sent", - file=sys.stderr) - return None - blocks = {} - if buttons and not isinstance(buttons, dict) and self.takes_buttons: - blocks['blocks'] = json.dumps(text_blocks(text, buttons)) - try: - answer = slack_api('chat.postMessage', channel=self.where, text=text, unfurl_links='false', - unfurl_media='false', **blocks) - except Exception as e: - print("Slack chat.postMessage to", self.where, "failed:", net.redact(e), file=sys.stderr) - return None - print("OK slack message to", self.where, file=sys.stderr) - return answer - - def post_chart(self, image, caption, buttons=None): - """Upload the chart into the channel with the report as its message, the way Slack takes files since it retired - files.upload: ask for an upload URL, send the bytes there, then complete the upload into the channel (the bot - needs the files:write and files:read scopes). With period buttons, where they work, it goes in a Block Kit - message instead (post_blocks), or without them if that fails. Slack's answer, MAYBE_SENT, or None if a step - failed.""" - data = image.read() if hasattr(image, 'read') else bytes(image) - if buttons and not isinstance(buttons, dict) and self.takes_buttons: - try: - return self.post_blocks(data, caption, buttons) - except Exception as e: - print("Slack chart with buttons failed, uploading it without them:", net.redact(e), file=sys.stderr) - try: - share = {'files': json.dumps([{'id': self.upload(data)}]), 'channel_id': self.where, - 'initial_comment': caption} - try: - output = slack_api('files.completeUploadExternal', **share) - except requests.exceptions.ReadTimeout as e: - print("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e), - file=sys.stderr) - return MAYBE_SENT - except Exception as e: - print("Slack chart upload failed:", net.redact(e), file=sys.stderr) - return None - print("OK slack chart upload to", self.where, file=sys.stderr) - return output - - def upload(self, data): - """Send a chart's bytes to Slack: the id of the file, not completed yet. Raises RuntimeError on a refusal.""" - ticket = slack_api('files.getUploadURLExternal', filename='chart.png', length=len(data)) - r = net.post(ticket['upload_url'], data=data) - if r.status_code != 200: - raise RuntimeError(f"Slack upload: {r.status_code} {r.text[:200]}") - return ticket['file_id'] - - def unshared(self, data): - """Upload a chart as a file that is in no channel, for an image block to show: its id.""" - file_id = self.upload(data) - slack_api('files.completeUploadExternal', files=json.dumps([{'id': file_id, 'title': 'chart'}])) - return file_id - - def post_blocks(self, data, caption, buttons): - """The chart in a Block Kit message, the report above it and the period buttons under it. Slack's answer, or - MAYBE_SENT when posting the message timed out (it may have arrived); raises RuntimeError when Slack refuses.""" - file_id = self.unshared(data) - blocks = chart_blocks(caption, buttons, file_id) - try: - answer = until_shown(lambda: slack_api('chat.postMessage', channel=self.where, - text=caption[:SECTION_LIMIT], blocks=json.dumps(blocks), unfurl_links='false', unfurl_media='false')) - except requests.exceptions.ReadTimeout as e: - print("Slack chat.postMessage timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT - if answer.get('ts'): - self.delete(self.shows(answer['ts'], file_id, caption, buttons)) - return answer - - def reshow(self, message, buttons): - """Show a chart message's period again from its upload of under SLACK_REUSE_SECONDS ago (for this message or - another), with the report it came with: nothing is drawn or uploaded. True when it did; False when there is no - such upload, or Slack refuses it (a file deleted outside finbot, say).""" - view, now = marked(buttons), time.time() - uploads = state.load(SLACK_CHARTS, {}).get('uploads', {}) - recent = [(at, file_id, caption) for file_id, (at, shows, caption) in uploads.items() - if view and shows == view and now - at < SLACK_REUSE_SECONDS] - if not recent: - return False - _, file_id, caption = max(recent) - try: - slack_api('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], - blocks=json.dumps(chart_blocks(caption, buttons, file_id))) - except Exception as e: - print("Slack chat.update with an earlier upload failed, uploading anew:", net.redact(e), file=sys.stderr) - return False - self.delete(self.shows(message, file_id)) - return True - - def edit_chart(self, message, image, caption, buttons=None): - """Show another period in a chart message: the new chart uploaded (a file cannot be replaced) and shown in its - image block, with its caption and buttons. A press for that period within SLACK_REUSE_SECONDS shows this upload - again (reshow). Raises RuntimeError when Slack refuses the update.""" - if hasattr(image, 'seek'): - image.seek(0) - file_id = self.unshared(image.read() if hasattr(image, 'read') else bytes(image)) - blocks = chart_blocks(caption, buttons or [], file_id) - until_shown(lambda: slack_api('chat.update', channel=self.where, ts=message, text=caption[:SECTION_LIMIT], - blocks=json.dumps(blocks))) - self.delete(self.shows(message, file_id, caption, buttons)) - - def shows(self, message, file_id, caption=None, buttons=None): - """Note that a chart message shows this file now; given its report, that the file was uploaded now for the - period its buttons mark (see reshow). The uploads to delete: those no message shows, a minute past their reuse - (a press reusing one has that minute to show it).""" - now = time.time() - with state.edit(SLACK_CHARTS, {}) as noted: - if set(noted) != {'shown', 'uploads'}: # new, or the {'channel ts': file} of before, whose files stay - noted.clear() - noted.update(shown={}, uploads={}) - shown, uploads = noted['shown'], noted['uploads'] # 'channel ts': file; file: [time, period, report] - if caption is not None: - uploads[file_id] = [now, marked(buttons), caption] - key = f'{self.where} {message}' - shown.pop(key, None) - shown[key] = file_id - for old in list(shown)[:-SLACK_CHARTS_KEPT]: # its message may show the file for good - uploads.pop(shown.pop(old), None) - showing = set(shown.values()) - gone = [old for old, (at, _, _) in uploads.items() - if old not in showing and now - at > SLACK_REUSE_SECONDS + 60] - for old in gone: - del uploads[old] - for upload in uploads.values(): # past reuse, a file shown is kept only to be deleted later - if now - upload[0] >= SLACK_REUSE_SECONDS: - upload[1:] = [None, None] - return gone - - def delete(self, files): - """Delete uploaded charts that no message shows any more; a failure is only logged (the file stays among the - bot's files, in no channel).""" - for file_id in files: - try: - slack_api('files.delete', file=file_id) - except Exception as e: - print("Slack files.delete failed:", net.redact(e), file=sys.stderr) - - def edit_text(self, message, text, buttons=None): - """Edit a message's text, and its buttons unless buttons is None ([] removes them); a failure is only logged.""" - data = {'channel': self.where, 'ts': message, 'text': text} - if buttons is not None: - data['blocks'] = json.dumps(text_blocks(text, buttons)) - try: - slack_api('chat.update', **data) - except Exception as e: - print("Slack chat.update failed:", net.redact(e), file=sys.stderr) - - -class SlackWebhook(Transport): - """A Slack incoming webhook: text only, into the channel it was made for.""" - name = 'slack' - dialect = dialects.SLACK - - def post_text(self, text, buttons=None): - """Post to the webhook, without link previews.""" - return self._post_json(self.where, {'text': text, 'unfurl_links': False, 'unfurl_media': False}) - - -class DiscordWebhook(Transport): - """A Discord webhook's native endpoint (the Slack one takes no flags): no previews, no pings, charts attached.""" - name = 'discord' - dialect = dialects.DISCORD - takes_charts = True - - def url(self): - """The webhook URL that answers with the message posted (200) instead of an empty 204.""" - return self.where + ('&' if '?' in self.where else '?') + 'wait=true' - - def post_text(self, text, buttons=None): - """Post the message.""" - return self._post_json(self.url(), {'content': text, 'flags': SUPPRESS_EMBEDS, 'allowed_mentions': NO_MENTIONS}) - - def post_chart(self, image, caption, buttons=None): - """The chart as an attachment, the report as its message: Discord's answer, MAYBE_SENT, or None if refused.""" - payload = {'content': caption, 'flags': SUPPRESS_EMBEDS, 'allowed_mentions': NO_MENTIONS, - 'attachments': [{'id': 0, 'filename': 'chart.png'}]} - try: - r = net.post(self.url(), data={'payload_json': json.dumps(payload)}, - files={'files[0]': ('chart.png', image, 'image/png')}) - except requests.exceptions.ReadTimeout as e: - print("Discord chart timed out, it may still arrive:", net.redact(e), file=sys.stderr) - return MAYBE_SENT - except Exception as e: - print("Discord chart failed:", net.redact(e), file=sys.stderr) - return None - if r.status_code == 200: - print(r.status_code, "OK discord chart", file=sys.stderr) - try: - return r.json() or {} - except ValueError: - return {} - print(r.status_code, "error discord chart", net.redact(r.text[:300]), file=sys.stderr) - return None - - -def slack_api(method, **data): - """POST a Slack Web API method as the bot (form-encoded) and return the answer; an error raises RuntimeError.""" - if not config.env.slackBotToken: - raise RuntimeError(f"Slack {method}: needs slackBotToken in finbot.env") - r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + config.env.slackBotToken}, - data=data) - try: - output = r.json() - except ValueError: - output = {} - if not output.get('ok'): - raise RuntimeError(f"Slack {method}: {output.get('error') or r.status_code}") - return output - - -SECTION_LIMIT = 3000 # characters of text in a Block Kit section - - -def section_blocks(text): - """Block Kit sections of a message's text, as many as its length needs (none for no text).""" - return [{'type': 'section', 'text': {'type': 'mrkdwn', 'text': '\n'.join(part)}} - for part in chunked(text.split('\n'), SECTION_LIMIT) if '\n'.join(part).strip()] - - -def actions_block(row): - """A Block Kit row of buttons, the one shown now marked.""" - return {'type': 'actions', 'elements': [{'type': 'button', 'text': {'type': 'plain_text', 'text': button.label}, - 'value': button.data, 'action_id': button.data, **({'style': 'primary'} if button.active else {})} - for button in row]} - - -def chart_blocks(caption, buttons, file_id): - """Block Kit for a chart message: the report, the chart (an uploaded file, by its id) and the buttons in one - row.""" - row = [button for line in buttons for button in line] - return section_blocks(caption) + [{'type': 'image', 'slack_file': {'id': file_id}, 'alt_text': 'chart'}] + \ - ([actions_block(row)] if row else []) - - -def marked(buttons): - """The data of the button marked as shown now, which names a chart message's period (None when none is).""" - return next((button.data for row in buttons or [] for button in row if button.active), None) - - -def text_blocks(text, buttons): - """Block Kit for a message with buttons: the text, then the buttons, a row of them to a block.""" - return section_blocks(text) + [actions_block(row) for row in buttons if row] - - -def until_shown(post, waits=(1, 2, 4)): - """post() of a message that shows a file just uploaded, tried again after each wait (in seconds) while Slack says - invalid_blocks: a file can take a moment before it can be shown.""" - for wait in waits: - try: - return post() - except RuntimeError as e: - if 'invalid_blocks' not in str(e): - raise - print(f"Slack cannot show the chart yet ({e}), trying again in {wait} s", file=sys.stderr) - time.sleep(wait) - return post() - def chunked(lines, limit): """The lines in messages of at most limit characters each, whole lines to a message; a line longer than limit on diff --git a/lib/watchlist.py b/lib/watchlist.py index 27a47af0..3aee9397 100644 --- a/lib/watchlist.py +++ b/lib/watchlist.py @@ -1,5 +1,6 @@ -"""The watchlist: tickers followed beside the Sharesight holdings, in DATA_DIR/finbot_watchlist.json. Every call reads -the file, so the bot and its scheduled reports (other processes) always see the same list.""" +"""The watchlist: each chat's tickers followed beside the Sharesight holdings, in DATA_DIR/finbot_watchlist.json, by +the chat's key ({'chats': {chat key: [tickers]}}, as state.load_for keeps it). Every call reads the file, so every +thread sees the same lists. The holdings are the same for every chat: a portfolio is not a chat's.""" from lib import config from lib import markets from lib import sharesight @@ -8,48 +9,63 @@ FILE = 'finbot_watchlist.json' -def tickers(): - """The watched tickers, in the order they were added.""" - return state.load(FILE, []) +def tickers(chat): + """A chat's watched tickers (by its key), in the order they were added.""" + return state.load_for(chat, FILE, []) -def watched(typed): - """The watched ticker that a ticker as written names: BHP.AX as written, or BHP on the local exchange (the - exchange_suffix preference) when only BHP.AX is watched; else Yahoo's spelling of what was written. Yahoo is not - asked, so a ticker it no longer knows can still be removed.""" - ticker = markets.transform_to_yahoo(typed.upper()) +def watched(chat, typed): + """The ticker of a chat's watchlist that a ticker as written names: BHP.AX as written, or BHP on the local exchange + (the exchange_suffix preference) when only BHP.AX is watched; else Yahoo's spelling of what was written. Yahoo is + not asked, so a ticker it no longer knows can still be removed.""" + ticker, watching = markets.transform_to_yahoo(typed.upper()), tickers(chat) local = f'{ticker}.{config.prefs.exchange_suffix}' - if ticker not in tickers() and config.prefs.exchange_suffix and '.' not in ticker and local in tickers(): + if ticker not in watching and config.prefs.exchange_suffix and '.' not in ticker and local in watching: return local return ticker -def add(ticker): - """Watch a ticker (check it with find first). False when it was already watched.""" - with state.edit(FILE, []) as watched: - if ticker in watched: +def add(chat, ticker): + """Add a ticker to a chat's watchlist (check it with find first). False when it was there already.""" + with state.edit_for(chat, FILE, []) as watching: + if ticker in watching: return False - watched.append(ticker) + watching.append(ticker) return True -def remove(ticker): - """Stop watching a ticker. False when it was not watched.""" - with state.edit(FILE, []) as watched: - if ticker not in watched: +def remove(chat, ticker): + """Take a ticker off a chat's watchlist. False when it was not on it.""" + with state.edit_for(chat, FILE, []) as watching: + if ticker not in watching: return False - watched.remove(ticker) + watching.remove(ticker) return True -def covered(ticker=None): - """The tickers a report covers: the one asked about, else every stock followed (with_holdings).""" - return [ticker] if ticker else with_holdings() +def covered(chat, ticker=None): + """The tickers a report covers in a chat: the one asked about, else every stock the chat follows (with_holdings).""" + return [ticker] if ticker else with_holdings(chat) -def with_holdings(): - """The watchlist and every Sharesight holding: the tickers the scheduled reports cover, sorted.""" - followed = set(sharesight.tickers()) | set(tickers()) +def with_holdings(chat): + """A chat's watchlist and every Sharesight holding: the tickers its reports cover, sorted.""" + return _followed(tickers(chat)) + + +def all_watched(): + """Every chat's watched tickers, each once, sorted.""" + return sorted({ticker for watching in state.load(FILE, {}).get('chats', {}).values() for ticker in watching}) + + +def all_followed(): + """Every chat's watchlist and every Sharesight holding, sorted: what yahoo_warm keeps warm.""" + return _followed(all_watched()) + + +def _followed(watching): + """The holdings and the tickers watched, sorted; GOOGL left out beside GOOG, its other class of shares.""" + followed = set(sharesight.tickers()) | set(watching) if 'GOOG' in followed and 'GOOGL' in followed: followed.remove("GOOGL") return sorted(followed) diff --git a/lib/worker.py b/lib/worker.py index db14f579..442688d8 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -5,8 +5,8 @@ import json import math import re -import sys import typing +from lib import log from lib import answer from lib import dialects from lib import state @@ -17,7 +17,7 @@ from lib import charts from lib.reports import about, cal, holdings, performance, price, profile, rankings, shorts, trades from lib import sharesight -from lib import transports +from lib import telegram_common from lib import jobs from lib import watchlist from lib import yahoo @@ -31,10 +31,11 @@ class Request: user_name: str # what to call them bot_name: str message_id: str = None # the message being answered + interaction: str = '' # what the network answers through, where it has one (Discord's slash command) def __post_init__(self): """The chat's transport, which also keeps the typing notice going.""" - self.dest = delivery.reply_to(self.service, self.chat_id, self.message_id) + self.dest = delivery.reply_to(self.service, self.chat_id, self.message_id, self.interaction) def reply(self, lines, chart=None, buttons=None): """Answer in the chat (see delivery.deliver): the lines, perhaps a chart (drawn, or a spec drawn only where it @@ -42,44 +43,46 @@ def reply(self, lines, chart=None, buttons=None): return delivery.deliver(self.dest, delivery.Report(lines, chart, buttons)) def process_job_callback(press, pressed): - """A command button of .jobs (an inbound.Press; pressed, what its data says): run that command now and answer in - the chat.""" + """A job's button (an inbound.Press; pressed, what its data says): run the job now and say how it went, or switch + it on or off and redraw the list in place. A button from an older list does neither: the list is redrawn as it is + now.""" service = dialects.of(press.service) - chat = delivery.reply_to(service, press.chat_id) + chat = delivery.reply_to(service, press.chat_id, interaction=press.interaction, pressed=True) + def redraw(): + """Put the chat's list, as it is now, in place of the pressed one, with the same kind of buttons.""" + payload, markup = about.prepare_jobs_payload(service, chat.key, pressed.action, chat.takes_buttons) + chat.edit_text(press.message_id, '\n'.join(payload), markup or []) try: - if jobs.current is None: # the scheduler is off now - raise jobs.ScheduleError('the scheduler is off') - command = jobs.current.pick(pressed.number, pressed.checksum) # only a command .jobs offers; anything else fails + if jobs.current is None: + raise jobs.ScheduleError('no schedule is running') + job = jobs.current.pick(chat.key, pressed.number, pressed.checksum) # only this chat's job, as listed except jobs.ScheduleError: - chat.acknowledge(press.press_id, 'Buttons updated, try again') - payload, markup = about.prepare_jobs_payload(service, '', chat.takes_buttons) # the list now, for the stale one - chat.edit_text(press.message_id, '\n'.join(payload), markup or []) - return - chat.acknowledge(press.press_id, f'Running {jobs.display(command[0])}…'[:100]) + return redraw() try: - line = about.job_line(jobs.current, *jobs.current.run_command(command)) - delivery.deliver(chat, delivery.Report([line])) + if pressed.action == 'toggle': + jobs.current.toggle(job) + redraw() + else: + delivery.deliver(chat, delivery.Report([about.job_line(pressed.number, job, jobs.current.run(job))])) except Exception as e: delivery.report_error(e, chat, context='jobs') def process_callback(press): - """A button pressed (an inbound.Press): run a scheduled command from .jobs, or rebuild a chart message for the - chosen period and edit it in place.""" + """A button pressed (an inbound.Press, which bot.take acknowledged as it arrived): a job's (run it, or switch it), + or a chart's, whose message is rebuilt for the chosen period and edited in place. Data of neither is ignored.""" pressed = buttons.parse(press.data) if isinstance(pressed, buttons.JobPress): return process_job_callback(press, pressed) - service = dialects.of(press.service) - chat = delivery.reply_to(service, press.chat_id) - # answering now shows Telegram's toast bubble ("Loading 1Y…") while the new chart is built - chat.acknowledge(press.press_id, f"Loading {buttons.PERIODS[pressed.period][0]}…" if pressed else None) if not pressed: return + service = dialects.of(press.service) + chat = delivery.reply_to(service, press.chat_id, interaction=press.interaction, pressed=True) try: args = buttons.resolve(pressed.ref) markup = buttons.keyboard(pressed.kind, args, pressed.period) if chat.reshow(press.message_id, markup): # that period as posted lately, where the network keeps it (Slack) return - caption, chart = charts.build(pressed.kind, args, service, period=pressed.period) + caption, chart = charts.build(pressed.kind, args, service, chat.key, period=pressed.period) if chart is None: # too few prices in that period to draw a line raise answer.Answer(f"Too few prices to draw {buttons.PERIODS[pressed.period][0]}") chat.edit_chart(press.message_id, render.draw(chart), caption, markup) @@ -94,7 +97,7 @@ def process_request(message): none gets no answer. Arguments the command cannot take get its usage, and any failure a one-line reply in the chat.""" request = Request(dialects.of(message.service), message.chat_id, message.user, message.user_name, message.bot_name, - message.message_id) + message.message_id, message.interaction) command, args = read(message.text, message.bot_name) if command: try: @@ -108,11 +111,13 @@ def process_request(message): delivery.report_error(e, request.dest, context=message.text.split()[0][:30]) finally: request.dest.end_typing() + elif message.interaction: # a slash command that reads as none: its notice taken off (Discord's "thinking…") + request.dest.end_typing() if message.service == 'telegram': try: ensure_keyboard(message.chat_id) # keep DM keyboards current after the reply except Exception as e: - print("keyboard update failed:", e, file=sys.stderr) + log.warn("keyboard update failed:", e) @dataclasses.dataclass class Args: @@ -210,21 +215,24 @@ def portfolios(): return 'Portfolios: ' + ', '.join(portfolio.name for portfolio in sharesight.portfolios()) def on_watchlist(request, args): - """.watchlist [add|del SYMBOL]: the watchlist, or a change to it.""" + """.watchlist [add|del SYMBOL]: this chat's watchlist, or a change to it.""" + chat = request.dest.key if args.keyword and len(args.words) == 1: # what is removed is what is watched: Yahoo may know it no longer - symbol = ticker(args) if args.keyword == 'add' else watchlist.watched(args.words[0]) - request.reply(holdings.prepare_watchlist(request.service, request.user, args.keyword, symbol)) + symbol = ticker(args) if args.keyword == 'add' else watchlist.watched(chat, args.words[0]) + request.reply(holdings.prepare_watchlist(request.service, chat, request.user, args.keyword, symbol)) elif args.keyword or args.words: raise Usage else: - request.reply(holdings.prepare_watchlist(request.service, request.user)) + request.reply(holdings.prepare_watchlist(request.service, chat, request.user)) def on_help(request, args): - """.help: every command; in a Telegram DM the command keyboard too.""" + """.help: every command (on Discord as its slash command, which works in both modes); in a Telegram DM the command + keyboard too.""" markup = None if request.service == 'telegram' and int(request.chat_id) > 0: # positive chat ids are DMs; groups are negative markup = command_keyboard() - request.reply(about.prepare_help(request.service, request.bot_name, help_sections()), buttons=markup) + sections = help_sections(slash=request.service == 'discord') + request.reply(about.prepare_help(request.service, request.bot_name, sections), buttons=markup) if markup: mark_keyboard(request.chat_id) # this chat now has the current keyboard @@ -276,7 +284,8 @@ def on_marketcap(request, args): if args.words: request.reply([rankings.marketcap_line(request.service, ticker(args))]) else: - request.reply(rankings.marketcap_list(request.service, bottom=args.keyword == 'bottom', length=15)) + request.reply(rankings.marketcap_list(request.service, request.dest.key, bottom=args.keyword == 'bottom', + length=15)) def on_ratio(kind): """The handler of .pe, .forward pe or .peg (kind: pe, forward pe or peg): the top (by default) or bottom 15 by the @@ -286,12 +295,13 @@ def handler(request, args): if args.words: request.reply([rankings.ratio_line(request.service, ticker(args), kind)]) else: - request.reply(*rankings.ratio_list(request.service, kind, args.keyword or 'top', length=15)) + mode = args.keyword or 'top' + request.reply(*rankings.ratio_list(request.service, request.dest.key, kind, mode, length=15)) return handler def on_beta(request, args): """.beta: the volatile small caps among the stocks followed.""" - payload, chart = rankings.prepare_beta_payload(request.service) + payload, chart = rankings.prepare_beta_payload(request.service, request.dest.key) request.reply(payload, chart) def on_pie(request, args): @@ -304,7 +314,7 @@ def on_pie(request, args): def on_rating(request, args): """.buy, .sell or both (.buy & sell, a keyboard button): the analysts' strongest buy or sell ratings, as asked.""" for action in dict.fromkeys(re.findall('buy|sell', f'{args.word} {args.text}'.lower())): # each once, as asked - payload, chart = rankings.prepare_rating_payload(request.service, action, length=15) + payload, chart = rankings.prepare_rating_payload(request.service, request.dest.key, action, length=15) request.reply(payload, chart) def on_compare(request, args): @@ -325,18 +335,18 @@ def on_history(request, args): def chart_reply(request, kind, args, days=None): """The first view of a chart with period buttons, built as a button press builds it (charts.build): its caption, chart and buttons where they go, the period shown marked (Max when days is None).""" - caption, chart = charts.build(kind, args, request.service, days=days) + caption, chart = charts.build(kind, args, request.service, request.dest.key, days=days) markup = None if chart and request.dest.takes_buttons: markup = buttons.keyboard(kind, args, buttons.period_for_days(days) if days else 'x') request.reply(caption.split('\n'), chart, markup) def on_plan(request, args): - """.plan [text]: record the sender's plan, and list everyone's.""" + """.plan [text]: record the sender's plan in this chat, and list everyone's here.""" if args.text and not args.text.startswith('@'): - with state.edit('finbot_plan.json', {}) as plans: + with state.edit_for(request.dest.key, PLANS, {}) as plans: plans[request.user] = args.text - plan = state.load('finbot_plan.json', {}) + plan = state.load_for(request.dest.key, PLANS, {}) if not plan: raise answer.Answer("No plans yet: .plan TEXT records yours") def who(user): @@ -354,15 +364,52 @@ def on_who(request, args): request.reply(holdings.prepare_who(request.service, re.split(r'[.:]', args.words[0].upper())[0])) def on_jobs(request, args): - """.jobs [run NUMBER|COMMAND]: the schedule, or one of its commands run now.""" - if bool(args.keyword) != bool(args.words): # run without a command, or words without run - raise Usage - payload, markup = about.prepare_jobs_payload(request.service, args.rest or None, request.dest.takes_buttons) - request.reply(payload, buttons=markup) + """.jobs: this chat's jobs; .jobs run or toggle [NUMBER]: one run now or switched on or off (without a number, the + list with a button per job that does it); .jobs add [CRON_TZ=ZONE] TIME REPORT: one more (alone, the reports it can + run); .jobs del NUMBER: one fewer.""" + scheduler, chat, service = jobs.current, request.dest.key, request.service + if scheduler is None: + raise answer.Answer('No schedule is running') + try: + if args.keyword is None and not args.words: + request.reply(about.prepare_jobs_payload(service, chat)[0]) + elif args.keyword == 'add': + first, line = (args.text.split(None, 1) + [''])[:2] + if first.lower() != 'add': # the job's line follows add + raise Usage + if not line: + return request.reply(about.prepare_reports(service)) + number = scheduler.add(chat, line) + request.reply([about.job_entry(service, scheduler, number, scheduler.chat_jobs(chat)[number - 1]), + 'Added: .jobs lists them all']) + elif args.keyword in ('run', 'toggle') and not args.words: + payload, markup = about.prepare_jobs_payload(service, chat, args.keyword, request.dest.takes_buttons) + request.reply(payload, buttons=markup) + elif args.keyword in ('run', 'toggle', 'del') and len(args.words) == 1: + number = args.words[0] + job = scheduler.job(chat, number) + if args.keyword == 'run': + request.reply([about.job_line(number, job, scheduler.run(job))]) + elif args.keyword == 'toggle': + scheduler.toggle(job) + request.reply([about.job_entry(service, scheduler, number, scheduler.job(chat, number))]) + else: + scheduler.delete(job) + request.reply([f"Deleted {number}. {job.line}"]) + else: + raise Usage + except jobs.ScheduleError as e: # a job that cannot be, or a number that is none: said, as an answer + raise answer.Answer(str(e)) def on_profile(request, args): - """.SYMBOL: the company's profile.""" - request.reply(profile.prepare_profile_payload(request.service, request.user, yahoo.symbol(args.word))) + """.SYMBOL, or .stock (.profile) SYMBOL: the company's profile.""" + if args.word.lower() in PROFILE.names: + if len(args.words) != 1: + raise Usage + symbol = ticker(args) + else: + symbol = yahoo.symbol(args.word) + request.reply(profile.prepare_profile_payload(request.service, request.user, symbol)) @dataclasses.dataclass(frozen=True) class Command: @@ -384,9 +431,13 @@ class Command: TOP = {'top': 'top', 'bottom': 'bottom'} # the ends of a ranked list NEGATIVE = TOP | dict.fromkeys(('neg', 'negative', 'negatives'), 'negative') # the ratios that can be negative CHANGE = {'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), 'delete') # of the watchlist +PLANS = 'finbot_plan.json' # each chat's plans, {user: plan}, as state.load_for keeps them BY = dict.fromkeys(('sector', 'sectors'), 'sector') | dict.fromkeys(('industry', 'industries'), 'industry') # .pie's +JOB_CHANGES = {'run': 'run', 'toggle': 'toggle', 'add': 'add'} | dict.fromkeys(('del', 'delete', 'rem', 'remove', 'rm'), + 'del') # what .jobs does to a chat's jobs BUY_SELL = ('buy', 'sell', *(a + sep + b for a, b in (('buy', 'sell'), ('sell', 'buy')) for sep in ('', '&', '/', '+'))) -PROFILE = Command((), on_profile, ('.SYMBOL',), COMPANY) # any other first word that may be a ticker +PROFILE = Command(('stock', 'profile'), on_profile, ('.SYMBOL', '.stock SYMBOL'), COMPANY) # any first word that is no +# command's name but may be a ticker calls it too COMMANDS = [ # .help lists a section's commands in this order. The odd spellings (earrings, prince, hospital...) are # kept on purpose. @@ -418,15 +469,141 @@ class Command: Command(('shorts', 'short'), on_shorts, ('.shorts [percent|SYMBOL]',), RISK, ('.shorts',), percent=True), Command(('trades', 'trade'), on_trades, ('.trades [period] [portfolio]',), TRADES, ('.trades',), period=True), Command(('plan',), on_plan, ('.plan [your plan]',), TRADES), - Command(('jobs', 'cron', 'crons', 'schedule'), on_jobs, ('.jobs [run NUMBER|COMMAND]',), JOBS, - keywords={'run': 'run'}), + Command(('jobs', 'job', 'cron', 'crons', 'schedule'), on_jobs, ('.jobs [run|toggle] [NUMBER]', + '.jobs add [CRON_TZ=ZONE] TIME REPORT', '.jobs del NUMBER'), JOBS, keywords=JOB_CHANGES), PROFILE, ] -def help_sections(): - """The sections of .help: (heading, the .help lines of its commands, in table order), in SECTIONS order.""" - return [(section, [line for command in COMMANDS if command.section == section for line in command.usage]) - for section in SECTIONS] +def help_sections(slash=False): + """The sections of .help: (heading, the .help lines of its commands, in table order), in SECTIONS order; with slash, + each line as its slash command (/forwardpe for .forward pe, /stock SYMBOL for .SYMBOL).""" + def shown(command, line): + """The line, or its slash command.""" + form = slash_form(command, line) if slash else None + return f'/{form[0]} {form[1]}'.strip() if form else line + return [(section, list(dict.fromkeys(shown(command, line) for command in COMMANDS if command.section == section + for line in command.usage))) for section in SECTIONS] + +SLASH_NAME = re.compile(r'[a-z0-9_]{1,32}') # what a slash command may be named, on every network +SLASH_ALIASES = {'stock': 'profile', 'jobs': 'job'} # slash commands with a second name, which the networks that list +# every slash command list too (Discord, Slack); typed, the second name works on Telegram too, as every name does + +def slash_form(command, line): + """The slash command of one of a command's .help lines, (name, the arguments it shows): named by the command's name + the line starts with where a slash command may be so named, else by the command's first name that may (.forward pe + is /forwardpe, .SYMBOL /stock SYMBOL); None when none may.""" + valid = [name for name in command.names if SLASH_NAME.fullmatch(name)] + if not valid: + return None + words = line.removeprefix('.').split() + named = next((name for name in command.names if [w.lower() for w in words[:len(name.split())]] == name.split()), + None) + if named is None: + return valid[0], ' '.join(words) + return (named if named in valid else valid[0]), ' '.join(words[len(named.split()):]) + +class Slash(typing.NamedTuple): + """A slash command to register: its name, its description, the arguments it shows, and whether Discord asks for + suggestions while they are typed (suggest).""" + name: str + description: str + arguments: str + completes: bool = False + +def slash_commands(aliases=True): + """The slash commands to register (Slash), from the .help lines (slash_form), and /help; the arguments of a + command's several lines joined by " | ". With aliases, the second names of SLASH_ALIASES too, each after its + first.""" + found = {'help': (None, [])} # name -> (its command, the arguments of its lines) + for command in COMMANDS: + for line in command.usage: + if form := slash_form(command, line): + name, arguments = form + for each in (name, SLASH_ALIASES.get(name) if aliases else None): + if each and arguments not in found.setdefault(each, (command, []))[1]: + found[each][1].append(arguments) + return [Slash('help', 'every command, and what it takes', '') if name == 'help' else + Slash(name, f"{name} {' | '.join(lines)}".strip(), ' | '.join(lines), completes(command)) + for name, (command, lines) in found.items()] + +PERIOD_WORDS = ('1d', '7d', '1m', '3m', '6m', 'ytd', '1y', '5y', '10y') # suggested where a command takes a period +JOB_WORDS = ('run', 'toggle', 'add', 'del') # what /jobs suggests first + +def completes(command): + """Whether suggest has words to offer for a command's arguments: its keywords, periods, tickers, portfolios, or + those of /jobs.""" + usage = ' '.join(command.usage) + return bool(command.handler is on_jobs or command.keywords or command.period or 'SYMBOL' in usage or + 'portfolio' in usage.lower()) + +def suggest(chat, text): + """What to suggest while a slash command's arguments are typed (Discord's autocomplete), from the message they make + so far (text: /price 5 7) in a chat (its key): up to 25 (name shown, value) pairs, each value the arguments with the + word being typed completed. The words come from the command's entry in the table: its keywords, periods, portfolios + and tickers (the chat's watchlist and the holdings, as cached, never a request: suggestions must come at once); for + /jobs, its subcommands, this chat's job numbers and the reports. Never raises: the failure is logged instead, and + nothing suggested.""" + try: + command, _ = read(text, '') + if command is None: + return [] + given = (text.split(None, 1) + [''])[1] + words = given.split() + typing = words.pop() if words and not given[-1].isspace() else '' + done = ' '.join(words) + if command.handler is on_jobs: + offered = job_words(chat, words) + else: + offered = argument_words(command, chat, read_args(command, '', done)) + found = [(f'{done} {shown}'.strip()[:100], f'{done} {word}'.strip()) for word, shown in offered + if word.lower().startswith(typing.lower())] + return [(shown, value) for shown, value in found if len(value) <= 100][:25] + except Exception as e: + log.warn('suggest:', text, delivery.error_line(e)) + return [] + +def argument_words(command, chat, given): + """(word, shown) pairs that a command's arguments may hold next, for suggest, after those given (Args): the + keywords its usage shows and periods, unless it has one already; portfolios; tickers (the chat's watchlist and the + holdings, as cached).""" + usage = ' '.join(command.usage) + words = [word for word in re.findall(r'[\w-]+', usage) + if word in command.keywords and word not in command.names and given.keyword is None] + if command.period and given.days is None: + words += PERIOD_WORDS + if 'SYMBOL' in usage or 'portfolio' in usage.lower(): + names, held = sharesight.cached() + if 'portfolio' in usage.lower(): + words += names + if 'SYMBOL' in usage: + words += sorted(set(watchlist.tickers(chat)) | set(held)) + return [(word, word) for word in dict.fromkeys(words)] + +def job_words(chat, words): + """(word, shown) pairs for /jobs, for suggest, after the words given: first its subcommands; after run, toggle or + del, this chat's job numbers, each shown with its line; after add and a time, the reports, each shown with what it + takes; after a report's name, the words it takes.""" + if not words: + return [(word, word) for word in JOB_WORDS] + action = JOB_CHANGES.get(words[0].lower()) + if action in ('run', 'toggle', 'del') and len(words) == 1: + own = jobs.current.chat_jobs(chat) if jobs.current else [] + return [(str(number), f'{number}: {job.line}') for number, job in enumerate(own, 1)] + if action != 'add': + return [] + line = words[1:][1:] if words[1:2] and words[1].startswith('CRON_TZ=') else words[1:] + time = 1 if line and line[0].startswith('@') else 5 + if len(line) < time: + return [] + after = line[time:] + if not after or after[-1].endswith(';'): # a report's name is next + return [(name, f'{name} {module.ARGUMENTS}'.strip()) for name, module in sorted(jobs.SCRIPTS.items()) + if name not in jobs.SYSTEM_SCRIPTS] + start = max(i for i in range(len(after)) if i == 0 or after[i - 1].endswith(';')) # the report being given + module = jobs.SCRIPTS.get(after[start].rstrip(';')) + if module is None or len(after) - start > 1 or after[start].endswith(';'): + return [] + return [(word, word) for word in re.findall(r'[a-z][a-z-]*', module.ARGUMENTS)] def command_keyboard(per_row=3): """Persistent reply keyboard of the commands that have buttons (Telegram DMs), alphabetical, row by row; a press @@ -448,7 +625,7 @@ def mark_keyboard(chat_id): def push_keyboard(chat_id): """Telegram can only change a reply keyboard by sending a message, so send a short one carrying it.""" - result = transports.TelegramBot(chat_id).post_text('⌨️ Keyboard updated', command_keyboard()) + result = telegram_common.TelegramBot(chat_id).post_text('⌨️ Keyboard updated', command_keyboard()) if result is not None: mark_keyboard(chat_id) return result @@ -467,4 +644,4 @@ def refresh_keyboards(): try: push_keyboard(chat_id) except Exception as e: - print("keyboard refresh failed for", chat_id, e, file=sys.stderr) + log.warn("keyboard refresh failed for", chat_id, e) diff --git a/lib/yahoo.py b/lib/yahoo.py index c9a0e9c5..0668eed1 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -5,7 +5,7 @@ import threading import typing import zoneinfo -import sys +from lib import log from lib import answer from lib import config from lib import cache @@ -37,13 +37,13 @@ def credentials(renew=False): try: r = net.get(url, headers={**HEADERS, 'Cookie': cookie}) except Exception as e: - print("Yahoo crumb:", net.redact(e), file=sys.stderr) + log.warn("Yahoo crumb:", net.redact(e)) continue if r.status_code == 200 and r.text.strip(): cache.write(CREDENTIALS, [cookie, r.text.strip()]) - print("Got a new Yahoo cookie and crumb", file=sys.stderr) + log.info("Got a new Yahoo cookie and crumb") return cookie, r.text.strip() - print(r.status_code, "from Yahoo for a crumb", file=sys.stderr) + log.warn(r.status_code, "from Yahoo for a crumb") raise RuntimeError("Yahoo gave no crumb") def _hosts(path): @@ -62,14 +62,14 @@ def _answer(query, what): r = net.get(url, headers={**HEADERS, 'Cookie': cookie}) except Exception as e: refusal = f"Yahoo unreachable for {what}: {net.redact(e)}" - print(refusal, file=sys.stderr) + log.warn(refusal) continue if r.status_code == 200: return r.json() if r.status_code == 404 and not_found(r): return None refusal = f"{r.status_code} error for {what} from Yahoo" - print(refusal + ':', r.text.strip()[:200], file=sys.stderr) + log.warn(refusal + ':', r.text.strip()[:200]) if not refusal.startswith('401 '): # only a refused crumb is worth a new one break raise RuntimeError(refusal) @@ -111,10 +111,13 @@ def _positive(value): return value def _convert(convert, value): - """convert(value), or None for garbage where a number belongs (left out, like a missing field).""" + """convert(value); None for a field Yahoo leaves out or sends as null (an ETF has no analysts' recommendation), and + for garbage, such as text where a number belongs (left out, like a missing field).""" + if value is None: + return None try: return convert(value) if convert else value - except (TypeError, ValueError, OverflowError): + except (TypeError, ValueError, OverflowError, AttributeError): return None # (key, Yahoo's name, conversion) for the fields of a v7 quote; one Yahoo leaves out, sends as null or garbles is left @@ -140,13 +143,13 @@ def fetch(tickers): """v7 quotes for many tickers in one request: {ticker: {...}}. Cached for 5 minutes (DO NOT CACHE LONGER). A ticker Yahoo gives no name or price for is left out. Raises RuntimeError when Yahoo cannot be reached.""" if not tickers: - print("no tickers provided to yahoo.fetch()", file=sys.stderr) + log.info("no tickers provided to yahoo.fetch()") return {} tickers = sorted(set(tickers)) # de-dupe name = "finbot_yahoo_quote_" + hashlib.sha256("_".join(tickers).encode()).hexdigest() + '.json' data = cache.cached(name, 300, lambda: _quote_answer(tickers)) if data is None: - print("Yahoo has no quotes for", ', '.join(tickers), file=sys.stderr) + log.warn("Yahoo has no quotes for", ', '.join(tickers)) return {} # callers iterate the result now = datetime.datetime.now().timestamp() yahoo_output = {} @@ -223,12 +226,12 @@ def fetch_detail(ticker, seconds=None): data = cache.cached("finbot_yahoo_detail_" + ticker + '.json', seconds, lambda: _detail_answer(ticker)) result = dig(data, 'quoteSummary', 'result', 0) if result is None: # Yahoo answers 404 Not Found, or result=null, for a symbol it does not know - print(ticker, "is unknown to Yahoo", file=sys.stderr) + log.warn(ticker, "is unknown to Yahoo") return {} title = next((name for name in (dig(result, 'price', 'longName'), dig(result, 'price', 'shortName')) if isinstance(name, str) and name), None) if not title: # catches some delisted stocks like "DUB" - print(ticker, "has no name at Yahoo", file=sys.stderr) + log.warn(ticker, "has no name at Yahoo") return {} detail = {'profile_title': formats.transform_title(title)} for key, path, *convert in DETAIL_FIELDS: @@ -364,18 +367,15 @@ def price_history(ticker, days=None, quote=None): tuple((str(day), float(last)) for day, last in closes)) def price_series(ticker): - """A ticker's daily closes over 10 years, from the same chart as price_history(): (name, exchange, pandas Series - indexed by date). Raises NoData when there is no history.""" - import pandas as pd + """A ticker's daily closes over 10 years, from the same chart as price_history(): (name, exchange, ((date, + close), ...)), oldest first, each date a datetime.date. Raises NoData when there is no history.""" data = fetch_chart_json(ticker) meta = dig(data, 'chart', 'result', 0, 'meta') or {} - closes = [(day, last) for day, _, last in bars(data) if last is not None] + closes = tuple((day, float(last)) for day, _, last in bars(data) if last is not None) if not closes: raise NoData(f"no price history for {ticker}") - series = pd.Series([float(last) for _, last in closes], index=pd.to_datetime([str(day) for day, _ in closes]), - name=ticker) name = formats.transform_title(meta.get('longName') or meta.get('shortName') or ticker) - return name, meta.get('fullExchangeName') or meta.get('exchangeName') or '', series + return name, meta.get('fullExchangeName') or meta.get('exchangeName') or '', closes def fetch_each(get, tickers, workers=8): """get(ticker) for each ticker, `workers` at a time so the waits overlap: [its result or the exception raised].""" @@ -420,7 +420,7 @@ def fetch_details(tickers, strict=False): if isinstance(detail, Exception): if strict: raise detail - print(ticker, detail, file=sys.stderr) + log.warn(ticker, detail) else: details |= detail return details @@ -452,7 +452,7 @@ def prefetch_history(tickers, workers=8, seconds=None): failed = [] for ticker, result in zip(tickers, fetch_each(lambda t: fetch_chart_json(t, seconds=seconds), tickers, workers)): if isinstance(result, Exception): - print("prefetch failed for", ticker, result, file=sys.stderr) + log.warn("prefetch failed for", ticker, result) failed.append(ticker) return failed diff --git a/milestone.py b/milestone.py deleted file mode 100755 index 29829cf3..00000000 --- a/milestone.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./milestone.py [arguments]: runs lib/reports/milestone.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/performance.py b/performance.py deleted file mode 100755 index e58e5d5b..00000000 --- a/performance.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./performance.py [arguments]: runs lib/reports/performance.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/podcasts.py b/podcasts.py deleted file mode 100755 index c86a624e..00000000 --- a/podcasts.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./podcasts.py [arguments]: runs lib/reports/podcasts.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/price.py b/price.py deleted file mode 100755 index 4aee8e91..00000000 --- a/price.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./price.py [arguments]: runs lib/reports/price.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/rating.py b/rating.py deleted file mode 100755 index 4a44d44f..00000000 --- a/rating.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./rating.py [arguments]: runs lib/reports/rating.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/reminder.py b/reminder.py deleted file mode 100755 index 09adbc26..00000000 --- a/reminder.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./reminder.py [arguments]: runs lib/reports/reminder.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/requirements-optional.txt b/requirements-optional.txt new file mode 100644 index 00000000..aeb22238 --- /dev/null +++ b/requirements-optional.txt @@ -0,0 +1,6 @@ +# Each is needed only by one way of receiving updates, and imported only when that way is configured. CI installs them +# all; an install needs only those of the modes it uses (Debian: the python3-* package named). +waitress>=3.0.2 # any network in http mode (python3-waitress) +websocket-client>=1.8.0 # Slack in pull mode, Socket Mode (python3-websocket) +discord.py>=2.5.0 # Discord in pull mode, the gateway (python3-discord) +PyNaCl>=1.5.0 # Discord in http mode, the interactions' signatures (python3-nacl) diff --git a/requirements.txt b/requirements.txt index 4fbb9d88..fe66405a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ # Targets Python 3.13.5 (see .python-version). Lower bounds are the first releases with 3.13 support. croniter>=1.3.15 -gevent>=24.11.1 matplotlib>=3.9.2,<4 numpy>=2.1.0,<3 -pandas>=2.2.3,<3 # 3.x not yet tried python-dateutil>=2.9.0 requests>=2.32.3 diff --git a/sharesight_warm.py b/sharesight_warm.py deleted file mode 100755 index b04e2850..00000000 --- a/sharesight_warm.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./sharesight_warm.py [arguments]: runs lib/reports/sharesight_warm.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/shorts.py b/shorts.py deleted file mode 100755 index 14347057..00000000 --- a/shorts.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./shorts.py [arguments]: runs lib/reports/shorts.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/tests/common.py b/tests/common.py index d11f9e61..48690d76 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,9 +1,8 @@ """Shared fixture: points finbot at a temp cache dir and replaces every external service (Telegram, Yahoo, Sharesight, Short Man) with canned data. Nothing here touches the network. ProviderCase runs the real parsers instead, on the API answers saved under doc/ (sample) as a stand-in network returns them (Resp, serve).""" -import collections, dataclasses, io, json, os, shutil, sys, tempfile, time, unittest, zoneinfo +import collections, dataclasses, datetime, io, json, os, shutil, sys, tempfile, time, unittest, zoneinfo from pathlib import Path -import pandas as pd from unittest import mock ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -12,10 +11,10 @@ os.environ.update(telegramOutgoingWebhook='https://example.com/telegram', telegramOutgoingToken='tok', telegramChatID='1') -# first, and once: it monkey-patches with gevent, which must happen before requests imports ssl and never mid-test import bot -from lib import answer, buttons, cache, charts, config, delivery, dialects, formats, inbound, links, markets, render -from lib import sharesight, shortman, state, telegram, transports, watchlist, worker, yahoo +from lib import answer, buttons, cache, charts, config, delivery, dialects, formats, inbound, links, markets, net +from lib import render, sharesight, shortman, state, transports, watchlist, worker, yahoo +from lib import discord_common, slack_common, telegram_common # each network's reader and bot from lib.reports import about, cal, history, holdings, milestone, performance, price, profile, rankings, rating from lib.reports import reminder, shorts, trades @@ -23,12 +22,19 @@ (yahoo, 'price_history'), (yahoo, 'historic_high'), (sharesight, '_portfolios'), (sharesight, '_trades'), (sharesight, '_performance'))} REAL_GET_HOLDINGS = watchlist.with_holdings +REAL_DISPATCH = bot.dispatch # FinbotCase notes the worker thread of each update, so that a test can wait for it +REAL_DOUBLED = bot.doubled # FinbotCase takes every message, a repeat sent on purpose too; DoubledMessages puts it back + + +def no_jobs(data): + """No chat's jobs in a data dir, so that a bot started there runs no scheduled report (and migrates none).""" + Path(data, 'finbot_schedule.json').write_text('{}') def bare_environment(): - """os.environ without finbot's settings (the startup keys, the preferences, the flag that hands a job its - preferences): for a process that must start from the defaults.""" - settings = {*config.STARTUP_KEYS, *config.PREFERENCE_KEYS, config.PASSED} + """os.environ without finbot's settings (the startup keys and the preferences): for a process that must start from + the defaults.""" + settings = {*config.STARTUP_KEYS, *config.PREFERENCE_KEYS} return {k: v for k, v in os.environ.items() if k not in settings} @@ -47,12 +53,14 @@ def temp_dir(case): (delivery, 'report_error', REAL_REPORT_ERROR)) TICKERS = [f'T{i}' for i in range(25)] TELEGRAM, SLACK, DISCORD = dialects.TELEGRAM, dialects.SLACK, dialects.DISCORD +READERS = {'telegram': telegram_common.READER, 'slack': slack_common.READER, 'discord': discord_common.READER} BOT = '@bot' def wsgi_environ(path, body, **headers): """A WSGI environ posting `body` (bytes) to `path`, with these headers (as HTTP_* keys).""" - return {'PATH_INFO': path, 'wsgi.input': io.BytesIO(body), 'CONTENT_LENGTH': str(len(body)), **headers} + return {'REQUEST_METHOD': 'POST', 'PATH_INFO': path, 'wsgi.input': io.BytesIO(body), + 'CONTENT_LENGTH': str(len(body)), **headers} def skip_if_old_env(case): @@ -64,17 +72,20 @@ def skip_if_old_env(case): STATES = ('REGULAR', 'PRE', 'POST', 'CLOSED') +NOW = time.time() # one moment for every fixture quote: fetched together in threads, they still tie on their dates + + def market_entry(ticker): """A Yahoo quote for a fixture ticker; T0 to T24 vary in market state, move, rating and so on, so that every threshold, sort and branch has rows.""" i = int(ticker[1:]) if ticker[1:].isdigit() else 0 # varied data so threshold/sort/sell branches all have rows - return {'marketState': STATES[i % 4], 'regularMarketTime': time.time(), 'quoteType': 'EQUITY', + return {'marketState': STATES[i % 4], 'regularMarketTime': NOW, 'quoteType': 'EQUITY', 'profile_title': 'Co ' + ticker, 'profile_exchange': 'NasdaqGS', 'percent_change': (i - 12) * 1.5, 'percent_change_premarket': (i - 12) * 1.5, 'percent_change_postmarket': (i - 12) * 1.5, 'market_cap': 1e9, 'price_to_earnings_trailing': 9, 'price_to_earnings_forward': 9, 'beta': 2.0, 'short_percent': 20, 'fiftyTwoWeekHigh': 120.0, 'fiftyTwoWeekLow': 80.0, - 'regularMarketPrice': 100.0, 'earnings_date': time.time() + 86400 * 3, - 'ex_dividend_date': time.time() + 86400 * 3, 'recommend': 'buy' if i % 2 else 'sell', + 'regularMarketPrice': 100.0, 'earnings_date': NOW + 86400 * 3, + 'ex_dividend_date': NOW + 86400 * 3, 'recommend': 'buy' if i % 2 else 'sell', 'recommend_index': 1.2 if i % 2 else 4.2, 'recommend_analysts': 8, 'price_to_earnings_peg': 1.2, 'currency': 'USD'} @@ -88,6 +99,18 @@ def patch(self, target, attr, value): p.start() self.addCleanup(p.stop) + def join_workers(self): + """Wait for the worker thread of every update taken so far (bot.dispatch starts one each).""" + for thread in self.workers: + thread.join(10) + + def press_button(self, data, chat_id=55, message_id=9): + """Press a button under one of the bot's Telegram messages, as the update arrives: bot.take acknowledges it + (answerCallbackQuery) and hands it to the worker, whose thread this waits for.""" + bot.take(telegram_common.READER, {'callback_query': {'id': 'cb', 'from': {'username': 'u'}, 'data': data, + 'message': {'message_id': message_id, 'chat': {'id': chat_id}}}}) + self.join_workers() + def env(self, **changes): """Startup settings for this test: config.env with these fields changed.""" self.patch(config, 'env', dataclasses.replace(config.env, **changes)) @@ -104,8 +127,13 @@ def setUp(self): self.patch(__import__('requests').Session, 'request', mock.Mock(return_value=fake)) # no real network traffic tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) - self.env(data_dir=tmp.name, cache_dir=tmp.name, webhooks={'telegram': 'https://api.telegram.org/botX/'}) + self.env(data_dir=tmp.name, cache_dir=tmp.name, telegramBotToken='X', + telegram_api='https://api.telegram.org/botX/') self.patch(bot, '_updates_seen', collections.OrderedDict()) # each test starts with no update taken + self.patch(bot, '_said', {}) # nor any message + self.patch(bot, 'doubled', lambda message: False) # a test may send the same message twice on purpose + self.workers = [] + self.patch(bot, 'dispatch', lambda event: self.workers.append(REAL_DISPATCH(event))) self.sent = [] self.markups = [] # reply_markup of each photo sent self.errors = [] @@ -114,7 +142,7 @@ def setUp(self): self.patch(transports.Transport, 'chart', lambda dest, image, caption, markup=None: self.record_chart(dest, image, caption, markup)) self.patch(transports.Transport, 'text', lambda dest, lines, markup=None: self.record_text(dest, lines)) - self.patch(watchlist, 'with_holdings', lambda: TICKERS) + self.patch(watchlist, 'with_holdings', lambda chat: TICKERS) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts}) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t)}) moved = {t: float(i * 4 - 40) for i, t in enumerate(TICKERS)} # each stock's move over any period @@ -123,9 +151,9 @@ def setUp(self): ()))) self.patch(yahoo, 'prefetch_history', lambda ts: None) self.patch(yahoo, 'historic_high', lambda t: (100.0, 50.0)) - dates = pd.date_range(end=pd.Timestamp.today().normalize(), periods=400) - self.patch(yahoo, 'price_series', - lambda t: ('Co ' + t, 'NASDAQ', pd.Series(range(100, 500), index=dates, name=t, dtype=float))) + today = datetime.date.today() # 400 days of closes, to today: 100, 101, ... 499 + closes = tuple((today - datetime.timedelta(days=399 - i), float(100 + i)) for i in range(400)) + self.patch(yahoo, 'price_series', lambda t: ('Co ' + t, 'NASDAQ', closes)) self.patch(shortman, 'fetch', lambda md: md) held = [{'id': i, 'instrument': {'code': t, 'market_code': 'NASDAQ'}, 'portfolio': {'name': 'Rob'}, 'capital_gain_percent': str(i * 4 - 40), 'value': '1000'} for i, t in enumerate(TICKERS)] @@ -156,7 +184,7 @@ def record_chart(self, dest, image, caption, markup=None): dest.stop_typing() image.seek(0) self.sent.append(('photo', len(image.read()), caption.split('\n')[0])) - self.markups.append(transports.TelegramBot.markup(markup) if markup else None) + self.markups.append(telegram_common.TelegramBot.markup(markup) if markup else None) return {'ok': True} def no_typing_left(self): diff --git a/tests/test_charts.py b/tests/test_charts.py index 9407a57d..df98ac6c 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -4,11 +4,9 @@ 3. one golden image per chart type, compared with a tolerance. Refresh after an intended restyle with: UPDATE_GOLDEN=1 python -m unittest tests.test_charts """ -import datetime, os, time, unittest -import gevent +import datetime, os, subprocess, sys, tempfile, threading, unittest from unittest import mock import numpy as np -import pandas as pd from PIL import Image from matplotlib.axes import Axes from matplotlib.figure import Figure @@ -48,7 +46,7 @@ def line(rows, title='Apple Inc (AAPL) 120 days', subtitle='164.12 USD ▲ 6.9 def compare_rows(): """Three made-up price series over the same 120 days, as a compare chart's rows.""" - days = [str(day.date()) for day in pd.date_range('2026-01-01', periods=120)] + days = [str(datetime.date(2026, 1, 1) + datetime.timedelta(days=i)) for i in range(120)] return tuple((name, tuple(zip(days, 100 + np.cumsum(np.sin(np.arange(120) / k) + drift)))) for name, k, drift in (('AAA', 7, 0.2), ('BBB', 11, -0.1), ('CCC', 5, 0.05))) @@ -87,11 +85,11 @@ def shorts(rows, threshold=15): class ImageSanity(unittest.TestCase): - def check_image(self, buf, name, bg=BG, edge_clean=True): - """Fail unless buf is a 1280x800 PNG that is not blank, starts with the background, and (edge_clean) draws - nothing on its edges, where clipped text or bars would show.""" + def check_image(self, buf, name, bg=BG, edge_clean=True, size=(1280, 800)): + """Fail unless buf is a PNG of this size (1280x800) that is not blank, starts with the background, and + (edge_clean) draws nothing on its edges, where clipped text or bars would show.""" image = decode(buf) - self.assertEqual(image.size, (1280, 800), name) + self.assertEqual(image.size, size, name) pixels = np.asarray(image) self.assertGreater(len({tuple(p) for p in pixels.reshape(-1, 3)[::97]}), 8, f'{name} looks blank') self.assertGreater(pixels.std(), 5, f'{name} looks blank') @@ -119,6 +117,11 @@ def test_edge_case_rows(self): self.check_image(bars(rows, 'Title', 'sub'), name) self.check_image(uncached(render.Chart('bars', 'Title', rows=tuple(rows))), name + '/drawn') + def test_a_long_bar_chart_grows_taller_and_stays_sane(self): + """Past 40 rows a bar chart's image grows taller, each row as tall as at 40, and draws as sanely.""" + rows = [(f'T{i}', (i - 30) * 1.5) for i in range(60)] + self.check_image(bars(rows, 'Title', 'sub'), 'sixty rows', size=(1280, 1200)) + def test_zoomed_axis(self): """A bar chart with a narrow x range (ratings, 1 to 5) draws sane images, wide labels and many rows included.""" self.check_image(bars([('AAA', 1.2), ('BBB', 3.4)], 'Ratings', xlim=(0.9, 4.0)), 'zoomed') @@ -158,41 +161,28 @@ def test_a_long_title_is_shrunk_to_fit(self): self.check_image(render.draw_line(line(price_rows(), title)), 'long line title') -class RenderPool(unittest.TestCase): - """The bot draws charts in one native thread (render.render_off_hub): the hub keeps serving while one is drawn.""" - - def setUp(self): - """Draw in the render thread, as the bot does, for this test only.""" - render.render_off_hub() - self.addCleanup(setattr, render, '_render_pool', None) +class DrawingLock(unittest.TestCase): + """Charts asked for from several threads at once are drawn one at a time (render._drawn).""" def test_charts_asked_for_at_once_all_finish(self): - """Six charts asked for at once all finish: one render thread, as several deadlock matplotlib under gevent.""" + """Six charts asked for from six threads at once all finish, whole.""" pie = lambda title: render.Chart('pie', title, rows=(('a', 1.0), ('b', 2.0))) - jobs = [gevent.spawn(render.draw_pie, pie(f't{i}')) for i in range(4)] - jobs += [gevent.spawn(bars, MIXED)] - jobs += [gevent.spawn(uncached, render.Chart('bars', 'T', rows=tuple(MIXED)))] - gevent.joinall(jobs, timeout=120) - self.assertTrue(all(job.successful() for job in jobs), - 'a chart did not finish (several native threads deadlock matplotlib)') - self.assertTrue(all(decode(job.value).size == (1280, 800) for job in jobs)) - - def test_the_hub_keeps_serving_while_a_chart_is_drawn(self): - """While the shorts pictogram draws (about a second), other greenlets run on: no gap over half a second.""" - gaps, done = [], [] - def ticker(): - """Note the time between wake-ups until the chart is done.""" - last = time.monotonic() - while not done: - gevent.sleep(0.01) - now = time.monotonic() - gaps.append(now - last) - last = now - tick = gevent.spawn(ticker) - shorts([(f'T{i}', 60.0 - 2 * i) for i in range(24)]) # about a second of drawing - done.append(True) - tick.join() - self.assertLess(max(gaps), 0.5) # drawn in the hub, the ticker stopped for the whole chart + draws = [lambda i=i: render.draw_pie(pie(f't{i}')) for i in range(4)] + draws += [lambda: bars(MIXED), lambda: uncached(render.Chart('bars', 'T', rows=tuple(MIXED)))] + images = {} + threads = [threading.Thread(target=lambda n=n, draw=draw: images.__setitem__(n, draw())) + for n, draw in enumerate(draws)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(120) + self.assertEqual(sorted(images), list(range(6)), 'a chart did not finish') + self.assertTrue(all(decode(image).size == (1280, 800) for image in images.values())) + + def test_one_chart_function_may_call_another(self): + """The lock is re-entrant: a drawn function that calls another drawn one finishes.""" + inner = render._drawn(lambda: 'drawn') + self.assertEqual(render._drawn(inner)(), 'drawn') class Contracts(unittest.TestCase): @@ -306,11 +296,13 @@ def run_report(self, call): self.assertTrue(captured, 'no chart was drawn') return captured[0] - def test_over_forty_bars_keep_both_ends_with_a_gap(self): - """A bar chart of over 40 rows keeps the first 20 and the last 20, a gap between them; the shorts pictogram, - over 24, the first and last 12, with the same gap (it used to drop the middle without a mark).""" - rows = render.trimmed(render.Chart('bars', 'T', rows=tuple((f'T{i}', i) for i in range(50)))).rows - self.assertEqual((len(rows), rows[20], rows[0][0], rows[-1][0]), (41, ('', None), 'T0', 'T49')) + def test_a_bar_chart_keeps_every_row_and_the_shorts_both_ends(self): + """A bar chart keeps every row, however many: its image grows instead (the caption is cut, so the chart must + show them all); the shorts pictogram, a scene of a fixed size, keeps the first and last 12 of over 24, a gap + between.""" + chart = render.Chart('bars', 'T', rows=tuple((f'T{i}', i) for i in range(50))) + self.assertEqual(render.trimmed(chart).rows, chart.rows) + self.assertEqual(decode(render.draw_bars(chart)).size, (1280, 1000)) rows = render.trimmed(render.Chart('shorts', rows=tuple((f'T{i}', 60 - i) for i in range(30)))).rows self.assertEqual((len(rows), rows[12], rows[11][0], rows[13][0]), (25, ('', None), 'T11', 'T18')) self.assertEqual(render.trimmed(render.Chart('pie', rows=(('a', 1.0),) * 50)).rows, (('a', 1.0),) * 50) @@ -360,6 +352,23 @@ def test_shorts_is_sorted_and_carries_the_threshold(self): self.assertTrue(all(v > chart.threshold for v in values)) +class WithoutPandas(unittest.TestCase): + def test_every_chart_draws_with_pandas_blocked(self): + """Every kind of chart draws in a process that cannot import pandas: the line charts' dates are plain dates and + their closes numpy arrays, which matplotlib takes as they are.""" + code = ('import sys\nfrom tests import test_charts\nfor draw in test_charts.CHARTS.values(): draw()\n' + 'print(sorted(test_charts.CHARTS))') + with tempfile.TemporaryDirectory() as data: + stubs = Path(data, 'stubs') + stubs.mkdir() + (stubs / 'pandas.py').write_text('raise ImportError("blocked by the test")\n') + env = bare_environment() + env.update(DATA_DIR=data, PYTHONPATH=str(stubs)) + out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=env, capture_output=True, text=True, + timeout=120) + self.assertEqual(out.stdout.strip().splitlines()[-1:], [str(sorted(CHARTS))], out.stderr) + + class GoldenImages(unittest.TestCase): """One reference image per chart type. Tolerant on purpose: compares 160x100 thumbnails, so font anti-aliasing and minor matplotlib differences pass, while changed colours, layout, bar sizes or missing text do not.""" diff --git a/tests/test_config.py b/tests/test_config.py index 1dd7cfaa..79ec7701 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,5 @@ """The two config files: finbot.env (the startup file, which the launcher loads) and preferences.conf (reloadable).""" -import ast, contextlib, dataclasses, io, os, re, subprocess, sys, unittest +import ast, contextlib, dataclasses, io, os, re, shutil, subprocess, sys, types, unittest from pathlib import Path from unittest import mock from tests import common # first: it sets the environment that lib.config reads as it loads @@ -34,7 +34,7 @@ def test_the_two_key_sets_are_disjoint_and_cover_everything_config_reads(self): def test_secrets_are_startup_keys(self): """Every secret is a startup key: preferences.conf holds none.""" self.assertTrue(set(config.SECRET_KEYS) <= set(config.STARTUP_KEYS)) - self.assertEqual(len(config.SECRET_KEYS), 12) + self.assertEqual(len(config.SECRET_KEYS), 13) def test_unset_preferences_take_their_defaults(self): """An unset preference takes its field's default, as it is: reading tidies nothing away.""" @@ -42,12 +42,15 @@ def test_unset_preferences_take_their_defaults(self): self.assertEqual(fields(values), fields(config.Prefs())) def test_templates_have_every_key_and_the_real_defaults(self): - """defaults/preferences.conf and finbot.env list every key, change no default, and are valid as shipped.""" - self.assertEqual(sorted(keys_in(REPO / 'defaults' / 'preferences.conf')), sorted(config.PREFERENCE_KEYS)) + """defaults/preferences.conf lists every key, commented out at its built-in default, so that it sets none; + finbot.env lists every key; both are valid as shipped.""" + template = REPO / 'defaults' / 'preferences.conf' + self.assertEqual(sorted(keys_in(template)), sorted(config.PREFERENCE_KEYS)) self.assertEqual(sorted(keys_in(REPO / 'defaults' / 'finbot.env')), sorted(config.STARTUP_KEYS)) - prefs = config.read_preferences(REPO / 'defaults' / 'preferences.conf') - shipped = config.preference_values(config._getenv_for(prefs)) - self.assertEqual(fields(shipped), fields(config.Prefs())) # the template changes nothing + self.assertEqual((config.read_preferences(template), config.template_keys()), ({}, [])) # it sets nothing + shown = dict(re.findall(r'^#([A-Za-z_][A-Za-z0-9_]*)=(.*)$', template.read_text(), re.M)) # #key=value lines + self.assertEqual(sorted(shown), sorted(config.PREFERENCE_KEYS)) + self.assertEqual(fields(config.preference_values(config._getenv_for(shown))), fields(config.Prefs())) startup = config.read_startup_file(REPO / 'defaults' / 'finbot.env') config.startup_values(lambda name, default=None: startup.get(name, default)) # and is valid as it stands @@ -84,7 +87,7 @@ def test_unknown_key_gets_a_hint_and_a_line_number(self): def test_a_startup_key_in_preferences_says_where_it_belongs(self): """A startup key in preferences.conf is sent to finbot.env, without printing its value.""" - errors = self.errors('slack_webhook=https://x\nip=0.0.0.0\ntelegramChatID=5\nscheduler=False\n') + errors = self.errors('slackOutgoingWebhook=https://x\nip=0.0.0.0\ntelegramChatID=5\nscheduler_catchup=3\n') self.assertEqual(len(errors), 4) for line in errors: self.assertIn('belongs in finbot.env (a change needs a restart)', line) @@ -131,16 +134,38 @@ def test_a_blank_value_means_the_default(self): self.assertEqual(values.exchange_suffix, suffix) self.assertEqual(config.preference_values(config._getenv_for({})).exchange_suffix, 'AX') # unset: the default + def test_log_level_is_one_of_syslogs_levels_in_any_case(self): + """log_level takes ERROR, WARN, INFO or DEBUG, in any case, INFO by default; anything else is a mistake that + names the levels.""" + for given, level in (('warn', 'WARN'), ('Debug', 'DEBUG'), ('ERROR', 'ERROR'), ('', 'INFO')): + with self.subTest(log_level=given): + values = config.preference_values(config._getenv_for({'log_level': given})) + self.assertEqual(values.log_level, level) + with self.assertRaises(config.ConfigError) as caught: + config.preference_values(config._getenv_for({'log_level': 'loud'})) + self.assertEqual(caught.exception.errors, ["log_level='loud' is not one of ERROR, WARN, INFO, DEBUG"]) + def test_a_retired_key_is_accepted_and_said_to_do_nothing(self): - """A key finbot no longer reads (alliterate, say) does not stop a start, as an unknown key would: it is ignored, - and the log at start says to delete it.""" + """A key finbot no longer reads (alliterate in preferences.conf, slack_webhook in finbot.env, say) does not stop + a start, as an unknown key would: it is ignored, and the log at start (and check_config) says what to do + instead.""" path = Path(common.temp_dir(self), 'preferences.conf') path.write_text(''.join(f'{key}=True\n' for key in config.RETIRED) + 'future_days=45\n') values = config.read_preferences(path) self.assertEqual(config.preference_values(config._getenv_for(values)).future_days, 45) - notes = [f'{key} in preferences.conf does nothing any more: delete the line' for key in config.RETIRED] + notes = [f'{key} in preferences.conf does nothing any more: {instead}' + for key, instead in config.RETIRED.items()] self.assertEqual(config.retired_notes(values), notes) self.assertEqual(check_config.notes(path), config.retired_notes(values)) + startup = Path(common.temp_dir(self), 'finbot.env') + startup.write_text('slack_webhook=https://hooks.slack.com/services/x\nport=80\n') + self.assertEqual(check_config.check(startup, path), []) + self.assertEqual(check_config.notes(path, startup)[0], 'slack_webhook in finbot.env does nothing any more:' + ' the scheduled reports go to slackChannelID as the bot (slackBotToken); set those, and delete the line') + with mock.patch.dict(os.environ, {'discord_webhook': 'https://discord.com/api/webhooks/1/x'}): + report = config.startup_report() + self.assertIn('discord_webhook in finbot.env does nothing any more: the scheduled reports go to' + ' discordChannelID as the bot (discordBotToken); set those, and delete the line', report) def test_unreadable_file_is_an_error_naming_it(self): """A preferences file that cannot be read is an error that names it.""" @@ -170,17 +195,28 @@ def test_the_old_multiplier_name_is_gone_everywhere(self): def test_startup_file_linting(self): """util/check_config.py reports bad lines in finbot.env first, then bad values, and nothing for a good file.""" path = Path(common.temp_dir(self), 'finbot.env') - path.write_text('ip=1.2.3.4\nfuture_days=3\nslak_webhook=x\nport=http\n') + path.write_text('ip=1.2.3.4\nfuture_days=3\nslackBotTokn=x\nport=http\n') problems = check_config.check(path, Path(common.temp_dir(self), 'none')) self.assertEqual(len(problems), 2) # the two bad lines; the bad value is only found once the lines are valid self.assertIn('future_days belongs in preferences.conf (it is reloadable)', problems[0]) - self.assertIn('unknown key slak_webhook (did you mean slack_webhook?)', problems[1]) + self.assertIn('unknown key slackBotTokn (did you mean slackBotToken?)', problems[1]) path.write_text('port=http\n') nowhere = Path(common.temp_dir(self), 'none') self.assertIn("port='http' is not a whole number", check_config.check(path, nowhere)[0]) path.write_text('port=80\n') self.assertEqual(check_config.check(path, Path(common.temp_dir(self), 'none')), []) + def test_check_config_gives_the_startup_warnings_of_a_valid_file(self): + """check_config shows what bot.py would warn about at start for the file it checks: here a Telegram token for + a webhook that is not set (pull mode).""" + path = Path(common.temp_dir(self), 'finbot.env') + path.write_text('telegramOutgoingToken=tok\n') + self.assertEqual(check_config.check(path, Path(common.temp_dir(self), 'none')), []) + self.assertEqual(check_config.warnings(path), ['telegramOutgoingToken is set but telegramOutgoingWebhook is' + ' not: finbot fetches its Telegram updates (pull mode), and the token is not used']) + path.write_text('telegramOutgoingToken=tok\ntelegramOutgoingWebhook=https://example.com/t\n') + self.assertEqual(check_config.warnings(path), []) + class Reload(unittest.TestCase): def setUp(self): @@ -204,9 +240,8 @@ def write(self, text): def test_a_change_is_seen_at_once(self): """A reload applies every change, says which, and swaps in a new object, so no request sees half a reload.""" self.write('future_days=45\nprice_percent=7\nshorts_percent=12\nexclude_portfolios=1156263\n' - 'exchange_suffix=nz\nhyperlinkProvider=yahoo\n' - 'hyperlinkFooter=false\ndebug=1\ngraph=0\ndemote_volatile=0\nvolatility_multiplier=3\npast_days=14\n' - 'min_analysts=3\n') + 'exchange_suffix=nz\nhyperlinkProvider=yahoo\nhyperlinkFooter=false\nlog_level=debug\ngraph=0\n' + 'demote_volatile=0\nvolatility_multiplier=3\npast_days=14\nmin_analysts=3\n') before = config.prefs changes = config.reload() self.assertEqual(len(changes), 13) @@ -217,8 +252,8 @@ def test_a_change_is_seen_at_once(self): prefs = config.prefs self.assertEqual((prefs.exclude_portfolios, prefs.exchange_suffix, prefs.hyperlinkProvider, prefs.hyperlinkFooter), (('1156263',), 'NZ', 'yahoo', False)) - self.assertEqual((config.prefs.debug, config.prefs.graph, config.prefs.demote_volatile, - config.prefs.volatility_multiplier), (True, False, False, 3.0)) + self.assertEqual((config.prefs.log_level, config.prefs.graph, config.prefs.demote_volatile, + config.prefs.volatility_multiplier), ('DEBUG', False, False, 3.0)) self.assertEqual(config.reload(), []) # nothing more to change def test_a_value_removed_from_the_file_goes_back_to_its_default(self): @@ -249,7 +284,7 @@ def test_an_error_applies_nothing(self): self.write('future_days=45\n') config.reload() for text in ('future_days=3\nprice_percent=abc\n', 'future_days=3\nfutur_days=4\n', - 'future_days=3\nslack_webhook=x\n'): + 'future_days=3\nslackBotToken=x\n'): self.write(text) with self.assertRaises(config.ConfigError): config.reload() @@ -289,38 +324,17 @@ def reads_prefs(node): offenders.append(f'{path.name}:{node.lineno} module constant') self.assertEqual(offenders, []) - def test_a_job_process_runs_on_the_snapshot_not_the_edited_file(self): - """A job process gets the bot's preferences in its environment and does not read the file, which may have been - edited since; a script run by hand reads the file.""" - self.write('future_days=45\n') - config.reload() - env = config.child_environment() - self.assertEqual((env['future_days'], env[config.PASSED]), ('45', '1')) - self.write('future_days=99\n') # edited, no reload sent - data = common.temp_dir(self) - Path(data, 'preferences.conf').write_text('future_days=99\n') - show = "from lib import config as c; print(c.prefs.future_days)" - parent = {k: v for k, v in os.environ.items() if k not in config.PREFERENCE_KEYS and k != config.PASSED} - run = lambda **extra: subprocess.run([sys.executable, '-c', show], cwd=REPO, capture_output=True, text=True, - check=True, env={**parent, 'DATA_DIR': data, **extra}).stdout.split()[-1] - self.assertEqual(run(**env), '45') # the parent's values, and the file is not read - self.assertEqual(run(), '99') # a script run by hand reads the file itself - - def test_run_child_hands_the_snapshot_to_the_job(self): - """run_child passes the preferences in force to the job process.""" + def test_a_scheduled_job_runs_on_the_preferences_in_force_not_the_edited_file(self): + """A scheduled job runs in the bot's process, on the preferences of the last reload: a file edited since, and + not reloaded, is not read.""" from lib import jobs - seen = {} - real = jobs.subprocess.Popen - def popen(args, **kw): - """Note the environment the job is started with, and run a no-op instead.""" - seen.update(kw.get('env', {})) - return real([sys.executable, '-c', 'pass']) + seen = [] self.write('future_days=45\n') config.reload() - with mock.patch.object(jobs.subprocess, 'Popen', popen): - from lib.reports import cleanup - self.assertEqual(jobs.run_child('cleanup.py', cleanup, {}), 0) - self.assertEqual((seen['future_days'], seen[config.PASSED]), ('45', '1')) + self.write('future_days=99\n') # edited, no reload sent + module = types.SimpleNamespace(run=lambda: seen.append(config.prefs.future_days)) + jobs.run_in_process('look.py', module, {}) + self.assertEqual(seen, [45]) class Startup(unittest.TestCase): @@ -346,12 +360,33 @@ def test_a_broken_preferences_file_refuses_to_start(self): """A bad value, unknown key, startup key or malformed line in preferences.conf stops the start, saying why.""" for prefs, expect in (('future_days=abc\n', "future_days='abc' is not a whole number"), ('futur_days=1\n', 'unknown key futur_days'), - ('slack_webhook=x\n', 'belongs in finbot.env'), ('what\n', 'expected KEY=value')): + ('slackBotToken=x\n', 'belongs in finbot.env'), ('what\n', 'expected KEY=value')): out = self.run_config(prefs=prefs) self.assertEqual(out.returncode, 1, prefs) self.assertIn('ERROR config:', out.stderr) self.assertIn(expect, out.stderr) + def test_the_template_is_never_read_and_a_key_set_there_is_pointed_elsewhere(self): + """defaults/preferences.conf is only a template: a key set there changes nothing, and the log at start says to + set it in the data dir's preferences.conf, or to make that file first with util/sync_defaults.py.""" + tree = Path(common.temp_dir(self)) # config.py and a template of its own, which sets a key + (tree / 'lib').mkdir() + shutil.copy(REPO / 'lib' / 'config.py', tree / 'lib') + (tree / 'defaults').mkdir() + (tree / 'defaults' / 'preferences.conf').write_text('# a comment\n#past_days=28\nfuture_days=45\n') + code = 'from lib import config as c; print(c.prefs.future_days); print(*c.startup_warnings(), sep="\\n")' + for made in (False, True): + data = common.temp_dir(self) + if made: + Path(data, 'preferences.conf').write_text('#future_days=30\n') + with self.subTest(preferences_conf=made): + out = subprocess.run([sys.executable, '-c', code], cwd=tree, + env={**common.bare_environment(), 'DATA_DIR': data}, capture_output=True, text=True) + conf = f'{data}/preferences.conf' + fix = f'edit {conf} instead' if made else f'run util/sync_defaults.py --apply, then edit {conf}' + self.assertEqual(out.stdout.split('\n')[:2], ['30', + f'defaults/preferences.conf sets future_days, but finbot never reads it: {fix}'], out.stderr) + def test_an_unreadable_preferences_file_refuses_to_start(self): """A preferences.conf that cannot be read stops the start.""" data = common.temp_dir(self) @@ -370,14 +405,12 @@ def test_a_bad_startup_value_refuses_to_start(self): def test_a_preference_in_the_environment_is_refused_by_the_bot(self): """A preference in the bot's environment (finbot.env) is refused, but a job process gets its preferences that way, and startup keys belong there.""" - with mock.patch.dict(os.environ, {'future_days': '30', 'debug': '1'}): + with mock.patch.dict(os.environ, {'future_days': '30', 'log_level': 'DEBUG'}): with self.assertRaises(config.ConfigError) as caught: config.check_startup_environment() self.assertEqual(len(caught.exception.errors), 2) self.assertIn('future_days is a preference; move it from finbot.env to preferences.conf', '\n'.join(caught.exception.errors)) - with mock.patch.dict(os.environ, {'future_days': '30', config.PASSED: '1'}): - config.check_startup_environment() # a job process gets its preferences this way with mock.patch.dict(os.environ, {'ip': '1.2.3.4', 'telegramChatID': '5'}): config.check_startup_environment() # startup keys are what the environment is for @@ -401,42 +434,92 @@ def test_a_leftover_env_in_the_checkout_refuses_to_start(self): self.assertIn('/util/split_env.py, which writes finbot.env.new and preferences.conf.new', caught.exception.errors[1]) - def test_a_discord_webhook_uses_the_native_endpoint(self): - """A Discord webhook URL is used as is, its old /slack suffix removed.""" - # the old setup's suffix - for url in ('https://discord.com/api/webhooks/1/abc', 'https://discord.com/api/webhooks/1/abc/slack/'): - values = config.startup_values(lambda name, default=None: url if name == 'discord_webhook' else default) - self.assertEqual(values.webhooks['discord'], 'https://discord.com/api/webhooks/1/abc') + def test_the_telegram_api_is_the_bot_tokens(self): + """Telegram's Bot API base is made from the bot token (a trailing slash dropped), and is empty without one.""" + for token, api in (('123:abc', 'https://api.telegram.org/bot123:abc/'), ('123:abc/', + 'https://api.telegram.org/bot123:abc/'), ('', '')): + values = config.startup_values(lambda name, default=None: token if name == 'telegramBotToken' else default) + self.assertEqual(values.telegram_api, api) def test_settings_that_leave_part_of_the_bot_unable_to_work_are_warned_about(self): """A startup warning for Slack commands or slackChannelID without slackBotToken (no reply, no reports there), and for an inbound webhook without its secret (every request would be refused).""" hooks = dict(slackOutgoingWebhook='https://example.com/slack', slackOutgoingToken='stok', slackBotToken='xoxb-1', telegramOutgoingWebhook='https://example.com/telegram', telegramOutgoingToken='tok') + deaf = ('slackBotToken is set but neither slackAppToken nor slackOutgoingWebhook is: finbot hears nothing from' + ' Slack, so its commands and buttons get no answer') no_reply = 'slackOutgoingWebhook is set but slackBotToken is not: Slack commands will get no reply' slack_refused = ('slackOutgoingWebhook is set but neither slackSigningSecret nor slackOutgoingToken is:' ' every Slack request will be refused') telegram_refused = ('telegramOutgoingWebhook is set but telegramOutgoingToken is not: every Telegram update' ' will be refused') no_bot = 'slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot' + unused = ('telegramOutgoingToken is set but telegramOutgoingWebhook is not: finbot fetches its Telegram updates' + ' (pull mode), and the token is not used') + app_no_reply = 'slackAppToken is set but slackBotToken is not: Slack commands will get no reply' + app_unused = ('slackAppToken is set but so is slackOutgoingWebhook: Slack posts its updates there (http mode),' + ' and the app-level token is not used') + not_app = ("slackAppToken does not start with xapp-: it must be an app-level token (the app's Basic" + ' Information, App-Level Tokens), not the bot token') + bad_secret = ("telegramOutgoingToken is not 1 to 256 letters, digits, _ or -: Telegram takes no other as the" + " webhook's secret, so setting the webhook will fail") cases = (({}, []), ({'slackBotToken': ''}, [no_reply]), ({'slackOutgoingToken': ''}, [slack_refused]), ({'slackChannelID': 'C9'}, []), ({'slackChannelID': 'C9', 'slackBotToken': ''}, [no_reply, no_bot]), ({'slackOutgoingToken': '', 'slackSigningSecret': 's'}, []), - ({'telegramOutgoingToken': ''}, [telegram_refused]), ({'telegramOutgoingWebhook': ''}, [])) + ({'telegramOutgoingToken': ''}, [telegram_refused]), ({'telegramOutgoingWebhook': ''}, [unused]), + ({'telegramOutgoingWebhook': '', 'telegramOutgoingToken': ''}, []), + ({'telegramOutgoingToken': 'A-z_09' * 40}, []), ({'telegramOutgoingToken': 'x' * 257}, [bad_secret]), + ({'telegramOutgoingToken': 'pass word!'}, [bad_secret]), + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1'}, []), + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xapp-1', 'slackBotToken': ''}, [app_no_reply]), + ({'slackAppToken': 'xapp-1'}, [app_unused]), + ({'slackOutgoingWebhook': '', 'slackAppToken': 'xoxb-1'}, [not_app]), + ({'slackOutgoingWebhook': ''}, [deaf])) for changes, warnings in cases: with self.subTest(changes=changes), \ mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**hooks, **changes})): self.assertEqual(config.startup_warnings(), warnings) + def test_discord_settings_that_leave_part_of_it_unable_to_work_are_warned_about(self): + """A startup warning for Discord's endpoint or channel without the bot token, the endpoint without the app's + public key (every interaction refused), and the key without the endpoint (the gateway is used).""" + key = 'ab' * 32 + http = dict(discordBotToken='x', discordOutgoingWebhook='https://example.com/discord', discordPublicKey=key) + no_reply = 'discordOutgoingWebhook is set but discordBotToken is not: Discord commands will get no reply' + no_bot = 'discordChannelID is set but discordBotToken is not: scheduled reports cannot go there as the bot' + refused = ("discordOutgoingWebhook is set but discordPublicKey is not the app's public key (64 hex digits):" + ' every Discord interaction will be refused') + unused = ('discordPublicKey is set but discordOutgoingWebhook is not: finbot keeps to the Discord gateway (pull' + ' mode), and the key is not used') + cases = (({}, []), ({'discordBotToken': ''}, [no_reply]), ({'discordChannelID': '77'}, []), + ({'discordChannelID': '77', 'discordBotToken': '', 'discordOutgoingWebhook': ''}, [no_bot, unused]), + ({'discordPublicKey': 'not-hex'}, [refused]), ({'discordPublicKey': ''}, [refused]), + ({'discordOutgoingWebhook': ''}, [unused]), ({'discordOutgoingWebhook': '', 'discordPublicKey': ''}, [])) + for changes, warnings in cases: + with self.subTest(changes=changes), \ + mock.patch.object(config, 'env', dataclasses.replace(config.env, **{**http, **changes})): + self.assertEqual(config.startup_warnings(), warnings) + def test_startup_report_is_a_short_count_never_names_or_values(self): - """The startup report counts the startup keys set, and never names them or shows their values.""" - with mock.patch.dict(os.environ, {'slack_webhook': 'https://secret.example/abc', 'port': '5000'}, clear=True): + """The startup report counts the startup keys set and the preferences, and never names them or shows their + values.""" + with mock.patch.dict(os.environ, {'slackOutgoingWebhook': 'https://secret.example/abc', 'port': '5000'}, + clear=True): lines = config.startup_report() - keys = len(config.STARTUP_KEYS) - self.assertEqual(lines[0], f'startup environment provides 2 of {keys} startup keys (values hidden)') - self.assertLessEqual(len('config: ' + lines[0]), 90) # the path in the second line depends on where DATA_DIR is + self.assertEqual(lines[0], 'startup environment (2 keys set)') self.assertNotIn('secret.example', ''.join(lines)) - self.assertNotIn('slack_webhook', ''.join(lines)) + self.assertNotIn('slackOutgoingWebhook', ''.join(lines)) + with mock.patch.dict(os.environ, {'port': '5000'}, clear=True): + self.assertEqual(config.startup_report()[0], 'startup environment (1 key set)') + conf = Path(common.temp_dir(self), 'preferences.conf') + conf.write_text('graph = false\n') + for source, count in (({'graph': 'false'}, '1 key'), ({'log_level': 'DEBUG', 'graph': 'false'}, '2 keys')): + with self.subTest(source=source), mock.patch.object(config, 'preferences_path', str(conf)), \ + mock.patch.object(config, 'prefs', dataclasses.replace(config.prefs, source=source)): + self.assertEqual(config.startup_report()[1], f'{conf} ({count} set; SIGHUP to reload)') + with mock.patch.object(config, 'preferences_path', str(conf.with_name('missing.conf'))): + self.assertEqual(config.startup_report()[1], f'no preferences.conf in {config.env.data_dir}, reload' + ' unavailable') class SigHup(unittest.TestCase): @@ -446,54 +529,58 @@ def restore(self, file): file.write_text('') config.reload() - def test_reload_all_reloads_preferences_and_the_schedule_independently(self): - """SIGHUP reloads preferences, keeping the running ones on an error (logged), and the schedule regardless.""" + def test_reload_all_reloads_preferences_and_keeps_them_on_an_error(self): + """SIGHUP reloads preferences; on an error (logged) the running ones stay, and the bot survives it. The jobs + need no reload: a change made with .jobs counts at once.""" file = Path(common.temp_dir(self), 'preferences.conf') file.write_text('future_days=45\n') - scheduler = mock.Mock() self.addCleanup(self.restore, file) - with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(bot, 'scheduler', scheduler): + with mock.patch.object(config, 'preferences_path', str(file)): err = io.StringIO() with contextlib.redirect_stderr(err): bot.reload_all() self.assertIn('config: future_days 30 -> 45', err.getvalue()) - scheduler.reload.assert_called_once() - # a preferences error does not stop the schedule reload, and the bot survives it file.write_text('futur_days=1\n') err = io.StringIO() with contextlib.redirect_stderr(err): bot.reload_all() self.assertIn('ERROR config:', err.getvalue()) - self.assertEqual(scheduler.reload.call_count, 2) self.assertEqual(config.prefs.future_days, 45) - def test_reload_with_the_scheduler_off_still_reloads_preferences(self): - """With the scheduler off, SIGHUP still reloads preferences.""" + def test_a_reload_points_a_key_set_in_the_template_at_the_file_to_set_it_in(self): + """A key set in defaults/preferences.conf, which finbot never reads, is reported on a reload too, with the file + to set it in.""" file = Path(common.temp_dir(self), 'preferences.conf') - file.write_text('future_days=11\n') + file.write_text('') self.addCleanup(self.restore, file) - with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(bot, 'scheduler', None): - with contextlib.redirect_stderr(io.StringIO()): + template = Path(common.temp_dir(self), 'template.conf') + template.write_text('#past_days=28\nlog_level=DEBUG\n') + with mock.patch.object(config, 'preferences_path', str(file)), mock.patch.object(config, 'TEMPLATE', template): + with contextlib.redirect_stderr(io.StringIO()) as err: bot.reload_all() - self.assertEqual(config.prefs.future_days, 11) + self.assertIn(f'WARN config: defaults/preferences.conf sets log_level, but finbot never reads it: edit {file}' + ' instead', err.getvalue()) + self.assertEqual(config.prefs.log_level, 'INFO') class Tools(unittest.TestCase): def test_split_env(self): """split_env sorts an old .env into startup keys and prefs, renames volatile_multiplier, lists what it drops.""" - startup, prefs, notes = split_env.split('slack_webhook=x\nfuture_days=3\nvolatile_multiplier=2\nslackToken=y\n' - 'STATE_DIRECTORY=/a\nbogus=1\nnot a setting\n# c\nport=80\n') - self.assertEqual(startup, ['slack_webhook=x', 'port=80']) + startup, prefs, notes = split_env.split('slackBotToken=x\nfuture_days=3\nvolatile_multiplier=2\nslackToken=y\n' + 'STATE_DIRECTORY=/a\nbogus=1\nnot a setting\n# c\nport=80\nslack_webhook=z\n') + self.assertEqual(startup, ['slackBotToken=x', 'port=80']) self.assertEqual(prefs, ['future_days=3', 'volatility_multiplier=2']) - self.assertEqual(len(notes), 4) + self.assertEqual(len(notes), 5) self.assertTrue(any('slackToken dropped' in n for n in notes) and any('unknown key bogus' in n for n in notes)) + self.assertIn('line 10: slack_webhook dropped: the scheduled reports go to slackChannelID', notes[-1]) def test_split_env_reads_the_old_dotenv_format_too(self): """split_env reads the old dotenv format: quotes, export, comments after values and # inside URLs.""" - old = ("# c\nfuture_days = '30' # note\nexport slack_webhook = \"https://x/#frag\"\nport = 80 # comment\nbare\n" - "discord_webhook=http://y/#z\n\n") + old = ("# c\nfuture_days = '30' # note\nexport slackOutgoingWebhook = \"https://x/#frag\"\nport = 80 # note\n" + "bare\ndiscordOutgoingWebhook=http://y/#z\n\n") startup, prefs, notes = split_env.split(old) - self.assertEqual(startup, ['slack_webhook=https://x/#frag', 'port=80', 'discord_webhook=http://y/#z']) + self.assertEqual(startup, ['slackOutgoingWebhook=https://x/#frag', 'port=80', + 'discordOutgoingWebhook=http://y/#z']) self.assertEqual(prefs, ['future_days=30']) self.assertEqual(notes, ['.env line 5: not KEY=value, left out']) diff --git a/tests/test_datadir.py b/tests/test_datadir.py index 8f3d3e6d..95701d8c 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -2,8 +2,9 @@ from unittest import mock from pathlib import Path from lib import config -from util import sync_defaults -from tests.common import FinbotCase +import contextlib, io +from util import flush, sync_defaults +from tests.common import FinbotCase, ROOT, bare_environment, temp_dir REPO = Path(config.__file__).resolve().parent.parent @@ -91,7 +92,9 @@ def test_sync_defaults(self): self.assertFalse(Path(config.env.data_dir, 'finbot_adr.json').exists()) # dry run sync_defaults.sync(apply=True) self.assertTrue(Path(config.env.data_dir, 'finbot_adr.json').exists()) - self.assertTrue(Path(config.env.data_dir, 'preferences.conf').exists()) + copied = Path(config.env.data_dir, 'preferences.conf') # the template as it is: every key commented out + self.assertEqual(copied.read_bytes(), (config.REPO_DIR / 'defaults' / 'preferences.conf').read_bytes()) + self.assertEqual(config.read_preferences(copied), {}) # the startup file is never copied into the data dir self.assertFalse(Path(config.env.data_dir, 'finbot.env').exists()) mine = Path(config.env.data_dir, 'finbot_adr.json') @@ -105,10 +108,93 @@ def test_sync_defaults(self): self.assertEqual(Path(str(mine) + '.bak').read_text(), '{"mine": 1}') def test_sync_defaults_to_another_destination(self): - """util/sync_defaults.py --dest copies there and leaves the data dir alone.""" + """util/sync_defaults.py --dest copies there and leaves the data dir alone; a destination that does not exist + is an error that names --create, which makes it, mode 750, as it copies.""" other = Path(config.env.data_dir, 'elsewhere') - self.assertIn(('finbot_adr.json', 'would copy'), sync_defaults.sync(dest=other)) + with self.assertRaisesRegex(FileNotFoundError, 'does not exist: --create makes it'): + sync_defaults.sync(apply=True, dest=other) + self.assertIn(('finbot_adr.json', 'would copy'), sync_defaults.sync(dest=other, create=True)) self.assertFalse(other.exists()) # dry run - sync_defaults.sync(apply=True, dest=other) + sync_defaults.sync(apply=True, dest=other, create=True) self.assertTrue((other / 'finbot_adr.json').exists()) + self.assertEqual(other.stat().st_mode & 0o777, 0o750) self.assertFalse(Path(config.env.data_dir, 'finbot_adr.json').exists()) # the data dir is untouched + self.assertTrue(sync_defaults.sync(dest=other)) # there now: no --create needed + + def test_sync_defaults_runs_with_any_python_and_creates_nothing_unasked(self): + """The tool needs only the standard library, so a bare python3 runs it (Docker's setup script does, on the + host); importing lib.config creates no data directory, and a missing destination is refused, naming + --create.""" + with tempfile.TemporaryDirectory() as tmp: + dest = Path(tmp, 'data') + env = {**bare_environment(), 'DATA_DIR': str(Path(tmp, 'never'))} + run = lambda *args: subprocess.run([sys.executable, '-I', 'util/sync_defaults.py', *args], cwd=ROOT, + env=env, capture_output=True, text=True, timeout=60) + refused = run('--apply', '--dest', str(dest)) + self.assertEqual(refused.returncode, 2, refused.stderr) + self.assertIn('does not exist: --create makes it', refused.stderr) + made = run('--create', '--apply', '--dest', str(dest)) + self.assertEqual(made.returncode, 0, made.stderr) + self.assertTrue((dest / 'preferences.conf').exists()) + self.assertFalse(Path(tmp, 'never').exists()) + + +class Flush(FinbotCase): + """util/flush.py: the cache files written before or after a time ago, or all of them; never the state.""" + + def setUp(self): + """A data dir of this test's own: a state file, and cache files written 60 seconds, 400 seconds and 8 days ago, + with a file in the cache that is not finbot's.""" + super().setUp() + self.data = Path(temp_dir(self)) + self.cache = self.data / 'cache' + self.cache.mkdir() + (self.data / 'finbot_watchlist.json').write_text('{}') + + def make(self): + """The cache files, written that long ago.""" + for name, ago in (('finbot_new.json', 60), ('finbot_old.json', 400), ('finbot_week.json', 8 * 86400), + ('other.txt', 400)): + path = self.cache / name + path.write_text('{}') + os.utime(path, (time.time() - ago,) * 2) + + def flush(self, *args): + """Run the tool on this data dir: (its exit status, what it printed, the cache files left).""" + with contextlib.redirect_stdout(io.StringIO()) as out: + status = flush.main([*args, '--data-dir', str(self.data)]) + return status, out.getvalue().strip(), sorted(path.name for path in self.cache.iterdir()) + + def test_before_after_or_all_and_never_the_state(self): + """--before TIME deletes the cache files written more than TIME ago, --after TIME those written in the last + TIME, all every one of them; nothing but finbot's files in the cache, and nothing in the data dir.""" + for args, left in ((['--before', '180s'], ['finbot_new.json', 'other.txt']), + (['--after', '24h'], ['finbot_week.json', 'other.txt']), + (['--after', '7d'], ['finbot_week.json', 'other.txt']), (['--after', '1y'], ['other.txt']), + (['all'], ['other.txt'])): + with self.subTest(args=args): + self.make() + status, said, files = self.flush(*args) + self.assertEqual((status, files), (0, left)) + self.assertEqual(said, f"flushed {4 - len(left)} files from {self.cache}") + self.assertTrue((self.data / 'finbot_watchlist.json').exists()) + + def test_alone_it_shows_its_help_and_a_time_is_required(self): + """On its own it prints its help, with the examples; a time it does not know, or two of --before, --after and + all, is refused.""" + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(flush.main([]), 2) + for example in ('--before 180s', '--after 24h', '--after 7d', '--after 3m', '--after 1y', 'util/flush.py all'): + self.assertIn(example, out.getvalue()) + for args in (['--after', '7x'], ['all', '--before', '1h'], ['--before', '1h', '--after', '2h'], ['some']): + with self.subTest(args=args), contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + flush.main(args) + + def test_it_runs_on_its_own_and_creates_nothing(self): + """As a script, it flushes the data dir it is given and creates no data dir of its own.""" + self.make() + env = {**bare_environment(), 'DATA_DIR': str(self.data / 'never')} + run = subprocess.run([sys.executable, 'util/flush.py', 'all', '--data-dir', str(self.data)], cwd=ROOT, env=env, + capture_output=True, text=True, timeout=60) + self.assertEqual((run.returncode, run.stdout.strip()), (0, f'flushed 3 files from {self.cache}'), run.stderr) + self.assertFalse((self.data / 'never').exists()) diff --git a/tests/test_discord.py b/tests/test_discord.py new file mode 100644 index 00000000..50b94446 --- /dev/null +++ b/tests/test_discord.py @@ -0,0 +1,548 @@ +"""The Discord bot: interactions and messages read as the other networks' updates, the interactions endpoint and its +signatures, REST calls and their rate limits, messages with charts and buttons, slash commands from the command table, +and the gateway.""" +import contextlib, io, json, sys, types, unittest +from tests.common import * +from lib import discord_websocket, jobs + +try: + from nacl.signing import SigningKey +except ImportError: # PyNaCl is optional: only Discord's http mode needs it + SigningKey = None + + +def interaction(kind, **fields): + """An interaction of this type in channel 77, from Rob in a server (fields replace the defaults).""" + return {'id': '900', 'application_id': '42', 'token': 'tok', 'type': kind, 'channel_id': '77', + 'member': {'user': {'id': '5', 'username': 'rob', 'global_name': 'Rob'}}, **fields} + + +COMMAND = interaction(2, data={'name': 'price', 'options': [{'name': 'arguments', 'type': 3, 'value': '5 7d'}]}) +TYPING = interaction(4, id='902', data={'name': 'price', 'options': [{'name': 'arguments', 'type': 3, 'value': '5 7', + 'focused': True}]}) # /price, its arguments being typed +PRESS = interaction(3, id='901', data={'custom_id': 'c|h|y|T1', 'component_type': 2}, message={'id': '333'}) +MESSAGE = {'id': '700', 'channel_id': '77', 'author': {'id': '5', 'username': 'rob'}, 'content': '<@99> price 7d'} + + +class Now: + """threading.Thread that runs its target at start(), so that a test needs no waiting.""" + def __init__(self, target, args=(), daemon=None): + """The target and its arguments.""" + self.target, self.args = target, args + + def start(self): + """Run the target now.""" + self.target(*self.args) + + +class Reading(FinbotCase): + """An interaction reads the same however it arrives; a gateway message reads as Slack's do.""" + + def setUp(self): + """The bot's user id is 99.""" + super().setUp() + self.patch(discord_common.READER, 'bot_id', '99') + + def test_a_slash_command_is_the_message_it_stands_for(self): + """/price with its arguments is the message '/price 5 7d', answered through its interaction; in a DM the person + is the interaction's user.""" + expected = inbound.Message('discord', '77', '<@5>', '/price 5 7d', '@99', 'Rob', '', '42/tok') + self.assertEqual(discord_common.READER.parse(COMMAND), expected) + dm = {k: v for k, v in COMMAND.items() if k != 'member'} | {'user': {'id': '5', 'username': 'rob'}} + self.assertEqual(discord_common.READER.parse(dm), dataclasses.replace(expected, user_name='rob')) + self.assertEqual(discord_common.READER.parse({**COMMAND, 'data': {'name': 'beta'}}).text, '/beta') + + def test_a_press_is_a_press_of_its_message(self): + """A button pressed is a Press of its message, its custom_id the button's data.""" + self.assertEqual(discord_common.READER.parse(PRESS), + inbound.Press('discord', '901', '77', '333', 'c|h|y|T1', '@rob', '42/tok')) + + def test_a_message_by_a_person_reads_with_its_mentions_as_slacks(self): + """A gateway message's mentions read as @id, so that '@bot price 7d' calls .price; one by a bot or a webhook + (this bot's replies included) is not answered.""" + message = discord_common.READER.parse(MESSAGE) + self.assertEqual(message, inbound.Message('discord', '77', '<@5>', '@99 price 7d', '@99', 'rob', '700')) + command, args = worker.read(message.text, message.bot_name) + self.assertEqual((command.names[0], args.days), ('price', 7)) + for other in ({**MESSAGE, 'author': {'id': '99', 'bot': True}}, {**MESSAGE, 'webhook_id': '1'}): + self.assertIs(discord_common.READER.parse(other), inbound.OK) + + def test_the_endpoints_check_and_other_interactions(self): + """A PING gets its PONG at once; an interaction of another kind is not taken.""" + self.assertEqual(json.loads(discord_common.READER.parse(interaction(1)).body), {'type': 1}) + self.assertIs(discord_common.READER.parse(interaction(5)), inbound.UNSUPPORTED) + + def test_an_interaction_is_answered_before_its_work_starts(self): + """A slash command is answered "thinking…" and a press with nothing to see, and the work waits in the answer + until whoever delivers it starts it; a gateway message needs no answer, and its work starts at once.""" + taken = [] + self.patch(worker, 'process_request', taken.append) + self.patch(worker, 'process_callback', taken.append) + answer = bot.take(discord_common.READER, COMMAND) + self.assertEqual(json.loads(answer.body), {'type': 5}) + self.join_workers() + self.assertEqual(taken, []) + answer.then() + self.assertEqual(json.loads(bot.take(discord_common.READER, PRESS).body), {'type': 6}) + self.assertIs(bot.take(discord_common.READER, MESSAGE), inbound.OK) + self.join_workers() + self.assertEqual([event.text if isinstance(event, inbound.Message) else event.data for event in taken], + ['/price 5 7d', '@99 price 7d']) + + +@unittest.skipUnless(SigningKey, 'PyNaCl is not installed') +class Endpoint(FinbotCase): + """Discord's interactions endpoint: each request signed with the app's key; a bad signature gets the 401 Discord + looks for.""" + + def setUp(self): + """Discord in http mode, with a key of the test's own.""" + super().setUp() + self.key = SigningKey.generate() + self.env(discordBotToken='x', discordOutgoingWebhook='https://example.com/discord', + discordPublicKey=self.key.verify_key.encode().hex()) + self.taken = [] + self.patch(worker, 'process_request', self.taken.append) + self.patch(worker, 'process_callback', self.taken.append) + + def post(self, update, signature=None, timestamp='1700000000'): + """Post an interaction to the endpoint, signed unless a signature is given: (status, answer, body object).""" + body = json.dumps(update).encode() + if signature is None: + signature = self.key.sign(timestamp.encode() + body).signature.hex() + status = [] + with contextlib.redirect_stderr(io.StringIO()): + out = bot.main(wsgi_environ('/discord', body, HTTP_X_SIGNATURE_ED25519=signature, + HTTP_X_SIGNATURE_TIMESTAMP=timestamp), lambda s, *a: status.append(s)) + return status[0], b''.join(out), out + + def test_a_signed_check_gets_its_pong(self): + """Discord's check of the URL, signed, gets its PONG.""" + status, answer, _ = self.post(interaction(1)) + self.assertEqual((status, json.loads(answer)), ('200 OK', {'type': 1})) + + def test_a_bad_or_missing_signature_gets_401(self): + """A request signed with another key, or with no signature, is refused with a 401 and taken no further.""" + other = SigningKey.generate().sign(b'1700000000' + json.dumps(COMMAND).encode()).signature.hex() + self.assertEqual(self.post(COMMAND, signature=other)[0], '401 Unauthorized') + self.assertEqual(self.post(COMMAND, signature='')[0], '401 Unauthorized') + self.join_workers() + self.assertEqual(self.taken, []) + + def test_arguments_being_typed_get_their_suggestions_at_once(self): + """While a slash command's arguments are typed, Discord asks what to suggest: the answer is the suggestions + (type 8), at once, and no work starts.""" + status, answer, _ = self.post(TYPING) + self.join_workers() + self.assertEqual((status, json.loads(answer)), ('200 OK', {'type': 8, 'data': {'choices': [ + {'name': '5 7d', 'value': '5 7d'}]}})) + self.assertEqual(self.taken, []) + + def test_the_work_starts_once_discord_has_its_answer(self): + """A slash command is answered "thinking…" (a press, nothing to see); the work starts only once the server has + sent that answer, which it says by closing the body.""" + status, answer, out = self.post(COMMAND) + self.assertEqual((status, json.loads(answer)), ('200 OK', {'type': 5})) + self.join_workers() + self.assertEqual(self.taken, []) + out.close() + self.join_workers() + self.assertEqual([message.text for message in self.taken], ['/price 5 7d']) + self.assertEqual(json.loads(self.post(PRESS)[1]), {'type': 6}) + + +class Sending(FinbotCase): + """DiscordBot: what each message, chart, edit and typing notice sends, and the rate limits waited out.""" + + def setUp(self): + """The bot token set, and Discord's API answered from self.answers (else 200 and a message).""" + super().setUp() + self.env(discordBotToken='x', discordChannelID='77') + self.calls, self.answers = [], [] + def request(method, url, **kwargs): + """Note the call, and give the next answer.""" + self.calls.append((method, url.removeprefix(discord_common.API), kwargs)) + return self.answers.pop(0) if self.answers else Resp({'id': '1'}) + self.patch(net, 'request', request) + self.patch(transports.Transport, 'text', REAL_TEXT) + self.patch(transports.Transport, 'chart', REAL_CHART) + + def test_a_message_has_no_previews_or_pings_and_its_buttons_in_rows_of_five(self): + """A message suppresses link previews and pings nobody; its buttons go in rows of five, the one shown now + blurple, and past 25 they are left out.""" + row = [buttons.Button(f'B{i}', f'c|h|w|{i}', active=i == 0) for i in range(12)] + with contextlib.redirect_stderr(io.StringIO()): + discord_common.DiscordBot('77').text(['hello'], [row[:7], row[7:]]) + method, path, kwargs = self.calls[0] + self.assertEqual((method, path), ('POST', '/channels/77/messages')) + payload = kwargs['json'] + self.assertEqual((payload['content'], payload['flags'], payload['allowed_mentions']), ('hello', 4, + {'parse': []})) + self.assertEqual([len(r['components']) for r in payload['components']], [5, 5, 2]) + self.assertEqual([b['style'] for b in payload['components'][0]['components']], [1, 2, 2, 2, 2]) + self.assertEqual(len(discord_common.components([[buttons.Button('x', 'y')] * 30])), 5) + + def test_a_chart_is_attached_to_its_report(self): + """A chart goes as an attachment of the message that carries the report and the buttons.""" + with contextlib.redirect_stderr(io.StringIO()): + discord_common.DiscordBot('77').chart(io.BytesIO(b'png'), 'caption', [[buttons.Button('7D', 'c|h|w|T1')]]) + method, path, kwargs = self.calls[0] + payload = json.loads(kwargs['data']['payload_json']) + self.assertEqual((method, path, payload['content'], payload['attachments']), ('POST', '/channels/77/messages', + 'caption', [{'id': 0, 'filename': 'chart.png'}])) + self.assertEqual(kwargs['files']['files[0]'], ('chart.png', b'png', 'image/png')) + self.assertEqual(payload['components'][0]['components'][0]['custom_id'], 'c|h|w|T1') + + def test_a_slash_commands_answer_replaces_its_thinking_and_the_rest_follow_up(self): + """A slash command's first message replaces Discord's "thinking…", through its interaction, and the rest + follow it up; there is no typing notice besides.""" + chat = discord_common.DiscordBot('77', interaction='42/tok') + self.assertFalse(chat.shows_typing) + with contextlib.redirect_stderr(io.StringIO()): + chat.text(['x' * 1500, 'y' * 1500]) + self.assertEqual([(m, p) for m, p, _ in self.calls], [('PATCH', '/webhooks/42/tok/messages/@original'), + ('POST', '/webhooks/42/tok')]) + chat.typing_done() + self.assertEqual(len(self.calls), 2) # answered: nothing to delete + + def test_a_slash_command_that_says_nothing_has_its_thinking_deleted(self): + """A slash command over without a word, or one that reads as no command, has its "thinking…" deleted, or it + would show for 15 minutes.""" + discord_common.DiscordBot('77', interaction='42/tok').typing_done() + worker.process_request(inbound.Message('discord', '77', '<@5>', '/nothing@else', '@99', 'Rob', '', '42/t2')) + self.assertEqual([(m, p) for m, p, _ in self.calls], [('DELETE', '/webhooks/42/tok/messages/@original'), + ('DELETE', '/webhooks/42/t2/messages/@original')]) + + def test_a_press_edits_its_message_through_its_interaction(self): + """A press edits its message through its own interaction (a message that answered a slash command can be + edited only so); a new message about it goes into the channel.""" + chat = discord_common.DiscordBot('77', interaction='42/tok', pressed=True) + chat.edit_chart('333', io.BytesIO(b'png'), 'caption', [[buttons.Button('1Y', 'c|h|y|T1', True)]]) + chat.edit_text('333', 'list', []) + with contextlib.redirect_stderr(io.StringIO()): + chat.text(['Too few prices']) + self.assertEqual([(m, p) for m, p, _ in self.calls], [('PATCH', '/webhooks/42/tok/messages/@original'), + ('PATCH', '/webhooks/42/tok/messages/@original'), ('POST', '/channels/77/messages')]) + self.assertEqual(json.loads(self.calls[0][2]['data']['payload_json'])['attachments'], + [{'id': 0, 'filename': 'chart.png'}]) + self.assertEqual(self.calls[1][2]['json'], {'content': 'list', 'components': []}) + + def test_a_command_in_a_message_shows_typing(self): + """A command in a message (the gateway's) shows "typing…" in its channel, which lasts 10 seconds (POST + /typing): it is sent again sooner, so that it shows the whole time the command works.""" + chat = discord_common.DiscordBot('77', '700') + self.assertTrue(chat.shows_typing) + chat.typing() + self.assertEqual(self.calls[0][:2], ('POST', '/channels/77/typing')) + self.assertLess(chat.TYPING[1], 10) + + def test_each_request_names_finbot_as_discord_asks(self): + """Each request carries the bot token, and the User-Agent that Discord asks of a bot: DiscordBot, its URL and + its version (the commit, which bot.py notes as it starts).""" + self.patch(discord_common, 'version', '0123456') + discord_common.call('GET', '/users/@me') + self.assertEqual(self.calls[0][2]['headers'], {'Authorization': 'Bot x', + 'User-Agent': 'DiscordBot (https://github.com/robdevops/finbot, 0123456)'}) + + def test_rate_limits_are_waited_out_and_refusals_raise(self): + """A 429 waits as long as Discord asks, then tries again, three times at most; a refusal raises with Discord's + reason; the token in an interaction's path is never logged.""" + slept = [] + self.patch(discord_common.time, 'sleep', slept.append) + self.answers += [Resp({'retry_after': 0.25}, 429), Resp({'id': '5'})] + self.assertEqual(discord_common.call('GET', '/users/@me'), {'id': '5'}) + self.assertEqual(slept, [0.25]) + self.answers += [Resp({'retry_after': 99}, 429)] * 3 + with self.assertRaisesRegex(RuntimeError, '429'): + discord_common.call('GET', '/users/@me') + self.assertEqual(slept, [0.25, 10, 10]) + self.answers.append(Resp({'message': 'Missing Access', 'code': 50001}, 403)) + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertIsNone(discord_common.try_call('PATCH', '/webhooks/42/s3cret-token/messages/@original')) + self.assertIn('403 Missing Access (50001)', err.getvalue()) + self.assertNotIn('s3cret-token', err.getvalue()) + + def test_scheduled_reports_go_to_the_channel_with_their_buttons(self): + """With discordChannelID and the bot token, scheduled reports go to that channel as the bot, buttons and all; + without the token, nowhere on Discord.""" + self.env(telegramChatID='') + chats = delivery.default_chats() + self.assertEqual([(type(chat).__name__, chat.where) for chat in chats], [('DiscordBot', '77')]) + self.assertTrue(chats[0].takes_buttons and chats[0].takes_charts) + self.env(discordBotToken='') + with self.assertRaisesRegex(RuntimeError, 'nowhere to post'): + delivery.default_chats() + + +class Commands(FinbotCase): + """The slash commands, one per .help line, from the command table, as every network registers them.""" + + def test_each_help_line_is_a_slash_command_that_reads_as_its_command(self): + """Each .help line has a slash command, named as a slash command may be (.forward pe is /forwardpe), and + /help, /stock and /profile besides; each reads as its command, its arguments where it takes any.""" + commands = worker.slash_commands() + names = [name for name, _, _, _ in commands] + self.assertEqual(len(names), len(set(names))) + for expected in ('help', 'dividend', 'earnings', 'buy', 'sell', 'session', 'premarket', 'forwardpe', 'stock', + 'profile', 'jobs'): + self.assertIn(expected, names) + for name, description, arguments, _ in commands: + with self.subTest(name=name): + self.assertRegex(name, r'^[a-z0-9_-]{1,32}$') + self.assertTrue(0 < len(description) <= 100) + command, args = worker.read(f'/{name} {arguments}'.strip(), BOT) + self.assertIsNotNone(command) + self.assertEqual(command.handler, (worker.COMMANDS[0] if name == 'help' else command).handler) + self.assertEqual(dict((n, a) for n, _, a, _ in commands)['price'], '[percent|SYMBOL|top|bottom] [period]') + self.assertEqual(dict((n, a) for n, _, a, _ in commands)['beta'], '') + + def test_stock_and_profile_are_the_profile_command(self): + """.stock SYMBOL and /profile SYMBOL are .SYMBOL; without a symbol, the usage.""" + self.assertEqual(self.command('.stock T1'), self.command('.T1')) + self.assertEqual(self.command('/profile T1'), self.command('.T1')) + self.assertEqual(self.command('.stock'), [('text', ['Usage: .SYMBOL', 'Usage: .stock SYMBOL'])]) + + def test_a_command_with_words_to_suggest_asks_for_suggestions(self): + """The arguments of a command with words to suggest ask Discord for suggestions as they are typed; /plan's, + free text, do not, and a command without arguments has none.""" + commands = (discord_common.command(*each) for each in worker.slash_commands()) + registered = {entry['name']: entry for entry in commands} + for name, completes in (('price', True), ('stock', True), ('jobs', True), ('performance', True), + ('pie', True), ('plan', False)): + with self.subTest(name=name): + self.assertEqual(registered[name]['options'][0]['autocomplete'], completes) + self.assertEqual(registered['beta']['options'], []) + + def test_registering_replaces_the_commands_only_when_they_differ(self): + """Registering puts the commands only when Discord's differ from them, and gives the bot's user id.""" + wanted = [discord_common.command('price', 'price [period]', '[period]')] + calls = [] + def call(method, path, **kwargs): + """Answer as Discord would, the commands as registered now.""" + calls.append((method, path)) + return {'/applications/@me': {'id': '42'}, '/users/@me': {'id': '99'}}.get(path, registered) + self.patch(discord_common, 'call', call) + registered = [{**wanted[0], 'id': '1', 'version': '2', 'application_id': '42'}] + self.assertEqual(discord_common.register([('price', 'price [period]', '[period]')]), '99') + self.assertNotIn(('PUT', '/applications/42/commands'), calls) + registered = [] + with contextlib.redirect_stderr(io.StringIO()): + discord_common.register([('price', 'price [period]', '[period]')]) + self.assertIn(('PUT', '/applications/42/commands'), calls) + + def test_help_on_discord_lists_the_slash_commands(self): + """.help on Discord lists each command as its slash command, which works in both modes; the messages that also + work are given only over the gateway.""" + self.patch(discord_common.READER, 'bot_id', '99') + sent = [] + self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: sent.append(lines) or []) + message = inbound.Message('discord', '77', '<@5>', '/help', '@99', 'Rob', '', '42/tok') + worker.process_request(message) + self.assertIn('/forwardpe [SYMBOL|top|bottom|negative]', sent[0]) + self.assertIn('/stock SYMBOL', sent[0]) + self.assertIn('.price, or <@99> price', sent[0]) + self.env(discordOutgoingWebhook='https://example.com/discord') + worker.process_request(message) + self.assertNotIn('.price, or <@99> price', sent[1]) + + +class Suggestions(FinbotCase): + """What a slash command suggests while its arguments are typed (Discord's autocomplete): the command table's words, + the chat's watchlist and the cached holdings, this chat's jobs and the reports; never a request.""" + + def setUp(self): + """Two portfolios cached, BHP and NVDA held; MSFT watched in channel 77, TSLA in 88; any request fails.""" + super().setUp() + cache.write(sharesight.PORTFOLIOS_CACHE, {'portfolios': [{'name': 'Rob', 'id': 1}, + {'name': 'Jo Smith', 'id': 2}]}) + held = [{'instrument': {'code': code, 'market_code': market}} for code, market in (('BHP', 'ASX'), + ('NVDA', 'NASDAQ'))] + cache.write(sharesight.performance_cache_file(1, 0), {'report': {'holdings': held}}) + watchlist.add('discord:77', 'MSFT') + watchlist.add('discord:88', 'TSLA') + def no_request(*args, **kwargs): + """A request, which suggestions must never make.""" + raise AssertionError('a suggestion made a request') + for module, name in ((sharesight, '_api'), (yahoo, 'fetch'), (yahoo, 'fetch_detail'), (net, 'post')): + self.patch(module, name, no_request) + + def values(self, text, chat='discord:77'): + """The values suggested for the text typed so far, in a chat.""" + return [value for _, value in worker.suggest(chat, text)] + + def test_keywords_periods_portfolios_and_tickers(self): + """A command suggests its keywords and periods until it has one, its portfolios, and tickers: the chat's + watchlist (not another chat's) and the holdings; the word being typed matches in any case.""" + periods = ['1d', '7d', '1m', '3m', '6m', 'ytd', '1y', '5y', '10y'] + self.assertEqual(self.values('/price '), ['top', 'bottom', *periods, 'BHP.AX', 'MSFT', 'NVDA']) + self.assertEqual(self.values('/price 5 7'), ['5 7d']) + self.assertEqual(self.values('/price B'), ['bottom', 'BHP.AX']) + self.assertEqual(self.values('/price 7d t'), ['7d top']) + self.assertEqual(self.values('/price 7d 1'), []) # a period given: none again + self.assertEqual(self.values('/forwardpe n'), ['negative', 'NVDA']) + self.assertEqual(self.values('/compare MSFT n'), ['MSFT NVDA']) + self.assertEqual(self.values('/stock '), ['BHP.AX', 'MSFT', 'NVDA']) + self.assertEqual(self.values('/stock ', chat='discord:88'), ['BHP.AX', 'NVDA', 'TSLA']) + self.assertEqual(self.values('/performance 7d '), ['7d Rob', '7d Jo Smith']) + self.assertEqual(self.values('/pie '), ['sector', 'Rob', 'Jo Smith']) + self.assertEqual(self.values('/plan buy '), []) + + def test_the_jobs_subcommands_numbers_and_reports(self): + """/jobs suggests its subcommands; after run, toggle or del this chat's job numbers, each shown with its line; + after add and a time (five fields, an @macro, a zone first), the reports a chat can schedule, each shown with + what it takes, and then what the report takes.""" + self.patch(jobs, 'SYSTEM', ()) + state.save(jobs.SCHEDULE_FILE, {'discord:77': [ + {'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}, + {'when': '0 22 * * *', 'reports': 'shorts', 'zone': 'UTC', 'on': False}]}) + scheduler = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + scheduler.load() + self.patch(jobs, 'current', scheduler) + self.assertEqual(self.values('/jobs '), ['run', 'toggle', 'add', 'del']) + self.assertEqual(self.values('/job T'), ['toggle']) + self.assertEqual(worker.suggest('discord:77', '/jobs run '), [('run 1: */20 * * * Mon-Fri trades', 'run 1'), + ('run 2: 0 22 * * * shorts', 'run 2')]) + self.assertEqual(self.values('/jobs del 2'), ['del 2']) + self.assertEqual(self.values('/jobs run ', chat='discord:88'), []) + self.assertEqual(self.values('/jobs add 0 22 * * '), []) + reports = worker.suggest('discord:77', '/jobs add 0 22 * * * ') + self.assertIn(('add 0 22 * * * price midsession|interday|premarket|close|DAYS', 'add 0 22 * * * price'), + reports) + self.assertFalse(any(name in value for _, value in reports for name in jobs.SYSTEM_SCRIPTS)) + self.assertEqual(self.values('/jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price m'), + ['add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession']) + self.assertEqual(self.values('/jobs add @daily cal earnings; s'), ['add @daily cal earnings; shorts']) + self.assertEqual(self.values('/jobs add 0 22 * * * trades '), []) + + def test_at_most_25_each_of_100_characters_and_a_failure_suggests_nothing(self): + """Discord takes 25 suggestions at most, each up to 100 characters; a failure suggests nothing, and is + logged.""" + for ticker in TICKERS: + watchlist.add('discord:77', ticker) + self.assertEqual(len(self.values('/price ')), 25) + self.assertEqual(self.values('/compare ' + 'LONGTICKER ' * 9), []) + self.patch(worker, 'argument_words', mock.Mock(side_effect=RuntimeError('broken'))) + with contextlib.redirect_stderr(io.StringIO()) as log: + self.assertEqual(self.values('/price '), []) + self.assertIn('suggest: /price ', log.getvalue()) + + +class FakeClient: + """discord.py's Client, enough for the gateway listener: its handlers get the raw messages of the test.""" + raw = [] + + def __init__(self, intents, enable_debug_events=False): + """Note how it was made.""" + self.intents, self.debug, self.handlers = intents, enable_debug_events, {} + FakeClient.made = self + + def event(self, handler): + """Register a handler by its name.""" + self.handlers[handler.__name__] = handler + return handler + + async def __aenter__(self): + """Enter.""" + return self + + async def __aexit__(self, *exc): + """Leave: closed.""" + self.closed = True + + async def start(self, token): + """Connect: ready, then the raw messages, then fail as Discord does on a refused token.""" + self.token = token + await self.handlers['on_ready']() + for message in self.raw: + await self.handlers['on_socket_raw_receive'](json.dumps(message)) + raise RuntimeError('Improper token has been passed.') + + +class Gateway(FinbotCase): + """The gateway through discord.py: its raw events taken as the endpoint's, an interaction answered over REST.""" + + def setUp(self): + """Discord in pull mode, with discord.py replaced by the fake client, and threads that run at once.""" + super().setUp() + self.env(discordBotToken='x', discordOutgoingWebhook='') + intents = types.SimpleNamespace(message_content=False) + library = types.SimpleNamespace(Client=FakeClient, VoiceClient=types.SimpleNamespace(warn_nacl=True), + Intents=types.SimpleNamespace(default=lambda: intents)) + self.library = library + self.patch(sys, 'modules', {**sys.modules, 'discord': library}) + self.patch(discord_websocket, 'threading', types.SimpleNamespace(Thread=Now)) # the listener's threads only + self.calls = [] + self.patch(discord_common, 'call', lambda method, path, **kwargs: self.calls.append((method, path, kwargs))) + + def test_messages_and_interactions_are_taken_and_an_interaction_answered_first(self): + """The client asks for the message intents and raw events; MESSAGE_CREATE and INTERACTION_CREATE go to take, the + rest is the client's; an interaction's answer goes to its callback before its work starts; a client that stops + waits the back-off.""" + taken, order = [], [] + def take(network, update): + """Note the update; answer an interaction as the reader would, its work noted when started.""" + taken.append(update.get('content') or update['type']) + if 'token' in update: + return inbound.Response(body=b'{"type": 5}', then=lambda: order.append(('work', len(self.calls)))) + return inbound.OK + FakeClient.raw = [{'op': 11}, {'op': 0, 't': 'READY', 'd': {}}, + {'op': 0, 't': 'MESSAGE_CREATE', 'd': MESSAGE}, {'op': 0, 't': 'INTERACTION_CREATE', 'd': COMMAND}] + listener = discord_websocket.DiscordGateway() + listener.failures = 3 + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(listener.poll(take), 5) # ready reset the failures; then the refused token + client = FakeClient.made + self.assertTrue(client.debug and client.intents.message_content and client.closed) + self.assertFalse(self.library.VoiceClient.warn_nacl) + self.assertEqual(taken, ['<@99> price 7d', 2]) + self.assertEqual(self.calls, [('POST', '/interactions/900/tok/callback', {'json': {'type': 5}})]) + self.assertEqual(order, [('work', 1)]) + self.assertIn('discord: the gateway failed, trying again in 5 s: Improper token', err.getvalue()) + + def test_arguments_being_typed_over_the_gateway_get_their_suggestions_through_the_callback(self): + """Over the gateway, the suggestions for what is being typed go to the interaction's callback.""" + FakeClient.raw = [{'op': 0, 't': 'INTERACTION_CREATE', 'd': TYPING}] + with contextlib.redirect_stderr(io.StringIO()): + discord_websocket.DiscordGateway().poll(bot.take) + self.assertEqual(self.calls, [('POST', '/interactions/902/tok/callback', {'json': {'type': 8, 'data': { + 'choices': [{'name': '5 7d', 'value': '5 7d'}]}}})]) + + @unittest.skipUnless(SigningKey, 'PyNaCl is not installed') + def test_a_press_over_the_gateway_is_the_press_the_endpoint_gets(self): + """The same press, posted to the endpoint (http mode) or sent over the gateway (pull mode), reaches the worker + as the same Press.""" + presses = [] + self.patch(worker, 'process_callback', presses.append) + key = SigningKey.generate() + self.env(discordOutgoingWebhook='https://example.com/discord', discordPublicKey=key.verify_key.encode().hex()) + body = json.dumps(PRESS).encode() + signature = key.sign(b'1700000000' + body).signature.hex() + with contextlib.redirect_stderr(io.StringIO()): + bot.main(wsgi_environ('/discord', body, HTTP_X_SIGNATURE_ED25519=signature, + HTTP_X_SIGNATURE_TIMESTAMP='1700000000'), lambda *a: None).close() + self.join_workers() + self.patch(bot, '_updates_seen', collections.OrderedDict()) # a restart in the other mode + self.env(discordOutgoingWebhook='') + FakeClient.raw = [{'op': 0, 't': 'INTERACTION_CREATE', 'd': PRESS}] + discord_websocket.DiscordGateway().poll(bot.take) + self.join_workers() + self.assertEqual(len(presses), 2) + self.assertEqual(presses[0], presses[1]) + self.assertEqual(self.calls[0][1:], ('/interactions/901/tok/callback', {'json': {'type': 6}})) + + def test_discord_is_listened_to_with_its_token_and_no_endpoint(self): + """The gateway is used with the bot token and no discordOutgoingWebhook; with the URL (http mode), or without + the token, it is not.""" + self.env(telegramBotToken='') + self.assertEqual([type(listener) for listener in bot.listeners()], [discord_websocket.DiscordGateway]) + self.env(discordOutgoingWebhook='https://example.com/discord') + self.assertEqual(bot.listeners(), []) + self.env(discordOutgoingWebhook='', discordBotToken='') + self.assertEqual(bot.listeners(), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_edges.py b/tests/test_edges.py index b25ec1e1..7f394c79 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -1,8 +1,9 @@ """Failure and no-data scenarios: empty or broken upstream data, bad arguments, unknown portfolios, secrets kept out of logs, and the bot's signals.""" -import contextlib, io, os, signal, subprocess, sys, tempfile, threading, time, unittest +import contextlib, io, os, re, signal, subprocess, sys, tempfile, threading, time, unittest from pathlib import Path from tests.common import * +from lib import log COMMANDS = ('.price', '.price 7d', '.price top', '.session', '.premarket', '.performance 7d', '.shorts', '.peg', '.buy', @@ -61,8 +62,8 @@ def down(*a, **k): self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) typing = [] - self.patch(transports.TelegramBot, 'post_text', lambda dest, *a, **k: typing.append(dest.is_typing())) - self.patch(transports.TelegramBot, 'post_chart', lambda dest, *a, **k: typing.append(dest.is_typing()) or + self.patch(telegram_common.TelegramBot, 'post_text', lambda dest, *a, **k: typing.append(dest.is_typing())) + self.patch(telegram_common.TelegramBot, 'post_chart', lambda dest, *a, **k: typing.append(dest.is_typing()) or {'ok': True}) for cmd in ('.help', '.price', '.trades 7d', '.performance 7d'): with self.subTest(cmd=cmd), contextlib.redirect_stderr(io.StringIO()): @@ -72,7 +73,7 @@ def down(*a, **k): def test_no_holdings_and_empty_watchlist(self): """With nothing held and nothing watched, the watchlist, price and earnings commands still answer.""" - self.patch(watchlist, 'with_holdings', lambda: []) + self.patch(watchlist, 'with_holdings', lambda chat: []) for cmd in ('.watchlist', '.price', '.earnings'): with self.subTest(cmd=cmd): self.command(cmd) @@ -83,12 +84,12 @@ def test_add_duplicate_delete_and_unknown_action(self): """Add, add again (said differently), remove, and an unknown action is refused by name.""" add = self.command('.watchlist add MSFT') self.assertTrue(add) - self.assertEqual(state.load('finbot_watchlist.json'), ['MSFT']) + self.assertEqual(state.load('finbot_watchlist.json'), {'chats': {'telegram:55': ['MSFT']}}) again = self.command('.watchlist add MSFT') self.assertNotEqual(add, again) # a duplicate is reported differently from a first add - self.assertEqual(state.load('finbot_watchlist.json'), ['MSFT']) + self.assertEqual(watchlist.tickers('telegram:55'), ['MSFT']) self.command('.watchlist rm MSFT') - self.assertEqual(state.load('finbot_watchlist.json'), []) + self.assertEqual(watchlist.tickers('telegram:55'), []) self.assertEqual(self.command('.watchlist frobnicate MSFT'), [('text', ['Usage: .watchlist [add|del SYMBOL]'])]) def test_a_bare_symbol_yahoo_does_not_know_is_tried_on_the_asx(self): @@ -99,9 +100,9 @@ def test_a_bare_symbol_yahoo_does_not_know_is_tried_on_the_asx(self): self.assertIn('already tracking', str(self.command('.watchlist add BHP'))) # a ticker with a market is not retried self.assertIn('could not find', str(self.command('.watchlist add BHP.L'))) - self.assertEqual(watchlist.tickers(), ['BHP.AX']) + self.assertEqual(watchlist.tickers('telegram:55'), ['BHP.AX']) self.assertIn('I deleted', str(self.command('.watchlist del bhp'))) - self.assertEqual(watchlist.tickers(), []) + self.assertEqual(watchlist.tickers('telegram:55'), []) def test_two_chats_adding_at_once_both_count(self): """Two chats adding at once both count: Yahoo is asked before the file is opened, where both adds used to read @@ -111,21 +112,22 @@ def slow_fetch(tickers): time.sleep(0.01) return {t: market_entry(t) for t in tickers} self.patch(yahoo, 'fetch', slow_fetch) - threads = [threading.Thread(target=holdings.prepare_watchlist, args=(TELEGRAM, '@u', 'add', t)) for t in ('T1', - 'T2')] + threads = [threading.Thread(target=holdings.prepare_watchlist, args=(TELEGRAM, chat, '@u', 'add', ticker)) + for chat, ticker in (('telegram:55', 'T1'), ('telegram:55', 'T2'), ('slack:C1', 'T3'))] for thread in threads: thread.start() for thread in threads: thread.join() - self.assertEqual(sorted(watchlist.tickers()), ['T1', 'T2']) + self.assertEqual(sorted(watchlist.tickers('telegram:55')), ['T1', 'T2']) + self.assertEqual(watchlist.tickers('slack:C1'), ['T3']) def test_removing_a_ticker_yahoo_no_longer_knows(self): """A ticker Yahoo no longer knows is still removed, and said to be; removing it again says it is not there.""" - state.save('finbot_watchlist.json', ['GONE', 'T1']) + state.save('finbot_watchlist.json', {'chats': {'telegram:55': ['GONE', 'T1']}}) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts if t != 'GONE'}) self.assertIn('I deleted', str(self.command('.watchlist rm GONE'))) self.assertIn('could not find', str(self.command('.watchlist rm GONE'))) - self.assertEqual(watchlist.tickers(), ['T1']) + self.assertEqual(watchlist.tickers('telegram:55'), ['T1']) class BadArguments(FinbotCase): @@ -146,7 +148,6 @@ def no_history(t, *a, **k): self.assertEqual(self.command('.compare T1 T2'), [('text', ['no price history for T1'])]) self.assertEqual(self.command('.history T1'), [('text', ['no price history for T1'])]) self.sent.clear() - self.patch(telegram, 'answerCallbackQuery', lambda *a, **k: None) worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', 'c|h|m|T1')) self.assertEqual((self.sent, self.errors), ([('text', ['no price history for T1'])], [])) @@ -220,9 +221,9 @@ def test_a_command_is_its_whole_name_in_any_case(self): ('.price @bot', 'price')): with self.subTest(message=message): self.assertEqual(self.args(message)[0], (name,)) - for message in ('.planet', '.shortman', '.pricey'): + for message in ('.planet', '.shortman', '.pricey', '.stock T1', '/profile T1'): with self.subTest(message=message): - self.assertEqual(self.args(message)[0], (), message) # .SYMBOL + self.assertEqual(self.args(message)[0], ('stock',), message) # .SYMBOL self.assertEqual(self.args('.price @bot')[1].words, []) for message in ('price 7d', '/price@otherbot', '...', '. ?'): with self.subTest(message=message): @@ -237,7 +238,7 @@ def test_keywords_in_any_case(self): self.assertIn('Top 10 performers', str(self.command('.price TOP 7d'))) def test_a_bare_number_on_price_is_a_percent(self): - """As on .session, .premarket and .shorts, and as the README has it; a period carries its unit.""" + """As on .session, .premarket and .shorts, and as doc/commands.md has it; a period carries its unit.""" self.assertIn('3% at close', str(self.command('.price 3'))) # it used to be a 3-day report self.assertIn('Price movement ≥ 3%', str(self.command('.price 3 7d'))) # the 3 used to be dropped self.assertIn('Price movement ≥ 5%', str(self.command('.price 7d'))) @@ -289,33 +290,150 @@ def stderr_of(self, call): return err.getvalue() def test_a_failed_send_logs_no_token(self): - """A failed send logs no Telegram token, Slack webhook path or bot token.""" + """A failed send logs no Telegram, Slack or Discord token, nor the token of a Discord interaction.""" def refuse(url, **kwargs): """Fail like requests does, the URL and headers in the message.""" raise ConnectionError(f"Max retries exceeded with url: {url} (headers {kwargs.get('headers')})") - self.patch(transports.net, 'post', refuse) - self.env(slackBotToken='xoxb-SECRET', webhooks={'telegram': self.SECRET_URL.split('sendMessage')[0]}) - log = self.stderr_of(lambda: transports.TelegramBot('1').post_text('hi')) - log += self.stderr_of(lambda: transports.SlackBot('C1').post_text('hi')) - hook = transports.SlackWebhook('https://hooks.slack.com/services/T0/B0/SECRETpath') - log += self.stderr_of(lambda: hook.post_text('hi')) - self.assertIn('Failure sending', log) - for secret in ('SECRET-tok_en', 'SECRETpath', 'xoxb-SECRET'): + self.patch(net, 'post', refuse) + self.patch(net, 'request', lambda method, url, **kwargs: refuse(url, **kwargs)) + token = 'MTAwMDAwMDAwMDAwMDAwMDAwMDA.SECRET.' + 'a' * 38 # a Discord bot token's shape + self.env(slackBotToken='xoxb-SECRET', telegram_api=self.SECRET_URL.split('sendMessage')[0], + discordBotToken=token) + log = self.stderr_of(lambda: telegram_common.TelegramBot('1').post_text('hi')) + log += self.stderr_of(lambda: slack_common.SlackBot('C1').post_text('hi')) + slash = discord_common.DiscordBot('77', interaction='42/SECRET-interaction') + log += self.stderr_of(lambda: slash.post_text('hi')) + self.assertIn('failed', log) + for secret in ('SECRET-tok_en', 'xoxb-SECRET', 'SECRET-interaction', token): self.assertNotIn(secret, log) def test_the_traceback_of_a_reported_error_is_redacted(self): """The traceback logged with a reported error is redacted.""" self.patch(delivery, 'report_error', REAL_REPORT_ERROR) - self.patch(transports.TelegramBot, 'post_text', lambda *a, **k: None) + self.patch(telegram_common.TelegramBot, 'post_text', lambda *a, **k: None) try: raise ConnectionError('failed: ' + self.SECRET_URL) except ConnectionError as e: - log = self.stderr_of(lambda: delivery.report_error(e, transports.TelegramBot('55'), context='x')) + log = self.stderr_of(lambda: delivery.report_error(e, telegram_common.TelegramBot('55'), context='x')) self.assertIn('Traceback', log) self.assertIn('bot', log) self.assertNotIn('SECRET-tok_en', log) +class Unbuffered(io.StringIO): + """A stream like stderr under PYTHONUNBUFFERED (finbot.service, the image): each write is a write to the journal, + whose system call lets another thread run.""" + + def write(self, text): + """Let another thread run, then write.""" + time.sleep(0) + return super().write(text) + + +class WholeLogLines(unittest.TestCase): + """bot.WholeLines, the bot's stderr: a line goes out whole, whichever threads log at once.""" + + def test_lines_that_threads_print_at_once_stay_whole(self): + """Lines that threads print at the same time come out whole, never mixed ("TTL:cache hit: 34254 s/var/..."): + print() writes its pieces one by one, and a thread's pieces wait for the end of its line.""" + out = Unbuffered() + stream = bot.WholeLines(out) + def log(n): + """Print lines of several pieces each.""" + for i in range(200): + print('cache hit:', f'/var/lib/finbot/cache/{n}.json', 'TTL:', f'{i}s', file=stream) + threads = [threading.Thread(target=log, args=(n,)) for n in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + lines = out.getvalue().splitlines() + self.assertEqual(len(lines), 8 * 200) + self.assertEqual([line for line in lines if not re.fullmatch(r'cache hit: \S+/\d\.json TTL: \d+s', line)], []) + + def test_a_flush_writes_a_line_that_has_not_ended(self): + """A flush writes what the thread holds, though its line has not ended; the rest passes through.""" + out = io.StringIO() + stream = bot.WholeLines(out) + print('partial', end='', file=stream) + self.assertEqual(out.getvalue(), '') + stream.flush() + self.assertEqual(out.getvalue(), 'partial') + self.assertEqual(stream.getvalue(), 'partial') # the stream's own + + +class LogLevels(FinbotCase): + """log_level: each of syslog's levels lets in its own lines and those of the levels above it.""" + + def logged(self, level, run): + """What run() logs with log_level at this level.""" + self.prefs(log_level=level) + with contextlib.redirect_stderr(io.StringIO()) as err: + run() + return err.getvalue().splitlines() + + def test_each_level_lets_in_its_own_lines_and_those_above_it(self): + """An error always reaches the log; WARN adds the warnings, INFO what finbot does, DEBUG its detail.""" + def each(): + """A line at each level.""" + log.error('e') + log.warn('w') + log.info('i') + log.debug('d') + for level, shown in (('ERROR', 'e'), ('WARN', 'ew'), ('INFO', 'ewi'), ('DEBUG', 'ewid')): + with self.subTest(log_level=level): + self.assertEqual(self.logged(level, each), list(shown)) + + def test_a_cache_miss_hit_or_write_is_debug(self): + """A cache miss (of a file not there, or too old), a hit, and the write after a miss are logged at DEBUG only: + at INFO, the request a miss sends is the line.""" + def twice(): + """A miss and its write, then a hit, then a miss of a file too old.""" + cache.cached('finbot_test.json', 60, lambda: {'a': 1}) + cache.cached('finbot_test.json', 60, lambda: {'a': 1}) + cache.cached('finbot_test.json', -1, lambda: {}) + where = cache.path('finbot_test.json') + self.assertEqual(self.logged('INFO', twice), []) + os.remove(where) + lines = self.logged('DEBUG', twice) + kinds = [line.split(':')[0] for line in lines] + self.assertEqual(kinds, ['cache miss', 'cache write', 'cache hit', 'cache miss']) + self.assertRegex(lines[2], r'^cache hit: \S+finbot_test\.json TTL: \d+s$') + + + def test_a_stock_without_a_market_cap_is_debug(self): + """A stock that .marketcap cannot rank, without a market cap (an ETF), is logged at DEBUG only: nothing went + wrong.""" + self.patch(yahoo, 'fetch', lambda tickers: {'ETF1': {'profile_title': 'A Fund', 'profile_exchange': 'NYSE'}}) + self.patch(yahoo, 'add_usd_market_caps', lambda quotes: quotes) + rank = lambda: rankings.marketcap_list(TELEGRAM, '55') + self.assertEqual((self.logged('INFO', rank), self.logged('DEBUG', rank)), ([], ['ETF1 no market cap'])) + + +class RequestLog(FinbotCase): + """Each request that goes out (a cache miss, or a call that has no cache) is a line at INFO.""" + + def test_each_request_is_a_short_line_without_secrets(self): + """A request logs its method and URL at INFO, without the scheme or a secret, cut to 73 bytes (the ellipsis + takes 3); Telegram's long polling (logged=False) does not, and nothing does below INFO.""" + with contextlib.redirect_stderr(io.StringIO()) as err: + net.get('https://query1.finance.yahoo.com/v7/finance/quote?crumb=s3cret') + net.post('https://api.telegram.org/bot123:ABC-def/sendMessage', json={}) + net.request('PATCH', 'https://discord.com/api/v10/channels/7/messages/9', json={}) + net.post('https://api.telegram.org/bot123:ABC-def/getUpdates', json={}, logged=False) + net.get('https://query2.finance.yahoo.com/v10/finance/quoteSummary/ARM.L?modules=calendarEvents,' + 'defaultKeyStatistics,earnings,financialData,price&crumb=s3cret') + lines = err.getvalue().splitlines() + self.assertEqual(lines, ['GET query1.finance.yahoo.com/v7/finance/quote?crumb=', + 'POST api.telegram.org/bot/sendMessage', 'PATCH discord.com/api/v10/channels/7/messages/9', + 'GET query2.finance.yahoo.com/v10/finance/quoteSummary/ARM.L?modules=ca…']) + self.assertEqual(len(lines[3].encode()), 73) + self.prefs(log_level='WARN') + with contextlib.redirect_stderr(io.StringIO()) as err: + net.get('https://example.com/') + self.assertEqual(err.getvalue(), '') + + class SharedSession(unittest.TestCase): def test_the_session_keeps_no_cookies(self): """Yahoo's cookie goes in a header: a stored one would go to fc.yahoo.com, which then sets no new one.""" @@ -360,6 +478,53 @@ def test_a_flush_fills_the_cache_again_at_once(self): self.assertEqual(started, [(bot.jobs.warm_now, ('the scheduler',))]) +OPTIONAL = ('waitress', 'websocket', 'discord', 'nacl') # each imported only by the mode that needs it + + +class OptionalLibraries(unittest.TestCase): + """An install without the optional libraries (requirements-optional.txt) runs every mode that does not need them.""" + + def blocked(self, data): + """The environment of a process for which no optional library can be imported, and that has no webhook set.""" + stubs = Path(data, 'stubs') + stubs.mkdir() + for name in OPTIONAL: + (stubs / f'{name}.py').write_text('raise ImportError("blocked by the test")\n') + env = bare_environment() + env.update(DATA_DIR=data, port='0', PYTHONPATH=str(stubs)) + no_jobs(data) + return env + + def test_every_module_imports_without_them(self): + """bot and every module of lib and lib.reports import with the optional libraries blocked, and none of them is + imported.""" + code = ('import importlib, pkgutil, sys, bot, lib, lib.reports\n' + 'for package in (lib, lib.reports):\n' + ' for module in pkgutil.iter_modules(package.__path__, package.__name__ + "."):\n' + ' importlib.import_module(module.name)\n' + f'print(sorted(name for name in {OPTIONAL!r} if sys.modules.get(name)))') + with tempfile.TemporaryDirectory() as data: + out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, env=self.blocked(data), capture_output=True, + text=True, timeout=60) + self.assertEqual(out.stdout.strip().splitlines()[-1:], ['[]'], out.stderr) + + def test_finbot_starts_without_them(self): + """With no webhook set, bot.py starts with the optional libraries blocked: no HTTP server, so no waitress.""" + skip_if_old_env(self) + with tempfile.TemporaryDirectory() as data: + proc = subprocess.Popen([sys.executable, '-u', 'bot.py'], cwd=ROOT, env=self.blocked(data), + stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True) + self.addCleanup(proc.wait) + self.addCleanup(proc.kill) + lines = [] + for line in proc.stderr: + lines.append(line) + if line.startswith('finbot is running'): + break + self.assertIsNone(proc.poll(), ''.join(lines)) + self.assertIn('No webhook is set: finbot takes no HTTP requests', ''.join(lines)) + + class StartupSignals(unittest.TestCase): """A reload (SIGHUP) that arrives while the bot is still starting must not kill it.""" @@ -367,8 +532,9 @@ def test_sighup_during_startup_is_ignored_and_works_afterwards(self): """A SIGHUP while bot.py starts is ignored, where its default action would kill it; once serving, it reloads.""" skip_if_old_env(self) with tempfile.TemporaryDirectory() as data: + no_jobs(data) env = bare_environment() - env.update(DATA_DIR=data, port='0', scheduler='False') + env.update(DATA_DIR=data, port='0') proc = subprocess.Popen([sys.executable, '-u', 'bot.py'], cwd=ROOT, env=env, stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True) self.addCleanup(proc.wait) @@ -382,10 +548,9 @@ def test_sighup_during_startup_is_ignored_and_works_afterwards(self): lines = [] for line in proc.stderr: lines.append(line) - if line.startswith('Opening socket'): + if line.startswith('finbot is running'): # the handlers are in break self.assertIsNone(proc.poll(), ''.join(lines)) - time.sleep(1) # the handler goes in just before serving proc.send_signal(signal.SIGHUP) for line in proc.stderr: # a reload now runs (there is no preferences.conf here, so it says so) lines.append(line) @@ -405,8 +570,9 @@ def test_usr1_flushes_the_cache_and_keeps_the_state(self): (cache / name).write_text('{}') Path(data, 'finbot_watchlist.json').write_text('["T1"]') # state, not cache Path(data, 'preferences.conf').write_text('cache=False\n') # the warm jobs a flush starts fetch nothing + no_jobs(data) env = bare_environment() - env.update(DATA_DIR=data, port='0', scheduler='False') + env.update(DATA_DIR=data, port='0') proc = subprocess.Popen([sys.executable, '-u', 'bot.py'], cwd=ROOT, env=env, stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, text=True) self.addCleanup(proc.wait) @@ -420,18 +586,17 @@ def test_usr1_flushes_the_cache_and_keeps_the_state(self): lines = [] for line in proc.stderr: lines.append(line) - if line.startswith('Opening socket'): + if line.startswith('finbot is running'): # the handlers are in break self.assertIsNone(proc.poll(), ''.join(lines)) self.assertEqual(len(os.listdir(cache)), 4) # nothing flushed yet - time.sleep(1) # the handler goes in just before serving proc.send_signal(signal.SIGUSR1) for line in proc.stderr: lines.append(line) if 'cache: warm jobs done' in line: # each in a child process, as the scheduler is off break self.assertIn('cache: flushed 2 files', ''.join(lines)) - self.assertIn('cache: filling it again: sharesight_warm.py, yahoo_warm.py', ''.join(lines)) + self.assertIn('cache: filling it again: sharesight_warm, yahoo_warm', ''.join(lines)) self.assertEqual(sorted(os.listdir(cache)), ['finbot_x.json.abc.tmp', 'matplotlib']) self.assertTrue(Path(data, 'finbot_watchlist.json').exists()) diff --git a/tests/test_finbot.py b/tests/test_finbot.py index ff4a9e5f..516ffe72 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -1,5 +1,5 @@ """Plumbing: message chunking, timezones, the WSGI handler.""" -import ast, contextlib, datetime, io, json, subprocess, tempfile, unittest, zoneinfo +import ast, contextlib, datetime, io, json, re, subprocess, tempfile, unittest, zoneinfo from pathlib import Path from unittest import mock from tests.common import * @@ -21,8 +21,8 @@ def post(url, **kwargs): posted.append(kwargs['json']) return Response() - with mock.patch.object(transports.net, 'post', post), mock.patch.object(transports.time, 'sleep', lambda s: 0): - results = REAL_TEXT(transports.TelegramBot('5'), ['x' * 60] * 300, {'keyboard': []}) + with mock.patch.object(net, 'post', post), mock.patch.object(transports.time, 'sleep', lambda s: 0): + results = REAL_TEXT(telegram_common.TelegramBot('5'), ['x' * 60] * 300, {'keyboard': []}) self.assertGreater(len(results), 1) self.assertEqual(len(posted), len(results)) self.assertIn('reply_markup', posted[-1]) @@ -47,7 +47,8 @@ def test_reminder_runs(self): reminder.run() def test_bot_handler(self): - """A sticker is answered 200 and not logged; git_version matches what git says.""" + """A sticker is answered 200, with nothing logged but the request's line (INFO); git_version matches what git + says.""" def post(message): """Post a Telegram message update to the WSGI handler: (its answer, what it logged).""" body = json.dumps({'update_id': 1, 'message': dict({'message_id': 5, 'chat': {'id': 55, 'type': 'private'}, @@ -57,7 +58,7 @@ def post(message): with contextlib.redirect_stderr(err): out = bot.main(env, lambda *a, **k: None) return out, err.getvalue() - self.assertEqual(post({'sticker': {'file_id': 'x'}}), ([b''], '')) # stickers: silent, not logged + self.assertEqual(post({'sticker': {'file_id': 'x'}}), ([b''], 'POST from telegram: we answered 200 OK\n')) self.assertGreaterEqual(len(bot.git_version()), 7) head = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=bot.__file__.rsplit('/', 1)[0], capture_output=True, text=True).stdout.strip() @@ -77,11 +78,11 @@ def test_every_function_has_a_docstring(self): self.assertEqual(missing, []) def test_every_line_fits_in_120_columns(self): - """Every line of the code, the tests, the shipped defaults, the units, the tools and CI fits in 120 - columns, tabs counting 4: an edit quotes whole lines. README.md, CLAUDE.md and doc/ are left out (CI runs no - tests for a change to those alone, so no test may read them), and so are data files.""" + """Every line of the code, the tests, the shipped defaults, the units, the tools, CI and the Docker files fits + in 120 columns, tabs counting 4: an edit quotes whole lines. README.md, CLAUDE.md and doc/ are left out (CI runs + no tests for a change to those alone, so no test may read them), and so are data files.""" patterns = ('*.py', '*.service', 'lib/**/*.py', 'tests/*.py', 'util/*', 'defaults/*', '.github/workflows/*.yml', - '.claude/hooks/*') + '.claude/hooks/*', 'Dockerfile', 'compose.yaml', '.dockerignore') files = sorted({path for pattern in patterns for path in Path(ROOT).glob(pattern) if path.is_file() and path.suffix != '.json'}) long = [f'{path.relative_to(ROOT)}:{number}' for path in files @@ -90,6 +91,17 @@ def test_every_line_fits_in_120_columns(self): self.assertGreater(len(files), 60) self.assertEqual(long, []) + def test_nothing_imports_pandas(self): + """finbot has no use for pandas, which took 0.8 s and 37 MB to load for three small jobs on dated closes: no + module imports it, not even inside a function.""" + files = [*Path(ROOT).glob('*.py'), *Path(ROOT, 'lib').rglob('*.py'), *Path(ROOT, 'util').glob('*.py'), + *Path(ROOT, 'tests').glob('*.py')] + found = [f'{path.relative_to(ROOT)}:{number}' for path in files + for number, line in enumerate(path.read_text(encoding='utf-8').splitlines(), 1) + if re.match(r'\s*(import|from) pandas\b', line)] + self.assertGreater(len(files), 60) + self.assertEqual(found, []) + def test_the_bot_and_its_reports_print_only_to_stderr(self): """systemd discards stdout (StandardOutput=null), so only util/ tools, run by hand, may print there.""" offenders = [] @@ -101,6 +113,22 @@ def test_the_bot_and_its_reports_print_only_to_stderr(self): self.assertEqual(offenders, []) +class Timeouts(unittest.TestCase): + def test_every_request_goes_through_net_with_a_timeout(self): + """The scheduled reports run in the bot's threads, which cannot be stopped: every request must end. So no module + but lib/net.py calls requests itself (net gives each call its timeouts), and none passes timeout=None.""" + call = re.compile(r'\brequests\.(?:get|post|put|patch|delete|head|request|Session)\(') + offenders = [] + for path in sorted([*Path(ROOT, 'lib').rglob('*.py'), Path(ROOT, 'bot.py'), *Path(ROOT, 'util').glob('*.py')]): + for number, line in enumerate(path.read_text().splitlines(), 1): + where = f'{path.relative_to(ROOT)}:{number}' + if call.search(line) and path.name != 'net.py': + offenders.append(f'{where} calls requests itself') + if re.search(r'\btimeout\s*=\s*None\b', line): + offenders.append(f'{where} passes timeout=None') + self.assertEqual(offenders, []) + + class OutputComparison(unittest.TestCase): def test_steps_are_compared_by_name_and_a_repeated_one_by_its_count(self): """util/record_outputs.py --diff shows a step that changed, appeared or went; a step run twice is told apart.""" @@ -166,6 +194,18 @@ def test_unreadable_is_unknown(self): self.repo(empty, 'garbage') self.assertEqual(bot.git_version(empty), 'unknown') + def test_an_image_without_git_names_the_commit_it_was_built_from(self): + """A Docker image has no .git: the commit it was built from (FINBOT_COMMIT, from the build's GIT_COMMIT) names + the version; a value that is no commit is 'unknown', and a .git still wins.""" + with tempfile.TemporaryDirectory() as empty, tempfile.TemporaryDirectory() as checkout: + self.repo(checkout, self.SHA) + for built, expected in ((self.SHA, '0123456'), ('abcdef1', 'abcdef1'), ('unknown', 'unknown'), + ('', 'unknown'), ('$(id)', 'unknown')): + with self.subTest(built=built), mock.patch.dict(os.environ, {'FINBOT_COMMIT': built}): + self.assertEqual(bot.git_version(empty), expected) + with mock.patch.dict(os.environ, {'FINBOT_COMMIT': 'f' * 40}): + self.assertEqual(bot.git_version(checkout), '0123456') + def test_does_not_run_git(self): """git_version reads .git itself: no git binary is run.""" with mock.patch('subprocess.run', side_effect=AssertionError('git was run')): diff --git a/tests/test_inbound.py b/tests/test_inbound.py index 71641921..68f7e576 100644 --- a/tests/test_inbound.py +++ b/tests/test_inbound.py @@ -1,9 +1,9 @@ -"""What reaches the bot: webhook requests turned away or taken, Slack's events, presses and signatures, -and user text escaped where it arrives.""" -import contextlib, hashlib, hmac, io, json, time, unittest +"""What reaches the bot: webhook requests turned away or taken, Slack's events, presses, slash commands and signatures, +the Slack app's manifest, and user text escaped where it arrives.""" +import contextlib, hashlib, hmac, io, json, re, time, types, unittest from urllib.parse import urlencode -import gevent from tests.common import * +from util import slack_manifest PRESS = {'type': 'block_actions', 'token': 'stok', 'trigger_id': 't1', 'user': {'id': 'U1', 'username': 'rob'}, @@ -95,7 +95,7 @@ def test_a_failure_before_authentication_is_only_logged(self): def broken(*a, **k): """Fail while routing.""" raise RuntimeError('boom') - self.patch(inbound, 'urlparse', broken) # fails while routing, before the secret is even looked at + self.patch(bot, 'route', broken) # fails while routing, before the secret is even looked at env = wsgi_environ('/telegram', b'{}', **self.SECRET) status = [] with contextlib.redirect_stderr(io.StringIO()) as err: @@ -134,7 +134,7 @@ def test_a_telegram_update_sent_again_runs_once(self): 'message': {'message_id': 1, 'chat': {'id': 5}, 'from': {'first_name': 'A'}, 'text': '.price'}} for _ in range(2): # Telegram sends an update again when it got no answer self.assertEqual(self.request(body=body, headers=self.SECRET)[1], '200 OK') - gevent.sleep(0.05) + self.join_workers() self.assertEqual(len(calls), 1) def test_an_update_is_taken_however_it_arrived(self): @@ -147,9 +147,9 @@ def test_an_update_is_taken_however_it_arrived(self): 'event': {'type': 'message', 'channel': 'C1', 'user': 'U1', 'text': '.trades', 'ts': '1.2'}} with contextlib.redirect_stderr(io.StringIO()): for _ in range(2): - self.assertEqual(bot.take(inbound.TELEGRAM, {'update_id': 70, 'message': message}), inbound.OK) - self.assertEqual(bot.take(inbound.SLACK, event), inbound.OK) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.assertEqual(bot.take(telegram_common.READER, {'update_id': 70, 'message': message}), inbound.OK) + self.assertEqual(bot.take(slack_common.READER, event), inbound.OK) + self.join_workers() self.assertEqual([(m.service, m.chat_id, m.text) for m in got], [('telegram', '5', '.price'), ('slack', 'C1', '.trades')]) @@ -167,6 +167,67 @@ def test_button_presses_are_logged(self): bot.main(wsgi_environ('/telegram', body, **secret), lambda *a, **k: None) self.assertIn(f'[Telegram button]: {who} chat 55 j|7|abcd', err.getvalue()) + def test_a_press_is_acknowledged_once_before_the_worker_gets_it(self): + """bot.take acknowledges a press, the one place that does: Telegram's with the toast its data makes, before the + worker has it; a resent copy is neither acknowledged nor run again. Slack's needs no call: the answer to the + update is the acknowledgement.""" + order = [] + self.patch(telegram_common, 'answerCallbackQuery', + lambda cid, text=None: order.append(('acknowledged', cid, text))) + self.patch(worker, 'process_callback', lambda press: order.append(('worker', press.service, press.data))) + update = {'update_id': 71, 'callback_query': {'id': 'q1', 'from': {'username': 'u'}, 'data': 'c|h|y|T1', + 'message': {'message_id': 9, 'chat': {'id': 55}}}} + with contextlib.redirect_stderr(io.StringIO()): + for _ in range(2): + self.assertEqual(bot.take(telegram_common.READER, update), inbound.OK) + self.join_workers() + self.assertEqual(bot.take(slack_common.READER, PRESS), inbound.OK) + self.join_workers() + self.assertEqual(order, [('acknowledged', 'q1', 'Loading 1Y…'), ('worker', 'telegram', 'c|h|y|T1'), + ('worker', 'slack', 'c|h|w|T1')]) + + +class DoubledMessages(FinbotCase): + """The same message from the same person in the same chat within a second is taken once: Telegram's mobile app + can send a command twice (/start in a new chat), each its own update.""" + + def setUp(self): + """The real check, and a clock of this test's own.""" + super().setUp() + self.patch(bot, 'doubled', REAL_DOUBLED) + self.now = [1000.0] + self.patch(bot, 'time', types.SimpleNamespace(monotonic=lambda: self.now[0])) + + def send(self, update_id, text='/start', user='rob', chat=55): + """Take a Telegram message, as an update of this id arrives.""" + with contextlib.redirect_stderr(io.StringIO()): + bot.take(telegram_common.READER, {'update_id': update_id, 'message': {'message_id': update_id, + 'chat': {'id': chat}, 'from': {'first_name': 'Rob', 'username': user}, 'text': text}}) + self.join_workers() + + def test_a_message_sent_again_within_a_second_is_taken_once(self): + """A second /start within a second is not run again; another person's, another chat's, another text, or one + over a second later is.""" + self.send(1) + self.send(2) + self.assertEqual(len(self.workers), 1) + self.send(3, user='ann') + self.send(4, chat=66) + self.send(5, text='.help') + self.assertEqual(len(self.workers), 4) + self.now[0] += 1.5 + self.send(6) + self.assertEqual(len(self.workers), 5) + + def test_a_discord_slash_command_is_always_answered(self): + """A slash command sent again at once still gets its answer: Discord shows an error for an interaction left + unanswered.""" + command = {'type': 2, 'id': '1', 'token': 't', 'application_id': '9', 'channel_id': '77', + 'member': {'user': {'id': '5', 'username': 'rob'}}, 'data': {'name': 'start'}} + with contextlib.redirect_stderr(io.StringIO()): + answers = [bot.take(discord_common.READER, dict(command, id=str(n))) for n in (1, 2)] + self.assertEqual([json.loads(answer.body) for answer in answers], [{'type': 5}, {'type': 5}]) + class SlackEvents(FinbotCase): """Slack sends every message in a subscribed channel, the bot's own posts included.""" @@ -188,7 +249,7 @@ def event(self, event_id=None, **event): status = [] with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): bot.main(wsgi_environ('/slack', body), lambda s, *a, **k: status.append(s)) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() return status[-1] def press(self, payload): @@ -197,7 +258,7 @@ def press(self, payload): with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): body = urlencode({'payload': json.dumps(payload)}).encode() bot.main(wsgi_environ('/slack', body), lambda s, *a, **k: status.append(s)) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() return status[-1] def test_a_button_press_reaches_the_worker(self): @@ -255,9 +316,9 @@ def test_a_retried_event_runs_once(self): def test_slack_does_not_change_the_telegram_bot_name(self): """A Slack event leaves the Telegram bot's name alone.""" - self.patch(inbound.TELEGRAM, 'bot_name', '@finbot_tg') + self.patch(telegram_common.READER, 'bot_name', '@finbot_tg') self.event(user='U1', text='.price') - self.assertEqual(inbound.TELEGRAM.bot_name, '@finbot_tg') + self.assertEqual(telegram_common.READER.bot_name, '@finbot_tg') class SlackSigning(FinbotCase): @@ -284,7 +345,7 @@ def post(self, secret='sss', age=0, token='stok', headers=None, tamper=False): status = [] with contextlib.redirect_stderr(io.StringIO()): bot.main(env, lambda s, *a, **k: status.append(s)) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() return status[-1] def test_a_button_press_is_signed_over_its_form(self): @@ -299,7 +360,7 @@ def test_a_button_press_is_signed_over_its_form(self): answers = [] with contextlib.redirect_stderr(io.StringIO()): bot.main(wsgi_environ('/slack', sent, **headers), lambda s, *a, **k: answers.append(s)) - gevent.sleep(0.05) + self.join_workers() self.assertEqual(answers, [status]) self.assertEqual(presses, [inbound.Press('slack', 't1', 'C1', '1.5', 'c|h|w|T1', '@rob')]) @@ -318,6 +379,102 @@ def test_unsigned_wrongly_signed_old_or_altered_requests_are_refused(self): self.assertEqual((self.requests, self.errors), ([], [])) +SLASH = {'token': 'stok', 'team_id': 'T1', 'channel_id': 'C1', 'channel_name': 'stocks', 'user_id': 'U1', + 'user_name': 'rob', 'command': '/price', 'text': '5 & 7d', 'response_url': 'https://hooks.slack.com/r/1', + 'trigger_id': 't9', 'api_app_id': 'A1'} # a Slack slash command, as its form's fields or a socket's payload + + +class SlackSlashCommands(FinbotCase): + """A Slack slash command posted to slackOutgoingWebhook as a form, signed as an event is.""" + + def setUp(self): + """Slack in HTTP mode with a signing secret, and the worker replaced by a list of what it would have run.""" + super().setUp() + self.env(slackOutgoingWebhook='https://example.com/slack', slackSigningSecret='sss') + self.requests = [] + self.patch(worker, 'process_request', self.requests.append) + + def post(self, fields, secret='sss'): + """Post the form of a slash command, signed with secret: (the status, the body).""" + body = urlencode(fields).encode() + timestamp = str(int(time.time())) + mac = hmac.new(secret.encode(), b'v0:' + timestamp.encode() + b':' + body, hashlib.sha256).hexdigest() + env = wsgi_environ('/slack', body, HTTP_X_SLACK_REQUEST_TIMESTAMP=timestamp, HTTP_X_SLACK_SIGNATURE='v0=' + mac) + status = [] + with contextlib.redirect_stderr(io.StringIO()): + answer = b''.join(bot.main(env, lambda s, *a, **k: status.append(s))) + self.join_workers() + return status[-1], answer + + def test_a_slash_command_is_the_message_that_says_the_same(self): + """/price with 5 & 7d is the message /price 5 & 7d, from <@U1>, in the channel or DM it was sent in, which + the reply goes to; Slack gets an empty 200 at once. There is no message of it, so no ⏳.""" + self.assertEqual(self.post(SLASH), ('200 OK', b'')) + self.assertEqual(self.post({**SLASH, 'channel_id': 'D1', 'command': '/job', 'text': ''}), ('200 OK', b'')) + self.assertEqual(self.requests, [inbound.Message('slack', 'C1', '<@U1>', '/price 5 & 7d', '', '<@U1>'), + inbound.Message('slack', 'D1', '<@U1>', '/job', '', '<@U1>')]) + self.assertEqual([worker.read(r.text, r.bot_name)[0].names[0] for r in self.requests], ['price', 'jobs']) + self.assertFalse(delivery.reply_to('slack', 'C1', self.requests[0].message_id).shows_typing) + + def test_a_slash_command_unsigned_or_altered_is_refused(self): + """A slash command signed with another secret is refused, and none reaches the worker.""" + self.assertEqual(self.post(SLASH, secret='wrong')[0], '403 Forbidden') + self.assertEqual(self.requests, []) + + +class SlackManifest(unittest.TestCase): + """util/slack_manifest.py: the Slack app's manifest, with the slash commands of the command table.""" + + def test_socket_mode_or_http_mode(self): + """Without a URL, Socket Mode and no Request URL anywhere; with one, HTTP mode and that URL for the events, the + buttons and every slash command. Either way: the messages tab for DMs, interactivity, and the scopes that the + replies, the slash commands, the charts, ⏳ and each event need.""" + socket = slack_manifest.manifest() + self.assertTrue(socket['settings']['socket_mode_enabled']) + self.assertNotIn('request_url', json.dumps(socket)) + self.assertNotIn('"url"', json.dumps(socket)) + url = 'https://www.example.com:8443/slack' + http = slack_manifest.manifest(url=url) + self.assertFalse(http['settings']['socket_mode_enabled']) + self.assertEqual((http['settings']['event_subscriptions']['request_url'], + http['settings']['interactivity']['request_url']), (url, url)) + self.assertTrue(all(command['url'] == url for command in http['features']['slash_commands'])) + for manifest in (socket, http): + self.assertTrue(manifest['features']['app_home']['messages_tab_enabled']) + self.assertTrue(manifest['settings']['interactivity']['is_enabled']) + self.assertEqual(manifest['settings']['event_subscriptions']['bot_events'], ['app_mention', 'message.im']) + self.assertEqual(manifest['oauth_config']['scopes']['bot'], ['app_mentions:read', 'chat:write', 'commands', + 'files:read', 'files:write', 'im:history', 'reactions:write']) + + def test_every_message_and_the_slash_commands(self): + """With every_message the bot hears every message in its channels (not only mentions, which it would then + answer twice). The slash commands are the command table's, second names included, each named and described + as Slack allows, and each reads as its command.""" + manifest = slack_manifest.manifest(every_message=True) + self.assertEqual(manifest['settings']['event_subscriptions']['bot_events'], + ['message.channels', 'message.groups', 'message.im']) + self.assertIn('channels:history', manifest['oauth_config']['scopes']['bot']) + self.assertNotIn('app_mentions:read', manifest['oauth_config']['scopes']['bot']) + commands = manifest['features']['slash_commands'] + self.assertEqual([c['command'] for c in commands], ['/' + command.name for command in worker.slash_commands()]) + self.assertTrue({'/profile', '/job', '/forwardpe'} <= {c['command'] for c in commands}) + for command in commands: + with self.subTest(command=command['command']): + self.assertRegex(command['command'], r'^/[a-z0-9_]{1,31}$') + self.assertTrue(0 < len(command['description']) <= 2000) + self.assertLessEqual(len(command.get('usage_hint', '')), 1000) + self.assertIsNotNone(worker.read(f"{command['command']} {command.get('usage_hint', '')}", '')[0]) + + def test_the_tool_prints_json_and_refuses_a_plain_http_url(self): + """The tool prints the manifest as JSON, to paste; a URL that is not HTTPS is refused, as Slack would.""" + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(slack_manifest.main(['--name', 'stockbot']), 0) + self.assertEqual(json.loads(out.getvalue())['features']['bot_user']['display_name'], 'stockbot') + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(slack_manifest.main(['--url', 'http://www.example.com/slack']), 2) + self.assertIn('--url must be https://', err.getvalue()) + + class UserTextIsEscaped(FinbotCase): """Replies go out as Telegram HTML: text a user wrote is shown, never read as markup. One stray < in a .plan entry made Telegram refuse that reply and every .plan reply after it.""" @@ -329,11 +486,10 @@ def test_a_plan_is_shown_as_text(self): self.assertEqual(self.command('.plan')[0][1], reply) # and the same way again self.sent.clear() # inbound has unescaped Slack's text - worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.plan sell ', BOT, 'U')) + worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.plan sell & run', BOT, 'U')) text = '\n'.join(self.sent[0][1]) self.assertIn('*<@U1>*', text) # a Slack mention stays markup - self.assertIn('_sell <dear>_', text) # escaped once, not twice - self.assertIn('_buy <cheap> & hold_', text) + self.assertIn('_sell <dear> & run_', text) # escaped once, not twice def test_a_telegram_name_is_escaped_where_it_arrives(self): """A Telegram first name is escaped as the update arrives, in the handle and the name.""" @@ -343,7 +499,7 @@ def test_a_telegram_name_is_escaped_where_it_arrives(self): env = wsgi_environ('/telegram', json.dumps(body).encode(), HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN='tok') with contextlib.redirect_stderr(io.StringIO()): bot.main(env, lambda *a, **k: None) - gevent.sleep(0.05) # the worker runs in a (gevent) thread + self.join_workers() self.assertEqual((requests[0].user, requests[0].user_name), ('@A<b>&',) * 2) # the handle and the name diff --git a/tests/test_pull.py b/tests/test_pull.py new file mode 100644 index 00000000..908a4975 --- /dev/null +++ b/tests/test_pull.py @@ -0,0 +1,305 @@ +"""The pull modes: a network without a webhook URL is asked for its updates, which reach the same reader as the +webhook's. Telegram's long polling, Slack's Socket Mode.""" +import contextlib, io, json, sys, types, unittest +from urllib.parse import urlencode +from tests.common import * +from lib import pull, slack_websocket, telegram_longpolling + + +def callback_query(update_id, data='c|h|y|T1'): + """A Telegram update: a button pressed under message 9 in chat 55.""" + return {'update_id': update_id, 'callback_query': {'id': f'q{update_id}', 'from': {'username': 'rob'}, + 'data': data, 'message': {'message_id': 9, 'chat': {'id': 55}}}} + + +class Stop(Exception): + """Raised by a test's fake to end a listener's loop.""" + + +class TelegramPolling(FinbotCase): + """getUpdates in a loop: each update handed on in order, the next asked for after it, and failures waited out.""" + + def setUp(self): + """Telegram in pull mode: a bot token and no webhook URL; telegram_common.call answers from self.answers.""" + super().setUp() + self.env(telegramBotToken='123:abc', telegramOutgoingWebhook='') + self.calls, self.answers = [], [] + def call(method, **kwargs): + """Note the call, and give the next answer (an exception is raised).""" + self.calls.append((method, kwargs)) + answer = self.answers.pop(0) + if isinstance(answer, Exception): + raise answer + return answer + self.patch(telegram_common, 'call', call) + + def test_each_update_is_handed_on_in_order_and_the_next_asked_for_after_it(self): + """Each update goes to take in order; the next getUpdates asks for those after the last (which also tells + Telegram it arrived), waits up to 50 seconds for one, and gives the request 65.""" + taken = [] + message = {'update_id': 10, 'message': {'message_id': 1, 'chat': {'id': 5}, 'from': {'first_name': 'A'}, + 'text': '.price'}} + self.answers += [[message, callback_query(11)], []] + poll = telegram_longpolling.TelegramPoll() + self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) + self.assertEqual(poll.poll(lambda network, update: taken.append((network, update))), 0) + self.assertEqual(taken, [(telegram_common.READER, message), (telegram_common.READER, callback_query(11))]) + allowed = ['message', 'callback_query'] + asked = {'timeout': (5, 65), 'logged': False} # not logged: it is asked again and again + self.assertEqual(self.calls, [('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed}, **asked}), + ('getUpdates', {'json': {'timeout': 50, 'allowed_updates': allowed, 'offset': 12}, **asked})]) + + def test_failures_are_waited_out_longer_each_time_and_logged_without_the_token(self): + """A failed getUpdates (a 409 when a webhook is set or another finbot polls, the network down) waits 5 seconds, + then twice as long each time up to a minute; a success starts again from 5. The log never shows the token.""" + conflict = RuntimeError('Telegram getUpdates: Conflict: terminated by other getUpdates request') + down = ConnectionError("Max retries exceeded with url: /bot123:SECRET-token/getUpdates") + self.answers += [conflict, down, conflict, conflict, conflict, conflict, [], down] + poll = telegram_longpolling.TelegramPoll() + err = io.StringIO() + with contextlib.redirect_stderr(err): + waits = [poll.poll(lambda *a: None) for _ in range(8)] + self.assertEqual(waits, [5, 10, 20, 40, 60, 60, 0, 5]) + self.assertIn('telegram: getUpdates failed, trying again in 5 s: Telegram getUpdates: Conflict', err.getvalue()) + self.assertNotIn('SECRET', err.getvalue()) + + def test_an_update_that_fails_is_reported_once_and_not_fetched_again(self): + """An update whose handling fails is reported to the default chats, as a failed webhook request is, and the + ones after it still go; the next getUpdates asks for those after all of them.""" + taken = [] + def take(network, update): + """Fail on update 20, note the rest.""" + if update['update_id'] == 20: + raise KeyError('chat') + taken.append(update['update_id']) + self.answers += [[callback_query(20), callback_query(21)], []] + poll = telegram_longpolling.TelegramPoll() + poll.poll(take) + poll.poll(take) + self.assertEqual(taken, [21]) + self.assertEqual(self.errors, ["⚠️ telegram: KeyError: 'chat'"]) + self.assertEqual(self.calls[1][1]['json']['offset'], 22) + + def test_run_polls_for_good_waiting_as_each_round_says(self): + """run() logs the mode, then polls round after round, sleeping what each round returns.""" + rounds, slept = iter([0, 5, 0]), [] + poll = telegram_longpolling.TelegramPoll() + self.patch(poll, 'poll', lambda take: next(rounds)) + def sleep(seconds): + """Note the wait; end the loop after three.""" + slept.append(seconds) + if len(slept) == 3: + raise Stop + self.patch(pull.time, 'sleep', sleep) + err = io.StringIO() + with contextlib.redirect_stderr(err), self.assertRaises(Stop): + poll.run(bot.take) + self.assertEqual(slept, [0, 5, 0]) + self.assertIn('telegram: long polling for updates (no telegramOutgoingWebhook)', err.getvalue()) + + def test_a_listener_for_each_network_without_a_webhook(self): + """Telegram is polled with its bot token and no telegramOutgoingWebhook; with the URL, or without a token, it + is not.""" + self.assertEqual([type(listener) for listener in bot.listeners()], [telegram_longpolling.TelegramPoll]) + self.env(telegramOutgoingWebhook='https://example.com/telegram') + self.assertEqual(bot.listeners(), []) + self.env(telegramBotToken='', telegramOutgoingWebhook='') + self.assertEqual(bot.listeners(), []) + + def test_a_press_polled_for_is_the_press_the_webhook_gets(self): + """The same button press, posted to the webhook (http mode) or polled for (pull mode), reaches the worker as the + same Press, and is acknowledged the same way.""" + presses, toasts = [], [] + self.patch(worker, 'process_callback', presses.append) + self.patch(telegram_common, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) + body = json.dumps(callback_query(80)).encode() + with contextlib.redirect_stderr(io.StringIO()): + self.env(telegramOutgoingWebhook='https://example.com/telegram') + bot.main(wsgi_environ('/telegram', body, HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN='tok'), lambda *a: None) + self.env(telegramOutgoingWebhook='') + update = callback_query(81) + update['callback_query']['id'] = 'q80' # the same press, as getUpdates gives it + self.answers.append([update]) + telegram_longpolling.TelegramPoll().poll(bot.take) + self.join_workers() + self.assertEqual(len(presses), 2) + self.assertEqual(presses[0], presses[1]) + self.assertEqual(toasts, ['Loading 1Y…'] * 2) + + +SLACK_PRESS = {'type': 'block_actions', 'token': 'stok', 'trigger_id': 't1', 'user': {'id': 'U1', 'username': 'rob'}, + 'container': {'type': 'message', 'message_ts': '1.5', 'channel_id': 'C1'}, + 'actions': [{'action_id': 'c|h|w|T1', 'value': 'c|h|w|T1', 'type': 'button'}]} # a period button pressed +SLACK_EVENT = {'type': 'event_callback', 'event_id': 'Ev1', 'authorizations': [{'user_id': 'UBOT'}], + 'event': {'type': 'message', 'channel': 'C1', 'user': 'U1', 'text': '.price', 'ts': '1.2'}} + + +class FakeSocket: + """A connection that gives the messages it was made with, a dict sent as JSON, and notes what is sent and when.""" + + def __init__(self, messages, log): + """messages: what receive() gives in turn (dicts as their JSON, None for a silence); log: a list to note in.""" + self.messages, self.log = list(messages), log + + def receive(self, timeout): + """The next message; when there are none left, the socket fails.""" + if not self.messages: + raise ConnectionError('the test socket has no more') + message = self.messages.pop(0) + return json.dumps(message) if isinstance(message, dict) else message + + def send(self, text): + """Note an ack.""" + self.log.append(('ack', json.loads(text)['envelope_id'])) + + def ping(self): + """Note a ping.""" + self.log.append(('ping',)) + + def close(self): + """Note the close.""" + self.log.append(('close',)) + + +class SlackSocketMode(FinbotCase): + """Socket Mode: envelopes acked before anything else, their payloads to the same reader as the webhook's, a socket + opened again when Slack closes it, and a silent one taken for dead.""" + + def setUp(self): + """Slack in pull mode: the bot and app-level tokens, no webhook URL; apps.connections.open answers a URL.""" + super().setUp() + self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') + self.log = [] + self.patch(slack_common, 'call', lambda method, as_app=False, **data: + self.log.append((method, as_app)) or {'ok': True, 'url': 'wss://example.com/link'}) + + def socket(self, *messages): + """A SlackSocket whose connections give these messages.""" + listener = slack_websocket.SlackSocket() + self.patch(listener, 'connect', lambda url: self.log.append(('connect', url)) or FakeSocket(messages, self.log)) + return listener + + def test_each_envelope_is_acked_before_its_payload_is_handed_on(self): + """After hello, an event and a button press each get their ack first, then go to take as Slack would post + them; Slack's disconnect ends the socket, and the next round opens another at once.""" + take = lambda network, update: self.log.append(('take', network.service, update['type'])) + listener = self.socket({'type': 'hello'}, + {'envelope_id': 'e1', 'type': 'events_api', 'payload': SLACK_EVENT}, + {'envelope_id': 'e2', 'type': 'interactive', 'payload': SLACK_PRESS}, + {'type': 'disconnect', 'reason': 'refresh_requested'}) + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(listener.poll(take), 0) + self.assertEqual(self.log, [('apps.connections.open', True), ('connect', 'wss://example.com/link'), + ('ack', 'e1'), ('take', 'slack', 'event_callback'), ('ack', 'e2'), ('take', 'slack', 'block_actions'), + ('close',)]) + self.assertIn('slack: Slack is closing the socket (refresh_requested), opening another', err.getvalue()) + + def test_a_silent_socket_is_pinged_then_taken_for_dead(self): + """Silence sends a ping; a pong (or anything) keeps the socket; silence after a ping fails it, and the next + round waits the back-off.""" + listener = self.socket({'type': 'hello'}, None, '', None, None) + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual(listener.poll(lambda *a: None), 5) + self.assertEqual([entry for entry in self.log if entry[0] in ('ping', 'close')], [('ping',), ('ping',), + ('close',)]) + self.assertIn('slack: the socket failed, trying again in 5 s: nothing from Slack for 60 s', err.getvalue()) + + def test_failures_wait_longer_each_time_and_hello_starts_again(self): + """A socket that cannot be opened, or fails, waits longer each time; one that says hello starts again from 5.""" + self.patch(slack_common, 'call', mock.Mock(side_effect=RuntimeError('Slack apps.connections.open:' + ' invalid_auth'))) + listener = slack_websocket.SlackSocket() + with contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual([listener.poll(lambda *a: None) for _ in range(3)], [5, 10, 20]) + self.assertIn('slack: opening a socket failed, trying again in 5 s: Slack apps.connections.open:' + ' invalid_auth', err.getvalue()) + self.patch(slack_common, 'call', lambda *a, **k: {'url': 'wss://example.com/link'}) + self.patch(listener, 'connect', lambda url: FakeSocket([{'type': 'hello'}], self.log)) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(listener.poll(lambda *a: None), 5) # the socket failed, after a hello + + def test_a_press_over_the_socket_is_the_press_the_webhook_gets(self): + """The same button press, posted to slackOutgoingWebhook (http mode) or sent over the socket, reaches the worker + as the same Press; an event already taken over one is not run again over the other (its event_id).""" + presses, messages = [], [] + self.patch(worker, 'process_callback', presses.append) + self.patch(worker, 'process_request', messages.append) + self.env(slackOutgoingWebhook='https://example.com/slack', slackOutgoingToken='stok') + with contextlib.redirect_stderr(io.StringIO()): + body = urlencode({'payload': json.dumps(SLACK_PRESS)}).encode() + bot.main(wsgi_environ('/slack', body), lambda *a: None) + bot.main(wsgi_environ('/slack', json.dumps({**SLACK_EVENT, 'token': 'stok'}).encode()), lambda *a: None) + self.env(slackOutgoingWebhook='') + self.socket({'type': 'hello'}, {'envelope_id': 'e1', 'type': 'interactive', 'payload': SLACK_PRESS}, + {'envelope_id': 'e2', 'type': 'events_api', 'payload': SLACK_EVENT}, + {'type': 'disconnect', 'reason': 'warning'}).poll(bot.take) + self.join_workers() + self.assertEqual(len(presses), 2) + self.assertEqual(presses[0], presses[1]) + self.assertEqual([m.text for m in messages], ['.price']) + + def test_a_slash_command_over_the_socket_is_the_message_the_webhook_gets(self): + """The same slash command, posted to slackOutgoingWebhook as a form (http mode) or sent over the socket (its + envelope acked first), reaches the worker as the same Message.""" + messages = [] + self.patch(worker, 'process_request', messages.append) + slash = {'token': 'stok', 'channel_id': 'C1', 'user_id': 'U1', 'user_name': 'rob', 'command': '/forwardpe', + 'text': 'top', 'trigger_id': 't9'} + self.env(slackOutgoingWebhook='https://example.com/slack', slackOutgoingToken='stok') + with contextlib.redirect_stderr(io.StringIO()): + bot.main(wsgi_environ('/slack', urlencode(slash).encode()), lambda *a: None) + self.env(slackOutgoingWebhook='') + self.socket({'type': 'hello'}, {'envelope_id': 'e1', 'type': 'slash_commands', 'payload': slash}, + {'type': 'disconnect', 'reason': 'warning'}).poll(bot.take) + self.join_workers() + self.assertEqual(messages, [inbound.Message('slack', 'C1', '<@U1>', '/forwardpe top', '', '<@U1>')] * 2) + self.assertIn(('ack', 'e1'), self.log) + + def test_the_websocket_is_imported_only_to_connect_and_read_as_slack_needs(self): + """websocket-client is imported when a socket is opened, not before; its frames read as text, '' for a pong, + None for a timeout, and a close raises ConnectionError.""" + class Timeout(Exception): + """websocket-client's timeout.""" + frames = [(1, b'{"type": "hello"}'), (10, b''), Timeout(), (8, b'')] + class WS: + """A websocket-client connection that gives the frames.""" + def settimeout(self, timeout): + """Note the timeout.""" + self.timeout = timeout + def recv_data(self, control_frame=False): + """The next frame, with control frames (pings, pongs) too.""" + frame = frames.pop(0) + if isinstance(frame, Exception): + raise frame + return frame + stub = types.SimpleNamespace(create_connection=lambda url, timeout: WS(), WebSocketTimeoutException=Timeout, + ABNF=types.SimpleNamespace(OPCODE_TEXT=1, OPCODE_CLOSE=8)) + self.patch(sys, 'modules', {**sys.modules, 'websocket': stub}) + socket = slack_websocket.WebSocket('wss://example.com/link') + self.assertEqual([socket.receive(30) for _ in range(3)], ['{"type": "hello"}', '', None]) + with self.assertRaises(ConnectionError): + socket.receive(30) + + def test_slack_is_listened_to_with_both_tokens_and_no_webhook(self): + """The socket is opened with the bot and app-level tokens and no slackOutgoingWebhook; with the URL (http mode), + or without either token, it is not.""" + self.env(telegramBotToken='') + self.assertEqual([type(listener) for listener in bot.listeners()], [slack_websocket.SlackSocket]) + for changes in ({'slackOutgoingWebhook': 'https://example.com/slack'}, {'slackAppToken': ''}, + {'slackBotToken': ''}): + with self.subTest(changes=changes): + self.env(**changes) + self.assertEqual(bot.listeners(), []) + self.env(slackBotToken='xoxb-1', slackAppToken='xapp-1', slackOutgoingWebhook='') + + +class Backoff(unittest.TestCase): + """The wait after failures in a row.""" + + def test_it_doubles_from_five_seconds_up_to_a_minute(self): + """5, 10, 20, 40, then a minute for good.""" + self.assertEqual([pull.backoff(n) for n in range(1, 8)], [5, 10, 20, 40, 60, 60, 60]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_reports.py b/tests/test_reports.py index d6ca0cd8..72906149 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,4 +1,5 @@ """Every chat command and scheduled report produces output on mocked data.""" +import contextlib import unittest import datetime import re @@ -113,12 +114,12 @@ def test_a_portfolio_command_without_a_portfolio_lists_them(self): def test_prepare_marketcap(self): """The market cap list is built.""" - self.assertTrue(rankings.marketcap_list(TELEGRAM, length=5)) + self.assertTrue(rankings.marketcap_list(TELEGRAM, 'telegram:55', length=5)) def test_prepare_rating(self): """The buy and sell lists are built.""" for action in ('buy', 'sell'): - self.assertTrue(rankings.prepare_rating_payload(TELEGRAM, action, length=5)[0]) + self.assertTrue(rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', action, length=5)[0]) def test_ratings_list_the_strongest_first(self): """1 is a strong buy and 5 a strong sell: .sell used to list the mildest sells first. On equal ratings, more @@ -128,19 +129,19 @@ def test_ratings_list_the_strongest_first(self): 'T7': ('strong_buy', 1.0, 6), 'T4': ('sell', 4.2, 7), 'T5': ('underperform', 4.6, 10), 'T6': ('sell', 4.2, 8), 'T8': ('strong_buy', 1.0, 2)} - self.patch(watchlist, 'with_holdings', lambda: list(ratings)) + self.patch(watchlist, 'with_holdings', lambda chat: list(ratings)) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | dict(zip(('recommend', 'recommend_index', 'recommend_analysts'), ratings[t]))}) def order(action): """(the tickers listed, the tickers charted) of a ratings report, in order.""" - payload, chart = rankings.prepare_rating_payload(TELEGRAM, action) + payload, chart = rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', action) return [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], [label for label, _ in chart.rows] self.assertEqual(order('buy'), (['T2', 'T3', 'T1'], ['T2', 'T3', 'T1'])) self.assertEqual(order('sell'), (['T5', 'T6', 'T4'], ['T5', 'T6', 'T4'])) - self.assertIn('(7+ analysts)', rankings.prepare_rating_payload(TELEGRAM, 'buy')[0][0]) - self.patch(watchlist, 'with_holdings', lambda: ['T7', 'T8']) - self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'buy'), + self.assertIn('(7+ analysts)', rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', 'buy')[0][0]) + self.patch(watchlist, 'with_holdings', lambda chat: ['T7', 'T8']) + self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', 'buy'), (['No tracked stocks rated buy by 7+ analysts'], None)) def test_a_small_cap_quoted_in_won_needs_the_bigger_move_too(self): @@ -150,7 +151,7 @@ def test_a_small_cap_quoted_in_won_needs_the_bigger_move_too(self): 'market_cap': 1e11}) for t in ts}) self.prefs(demote_volatile=True) - quote = price.gather()[1]['T1'] + quote = price.gather(TICKERS)['T1'] self.assertAlmostEqual(quote['market_cap_usd'], 1e11 / 1380.0) # US$72M self.assertEqual(price.trigger_bar('T1', quote, 5, {}), 5 * config.prefs.volatility_multiplier) @@ -159,8 +160,7 @@ def test_a_short_top_list_repeats_none_of_its_best_as_the_worst(self): with none left there is no Bottom list and no gap in the chart; the subtitle says "over the past 3 months".""" for count, bottom, bars in ((12, 2, 13), (8, 0, 8)): with self.subTest(count=count): - self.patch(watchlist, 'with_holdings', lambda: TICKERS[:count]) - found = price.moves('period', 0, 90, interactive=True) + found = price.moves(TICKERS[:count], 'period', 0, 90, interactive=True) payload, chart = price.report(TELEGRAM, found, 'period', 0, 90, top=10, interactive=True) shown = payload[payload.index('Bottom 10') + 1:] if bottom else [] self.assertEqual((len(shown), len(chart.rows)), (bottom, bars)) @@ -183,13 +183,13 @@ def test_the_small_caps_betas_are_fetched_together(self): def test_forward_pe_lists_need_as_many_analysts_as_buy(self): """A forward P/E is analysts' estimate: from few, it says little, as a perfect 1.0 buy from two brokers does.""" analysts = {'T1': 7, 'T2': 6, 'T3': 12, 'T4': 2} - self.patch(watchlist, 'with_holdings', lambda: list(analysts)) + self.patch(watchlist, 'with_holdings', lambda chat: list(analysts)) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) | {'price_to_earnings_forward': -5 if t == 'T3' else 20} for t in ts}) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'recommend_analysts': analysts[t]}}) def listed(mode='top'): """(heading, the tickers listed, the chart) of a forward P/E list.""" - payload, chart = rankings.ratio_list(TELEGRAM, 'forward pe', mode) + payload, chart = rankings.ratio_list(TELEGRAM, 'telegram:55', 'forward pe', mode) return payload[0], [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], chart heading, names, chart = listed() # T2 and T4: too few analysts; T3: negative @@ -199,7 +199,7 @@ def listed(mode='top'): self.assertIn('Co T2', rankings.ratio_line(TELEGRAM, 'T2', 'forward pe')) # one ticker asked for: shown self.prefs(min_analysts=13) # the one setting moves .buy and .forward pe together self.assertEqual(listed()[0], 'No tracked stocks with 13+ analysts have a forward pe ratio') - self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'buy')[0], + self.assertEqual(rankings.prepare_rating_payload(TELEGRAM, 'telegram:55', 'buy')[0], ['No tracked stocks rated buy by 13+ analysts']) def test_market_caps_compare_in_us_dollars(self): @@ -214,15 +214,15 @@ def fetch(tickers): for t in tickers if t in fx or t in stocks} self.patch(yahoo, 'fetch', fetch) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: fetch([t])) - self.patch(watchlist, 'with_holdings', lambda: list(stocks)) - payload = rankings.marketcap_list(TELEGRAM) + self.patch(watchlist, 'with_holdings', lambda chat: list(stocks)) + payload = rankings.marketcap_list(TELEGRAM, 'telegram:55') self.assertIn('by market cap (USD)', payload[0]) # Z1: no rate, left out self.assertEqual([line.split('Co ')[1].split(' ')[0] for line in payload[1:]], ['U1', 'K1.KS', 'L1.L', 'K2.KS']) self.assertIn('mkt cap: 731.88 B', payload[2]) # 1.01 Q won self.assertIn('mkt cap: KRW 1.01 Q (USD 731.88 B)', rankings.marketcap_line(TELEGRAM, 'K1.KS')) self.assertIn('mkt cap: GBP 62.50 B (USD 79.11 B)', rankings.marketcap_line(TELEGRAM, 'L1.L')) # pence: pounds - beta = rankings.prepare_beta_payload(TELEGRAM)[0] + beta = rankings.prepare_beta_payload(TELEGRAM, 'telegram:55')[0] self.assertIn('Co K2.KS', str(beta)) # 500 billion won is US$362M: a small cap, which it never was unconverted self.assertNotIn('Co K1.KS', str(beta)) @@ -233,7 +233,7 @@ def test_prepare_value(self): ('peg', 'top', 'Top 5 tracked stocks by peg ratio'), ('peg', 'negative', 'negative peg: no data found')): with self.subTest(kind=kind, mode=mode): - payload, chart = rankings.ratio_list(TELEGRAM, kind, mode, length=5) + payload, chart = rankings.ratio_list(TELEGRAM, 'telegram:55', kind, mode, length=5) self.assertIn(heading, payload[0]) def test_one_stocks_ratio_shows_what_yahoo_has(self): @@ -253,11 +253,11 @@ def test_pe_sends_its_chart_as_forward_pe_and_peg_do(self): def test_prepare_beta(self): """The beta report charts a bar per stock listed, and says so when none qualify.""" - payload, chart = rankings.prepare_beta_payload(TELEGRAM) + payload, chart = rankings.prepare_beta_payload(TELEGRAM, 'telegram:55') self.assertIn('Beta over 1.5', payload[0]) self.assertEqual(len(chart.rows), len(payload) - 1) # a bar per stock listed self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'beta': 1.0}}) - self.assertEqual(rankings.prepare_beta_payload(TELEGRAM), + self.assertEqual(rankings.prepare_beta_payload(TELEGRAM, 'telegram:55'), (['No tracked stocks have a beta over 1.5 and mkt cap under US$2B'], None)) def test_prepare_profile(self): @@ -298,7 +298,38 @@ def test_a_profile_warns_of_a_ps_ratio_only_above_15(self): def test_prepare_watchlist(self): """The watchlist is listed.""" - self.assertTrue(holdings.prepare_watchlist(TELEGRAM, '@u')) + self.assertTrue(holdings.prepare_watchlist(TELEGRAM, 'telegram:55', '@u')) + + def test_each_chat_follows_its_own_watchlist_besides_the_holdings(self): + """Each chat has a watchlist of its own: .watchlist lists it, and a report, asked for or scheduled, covers that + chat's watchlist and the holdings, which every chat shares, and no other chat's watchlist.""" + self.patch(watchlist, 'with_holdings', REAL_GET_HOLDINGS) + self.patch(sharesight, 'tickers', lambda: ['T1']) + fetched = [] + self.patch(yahoo, 'fetch', lambda ts: fetched.append(sorted(ts)) or {t: market_entry(t) for t in ts}) + self.command('.watchlist add T2') + self.command('.watchlist add T3', chat='56') + listed = [str(self.command('.watchlist', chat=chat)) for chat in ('55', '56')] + self.assertEqual([('Co T2' in text, 'Co T3' in text) for text in listed], [(True, False), (False, True)]) + for chat, followed in (('55', ['T1', 'T2']), ('56', ['T1', 'T3'])): + with self.subTest(chat=chat): + fetched.clear() + self.command('.price', chat=chat) + self.assertEqual(fetched[0], followed) + fetched.clear() + self.command('.marketcap', chat=chat) + self.assertEqual(fetched[0], followed) + fetched.clear() + shorts.run(delivery.chat('telegram:56', scheduled=True)) + self.assertEqual(fetched[0], ['T1', 'T3']) + + def test_plans_are_kept_per_chat(self): + """.plan records the sender's plan in this chat, and lists this chat's plans, not another chat's.""" + self.command('.plan buy low') + self.assertIn('buy low', str(self.command('.plan'))) + self.assertEqual(self.command('.plan', chat='56'), [('text', ['No plans yet: .plan TEXT records yours'])]) + self.assertNotIn('buy low', str(self.command('.plan sell high', chat='56'))) + self.assertNotIn('sell high', str(self.command('.plan'))) def test_compare_usage(self): """.compare with one ticker gets its usage, from the command table.""" @@ -336,27 +367,64 @@ def test_only_the_scheduled_earnings_report_is_pinned(self): """The scheduled earnings report is pinned on Telegram; a reply to .earnings is not, or the chat would collect pinned copies, and neither is the ex-dividend report.""" pins = [] - self.patch(transports.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) + self.patch(telegram_common.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) self.command('.earnings') self.assertEqual(pins, []) - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}, telegramChatID='1') + self.env(telegramChatID='1') cal.run(event='ex-dividend') cal.run(event='earnings') self.assertEqual(pins, ['1']) + def test_a_chats_job_runs_its_report_as_scheduled_not_as_a_command(self): + """A report a chat's job runs is a scheduled run, as the home chats' is: it says nothing when nothing passes, + where a command's reply says so, and the earnings report is pinned. price, shorts and cal took any chat they + were given for a command.""" + pins = [] + self.patch(telegram_common.TelegramBot, 'pin', lambda chat, answers: pins.append(chat.where)) + job_chat = delivery.chat('telegram:5', scheduled=True) + for module, kwargs in ((price, {'mode': 'premarket', 'threshold': 1000}), (shorts, {'threshold': 1000})): + with self.subTest(report=module.__name__): + self.sent.clear() + module.run(job_chat, **kwargs) + self.assertEqual(self.sent, []) + module.run(telegram_common.TelegramBot('5'), **kwargs) # a command's reply says that nothing passed + self.assertTrue(self.sent) + cal.run(job_chat, event='earnings') + self.assertEqual(pins, ['5']) + def test_earnings_skip_an_adr_whose_primary_listing_yahoo_does_not_know(self): - """T1 has no earnings date, so its primary listing is looked up: Yahoo not knowing it crashed the report.""" + """T1 has no earnings date, even in its details, so its primary listing is looked up: Yahoo not knowing it + crashed the report.""" self.patch(cal, 'primary_listings', lambda: {'T1': 'T1.XX'}) undated = lambda t: {k: v for k, v in market_entry(t).items() if not (t == 'T1' and k == 'earnings_date')} self.patch(yahoo, 'fetch', lambda ts: {t: undated(t) for t in ts}) - self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {} if t == 'T1.XX' else {t: market_entry(t)}) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {} if t == 'T1.XX' else {t: undated(t)}) cal.run(event='earnings') self.assertIn('Co T2 ', str(self.sent)) self.assertNotIn('Co T1 ', str(self.sent)) + def test_a_us_listing_of_a_foreign_company_keeps_its_own_dates(self): + """A foreign company's US listing (ARM, CLBT: held on a US market) whose quote lacks the date takes it from its + own details, and is never swapped for its home market's listing (ARM.L, CLBT.TA), even when it has no date at + all: then it has none.""" + undated = lambda t: {k: v for k, v in market_entry(t).items() if not (t == 'T1' and k == 'earnings_date')} + self.patch(yahoo, 'fetch', lambda ts: {t: undated(t) for t in ts}) + for own_date, listed in ((True, True), (False, False)): + with self.subTest(own_date=own_date): + asked = [] + def details(t, *a, **k): + """T1's details, of a British company, with its date or without; asked for, noted.""" + asked.append(t) + return {t: {**(market_entry(t) if own_date else undated(t)), 'profile_country': 'United Kingdom'}} + self.patch(yahoo, 'fetch_detail', details) + self.sent.clear() + cal.run(event='earnings') + self.assertEqual('Co T1 ' in str(self.sent), listed) + self.assertEqual([t for t in asked if '.' in t], []) # no T1.L + def test_trades(self): """.trades run from a chat posts.""" - trades.run(transports.TelegramBot('5')) + trades.run(telegram_common.TelegramBot('5')) self.assertTrue(self.sent) def test_a_trade_is_in_its_market_currency_not_the_brokerage_one(self): @@ -380,13 +448,33 @@ def test_trades_asks_yahoo_only_for_new_trades(self): def test_a_scheduled_run_keeps_the_trades_another_run_recorded_meanwhile(self): """The seen trades are read again as the run ends, so a trade another run recorded meanwhile is kept.""" - meanwhile = lambda tickers: state.save('finbot_sharesight_trades.json', - [999]) or {t: market_entry(t) for t in tickers} + meanwhile = lambda tickers: state.save(trades.STATE_FILE, + {'chats': {'telegram:1': [999]}}) or {t: market_entry(t) for t in tickers} self.patch(yahoo, 'fetch', meanwhile) # called after the seen trades were read, before the new ones are recorded trades.run() - known = state.load('finbot_sharesight_trades.json') + known = state.load_for('telegram:1', trades.STATE_FILE, []) self.assertEqual((known[0], len(known)), (999, 2)) + def test_a_trade_or_a_rating_change_reaches_each_chat_that_schedules_the_report(self): + """Each chat keeps its own record of what a report told it, so a trade or a rating change reaches every chat + whose job runs the report, each once: the first chat's run used to mark it told for all.""" + told = [] + self.patch(transports.Transport, 'text', lambda dest, lines, markup=None: + told.append((dest.key, ' '.join(lines))) or []) + chats = [delivery.chat(key, scheduled=True) for key in ('telegram:5', 'slack:C9')] + self.env(slackBotToken='xoxb-test') + for chat in chats * 2: + trades.run(chat) + self.assertEqual([key for key, text in told if 'T1' in text], ['telegram:5', 'slack:C9']) + for chat in chats: + rating.run(chat) # the first sighting, recorded + told.clear() + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | ({'recommend': 'sell', + 'recommend_index': 4.5} if t == 'T1' else {})}) + for chat in chats * 2: + rating.run(chat) + self.assertEqual([key for key, text in told if 'Co T1' in text], ['telegram:5', 'slack:C9']) + def test_milestone(self): """The milestone report posts.""" milestone.run() @@ -398,21 +486,23 @@ def test_milestone_records_a_ticker_only_with_its_new_values(self): self.patch(yahoo, 'fetch', lambda ts: {t: {k: v for k, v in market_entry(t).items() if not k.startswith('fiftyTwoWeek')} for t in ts}) milestone.run() - self.assertFalse(state.load('finbot_milestone.json')) + self.assertFalse(state.load_for('telegram:1', milestone.STATE_FILE, {})) self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts}) self.sent.clear() milestone.run() self.assertNotIn('Record high', str(self.sent)) - self.assertEqual(state.load('finbot_milestone.json')['T1'], {'all_time_high': 150.0, 'all_time_low': 50.0}) + self.assertEqual(state.load_for('telegram:1', milestone.STATE_FILE, {})['T1'], + {'all_time_high': 150.0, 'all_time_low': 50.0}) def test_a_record_kept_from_ten_years_is_seeded_again_from_the_whole_history(self): """Records kept before they came from the whole history (record_high, from 10 years) are seeded again: a stock above its 10-year high but below its all-time one posted a false record.""" - state.save('finbot_milestone.json', {'T1': {'record_high': 110.0, 'record_low': 90.0}}) + state.save(milestone.STATE_FILE, {'chats': {'telegram:1': {'T1': {'record_high': 110.0, 'record_low': 90.0}}}}) self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) milestone.run() # the 52-week range (120/80) was past the old record, but not the whole history's self.assertNotIn('Co T1 ', str(self.sent)) - self.assertEqual(state.load('finbot_milestone.json')['T1'], {'all_time_high': 150.0, 'all_time_low': 50.0}) + self.assertEqual(state.load_for('telegram:1', milestone.STATE_FILE, {})['T1'], + {'all_time_high': 150.0, 'all_time_low': 50.0}) def test_milestone_records_are_all_time(self): """A new 52-week high below the all-time high is no record. The record used to become the 52-week high after the @@ -428,14 +518,15 @@ def test_milestone_records_are_all_time(self): def test_a_52_week_record_kept_before_is_replaced_by_the_history(self): """A record kept in the old 52-week format is replaced by the history: no false record high.""" - state.save('finbot_milestone.json', {f'T{i}': {'high': 110.0, 'low': 90.0} for i in range(25)}) + state.save(milestone.STATE_FILE, {'chats': {'telegram:1': {f'T{i}': {'high': 110.0, 'low': 90.0} + for i in range(25)}}}) self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) milestone.run() # 120 beats the old 52-week record of 110, but not the history's 150 self.assertNotIn('Record high', str(self.sent)) def test_rating(self): """.rating run from a chat posts.""" - rating.run(transports.TelegramBot('5')) + rating.run(telegram_common.TelegramBot('5')) self.assertTrue(self.sent) def test_rating_keeps_the_last_rating_of_a_ticker_missing_from_a_run(self): @@ -452,35 +543,37 @@ def test_rating_keeps_the_last_rating_of_a_ticker_missing_from_a_run(self): def test_an_interactive_rating_records_nothing(self): """A rating report asked for in a chat leaves the record for the scheduled runs alone.""" - rating.run(transports.TelegramBot('5')) - self.assertIsNone(state.load('finbot_rating.json')) - - def test_slack_webhook_without_a_bot_token_gets_scheduled_reports(self): - """Scheduled reports reach Slack through its incoming webhook, which takes no channel and no token. They used to - send Telegram's chat id along, which made write() need slackBotToken and crash without one, before Telegram got - its copy.""" - self.env(slackBotToken=False) + rating.run(telegram_common.TelegramBot('5')) + self.assertIsNone(state.load(rating.STATE_FILE)) - self.env(webhooks={**config.env.webhooks, 'slack': 'https://hooks.slack.com/services/T/B/x'}) + def test_scheduled_reports_reach_each_home_chat_with_its_own_token(self): + """Scheduled reports reach Telegram's and Slack's home chats, each with its own bot's credentials: Slack's + request carries the Slack token and channel, and Telegram's chat id goes only to Telegram.""" + self.env(slackBotToken='xoxb-test', slackChannelID='C9') self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) + self.patch(render, 'draw', lambda chart: io.BytesIO(b'png')) posts = [] def post(url, **kwargs): - """Record (host, whether a bot token was sent, the channel) and answer ok.""" - posts.append((url.split('/')[2], 'Authorization' in kwargs.get('headers', {}), - (kwargs.get('json') or {}).get('channel'))) - return mock.Mock(status_code=200, json=lambda: {'ok': True, 'result': {'message_id': 1, 'chat': {'id': 1}}}) - self.patch(transports.net, 'post', post) + """Record (host, the Slack token sent, the channel or chat) and answer ok.""" + data = kwargs.get('data') or kwargs.get('json') or {} + posts.append((url.split('/')[2], kwargs.get('headers', {}).get('Authorization'), + data.get('channel') or data.get('channel_id') or data.get('chat_id'))) + return mock.Mock(status_code=200, text='OK', json=lambda: {'ok': True, 'upload_url': 'https://files.x/u', + 'file_id': 'F1', 'ts': '1.5', 'result': {'message_id': 1, 'chat': {'id': 1}}}) + self.patch(net, 'post', post) for report in (price.run, shorts.run, milestone.run): with self.subTest(report=report): posts.clear() - report() - self.assertIn(('hooks.slack.com', False, None), posts) - self.assertTrue(any(host == 'api.telegram.org' for host, _, _ in posts)) + with contextlib.redirect_stderr(io.StringIO()): + report() + self.assertIn(('slack.com', 'Bearer xoxb-test', 'C9'), posts) + self.assertTrue(any(host == 'api.telegram.org' and chat == '1' for host, _, chat in posts)) + self.assertFalse(any(host == 'slack.com' and chat == '1' for host, _, chat in posts)) def test_nowhere_to_post_is_an_error(self): """A scheduled report with nowhere to post is an error that says so.""" - self.env(webhooks={}) + self.env(telegramChatID='') with self.assertRaisesRegex(RuntimeError, 'nowhere to post'): price.run() @@ -533,25 +626,25 @@ def test_a_long_list_is_stored_and_only_the_newest_are_kept(self): with self.assertRaises(KeyError): buttons.resolve(refs[0]) - def test_no_buttons_where_their_presses_would_go_nowhere(self): - """Without telegramOutgoingWebhook (reports only), Telegram hears nothing from finbot's buttons, so a chart, a - scheduled one included, comes without them.""" - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}, telegramChatID='1') + def test_telegram_charts_have_buttons_in_both_modes(self): + """A Telegram chart, a scheduled one included, has its period buttons whether Telegram posts their presses to + telegramOutgoingWebhook (http mode) or finbot fetches them (pull mode).""" + self.env(telegramChatID='1') label, kwargs = price.job(['7']) # as scheduled: price.py 7 - for hook, buttons in (('https://example.com/telegram', True), ('', False)): + for hook in ('https://example.com/telegram', ''): with self.subTest(telegramOutgoingWebhook=hook): self.env(telegramOutgoingWebhook=hook) self.markups.clear() price.run(**kwargs) - self.assertEqual(bool(self.markups and self.markups[0]), buttons) + self.assertTrue(self.markups and self.markups[0]) def press(self, data): """Press a period button: (the edits made, the toasts shown).""" edits, toasts = [], [] - self.patch(transports.TelegramBot, 'edit_chart', lambda dest, msg, img, caption, buttons=None: - edits.append((caption.split('\n')[0], transports.TelegramBot.markup(buttons)))) - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) - worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', data)) + self.patch(telegram_common.TelegramBot, 'edit_chart', lambda dest, msg, img, caption, buttons=None: + edits.append((caption.split('\n')[0], telegram_common.TelegramBot.markup(buttons)))) + self.patch(telegram_common, 'answerCallbackQuery', lambda cid, text=None: toasts.append(text)) + self.press_button(data) return edits, toasts def test_pressing_1d_rebuilds_todays_list_and_moves_the_marker(self): @@ -686,11 +779,12 @@ def test_unknown_or_missing_portfolio_lists_known(self): self.assertEqual(self.command('.pie'), [('text', ['Usage: .pie PORTFOLIO [sector]', 'Portfolios: Rob'])]) self.assertEqual(self.command('.pie Nope'), [('text', ['Nope portfolio not found. I only know about:', 'Rob'])]) - def test_a_service_that_takes_no_images_gets_text(self): - """Slack without the bot token takes no images: the report goes as text.""" - self.env(slackBotToken=False) - worker.Request(SLACK, 'C1', '@u', 'U', BOT).reply(['x'], render.Chart('pie', 't', rows=(('a', 1.0),))) - self.assertEqual(self.sent[-1][0], 'text') + def test_with_the_graph_preference_off_a_chart_goes_as_text(self): + """With graph off, a report's chart is left out and its lines go as text, on every network.""" + self.prefs(graph=False) + for service in (TELEGRAM, SLACK, DISCORD): + worker.Request(service, 'C1', '@u', 'U', BOT).reply(['x'], render.Chart('pie', 't', rows=(('a', 1.0),))) + self.assertEqual(self.sent[-1], ('text', ['x'])) if __name__ == '__main__': diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 884bdc33..4e67246c 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -1,191 +1,246 @@ -"""The schedule file (lib/jobs.py): its lines parsed, each script's job(argv), and a schedule the bot refuses.""" -import hashlib, io, os, subprocess, sys, tempfile, unittest +"""Jobs as .jobs add takes them (lib/jobs.py): their lines, and each report's job(argv); and the one-time migration +(lib/migrate.py) of the old schedule.cron, and of the reports' records, to the home chats.""" +import contextlib, hashlib, io, os, unittest from pathlib import Path from unittest import mock from tests.common import * -from lib import jobs -from lib.reports import cal, cleanup, performance, podcasts, price - - -class ParseTests(unittest.TestCase): - def test_shipped_schedule_has_no_errors(self): - """defaults/schedule.cron parses without errors, its Saturday line into its five commands; the warm jobs run - within their caches' lifetimes: Sharesight's every 30 minutes, Yahoo's every 8 hours.""" - parsed, errors = jobs.parse_schedule(DEFAULT_SCHEDULE.read_text(), UTC) - self.assertEqual(errors, []) - self.assertEqual(len(parsed), 11) - lines = [(j.expr, [c[0] for c in j.commands]) for j in parsed] - self.assertIn(('14,44 * * * *', ['sharesight_warm.py']), lines) - self.assertIn(('07 */8 * * *', ['yahoo_warm.py']), lines) - weekly = next(j for j in parsed if j.expr == '28 21 * * Sat') - self.assertEqual([label for label, _, _ in weekly.commands], - ['cal.py earnings', 'cal.py ex-dividend', 'price.py 7', 'performance.py 7', 'rating.py']) - - def test_pasted_crontab_lines(self): - """Lines pasted from a crontab work: variables, paths, redirects, braces, comments and @daily.""" - text = ('MAILTO=me\n' - '*/20 * * * Mon-Fri ~/finbot/trades.py > ~/finbot.trades 2>&1\n' - '28 21 * * Sat { ./cal.py earnings; ./price.py 7 ;} &> /dev/null # weekly (UTC)\n' - '@daily /opt/finbot/reminder.py\n') - parsed, errors = jobs.parse_schedule(text, UTC) - self.assertEqual(errors, []) - self.assertEqual([[label for label, _, _ in j.commands] for j in parsed], - [['trades.py'], ['cal.py earnings', 'price.py 7'], ['reminder.py']]) +from lib import jobs, migrate, state, watchlist, worker +from lib.reports import cal, cleanup, milestone, performance, podcasts, price, rating, trades + +LOCAL_ZONE = migrate.local_zone # the real one: the migration tests put UTC in its place + + +class JobLines(unittest.TestCase): + def test_a_job_is_a_time_then_its_reports(self): + """A job is five time fields, or an @macro, then one or more reports with their arguments, separated by ";"; + it is on, in UTC, and its reports are kept as written.""" + job = jobs.parse_job('28 21 * * Sat cal earnings; cal ex-dividend;price 7 ; performance 7; rating') + self.assertEqual((job.when, job.zone.key, job.on, job.chat), ('28 21 * * Sat', 'UTC', True, '')) + self.assertEqual([label for label, _, _ in job.commands], + ['cal earnings', 'cal ex-dividend', 'price 7', 'performance 7', 'rating']) + self.assertEqual(job.text, 'cal earnings; cal ex-dividend;price 7 ; performance 7; rating') + self.assertEqual(job.line, '28 21 * * Sat ' + job.text) + self.assertEqual(jobs.parse_job('@daily reminder', 'telegram:1').when, '@daily') + + def test_a_job_may_name_its_zone_first(self): + """A job may name its zone, as crontab writes it, to follow that market's clocks; its line keeps it.""" + job = jobs.parse_job('CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession') + self.assertEqual((job.zone.key, job.when, job.text), + ('America/New_York', '15 13 * * Mon-Fri', 'price midsession')) + self.assertEqual(job.line, 'CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession') + for zone in ('Nowhere/Land', '', '../etc/passwd'): + with self.subTest(zone=zone), self.assertRaisesRegex(jobs.ScheduleError, 'no time zone'): + jobs.parse_job(f'CRON_TZ={zone} 0 1 * * * trades') def test_quoted_arguments_may_hold_anything(self): - """A quoted argument may hold shell characters; commands after a ; still split.""" + """An argument in quotes may hold anything, a ";" included; commands after a ";" outside quotes still split.""" url = 'https://example.com/feed?a=1&b=2;c>d $x |y' - line = f'01 0-10,23 * * Mon-Fri podcasts.py "{url}" > ~/out 2>&1 # hourly (UTC)\n' - parsed, errors = jobs.parse_schedule(line, UTC) - self.assertEqual(errors, []) - self.assertEqual(parsed[0].commands[0][2], {'feed_url': url}) - parsed, errors = jobs.parse_schedule(f"5 4 * * * podcasts.py '{url}'; shorts.py\n", UTC) - self.assertEqual([(label[:9], kw) for label, _, kw in parsed[0].commands], - [('podcasts.', {'feed_url': url}), ('shorts.py', {})]) - - def test_unquoted_shell_syntax_is_still_refused(self): - """Pipes, &&, a trailing &, $(), backticks, input redirects, subshells and an unterminated quote are refused.""" - for line in ('0 1 * * * trades.py | tee x', '0 1 * * * trades.py && shorts.py & ', - '0 1 * * * trades.py $(date)', '0 1 * * * trades.py `x`', - '0 1 * * * trades.py < in', '0 1 * * * (trades.py)', '0 1 * * * trades.py "unterminated'): - parsed, errors = jobs.parse_schedule(line + '\n', UTC) - self.assertEqual((parsed, len(errors)), ([], 1), line) - - def test_redirects_and_braces_are_dropped(self): - """Output redirects, braces, paths and trailing comments are dropped from a command.""" - for command in ('trades.py > /dev/null', 'trades.py >/dev/null 2>&1', 'trades.py &> /dev/null', - 'trades.py >> ~/x 2>&1', - '{ trades.py ;} > ~/x', '/opt/finbot/trades.py # note (x)'): - parsed, errors = jobs.parse_schedule(f'0 1 * * * {command}\n', UTC) - self.assertEqual(errors, [], command) - self.assertEqual([label for label, _, _ in parsed[0].commands], ['trades.py'], command) - - def test_bin_run_wrapper_is_no_longer_accepted(self): - """The old bin/run wrapper and a cd before the command are refused.""" - for command in ('bin/run trades.py', 'cd /opt/finbot; trades.py'): - parsed, errors = jobs.parse_schedule(f'0 1 * * * {command}\n', UTC) - self.assertEqual((parsed, len(errors)), ([], 1), command) - - def test_bad_lines_are_skipped_and_named(self): - """Bad lines are skipped and reported by line number; the good ones load.""" - text = ('0 1 * * * ls | wc -l\n0 1 * * * price.py bogus\n0 1 * * * nothere.py\n61 1 * * * trades.py\n0 1 * *\n' - 'FOO=bar\n5 4 * * * shorts.py\n') - parsed, errors = jobs.parse_schedule(text, UTC) - self.assertEqual([j.expr for j in parsed], ['5 4 * * *']) - self.assertEqual([number for number, _, _ in errors], [1, 2, 3, 4, 5, 6]) - - def test_impossible_times_are_marked_and_reboot_is_just_unsupported(self): - """Times that never come (30 February, minute 61, month 13, @bogus) are ImpossibleTime; @reboot is only - unsupported, and 29 February does come.""" - for line in ('0 0 30 2 * trades.py', '0 9 31 4,6,9,11 * trades.py', '61 1 * * * trades.py', - '0 0 * 13 * trades.py', '@bogus trades.py'): - parsed, errors = jobs.parse_schedule(line + '\n', UTC) - self.assertEqual(parsed, [], line) - self.assertIsInstance(errors[0][2], jobs.ImpossibleTime, line) - parsed, errors = jobs.parse_schedule('@reboot trades.py\n0 0 29 2 * trades.py\n', UTC) - self.assertEqual([j.expr for j in parsed], ['0 0 29 2 *']) # a leap day does come - self.assertNotIsInstance(errors[0][2], jobs.ImpossibleTime) - - def test_cron_tz_applies_to_the_lines_below(self): - """CRON_TZ sets the zone of the lines below it; an unknown zone is skipped.""" - text = '1 1 * * * trades.py\nCRON_TZ=Australia/Melbourne\n2 2 * * * trades.py\nCRON_TZ=Nowhere/Land\n' - parsed, _ = jobs.parse_schedule(text, UTC) - self.assertEqual([j.zone.key for j in parsed], ['UTC', 'Australia/Melbourne']) - - def test_local_zone_follows_tz(self): - """The local zone follows TZ.""" - with mock.patch.dict(os.environ, {'TZ': 'Australia/Melbourne'}): - self.assertEqual(jobs.local_zone().key, 'Australia/Melbourne') + for quoted in (f'"{url}"', f"'{url}'"): + job = jobs.parse_job(f'5 4 * * * podcasts {quoted}; shorts') + self.assertEqual([(label.split()[0], kwargs) for label, _, kwargs in job.commands], + [('podcasts', {'feed_url': url}), ('shorts', {})]) + with self.assertRaisesRegex(jobs.ScheduleError, 'No closing quotation'): + jobs.parse_job('5 4 * * * podcasts "https://example.com') + + def test_shell_syntax_is_only_words_that_no_report_takes(self): + """Nothing runs a shell: pipes, redirects, $() and comments are mere words, which no report takes.""" + for line in ('0 1 * * * trades | tee x', '0 1 * * * trades > /dev/null', '0 1 * * * trades $(date)', + '0 1 * * * trades && shorts', '0 1 * * * trades # a note'): + with self.subTest(line=line), self.assertRaisesRegex(jobs.ScheduleError, 'trades takes no arguments'): + jobs.parse_job(line) + + def test_a_time_that_cannot_be_or_never_comes(self): + """A time that cannot be (minute 61, month 13, @bogus) or never comes (30 February) is ImpossibleTime, and so is + @reboot; 29 February does come.""" + for when, why in (('0 0 30 2 *', 'never comes'), ('0 9 31 4,6,9,11 *', 'never comes'), + ('61 1 * * *', 'is no time'), ('0 0 * 13 *', 'is no time'), ('@bogus', 'is no time'), + ('@reboot', 'the jobs start with the bot')): + with self.subTest(when=when), self.assertRaisesRegex(jobs.ImpossibleTime, why): + jobs.parse_job(f'{when} trades') + self.assertEqual(jobs.parse_job('0 0 29 2 * trades').when, '0 0 29 2 *') + + def test_what_is_no_report(self): + """A job runs reports by name: not a script's file, not a system job's (except as one), not a report that is + not there; with arguments it takes, and with at least one report.""" + for line, why in (('0 1 * * * nothere', "no report 'nothere'"), + ('0 1 * * * trades.py', "no report 'trades.py'"), + ('0 1 * * * ./trades', "no report './trades'"), ('0 1 * * * yahoo_warm', "no report 'yahoo_warm'"), + ('0 1 * * * cleanup 30', "no report 'cleanup'"), ('0 1 * * * trades; nothere', "no report 'nothere'"), + ('0 1 * * * price soon', 'price takes midsession|interday|premarket|close|DAYS'), + ('0 1 * * * ;', 'no report: send .jobs add'), ('0 1 * * *', 'a job is five time fields'), + ('trades', 'a job is five time fields')): + with self.subTest(line=line), self.assertRaisesRegex(jobs.ScheduleError, why): + jobs.parse_job(line) + self.assertEqual(jobs.parse_job('51 18 * * Sat cleanup 30', system=True).commands[0][2], {'days': 30}) class JobArguments(unittest.TestCase): def test_price(self): - """price.py takes one mode or a number of days (the top and bottom ten over it); anything else is its usage.""" - self.assertEqual(price.job(['midsession']), ('price.py midsession', {'mode': 'midsession'})) + """price takes one mode or a number of days (the top and bottom ten over it); anything else is its usage.""" + self.assertEqual(price.job(['midsession']), ('price midsession', {'mode': 'midsession'})) self.assertEqual(price.job(['premarket'])[1], {'mode': 'premarket'}) self.assertEqual(price.job(['interday'])[1], {'mode': 'interday'}) self.assertEqual(price.job(['close'])[1], {'mode': 'close'}) - self.assertEqual(price.job(['28']), ('price.py 28', {'mode': 'period', 'threshold': 0, 'days': 28, 'top': 10})) + self.assertEqual(price.job(['28']), ('price 28', {'mode': 'period', 'threshold': 0, 'days': 28, 'top': 10})) for bad in ([], ['soon'], ['7', '8']): - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, price.ARGUMENTS.replace('|', r'\|')): price.job(bad) def test_cal_and_performance(self): - """cal.py takes earnings or ex-dividend; performance.py an optional number of days.""" - self.assertEqual(cal.job(['earnings']), ('cal.py earnings', {'event': 'earnings'})) - self.assertEqual(cal.job(['ex-dividend']), ('cal.py ex-dividend', {'event': 'ex-dividend'})) - self.assertEqual(performance.job([]), ('performance.py', {})) - self.assertEqual(performance.job(['7']), ('performance.py 7', {'past_days': 7})) + """cal takes earnings or ex-dividend; performance an optional number of days.""" + self.assertEqual(cal.job(['earnings']), ('cal earnings', {'event': 'earnings'})) + self.assertEqual(cal.job(['ex-dividend']), ('cal ex-dividend', {'event': 'ex-dividend'})) + self.assertEqual(performance.job([]), ('performance', {})) + self.assertEqual(performance.job(['7']), ('performance 7', {'past_days': 7})) for bad, module in ((['x'], cal), ([], cal), (['x'], performance)): with self.assertRaises(ValueError): module.job(bad) def test_podcasts(self): - """podcasts.py takes one feed URL; its label carries the start of the hash that names its state file.""" + """podcasts takes one feed URL; its label carries the start of the hash that names its seen file.""" url = 'https://example.com/podcast.rss' label, kwargs = podcasts.job([url]) self.assertEqual(kwargs, {'feed_url': url}) - # the label is the start of the state file's hash self.assertIn('finbot_podcast_seen_' + label.split()[1], podcasts.get_seen_filename(url)) - self.assertEqual(label, 'podcasts.py ' + hashlib.md5(url.encode()).hexdigest()[:8]) + self.assertEqual(label, 'podcasts ' + hashlib.md5(url.encode()).hexdigest()[:8]) for bad in ([], [url, url]): with self.assertRaises(ValueError): podcasts.job(bad) def test_cleanup_arguments(self): - """cleanup.py takes an optional number of days.""" - self.assertEqual(cleanup.job([]), ('cleanup.py', {})) - self.assertEqual(cleanup.job(['7']), ('cleanup.py 7', {'days': 7})) + """cleanup takes an optional number of days.""" + self.assertEqual(cleanup.job([]), ('cleanup', {})) + self.assertEqual(cleanup.job(['7']), ('cleanup 7', {'days': 7})) for bad in (['x'], ['1', '2']): with self.assertRaises(ValueError): cleanup.job(bad) - def test_plain_scripts(self): - """The other reports take no arguments.""" + def test_plain_reports(self): + """The other reports take no arguments, and say so: their ARGUMENTS, which .jobs add lists, are empty.""" for name, module in jobs.SCRIPTS.items(): - if name not in ('cal', 'cleanup', 'performance', 'podcasts', 'price'): - self.assertEqual(module.job([]), (name + '.py', {})) - - def test_cli_prints_usage_and_exits(self): - """A bad argument on the command line prints the usage and exits 1.""" - with mock.patch('sys.stderr', new_callable=io.StringIO) as err, self.assertRaises(SystemExit) as exit_: - delivery.cli('price.py', price.job, lambda **k: None, ['bogus']) - self.assertEqual(exit_.exception.code, 1) - self.assertIn('Usage: price.py [midsession', err.getvalue()) - - def test_cli_runs_the_handler(self): - """Good arguments run the report with what job() made of them.""" - calls = [] - delivery.cli('price.py', price.job, lambda **k: calls.append(k), ['midsession']) - self.assertEqual(calls, [{'mode': 'midsession'}]) - - def test_each_scheduled_report_runs_from_its_top_level_script(self): - """./price.py and the rest run lib/reports/.py, so cron lines and runs by hand keep working.""" - self.assertEqual(sorted(path.stem for path in Path(ROOT).glob('*.py') if path.stem not in ('bot', - 'bot_send_stdin')), - sorted(jobs.SCRIPTS)) - with tempfile.TemporaryDirectory() as data: - run = lambda *args: subprocess.run([sys.executable, *args], cwd=ROOT, capture_output=True, text=True, - timeout=60, env={**os.environ, 'DATA_DIR': data}) - self.assertEqual(run('cleanup.py', '0').returncode, 0) - bad = run('price.py', 'bogus') - self.assertEqual(bad.returncode, 1) - self.assertIn('Usage: price.py [midsession', bad.stderr) - - -class StartupRefusal(unittest.TestCase): - def test_the_bot_will_not_start_with_an_impossible_time(self): - """bot.py will not start while the schedule holds a time that never comes, and says which.""" - skip_if_old_env(self) - with tempfile.TemporaryDirectory() as data: - Path(data, 'schedule.cron').write_text('*/20 * * * Mon-Fri trades.py\n0 9 31 6 * price.py close\n') - env = bare_environment() - env.update(DATA_DIR=data, port='0') - out = subprocess.run([sys.executable, 'bot.py'], cwd=ROOT, env=env, capture_output=True, text=True, - timeout=120) - self.assertEqual(out.returncode, 1, out.stderr) - self.assertIn(f"ERROR schedule: {data}/schedule.cron line 2: '0 9 31 6 *' never comes", out.stderr) - - -if __name__ == '__main__': - unittest.main() + with self.subTest(name=name): + self.assertIsInstance(module.ARGUMENTS, str) + if name not in ('cal', 'cleanup', 'performance', 'podcasts', 'price'): + self.assertEqual((module.job([]), module.ARGUMENTS), ((name, {}), '')) + with self.assertRaisesRegex(ValueError, 'no arguments'): + module.job(['7']) + + def test_the_checkout_holds_only_the_bot_at_its_top(self): + """The scheduled reports run in the bot (and `.jobs run`), so the checkout's top holds bot.py alone: the scripts + that ran each report from cron are gone.""" + self.assertEqual(sorted(path.name for path in Path(ROOT).glob('*.py')), ['bot.py']) + + +class Migration(FinbotCase): + """As finbot first starts with jobs per chat, the home chats get what was kept for all chats, once.""" + + def setUp(self): + """Two home chats, Telegram's and Slack's; the old schedule ran in UTC.""" + super().setUp() + self.env(slackBotToken='xoxb-1', slackChannelID='C1') + self.patch(migrate, 'local_zone', lambda: jobs.UTC) + + def migrate(self): + """Run the migration, as finbot does as it starts: what it logs.""" + with contextlib.redirect_stderr(io.StringIO()) as log: + migrate.run() + return log.getvalue() + + def test_the_old_schedule_becomes_each_home_chats_jobs_once(self): + """Each home chat gets the old schedule's jobs (the data dir's copy), pasted crontab syntax read as cron read + it, each in the zone it ran in, without the system jobs; a line that cannot be a job is logged and left out. + Then the old file is read no more.""" + url = 'https://example.com/feed?a=1&b=2' + Path(config.env.data_dir, 'schedule.cron').write_text('MAILTO=me\n' + '*/20 * * * Mon-Fri ~/finbot/trades.py > ~/finbot.trades 2>&1\n' + '28 21 * * Sat { ./cal.py earnings; ./price.py 7 ;} &> /dev/null # weekly\n' + '@daily /opt/finbot/reminder.py\n' + 'CRON_TZ=Australia/Melbourne\n' + '14,44 * * * * sharesight_warm.py\n' + f'01 * * * * podcasts.py "{url}"; cleanup.py 30\n' + '0 1 * * * ls | wc -l\n' + '0 9 31 6 * price.py close\n' + '0 2 * * * bin/run trades.py\n') + log = self.migrate() + jobs_each = [{'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}, + {'when': '28 21 * * Sat', 'reports': 'cal earnings; price 7', 'zone': 'UTC', 'on': True}, + {'when': '@daily', 'reports': 'reminder', 'zone': 'UTC', 'on': True}, + {'when': '01 * * * *', 'reports': f"podcasts '{url}'", 'zone': 'Australia/Melbourne', 'on': True}] + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {'telegram:1': jobs_each, 'slack:C1': jobs_each}) + path = config.env.data_dir + '/schedule.cron' + self.assertEqual([line for line in log.splitlines() if 'left out' in line], [ + f'schedule: {path} line 8 left out: only finbot scripts can be scheduled, not shell commands', + f"schedule: {path} line 9 left out: '0 9 31 6 *' never comes: no date matches it", + f"schedule: {path} line 10 left out: no report 'run': send .jobs add for the reports and what each takes"]) + self.assertIn(f"schedule: the 4 jobs of {path} are now each home chat's (telegram:1, slack:C1)", log) + s = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + s.load() + self.assertEqual([job.line for job in s.chat_jobs('slack:C1')][-1], f"CRON_TZ=Australia/Melbourne 01 * * * *" + f" podcasts '{url}'") + Path(config.env.data_dir, 'schedule.cron').write_text('0 3 * * * shorts.py\n') # read no more + self.assertEqual(self.migrate(), '') + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {'telegram:1': jobs_each, 'slack:C1': jobs_each}) + + def test_the_shipped_schedule_migrates_whole(self): + """Without a copy in the data dir, the home chats get the shipped schedule's jobs, every line of it but the + system jobs', the weekly line's five reports among them.""" + self.assertNotIn('left out', self.migrate()) + entries = state.load(jobs.SCHEDULE_FILE)['telegram:1'] + self.assertEqual(len(entries), 8) + self.assertIn({'when': '28 21 * * Sat', 'reports': 'cal earnings; cal ex-dividend; price 7; performance 7;' + ' rating', 'zone': 'UTC', 'on': True}, entries) + self.assertFalse(any(name in entry['reports'] for entry in entries for name in jobs.SYSTEM_SCRIPTS)) + + def test_an_empty_schedule_migrates_to_no_jobs_and_is_done(self): + """A schedule with no jobs gives the home chats none, and the migration is still done: it is not tried again.""" + Path(config.env.data_dir, 'schedule.cron').write_text('# nothing\n14,44 * * * * sharesight_warm.py\n') + self.migrate() + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {}) + Path(config.env.data_dir, 'schedule.cron').write_text('0 3 * * * shorts.py\n') + self.assertEqual(self.migrate(), '') + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {}) + + def test_with_no_home_chat_it_waits(self): + """With no home chat set there is nowhere to put the jobs and records, so the migration waits for one.""" + self.env(telegramChatID='', slackChannelID='') + state.save(trades.STATE_FILE, ['t1']) + self.assertEqual(self.migrate(), '') + self.assertFalse(os.path.exists(state.path(jobs.SCHEDULE_FILE))) + self.assertEqual(state.load(trades.STATE_FILE), ['t1']) + + def test_the_watchlist_plans_and_records_become_each_home_chats_once(self): + """The watchlist and the plans become each home chat's, and so does what a report has told the chats (trades, + ratings, record highs, each feed's episodes), so that nothing old is told again; a file kept per chat already is + left as it is, and so is one that cannot be read (said in the log).""" + feed = podcasts.get_seen_filename('https://example.com/podcast.rss') + old = {watchlist.FILE: ['MSFT', 'BHP.AX'], worker.PLANS: {'@rob': 'buy low'}, trades.STATE_FILE: ['t1', 't2'], + rating.STATE_FILE: {'NVDA': ['upgrade', 3]}, milestone.STATE_FILE: {'NVDA': {'all_time_high': 200}}, + feed: ['guid-1']} + for name, record in old.items(): + state.save(name, record) + log = self.migrate() + for name, record in old.items(): + with self.subTest(name=name): + self.assertEqual(state.load(name), {'chats': {'telegram:1': record, 'slack:C1': record}}) + self.assertIn(f'state: {feed} is kept per chat now, its record copied to telegram:1, slack:C1', log) + state.save(trades.STATE_FILE, {'chats': {'telegram:1': ['t3']}}) + Path(state.path(rating.STATE_FILE)).write_text('{"NVDA": ["upgr') + log = self.migrate() + self.assertEqual(state.load(trades.STATE_FILE), {'chats': {'telegram:1': ['t3']}}) + self.assertEqual(Path(state.path(rating.STATE_FILE)).read_text(), '{"NVDA": ["upgr') + self.assertIn(f'state: {rating.STATE_FILE} left as it is, unreadable', log) + + def test_pasted_redirects_braces_and_paths_are_dropped(self): + """Output redirects, braces, paths, a .py and trailing comments are dropped from an old line's command.""" + for command in ('trades.py > /dev/null', 'trades.py >/dev/null 2>&1', 'trades.py &> /dev/null', + 'trades.py >> ~/x 2>&1', '{ trades.py ;} > ~/x', '/opt/finbot/trades.py # note (x)', 'FOO=1 trades'): + with self.subTest(command=command): + lines = migrate.legacy_lines(f'0 1 * * * {command}\n', 'schedule.cron') + self.assertEqual([commands for _, _, _, commands in lines], [[('trades', [])]]) + + def test_the_local_zone_follows_tz(self): + """The old schedule ran in the server's zone, which TZ sets: UTC by that name when it is UTC by another.""" + for tz, zone in (('Australia/Melbourne', 'Australia/Melbourne'), ('Etc/UTC', 'UTC'), (':UTC', 'UTC')): + with self.subTest(tz=tz), mock.patch.dict(os.environ, {'TZ': tz}): + self.assertEqual(LOCAL_ZONE().key, zone) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9668777b..a9c1cc81 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,23 +1,17 @@ -"""The in-process scheduler (lib/jobs.py): firing on a fake clock, across the clocks' changes, runs from chat, -and each command in a child process.""" -import contextlib, datetime, io, json, os, signal, subprocess, sys, tempfile, threading, time, unittest +"""The scheduler (lib/jobs.py): each chat's jobs and the system jobs firing on a fake clock, across the clocks' changes; +.jobs in a chat, and its buttons; each command run in the bot's process, past its deadline too.""" +import contextlib, datetime, io, os, subprocess, sys, tempfile, time, unittest import zoneinfo from pathlib import Path from unittest import mock -import gevent from tests.common import * from lib import jobs from lib.reports import cleanup, podcasts, price, sharesight_warm, yahoo_warm +from tests import test_sending PREFETCH_HISTORY = yahoo.prefetch_history # FinbotCase replaces it; the warm job's test puts it back -def run_in_process(label, module, kwargs): - """The scheduler's runner in these tests: the command's handler, called here (no child process, no deadline).""" - module.run(**kwargs) - return 0 - - def at(day, hour, minute, second=0): """A UTC time in October 2026 (the 13th is a Tuesday).""" return datetime.datetime(2026, 10, day, hour, minute, second, tzinfo=UTC) @@ -92,16 +86,16 @@ def test_every_followed_stocks_history_is_fetched_afresh(self): self.assertEqual(cache.read('finbot_yahoo_history_T1.json'), {'history of': 'T1'}) self.assertIn(f'yahoo_warm: {len(TICKERS) - 1} of {len(TICKERS)} price histories fetched', log.getvalue()) - def test_without_sharesight_the_watchlist_is_fetched(self): - """While Sharesight cannot be asked for the holdings, the watchlist's histories are fetched; without Sharesight - configured, it is not asked.""" + def test_without_sharesight_the_watchlists_are_fetched(self): + """While Sharesight cannot be asked for the holdings, every chat's watchlist's histories are fetched; without + Sharesight configured, it is not asked.""" asked = [] def unreachable(): - """Sharesight failing, as watchlist.with_holdings meets it.""" + """Sharesight failing, as watchlist.all_followed meets it.""" asked.append(1) raise RuntimeError('Sharesight auth: down') - self.patch(watchlist, 'with_holdings', unreachable) - self.patch(watchlist, 'tickers', lambda: ['W1']) + self.patch(watchlist, 'all_followed', unreachable) + self.patch(watchlist, 'all_watched', lambda: ['W1']) with contextlib.redirect_stderr(io.StringIO()) as log: self.assertEqual(yahoo_warm.run(), 1) self.assertEqual(cache.read('finbot_yahoo_history_W1.json'), {'history of': 'W1'}) @@ -142,43 +136,45 @@ def down(): self.assertEqual(self.errors, []) -class SchedulerFlag(unittest.TestCase): - def flag(self, **env): - """config.env.scheduler in a fresh interpreter, given `env`.""" - code = 'from lib import config; print(config.env.scheduler)' - e = {k: v for k, v in os.environ.items() if k != 'scheduler'} - e.update(env) - repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - return subprocess.run([sys.executable, '-c', code], cwd=repo, env=e, capture_output=True, text=True, - check=True).stdout.split()[-1] +CHAT, OTHER = 'telegram:55', 'slack:C1' # two chats, by their keys - def test_on_by_default_and_off_for_false(self): - """The scheduler is on by default and with any true word, off with any false one.""" - self.assertEqual(self.flag(), 'True') - for value in ('True', 'true', '1', 't'): - self.assertEqual(self.flag(scheduler=value), 'True', value) - for value in ('False', 'false', '0', 'no'): - self.assertEqual(self.flag(scheduler=value), 'False', value) + +def entry(line, on=True): + """A job as SCHEDULE_FILE stores it, from its line as .jobs add takes it.""" + job = jobs.parse_job(line) + return {'when': job.when, 'reports': job.text, 'zone': job.zone.key, 'on': on} class SchedulerBase(FinbotCase): def setUp(self): - """A schedule of trades.py every 20 minutes on weekdays, a fake clock; trades and shorts recorded, not run.""" + """Chat telegram:55 runs trades every 20 minutes on weekdays; a fake clock; no system jobs (they have tests of + their own); the reports noted with the chat each posts to, not run.""" super().setUp() - tmp = tempfile.TemporaryDirectory() - self.addCleanup(tmp.cleanup) - self.file = Path(tmp.name, 'schedule.cron') - self.file.write_text('*/20 * * * Mon-Fri trades.py\n') + self.patch(jobs, 'SYSTEM', ()) + self.store(CHAT, '*/20 * * * Mon-Fri trades') self.clock = Clock(at(13, 21, 3)) - self.ran = [] - self.patch(jobs.SCRIPTS['trades'], 'run', lambda **k: self.ran.append(('trades', k))) - self.patch(jobs.SCRIPTS['shorts'], 'run', lambda **k: self.ran.append(('shorts', k))) + self.ran, self.dests = [], [] + for name in ('trades', 'shorts', 'price', 'rating', 'reminder'): + self.patch(jobs.SCRIPTS[name], 'run', self.recorder(name)) + + def recorder(self, name): + """A report's run() that notes the run, and the chat it would post to (its key; None for the home chats).""" + def run(dest=None, **kwargs): + """Note the run and where it posts.""" + self.ran.append((name, kwargs)) + self.dests.append(dest.key if dest else None) + return run + + def store(self, chat, *lines, on=True): + """Set a chat's jobs, as .jobs add would have stored them.""" + with state.edit(jobs.SCHEDULE_FILE, {}) as chats: + chats[chat] = [entry(line, on) for line in lines] def scheduler(self, catchup=5): """A loaded scheduler on the fake clock that runs commands in this process, at once.""" - s = jobs.Scheduler(lambda: self.file, catchup=catchup, clock=self.clock, sleep=lambda seconds: None, - spawn=lambda f, *a: f(*a), zone=UTC, runner=run_in_process) - self.assertTrue(s.load()) + s = jobs.Scheduler(catchup=catchup, clock=self.clock, sleep=lambda seconds: None, spawn=lambda f, *a: f(*a)) + with contextlib.redirect_stderr(io.StringIO()): + s.load() return s @@ -191,25 +187,43 @@ def test_the_warm_jobs_run_through_the_scheduler(self): s = self.scheduler() with contextlib.redirect_stderr(io.StringIO()) as log: jobs.warm_now(s) - s.busy.add('yahoo_warm.py') # on its schedule + s.busy.add(('', 'yahoo_warm')) # on its schedule jobs.warm_now(s) self.assertEqual(warmed, ['sharesight', 'yahoo', 'sharesight']) - self.assertIn('cache: filling it again: sharesight_warm.py, yahoo_warm.py', log.getvalue()) + self.assertIn('cache: filling it again: sharesight_warm, yahoo_warm', log.getvalue()) - def test_without_a_scheduler_each_runs_in_a_child_process(self): - """With the scheduler off, each warm job runs in a child process of its own, a failure reported.""" + def test_without_a_scheduler_each_runs_here(self): + """Without a scheduler (a test, a tool), each warm job runs here, one after the other, a failure reported.""" started = [] - def child(label, module, kwargs): - """Note the child process started; the second one runs out of time.""" + def run(label, module, kwargs, dest=None): + """Note the run; the first one fails.""" started.append(label) - if label == 'yahoo_warm.py': - raise TimeoutError('still running after 5 minutes') - return 0 - self.patch(jobs, 'run_child', child) + if label == 'sharesight_warm': + raise RuntimeError('Sharesight unreachable') + self.patch(jobs, 'run_in_process', run) with contextlib.redirect_stderr(io.StringIO()): jobs.warm_now() - self.assertEqual(started, ['sharesight_warm.py', 'yahoo_warm.py']) - self.assertTrue(any('still running after 5 minutes' in e for e in self.errors)) + self.assertEqual(started, ['sharesight_warm', 'yahoo_warm']) + self.assertEqual(self.errors, ['⚠️ sharesight_warm: RuntimeError: Sharesight unreachable']) + + +class SystemJobs(FinbotCase): + def test_the_system_jobs_are_fixed_and_no_chats(self): + """The system jobs run on a schedule of their own, the same for every install: each warm job within its + cache's lifetime (Sharesight's every 30 minutes, cached 40; Yahoo's every 8 hours, cached 12), and cleanup + weekly. A chat can neither add one nor see it among its own jobs.""" + s = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + s.load() + system = {job.text: job.when for job in s.system_jobs()} + self.assertEqual(system, {'sharesight_warm': '14,44 * * * *', 'yahoo_warm': '7 */8 * * *', + 'cleanup 30': '51 18 * * Sat'}) + self.assertLess(30 * 60, config.prefs.performance_cache_seconds) + self.assertLess(8 * 3600, config.prefs.cache_seconds) + self.assertEqual(s.chat_jobs(''), s.system_jobs()) + for name in jobs.SYSTEM_SCRIPTS: + with self.subTest(name=name), self.assertRaisesRegex(jobs.ScheduleError, f"no report '{name}'"): + s.add(CHAT, f'0 1 * * * {name}') class SchedulerTests(SchedulerBase): @@ -228,20 +242,49 @@ def test_fires_on_the_minute_on_weekdays_only(self): self.clock.now = at(17, 21, 20) # a Saturday self.assertEqual(s.due(self.clock.now), []) - def test_tick_runs_the_job_and_returns_the_wait(self): - """tick() runs what is due and returns how long to sleep: up to 30 seconds, less when the next run is nearer.""" + def test_tick_runs_the_job_for_its_chat_and_returns_the_wait(self): + """tick() runs what is due, posting to the job's chat as a scheduled run, and returns how long to sleep: up to + 30 seconds, less when the next run is nearer.""" + dests = [] + self.patch(jobs.SCRIPTS['trades'], 'run', lambda dest=None, **k: dests.append(dest)) s = self.scheduler() self.clock.now = at(13, 21, 20) - wait = s.tick() - self.assertEqual(self.ran, [('trades', {})]) - self.assertEqual(wait, 30) # the next run is 20 minutes away: wake at most every 30 seconds + self.assertEqual(s.tick(), 30) # the next run is 20 minutes away: wake at most every 30 seconds + self.assertEqual([(dest.key, dest.scheduled) for dest in dests], [(CHAT, True)]) self.clock.now = at(13, 21, 39, 50) self.assertEqual(s.tick(), 10) + def test_each_chats_jobs_post_to_that_chat(self): + """Two chats' jobs at the same time each run, each posting to its own chat.""" + self.store(OTHER, '*/20 * * * Mon-Fri trades', '0 22 * * * shorts') + s = self.scheduler() + self.clock.now = at(13, 21, 20) + s.tick() + self.assertEqual(set(zip(self.dests, [name for name, _ in self.ran])), {(CHAT, 'trades'), (OTHER, 'trades')}) + self.ran.clear() + self.dests.clear() + self.clock.now = at(13, 22, 0) + s.tick() + self.assertEqual(set(zip(self.dests, [name for name, _ in self.ran])), {(CHAT, 'trades'), (OTHER, 'trades'), + (OTHER, 'shorts')}) + + def test_a_job_that_is_off_never_fires_nor_catches_up_once_on(self): + """A job that is off does not fire; switched on, it fires at its next time, not for the runs it missed.""" + self.store(CHAT, '*/20 * * * Mon-Fri trades', on=False) + s = self.scheduler() + self.clock.now = at(13, 21, 20) + self.assertEqual(s.due(self.clock.now), []) + self.clock.now = at(13, 21, 30) + s.toggle(s.chat_jobs(CHAT)[0]) + self.assertTrue(s.chat_jobs(CHAT)[0].on) + self.assertEqual(s.due(self.clock.now), []) # 21:20 is not caught up + self.clock.now = at(13, 21, 40) + self.assertEqual([t for _, t in s.due(self.clock.now)], [at(13, 21, 40)]) + def test_a_run_missed_within_the_catch_up_window_fires_once(self): """After a restart, a run missed within the catch-up window fires once; an older one is skipped.""" first = self.scheduler() - first.last = {first.jobs[0].key: at(13, 21, 0).isoformat()} + first.last = {first.chat_jobs(CHAT)[0].key: at(13, 21, 0).isoformat()} state.save(jobs.STATE_FILE, first.last) self.clock.now = at(13, 21, 43) s = self.scheduler() @@ -250,16 +293,13 @@ def test_a_run_missed_within_the_catch_up_window_fires_once(self): def test_a_corrupt_state_file_is_reported_and_the_schedule_starts_afresh(self): """A state file cut off mid-write is reported once, and the schedule runs as usual and writes it afresh.""" - # cut off mid-write - Path(config.env.data_dir, jobs.STATE_FILE).write_text('{"*/20 * * * Mon-Fri|UTC|trades.py": "2026-10-13T21:') + Path(config.env.data_dir, jobs.STATE_FILE).write_text('{"telegram:55|*/20 * * * Mon-Fri|UTC|trades": "2026-1') s = self.scheduler() self.assertEqual((s.last, len(self.errors)), ({}, 1)) self.assertIn('cannot read finbot_jobs.json', self.errors[0]) self.clock.now = at(13, 21, 20) self.assertEqual(len(s.due(self.clock.now)), 1) # runs as usual, and the record is written afresh - self.assertEqual(state.load(jobs.STATE_FILE), {s.jobs[0].key: at(13, 21, 20).isoformat()}) - s.reload() # and a reload is no trouble either - self.assertEqual(len(self.errors), 1) + self.assertEqual(state.load(jobs.STATE_FILE), {s.chat_jobs(CHAT)[0].key: at(13, 21, 20).isoformat()}) def test_no_record_means_nothing_is_missed(self): """With no record of past runs, nothing counts as missed.""" @@ -276,78 +316,73 @@ def test_a_restart_just_after_a_run_does_not_repeat_it(self): again = self.scheduler() self.assertEqual(again.due(self.clock.now), []) - def test_a_failing_job_is_reported_and_the_next_command_still_runs(self): - """A command that fails is reported, and the next command on its line still runs.""" - self.file.write_text('*/20 * * * * trades.py; shorts.py\n') - def boom(**k): + def test_a_failing_job_is_reported_to_its_chat_and_the_next_command_still_runs(self): + """A command that fails is reported to the chat whose job it is, and the next command of the job still runs.""" + self.store(CHAT, '*/20 * * * * trades; shorts') + def boom(dest=None, **k): """Fail.""" raise RuntimeError('boom') self.patch(jobs.SCRIPTS['trades'], 'run', boom) + reported = [] + self.patch(delivery, 'report_error', lambda e, dest=None, context=None: reported.append((context, dest.key))) s = self.scheduler() - s.execute(s.jobs[0].commands) + self.assertEqual(s.execute(s.chat_jobs(CHAT)[0].commands, CHAT), 'failed') + self.assertEqual(reported, [('trades', CHAT)]) self.assertEqual(self.ran, [('shorts', {})]) - self.assertTrue(any('boom' in e for e in self.errors)) def test_system_exit_inside_a_job_is_reported_not_raised(self): - """A SystemExit inside a job is reported, not raised, and the job is marked done.""" - def quit_(**k): - """Exit, as a script calling sys.exit would.""" - raise SystemExit(3) + """A report calling sys.exit is reported like any failure and does not stop the scheduler.""" + def quit_(dest=None, **k): + """Exit, as a report may.""" + raise SystemExit(1) self.patch(jobs.SCRIPTS['trades'], 'run', quit_) s = self.scheduler() - s.execute(s.jobs[0].commands) + s.run(s.chat_jobs(CHAT)[0]) self.assertEqual(len(self.errors), 1) - def test_a_command_stopped_at_its_deadline_is_reported_and_the_next_still_runs(self): - """A command still running at its deadline (run_child stops it) is reported, and the rest of its line runs.""" - self.file.write_text('*/20 * * * * trades.py; shorts.py\n') - def runner(label, module, kwargs): - """trades.py hangs until run_child stops it; the rest run here.""" - if label == 'trades.py': - raise TimeoutError('still running after 5 minutes') - return run_in_process(label, module, kwargs) + def test_a_command_past_its_deadline_is_reported_once_and_left_to_finish(self): + """A command still running at its deadline is reported as late, once, and runs on to its end (a thread cannot + be stopped); meanwhile it stays marked running, so a second run of it is turned away; then the rest of its job + runs.""" + self.patch(jobs, 'JOB_TIMEOUT', 0.1) + self.store(CHAT, '*/20 * * * * trades; shorts') s = self.scheduler() - s.runner = runner - self.assertEqual(s.execute(s.jobs[0].commands), 'timed out') - self.assertEqual(self.errors, ['⚠️ trades.py: TimeoutError: still running after 5 minutes']) - self.assertEqual(self.ran, [('shorts', {})]) + second = [] + def slow(dest=None, **k): + """Take three deadlines, and try a second run of this command meanwhile.""" + time.sleep(0.3) + second.append(s.execute(s.chat_jobs(CHAT)[0].commands[:1], CHAT)) + self.ran.append(('trades', k)) + self.patch(jobs.SCRIPTS['trades'], 'run', slow) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(s.run(s.chat_jobs(CHAT)[0]), 'done') + self.assertEqual(self.errors, ['⚠️ trades: TimeoutError: still running after 0.00166667 minutes, and left to' + ' finish']) + self.assertEqual(second, ['already running']) + self.assertEqual(self.ran, [('trades', {}), ('shorts', {})]) self.assertEqual(s.busy, set()) - def test_reload_applies_edits_keeps_state_and_survives_a_bad_file(self): - """A reload applies edits and keeps unchanged jobs' timing; an unusable file keeps the running schedule.""" - s = self.scheduler() - old_key = s.jobs[0].key - base = s.base[old_key] - self.file.write_text('*/20 * * * Mon-Fri trades.py\n5 4 * * * shorts.py\n') - self.assertTrue(s.load()) - self.assertEqual(len(s.jobs), 2) - self.assertEqual(s.base[old_key], base) - self.file.write_text('# nothing usable\nnot a job\n') - self.assertFalse(s.load()) - self.assertEqual(len(s.jobs), 2) - - def test_an_impossible_time_refuses_the_whole_file(self): - """A time that never comes refuses the whole file: a reload says why and the running schedule carries on.""" + def test_each_command_of_a_job_has_a_deadline_of_its_own(self): + """The deadline is per command (it once covered the whole line): two that each take most of it are not late.""" + self.patch(jobs, 'JOB_TIMEOUT', 0.4) + self.store(CHAT, '*/20 * * * * trades; shorts') + self.patch(jobs.SCRIPTS['trades'], 'run', lambda dest=None, **k: time.sleep(0.25)) + self.patch(jobs.SCRIPTS['shorts'], 'run', lambda dest=None, **k: time.sleep(0.25)) s = self.scheduler() - self.file.write_text('*/20 * * * Mon-Fri trades.py\n0 9 30 2 * shorts.py\n5 4 * * * nothere.py\n') - with self.assertRaises(jobs.ImpossibleTime) as caught: - s.load() - self.assertEqual(len(caught.exception.lines), 1) - self.assertIn("line 2: '0 9 30 2 *' never comes", caught.exception.lines[0]) - self.assertEqual(self.errors, []) # load reports nothing for a refused file... - s.reload() # ...a reload says why it was refused, and the running schedule carries on - self.assertEqual(len(self.errors), 1) - self.assertIn('not reloaded, the running schedule is unchanged', self.errors[0]) - self.assertEqual([j.text for j in s.jobs], ['*/20 * * * Mon-Fri trades.py']) - self.clock.now = at(13, 21, 20) - self.assertEqual(len(s.due(self.clock.now)), 1) + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(s.run(s.chat_jobs(CHAT)[0]), 'done') + time.sleep(0.3) # past where a deadline of the whole job would have struck + self.assertEqual(self.errors, []) - def test_load_reports_bad_lines_once_and_runs_the_rest(self): - """A bad line is reported once and the rest of the schedule runs.""" - self.file.write_text('0 1 * * * nothere.py\n*/20 * * * * trades.py\n') + def test_a_stored_job_that_cannot_be_is_reported_and_the_rest_run(self): + """A stored job that cannot be any more (a report since retired, say) is reported and left out; the chat's other + jobs load.""" + with state.edit(jobs.SCHEDULE_FILE, {}) as chats: + chats[CHAT].append({'when': '0 1 * * *', 'reports': 'nothere', 'zone': 'UTC', 'on': True}) s = self.scheduler() - self.assertEqual(len(s.jobs), 1) + self.assertEqual([job.text for job in s.chat_jobs(CHAT)], ['trades']) self.assertEqual(len(self.errors), 1) + self.assertIn("a job of telegram:55 left out: no report 'nothere'", self.errors[0]) def test_run_forever_survives_an_error(self): """run_forever reports a failed tick and carries on.""" @@ -366,15 +401,15 @@ def sleep(seconds): self.assertTrue(any('tick failed' in e for e in self.errors)) -class DaylightSaving(unittest.TestCase): +class DaylightSaving(FinbotCase): """Melbourne's clocks go back an hour at 03:00 on 5 April 2026 and forward at 02:00 on 4 October 2026. The fire times must be right on any croniter: the distro packages (up to 6.0) got zoneinfo's changes wrong.""" MEL = zoneinfo.ZoneInfo('Australia/Melbourne') def fires(self, expr, start, end): - """When a job fires between start and end, as Melbourne times ('04 09:00 AEDT').""" - job = jobs.parse_schedule(f'{expr} trades.py\n', self.MEL)[0][0] - s = jobs.Scheduler(lambda: None, zone=self.MEL) + """When a Melbourne job fires between start and end, as Melbourne times ('04 09:00 AEDT').""" + job = jobs.parse_job(f'CRON_TZ=Australia/Melbourne {expr} trades') + s = jobs.Scheduler() times, t = [], start while (t := s.next_fire(job, t)) <= end: # by way of UTC: the real local time @@ -412,396 +447,226 @@ def test_starting_in_the_repeated_hour_runs_nothing_twice(self): self.assertEqual(self.fires('30 2 * * *', second_time_round, self.mel(2026, 4, 6, 12)), ['06 02:30 AEST']) def test_the_scheduler_runs_a_daily_job_once_across_the_change(self): - """The scheduler runs a daily Melbourne job once a day across the change to daylight time.""" - file = Path(temp_dir(self), 'schedule.cron') - file.write_text('CRON_TZ=Australia/Melbourne\n0 9 * * * trades.py\n') + """The scheduler runs a chat's daily Melbourne job once a day across the change to daylight time.""" + self.patch(jobs, 'SYSTEM', ()) + with state.edit(jobs.SCHEDULE_FILE, {}) as chats: + chats[CHAT] = [entry('CRON_TZ=Australia/Melbourne 0 9 * * * trades')] clock = Clock(self.mel(2026, 10, 3, 12).astimezone(UTC)) - s = jobs.Scheduler(lambda: file, clock=clock, zone=UTC, runner=lambda *a: 0) - elsewhere = dataclasses.replace(config.env, data_dir=temp_dir(self)) - with mock.patch.object(config, 'env', elsewhere), contextlib.redirect_stderr(io.StringIO()): - self.assertTrue(s.load()) - fired = [] - while clock.now < self.mel(2026, 10, 5, 12): - fired += [t.astimezone(UTC).astimezone(self.MEL).strftime('%d %H:%M') for _, t in s.due(clock.now)] - clock.now += datetime.timedelta(minutes=10) + s = jobs.Scheduler(clock=clock, runner=lambda *a: None) + with contextlib.redirect_stderr(io.StringIO()): + s.load() + fired = [] + while clock.now < self.mel(2026, 10, 5, 12): + fired += [t.astimezone(UTC).astimezone(self.MEL).strftime('%d %H:%M') for _, t in s.due(clock.now)] + clock.now += datetime.timedelta(minutes=10) self.assertEqual(fired, ['04 09:00', '05 09:00']) -class ManualRuns(SchedulerBase): +class JobsInChat(SchedulerBase): + """What a chat does with its jobs: .jobs lists them; run, toggle, add and del; their buttons.""" + def setUp(self): - """A schedule with repeats, comments, braces and the commands that cannot be picked (cleanup, podcasts).""" + """Chat telegram:55 with its one job, the scheduler running, Telegram's toasts and edits noted.""" super().setUp() - self.file.write_text('# a comment\n*/20 * * * Mon-Fri trades.py > /dev/null\n' - '28 21 * * Sat price.py close; trades.py # weekly\n5 4 * * * { price.py close ;}\n6 4 * * * price.py 7\n' - '0 3 * * * performance.py 365; performance.py 7; performance.py 90; performance.py 30; performance.py 180\n' - '0 2 * * * cleanup.py\n1 * * * * podcasts.py "https://example.com/feed"\n') self.s = self.scheduler() self.patch(jobs, 'current', self.s) - self.patch(jobs.SCRIPTS['price'], 'run', lambda **k: self.ran.append(('price', k))) - - def test_commands_are_unique_clean_and_in_order(self): - """The commands offered are each once, without redirects, sorted by name with numbers in numeric order.""" - self.assertEqual([label for label, _, _ in self.s.commands()], - ['performance.py 7', 'performance.py 30', 'performance.py 90', 'performance.py 180', - 'performance.py 365', 'price.py 7', - 'price.py close', 'trades.py']) - - def test_select_by_number_or_command_text(self): - """A command is picked by its number or its text, quotes, paths and redirects aside.""" - self.assertEqual(self.s.select('7')[0], 'price.py close') - self.assertEqual(self.s.select('8')[0], 'trades.py') - self.assertEqual(self.s.select(' "price.py" close > /dev/null')[0], 'price.py close') - self.assertEqual(self.s.select('~/finbot/price.py 7')[0], 'price.py 7') - - def test_select_refuses_what_is_not_offered(self): - """Numbers out of range, cleanup, podcasts, unscheduled commands, shell syntax and two commands are refused.""" - for text in ('0', '9', 'cleanup.py', 'podcasts.py "https://example.com/feed"', 'price.py 99', 'cal.py earnings', - 'nonsense', 'ls | wc', '', 'price.py close; trades.py'): - with self.assertRaises(jobs.ScheduleError, msg=text): - self.s.select(text) - with self.assertRaisesRegex(jobs.ScheduleError, '^cleanup is a system job: it runs on its schedule only$'): - self.s.select('cleanup.py') - - def test_run_now_runs_one_command_and_leaves_the_schedule_alone(self): - """run_now runs one command and leaves the schedule's timing alone.""" - base, last = dict(self.s.base), dict(self.s.last) - self.assertEqual(self.s.run_now('7'), ('price.py close', 'done')) - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) - self.assertEqual((self.s.base, self.s.last), (base, last)) - self.assertEqual(self.s.busy, set()) - - def test_run_now_refuses_while_the_command_is_running_and_reports_failures(self): - """run_now refuses a command that is running, and reports one that fails.""" - self.s.busy.add('trades.py') - self.assertEqual(self.s.run_now('8'), ('trades.py', 'already running')) - self.assertEqual(self.ran, []) - self.s.busy.clear() - def boom(**k): - """Fail.""" - raise RuntimeError('boom') - self.patch(jobs.SCRIPTS['trades'], 'run', boom) - self.assertEqual(self.s.run_now('8'), ('trades.py', 'failed')) - self.assertTrue(any('boom' in e for e in self.errors)) - self.assertEqual(self.s.busy, set()) - - def test_a_scheduled_run_skips_a_command_that_a_manual_run_is_in_the_middle_of(self): - """A scheduled run skips a command still running (from chat, or the line's last run), and runs the rest of its - line.""" - self.s.busy.add('trades.py') - self.assertEqual(self.s.execute(self.s.jobs[1].commands), 'already running') - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) # price.py close ran, trades.py was skipped - - def test_run_now_times_out(self): - """A manual run stopped at its deadline says so.""" - def hang(label, module, kwargs): - """Hang until run_child stops it.""" - raise TimeoutError('still running after 5 minutes') - self.s.runner = hang - self.assertEqual(self.s.run_now('8'), ('trades.py', 'timed out')) - self.assertEqual(self.s.busy, set()) - - def test_two_runs_of_one_command_at_once_run_it_once(self): - """Of two runs of the same command started together, one runs it and the other is told it is running.""" - def slow(**k): - """Take a while, letting the other run start.""" - time.sleep(0.2) - self.ran.append(('trades', k)) - self.patch(jobs.SCRIPTS['trades'], 'run', slow) - outcomes = [] - threads = [threading.Thread(target=lambda: outcomes.append(self.s.run_now('8'))) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - self.assertEqual(sorted(outcomes), [('trades.py', 'already running'), ('trades.py', 'done')]) - self.assertEqual(self.ran, [('trades', {})]) - self.assertEqual(self.s.busy, set()) - - def list_in_chat(self, text='.jobs'): - """The lines .jobs (or `text`) answers with.""" - return self.command(text)[0][1] - - def test_list_in_chat_is_just_the_commands(self): - """.jobs and aliases number the commands, without times, comments or redirects, and mark one that is running; - the system jobs follow under their own heading, without a number.""" - feed = jobs.display(podcasts.job(['https://example.com/feed'])[0]) # podcasts and the feed's hash - for alias in ('.jobs', '.cron', '.crons', '.schedule', '@' + BOT.lstrip('@') + ' jobs'): - text = '\n'.join(map(str, self.list_in_chat(alias))) - for line in ('1. performance 7', '2. performance 30', '5. performance 365', '6. price 7', '7. price close', - '8. trades\n\nSystem:\ncleanup\n' + feed + '\n'): - self.assertIn(line, text, alias) - for noise in ('*/20', '#', 'Mon-Fri', 'next', 'last', '/dev/null', 'weekly', '.py'): - self.assertNotIn(noise, text, alias) - self.s.busy.add('price.py 7') - self.assertIn('6. price 7 (running)', '\n'.join(map(str, self.list_in_chat()))) - - def test_a_schedule_of_system_jobs_alone_lists_them(self): - """A schedule of system jobs alone lists them, with nothing to run or tap.""" - self.file.write_text('0 2 * * * cleanup.py\n') - self.patch(jobs, 'current', self.scheduler()) - self.assertEqual(about.prepare_jobs_payload(TELEGRAM, '', tappable=True), (['System:', 'cleanup'], None)) - - def test_a_chat_that_takes_buttons_gets_one_per_command(self): - """Where buttons work each command gets one, whose data fits in Telegram's 64 bytes, and the list says to tap - it; elsewhere there are none, and it does not.""" - payload, markup = about.prepare_jobs_payload(TELEGRAM, '', tappable=True) - pressable = [b for row in markup for b in row] - self.assertEqual([b.label for b in pressable], ['1. performance 7', '2. performance 30', '3. performance 90', - '4. performance 180', '5. performance 365', '6. price 7', '7. price close', '8. trades']) - for b in pressable: - self.assertLessEqual(len(b.data.encode()), 64) - self.assertTrue(payload[-1].endswith('or tap it')) - payload, markup = about.prepare_jobs_payload(SLACK, '') - self.assertIsNone(markup) - self.assertTrue(payload[-1].endswith('.jobs run NUMBER')) - - def test_slack_gets_the_buttons_in_block_kit_and_a_stale_list_is_replaced(self): - """With the bot token and slackOutgoingWebhook, Slack's .jobs list carries a button per command, two to a row - of Block Kit, and a stale button's press puts the current list in its place.""" - self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') - self.patch(transports.Transport, 'text', REAL_TEXT) - posts = [] - self.patch(transports.net, 'post', lambda url, **kwargs: posts.append((url, kwargs)) or - mock.Mock(status_code=200, json=lambda: {'ok': True, 'ts': '1.5'})) - with contextlib.redirect_stderr(io.StringIO()): - worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.jobs', BOT, 'U')) - url, kwargs = posts[-1] - blocks = json.loads(kwargs['data']['blocks']) - self.assertTrue(url.endswith('/chat.postMessage')) - self.assertEqual([block['type'] for block in blocks], ['section'] + ['actions'] * 4) - self.assertEqual([button['text']['text'] for button in blocks[4]['elements']], ['7. price close', '8. trades']) - with contextlib.redirect_stderr(io.StringIO()): - worker.process_callback(inbound.Press('slack', 't1', 'C1', '1.5', 'j|7|0000')) - url, kwargs = posts[-1] - self.assertTrue(url.endswith('/chat.update')) - self.assertEqual((kwargs['data']['channel'], kwargs['data']['ts']), ('C1', '1.5')) - self.assertEqual(len(json.loads(kwargs['data']['blocks'])), 5) - self.assertEqual(self.ran, []) - - def test_the_chat_command_passes_the_keyboard_on(self): - """.jobs sends its buttons with the list.""" - seen = [] - self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: seen.append(buttons) or []) - worker.process_request(inbound.Message('telegram', '55', '@u', '.jobs', BOT, 'U')) - self.assertEqual(len(seen[0][0]), 2) # two to a row - - def test_button_press_runs_that_command(self): - """Pressing a command's button runs it, says so in a toast, and reports how it went.""" - sent = [] - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: sent.append(text)) - markup = about.prepare_jobs_payload(TELEGRAM, '', tappable=True)[1] - data = next(b.data for row in markup for b in row if b.label.startswith('7.')) - self.sent.clear() - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', data)) - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) - self.assertEqual(self.sent, [('text', ['7. price close: done'])]) - self.assertEqual(sent, ['Running price close…']) - - def stale_buttons(self): - """Capture the toast and the in-place edit that a refused button press produces.""" self.toasts, self.edits = [], [] - self.patch(telegram, 'answerCallbackQuery', lambda cid, text=None: self.toasts.append(text)) - self.patch(transports.TelegramBot, 'edit_text', lambda dest, mid, text, buttons=None: + self.patch(telegram_common, 'answerCallbackQuery', lambda cid, text=None: self.toasts.append(text)) + self.patch(telegram_common.TelegramBot, 'edit_text', lambda dest, mid, text, buttons=None: self.edits.append((dest.where, mid, text, buttons))) + self.replies = [] + self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: + self.replies.append((dest.key, list(lines), buttons)) or []) - def test_forged_button_data_never_runs_anything_else(self): - """Bad or forged button data runs nothing, refreshes buttons in place; no number picks cleanup or podcasts.""" - self.stale_buttons() - good = jobs.checksum('trades.py') - for data in ('j|-1|' + good, 'j|0|' + good, 'j|9|' + good, 'j|8|0000', 'j|8|', 'j|8', 'j|8|' + good + '|x', - 'j||' + good, - 'j|٨|' + good, 'j| 8|' + good, 'j|8.0|' + good, 'j|trades.py|' + good, 'j|1|' + good, - 'j|8|price.py close', - 'j|8|' + good.upper() + 'x', 'j|99999999999999999999|' + good): - self.sent.clear() - self.toasts.clear(); self.edits.clear() - worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', data)) - self.assertEqual(self.ran, [], data) - self.assertEqual(self.toasts, ['Buttons updated, try again'], data) - self.assertEqual(len(self.edits), 1, data) - self.assertEqual(self.sent, [], data) # no new message, the old one is edited - # cleanup and podcasts are scheduled but cannot be picked, by number or by their checksum - for label in ('cleanup.py', 'podcasts.py'): - for n in range(1, 12): - with self.assertRaises(jobs.ScheduleError): - self.s.pick(str(n), jobs.checksum(label)) - worker.process_callback(inbound.Press('telegram', 'cb', '55', '9', 'j|8|' + good)) # the genuine one still works + def say(self, text, chat='55'): + """Send text in a Telegram chat: the lines of the reply.""" + self.replies.clear() + with contextlib.redirect_stderr(io.StringIO()): + worker.process_request(inbound.Message('telegram', chat, '@u', text, BOT, 'U')) + self.assertEqual(self.errors, [], text) + return self.replies[-1][1] if self.replies else None + + def test_add_list_and_del(self): + """A job is added, listed with its number, line, state and next run (in UTC and Melbourne), and deleted.""" + added = self.say('.jobs add 29 21 * * Mon-Fri price interday') + self.assertEqual(added, ['2. 29 21 * * Mon-Fri price interday: on, next Tue 21:29 UTC (Wed 08:29 AEDT)', + 'Added: .jobs lists them all']) + listed = self.say('.jobs') + self.assertEqual(listed[:3], ['Jobs here:', '1. */20 * * * Mon-Fri trades: on, next Tue 21:20 UTC (Wed' + ' 08:20 AEDT)', '2. 29 21 * * Mon-Fri price interday: on, next Tue 21:29 UTC (Wed 08:29 AEDT)']) + self.assertEqual(listed[-4:], ['.jobs run [NUMBER]: run one now', '.jobs toggle [NUMBER]: switch one on or off', + '.jobs add [CRON_TZ=ZONE] TIME REPORT: add one (.jobs add lists the reports)', + '.jobs del NUMBER: delete one']) + self.assertIsNone(self.replies[-1][2]) # .jobs alone has no buttons + self.assertEqual(self.say('.jobs del 1'), ['Deleted 1. */20 * * * Mon-Fri trades']) + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {CHAT: [entry('29 21 * * Mon-Fri price interday')]}) + self.assertEqual([job.text for job in self.s.chat_jobs(CHAT)], ['price interday']) # at once, no reload + self.assertEqual(self.say('.job del 1'), ['Deleted 1. 29 21 * * Mon-Fri price interday']) + self.assertEqual(state.load(jobs.SCHEDULE_FILE), {}) + self.assertEqual(self.say('.jobs')[0], 'No jobs here yet: add one with .jobs add') + + def test_a_job_names_its_zone_runs_several_reports_and_quotes_its_arguments(self): + """A job is in UTC unless it names its zone; it may run several reports in turn, and an argument in quotes may + hold anything.""" + url = 'https://example.com/feed?a=1&b=2;c' + self.say(f'.jobs add CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession') + self.say(f'.jobs add 5 4 * * * podcasts "{url}"; shorts') + self.say('.jobs add @daily reminder') + ny, feed, daily = self.s.chat_jobs(CHAT)[1:] + self.assertEqual((ny.zone.key, ny.line), ('America/New_York', 'CRON_TZ=America/New_York 15 13 * * Mon-Fri' + ' price midsession')) + self.assertEqual([(label.split()[0], kwargs) for label, _, kwargs in feed.commands], + [('podcasts', {'feed_url': url}), ('shorts', {})]) + self.assertEqual((daily.when, daily.zone.key), ('@daily', 'UTC')) + self.assertIn('2. CRON_TZ=America/New_York 15 13 * * Mon-Fri price midsession: on, next Wed 17:15 UTC', + self.say('.jobs')[2]) + + def test_add_alone_lists_the_reports_a_job_can_run(self): + """.jobs add alone lists the reports with what each takes, the system jobs' aside, and examples.""" + lines = self.say('.jobs add') + self.assertEqual(lines[0], 'Reports a job can run:') + for line in ('cal earnings|ex-dividend', 'performance [DAYS]', 'podcasts FEED_URL', + 'price midsession|interday|premarket|close|DAYS', 'trades'): + self.assertIn(line, lines) + self.assertFalse(any(name in line for line in lines for name in jobs.SYSTEM_SCRIPTS)) + + def test_a_job_that_cannot_be_is_refused_saying_why(self): + """A time that cannot be or never comes, an unknown report, a script's file name, a system job, a bad argument, + an unknown zone or too few fields: refused, saying why, and nothing stored.""" + for line, why in (('61 1 * * * trades', "'61 1 * * *' is no time"), + ('0 0 30 2 * trades', "'0 0 30 2 *' never comes"), ('@reboot trades', '@reboot is no time'), + ('0 1 * * * nothere', "no report 'nothere'"), ('0 1 * * * trades.py', "no report 'trades.py'"), + ('0 1 * * * yahoo_warm', "no report 'yahoo_warm'"), + ('0 1 * * * price bogus', 'price takes midsession|interday|premarket|close|DAYS'), + ('0 1 * * * trades 7', 'trades takes no arguments'), + ('CRON_TZ=Nowhere/Land 0 1 * * * trades', "no time zone 'Nowhere/Land'"), + ('0 1 * *', 'a job is five time fields')): + with self.subTest(line=line): + self.assertIn(why, self.say(f'.jobs add {line}')[0]) + self.assertEqual(len(self.s.chat_jobs(CHAT)), 1) + + def test_a_chat_has_thirty_jobs_at_most(self): + """A chat may have 30 jobs; the 31st is refused.""" + for minute in range(1, 30): + self.s.add(CHAT, f'{minute} 1 * * * trades') + self.assertIn('30 jobs is the most a chat may have', self.say('.jobs add 31 1 * * * trades')[0]) + self.assertEqual(len(self.s.chat_jobs(CHAT)), 30) + + def test_run_a_job_now(self): + """.jobs run N runs job N now, posting to this chat, and says how it went; a number that is none says so.""" + self.assertEqual(self.say('.jobs run 1'), ['1. trades: done']) + self.assertEqual((self.ran, self.dests), ([('trades', {})], [CHAT])) + self.assertEqual(self.say('.jobs run 2'), ['no job 2: send .jobs for the list']) + self.assertEqual(self.say('.jobs run 1', chat='56'), ['no jobs here yet: .jobs add']) # another chat's list self.assertEqual(self.ran, [('trades', {})]) - def test_button_from_an_old_schedule_updates_the_message(self): - """A stale button runs nothing; the message gets the current list and buttons (scheduler off: a notice).""" - self.stale_buttons() - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|7|0000')) - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|9|0000')) - self.assertEqual(self.ran, []) - self.assertEqual(self.sent, []) - self.assertEqual(self.toasts, ['Buttons updated, try again'] * 2) - chat, mid, text, markup = self.edits[0] - self.assertEqual((chat, mid), ('55', '9')) # the message with the old buttons is edited in place - self.assertIn('7. price close', text) - self.assertEqual(markup, about.prepare_jobs_payload(TELEGRAM, '', tappable=True)[1]) # the current buttons - self.patch(jobs, 'current', None) # scheduler off: the list is replaced by the notice and the buttons removed - worker.process_callback(inbound.Press('telegram', 'cb1', '55', '9', 'j|7|0000')) - self.assertEqual(self.edits[-1][2:], ('Scheduled jobs are off (scheduler=False)', [])) # no buttons left - - def test_run_in_chat(self): - """.jobs run takes a number or a command's text.""" - self.assertEqual(self.command('.jobs run 8'), [('text', ['8. trades: done'])]) + def test_toggle_a_job_off_and_on(self): + """.jobs toggle N switches job N off (it no longer fires) and on again.""" + self.assertEqual(self.say('.jobs toggle 1'), ['1. */20 * * * Mon-Fri trades: off']) + self.clock.now = at(13, 21, 20) + self.assertEqual(self.s.due(self.clock.now), []) + self.assertEqual(self.say('.jobs toggle 1')[0][:30], '1. */20 * * * Mon-Fri trades: ') + self.clock.now = at(13, 21, 40) + self.assertEqual(len(self.s.due(self.clock.now)), 1) + + def test_the_run_and_toggle_lists_have_a_button_per_job(self): + """.jobs run and .jobs toggle list the jobs with a button each that runs it now or switches it.""" + self.s.add(CHAT, '0 22 * * * shorts') + for action, kind in (('run', 'j'), ('toggle', 'e')): + with self.subTest(action=action): + self.say(f'.jobs {action}') + lines, markup = self.replies[-1][1:] + tap = 'Tap one to run it now' if action == 'run' else 'Tap one to switch it on or off' + self.assertIn(tap, lines[-1]) + job = self.s.chat_jobs(CHAT)[1] + self.assertEqual(markup[0][1], buttons.Button('2. shorts', f'{kind}|2|{jobs.checksum(job.key)}')) + + def test_a_run_button_runs_its_job_and_a_toggle_button_redraws_the_list(self): + """A run button runs its job and says how it went; a toggle button switches its job and redraws the list in + place, with the buttons still there.""" + job = self.s.chat_jobs(CHAT)[0] + self.press_button(f'j|1|{jobs.checksum(job.key)}') + self.assertEqual((self.toasts, self.ran), (['Running…'], [('trades', {})])) + self.assertEqual(self.replies[-1][1], ['1. trades: done']) + self.press_button(f'e|1|{jobs.checksum(job.key)}') + self.assertEqual(self.toasts[-1], 'Switching…') + self.assertFalse(self.s.chat_jobs(CHAT)[0].on) + chat, message, text, markup = self.edits[-1] + self.assertEqual((chat, message), ('55', '9')) + self.assertIn('1. */20 * * * Mon-Fri trades: off', text) + self.assertEqual(markup[0][0].data, f'e|1|{jobs.checksum(job.key)}') # the key does not hold the switch + + def test_a_stale_or_forged_button_redraws_the_list_and_does_nothing(self): + """A button whose number or checksum is not this chat's job (an older list, another chat's, forged data) runs + and switches nothing: the list is redrawn in its place, with that kind of buttons.""" + self.store(OTHER, '*/20 * * * Mon-Fri trades') + self.s.load() + good = jobs.checksum(self.s.chat_jobs(CHAT)[0].key) + others = jobs.checksum(self.s.chat_jobs(OTHER)[0].key) + for data in ('j|2|' + good, 'j|0|' + good, 'j|1|0000', 'j|1|' + others, 'e|1|0000', 'j|1', 'j|٨|' + good, + 'j| 1|' + good, 'e|99999999999999999999|' + good): + with self.subTest(data=data): + self.edits.clear() + self.press_button(data) + self.assertEqual(self.ran, []) + self.assertTrue(self.s.chat_jobs(CHAT)[0].on) + self.assertEqual(len(self.edits), 1) + self.assertIn('1. */20 * * * Mon-Fri trades: on', self.edits[0][2]) + self.assertEqual(self.edits[0][3][0][0].data[0], data[0]) # the same kind of buttons + self.press_button('j|1|' + good) # the genuine one still works self.assertEqual(self.ran, [('trades', {})]) - self.ran.clear() - self.assertEqual(self.command('.cron run price close')[0][1], ['7. price close: done']) - self.assertEqual(self.ran, [('price', {'mode': 'close'})]) - - def test_run_in_chat_refusals(self): - """.jobs run refuses an unscheduled command, an unknown number and two commands, else gives its usage.""" - self.assertIn('not in the schedule', self.list_in_chat('.jobs run cal.py earnings')[0]) - self.assertIn('no command 99', self.list_in_chat('.jobs run 99')[0]) - self.assertIn('one command at a time', self.list_in_chat('.jobs run price.py close; trades.py')[0]) - self.assertIn('Usage', self.list_in_chat('.jobs run')[0]) - self.assertIn('Usage', self.list_in_chat('.jobs bogus')[0]) - self.assertEqual(self.ran, []) - - def test_scheduler_off(self): - """With the scheduler off, .jobs says so.""" + + def test_no_schedule_running(self): + """Without a schedule running, .jobs says so.""" self.patch(jobs, 'current', None) - self.assertIn('off', self.list_in_chat('.jobs')[0]) - self.assertIn('off', self.list_in_chat('.jobs run 8')[0]) - - def test_job_is_still_a_ticker(self): - """.job is a ticker, not .jobs.""" - self.assertNotIn('Scheduled', str(self.command('.job'))) - - -class ChildProcess(FinbotCase): - """Jobs run in their own process, so heavy work cannot hold up the bot's web server.""" - - def sleeper(self, seconds): - """Have run_child start a process that sleeps for `seconds`, kept in self.children.""" - real = jobs.subprocess.Popen - self.children = [] - def popen(args, **kw): - """Start the sleeper instead of the job.""" - self.children.append(real([sys.executable, '-c', f'import time; time.sleep({seconds})'])) - return self.children[-1] - self.patch(jobs.subprocess, 'Popen', popen) - - def test_every_command_can_be_handed_to_a_child(self): - """Every shipped command's arguments survive JSON, the way they reach the child.""" - parsed, _ = jobs.parse_schedule(DEFAULT_SCHEDULE.read_text(), UTC) - for job in parsed: - for label, module, kwargs in job.commands: - self.assertEqual(json.loads(json.dumps(kwargs)), kwargs, label) - - def test_a_real_child_runs_the_handler_and_reports_its_exit_status(self): - """A real child runs the report and exits 0, or REPORTED when it failed and said so.""" - real = jobs.subprocess.Popen - self.patch(jobs.subprocess, 'Popen', - lambda args, **kw: real(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, **kw)) - with tempfile.TemporaryDirectory() as data: - env = {'DATA_DIR': data, 'telegramBotToken': '', 'slack_webhook': '', 'discord_webhook': ''} - with mock.patch.dict(os.environ, env): - self.assertEqual(jobs.run_child('cleanup.py', cleanup, {}), 0) - self.assertEqual(jobs.run_child('cleanup.py 0', cleanup, {'days': 'not a number'}), - jobs.jobrun.REPORTED) - - def test_a_job_process_survives_a_reload_signal(self): - """A job process ignores SIGHUP, which systemctl kill -s HUP sends to every process of the unit.""" - with tempfile.TemporaryDirectory() as scripts, tempfile.TemporaryDirectory() as data: - Path(scripts, 'sleeper.py').write_text( - 'import time\ndef run():\n print("started", flush=True)\n time.sleep(1)\n') - env = {**os.environ, 'PYTHONPATH': scripts, 'DATA_DIR': data, 'telegramBotToken': '', 'slack_webhook': '', - 'discord_webhook': ''} - proc = subprocess.Popen([sys.executable, '-m', 'lib.jobrun', 'sleeper', '{}'], cwd=ROOT, env=env, - stdout=subprocess.PIPE, text=True) - self.addCleanup(proc.kill) - for line in proc.stdout: - if line.strip() == 'started': - break - proc.send_signal(signal.SIGHUP) # what `systemctl kill -s HUP finbot` does to every process of the unit - self.assertEqual(proc.wait(timeout=30), 0) - - def test_a_script_that_cannot_be_imported_is_reported(self): - """A report that cannot be imported is reported like any other failure.""" - with tempfile.TemporaryDirectory() as data: - env = {**os.environ, 'DATA_DIR': data, 'telegramBotToken': '', 'slack_webhook': '', 'discord_webhook': ''} - out = subprocess.run([sys.executable, '-m', 'lib.jobrun', 'nosuchscript', '{}'], cwd=ROOT, env=env, - capture_output=True, text=True) - self.assertEqual(out.returncode, jobs.jobrun.REPORTED, out.stderr) - self.assertIn('⚠️ nosuchscript.py: ModuleNotFoundError', out.stderr) - - def test_scheduler_reports_a_child_that_ended_without_saying_why(self): - """A child killed by a signal or exiting non-zero without a report is reported by the scheduler; one that - reported itself is not reported twice.""" - file = Path(temp_dir(self), 'schedule.cron') - file.write_text('* * * * * trades.py\n') - for status, reason in ((-signal.SIGHUP, 'killed by SIGHUP'), (-signal.SIGKILL, 'killed by SIGKILL'), - (1, 'exited with status 1 without saying why'), - (jobs.jobrun.REPORTED, None)): # the child reported this one itself - with self.subTest(status=status): - self.errors.clear() - s = jobs.Scheduler(lambda: file, zone=UTC, runner=lambda label, module, kwargs: status) - self.assertTrue(s.load()) - self.assertEqual(s.run_now('1'), ('trades.py', 'failed')) - self.assertEqual(self.errors, [f'⚠️ trades.py: RuntimeError: {reason}'] if reason else []) - self.assertEqual(s.busy, set()) - - def test_a_report_script_loads_neither_pandas_nor_the_scheduler(self): - """A job imports only its script; price once pulled in pandas and every script (via charts, reports, jobs).""" + self.assertEqual(self.say('.jobs'), ['No schedule is running']) + + def test_a_job_is_its_chats_whichever_way_its_updates_arrive(self): + """A chat's jobs are kept under its network and its id, which a network gives the same in both modes: a job + added in http mode is that chat's in pull mode, listed and fired there, and the other way round.""" + updates = test_sending.EveryNetworkAndMode() + updates.updates = 0 + self.patch(discord_common.READER, 'bot_id', '99') + self.env(slackBotToken='xoxb-1', discordBotToken='x') + for network, chat in (('telegram', '-1001'), ('slack', 'C1'), ('discord', '77')): + for first, then in (('http', 'pull'), ('pull', 'http')): + with self.subTest(network=network, added=first): + self.env(**test_sending.EveryNetworkAndMode.MODES[(network, first)]) + reader = READERS[network] + with contextlib.redirect_stderr(io.StringIO()): + answer = bot.take(reader, updates.update(network, first, chat, '.jobs add 0 23 * * * shorts')) + if answer.then: + answer.then() + self.join_workers() + key = f'{network}:{chat}' + self.assertEqual([job.text for job in self.s.chat_jobs(key)], ['shorts']) + self.env(**test_sending.EveryNetworkAndMode.MODES[(network, then)]) + self.replies.clear() + answer = bot.take(reader, updates.update(network, then, chat, '.jobs')) + if answer.then: + answer.then() + self.join_workers() + self.assertIn('1. 0 23 * * * shorts: on', self.replies[-1][1][1]) + self.assertEqual(self.replies[-1][0], key) + self.dests.clear() + self.s.run(self.s.chat_jobs(key)[0]) + self.assertEqual(self.dests, [key]) # posted to the same chat + self.s.delete(self.s.chat_jobs(key)[0]) + + +class ReportImports(FinbotCase): + """What importing the reports brings in.""" + + def test_a_report_loads_neither_matplotlib_nor_the_scheduler(self): + """A report imports only its own module: price once pulled in every report, through charts and jobs, and the + chart code until a chart is drawn.""" code = ('import sys; from lib.reports import price, performance, trades, shorts, cal; ' - 'print(sorted(m for m in ("pandas", "lib.jobs", "matplotlib") if m in sys.modules))') + 'print(sorted(m for m in ("lib.jobs", "matplotlib") if m in sys.modules))') with tempfile.TemporaryDirectory() as data: out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True, env={**os.environ, 'DATA_DIR': data}) self.assertEqual(out.stdout.strip().splitlines()[-1:], ['[]'], out.stderr) - def test_the_event_loop_keeps_running_while_a_child_works(self): - """Waiting for a child does not stop the event loop.""" - self.sleeper(1) - ticks = [] - def ticker(): - """Tick five times, 0.1 seconds apart.""" - for _ in range(5): - gevent.sleep(0.1) - ticks.append(time.monotonic()) - greenlet = gevent.spawn(ticker) - started = time.monotonic() - self.assertEqual(jobs.run_child('cleanup.py', cleanup, {}), 0) - greenlet.join() - self.assertGreater(time.monotonic() - started, 0.9) - self.assertEqual(len(ticks), 5) # the loop was not blocked by the child - self.assertLess(ticks[-1] - started, 0.8) - - def test_a_child_still_running_at_its_deadline_is_stopped(self): - """A child still running after JOB_TIMEOUT is stopped, rather than left running, and TimeoutError raised.""" - self.patch(jobs, 'JOB_TIMEOUT', 0.3) - self.sleeper(30) - started = time.monotonic() - with self.assertRaises(TimeoutError): - jobs.run_child('cleanup.py', cleanup, {}) - self.assertLess(time.monotonic() - started, 5) - self.assertIsNotNone(self.children[0].poll()) # the child was stopped, not left running - - def test_another_interruption_stops_the_child_too(self): - """An interruption while a child runs (a greenlet killed, say) stops the child as well.""" - self.sleeper(30) - with self.assertRaises(gevent.Timeout): - with gevent.Timeout(0.3): - jobs.run_child('cleanup.py', cleanup, {}) - self.assertIsNotNone(self.children[0].poll()) - - def test_each_command_of_a_line_has_a_deadline_of_its_own(self): - """The deadline is per command (it once covered the whole line): two that each take most of it both finish.""" - self.patch(jobs, 'JOB_TIMEOUT', 0.8) - self.sleeper(0.5) - file = Path(temp_dir(self), 'schedule.cron') - file.write_text('* * * * * trades.py; shorts.py\n') - s = jobs.Scheduler(lambda: file, zone=UTC) - with contextlib.redirect_stderr(io.StringIO()): - self.assertTrue(s.load()) - self.assertEqual(s.execute(s.jobs[0].commands), 'done') - self.assertEqual(len(self.children), 2) - self.assertEqual(self.errors, []) - if __name__ == '__main__': unittest.main() diff --git a/tests/test_sending.py b/tests/test_sending.py index f2a61d9b..658b0d0b 100644 --- a/tests/test_sending.py +++ b/tests/test_sending.py @@ -2,7 +2,9 @@ buttons) and Discord.""" import contextlib, io, json, threading, unittest from unittest import mock +import requests from tests.common import * +from lib import jobs class TelegramSending(FinbotCase): @@ -15,8 +17,8 @@ def fake(url, **kwargs): if isinstance(outcome, Exception): raise outcome return outcome - self.patch(transports.net, 'post', fake) - return transports.TelegramBot('5').post_text('hi') + self.patch(net, 'post', fake) + return telegram_common.TelegramBot('5').post_text('hi') def test_success_returns_the_message(self): """A sent message returns the message, as Telegram gives it.""" @@ -37,12 +39,33 @@ def logged(self, outcome): self.post(outcome) return log.getvalue() + def test_text_goes_silent_and_without_link_previews(self): + """A message goes silent and without link previews, and so does an edit: link_preview_options, which replaced + disable_web_page_preview in Bot API 7.0.""" + sent = [] + self.patch(net, 'post', lambda url, **kwargs: sent.append((url.rsplit('/', 1)[1], kwargs['json'])) or + Resp({'ok': True, 'result': {'message_id': 8}})) + chat = telegram_common.TelegramBot('5') + chat.post_text('hi') + chat.edit_text('8', 'edited') + off = {'is_disabled': True} + self.assertEqual(sent, [('sendMessage', {'chat_id': '5', 'text': 'hi', 'parse_mode': 'HTML', + 'link_preview_options': off, 'disable_notification': True}), ('editMessageText', {'chat_id': '5', + 'message_id': '8', 'text': 'edited', 'parse_mode': 'HTML', 'link_preview_options': off})]) + + def test_typing_is_sent_again_before_it_lapses(self): + """"typing…" shows for 5 seconds at most (sendChatAction), so it is sent again sooner, and shows the whole time + a command works, for a minute at least.""" + delay, every, times = telegram_common.TelegramBot.TYPING + self.assertLess(every, 5) + self.assertGreaterEqual(delay + every * times, 60) + def test_a_chart_goes_without_a_parameter_sendphoto_does_not_take(self): """sendPhoto is sent the chat, caption and markup, and no link-preview flag: a photo has no preview.""" sent = [] - self.patch(transports.net, 'post', lambda url, **kwargs: sent.append((url, kwargs)) or + self.patch(net, 'post', lambda url, **kwargs: sent.append((url, kwargs)) or Resp({'ok': True, 'result': {'message_id': 8}})) - self.assertEqual(transports.TelegramBot('5').post_chart(io.BytesIO(b'png'), 'caption'), {'message_id': 8}) + self.assertEqual(telegram_common.TelegramBot('5').post_chart(io.BytesIO(b'png'), 'caption'), {'message_id': 8}) url, kwargs = sent[0] self.assertTrue(url.endswith('/sendPhoto')) self.assertEqual(kwargs['data'], {'chat_id': '5', 'caption': 'caption', 'parse_mode': 'HTML', @@ -60,16 +83,15 @@ def test_error_reports_are_escaped_for_telegram_and_slack(self): """An error line is escaped for Telegram's HTML and Slack (the same), and left as it is for Discord.""" sent = [] self.patch(transports.Transport, 'text', REAL_TEXT) - for kind in (transports.TelegramBot, transports.SlackWebhook, transports.DiscordWebhook): + for kind in (telegram_common.TelegramBot, slack_common.SlackBot, discord_common.DiscordBot): self.patch(kind, 'post_text', lambda dest, text, buttons=None: sent.append((dest.name, text))) - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/', 'slack': 'https://hooks.slack.com/x', - 'discord': 'https://discord.com/x/slack'}) - self.env(telegramChatID='5') + self.env(telegramChatID='5', slackChannelID='C9', slackBotToken='xoxb-1', discordChannelID='77', + discordBotToken='x') error = RuntimeError("404 for ZZ at https://x/?a=1&crumb=abc; '<' not supported between ") with contextlib.redirect_stderr(io.StringIO()): REAL_REPORT_ERROR(error, context='.compare ') - REAL_REPORT_ERROR(error, transports.TelegramBot('55'), context='.compare') + REAL_REPORT_ERROR(error, telegram_common.TelegramBot('55'), context='.compare') texts = dict(sent[:3]) self.assertIn('crumb=<redacted>', texts['telegram']) # Telegram's HTML parser rejects a bare self.assertNotIn('<', texts['telegram'] + sent[3][1]) @@ -81,19 +103,9 @@ def test_report_error_never_raises(self): """report_error survives a failed send.""" self.patch(transports.Transport, 'text', mock.Mock(side_effect=RuntimeError('send failed'))) with contextlib.redirect_stderr(io.StringIO()): - REAL_REPORT_ERROR(ValueError('boom'), transports.TelegramBot('5')) + REAL_REPORT_ERROR(ValueError('boom'), telegram_common.TelegramBot('5')) REAL_REPORT_ERROR(ValueError('boom')) - def test_guarded_reports_and_exits_nonzero(self): - """guarded reports a failure and exits 1; a success returns its value.""" - reported = [] - self.patch(delivery, 'report_error', lambda e, **k: reported.append(e)) - with self.assertRaises(SystemExit) as raised: - delivery.guarded('job', lambda: 1 / 0) - self.assertEqual(raised.exception.code, 1) - self.assertEqual(len(reported), 1) - self.assertEqual(delivery.guarded('job', lambda: 'fine'), 'fine') - class ChartDelivery(FinbotCase): """Charts reach Telegram and Slack replies (uploaded); a report whose chart fails still goes, as text.""" @@ -114,7 +126,7 @@ def post(url, **kwargs): if isinstance(outcome, Exception): raise outcome return outcome - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) return calls def upload_steps(self): @@ -122,36 +134,30 @@ def upload_steps(self): return [Resp({'ok': True, 'upload_url': self.UPLOAD_URL, 'file_id': 'F1'}), Resp(None, text='OK - 3'), Resp({'ok': True, 'files': [{'id': 'F1'}]})] - def test_where_charts_can_go(self): - """Charts always go to Telegram and Discord; Slack gets them as the bot, with the bot token, but not through its - incoming webhook.""" - self.env(slackBotToken='xoxb-test') - - kinds = (transports.TelegramBot, transports.SlackBot, transports.SlackWebhook, transports.DiscordWebhook) - self.assertEqual([kind(None).takes_charts for kind in kinds], [True, True, False, True]) - self.env(slackBotToken=False) - self.assertFalse(transports.SlackBot(None).takes_charts) - - def test_scheduled_reports_go_to_slack_as_the_bot_with_slackChannelID(self): - """With slackChannelID and the bot token, scheduled reports reach Slack as the bot in place of the incoming - webhook, which it does not need; without the token, the webhook carries them.""" - hook = 'https://hooks.slack.com/services/x' - telegram = config.env.webhooks['telegram'] - self.env(webhooks={'telegram': telegram, 'slack': hook}, telegramChatID='1', slackChannelID='C9', - slackBotToken='xoxb-test') - self.assertEqual(list(map(repr, delivery.default_chats())), ['', '']) - self.env(slackBotToken='') - self.assertEqual(list(map(repr, delivery.default_chats())), ['', f'']) - self.env(webhooks={'telegram': telegram}, slackBotToken='xoxb-test') - self.assertEqual(list(map(repr, delivery.default_chats())), ['', '']) - self.env(webhooks={}, telegramChatID='') # the bot alone - self.assertEqual(list(map(repr, delivery.default_chats())), ['']) + def test_every_bot_takes_charts_and_buttons(self): + """Every network's bot takes charts and buttons, whichever way its updates arrive: their presses reach finbot, + and no setting holds them back.""" + self.env(slackBotToken='', slackAppToken='', slackOutgoingWebhook='', telegramOutgoingWebhook='') + for kind in (telegram_common.TelegramBot, slack_common.SlackBot, discord_common.DiscordBot): + with self.subTest(kind=kind.__name__): + self.assertTrue(kind('1').takes_charts and kind('1').takes_buttons) + + def test_scheduled_reports_go_to_each_networks_home_chat_as_its_bot(self): + """Scheduled reports go to each network's home chat (telegramChatID, slackChannelID, discordChannelID), as its + bot: a chat without its bot token is left out.""" + self.env(telegramChatID='1', slackChannelID='C9', slackBotToken='xoxb-test', discordChannelID='77', + discordBotToken='x') + self.assertEqual(list(map(repr, delivery.default_chats())), ['', '', + '']) + self.env(slackBotToken='', telegramBotToken='') + self.assertEqual(list(map(repr, delivery.default_chats())), ['']) def test_slack_replies_upload_their_charts(self): - """A Slack reply uploads its chart in three steps (bot token), the report as its message, with no text copy.""" + """A Slack reply whose chart has no buttons uploads it in three steps (bot token), the report as its message, + with no text copy (a chart with buttons goes in a Block Kit message: SlackButtons).""" self.env(slackBotToken='xoxb-test') - for cmd, heading in (('.beta', '*Beta over 1.5'), ('.price', '*≥ 5% at close'), ('.compare T1 T2', '*')): + for cmd, heading in (('.beta', '*Beta over 1.5'), ('.pie Rob', '*')): with self.subTest(cmd=cmd): calls = self.slack_answers(*self.upload_steps()) self.sent.clear(); self.errors.clear() @@ -178,7 +184,7 @@ def test_a_slack_upload_that_fails_goes_out_as_text(self): self.patch(render, 'draw', lambda chart: io.BytesIO(b'png')) good = self.upload_steps() - slow = transports.requests.exceptions.ReadTimeout('slow') + slow = requests.exceptions.ReadTimeout('slow') text = [('text', ['*Heading*', 'line 2'])] for outcomes, expect in (([Resp({'ok': False, 'error': 'missing_scope'})], text), ([ConnectionError('down')], text), @@ -190,16 +196,12 @@ def test_a_slack_upload_that_fails_goes_out_as_text(self): self.slack_answers(*outcomes) self.sent.clear() with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): - delivery.deliver(transports.SlackBot('C1'), delivery.Report(['*Heading*', 'line 2'], CHART)) + delivery.deliver(slack_common.SlackBot('C1'), delivery.Report(['*Heading*', 'line 2'], CHART)) self.assertEqual(self.sent, expect) - def test_scheduled_reports_to_slack_carry_charts_only_as_the_bot(self): - """Scheduled reports reach Slack's incoming webhook as text; the Slack bot (slackChannelID) and Telegram get - their charts.""" - self.env(slackBotToken='xoxb-test') - - # an incoming webhook cannot upload - self.env(webhooks={**config.env.webhooks, 'slack': 'https://hooks.slack.com/services/x'}) + def test_scheduled_reports_carry_their_charts_to_every_home_chat(self): + """A scheduled report's chart goes to each network's home chat, as its bot.""" + self.env(slackBotToken='xoxb-test', slackChannelID='C9', discordBotToken='x', discordChannelID='77') sent = [] self.patch(transports.Transport, 'chart', lambda dest, image, caption, buttons=None: sent.append((dest.name, 'chart')) or {'ok': True}) @@ -209,39 +211,16 @@ def test_scheduled_reports_to_slack_carry_charts_only_as_the_bot(self): with self.subTest(report=report): sent.clear() report() - self.assertEqual(sorted(set(sent)), [('slack', 'text'), ('telegram', 'chart')]) - self.env(slackChannelID='C9') # as the bot, the report's chart goes too - sent.clear() - report() - self.assertEqual(sorted(set(sent)), [('slack', 'chart'), ('telegram', 'chart')]) - self.env(slackChannelID='') - - def test_slack_without_a_bot_token_gets_text_and_no_chart_is_drawn(self): - """Without the bot token, Slack replies are text and no chart is drawn for them.""" - self.env(slackBotToken=False) - - def refuse(*a, **k): - """Fail: no chart should be drawn.""" - raise AssertionError('a chart was drawn for Slack') - self.patch(render, 'draw', refuse) # every chart is drawn there - self.env(webhooks={**config.env.webhooks, 'slack': 'https://hooks.slack.com/services/x'}) - for cmd in ('.beta', '.pie Rob', '.buy', '.sell', '.peg', '.price', '.price top', '.price T1 7d', - '.performance 7d', '.shorts', '.history T1', '.compare T1 T2'): - with self.subTest(cmd=cmd): - self.sent.clear(); self.errors.clear() - worker.process_request(inbound.Message('slack', 'C1', '<@U1>', cmd, BOT, 'U')) - self.assertEqual(self.errors, []) - self.assertTrue(self.sent) - self.assertEqual({kind for kind, *_ in self.sent}, {'text'}) + self.assertEqual(sorted(set(sent)), [('discord', 'chart'), ('slack', 'chart'), ('telegram', 'chart')]) def test_the_graph_preference_off_sends_no_chart_anywhere(self): - """graph=False means no charts, as the README says; it used to hold back only the price-line charts.""" + """graph=False means no charts, as doc/configuration.md says; it held back only the price-line charts.""" self.prefs(graph=False) self.env(slackBotToken='xoxb-test') posts = [] - self.patch(transports.net, 'post', lambda url, **k: posts.append(url) or Resp({'ok': True, 'result': True})) + self.patch(net, 'post', lambda url, **k: posts.append(url) or Resp({'ok': True, 'result': True})) for service, chat in (('telegram', '55'), ('slack', 'C1')): for cmd in ('.price', '.price top', '.performance 7d', '.shorts', '.buy', '.peg', '.beta', '.compare T1 T2', '.pie Rob'): @@ -267,41 +246,37 @@ def fake(url, **kwargs): for outcome, expect in ((Resp({'ok': False, 'error_code': 400, 'description': "Bad Request: can't parse entities"}, status=400), text), (Resp(None, status=502), text), (ConnectionError('down'), text), - (transports.requests.exceptions.ReadTimeout('slow'), []), # it may be there + (requests.exceptions.ReadTimeout('slow'), []), # it may be there (Resp({'ok': True, 'result': {'message_id': 1}}), [])): with self.subTest(outcome=outcome): self.sent.clear() - self.patch(transports.net, 'post', post(outcome)) + self.patch(net, 'post', post(outcome)) with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): - delivery.deliver(transports.TelegramBot('5'), delivery.Report(['Heading', 'line 2'], CHART, [[]])) + report = delivery.Report(['Heading', 'line 2'], CHART, [[]]) + delivery.deliver(telegram_common.TelegramBot('5'), report) self.assertEqual(self.sent, expect) -class DiscordWebhook(FinbotCase): - """Discord's own webhook endpoint (the Slack-like one takes no flags): no previews, no pings, charts attached.""" - URL = 'https://discord.com/api/webhooks/1/abc' +class DiscordSending(FinbotCase): + """Discord's home channel, as the bot: a scheduled report's chart attached, within Discord's 2,000 characters.""" def setUp(self): - """Discord as the only destination, sent through the real path, its posts recorded.""" + """Discord as the only destination, sent through the real path, its calls recorded.""" super().setUp() - self.env(webhooks={'discord': self.URL}) + self.env(telegramChatID='', discordBotToken='x', discordChannelID='77') self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) - self.posts = [] - self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'id': '1'})) - - def test_text_goes_to_the_native_endpoint_without_previews_or_pings(self): - """Text goes to the native endpoint with ?wait=true, without previews or mentions.""" - transports.DiscordWebhook(self.URL).post_text('hello @everyone') - url, kwargs = self.posts[0] - self.assertEqual(url, self.URL + '?wait=true') # without wait Discord answers 204, which counted as a failure - self.assertEqual(kwargs['json'], {'content': 'hello @everyone', 'flags': 4, 'allowed_mentions': {'parse': []}}) + self.calls = [] + self.patch(net, 'request', lambda method, url, **kwargs: self.calls.append((url, kwargs)) + or Resp({'id': '1'})) def test_a_scheduled_report_carries_its_chart(self): - """A scheduled Discord report has its chart attached and the report as its message, within 2,000 characters.""" - price.run() - (url, kwargs), = self.posts - self.assertEqual(url, self.URL + '?wait=true') + """A scheduled Discord report has its chart attached and the report as its message, within 2,000 characters, + without previews or pings.""" + with contextlib.redirect_stderr(io.StringIO()): + price.run() + (url, kwargs), = self.calls + self.assertEqual(url, 'https://discord.com/api/v10/channels/77/messages') payload = json.loads(kwargs['data']['payload_json']) self.assertEqual((payload['flags'], payload['allowed_mentions'], payload['attachments'][0]['filename']), (4, {'parse': []}, 'chart.png')) @@ -309,13 +284,14 @@ def test_a_scheduled_report_carries_its_chart(self): self.assertLessEqual(len(payload['content']), 2000) name, image, kind = kwargs['files']['files[0]'] self.assertEqual((name, kind), ('chart.png', 'image/png')) - self.assertTrue(image.read().startswith(b'\x89PNG')) + self.assertTrue(image.startswith(b'\x89PNG')) def test_a_long_report_is_split_by_size(self): """A long report is split into messages of at most 2,000 characters.""" line = 'x' * 150 - transports.DiscordWebhook(self.URL).text([line] * 30) - sizes = [len(kwargs['json']['content']) for _, kwargs in self.posts] + with contextlib.redirect_stderr(io.StringIO()): + discord_common.DiscordBot('77').text([line] * 30) + sizes = [len(kwargs['json']['content']) for _, kwargs in self.calls] self.assertGreater(len(sizes), 2) self.assertTrue(all(size <= 2000 for size in sizes), sizes) # 20 lines a chunk used to make 3,019 characters @@ -325,7 +301,7 @@ def test_links_are_masked_links(self): class SlackSending(FinbotCase): - """Slack replies go through chat.postMessage with the bot token; scheduled reports through the incoming webhook.""" + """Slack's messages go through chat.postMessage with the bot token, replies and scheduled reports alike.""" def setUp(self): """The real send path, its posts recorded.""" @@ -333,26 +309,24 @@ def setUp(self): self.patch(transports.Transport, 'text', REAL_TEXT) self.patch(transports.Transport, 'chart', REAL_CHART) self.posts = [] - self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'ok': True})) + self.patch(net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or Resp({'ok': True})) def test_link_previews_are_turned_off_in_the_body(self): - """Link previews are turned off in the body, where Slack reads them, for replies and for the webhook.""" + """Link previews are turned off in the body, where Slack reads them.""" self.env(slackBotToken='xoxb-test') - transports.SlackBot('C1').post_text('hi') - transports.SlackWebhook('https://hooks.slack.com/services/T/B/x').post_text('hi') - (_, reply), (_, hook) = self.posts + with contextlib.redirect_stderr(io.StringIO()): + slack_common.SlackBot('C1').post_text('hi') + (_, reply), = self.posts self.assertEqual((reply['data']['unfurl_links'], reply['data']['unfurl_media']), ('false', 'false')) - self.assertEqual((hook['json']['unfurl_links'], hook['json']['unfurl_media']), (False, False)) - for kwargs in (reply, hook): # they used to go in the headers, where Slack does not look - self.assertNotIn('unfurl_links', kwargs['headers']) + self.assertNotIn('unfurl_links', reply['headers']) # they used to go in the headers, where Slack does not look def test_a_reply_slack_refuses_is_logged_as_failed(self): """Slack refuses with HTTP 200 and ok false (the bot is not in the channel, say): a failure, logged as one.""" self.env(slackBotToken='xoxb-test') - self.patch(transports.net, 'post', lambda url, **kwargs: Resp({'ok': False, 'error': 'not_in_channel'})) + self.patch(net, 'post', lambda url, **kwargs: Resp({'ok': False, 'error': 'not_in_channel'})) with contextlib.redirect_stderr(io.StringIO()) as err: - self.assertIsNone(transports.SlackBot('C1').post_text('hi')) + self.assertIsNone(slack_common.SlackBot('C1').post_text('hi')) self.assertIn('Slack chat.postMessage to C1 failed: Slack chat.postMessage: not_in_channel', err.getvalue()) self.assertNotIn('OK', err.getvalue()) @@ -370,7 +344,7 @@ def test_a_reply_without_a_bot_token_is_logged_not_sent(self): def test_a_command_at_work_shows_an_hourglass_on_its_message(self): """While a command works, ⏳ is on the message it answers (Slack has no typing notice for bots); it comes off - once the answer is out, so the answer does not wait for it. Only with that message and the bot token.""" + once the answer is out, so the answer does not wait for it. Only with that message.""" self.env(slackBotToken='xoxb-test') self.patch(transports.Transport, 'TYPING', (0, 7, 10)) added = threading.Event() @@ -380,7 +354,7 @@ def post(url, **kwargs): if url.endswith('/reactions.add'): added.set() return Resp({'ok': True}) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) help_text = about.prepare_help self.patch(about, 'prepare_help', lambda *a: (added.wait(5), help_text(*a))[1]) # a command slow enough for ⏳ worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.help', BOT, 'U', '1.2')) @@ -388,16 +362,14 @@ def post(url, **kwargs): ['reactions.add', 'chat.postMessage', 'reactions.remove']) reaction = {'channel': 'C1', 'timestamp': '1.2', 'name': 'hourglass_flowing_sand'} self.assertEqual((self.posts[0][1]['data'], self.posts[2][1]['data']), (reaction, reaction)) - self.assertFalse(transports.SlackBot('C1').shows_typing) # no message to put it on - self.env(slackBotToken='') - self.assertFalse(transports.SlackBot('C1', '1.2').shows_typing) + self.assertFalse(slack_common.SlackBot('C1').shows_typing) # no message to put it on def test_an_hourglass_slack_refuses_is_tried_once(self): """⏳ that Slack refuses (a missing scope) is logged and not tried again, and nothing is taken off.""" self.env(slackBotToken='xoxb-test') refuse = Resp({'ok': False, 'error': 'missing_scope'}) - self.patch(transports.net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or refuse) - chat = transports.SlackBot('C1', '1.2') + self.patch(net, 'post', lambda url, **kwargs: self.posts.append((url, kwargs)) or refuse) + chat = slack_common.SlackBot('C1', '1.2') with contextlib.redirect_stderr(io.StringIO()) as err: chat.typing() chat.typing() @@ -411,13 +383,13 @@ def test_the_slack_api_says_what_is_missing(self): with self.assertRaisesRegex(RuntimeError, 'Slack files.getUploadURLExternal: needs slackBotToken in finbot.env'): - transports.slack_api('files.getUploadURLExternal', filename='chart.png', length=1) # it was a TypeError + slack_common.call('files.getUploadURLExternal', filename='chart.png', length=1) # it was a TypeError self.assertEqual(self.posts, []) class SlackButtons(FinbotCase): """A chart's period buttons on Slack: a Block Kit message (the report, the chart uploaded into no channel, the - buttons) that a press redraws in place, where the presses reach finbot (the bot token and slackOutgoingWebhook).""" + buttons) that a press redraws in place, in either mode.""" UPLOAD_URL = 'https://files.slack.com/upload/v1/abc' def setUp(self): @@ -427,7 +399,7 @@ def setUp(self): self.patch(transports.Transport, 'chart', REAL_CHART) self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') self.posts, self.refusals, self.waits = [], {}, [] - self.patch(transports.time, 'sleep', self.waits.append) + self.patch(slack_common.time, 'sleep', self.waits.append) def post(url, **kwargs): """Record the post and answer it: a refusal set for the method, while there is one left, else success.""" self.posts.append((url, kwargs)) @@ -438,7 +410,7 @@ def post(url, **kwargs): return Resp(None, text='OK - 3') return Resp({'ok': True, 'upload_url': self.UPLOAD_URL, 'file_id': 'F1', 'files': [{'id': 'F1'}], 'ts': '1.5'}) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) def calls(self): """The methods posted, in order (the upload as 'upload').""" @@ -481,14 +453,14 @@ def test_a_press_redraws_the_chart_in_place(self): def uploading(self): """Answer each upload with a new file id, F1 first, as Slack does.""" uploads = iter(f'F{n}' for n in range(1, 10)) - record = transports.net.post + record = net.post def post(url, **kwargs): """Answer as Slack does, with a new file id for each upload.""" answer = record(url, **kwargs) if url.endswith('/files.getUploadURLExternal'): return Resp({'ok': True, 'upload_url': self.UPLOAD_URL, 'file_id': next(uploads)}) return answer - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) def test_a_period_shown_again_within_15_minutes_shows_its_upload_again(self): """A press for a period whose chart was uploaded in the last 15 minutes, for this message or another, shows @@ -506,7 +478,7 @@ def test_a_period_shown_again_within_15_minutes_shows_its_upload_again(self): blocks = self.blocks('chat.update') self.assertEqual(blocks[1]['slack_file'], {'id': 'F1'}) self.assertTrue(blocks[0]['text']['text'].startswith('*≥ 5% at close')) - with state.edit(transports.SLACK_CHARTS, {}) as noted: # 16 minutes on + with state.edit(slack_common.SLACK_CHARTS, {}) as noted: # 16 minutes on for upload in noted['uploads'].values(): upload[0] -= 16 * 60 + 1 worker.process_callback(inbound.Press('slack', 't1', 'C1', '1.5', 'c|l|d|5,0')) # 1D again: F3 @@ -528,10 +500,10 @@ def test_an_upload_slack_will_not_show_again_is_uploaded_anew(self): def test_the_charts_noted_before_uploads_were_reused_are_left_alone(self): """The chart messages noted as before ({'channel ts': file}) are forgotten, their files never deleted.""" - state.save(transports.SLACK_CHARTS, {'C1 1.5': 'F0'}) + state.save(slack_common.SLACK_CHARTS, {'C1 1.5': 'F0'}) worker.process_callback(inbound.Press('slack', 't1', 'C1', '1.5', 'c|l|w|5,0')) self.assertNotIn('files.delete', self.calls()) - self.assertEqual(state.load(transports.SLACK_CHARTS)['shown'], {'C1 1.5': 'F1'}) + self.assertEqual(state.load(slack_common.SLACK_CHARTS)['shown'], {'C1 1.5': 'F1'}) def test_a_chart_not_ready_to_show_is_shown_a_moment_later(self): """Slack refusing the blocks of a chart just uploaded (invalid_blocks) is asked again after a wait.""" @@ -549,31 +521,33 @@ def test_buttons_slack_will_not_show_leave_a_plain_chart(self): self.assertEqual(self.posts[-1][1]['data']['channel_id'], 'C1') self.assertEqual((self.waits, self.sent, self.errors), ([1, 2, 4], [], [])) - def test_without_slackOutgoingWebhook_a_chart_has_no_buttons(self): - """Without slackOutgoingWebhook, no press could reach finbot, so a chart goes without buttons.""" - self.env(slackOutgoingWebhook='') + def test_in_socket_mode_a_chart_has_its_buttons_too(self): + """In Socket Mode (no slackOutgoingWebhook) a press reaches finbot over the socket, so a chart has its buttons + as in http mode.""" + self.env(slackOutgoingWebhook='', slackAppToken='xapp-1') worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.price', BOT, 'U')) - self.assertEqual(self.calls(), ['files.getUploadURLExternal', 'upload', 'files.completeUploadExternal']) - self.assertEqual(self.posts[-1][1]['data']['channel_id'], 'C1') + self.assertEqual(self.calls(), ['files.getUploadURLExternal', 'upload', 'files.completeUploadExternal', + 'chat.postMessage']) + self.assertEqual(self.blocks('chat.postMessage')[-1]['type'], 'actions') def test_a_long_report_takes_several_sections(self): """A report longer than a section takes holds is split into sections, and an empty one takes none.""" caption = '\n'.join('x' * 99 for _ in range(70)) # 6,999 characters - blocks = transports.chart_blocks(caption, [], 'F1') + blocks = slack_common.chart_blocks(caption, [], 'F1') self.assertEqual([block['type'] for block in blocks], ['section'] * 3 + ['image']) self.assertTrue(all(len(block['text']['text']) <= 3000 for block in blocks[:3])) - self.assertEqual([block['type'] for block in transports.chart_blocks('', [], 'F1')], ['image']) + self.assertEqual([block['type'] for block in slack_common.chart_blocks('', [], 'F1')], ['image']) def test_a_chart_message_that_times_out_is_not_posted_again(self): """A chart with its buttons whose chat.postMessage times out may have arrived: no plain copy follows.""" - record = transports.net.post + record = net.post def post(url, **kwargs): """Time out on chat.postMessage, as a slow Slack does.""" if url.endswith('/chat.postMessage'): self.posts.append((url, kwargs)) - raise transports.requests.exceptions.ReadTimeout('slow') + raise requests.exceptions.ReadTimeout('slow') return record(url, **kwargs) - self.patch(transports.net, 'post', post) + self.patch(net, 'post', post) with contextlib.redirect_stderr(io.StringIO()): worker.process_request(inbound.Message('slack', 'C1', '<@U1>', '.price', BOT, 'U')) self.assertEqual(self.calls(), ['files.getUploadURLExternal', 'upload', 'files.completeUploadExternal', @@ -581,5 +555,84 @@ def post(url, **kwargs): self.assertEqual(self.errors, []) + +class EveryNetworkAndMode(FinbotCase): + """One rule for the six networks and modes, in a channel and in a DM: a chart has its period buttons and a .jobs + run list its buttons, a command shows the network's notice that finbot is at work, and the reply goes to the chat + that asked.""" + MODES = { + ('telegram', 'http'): {'telegramOutgoingWebhook': 'https://example.com/telegram'}, + ('telegram', 'pull'): {'telegramOutgoingWebhook': ''}, + ('slack', 'http'): {'slackBotToken': 'xoxb-1', 'slackOutgoingWebhook': 'https://example.com/slack'}, + ('slack', 'pull'): {'slackBotToken': 'xoxb-1', 'slackAppToken': 'xapp-1', 'slackOutgoingWebhook': ''}, + ('discord', 'http'): {'discordBotToken': 'x', 'discordOutgoingWebhook': 'https://example.com/discord', + 'discordPublicKey': 'ab' * 32}, + ('discord', 'pull'): {'discordBotToken': 'x', 'discordOutgoingWebhook': ''}, + } + CHATS = {'telegram': ('-1001', '55'), 'slack': ('C1', 'D1'), 'discord': ('77', '88')} # a channel, then a DM + + def setUp(self): + """A job in each chat for .jobs run to list, and what each command posts noted with its destination.""" + super().setUp() + self.patch(discord_common.READER, 'bot_id', '99') + every = [{'when': '*/20 * * * Mon-Fri', 'reports': 'trades', 'zone': 'UTC', 'on': True}] + state.save(jobs.SCHEDULE_FILE, {f'{network}:{chat}': every for network, chats in self.CHATS.items() + for chat in chats}) + scheduler = jobs.Scheduler() + with contextlib.redirect_stderr(io.StringIO()): + scheduler.load() + self.patch(jobs, 'current', scheduler) + self.posted = [] + self.patch(transports.Transport, 'chart', lambda dest, image, caption, buttons=None: + self.posted.append((dest, buttons)) or {'ok': True}) + self.patch(transports.Transport, 'text', lambda dest, lines, buttons=None: self.posted.append((dest, buttons)) + or []) + self.updates = 0 + + def update(self, network, mode, chat, text): + """The update the network sends for a message in this mode: Discord's http mode has only the slash command.""" + self.updates += 1 + n, person = self.updates, {'id': '5', 'username': 'rob', 'first_name': 'Rob'} + if network == 'telegram': + message = {'message_id': n, 'chat': {'id': int(chat)}, 'from': person, 'text': text} + return {'update_id': n, 'message': message} + if network == 'slack': + return {'type': 'event_callback', 'event_id': f'Ev{n}', 'authorizations': [{'user_id': 'UBOT'}], + 'event': {'type': 'message', 'channel': chat, 'user': 'U1', 'text': text, 'ts': f'{n}.1'}} + if mode == 'pull': + return {'id': str(700 + n), 'channel_id': chat, 'author': person, 'content': text} + name, _, arguments = text.removeprefix('.').partition(' ') + options = [{'name': 'arguments', 'type': 3, 'value': arguments}] if arguments else [] + who = {'user': person} if chat == '88' else {'member': {'user': person}} + return {'id': str(900 + n), 'application_id': '42', 'token': f't{n}', 'type': 2, 'channel_id': chat, **who, + 'data': {'name': name, 'options': options}} + + def test_every_network_in_both_modes_in_a_channel_and_a_dm(self): + """A chart and a .jobs run list each come with their buttons, in the chat that asked; the command showed the + notice that finbot is at work: "typing…" (Telegram, a Discord message), ⏳ on the message (Slack), or Discord's + "thinking…" for a slash command, which its answer replaces.""" + for (network, mode), keys in self.MODES.items(): + for chat in self.CHATS[network]: + with self.subTest(network=network, mode=mode, chat=chat): + self.env(**keys) + self.posted.clear(); self.typing.clear() + answers = [] + with contextlib.redirect_stderr(io.StringIO()): + for text in ('.price', '.jobs run'): + answer = bot.take(READERS[network], self.update(network, mode, chat, text)) + answers.append(answer) + if answer.then: + answer.then() + self.join_workers() + self.assertEqual(self.errors, []) + self.assertEqual([(dest.where, bool(buttons)) for dest, buttons in self.posted], [(chat, True)] * 2) + if network == 'discord' and mode == 'http': + self.assertEqual([json.loads(answer.body) for answer in answers], [{'type': 5}] * 2) + self.assertTrue(all(dest.interaction and not dest.shows_typing for dest, _ in self.posted)) + else: + self.assertEqual(len(self.typing), 2) + self.assertTrue(all(dest.shows_typing for dest in self.typing)) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_telegram_api.py b/tests/test_telegram_api.py index 4c2ef64d..a8e5b0e6 100644 --- a/tests/test_telegram_api.py +++ b/tests/test_telegram_api.py @@ -1,36 +1,50 @@ -"""Telegram's Bot API: the webhook registered once, as bot.py starts.""" -import contextlib, io, os, subprocess, unittest +"""Telegram's Bot API: the webhook set or deleted once, as bot.py starts, and the command menu.""" +import contextlib, io, os, re, subprocess, unittest +from unittest import mock from tests.common import * +from lib import telegram_longpolling, telegram_webhook class TelegramWebhook(FinbotCase): - """bot.py registers the webhook once as it starts; importing lib.telegram, as each job does, does nothing.""" + """bot.py registers the webhook once as it starts (or deletes it, for long polling); importing does nothing.""" def setUp(self): """A Telegram bot with an outgoing webhook configured.""" super().setUp() - self.env(webhooks={'telegram': 'https://api.telegram.org/botX/'}) + self.env(telegram_api='https://api.telegram.org/botX/') self.env(telegramOutgoingWebhook='https://example.com/telegram') - def test_register_unregisters_registers_and_names_the_bot(self): - """register() removes the webhook, sets it again and returns the bot's @name.""" + def test_register_sets_the_webhook_again_or_deletes_it_and_names_the_bot(self): + """register() deletes the webhook, keeping the updates not delivered yet, and sets it again with + telegramOutgoingWebhook (telegram_webhook, http mode), or leaves it unset for getUpdates (telegram_longpolling, + pull mode); bot_name() is the bot's @name. The log never shows the URL: its path is a secret.""" calls = [] def post(url, **kwargs): - """Record the method and URL, and answer like Telegram.""" - calls.append((url.rsplit('/', 1)[1], kwargs.get('params', {}).get('url'))) + """Record the method and what it was given, and answer like Telegram.""" + calls.append((url.rsplit('/', 1)[1], kwargs.get('params', {}).get('url') or kwargs.get('json'))) return Resp({'ok': True, 'result': {'username': 'finbot'} if url.endswith('getMe') else True}) - self.patch(telegram.net, 'post', post) - with contextlib.redirect_stderr(io.StringIO()): - self.assertEqual(telegram.register(), '@finbot') - self.assertEqual(calls, [('setWebhook', ''), ('setWebhook', 'https://example.com/telegram'), ('getMe', None)]) + self.patch(net, 'post', post) + kept = ('deleteWebhook', {'drop_pending_updates': False}) + for hook, mode, expected in (('https://example.com/telegram', telegram_webhook, + [kept, ('setWebhook', 'https://example.com/telegram'), ('getMe', None)]), + ('', telegram_longpolling, [kept, ('getMe', None)])): + with self.subTest(telegramOutgoingWebhook=hook): + self.env(telegramOutgoingWebhook=hook) + calls.clear() + err = io.StringIO() + with contextlib.redirect_stderr(err): + mode.register() + self.assertEqual(telegram_common.bot_name(), '@finbot') + self.assertEqual(calls, expected) + self.assertNotIn('example.com', err.getvalue()) def test_a_refused_call_raises(self): """A call Telegram refuses raises with its description.""" - self.patch(telegram.net, 'post', + self.patch(net, 'post', lambda url, **k: Resp({'ok': False, 'description': 'Bad Request: bad webhook: HTTPS url must be provided'})) with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(RuntimeError) as caught: - telegram.register() + telegram_webhook.register() self.assertIn('HTTPS url must be provided', str(caught.exception)) def test_edits_toasts_and_typing_go_through_call(self): @@ -47,27 +61,27 @@ def post(url, **kwargs): refused = Resp({'ok': False, 'description': 'Bad Request: message to edit not found'}) unchanged = Resp({'ok': False, 'description': 'Bad Request: message is not modified: specified new message content is the same'}) - chat = transports.TelegramBot('5') + chat = telegram_common.TelegramBot('5') for outcome, raises in ((refused, True), (unchanged, False), (ConnectionError('down'), False), (Resp({'ok': True, 'result': True}), False)): with self.subTest(outcome=outcome): - self.patch(telegram.net, 'post', answer(outcome)) + self.patch(net, 'post', answer(outcome)) with contextlib.redirect_stderr(io.StringIO()): if raises: with self.assertRaisesRegex(RuntimeError, 'not found'): chat.edit_chart('9', io.BytesIO(b'png'), 'caption') else: chat.edit_chart('9', io.BytesIO(b'png'), 'caption') - telegram.answerCallbackQuery('1', 'Loading…') + telegram_common.answerCallbackQuery('1', 'Loading…') chat.edit_text('9', 'text') chat.typing() def test_importing_never_calls_telegram(self): - """Importing the modules a job process uses makes no request.""" + """Importing Telegram's modules, and those they work with, makes no request.""" code = ('import requests\n' 'def refuse(*a, **k): raise SystemExit("network call on import")\n' 'requests.Session.request = refuse\n' - 'from lib import telegram, yahoo, worker, jobrun\n' + 'from lib import telegram_common, telegram_longpolling, telegram_webhook, yahoo, worker, jobs\n' 'print("quiet")') with tempfile.TemporaryDirectory() as data: out = subprocess.run([sys.executable, '-c', code], cwd=ROOT, capture_output=True, text=True, @@ -75,12 +89,39 @@ def test_importing_never_calls_telegram(self): 'telegramOutgoingWebhook': 'https://example.com/telegram'}) self.assertEqual(out.stdout.split()[-1:], ['quiet'], out.stderr) + def test_the_command_menu_is_set_only_when_it_differs(self): + """The menu is the slash commands, each with its usage, without their second names (/profile, /job: typed, they + work as any name does), set only when Telegram's menu differs; a failure is only logged.""" + calls, menu = [], [] + def call(method, **kwargs): + """Telegram's menu, as getMyCommands and setMyCommands see it.""" + calls.append(method) + if method == 'setMyCommands': + menu[:] = kwargs['json']['commands'] + return True + return list(menu) + self.patch(telegram_common, 'call', call) + with contextlib.redirect_stderr(io.StringIO()) as log: + telegram_common.set_commands(worker.slash_commands(aliases=False)) + telegram_common.set_commands(worker.slash_commands(aliases=False)) + self.assertEqual(calls, ['getMyCommands', 'setMyCommands', 'getMyCommands']) + self.assertIn(f'telegram: {len(menu)} commands in the menu', log.getvalue()) + names = [entry['command'] for entry in menu] + self.assertIn({'command': 'forwardpe', 'description': 'forwardpe [SYMBOL|top|bottom|negative]'}, menu) + self.assertEqual([name in names for name in ('stock', 'jobs', 'profile', 'job')], [True, True, False, False]) + self.assertTrue(all(re.fullmatch(r'[a-z0-9_]{1,32}', name) for name in names)) + self.patch(telegram_common, 'call', mock.Mock(side_effect=RuntimeError('Telegram getMyCommands: Unauthorized'))) + with contextlib.redirect_stderr(io.StringIO()) as log: + telegram_common.set_commands(worker.slash_commands(aliases=False)) + self.assertIn('Telegram setMyCommands failed: Telegram getMyCommands: Unauthorized', log.getvalue()) + def test_the_bot_will_not_start_unregistered_and_never_logs_the_token(self): """bot.py exits when the webhook cannot be registered, and never logs the token.""" skip_if_old_env(self) with tempfile.TemporaryDirectory() as data: env = {k: v for k, v in bare_environment().items() if 'proxy' not in k.lower()} - env.update(DATA_DIR=data, port='0', scheduler='False', telegramBotToken='123456:SECRET-token', + no_jobs(data) + env.update(DATA_DIR=data, port='0', telegramBotToken='123456:SECRET-token', telegramOutgoingToken='tok', telegramOutgoingWebhook='https://example.com/telegram', # unreachable, so the registration fails at once and offline @@ -88,7 +129,7 @@ def test_the_bot_will_not_start_unregistered_and_never_logs_the_token(self): out = subprocess.run([sys.executable, 'bot.py'], cwd=ROOT, env=env, capture_output=True, text=True, timeout=120) self.assertEqual(out.returncode, 1, out.stderr) - self.assertIn('ERROR ⚠️ telegram webhook:', out.stderr) + self.assertIn('ERROR ⚠️ telegram:', out.stderr) self.assertNotIn('SECRET-token', out.stderr + out.stdout) diff --git a/tests/test_util.py b/tests/test_util.py index 5965c47f..d62d91de 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -19,6 +19,17 @@ def test_days_from_human_days_rejects_non_periods(self): with self.subTest(arg=arg), self.assertRaises(ValueError): formats.days_from_human_days(arg) + def test_seconds_from_human(self): + """A time in seconds from 180s or 24h, or any period in days (7d, 2w, 3m, 1y, 7); anything else is a + ValueError.""" + for arg, seconds in (('180s', 180), ('24H', 86400), ('7d', 7 * 86400), ('2w', 14 * 86400), ('3m', 90 * 86400), + ('1y', 365 * 86400), ('7', 7 * 86400), ('0s', 0)): + with self.subTest(arg=arg): + self.assertEqual(formats.seconds_from_human(arg), seconds) + for arg in ('', 's', 'h5', '1.5h', '-5s', '5x', 'AAPL'): + with self.subTest(arg=arg), self.assertRaises(ValueError): + formats.seconds_from_human(arg) + def test_transform_to_yahoo(self): """Sharesight and Google tickers as Yahoo's: suffixes by market, none for US venues, a dash for class shares.""" for args, expected in ((('AAPL',), 'AAPL'), (('FANG', 'ASX'), 'FANG.AX'), (('ARM', 'NASDAQ'), 'ARM'), @@ -83,6 +94,30 @@ def test_transform_title(self): with self.subTest(title=title): # a suffix comes off the end only; no run of spaces is left behind self.assertEqual(formats.transform_title(title), expected) + def test_a_name_in_a_list_is_cut_to_fit_a_phone(self): + """A name over 30 characters is cut at a word, with an ellipsis (mid-word where a word break would cut it too + short); a shorter one stays whole.""" + for name, shown in (('Global X Asia Semiconductor ETF', 'Global X Asia Semiconductor…'), + ('Global X China EV ETF', 'Global X China EV ETF'), ('A' * 40, 'A' * 29 + '…'), + ('Short Supercalifragilisticexpialidocious', 'Short Supercalifragilisticexp…')): + with self.subTest(name=name): + self.assertEqual(formats.short_name(name), shown) + self.assertLessEqual(len(formats.short_name(name)), formats.NAME_LIMIT) + + def test_a_funds_name_loses_its_trusts(self): + """A fund's name loses the trust's before it, a repeated brand or ETF, and OFC; a company's repeated word, or a + fund's own Trust, stays.""" + for title, expected in (('Global X ETF Series OFC Global X China EV ETF', 'Global X China EV ETF'), + ('iShares Trust iShares Core S&P 500 ETF', 'iShares Core S&P 500 ETF'), + ('SPDR Series Trust SPDR S&P Biotech ETF', 'SPDR S&P Biotech ETF'), + ('Betashares ETF Trust Global Cybersecurity ETF', 'Betashares Global Cybersecurity ETF'), + ('Global X Uranium ETF ETF', 'Global X Uranium ETF'), + ('VanEck Semiconductor ETF', 'VanEck Semiconductor ETF'), + ('SPDR S&P 500 ETF Trust', 'SPDR S&P 500 ETF Trust'), ('Johnson & Johnson', 'Johnson & Johnson'), + ('Johnson & Johnson Health', 'Johnson & Johnson Health')): + with self.subTest(title=title): + self.assertEqual(formats.transform_title(title), expected) + class Formatting(unittest.TestCase): def test_signed_percent(self): @@ -120,6 +155,17 @@ def test_button_data_reads_back(self): with self.subTest(data=data): self.assertIsNone(buttons.parse(data)) + def test_a_press_is_told_what_is_coming_from_its_data_alone(self): + """The toast a press gets as it arrives: the period a period button loads, 'Running…' for any .jobs button + (the worker checks it later), and none for data of neither.""" + self.assertEqual([buttons.toast(button.data) for button in buttons.keyboard('l', ['5', '0'])[0]], + ['Loading 1D…', 'Loading 7D…', 'Loading 1M…', 'Loading 1Y…', 'Loading Max…']) + self.assertEqual(buttons.toast(buttons.job_button(8, 'trades', 'abcd').data), 'Running…') + self.assertEqual(buttons.toast('j|99|0000'), 'Running…') + for data in ('c|f|d|', 'c|zz|m|T1', '', 'x'): + with self.subTest(data=data): + self.assertIsNone(buttons.toast(data)) + def test_strip_url(self): """A link's text is its address without the scheme or www.""" for url, text in (('https://www.apple.com', 'apple.com'), ('http://example.org/a?b', 'example.org/a?b'), @@ -255,12 +301,20 @@ def test_binary_cache_roundtrip(self): self.assertEqual(cache.read_bytes('img.png').read(), b'png-bytes') def test_holdings_and_watchlist_without_a_watchlist_file(self): - """On the first run there is no watchlist file: the holdings alone, GOOGL dropped when GOOG is held.""" + """On the first run there is no watchlist file: the holdings alone, GOOGL dropped when GOOG is held. A chat's + watchlist adds to the holdings in that chat only; yahoo_warm keeps every chat's warm.""" self.patch(sharesight, 'tickers', lambda: ['AAPL', 'GOOG', 'GOOGL']) self.patch(watchlist, 'with_holdings', REAL_GET_HOLDINGS) - self.assertEqual(watchlist.with_holdings(), ['AAPL', 'GOOG']) # GOOGL dropped when GOOG is present - state.save('finbot_watchlist.json', ['MSFT']) - self.assertEqual(watchlist.with_holdings(), ['AAPL', 'GOOG', 'MSFT']) + self.assertEqual(watchlist.with_holdings('telegram:1'), ['AAPL', 'GOOG']) # GOOGL dropped when GOOG is present + self.assertEqual((watchlist.all_watched(), watchlist.all_followed()), ([], ['AAPL', 'GOOG'])) + watchlist.add('telegram:1', 'MSFT') + watchlist.add('slack:C1', 'NVDA') + watchlist.add('slack:C1', 'MSFT') + self.assertEqual(watchlist.with_holdings('telegram:1'), ['AAPL', 'GOOG', 'MSFT']) + self.assertEqual(watchlist.with_holdings('slack:C1'), ['AAPL', 'GOOG', 'MSFT', 'NVDA']) + self.assertEqual(watchlist.with_holdings('discord:7'), ['AAPL', 'GOOG']) + self.assertEqual(watchlist.all_watched(), ['MSFT', 'NVDA']) + self.assertEqual(watchlist.all_followed(), ['AAPL', 'GOOG', 'MSFT', 'NVDA']) if __name__ == '__main__': diff --git a/tests/test_yahoo.py b/tests/test_yahoo.py index 1511a4aa..f179b293 100644 --- a/tests/test_yahoo.py +++ b/tests/test_yahoo.py @@ -130,6 +130,19 @@ def test_a_symbol_yahoo_does_not_know_is_empty_and_an_outage_an_error(self): with self.assertRaises(RuntimeError): yahoo.fetch_detail('TSLA') + def test_a_fund_without_an_analysts_recommendation_parses(self): + """An ETF's quoteSummary has no analysts' recommendation (recommendationKey left out, or null), or a garbled + one: it parses without one, where .earnings failed on it ('NoneType' object has no attribute 'replace').""" + for ticker, financial in (('ETPMAG.AX', {}), ('MAGS', {'recommendationKey': None}), ('SMH', { + 'recommendationKey': 5, 'recommendationMean': {'raw': 2.1}, 'numberOfAnalystOpinions': {'raw': 9}})): + with self.subTest(ticker=ticker, financialData=financial): + body = self.combined() + body['quoteSummary']['result'][0]['financialData'] = financial + self.http(yahoo, Resp(body)) + data = yahoo.fetch_detail(ticker)[ticker] + self.assertIn('regularMarketPrice', data) + self.assertEqual([key for key in yahoo.RECOMMENDATION if key in data], []) + def test_missing_names_give_empty(self): """A detail without a name, or without a result, is empty.""" body = self.combined() @@ -264,7 +277,8 @@ def get(url, **kwargs): return Resp(status=404) if '/chart/T3?' in url or '/chart/SPY?' in url else good self.patch(yahoo.net, 'get', get) def slack(text): - """Run a command on Slack, where reports come as text, and return the text.""" + """Run a command on Slack, its report as text (no charts), and return the text.""" + self.prefs(graph=False) self.sent.clear(); self.errors.clear() worker.process_request(inbound.Message('slack', 'C1', '<@U1>', text, BOT, 'U')) self.assertEqual(self.errors, [], text) diff --git a/trades.py b/trades.py deleted file mode 100755 index d9f3cc9f..00000000 --- a/trades.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./trades.py [arguments]: runs lib/reports/trades.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__) diff --git a/util/check_config.py b/util/check_config.py index 75da666d..667ffcd7 100755 --- a/util/check_config.py +++ b/util/check_config.py @@ -6,7 +6,7 @@ finbot.env (the startup file) is never read by the bot, so this is the only place its typos are caught: an unknown key, a preference key in it, a malformed line or a bad value. preferences.conf defaults to /preferences.conf and is checked the way the bot checks it at start and on reload. Exits 1 and prints every problem if either file is wrong, so a -launcher can refuse to start (see the README).""" +launcher can refuse to start (see doc/install.md and doc/docker.md).""" import argparse import os @@ -39,14 +39,30 @@ def check(startup_path, preferences_path): return problems -def notes(preferences_path): - """The keys of a valid preferences.conf that do nothing any more, a line each.""" +def warnings(startup_path): + """What a valid finbot.env sets that keeps part of the bot from working or does nothing, a line each: what bot.py + logs at start.""" try: - return config.retired_notes(config.read_preferences(preferences_path) or {}) + values = config.read_startup_file(startup_path) + return config.startup_warnings(config.startup_values(lambda name, default=None: values.get(name, default))) except config.ConfigError: return [] +def notes(preferences_path, startup_path=None): + """The keys of the two files, where valid, that do nothing any more, a line each with what to do instead.""" + lines = [] + if startup_path: + try: + lines += config.retired_notes(config.read_startup_file(startup_path), 'finbot.env') + except config.ConfigError: + pass + try: + return lines + config.retired_notes(config.read_preferences(preferences_path) or {}) + except config.ConfigError: + return lines + + if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('startup', help='the finbot.env to check (the startup file)') @@ -59,6 +75,8 @@ def notes(preferences_path): if not problems: print('config: ok', args.startup, args.preferences if Path(args.preferences).is_file() else '(no preferences.conf, defaults)') - for line in notes(args.preferences): + for line in warnings(args.startup): + print('config: warning:', line) + for line in notes(args.preferences, args.startup): print('config:', line) sys.exit(1 if problems else 0) diff --git a/util/docker_setup.sh b/util/docker_setup.sh new file mode 100755 index 00000000..312d5fdb --- /dev/null +++ b/util/docker_setup.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# The host side of finbot's Docker install, set up once, as root; safe to run again (doc/docker.md). Beside the +# checkout, where compose.yaml looks for them: ../finbot-data (DATA_DIR) made, the defaults copied in, owned by the +# container's user (10001); and ../finbot.env from the template if there is none, readable by root and the container's +# group only. Cloned to /opt/finbot, they are /opt/finbot-data and /opt/finbot.env. +set -eu + +repo=$(cd "$(dirname "$0")/.." && pwd) +parent=$(dirname "$repo") +data="$parent/finbot-data" +startup="$parent/finbot.env" + +if [ "$(id -u)" -ne 0 ]; then + echo "docker_setup: run it as root: sudo $0" >&2 + exit 1 +fi + +python3 "$repo/util/sync_defaults.py" --create --apply --dest "$data" +chown -R 10001:10001 "$data" + +if [ ! -e "$startup" ]; then + cp "$repo/defaults/finbot.env" "$startup" + echo "docker_setup: $startup written from the template" +fi +chown root:10001 "$startup" +chmod 640 "$startup" + +cat <_common.py, or in a commit from before the + protocol code was split per network, lib/inbound.py's TELEGRAM, SLACK or DISCORD (None before the readers).""" + try: + return importlib.import_module(f'lib.{service}_common').READER + except ImportError: + return getattr(getattr(common, 'inbound', None), service.upper(), None) + + def pressing(service, *fields): + """A step that presses a button: acknowledged first where the tree's bot.take does that (Network.acknowledge), + then handed to the worker as handing() does.""" + run = handing(worker.process_callback, 'Press', service, *fields) + network = reader(service) + if not hasattr(network, 'acknowledge'): + return run + def press(): + """Acknowledge the press, then hand it to the worker.""" + network.acknowledge(common.inbound.Press(service, *fields)) + run() + return press + class Record(common.FinbotCase): """The steps, as one test, so that FinbotCase's fixtures (and their cleanup) apply.""" def runTest(self): """Record what each step posts: the real sending code runs, and net.post notes each request.""" posts = [] - def post(url, **kwargs): - """Note a request, the typing notices aside (Telegram's, Slack's ⏳: timing, not output), and answer - like the APIs do.""" - if 'sendChatAction' in url or '/reactions.' in url: + def post(url, method='POST', **kwargs): + """Note a request, the typing notices aside (Telegram's, Slack's ⏳, Discord's: timing, not output), and + answer like the APIs do.""" + if 'sendChatAction' in url or '/reactions.' in url or url.endswith('/typing'): return common.mock.Mock(status_code=200, json=lambda: {'ok': True, 'result': True}) - entry = {'url': url.replace('botX/', 'bot/')} + entry = {'url': url.replace('botX/', 'bot/'), **({'method': method} if method != 'POST' else {})} for key in ('json', 'data', 'params'): body = kwargs.get(key) if isinstance(body, dict): @@ -85,10 +109,25 @@ def post(url, **kwargs): answer = {**ANSWER, 'result': {'message_id': len(posts), 'chat': {'id': 5}}} return common.mock.Mock(status_code=200, json=lambda: answer, text='OK') self.patch(net, 'post', post) + if hasattr(net, 'request'): # Discord's calls, since the Discord bot + self.patch(net, 'request', lambda method, url, **kwargs: post(url, method, **kwargs)) for target, name, real in real_sending: # what the fixture replaces with recorders, put back self.patch(target, name, real) self.patch(common.time, 'sleep', lambda seconds: None) # no pauses between a long report's messages self.env(slackBotToken='xoxb-test', slackOutgoingWebhook='https://example.com/slack') # Slack's buttons too + if 'discordBotToken' in {field.name for field in common.dataclasses.fields(common.config.env)}: + self.env(discordBotToken='discord-test') + if hasattr(jobs, 'SCHEDULE_FILE'): # each chat's jobs, since jobs per chat + entries = [{'when': job.when, 'reports': job.text, 'zone': 'UTC', 'on': True} + for job in map(jobs.parse_job, JOBS)] + chats = ('telegram:55', 'slack:C1', 'discord:77') + common.state.save(jobs.SCHEDULE_FILE, {key: entries for key in chats}) + schedule = jobs.Scheduler() + else: # one schedule for all chats, the shipped one + schedule = jobs.Scheduler(lambda: REPO / 'defaults' / 'schedule.cron', zone=common.UTC) + with contextlib.redirect_stderr(io.StringIO()): + schedule.load() + self.patch(jobs, 'current', schedule) steps = [] def step(label, run): """Run one step and note what it posted, or what it raised.""" @@ -100,22 +139,29 @@ def step(label, run): except Exception as e: raised = f'{type(e).__name__}: {e}' steps.append({'step': label, 'posts': list(posts), 'raised': raised}) - for service, chat, user in (('telegram', '55', '@u'), ('slack', 'C1', '<@U1>')): + for service, chat, user in (('telegram', '55', '@u'), ('slack', 'C1', '<@U1>'), ('discord', '77', '<@5>')): for text in COMMANDS: step(f'{service} {text}', handing(worker.process_request, 'Message', service, chat, user, text, common.BOT, 'U')) - for service, chat, message in (('telegram', '55', '9'), ('slack', 'C1', '1.5')): + for service, chat, message in (('telegram', '55', '9'), ('slack', 'C1', '1.5'), ('discord', '77', '333')): for data in BUTTONS: label = f'button {data}' if service == 'telegram' else f'{service} button {data}' - step(label, handing(worker.process_callback, 'Press', service, 'cb', chat, message, data)) - self.env(webhooks=WEBHOOKS, telegramChatID='1') - for where in ('', ' (slackChannelID)'): # Slack's incoming webhook, then the channel as the bot - if where: - self.env(slackChannelID='C9') - for line in SCHEDULED: - label, module, kwargs = jobs.parse_command(line)[0] - run = getattr(module, 'run', None) or module.lambda_handler # lambda_handler: a commit before run() - step(f'scheduled {line}{where}', lambda: run(**kwargs)) + step(label, pressing(service, 'cb', chat, message, data)) + # the scheduled reports go to each network's home chat as its bot (a commit from before telegram_api had + # webhooks, the incoming ones among them, and one from before the Discord bot no discordChannelID) + fields = {field.name for field in common.dataclasses.fields(common.config.env)} + api = {'telegram_api': TELEGRAM_API} if 'telegram_api' in fields else { + 'webhooks': {'telegram': TELEGRAM_API}} + discord = {'discordChannelID': '78'} if 'discordChannelID' in fields else {} + self.env(telegramChatID='1', slackChannelID='C9', **api, **discord) + for line in SCHEDULED: + if hasattr(jobs, 'parse_reports'): + label, module, kwargs = jobs.parse_reports(line)[0] + else: # a commit from before jobs per chat, whose reports were scripts + name, _, arguments = line.partition(' ') + label, module, kwargs = jobs.parse_command(f'{name}.py {arguments}'.strip())[0] + run = getattr(module, 'run', None) or module.lambda_handler # lambda_handler: a commit before run() + step(f'scheduled {line}', lambda: run(**kwargs)) Path(output).write_text(json.dumps(steps, indent=1, sort_keys=True, default=str)) result = unittest.TextTestRunner(verbosity=0, stream=io.StringIO()).run(Record()) diff --git a/util/send_stdin.py b/util/send_stdin.py new file mode 100755 index 00000000..b167a5e3 --- /dev/null +++ b/util/send_stdin.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Post the lines piped in to the home chats, as the scheduled reports go: `echo hello | util/send_stdin.py`.""" +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from lib import delivery + + +if __name__ == '__main__': + if os.isatty(sys.stdin.fileno()): + sys.exit('send_stdin: pipe the lines to post into it, e.g. echo hello | util/send_stdin.py') + lines = [line.strip() for line in sys.stdin if line.strip()] + for dest in delivery.default_chats(): + delivery.deliver(dest, delivery.Report(lines)) diff --git a/util/slack_manifest.py b/util/slack_manifest.py new file mode 100755 index 00000000..4613a508 --- /dev/null +++ b/util/slack_manifest.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Print the manifest of finbot's Slack app, to paste into the app's settings (_App Manifest_, the JSON tab), which sets +it all up at once: the slash commands, from finbot's own command table; the bot's scopes and events; interactivity, for +the buttons; the messages tab of App Home, for direct messages; and Socket Mode, on or off. + + util/slack_manifest.py Socket Mode: finbot opens a socket to Slack (slackAppToken) + util/slack_manifest.py --url https://example.com/slack HTTP mode: Slack posts to slackOutgoingWebhook, this URL + util/slack_manifest.py --every-message the bot sees every message in its channels, not only the + ones that mention it, so that .price works there + +Slack makes slash commands only in an app's settings (there is no API for them without a configuration token), so this +is the way they get there: run it again, and paste what it prints, when finbot's commands change.""" + +import argparse +import json +import os +import sys +from pathlib import Path + +if __name__ == '__main__': + os.environ['FINBOT_CHECK_ONLY'] = '1' # importing lib.config must not create the data directories +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from lib import worker + +SCOPES = ('chat:write', 'commands', 'files:read', 'files:write', 'reactions:write') # replies, slash commands, charts, ⏳ +EVENT_SCOPES = {'app_mention': 'app_mentions:read', 'message.channels': 'channels:history', + 'message.groups': 'groups:history', 'message.im': 'im:history'} # what each event needs + + +def manifest(name='finbot', url='', every_message=False): + """The manifest, as a dict: for Socket Mode without url; for HTTP mode with it, the Request URL of the events, the + buttons and each slash command. The bot hears the messages that mention it, or with every_message each message in + the channels it is in, and its direct messages either way.""" + events = ['message.channels', 'message.groups'] if every_message else ['app_mention'] + events.append('message.im') + posted = {'url': url} if url else {} + commands = [{'command': '/' + command.name, 'description': command.description, 'should_escape': False, + **({'usage_hint': command.arguments} if command.arguments else {}), **posted} + for command in worker.slash_commands()] + return { + 'display_information': {'name': name, 'description': 'Stock prices, charts and Sharesight reports', + 'background_color': '#14181f'}, + 'features': { + 'app_home': {'home_tab_enabled': False, 'messages_tab_enabled': True, + 'messages_tab_read_only_enabled': False}, + 'bot_user': {'display_name': name, 'always_online': True}, + 'slash_commands': commands, + }, + 'oauth_config': {'scopes': {'bot': sorted({*SCOPES, *(EVENT_SCOPES[event] for event in events)})}}, + 'settings': { + 'event_subscriptions': {'bot_events': events, **({'request_url': url} if url else {})}, + 'interactivity': {'is_enabled': True, **({'request_url': url} if url else {})}, + 'org_deploy_enabled': False, + 'socket_mode_enabled': not url, + 'token_rotation_enabled': False, + }, + } + + +def main(argv=None): + """Print the manifest the arguments ask for; 2 for a URL that is not HTTPS, which Slack takes no other.""" + parser = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + parser.add_argument('--url', default='', help='HTTP mode: slackOutgoingWebhook, which Slack posts to') + parser.add_argument('--name', default='finbot', help="the app's and the bot's name (default finbot)") + parser.add_argument('--every-message', action='store_true', help='the bot sees every message in its channels') + args = parser.parse_args(argv) + if args.url and not args.url.startswith('https://'): + print('slack_manifest: --url must be https://: Slack posts to no other', file=sys.stderr) + return 2 + print(json.dumps(manifest(args.name, args.url, args.every_message), indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/util/split_env.py b/util/split_env.py index 9033ca96..253119e0 100755 --- a/util/split_env.py +++ b/util/split_env.py @@ -58,6 +58,8 @@ def split(text, path='.env'): notes.append(f'{path} line {number}: {key} dropped ({REMOVED[key]})') elif key == 'STATE_DIRECTORY': notes.append(f'{path} line {number}: {key} is set by systemd, left out') + elif key in config.RETIRED: + notes.append(f'{path} line {number}: {key} dropped: {config.RETIRED[key]}') elif key in config.STARTUP_KEYS: startup.append(f'{key}={value}') elif key in config.PREFERENCE_KEYS: diff --git a/util/sync_defaults.py b/util/sync_defaults.py index 081018cb..65ef4261 100755 --- a/util/sync_defaults.py +++ b/util/sync_defaults.py @@ -1,35 +1,44 @@ #!/usr/bin/env python3 """Compare the shipped defaults/ with the data dir (DATA_DIR, default var/), or with the directory given by --dest. Dry run by default. Every file is reported as copied, replaced or skipped. --apply copies files that are missing; ---force also replaces differing files (the old copy is kept as .bak).""" +--force also replaces differing files (the old copy is kept as .bak). A destination that does not exist is an error, +unless --create makes it (mode 750). It needs only the standard library, so any python3 runs it.""" import argparse import filecmp +import os import shutil import sys from pathlib import Path +if __name__ == '__main__': + os.environ['FINBOT_CHECK_ONLY'] = '1' # importing lib.config must not create the data dirs: --dest names another sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from lib import config -def sync(apply=False, force=False, dest=None): - """One (file, outcome) per default: copied / would copy, replaced (old copy kept as .bak), or skipped.""" +def sync(apply=False, force=False, dest=None, create=False): + """One (file, outcome) per default: copied / would copy, replaced (old copy kept as .bak), or skipped. A destination + that does not exist is made (mode 750) with create, when copying; without it, FileNotFoundError.""" apply = apply or force dest_dir = Path(dest or config.env.data_dir) + if not dest_dir.is_dir(): + if not create: + raise FileNotFoundError(f'{dest_dir} does not exist: --create makes it') + if apply: + dest_dir.mkdir(mode=0o750, parents=True) results = [] for src in sorted((config.REPO_DIR / 'defaults').iterdir()): if not src.is_file(): continue - if src.name == 'finbot.env': + if src.name in ('finbot.env', 'schedule.cron'): # the startup file is not data: it holds credentials, lives wherever the launcher loads it from, and the bot - # never reads it + # never reads it; the old schedule is read only to give the home chats their jobs once (lib/migrate.py) continue name = src.name target = dest_dir / name if not target.exists(): if apply: - dest_dir.mkdir(parents=True, exist_ok=True) shutil.copy2(src, target) results.append((name, 'copied' if apply else 'would copy')) elif filecmp.cmp(src, target, shallow=False): @@ -50,9 +59,15 @@ def sync(apply=False, force=False, dest=None): ap.add_argument('--apply', action='store_true', help='copy missing files') ap.add_argument('--force', action='store_true', help='also replace differing files (keeps .bak); implies --apply') ap.add_argument('--dest', metavar='DIR', help='destination directory (default: the data dir, DATA_DIR or var/)') + ap.add_argument('--create', action='store_true', help='make the destination directory (mode 750) if it is missing') args = ap.parse_args() print(f"{config.REPO_DIR / 'defaults'} -> {Path(args.dest or config.env.data_dir).resolve()}") - for name, outcome in sync(args.apply or args.force, args.force, args.dest): + try: + results = sync(args.apply or args.force, args.force, args.dest, args.create) + except FileNotFoundError as e: + print('sync_defaults:', e, file=sys.stderr) + sys.exit(2) + for name, outcome in results: label = {'copied': '', 'skipped - destination identical': '', 'skipped - destination differs': '[differs]'}.get(outcome, f'[{outcome}]') print(f"{name} {label}".rstrip()) diff --git a/yahoo_warm.py b/yahoo_warm.py deleted file mode 100755 index ed868699..00000000 --- a/yahoo_warm.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""./yahoo_warm.py [arguments]: runs lib/reports/yahoo_warm.py once, as its schedule line would.""" -if __name__ == '__main__': - from lib import reports - reports.cli(__file__)