diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh index e3468826..f442c27a 100755 --- a/.claude/hooks/session-start.sh +++ b/.claude/hooks/session-start.sh @@ -13,7 +13,8 @@ PYTHON_VERSION="$(tr -d '[:space:]' < .python-version)" uv python install "$PYTHON_VERSION" # (re)create .venv only when it is missing or on a different Python -if [ ! -x .venv/bin/python ] || [ "$(.venv/bin/python -c 'import platform; print(platform.python_version())')" != "$PYTHON_VERSION" ]; then +if [ ! -x .venv/bin/python ] || + [ "$(.venv/bin/python -c 'import platform; print(platform.python_version())')" != "$PYTHON_VERSION" ]; then rm -rf .venv uv venv --python "$PYTHON_VERSION" .venv fi diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..1951c484 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true + +[*.py] +indent_style = tab +indent_size = 4 + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9a0f38b..080ef50d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,8 @@ jobs: if [ "$GITHUB_EVENT_NAME" = pull_request ]; then files=$(git -c core.quotePath=false diff --name-only --no-renames "origin/$BASE_REF...HEAD") echo "$files" - if [ -n "$files" ] && ! grep -qvE '^(README\.md|CLAUDE\.md|img/[^/]+\.png)$' <<< "$files"; then + docs='^(README\.md|CLAUDE\.md|doc/[^/]+\.md|img/[^/]+\.png)$' + if [ -n "$files" ] && ! grep -qvE "$docs" <<< "$files"; then docs_only=true echo "::notice::Only docs changed: install, compile and tests are skipped" fi @@ -48,7 +49,7 @@ jobs: run: | pip install -r requirements.txt python -m compileall -q *.py lib util - python -m unittest discover -s tests -v + python -m unittest discover -s tests -v -b notify: # Telegram alert when the weekly run fails. Needs repository secrets TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID. diff --git a/.gitignore b/.gitignore index 4e27a261..dea55888 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .* !.gitignore +!.editorconfig !.python-version !.claude !.github diff --git a/CLAUDE.md b/CLAUDE.md index 9b1ea0ab..597836fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## Iterating (auto mode) The owner chats back and forth; every message that asks for a change gets a change. -1. Make the change, run the tests (`.venv/bin/python -m unittest discover -s tests`), fix what breaks. +1. Make the change, run the tests (`.venv/bin/python -m unittest discover -s tests -b`), fix what breaks. 2. Commit (with the attribution trailer) and **push to the work branch** (`git push -u origin `). Do this every iteration, so the branch always holds the latest working state. If a PR from the work branch is open, update its title and description in the same iteration (see below). 3. Reply briefly: what changed, test result, and the pushed commit's short hash. @@ -18,7 +18,98 @@ An open PR takes in every commit pushed to its branch, so a PR opened for one sm - To merge: `main` takes only a branch that is up to date with it and green on both CI jobs. Merge `origin/main` into the work branch (a merge commit, never a rebase), run the tests, push, wait for CI on that head, then merge the PR with a merge commit. ## Facts -- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, except for pull requests that change only README.md, CLAUDE.md or `img/*.png` (install, compile and tests are skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). -- Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); `requirements.txt` is for pip/uv users and CI. -- A new preference goes in five places: `PREFERENCE_KEYS` and `preference_values` (`lib/config.py`), `defaults/preferences.conf`, the README's preferences table and its key count, and `DEFAULTS` plus the key count in `tests/test_config.py`. The tests catch all but the README. -- If Yahoo and Sharesight are unreachable (the cloud environment's network policy can block them), check behaviour against the offline fixtures in `tests/common.py`. For a refactor, compare old and new output: load `git show HEAD:` as a module beside the new one and diff what both produce for the same inputs. +- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, except for + pull requests that change only README.md, CLAUDE.md, `doc/*.md` or `img/*.png` (install, compile and tests are + skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). +- Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); + `requirements.txt` is for pip/uv users and CI. +- A new preference goes in three places: a `Prefs` field with its default (`lib/config.py`; its type says how it is + read), `defaults/preferences.conf`, and the README's preferences table. The tests catch a key missing from the + template or a default that differs, but not the README (no test may read README.md: CI skips the tests for a + README-only change). +- If Yahoo and Sharesight are unreachable (the cloud environment's network policy can block them), check behaviour + against the offline fixtures in `tests/common.py`. +- For a refactor, `util/record_outputs.py` (or `--against COMMIT`) records what every command, button and scheduled + report posts, for this tree and for HEAD in the same run, and prints the steps that differ (exit status 1 when any + do). A change to how the tests capture output goes in `tests/common.py`'s `REAL_SENDING`, which the tool puts back to + record real requests. + +## Code map (open the one file you need) +- `bot.py`: WSGI entry (route, verify, parse, then a thread of `lib/worker.py` per message or button press); SIGHUP + reload and SIGUSR1 cache flush, each in a thread. `lib/inbound.py`: a class per network that reads its updates + (`Telegram`, `Slack`: a `Message`, a `Press`, or a `Response` given straight back), apart from a class per way they + arrive (`TelegramWebhook`, `SlackWebhook`: the path, how it proves itself); `bot.take(network, update)` takes an + update however it arrived. +- `lib/worker.py`: `COMMANDS`, a table of `Command(names, handler, usage, section, buttons, percent, period, + keywords)`, which also makes `.help` and the Telegram command keyboard, kept current in DMs. `read()` finds the + command by its whole name (any other first word that may be a ticker is `.SYMBOL`) and `read_args()` its `Args`: a + percent, a period and a keyword from either end, the words between; a handler raises `Usage` for arguments it cannot + take, and gets a ticker with `ticker(args)`. `Request` (`request.reply(lines, chart, buttons)`); each command gets + the typing notice while it works. `process_request(message)` and `process_callback(press)` take inbound's objects. +- `lib/reports/`: what finbot says. Scheduled (each has `run(dest=None, **kwargs)`, which posts to the chat that asked + or else the default chats, and `job(argv)`, its schedule-line parser): `price performance trades shorts cal rating + milestone reminder podcasts cleanup sharesight_warm yahoo_warm`; the top-level `price.py` and the rest are shims that + run them. Chat-only (`prepare_*`): `about` (.help .jobs), `history` (.history .compare), `holdings` (.holdings .who + .pie .watchlist), `profile` (.SYMBOL), `rankings` (.marketcap .pe .fpe .peg .beta .buy .sell). +- `lib/delivery.py`: `Report(lines, chart, buttons)`, the destinations (the chat that asked, or the default chats), + `deliver(dest, report)`, the one-line error report. `lib/transports.py`: a class per way of posting (`TelegramBot`, + `SlackBot`, `SlackWebhook`, `DiscordWebhook`): splitting, captions, charts, buttons, edits, and the typing notice + (`start_typing`: after half a second; Slack's ⏳ on the message), which `text()` and `chart()` stop before posting. + `lib/telegram.py`: Bot API calls (`call`, `try_call`). +- `lib/buttons.py`: `Button`, a chart's period buttons (`keyboard`), a `.jobs` command's (`job_button`), and what a + pressed button's data asks for (`parse`); each transport draws them. `lib/charts.py`: a chart with period buttons, + built by its kind's builder (`build`) for the command's first view (`worker.chart_reply`) and for each press. +- `lib/dialects.py`: each network's markup and message sizes. `lib/markets.py`: one table by market code (flag, + currency, Yahoo suffix, Google code; Sharesight's codes, so TSE is Toronto), and `code(name)` for any name an exchange + goes by. `lib/formats.py`: names, units, dates, percents in words. `lib/links.py`: stock links. `lib/render.py`: + every chart, `Chart` specs and `draw()`. +- `lib/yahoo.py lib/sharesight.py lib/shortman.py lib/simplywallst.py`: data sources. Yahoo's and Sharesight's answers + are cached in `DATA_DIR/cache` as sent, not parsed, so a parser fix needs no flush (`cache.cached`). + `yahoo.symbol(typed)` is Yahoo's symbol for a ticker as written, tried on the local exchange (`exchange_suffix`) + when Yahoo does not know it bare: every ticker a chat gives goes through it. `sharesight.portfolios(name=None)` gives + `Portfolio` objects: `holdings()`, `tickers()`, `gain(days)` (the portfolio's), `trades(days)`. +- `lib/cache.py`: cache files (any may vanish at any time). `lib/state.py`: state files in `DATA_DIR`, which must last; + change one with `with state.edit(name, default) as data:`, never load-then-save. `lib/watchlist.py`: the watchlist. +- `lib/config.py`: `config.env` (finbot.env, fixed at start) and `config.prefs` (preferences.conf, a new object on + each reload). `lib/jobs.py` + `lib/jobrun.py`: the scheduler, its shared state under one lock; each command runs in + a child process, stopped after `JOB_TIMEOUT`. +- `tests/common.py`: `FinbotCase` (offline fixtures). `self.command('.price')` returns what was sent: + `('text', lines)` or `('photo', size, caption)`. `wsgi_environ(path, body, **headers)` is a request for `bot.main`. + `ProviderCase` runs the real parsers on the answers saved under `doc/` (`sample`), as `Resp` and `serve` return + them; `temp_dir(self)` is a directory removed after the test, `bare_environment()` the environment without finbot's + settings. +- Which test file: commands and scheduled reports `test_reports`; failures, bad input, secrets in logs, signals + `test_edges`; webhook requests, Slack events and signatures `test_inbound`; Yahoo `test_yahoo`; Sharesight + `test_sharesight`; posting to each network `test_sending`; registering Telegram's webhook `test_telegram_api`; the + schedule file and job arguments `test_schedule`; the scheduler and manual runs `test_scheduler`; config + `test_config`; charts `test_charts` (refresh goldens with `UPDATE_GOLDEN=1`); markets, formats, links, buttons, + cache and state files `test_util`; data dir `test_datadir`; chunking, git_version, and the docstring, stdout and + line-width checks `test_finbot`. + +## Conventions +- Read settings as `config.env.` / `config.prefs.` where they are used, never into a default or module + constant (a reload must be seen); in tests, `self.env(...)` / `self.prefs(...)` change them. +- A report is a list of lines. `service` is a dialect (`lib/dialects.py`), which equals the network's name: markup + with `service.bold(text)`, `.italic`, `.strike`, `.link(url, text)`, user-written text through + `service.escape(text)`, stock links through `links.finance_link(symbol, exchange, service)`. Tests pass `TELEGRAM`, + `SLACK`, `DISCORD`. +- A command that cannot give what was asked (no such portfolio, no prices) raises `answer.Answer(*lines)`, which the + chat gets as a plain reply; `yahoo.NoData` is one. Any other exception becomes a one-line ⚠️ error report. +- Every function and method has a docstring, tests included (a test checks): one line where it fits, reworded rather + than spilling a few words onto a second. Tabs everywhere (`.editorconfig`). Every line stays within 120 columns, + tabs counting 4 (an edit quotes whole lines); a test checks all but README.md, CLAUDE.md and `doc/`. +- Log with `print(..., file=sys.stderr)`. systemd discards stdout (`StandardOutput=null`), so the bot and its reports + never print there (a test checks); only the util/ tools, run by hand, do. +- A report gives its chart as a `render.Chart` spec, not a drawing: `deliver()` draws it only for a chat that takes + charts, through `render.draw()`, which caches the image by the spec's hash (and the drawing code's). +- A chart renderer in `lib/render.py` takes its `Chart` and is listed in `RENDERERS` by kind; it gets `@_drawn` (the + bot draws in one native thread) and must not call another `@_drawn` function, or that thread waits on itself. +- A new command: a `Command` in `COMMANDS` (its names, and what its arguments may hold), whose usage, section and + buttons make its `.help` lines and keyboard buttons, and the README command list. + +## Saving tokens +- Run the tests quietly: `.venv/bin/python -m unittest discover -s tests -b` prints only failures (about 0.4 KB when + green). One module: `.venv/bin/python -m unittest tests.test_reports -b`. +- README.md is 36 KB: grep for the section instead of reading it whole. Each network's setup is in its own guide, + `doc/telegram.md`, `doc/slack.md` and `doc/discord.md`: a change to a network's setup or keys goes there. `doc/yahoo/` + and `doc/sharesight/` hold large API samples that the tests replay: inspect them with python, don't read them. diff --git a/README.md b/README.md index 978cbb2a..1a33be11 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,30 @@ * Shared watch list * List stocks (holdings + watchlist) by P/E, Forward P/E, PEG, market cap, or Yahoo analyst consensus +### Chat networks +What finbot can do on each network. Telegram takes everything; the others take what their APIs allow. + +| Feature | Telegram | Slack | Discord | +|---|---|---|---| +| Chat commands | ✓ | ✓ | ✗ (a webhook only posts) | +| "typing…" while a command works | ✓ | ⏳ on the command's message (Slack has no typing notice for bots) | ✗ | +| Charts in replies | ✓ | ✓ with `slackBotToken` | n/a | +| Charts in scheduled reports | ✓ | ✓ with `slackChannelID` and `slackBotToken` (the incoming webhook takes text only) | ✓ | +| Period and job buttons | ✓ | Period buttons, with the interactive bot | ✗ | +| No link previews | ✓ | ✓ | ✓ | +| Linked tickers | ✓ | ✓ | ✓ | +| Charts updated in place by their buttons | ✓ | ✓ | ✗ | + +**Setting up a network:** [Telegram](doc/telegram.md) · [Slack](doc/slack.md) · [Discord](doc/discord.md). Each guide +covers the whole setup of that network, and where there is a choice between reports only and the interactive bot, it +compares the two before the steps. + ![Screenshot of showing trade notifications on Slack](img/screenshot.png?raw=true "Screenshot showing trade notifications on Slack") ### Requirements * Most features require a Sharesight paid plan, preferably with automatic trade imports, and an API key -* Discord/Slack webhooks / Telegram bot user -* Interactive commands require a public web server with domain name and matching certificate to proxy chat commands to the bot. +* A Telegram bot, a Slack app or a Discord webhook (see the network's guide, above) +* Interactive commands require a public web server with domain name and matching certificate to proxy chat commands to the bot (see the [Telegram](doc/telegram.md#interactive-bot) and [Slack](doc/slack.md#interactive-bot) guides). * Python 3.13.5 (the version in `.python-version`; 3.12 is the minimum) * Python modules, pinned to versions with 3.13 support in `requirements.txt`: ``` @@ -37,53 +55,38 @@ croniter python-dateutil requests gevent pandas matplotlib numpy ### Supported commands Where: -* _percent_ is a number followed by '%' -* _period_ is a number followed by 'd' 'w' 'm' or 'y' +* _percent_ is a number, optionally followed by '%' +* _period_ is a number followed by 'd', 'w', 'm' or 'y', or 'ytd' * _portfolio_ is a Sharesight portfolio name -* _symbol_ is a Yahoo Finance symbol +* _symbol_ is a Yahoo Finance symbol; a bare one Yahoo does not know is tried again on your exchange (`exchange_suffix`: `BHP` may be `BHP.AX`) ``` .symbol +.beta +.buy +.sell +.compare symbol symbol [symbol...] [period] .dividend [period|symbol] .earnings [period|symbol] +.forward pe [symbol|top|bottom|negative] .history symbol .holdings portfolio .jobs [run number|command] .marketcap [symbol|top|bottom] +.pe [symbol|top|bottom] +.peg [symbol|top|bottom|negative] .performance [period] [portfolio] .pie portfolio [sector] +.plan [your plan] .premarket [percent|symbol] -.price [percent|symbol] [period] -.rating [strong buy|buy|hold|underperform|sell] +.price [percent|symbol|top|bottom] [period] .session [percent|symbol] .shorts [percent|symbol] .trades [period] [portfolio] -.value [pe|forward pe|peg|bottom pe|bottom forward pe|bottom peg|negative forward pe|negative peg] .watchlist [add|del symbol] .who symbol ``` -Alternative syntax: - -``` -@botname -@botname dividend [period|symbol] -@botname earnings [period|symbol] -@botname history symbol -@botname holdings [portfolio] -@botname jobs [run number|command] -@botname marketcap [symbol|top|bottom] -@botname performance [period] [portfolio] -@botname pie portfolio [sector] -@botname premarket [percent|symbol] -@botname price [percent|symbol] [period] -@botname rating [buy|hold|underperform|sell] -@botname session [percent|symbol] -@botname shorts [percent|symbol] -@botname trades [period] [portfolio] -@botname value [pe|forward pe|peg] -@botname watchlist [add|del symbol] -@botname who symbol -``` +`!` or `/` work in place of the dot, and so does the bot's name: `@botname price 7d`, `@botname AAPL`. A command is its whole name, in any case, and so are its keywords (`.price TOP`); any other first word is a symbol. `.help` lists the commands in chat. Trade notifications are peformed by polling the Sharesight trades API on a schedule (see [Scheduling](#scheduling)), and notifying your configured chat networks of any new trades. Thus, it works best if your trades are auto-imported into Sharesight through its broker integrations, and if your environment has persistent storage so that the bot can keep track of known trade IDs between runs. @@ -176,7 +179,7 @@ You can also specify a portfolio name to get trades for just that portfolio: ![Price chart triggered from Slack](img/finbot_price.png?raw=true "Price chart triggered from Slack") ![Price alert triggered by Cron](img/price.png?raw=true "Price alert triggered by Cron") -`prices.py` sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. +`price.py` sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. @@ -451,7 +454,8 @@ sudo cp ~/finbot/defaults/finbot.env /opt/finbot.env && sudo chmod 600 /opt/finb * **init.d / shell:** `util/check_config.py /opt/finbot.env && set -a && . /opt/finbot.env && set +a && ./bot.py` (`set -a` is what exports the values to the bot). * **Scheduled jobs:** they are started by `bot.py` (see [Scheduling](#scheduling)), so they run on the same settings. For a one-off run, see [Running a report by hand](#scheduling). -All 17 keys, read by your launcher. A change needs a restart. Blank means unset. +The startup keys, read by your launcher. A change needs a restart. Blank means unset. Each network's guide says which +of the chat network keys it needs. | Key | Default | Description | |---|---|---| @@ -468,19 +472,21 @@ All 17 keys, read by your launcher. A change needs a restart. Blank means unset. | `discord_webhook` | | Discord webhook, for outbound reports | | `telegramBotToken` | | Telegram bot token, for outbound reports | | `slackBotToken` | | Slack bot token, for replies and chart uploads in Slack | +| `slackChannelID` | | Channel id that scheduled reports go to on Slack as the bot, charts and all (with `slackBotToken`); `slack_webhook` is then not used | | `telegramOutgoingWebhook` | | Telegram webhook path the bot accepts (interactive mode) | | `telegramOutgoingToken` | | Secret token Telegram must present (interactive mode) | | `slackOutgoingWebhook` | | Slack webhook path the bot accepts (interactive mode) | -| `slackOutgoingToken` | | Token Slack must present (interactive mode) | +| `slackOutgoingToken` | | Token Slack must present (interactive mode): the older way, which Slack has deprecated; not needed with `slackSigningSecret` | +| `slackSigningSecret` | | Secret Slack signs each request with (interactive mode) | ### `preferences.conf` items Copies preferences.conf and metadata files into the data directory (never overwrites) ``` -util/sync_defaults.py [-d DEST] --apply +util/sync_defaults.py [--dest DIR] --apply ``` -All 26 keys, read by the bot itself. A change takes effect on a reload. +The preferences, read by the bot itself. A change takes effect on a reload. A true/false preference takes `true`/`false`, `yes`/`no`, `on`/`off` or `1`/`0`; anything else is a mistake, reported like any other. A blank value means the default. | Key | Default | Description | |---|---|---| @@ -495,23 +501,21 @@ All 26 keys, read by the bot itself. A change takes effect on a reload. | `exclude_portfolios` | | Sharesight portfolio IDs to leave out, space separated | | `include_portfolios` | | Sharesight portfolio IDs to report on, space separated (empty: all that are not excluded) | | `trades_use_yahoo` | `True` | Use Yahoo for trades; set `False` if Yahoo breaks | -| `performance_use_sharesight` | `False` | Use Sharesight rather than Yahoo for performance over a period | -| `alliterate` | `False` | Greeting uses alliterating adjectives (easter egg) | | `graph` | `True` | Send charts with reports | | `hyperlink` | `True` | Hyperlink tickers in reports | | `hyperlinkFooter` | `True` | Add a website link footer to stock lookups | | `hyperlinkProvider` | `google` | Where ticker links go: `google` or `yahoo` | -| `chunk_maxlines` | `20` | Maximum lines per message before a report is split | -| `country_code` | `AU` | Country, used by the reminder report | -| `timezone` | `Australia/Melbourne` | Time zone, used by the reminder report | +| `exchange_suffix` | `AX` | Regional: Yahoo's suffix for your exchange (`AX` for the ASX, `NZ`, `L`, `TO` and so on). A bare symbol Yahoo does not know is tried again with it, in every command that takes a symbol (`BHP` becomes `BHP.AX`); empty, a symbol is taken as written | +| `country_code` | `AU` | Regional: your country, for the reminder report | +| `timezone` | `Australia/Melbourne` | Regional: your time zone, for the reminder report | | `cache` | `True` | Cache Yahoo and Sharesight responses | | `cache_seconds` | `43200` | Cache lifetime in seconds (12 hours) | -| `performance_cache_seconds` | `1800` | Cache lifetime of Sharesight performance reports in seconds | +| `performance_cache_seconds` | `2400` | Cache lifetime of Sharesight performance reports in seconds: longer than the 30 minutes between `sharesight_warm.py` runs, so the holdings never run out | | `http_timeout` | `18` | Seconds to wait on outbound HTTP requests | | `debug` | `False` | Extra diagnostics in the log | | `print_headers` | `False` | Log inbound request headers | -A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. +A key in the wrong file, or one that is not listed, stops finbot starting; on a reload it is reported and nothing is applied. A key finbot no longer reads (`alliterate`, `chunk_maxlines`) is ignored, and the log at start says to delete it. edit `preferences.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. @@ -529,6 +533,8 @@ util/sync_defaults.py --apply # copy missing files (including a starter prefere util/sync_defaults.py --force # also replace changed files, keeping .bak copies util/sync_defaults.py --dest /srv/finbot --apply # target another directory instead of the data directory ``` +`reminder.json` holds the reminders that `reminder.py` posts on their day. Each entry has a `month_day` (`06-30`) and a `message`, which is one line or a list of lines, and may name a `year` (`2026`) and a `gate_country` (posted only when `country_code` in `preferences.conf` matches; `{flag}` in its message becomes that country's flag). A message is text, with two kinds of markup that each network gets in its own form: `[text](https://example.com)` is a link, and a line wrapped in underscores, `_like this_`, is in italics. The Australian end of financial year checklist (23 June) is such an entry: if you keep your own `reminder.json` in the data directory, `util/sync_defaults.py` shows what it lacks. + With systemd `DynamicUser=yes` and `StateDirectory=finbot`, `DATA_DIR` defaults to systemd's `$STATE_DIRECTORY` (`/var/lib/finbot`), so no setting is needed; move existing data once with `cp -a /opt/finbot/var/. /var/lib/finbot/`. An explicit `DATA_DIR` always wins. In Docker, one container is enough: `bot.py` runs the schedule too, so there is no cron or scheduler sidecar to set up. @@ -541,35 +547,10 @@ sharesight_client_id=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f sharesight_client_secret=01d692d4de7 samplefc64bc2e2f01d692d4de72986ea808f6e99813f ``` -### Discord -Discord support is currently only for trade notifications and scheduled reports. -The Discord webhook can be provisioned in Discord under _Server Settings > Integrations > Webhooks_. -Example: -``` -discord_webhook=https://discord.com/api/webhooks/1009998000000000000/aaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbb -``` - -### Slack -A Slack webhook can be provisioned by creating a new app from scratch at https://api.slack.com/apps/ then navigating to _Incoming Webhooks_. Once the link is generated, add it to `finbot.env`. Example: -``` -slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA -``` -The webhook is only used for the trade notifications and scheduled reports. The interactive bot requires further configuration of this Slack app (see below). - -### Telegram -* Set up the bot by messaging [BotFather](https://telegram.me/BotFather). -* Add your bot to a group or channel. -* For security, run `/setjoingroups` and set to `Disabled` -* Optionally, make the bot a group admin (for interactive features). -* In `finbot.env`, set `telegramBotToken` to the token BotFather gave you. -* In `finbot.env`, set `telegramChatID` to the chat group or channel id. - * For channels and supergroups, `telegramChatID` should be negative and 13 characters. Prepend `-100` if necessary. - * Be aware a group id can change if you edit group settings and it becomes a "supergroup". Currently, the bot does not automatically handle this. -* Example `finbot.env` entry: -``` -telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA -telegramChatID=-1001000000000 -``` +### Chat networks setup +Follow the guide for each network you use: [Telegram](doc/telegram.md), [Slack](doc/slack.md) or +[Discord](doc/discord.md). It covers the bot or webhook, its `finbot.env` keys, and for chat commands the HTTPS server in +front of finbot. ### Portfolios Portfolios are auto-discovered, including other people's portfolios which are shared to you. To exclude specific portfolios, add their IDs to `exclude_portfolios` in the `preferences.conf`: @@ -583,21 +564,39 @@ include_portfolios=100001 100002 ``` ### Caching -Many objects are cached for just under one day by default. Cache is controlled by the settings below. Trades IDs are stored on disk, but trade queries generally don't result in a cache hit for functional reasons. +Many objects are cached for 12 hours by default (`cache_seconds`). Cache is controlled by the settings below. Trades IDs are stored on disk, but trade queries generally don't result in a cache hit for functional reasons. ``` cache=True -cache_seconds=82800 -performance_cache_seconds=1800 +cache_seconds=43200 +performance_cache_seconds=2400 ``` -`performance_cache_seconds` (default 30 minutes) is the cache lifetime of Sharesight performance reports. +`performance_cache_seconds` (default 40 minutes) is the cache lifetime of Sharesight performance reports. + +Charts are cached by what they show (with `cache=True`): the same chart, for another chat, a later run or a period button, is drawn once, and a chart of new figures is a new chart, so a cached one is never stale. The scheduled `cleanup.py` removes cache files after 30 days. -While `bot.py` is running with Sharesight configured and `cache=True` (a reload that switches `cache` pauses or resumes it), a background job keeps the Sharesight portfolio list (`cache_seconds`) and each portfolio's holdings report (1 day, `performance_cache_seconds` cache) refreshed at 90% of their cache lifetime, so chat requests do not wait on Sharesight. If a refresh fails, the previous cache keeps being served and the job retries after 5 minutes. +To flush the cache, so that everything is fetched afresh, signal the running bot: it deletes its cache files (Yahoo and Sharesight +answers, charts, the Yahoo cookie) and keeps its state (watchlist, seen trades, ratings, schedule), then runs both warm jobs +(`sharesight_warm.py`, `yahoo_warm.py`) straight away to fill the cache again. There is no `systemctl flush`, since systemctl takes no custom verbs; any of these does it: +``` +sudo systemctl kill -s USR1 finbot # systemd +sudo systemctl start finbot-flush # the same, once finbot-flush.service is installed (see below) +docker kill -s USR1 # Docker +kill -USR1 # anything else +``` +The log says `cache: flushed N files`, then `cache: warm jobs done` once the cache is full again. To install the `finbot-flush` shortcut: `sudo cp -v finbot-flush.service /etc/systemd/system/ && +sudo systemctl daemon-reload`. `finbot.service` itself needs no change. With the bot stopped, `cleanup.py 0` deletes the same files. + +Two scheduled jobs keep the caches full, each on a fixed interval shorter than the cache lifetime, so that chat requests do not wait on Sharesight or Yahoo (see [Scheduling](#scheduling)): `sharesight_warm.py` fetches the Sharesight portfolio list and each portfolio's holdings report afresh every 30 minutes (`performance_cache_seconds`: 40 minutes), and `yahoo_warm.py` the price history of every stock followed every 8 hours (`cache_seconds`: 12 hours). Neither fetches anything with `cache=False`, and `sharesight_warm.py` nothing without Sharesight configured. A refresh that fails is logged, the previous cache keeps being served while fresh, and the next run tries again. A flush runs both at once (see above). A data directory with its own `schedule.cron` needs their lines: +``` +14,44 * * * * sharesight_warm.py +07 */8 * * * yahoo_warm.py +``` ### Hyperlinks These `preferences.conf` options relate to hyperlinks: -* `hyperlink`, if set to `False`, disables all hyperlinks. Note: it can not disable syntax highlighting and false positives by chat clients that try to detect domains. Setting it to `False` can be useful for chat networks like Discord that can't easily disable link previews. Defaults to `True`. +* `hyperlink`, if set to `False`, disables all hyperlinks. Note: it can not disable syntax highlighting and false positives by chat clients that try to detect domains. Defaults to `True`. ``` hyperlink=True ``` @@ -619,7 +618,7 @@ hyperlinkProvider=yahoo ## Scheduling The scheduled reports are started by `bot.py`, each run in its own child process (`python -m lib.jobrun`), so they use the same user, sandbox, environment and data directory as the bot, while heavy work (charts, large reports) cannot hold up incoming chat webhooks. A run that exceeds 5 minutes is stopped. There is no cron to set up, which matters under systemd `DynamicUser=` where only the bot's own user can read the state directory. **This is on by default.** Remove the equivalent entries from any old crontab before the first start, otherwise every job runs twice; or set `scheduler=False` in the environment file to keep using cron. -The schedulable scripts are `trades.py`, `price.py`, `reminder.py`, `cal.py`, `shorts.py`, `performance.py`, `rating.py`, `milestone.py`, `podcasts.py ""` and `cleanup.py [days]` (deletes cache files older than `days`, default 30). Other jobs, such as a DNS updater or backups, stay in cron or systemd timers. +The schedulable scripts are `trades.py`, `price.py`, `reminder.py`, `cal.py`, `shorts.py`, `performance.py`, `rating.py`, `milestone.py`, `podcasts.py ""`, `cleanup.py [days]` (deletes cache files older than `days`, default 30), `sharesight_warm.py` and `yahoo_warm.py` (they fill the cache: see [Caching](#caching)). Other jobs, such as a DNS updater or backups, stay in cron or systemd timers. The schedule is `defaults/schedule.cron`, written in **cron-compatible syntax**: five time fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists work) or an `@macro` such as `@daily`, then a finbot script and its arguments. Quote an argument that has spaces or special characters (a feed URL with `&`, say). Several commands separated by `;` run one after another. `MAILTO=`, `SHELL=`, `PATH=` and `HOME=` lines are ignored. Anything else a crontab allows (a user column, pipes, other programs) is not supported: such a line is reported at startup and skipped. A time that cannot be or never comes (minute 61, 30 February) is different: finbot refuses to start with it, and a reload with it is refused, leaving the running schedule as it was. Existing crontab lines can be pasted as they are: `> /dev/null` redirects, `{ a; b ;}` groups and a path before the script name are understood. ``` @@ -642,11 +641,11 @@ sudo systemctl reload finbot # the bot re-reads the file and logs how man ``` The bot logs the number of jobs when it starts or reloads, and each job as it runs (`journalctl -u finbot`); a line it cannot use is reported by name. If the edited file has no usable line the previous schedule keeps running. -**To run a job from chat**, send `.jobs` (also `.cron`, `.crons` or `.schedule`): it lists the scheduled commands, each once and without times, comments or `.py` (`cleanup` and `podcasts` are scheduled but not listed), and on Telegram adds a button per command. `.jobs run 2` (or tapping the button) runs that command now, and `.jobs run price.py close` runs it by name; anything not in the schedule is refused, and so is a chain such as `a; b` (run them one at a time). A manual run is exactly a cron run: it reports to all configured chat networks, updates the same state (seen trades, ratings and so on), and does not change the schedule. Anyone who can message the bot can do this; a command that is already running is not started twice. +**To run a job from chat**, send `.jobs` (also `.cron`, `.crons` or `.schedule`): it lists the scheduled commands, each once and without times, comments or `.py`, and on Telegram and Slack (with its buttons set up) adds a button per command. The system jobs (`cleanup`, `podcasts`, `sharesight_warm` and `yahoo_warm`) follow under _System_, without a number or a button: they run on their schedule only. `.jobs run 2` (or tapping the button) runs that command now, and `.jobs run price.py close` runs it by name; anything not in the schedule is refused, and so is a chain such as `a; b` (run them one at a time). A manual run is exactly a cron run: it reports to all configured chat networks, updates the same state (seen trades, ratings and so on), and does not change the schedule. Anyone who can message the bot can do this; a command that is already running is not started twice. Behaviour worth knowing: * A run that was due while the bot was down (a restart, a deploy) still fires once if it was due within the last `scheduler_catchup` minutes (default 5). A restart right after a run does not repeat it. -* A job still running when it is next due is skipped, and a job that takes more than 5 minutes is abandoned and reported. +* A job still running when it is next due is skipped. A command still running after 5 minutes is stopped and reported, and the rest of its line still runs: each command has 5 minutes of its own. * Failures are reported to your chat channels, as before. * With `scheduler=False` nothing is scheduled by the bot, so you can keep using cron or systemd timers and run the scripts yourself. @@ -656,70 +655,7 @@ sudo systemd-run --pty --wait --collect -p DynamicUser=yes -p StateDirectory=fin ``` The script prints its output on your terminal and sends its report to your chat channels as usual. The transient unit gets its own dynamic user, and systemd may re-own the state directory to it for the run: check `ls -ln /var/lib/private/finbot` afterwards and `systemctl restart finbot` if the bot can no longer write there. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports inside the bot, without any of this. Outside systemd, run a script from the checkout as your own user with the settings loaded into the shell: `set -a; . /opt/finbot.env; set +a; ./price.py 7` (this works while no value in the file contains spaces). -## Interactive bot setup -The backend `bot.py` needs a frontend https server on a valid domain name with a valid x509 certifcate. -It defaults to listening on http://127.0.0.1:5000/, which can be changed with `ip` and `port` in `finbot.env`. - -Example frontend https server config (nginx): -``` -server { - listen 8443 ssl; - deny all; - - server_name www.example.com; - ssl_certificate /etc/letsencrypt/live/www.example.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/www.example.com/privkey.pem; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers HIGH:!aNULL:!MD5; - - location /slack { - proxy_pass http://127.0.0.1:5000/slack; - include /etc/nginx/aws_subnets; - deny all; - } - location /telegram { - proxy_pass http://127.0.0.1:5000/telegram; - include /etc/nginx/telegram_subnets; - deny all; - } -} -``` -Note: The utils folder contains scripts to generate `/etc/nginx/aws_subnets` and `/etc/nginx/telegram_subnets`. They can be run once, or placed in `/etc/cron.daily/`. - -### Integrating the interactive bot with chat networks - -#### Telegram -* Message BotFather to create a bot user. - * Set `telegramBotToken` in `finbot.env` to the token given by BotFather. - * Add your Telegram bot user to a group - * Give the bot group admin access so it can read the group chat. - * For security, message BotFather and disable `/setjoingroups`. -* Set `telegramOutgoingWebhook` in `finbot.env` to your web server (e.g. https://www.example.com:8443/telegram). -* Set `telegramOutgoingToken` to a password of your choosing. Telegram will use this to authenticate with finbot. -With these options set, your bot will auto-subscribe your URL to events the bot sees, when you run `bot.py`. It unregisters and registers the webhook as it starts (and only then, which also wakes up a stale connection); if Telegram refuses, finbot does not start and the log says why. - -#### Slack -Visit https://api.slack.com/apps/ to create a new Slack app from scratch (if you already created one for a finbot webhook, you can reuse that app). Charts in replies are uploaded into the channel with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`, which replaced the retired `files.upload`). Scheduled reports reach Slack through the incoming webhook, which takes text only, so they come without charts. If an upload fails (a missing scope, say), the reply goes out as text and the log says why. -* From _Basic Information > Verification Token_, copy _Verification token_ into `finbot.env` as `slackOutgoingToken`. This allows Slack to authenticate with finbot. -* In `finbot.env`, put your web server URL (e.g. https://www.example.com:8443/slack) into `slackOutgoingWebhook` -* Save `finbot.env` and restart `bot.py` -* Under _Event Subscriptions_: - * Go to _Enable Events > On > Request URL_ and enter your web server URL (e.g. https://www.example.com:8443/slack). At this time, if the bot is reachable, it will auto respond to Slack's verification request. - * Go down to _Subscribe to bot events_ and add event `app_mention` for the bot to see _@botname_ mentions. - * Alternatively, you can subscribe to `message.channels` if you want your bot to see everything and respond to `.` commands. To avoid duplicate responses, don't subscribe to `app_mention` and `message.channels` at the same time. - * If you want to DM the bot: - * Subscribe to `message.im` - * Check the box _App Home > Allow users to send Slash commands and messages from the messages tab_ - * Save Changes -* Under _OAuth & Permissions_: - * Scroll down to _Scopes > Bot Token Scopes_, and add: `chat:write`, `files:write` and `files:read` (Slack's file upload API needs both `files` scopes). - * Adding scopes to an app that is already installed takes effect only after _Reinstall to Workspace_. - * Scroll up to _OAuth Tokens for Your Workspace_: - * If _Bot User OAuth Token_ is not visible, hit _Install to Workspace > Allow_ - * Copy _Bot User OAuth Token_ into `finbot.env` as `slackBotToken`. This allows finbot to authenticate with Slack so it can send replies. - * Restart `bot.py` - -### Daemonize (systemd) +## Daemonize (systemd) `finbot.service` can take care of keeping `bot.py` running in the background and starting it on boot. Copy `finbot.service` to `/etc/systemd/system/`, edit it to set the `User`, `ExecStart` and the path of [`finbot.env`](#setup) (`EnvironmentFile=`, and the `ExecStartPre=` check), then enable and start it. `systemctl reload finbot` re-reads `preferences.conf` and the [schedule](#scheduling) without a restart; a change to `finbot.env` needs `systemctl restart finbot`: ``` @@ -739,9 +675,6 @@ You can now monitor the bot's stderr with : journalctl -fu finbot ``` -## Limitations -* Discord shows garbage link previews from Sharesight. Set `hyperlink=False`, or disable this for each Discord client under _Settings > Text & Images > Embeds and link previews_ (must be repeated for each participating Discord client). - ## Suggestions * Is my code is doing something the hard way? * Something important is missing from this README? diff --git a/bot.py b/bot.py index 0b5cf865..e7f4ab2e 100755 --- a/bot.py +++ b/bot.py @@ -2,41 +2,59 @@ import signal if __name__ == '__main__': - signal.signal(signal.SIGHUP, signal.SIG_IGN) # ignored until startup is done (the reload handler is installed last): its default action kills the bot + # ignored until startup is done (the reload handler is installed last): its default action kills the bot + signal.signal(signal.SIGHUP, signal.SIG_IGN) + signal.signal(signal.SIGUSR1, signal.SIG_IGN) # likewise the cache flush import gevent.monkey gevent.monkey.patch_all() -import json, os, re, sys, traceback +import collections, json, os, re, sys, traceback from pathlib import Path from gevent import pywsgi import threading -from urllib.parse import urlparse -from lib.config import * from lib import config +from lib import inbound from lib import worker from lib import jobs -from lib import charts -from lib import sharesight from lib import net from lib import telegram -from lib import webhook +from lib import delivery +from lib import render +from lib.reports import cleanup -botName = None # the Telegram bot's @username, from telegram.register() at start -AUTHENTICATED = 'finbot.authenticated' # set in a request's WSGI environ once it has passed its service's check -SLACK_USER_SUBTYPES = (None, 'thread_broadcast') # Slack message subtypes a person writes; the others are bot posts, edits, joins and the like +ROUTED = 'finbot.routed' # a request's WSGI environ: the network whose webhook path it came to (the path is a secret) +AUTHENTICATED = 'finbot.authenticated' # set once it has passed that network's check +MAX_BODY = 1 << 20 # bytes in a request body, at most: an update from Telegram or Slack takes a few thousand +_updates_seen = collections.OrderedDict() # (service, update id) of the last updates taken, oldest first + +def seen_before(service, update_id, keep=1000): + """Whether this update was taken already: Telegram's update_id, Slack's event_id. Slack resends an event it got no + answer to within 3 seconds and Telegram one that failed, and a command must not run twice. An update without an id + is never a repeat.""" + if update_id is None: + return False + key = (service, update_id) + if key in _updates_seen: + return True + _updates_seen[key] = None + if len(_updates_seen) > keep: + _updates_seen.popitem(last=False) + return False def main(environ, start_response): - """WSGI entry point. Never lets an exception escape. A failure handling an authenticated request is reported as one line to the - default channel and answered 200, so the service does not retry it into the chat again. Before authentication a failure is only - logged: a stranger must not be able to post into the chats.""" + """WSGI entry point. Never lets an exception escape. A failure handling an authenticated request is reported as one + line to the default channel and answered 200, so the service does not retry it into the chat again. Before + authentication a failure is only logged: a stranger must not be able to post into the chats. Either names the + network, not the path: a webhook's path is a secret (telegramOutgoingWebhook, slackOutgoingWebhook).""" try: return handle(environ, start_response) except Exception as e: if environ.get(AUTHENTICATED): - webhook.report_error(e, context=environ.get('PATH_INFO', 'bot')) + delivery.report_error(e, context=environ[ROUTED]) status = '200 OK' else: - print(f"request to {environ.get('PATH_INFO')!r} failed:", webhook.error_line(e), file=sys.stderr) + where = environ.get(ROUTED) or repr(environ.get('PATH_INFO')) + print(f"request to {where} failed:", delivery.error_line(e), file=sys.stderr) print(net.redact(traceback.format_exc()), end='', file=sys.stderr) status = '500 Internal Server Error' try: @@ -46,129 +64,79 @@ def main(environ, start_response): return [b''] def handle(environ, start_response): - def print_body(): - shown = {k: '' if k == 'token' else v for k, v in inbound.items()} # Slack's verification token - print(f"inbound {uri} ", json.dumps(shown, indent=4, default=str), file=sys.stderr) + """Route a request by its path to the webhook of the network that posts there, turn it away unless it proves it + comes from that network (before its body is read, when the proof is a header), read the update (CONTENT_LENGTH + bytes, up to MAX_BODY) and take it (take): the network gets its answer at once.""" + def respond(response): + """Answer the request: a status line and a body.""" + start_response(response.status, [('Content-type', 'application/json')]) + return [response.body] def print_headers(): + """Log the request headers, Telegram's secret header redacted.""" for name, value in sorted(environ.items()): print((name, '' if name == 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' else value), file=sys.stderr) - def reply(status, body=b''): - start_response(status, [('Content-type', 'application/json')]) - return [body] - print_headers() if config_print_headers else None - # route on the path first, and turn strangers away before reading the body - uri = environ.get('PATH_INFO', '') - if config_telegramOutgoingWebhook and uri == urlparse(config_telegramOutgoingWebhook).path: - service = 'telegram' - elif config_slackOutgoingWebhook and uri == urlparse(config_slackOutgoingWebhook).path: - service = 'slack' - else: - print("Unknown URI", repr(uri), file=sys.stderr) - return reply('404 Not Found', b'

404

') - if service == 'telegram': # Telegram proves itself with a header - secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') - if secret != config_telegramOutgoingToken: - print_headers() - print("Fatal: Telegram authorisation header", "not present" if secret is None else "is present but incorrect", file=sys.stderr) - return reply('403 Forbidden', b'

Unauthorized

') - environ[AUTHENTICATED] = True + if config.prefs.print_headers: + print_headers() + path = environ.get('PATH_INFO', '') + source = inbound.route(path) + if source is None: + print("Unknown URI", repr(path), file=sys.stderr) + return respond(inbound.NOT_FOUND) + environ[ROUTED] = source.service + if why := source.refuse_unread(environ): # strangers are turned away before their body is read + print_headers() + print("Fatal:", why, file=sys.stderr) + return respond(inbound.FORBIDDEN) + length = environ.get('CONTENT_LENGTH') or '0' + if not length.isdigit() or int(length) > MAX_BODY: + print(f"[{source.service}]: a body of {length!r} bytes refused unread", file=sys.stderr) + return respond(inbound.TOO_LARGE if length.isdigit() else inbound.BAD_REQUEST) + body = environ['wsgi.input'].read(int(length)) # the body and no more: a server need not end the stream there try: - inbound = json.loads(environ['wsgi.input'].read()) + update = source.decode(body) except ValueError: - print(f"[{service}]: request body is not JSON", file=sys.stderr) - return reply('400 Bad Request', b'

Bad Request

') - if not isinstance(inbound, dict): - print(f"[{service}]: request body is not a JSON object", file=sys.stderr) - return reply('400 Bad Request', b'

Bad Request

') - if service == 'telegram': - bot_name = botName - if "callback_query" in inbound: # chart period buttons - cq = inbound["callback_query"] - cq_message = cq.get("message") - if not cq_message or not cq.get("data"): - return reply('200 OK', b'Unsupported') - sender = cq.get("from", {}) - print("[Telegram button]:", '@' + sender["username"] if "username" in sender else sender.get("first_name", "unknown"), "chat", cq_message["chat"]["id"], cq["data"], file=sys.stderr) - threading.Thread(target=worker.process_callback, args=('telegram', cq["id"], str(cq_message["chat"]["id"]), str(cq_message["message_id"]), cq["data"])).start() - return reply('200 OK') - if "message" not in inbound: - return reply('200 OK', b'Unsupported') - msg = inbound["message"] - sender = msg["from"] - message_id = str(msg["message_id"]) - chat_id = str(msg["chat"]["id"]) - if "username" in sender: - user = '@' + sender["username"] - userRealName = sender["first_name"] or user # what to call them: the handle when the first name is blank - else: - user = userRealName = '@' + sender["first_name"] - if "text" in msg: - message = msg["text"] - print("[Telegram]:", user, message, file=sys.stderr) if debug else None - elif "photo" in msg: - message = msg.get("caption", '') - file_id = msg["photo"][-1]["file_id"] - print("[Telegram photo]:", user, file_id, message) - elif IGNORED_TELEGRAM_CONTENT & msg.keys(): - return reply('200 OK') # stickers, GIFs, voice notes and the like: nothing to answer, nothing worth logging - else: - print(f"[{service}]: unhandled: 'message' without 'text/photo'", file=sys.stderr) - return reply('200 OK', b'

Unhandled

') - else: # Slack puts its token in the body - if 'token' not in inbound: - print("warning: Slack authorisation field not present", file=sys.stderr) - print_body() - return reply('403 Forbidden', b'

Unauthorized

') - if inbound['token'] != config_slackOutgoingToken: - print("Slack auth incorrect", file=sys.stderr) - print_body() - return reply('403 Forbidden', b'

Unauthorized

') - environ[AUTHENTICATED] = True - print("Incoming Slack request authenticated") - if 'type' not in inbound: - print(f"[{service}]: unhandled: no 'type'", file=sys.stderr) - return reply('200 OK', b'Unhandled') - if inbound['type'] == 'url_verification': - response = json.dumps({"challenge": inbound["challenge"]}) - print("replying with", response, file=sys.stderr) - return reply('200 OK', bytes(response, "utf-8")) - if inbound['type'] == 'event_callback': - event = inbound['event'] - bot_user = inbound['authorizations'][0]['user_id'] # QWERTY - if event.get('bot_id') or event.get('subtype') not in SLACK_USER_SUBTYPES or not event.get('user') or event['user'] == bot_user: - return reply('200 OK') # not written by a person: answering a bot (this one's replies and webhook reports included) could loop - if event["type"] not in ('message', 'app_mention') or "text" not in event: - print(f"[{service}]: unhandled event callback type", event["type"], file=sys.stderr) - return reply('200 OK', b'

Unhandled

') - message_id = str(event["ts"]) - message = event['text'] - message = re.sub(r'', r'\g<1>', message) # becomes dub.ax - message = re.sub(r'<(@[\w\.]+)>', r'\g<1>', message) # <@QWERTY> becomes @QWERTY - user = userRealName = '<@' + event['user'] + '>' # ZXCVBN becomes <@ZXCVBN> - bot_name = '@' + bot_user # QWERTY becomes @QWERTY - chat_id = event['channel'] - print(f"[{service}]:", user, message, file=sys.stderr) if debug else None - # this condition spawns a worker before returning - else: - print(f"[{service}]: unhandled 'type'", file=sys.stderr) - return reply('200 OK', b'

Unhandled

') - - # process in a background thread so we don't keep the requesting client waiting - threading.Thread(target=worker.process_request, args=(service, chat_id, user, message, bot_name, userRealName, message_id)).start() + print(f"[{source.service}]: request body is not a JSON object", file=sys.stderr) + return respond(inbound.BAD_REQUEST) + if why := source.refuse(environ, body, update): + shown = {k: '' if k == 'token' else v for k, v in update.items()} # Slack's verification token + print("warning:", why, file=sys.stderr) + print(f"[{source.service}]: refused update", json.dumps(shown, indent=4, default=str), file=sys.stderr) + return respond(inbound.FORBIDDEN) + environ[AUTHENTICATED] = True + return respond(take(source.network, update)) - # Return an empty response to the client - return reply('200 OK') +def take(network, update): + """An update from a network (an inbound.Network reads it), however it arrived: what it says goes to lib/worker in a + thread of its own (dispatch), unless it is a repeat. Returns the inbound.Response for the network.""" + event = network.parse(update) + if isinstance(event, inbound.Response): + return event + if seen_before(network.service, network.update_id(update)): + print(f"[{network.service}]: update {network.update_id(update)} seen already, not run again", file=sys.stderr) + return inbound.OK + dispatch(event) + return inbound.OK -IGNORED_TELEGRAM_CONTENT = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', 'venue', 'poll', 'dice'} +def dispatch(event): + """Hand a message or a button press to lib/worker, in a thread of its own.""" + if isinstance(event, inbound.Press): + print(f"[{event.service.capitalize()} button]:", event.user, "chat", event.chat_id, event.data, file=sys.stderr) + target = worker.process_callback + else: + if config.prefs.debug: + print(f"[{event.service.capitalize()}]:", event.user, event.text, file=sys.stderr) + target = worker.process_request + threading.Thread(target=target, args=(event,)).start() def git_version(checkout=None): - """Short hash of the checked-out commit, read straight from the .git directory: no git binary to run, and no "dubious - ownership" refusal when the service user does not own the checkout. 'unknown' when it cannot be read.""" + """Short hash of the checked-out commit, read straight from the .git directory: no git binary to run, and no + "dubious ownership" refusal when the service user does not own the checkout. 'unknown' when it cannot be read.""" try: - git = Path(checkout or REPO_DIR) / '.git' + git = Path(checkout or config.REPO_DIR) / '.git' if git.is_file(): # a linked worktree or submodule: .git is a file saying "gitdir: " git = (git.parent / git.read_text().split(':', 1)[1].strip()).resolve() - common = (git / (git / 'commondir').read_text().strip()).resolve() if (git / 'commondir').is_file() else git # refs of a worktree + # refs of a worktree + common = (git / (git / 'commondir').read_text().strip()).resolve() if (git / 'commondir').is_file() else git head = (git / 'HEAD').read_text().strip() if head.startswith('ref:'): ref = head.split(':', 1)[1].strip() @@ -182,20 +150,43 @@ def git_version(checkout=None): return 'unknown' scheduler = None +_signal_work = threading.Lock() # one signal's work at a time: a second reload waits for the first + +def in_thread(work): + """A signal handler that runs `work` in a thread of its own and returns at once: a handler must not wait on the + network, nor on a lock that the code it interrupted may hold.""" + def run(): + """The work, once any other signal's is done.""" + with _signal_work: + work() + return lambda *args: threading.Thread(target=run, daemon=True).start() def reload_all(*args): - """SIGHUP: re-read preferences.conf and, if the scheduler is running, schedule.cron. Each reports its own errors; a bad - preferences.conf leaves the running config untouched.""" + """SIGHUP: re-read preferences.conf and, if the scheduler is running, schedule.cron. Each reports its own errors; a + bad preferences.conf leaves the running config untouched.""" try: changes = config.reload() for change in changes: print('config:', change, file=sys.stderr) - print(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}", file=sys.stderr) + print(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}", + file=sys.stderr) except config.ConfigError as e: config.print_errors(e) if scheduler: scheduler.reload() +def flush_cache(*args): + """SIGUSR1 (`systemctl kill -s USR1 finbot`): delete the cache and keep the state (watchlist, seen trades, ratings, + schedule), then fill it again straight away with the warm jobs, in a thread of their own (a reload need not wait + for them).""" + try: + removed = cleanup.run(days=0) + print(f"cache: flushed {removed} file{'' if removed == 1 else 's'} from {config.env.cache_dir}", + file=sys.stderr) + except Exception as e: + print('ERROR', delivery.error_line(e, 'cache flush'), file=sys.stderr) + jobs.start_thread(jobs.warm_now, scheduler) + if __name__ == '__main__': print(f'Starting finbot {git_version()}', file=sys.stderr) try: @@ -205,10 +196,12 @@ def reload_all(*args): sys.exit(1) for line in config.startup_report(): print('config:', line, file=sys.stderr) + for line in config.startup_warnings(): + print('ERROR config:', line, file=sys.stderr) if os.getuid() == 0: print("Running as superuser. This is not recommended.", file=sys.stderr) - if config_scheduler: # read before anything else happens: an impossible time in it stops the start - scheduler = jobs.Scheduler(lambda: data_file('schedule.cron')) + if config.env.scheduler: # read before anything else happens: an impossible time in it stops the start + scheduler = jobs.Scheduler(lambda: config.data_file('schedule.cron')) try: if not scheduler.load(): scheduler = None @@ -219,23 +212,24 @@ def reload_all(*args): sys.exit(1) else: print('Scheduled reports are off (scheduler=False)', file=sys.stderr) - httpd = pywsgi.WSGIServer((config_ip, config_port), main) - httpd.secure_repr = False if debug else None - print(f'Opening socket on http://{config_ip}:{config_port}', file=sys.stderr) - if config_telegramBotToken and config_telegramOutgoingWebhook: + render.render_off_hub() # charts drawn in a thread of their own: no chat waits while another's chart is drawn + httpd = pywsgi.WSGIServer((config.env.ip, config.env.port), main) + httpd.secure_repr = False if config.prefs.debug else None + print(f'Opening socket on http://{config.env.ip}:{config.env.port}', file=sys.stderr) + if config.env.telegramBotToken and config.env.telegramOutgoingWebhook: try: - botName = telegram.register() # here and only here: the job processes import lib.telegram too + inbound.TELEGRAM.bot_name = telegram.register() # here only: the job processes import lib.telegram too except Exception as e: - print('ERROR', webhook.error_line(e, 'telegram webhook'), file=sys.stderr) + print('ERROR', delivery.error_line(e, 'telegram webhook'), file=sys.stderr) sys.exit(1) - if config_telegramBotToken: - threading.Thread(target=charts.refresh_keyboards, daemon=True).start() # update stale DM keyboards in the background - if config_sharesight_auth.get('client_id'): - threading.Thread(target=sharesight.keep_warm, daemon=True).start() # keep the portfolio list and holdings report cached, while cache is on + if config.env.telegramBotToken: + threading.Thread(target=worker.refresh_keyboards, daemon=True).start() # stale DM keyboards, in the background if scheduler: jobs.current = scheduler # for the .jobs chat command threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs - gevent.signal_handler(signal.SIGHUP, reload_all) # startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron + # Startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron + gevent.signal_handler(signal.SIGHUP, in_thread(reload_all)) + gevent.signal_handler(signal.SIGUSR1, in_thread(flush_cache)) # systemctl kill -s USR1 finbot: delete the cache try: httpd.serve_forever() except OSError as e: diff --git a/bot_send_stdin.py b/bot_send_stdin.py index 6f7849c1..3e9851a2 100755 --- a/bot_send_stdin.py +++ b/bot_send_stdin.py @@ -2,13 +2,13 @@ import sys import os -from lib import webhook +from lib import delivery if os.isatty(sys.stdin.fileno()): raise RuntimeError("No stdin input detected (not piped or redirected)") payload = [line.strip() for line in sys.stdin if line.strip()] -for service, url, chat in webhook.default_chats(): - webhook.send(service, url, chat, payload) +for dest in delivery.default_chats(): + delivery.deliver(dest, delivery.Report(payload)) diff --git a/cal.py b/cal.py index 1a63e487..e631d21d 100755 --- a/cal.py +++ b/cal.py @@ -1,95 +1,5 @@ #!/usr/bin/env python3 - -import datetime -import sys - -from lib.config import * -from lib import telegram -from lib import util -from lib import webhook -from lib import yahoo - -EVENTS = {'earnings': ('earnings_date', 'Upcoming earnings'), 'ex-dividend': ('ex_dividend_date', 'Upcoming ex-dividends')} # -> (field, heading) - -def lambda_handler(chat_id=config_telegramChatID, days=UNSET, service=None, specific_stock=None, message_id=None, interactive=False, event='earnings'): - """The tracked stocks' (or one stock's) upcoming earnings or ex-dividend dates, as `event` says. A scheduled earnings report - is pinned on Telegram.""" - if days is UNSET: - days = config_future_days - field, heading = EVENTS[event] - def prepare_payload(service, market_data): - now = datetime.datetime.now() - soon = now + datetime.timedelta(days=days) - found = [] # (when, flag, title, link) - for ticker in list(market_data): - if market_data[ticker]['quoteType'] == 'ETF': - continue - ticker_primary = ticker - if field not in market_data[ticker]: # an ADR or a second listing: its primary listing may have the date - title_orig = market_data[ticker]['profile_title'] - if ticker in primary_listing: # dict from lib.config - ticker_primary = primary_listing[ticker] - elif ticker.endswith('.AX') and market_data[ticker].get('financialCurrency') == 'NZD': - ticker_primary = ticker.replace('.AX', '.NZ') # fixes GTK.AX and SKO.AX - market_data = market_data | yahoo.fetch_detail(ticker_primary) - country = market_data.get(ticker_primary, {}).get('profile_country') # only in the details - if country in yahoo_country and '.' not in ticker_primary: # dict from lib.config - ticker_primary = ticker_primary + '.' + yahoo_country[country] # fixes ASML and INFY - market_data = market_data | yahoo.fetch_detail(ticker_primary) - primary = market_data.get(ticker_primary, {}) - if primary.get('profile_title') != title_orig or field not in primary: # not the same company, or no date there either - continue - when = datetime.datetime.fromtimestamp(market_data[ticker_primary][field]) - if now < when < soon or specific_stock: - found.append((when, util.flag_from_ticker(ticker_primary), market_data[ticker]['profile_title'], util.yahoo_link(ticker, service))) - found.sort(key=lambda item: item[0]) - payload = [] - for date in sorted({when.date() for when, *_ in found}): - if not specific_stock: - payload += ["", webhook.bold(f"{date:%b %d}", service)] - for when, flag, title, link in found: - if when.date() == date: - payload.append(f"{flag} {date:%b %d} {title} ({link})" if specific_stock else f"{flag} {title} ({link})") - if payload and not specific_stock: - payload.insert(0, webhook.bold(f"{heading} {util.days_english(days, 'for the next ')}:", service)) - elif not payload and interactive: - payload = [f"No events found for {tickers[0]}" if specific_stock else f"No events found {util.days_english(days, 'for the next ')}"] - return payload - - def pin_if_earnings(service, results): - """Pin every message of the Telegram earnings report (several if it was chunked).""" - if not (event == 'earnings' and not specific_stock and service == 'telegram' and results): - return - sent = [] - for result in results: - try: - sent.append((result['result']['chat']['id'], result['result']['message_id'])) - except (KeyError, TypeError): - print("Unable to pin: unexpected response", file=sys.stderr) - for chat, message_id in reversed(sent): # pin last-to-first so the report's first message ends up on top - telegram.pinChatMessage(chat, message_id) - - # MAIN # - tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() - if event == 'earnings': - market_data = yahoo.fetch(tickers) - else: # ex-dividend dates are in Yahoo's details - market_data = yahoo.fetch_details(tickers, strict=bool(specific_stock)) - print("") - - for service, url, chat in webhook.destinations(interactive, service, chat_id): - pin_if_earnings(service, webhook.send(service, url, chat, prepare_payload(service, market_data), message_id=message_id)) - - # make google cloud happy - return True - -def job(argv): - """Arguments of a scheduled run: (label, lambda_handler kwargs). Raises ValueError with the usage on a bad argument.""" - match argv: - case ['earnings' | 'ex-dividend' as event]: - return f'cal.py {event}', {'event': event} - case _: - raise ValueError('[earnings|ex-dividend]') - -if __name__ == "__main__": - webhook.cli(sys.argv[0], job, lambda_handler) +"""./cal.py [arguments]: runs lib/reports/cal.py once, as its schedule line would.""" +if __name__ == '__main__': + from lib import reports + reports.cli(__file__) diff --git a/cleanup.py b/cleanup.py index 95a6bf57..530ee071 100755 --- a/cleanup.py +++ b/cleanup.py @@ -1,36 +1,5 @@ #!/usr/bin/env python3 - -import os, sys, time -from lib.config import * -from lib import webhook - -def lambda_handler(days=30): - """Delete the cache files (finbot_*) that were last written more than `days` days ago.""" - cutoff = time.time() - days * 86400 - removed = 0 - try: - entries = list(os.scandir(config_cache_dir)) - except FileNotFoundError: - return 0 - for entry in entries: - if entry.is_file(follow_symlinks=False) and entry.name.startswith('finbot_') and entry.stat().st_mtime < cutoff: - try: - os.unlink(entry.path) - removed += 1 - except OSError as e: - print("cleanup: can't remove", entry.path, e, file=sys.stderr) - print(f"cleanup: removed {removed} cache files older than {days} days from {config_cache_dir}") - return removed - -def job(argv): - """Arguments of a scheduled run: (label, lambda_handler kwargs). Raises ValueError with the usage on a bad argument.""" - match argv: - case []: - return 'cleanup.py', {} - case [days] if days.isdigit(): - return f'cleanup.py {days}', {'days': int(days)} - case _: - raise ValueError('[days]') - -if __name__ == "__main__": - webhook.cli(sys.argv[0], job, lambda_handler) +"""./cleanup.py [arguments]: runs lib/reports/cleanup.py once, as its schedule line would.""" +if __name__ == '__main__': + from lib import reports + reports.cli(__file__) diff --git a/defaults/finbot.env b/defaults/finbot.env index 489bcf38..cdbb8783 100644 --- a/defaults/finbot.env +++ b/defaults/finbot.env @@ -4,18 +4,23 @@ # docker: --env-file /opt/finbot.env # init.d: set -a; . /opt/finbot.env; set +a; ./bot.py # Keep it private: chmod 600, owned by root. It holds the credentials. -# Flat KEY=value lines: no spaces around "=", no quotes (docker keeps quotes literally), no trailing comments; comments on their own line. -# Everything else (thresholds, windows, display, caching, ...) is in preferences.conf, which the bot reloads without a restart. -# A preference key in this file is an error: finbot refuses to start. Lint both files with: util/check_config.py /opt/finbot.env +# Flat KEY=value lines: no spaces around "=", no quotes (docker keeps quotes literally), no trailing comments; comments +# on their own line. +# Everything else (thresholds, windows, display, caching, ...) is in preferences.conf, which the bot reloads without a +# restart. +# A preference key in this file is an error: finbot refuses to start. Lint both files with: +# util/check_config.py /opt/finbot.env -# Where state and cache live. Leave unset under systemd with StateDirectory=finbot (it uses $STATE_DIRECTORY, /var/lib/finbot). +# Where state and cache live. Leave unset under systemd with StateDirectory=finbot (it uses $STATE_DIRECTORY, +# /var/lib/finbot). # DATA_DIR=/var/lib/finbot # The listening socket ip=127.0.0.1 port=5000 -# Run the scheduled reports (schedule.cron) inside the bot, and how many minutes late a run missed while the bot was down may still fire +# Run the scheduled reports (schedule.cron) inside the bot, and how many minutes late a run missed while the bot was +# down may still fire scheduler=True scheduler_catchup=5 @@ -32,9 +37,12 @@ slack_webhook= discord_webhook= telegramBotToken= slackBotToken= +# Slack channel id for scheduled reports as the bot, charts and all (needs slackBotToken; slack_webhook is then unused) +slackChannelID= # Chat networks: inbound (interactive mode) telegramOutgoingWebhook= telegramOutgoingToken= slackOutgoingWebhook= slackOutgoingToken= +slackSigningSecret= diff --git a/defaults/preferences.conf b/defaults/preferences.conf index 8f0355f1..b8b69bed 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -1,17 +1,20 @@ # Preferences for finbot: how the bot behaves. The bot reads this file itself, at start and on every reload. # Apply an edit without a restart: sudo systemctl reload finbot (docker: docker kill -s HUP ) # Check an edit first: util/check_config.py /path/to/finbot.env -# Flat KEY=value lines, one per line. This is NOT an INI file: no [sections]. No spaces around "=", no quotes, no trailing comments; -# put comments on their own line. A key that is not listed here is an error and finbot refuses to start. -# Each value shown is the built-in default. The startup keys (credentials, ip, port, telegramChatID, scheduler, ...) are NOT set here: -# they live in finbot.env, the startup file, which the launcher loads and the bot never reads. Changing them needs a restart. +# Flat KEY=value lines, one per line. This is NOT an INI file: no [sections]. No spaces around "=", no quotes, no +# trailing comments; put comments on their own line. A key that is not listed here is an error and finbot refuses to +# start. +# Each value shown is the built-in default. The startup keys (credentials, ip, port, telegramChatID, scheduler, ...) +# are NOT set here: they live in finbot.env, the startup file, which the launcher loads and the bot never reads. +# Changing them needs a restart. # Reports: how far back and forward to look, and when to alert past_days=28 future_days=30 price_percent=5 shorts_percent=15 -# .buy, .sell and .forward pe list only stocks that at least this many analysts cover: with fewer, a consensus says little +# .buy, .sell and .forward pe list only stocks that at least this many analysts cover: with fewer, a consensus says +# little min_analysts=7 # price alerts: volatile and leveraged stocks need a bigger move; the multiplier is how much bigger demote_volatile=True @@ -24,24 +27,25 @@ include_portfolios= # Data sources: set trades_use_yahoo to False if Yahoo breaks trades_use_yahoo=True -performance_use_sharesight=False # Display -alliterate=False graph=True hyperlink=True hyperlinkFooter=True hyperlinkProvider=google -chunk_maxlines=20 -# Locale, used by the reminder report +# Regional: where you are. +# exchange_suffix: a ticker Yahoo does not know is tried again on your exchange, by Yahoo's suffix for it (AX: BHP is +# tried as BHP.AX; NZ, L, TO and so on for others). Empty: a ticker is taken as written. +# country_code and timezone: the reminder report's country and time zone. +exchange_suffix=AX country_code=AU timezone=Australia/Melbourne # Caching and network, in seconds cache=True cache_seconds=43200 -performance_cache_seconds=1800 +performance_cache_seconds=2400 http_timeout=18 # Diagnostics diff --git a/defaults/reminder.json b/defaults/reminder.json index f6168f83..1b9d05e0 100644 --- a/defaults/reminder.json +++ b/defaults/reminder.json @@ -7,6 +7,24 @@ "month_day": "06-12", "message": "SPCX IPO anniversary" }, + { + "month_day": "06-23", + "gate_country": "AU", + "message": [ + "💸 Realise capital gains/losses by EOFY June 30{flag}", + "", + "💰 Finalise any deductables by EOF June 30: {flag}", + "• donations*", + "• Super contributions†", + "• work expenses‡", + "• investment subscriptions§", + "", + "_* Organisation must be [DGR-registered](https://abr.business.gov.au/Tools/DgrListing)_", + "_† Allow transfer time for Super, as contributions can only be deducted for the year they're received. Stake users: see [note about XX PCC in transaction description](https://hellostake.com/au/support/stake-super/employer-and-personal-contributions/33442327233305#h_01JQAPN9A6ZM41NZKP6XZWN1KP)_", + "_‡ See [eligible work expenses](https://www.ato.gov.au/individuals-and-families/income-deductions-offsets-and-records/deductions-you-can-claim)_", + "_§ Must relate directly to specific trades and gains/income (e.g. not general finance news)_" + ] + }, { "month_day": "06-30", "message": "🥳 Happy EOFY 🇦🇺 🇧🇩 🇪🇬 🇪🇹 🇰🇪 🇳🇵 🇵🇰" diff --git a/defaults/schedule.cron b/defaults/schedule.cron index d11f5fe4..5020d035 100644 --- a/defaults/schedule.cron +++ b/defaults/schedule.cron @@ -1,5 +1,6 @@ -# The scheduled reports, run by the bot (bot.py). Same time syntax as crontab(5): minute hour day-of-month month day-of-week, -# or an @macro such as @daily. The command is a finbot script and its arguments; separate several with ";" to run them in turn. +# The scheduled reports, run by the bot (bot.py). Same time syntax as crontab(5): minute hour day-of-month month +# day-of-week, or an @macro such as @daily. The command is a finbot script and its arguments; separate several with ";" +# to run them in turn. # Times are in the server's own time zone, like cron; add a line "CRON_TZ=Australia/Melbourne" above a job to pin one. # To change the schedule, copy this file to the data directory (util/sync_defaults.py --apply), edit the copy, then: # systemctl reload finbot @@ -21,7 +22,11 @@ # Weekly 28 21 * * Sat cal.py earnings; cal.py ex-dividend; price.py 7; performance.py 7; rating.py -# Housekeeping: delete cache files older than 30 days +# Housekeeping: fetch afresh the Sharesight portfolio list and holdings every 30 minutes (cached 40 minutes) and the +# price histories of the stocks followed every 8 hours (cached 12 hours), so requests never wait on Sharesight or +# Yahoo; delete cache files older than 30 days +14,44 * * * * sharesight_warm.py +07 */8 * * * yahoo_warm.py 51 18 * * Sat cleanup.py 30 # Monthly diff --git a/doc/discord.md b/doc/discord.md new file mode 100644 index 00000000..82db2477 --- /dev/null +++ b/doc/discord.md @@ -0,0 +1,24 @@ +# Discord + +finbot reaches Discord through a webhook, which posts the scheduled reports into a channel. A webhook only posts, so +there is no choice to make: on Discord, finbot does not answer commands. For commands, set up finbot on +[Telegram](telegram.md) or [Slack](slack.md) as well. + +What the webhook carries: +* the scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest; +* their charts, attached to the messages; +* linked tickers, without link previews, and without pinging anyone. + +## Webhook + +1. In Discord, open _Server Settings > Integrations > Webhooks_ and choose _New Webhook_. +2. Name it (finbot, say), pick the channel for the reports, and _Copy Webhook URL_. +3. Put the URL in `finbot.env` as `discord_webhook`. +4. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the channel. + +``` +discord_webhook=https://discord.com/api/webhooks/1009998000000000000/aaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbb +``` + +A URL ending in `/slack`, from older setups, works as it is: finbot uses Discord's own endpoint either way. A message +holds up to 2,000 characters, so a long report arrives as several messages. diff --git a/doc/sharesight/finbot_sharesight_trades_684141_1.json b/doc/sharesight/finbot_sharesight_trades_684141_1.json index 7d3c02a5..2c2200ad 100644 --- a/doc/sharesight/finbot_sharesight_trades_684141_1.json +++ b/doc/sharesight/finbot_sharesight_trades_684141_1.json @@ -26,8 +26,7 @@ "confirmed": true, "links": { "portfolio": "https://api.sharesight.com/api/v2/portfolios/684141" - }, - "portfolio": "RobSMSF" + } }, { "id": 104446458, @@ -55,8 +54,7 @@ "confirmed": true, "links": { "portfolio": "https://api.sharesight.com/api/v2/portfolios/684141" - }, - "portfolio": "RobSMSF" + } } ], "api_transaction": { diff --git a/doc/slack.md b/doc/slack.md new file mode 100644 index 00000000..038d6150 --- /dev/null +++ b/doc/slack.md @@ -0,0 +1,115 @@ +# Slack + +finbot reaches Slack through a Slack app of your own. The app's incoming webhook posts the scheduled reports into a +channel; the app's bot also answers commands. The bot builds on the webhook, which carries the scheduled reports unless +the bot posts them itself, charts and all (`slackChannelID`). + +## Incoming webhook, or the interactive bot? + +| | Incoming webhook | Interactive bot (with the webhook) | +|---|---|---| +| Scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest | ✓, as text | ✓, as text through the webhook, or with their charts as the bot (`slackChannelID`) | +| Chat commands: `.price`, `@finbot price` and the rest ([the list](../README.md#supported-commands)) | ✗ | ✓ | +| Charts in the replies to commands | n/a | ✓, uploaded into the channel | +| A sign that finbot is working on a command | ✗ | ✓: ⏳ on the command's message, Slack having no typing notice for bots | +| Period buttons under a chart, which redraw it for that period in place | ✗ | ✓ | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| `finbot.env` keys | `slack_webhook` | That one, plus `slackOutgoingWebhook`, `slackSigningSecret` and `slackBotToken` (and `slackChannelID`, for charts in the scheduled reports) | + +Start with [Incoming webhook](#incoming-webhook). For commands, go on to [Interactive bot](#interactive-bot) +afterwards, in the same app. + +## Incoming webhook + +1. Create the app: at https://api.slack.com/apps choose _Create New App > From scratch_, name it (finbot, say) and pick + your workspace. +2. Under _Features > Incoming Webhooks_, turn _Activate Incoming Webhooks_ on, then _Add New Webhook to Workspace_ and + pick the channel for the reports. +3. Copy the webhook URL into `finbot.env` as `slack_webhook`. +4. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the channel. + +``` +slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA +``` + +An incoming webhook takes text only, so the scheduled reports come without charts. The [interactive bot](#interactive-bot) +can post them with their charts. + +## Interactive bot + +Slack delivers the messages that mention the bot (or every message in a channel, as you choose) to finbot over HTTPS. +finbot listens on `http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, +on a domain name with a valid certificate. + +1. Set up the [incoming webhook](#incoming-webhook), above. +2. Put finbot behind an HTTPS server. An nginx example, which lets in only Amazon's addresses, where Slack's requests + come from: + ``` + server { + listen 8443 ssl; + deny all; + + server_name www.example.com; + ssl_certificate /etc/letsencrypt/live/www.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/www.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location /slack { + proxy_pass http://127.0.0.1:5000/slack; + include /etc/nginx/aws_subnets; + deny all; + } + } + ``` + `util/nginx_aws_cron` writes `/etc/nginx/aws_subnets`, an `allow` line for each of Amazon's networks, and reloads + nginx. Run it once as root before nginx loads this configuration, or put it in `/etc/cron.daily/` to keep the list + current. +3. In the app's _Basic Information > App Credentials_, copy the _Signing Secret_ into `finbot.env` as + `slackSigningSecret`. finbot checks Slack's signature on every request with it, and turns away a request signed more + than five minutes earlier: a replay, or a server clock that is wrong. + * The older _Verification Token_, set as `slackOutgoingToken`, still works in place of the signing secret, but + Slack has deprecated it: the token travels in every request, where the signing secret never does. +4. In `finbot.env`, set `slackOutgoingWebhook` to the URL Slack should post to, and restart finbot, so that it can + answer Slack's check of the URL in the next step. +5. Under _Event Subscriptions_: + * Turn _Enable Events_ on, and enter the same URL as the _Request URL_. Slack checks it at once, and finbot answers. + * Under _Subscribe to bot events_, add `app_mention`: the bot sees messages that mention it, such as + `@finbot price`. + * Or add `message.channels` instead, for the bot to see every message in its channels and answer the commands + that start with `.`. Not both: the bot would answer a mention twice. + * For direct messages to the bot, add `message.im`, and tick _App Home > Allow users to send Slash commands and + messages from the messages tab_. + * _Save Changes_. +6. Under _Interactivity & Shortcuts_, turn _Interactivity_ on, enter the same URL as the _Request URL_, and _Save + Changes_. Slack posts there the presses of the period buttons under a chart; without it, Slack answers a press with + an error. +7. Under _OAuth & Permissions > Scopes > Bot Token Scopes_, add `chat:write`, `files:write`, `files:read` and + `reactions:write`. The upload of a chart needs both `files` scopes, and the ⏳ on a command that takes a while + `reactions:write`. +8. At the top of _OAuth & Permissions_, _Install to Workspace_ (or _Reinstall to Workspace_ when you changed scopes of + an installed app), and copy the _Bot User OAuth Token_ into `finbot.env` as `slackBotToken`. finbot replies with it; + without it, commands get no reply, and the log says so at startup and for each reply. +9. Restart finbot, and invite the bot to the channel: `/invite @finbot`. +10. Send `@finbot help` in the channel (or `.help`, with `message.channels`). +11. For charts in the scheduled reports, set `slackChannelID` to the channel's id (in Slack, click the channel's name: + the _Channel ID_, which starts with C, is at the bottom of _About_), and restart finbot. The bot then posts the + scheduled reports there, as it posts its replies, and `slack_webhook` is no longer used. + +``` +slack_webhook=https://hooks.slack.com/services/XXXXXXXXXXX/YYYYYYYYYYY/AAAAAAAAmockupAAAAAAAAAAAA +slackOutgoingWebhook=https://www.example.com:8443/slack +slackSigningSecret=0123456789abcdef0123456789abcdef +slackBotToken=xoxb-your-bot-token +slackChannelID=C0123456789 +``` + +A chart is uploaded with Slack's file upload API (`files.getUploadURLExternal`, then `files.completeUploadExternal`). +With its period buttons, it goes in a message of blocks: the report, the chart (uploaded into no channel, and shown by +its id) and the buttons. A press shows the chart for that period in the message in place of the last one: the upload of +that period from the last 15 minutes, for this message or another, with the report that came with it, or else the chart +drawn and uploaded anew (Slack cannot replace a file). An upload that no message shows is deleted once those 15 minutes +are past, so the bot's files do not pile up. If Slack will not show a chart with its buttons, the chart is uploaded into +the channel without them; if an upload fails (a missing scope, say), the report goes out as text. The log says why. + +`.jobs` lists the scheduled commands with a button for each, as on Telegram; a button from a list that has since +changed puts the current list in its place. diff --git a/doc/telegram.md b/doc/telegram.md new file mode 100644 index 00000000..82520fc2 --- /dev/null +++ b/doc/telegram.md @@ -0,0 +1,88 @@ +# Telegram + +finbot talks to Telegram as a bot user that you create with BotFather. The same bot can post the scheduled reports +only, or also answer commands in your chats: the second builds on the first. + +## Reports only, or the interactive bot? + +| | Reports only | Interactive bot | +|---|---|---| +| Scheduled reports: trades, price alerts, earnings and dividend dates, performance and the rest | ✓ | ✓ | +| Charts with the reports | ✓ | ✓ | +| Chat commands: `.price`, `.history`, `.performance` and the rest ([the list](../README.md#supported-commands)) | ✗ | ✓ | +| Period buttons under charts, and `.jobs` buttons | ✗ | ✓ | +| "typing…" while a command works | ✗ | ✓ | +| A keyboard of the common commands in a direct chat with the bot | ✗ | ✓ | +| Public HTTPS server | Not needed | Needed: a domain name with a valid certificate | +| `finbot.env` keys | `telegramBotToken`, `telegramChatID` | Those two, plus `telegramOutgoingWebhook` and `telegramOutgoingToken` | + +Start with [Reports only](#reports-only). For commands, go on to [Interactive bot](#interactive-bot) afterwards. + +## Reports only + +1. Create the bot: message [@BotFather](https://telegram.me/BotFather), send `/newbot` and answer its questions. It + replies with the bot's token: put it in `finbot.env` as `telegramBotToken`. +2. Add the bot to the group or channel the reports should go to. In a channel, make it an administrator, so that it can + post. +3. Stop strangers adding your bot to their groups: send BotFather `/setjoingroups`, pick the bot and choose `Disable`. + Do this after step 2, as it stops you adding the bot to groups too. +4. Put the chat's id in `finbot.env` as `telegramChatID`. + * A group's id is negative. A channel's or a supergroup's is negative and 13 characters long: prepend `-100` if + yours is shorter. + * To find it, send `/start@` in the chat, then open + `https://api.telegram.org/bot/getUpdates` in a browser and look for `"chat":{"id":`. This works + only while the bot has no webhook, so before you set up the interactive bot. + * A group's id changes when Telegram turns it into a supergroup, which some changes to its settings do. Update + `telegramChatID` then: finbot does not follow the change. +5. Restart finbot (`sudo systemctl restart finbot`). The scheduled reports now arrive in the chat. + +``` +telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA +telegramChatID=-1001000000000 +``` + +## Interactive bot + +Telegram delivers the messages in your chats, and each button press, to finbot over HTTPS. finbot listens on +`http://127.0.0.1:5000` (`ip` and `port` in `finbot.env`), so it needs a web server in front of it, on a domain name +with a valid certificate. Telegram posts only to ports 443, 80, 88 and 8443. + +1. Set up the bot for [reports only](#reports-only), above. +2. Let the bot read the chat: make it an administrator of the group. A bot that is not an administrator sees only the + commands that start with `/`, and finbot's start with `.`. Alternatively, send BotFather `/setprivacy` and choose + `Disable`. +3. Put finbot behind an HTTPS server. An nginx example, which lets in only Telegram's addresses: + ``` + server { + listen 8443 ssl; + deny all; + + server_name www.example.com; + ssl_certificate /etc/letsencrypt/live/www.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/www.example.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + location /telegram { + proxy_pass http://127.0.0.1:5000/telegram; + include /etc/nginx/telegram_subnets; + deny all; + } + } + ``` + `util/nginx_telegram_cron` writes `/etc/nginx/telegram_subnets`, an `allow` line for each of Telegram's networks, + and reloads nginx. Run it once as root before nginx loads this configuration, or put it in `/etc/cron.daily/` to + keep the list current. `util/iptables_telegram_cron` goes further, in the firewall: of the traffic to ports 443 and + 8443 it lets in only Telegram's networks on 8443 and drops the rest, so nothing else can be served on those ports. +4. In `finbot.env`, set `telegramOutgoingWebhook` to the URL Telegram should post to, and `telegramOutgoingToken` to a + password of your choosing: 1 to 256 characters, of letters, digits, `_` and `-`. Telegram sends the password with + every update, and finbot turns away a request without it. +5. Restart finbot. As it starts, it registers the URL with Telegram, unregistering it first, which also wakes up a stale + connection. If Telegram refuses, finbot does not start, and the log says why. +6. Send `.help` in the chat. In a direct chat with the bot, the reply also brings a keyboard of the common commands. + +``` +telegramBotToken=0000000000:AAAAAAAAAAAAAAAAAAAAAAAAAA +telegramChatID=-1001000000000 +telegramOutgoingWebhook=https://www.example.com:8443/telegram +telegramOutgoingToken=a-long-random-password +``` diff --git a/finbot-flush.service b/finbot-flush.service new file mode 100644 index 00000000..d79c3580 --- /dev/null +++ b/finbot-flush.service @@ -0,0 +1,7 @@ +[Unit] +Description=Flush finbot's cache (sudo systemctl start finbot-flush) + +[Service] +Type=oneshot +# finbot deletes its cache files on SIGUSR1 and keeps its state; its job processes ignore the signal +ExecStart=/bin/systemctl kill --signal=USR1 finbot.service diff --git a/finbot.service b/finbot.service index 2fff0c7a..fd74abf0 100644 --- a/finbot.service +++ b/finbot.service @@ -8,7 +8,8 @@ User=CHANGEME ExecStart=/home/CHANGEME/finbot/bot.py # systemctl reload finbot re-reads preferences.conf and schedule.cron without a restart ExecReload=/bin/kill -HUP $MAINPID -# Refuse to start on a mistake in finbot.env or preferences.conf (the bot never reads finbot.env itself, so this is where its typos are caught). "+" runs it as root. +# Refuse to start on a mistake in finbot.env or preferences.conf (the bot never reads finbot.env itself, so this is +# where its typos are caught). "+" runs it as root. ExecStartPre=+/home/CHANGEME/finbot/util/check_config.py /opt/finbot.env Restart=on-failure RestartSec=30s @@ -16,7 +17,8 @@ StandardOutput=null StandardError=journal SyslogIdentifier=finbot Environment=PYTHONUNBUFFERED=1 -# finbot.env is the startup file: systemd loads it (as root), the bot only inherits the values. chmod 600. A change needs a restart. +# finbot.env is the startup file: systemd loads it (as root), the bot only inherits the values. chmod 600. A change +# needs a restart. EnvironmentFile=/opt/finbot.env [Install] diff --git a/lib/answer.py b/lib/answer.py new file mode 100644 index 00000000..f114ce48 --- /dev/null +++ b/lib/answer.py @@ -0,0 +1,12 @@ +"""A reply in place of the report asked for. A command that cannot give what was asked (a portfolio name that is none of +them, a ticker with no price history) raises Answer, and the chat gets its lines as an ordinary reply, where any other +exception becomes a one-line error report.""" + + +class Answer(Exception): + """The lines to reply with instead of the report.""" + + def __init__(self, *lines): + """lines: the reply, a line each.""" + self.lines = [str(line) for line in lines] + super().__init__(' '.join(self.lines)) diff --git a/lib/buttons.py b/lib/buttons.py new file mode 100644 index 00000000..cf8b0982 --- /dev/null +++ b/lib/buttons.py @@ -0,0 +1,89 @@ +"""Buttons under a message, and what their data says when one is pressed (parse): a chart's period buttons (1D | 7D | +1M | 1Y | Max), whose data carries what its builder needs (charts.BUILDERS), and a .jobs command's button. Each +transport draws Button rows its own way.""" +import hashlib +import typing + +from lib import state + +class Button(typing.NamedTuple): + """One button: its label, the data a press sends back, and whether it is the one shown now (marked).""" + label: str + data: str + active: bool = False + + +class ChartPress(typing.NamedTuple): + """A chart's period button pressed: the chart's kind, the period, and its builder's arguments (resolve them).""" + kind: str + period: str + ref: str + + +class JobPress(typing.NamedTuple): + """A .jobs command's button pressed: the command's number in the list and its checksum, both for + jobs.Scheduler.pick to check.""" + number: str + checksum: str + + +PERIODS = {'d': ('1D', 1), 'w': ('7D', 7), 'm': ('1M', 30), 'y': ('1Y', 365), 'x': ('Max', None)} +BUTTONS = {'h': 'wmyx', 'p': 'wmyx', 'c': 'wmyx', 'f': 'wmyx', 'l': 'dwmyx'} # chart kind -> its buttons, five at most +# (Discord's limit for a row): h history, p a stock's price chart, c compare, f performance, l the price list +MAX_DAYS = 3650 # what Max means for the Sharesight-based reports +MAX_CALLBACK_BYTES = 64 # Telegram limit on callback_data +REF_FILE = 'finbot_chart_buttons.json' +REF_LIMIT = 500 # the stored argument lists kept: a button older than that says it has expired + +def period_for_days(days, default=None): + """Button key for a day count, if it is one of the button periods; `default` when it is not (or days is None).""" + for key, (_, d) in PERIODS.items(): + if d and d == days: + return key + return default + +def _ref(args): + """A builder's arguments as callback data: inline if they fit, else a short id of the list, stored on disk with the + newest REF_LIMIT others.""" + inline = ','.join(args) + if len(('c|h|w|' + inline).encode()) <= MAX_CALLBACK_BYTES: + return inline + key = hashlib.sha1(inline.encode()).hexdigest()[:10] + with state.edit(REF_FILE, {}) as refs: + refs[key] = args + for old in list(refs)[:-REF_LIMIT]: + del refs[old] + return '#' + key + +def resolve(ref): + """The arguments a button's callback data carries: listed inline ('T1,T2'), or a stored list's '#id' (see _ref). + Raises KeyError when the stored list is gone.""" + if ref.startswith('#'): + args = state.load(REF_FILE, {}).get(ref[1:]) + if not args: + raise KeyError("these buttons have expired, please re-run the command") + return args + return ref.split(',') + +def keyboard(kind, args, active=None): + """A chart's period buttons, one row, the active period marked. args: what the kind's builder takes + (charts.BUILDERS): a ticker, tickers, a portfolio, or a price list's threshold and top, as strings.""" + ref = _ref(args) + return [[Button(PERIODS[key][0], f"c|{kind}|{key}|{ref}", key == active) for key in BUTTONS[kind]]] + +def job_button(number, label, checksum): + """A .jobs command's button: 'N. label', cut to 30 characters, its data the number and the command's checksum.""" + return Button(f"{number}. {label}"[:30], f"j|{number}|{checksum}") + +def parse(data): + """What a pressed button's data asks for: a JobPress for a .jobs button (whatever follows 'j|', which pick refuses + unless it is a command's number and checksum), a ChartPress for a period button of a known kind and period, or None + for data of neither (from an older finbot, or forged).""" + kind, _, rest = data.partition('|') + if kind == 'j': + number, _, checksum = rest.partition('|') + return JobPress(number, checksum) + parts = rest.split('|', 2) + if kind == 'c' and len(parts) == 3 and parts[0] in BUTTONS and parts[1] in BUTTONS[parts[0]]: + return ChartPress(*parts) + return None diff --git a/lib/cache.py b/lib/cache.py new file mode 100644 index 00000000..7c8bb7ac --- /dev/null +++ b/lib/cache.py @@ -0,0 +1,117 @@ +"""Cache files in DATA_DIR/cache: copies of Yahoo and Sharesight answers and of drawn charts, each good for so many +seconds. Anything here may be deleted at any time (cleanup.py, a flush); what must last lives in lib/state.py.""" +import io +import json +import os +import sys +import tempfile +import time +from pathlib import Path + +from lib import config + + +def path(name): + """Where a cache file lives.""" + return config.env.cache_dir + '/' + name + + +def age(name): + """Seconds since the cache file was written, or None when it does not exist.""" + try: + return time.time() - os.path.getmtime(path(name)) + except OSError: + return None + + +def _fresh(name, seconds): + """The cache file's path if written less than `seconds` ago (cache_seconds when None), else None.""" + if seconds is None: + seconds = config.prefs.cache_seconds + where, written = path(name), age(name) + if written is None: + print("cache miss:", where, file=sys.stderr) if config.prefs.debug else None + return None + if written >= seconds: + print("cache expired:", where, file=sys.stderr) if config.prefs.debug else None + return None + print("cache hit:", where, "TTL:", f"{seconds - written:.0f} s", file=sys.stderr) if config.prefs.debug else None + return where + + +def _read_fresh(name, seconds, read): + """read(path) of the cache file if fresh, else None; also None if it goes (cleanup, a flush) after the check.""" + where = _fresh(name, seconds) + if not where: + return None + try: + return read(Path(where)) + except FileNotFoundError: + return None + + +def read(name, seconds=None): + """A cache file's JSON, or None when it is missing or older than `seconds` (cache_seconds when None).""" + return _read_fresh(name, seconds, lambda where: json.loads(where.read_text(encoding="utf-8"))) + + +def read_bytes(name, seconds=None): + """A cache file's bytes (as a file object), or None when it is missing or older than `seconds`.""" + return _read_fresh(name, seconds, lambda where: io.BytesIO(where.read_bytes())) + + +def load(name): + """A cache file's JSON whatever its age, or None: for an answer with its own expiry (the Sharesight token).""" + where = Path(path(name)) + try: + return json.loads(where.read_text(encoding="utf-8")) + except FileNotFoundError: + return None + + +def cached(name, seconds, fetch): + """fetch()'s answer (JSON), as sent: from the cache file `name` while it is under `seconds` old (cache_seconds when + None) and the cache preference is on, else fetched and written there unless empty. Answers are cached as sent, not + parsed, so that a parser fix needs no flush.""" + if config.prefs.cache and (data := read(name, seconds)): + return data + data = fetch() + if data and config.prefs.cache: + write(name, data) + return data + + +def write(name, data): + """Write a cache file's JSON, compact: nobody reads it but finbot (a 10-year chart takes a third of the space, and + parses faster). See write_file.""" + where = path(name) + print("cache write:", where, file=sys.stderr) if config.prefs.debug else None + write_file(where, json.dumps(data, separators=(',', ':')).encode('utf-8')) + + +def write_bytes(name, data): + """Write a cache file's bytes from a file object (a drawn chart).""" + data.seek(0) + write_file(path(name), data.getvalue()) + + +def write_file(where, data, sync=False): + """Write `data` (bytes) to `where` whole or not at all, mode 0640: it goes to a temp file beside it, which then + replaces the file, so a failure part way (a full disk, a kill) leaves the old file as it was. sync=True also flushes + it to disk, for state that must survive a power cut. The process umask is left alone.""" + where = os.path.realpath(where) # through a symlink, as an in-place write would + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(where), prefix=os.path.basename(where) + '.', suffix='.tmp') + try: + with os.fdopen(fd, 'wb') as f: + f.write(data) + if sync: + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp, 0o640) + os.replace(tmp, where) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise diff --git a/lib/charts.py b/lib/charts.py index 124b7276..0dbf7781 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -1,160 +1,56 @@ -"""Period buttons (Week | Month | Quarter | Max) for Telegram chart messages, and the rebuild behind them.""" -import hashlib -import json -import sys -import threading - -from lib import util -from lib import webhook +"""A chart with period buttons, built by its kind's builder: the command's first view, and again for the period of a +button pressed, so that both look alike.""" +from lib import answer +from lib import buttons +from lib import formats +from lib import links from lib import yahoo -from lib import reports - -PERIODS = {'d': ('1D', 1), 'w': ('7D', 7), 'm': ('1M', 30), 'q': ('3M', 90), 'y': ('1Y', 365), 'x': ('Max', None)} -BUTTONS = {'h': 'wmqyx', 'p': 'wmqyx', 'c': 'wmqyx', 'f': 'wmqyx', 'l': 'dwmqyx'} # chart kind -> buttons offered (h=history, p=price chart, c=compare, f=performance, l=price list) -MAX_DAYS = 3650 # what Max means for the Sharesight-based reports -MAX_CALLBACK_BYTES = 64 # Telegram limit on callback_data -REF_FILE = 'finbot_chart_buttons.json' - -def period_for_days(days, default=None): - """Button key for a day count, if it is one of the button periods; `default` when it is not (or days is None).""" - for key, (_, d) in PERIODS.items(): - if d and d == days: - return key - return default - -def _ref(tickers): - """Tickers as callback data: inline if it fits, else a short id persisted to disk.""" - inline = ','.join(tickers) - if len(('c|h|w|' + inline).encode()) <= MAX_CALLBACK_BYTES: - return inline - key = hashlib.sha1(inline.encode()).hexdigest()[:10] - refs = util.json_load(REF_FILE, persist=True) or {} - if refs.get(key) != tickers: - refs[key] = tickers - util.json_write(REF_FILE, refs, persist=True) - return '#' + key - -def resolve(ref): - if ref.startswith('#'): - tickers = (util.json_load(REF_FILE, persist=True) or {}).get(ref[1:]) - if not tickers: - raise KeyError("these buttons have expired, please re-run the command") - return tickers - return ref.split(',') - -COMMAND_BUTTONS = sorted(['.beta', '.buy & sell', '.dividend', '.earnings', '.forward pe', '.marketcap', '.pe', '.peg', '.performance', '.premarket', - '.price', '.session', '.shorts', '.trades', '.watchlist']) # alphabetical, row by row; '.buy & sell' sends both rating reports - -def command_keyboard(per_row=3): - """Persistent reply keyboard of the common commands (Telegram DMs); a press sends the command text as a message.""" - rows = [[{'text': c} for c in COMMAND_BUTTONS[i:i + per_row]] for i in range(0, len(COMMAND_BUTTONS), per_row)] - return {'keyboard': rows, 'resize_keyboard': True, 'is_persistent': True} - -KEYBOARD_FILE = 'finbot_keyboards.json' # chat id -> version of the command keyboard that chat last received -_keyboard_lock = threading.Lock() - -def keyboard_version(): - return hashlib.sha1(json.dumps(command_keyboard(), sort_keys=True).encode()).hexdigest()[:8] - -def mark_keyboard(chat_id): - """Record that this chat now has the current command keyboard.""" - with _keyboard_lock: - state = util.json_load(KEYBOARD_FILE, persist=True) or {} - state[str(chat_id)] = keyboard_version() - util.json_write(KEYBOARD_FILE, state, persist=True) - -def push_keyboard(chat_id): - """Telegram can only change a reply keyboard by sending a message, so send a short one carrying it.""" - result = webhook.write('telegram', webhook.chat_url('telegram', chat_id), '⌨️ Keyboard updated', chat_id, reply_markup=command_keyboard()) - if result is not None: - mark_keyboard(chat_id) - return result - -def ensure_keyboard(chat_id): - """DMs only (positive chat ids): bring a chat's keyboard up to date if its buttons have changed or it never had them.""" - if int(chat_id) <= 0: - return - with _keyboard_lock: - current = (util.json_load(KEYBOARD_FILE, persist=True) or {}).get(str(chat_id)) == keyboard_version() - if not current: - push_keyboard(chat_id) - -def refresh_keyboards(): - """At startup: update every chat we know of whose keyboard is out of date.""" - with _keyboard_lock: - state = util.json_load(KEYBOARD_FILE, persist=True) or {} - for chat_id, version in state.items(): - if version != keyboard_version(): - try: - push_keyboard(chat_id) - except Exception as e: - print("keyboard refresh failed for", chat_id, e, file=sys.stderr) - -def keyboard(kind, tickers, active=None): - """Telegram inline keyboard; the active period is marked.""" - ref = _ref(tickers) - row = [] - for key in BUTTONS[kind]: - label = PERIODS[key][0] - row.append({'text': ('● ' + label) if key == active else label, 'callback_data': f"c|{kind}|{key}|{ref}"}) - return {'inline_keyboard': [row]} - -def history_payload(ticker, service, highlight=None, graph=False): - """Caption lines for .history: performance table, with the selected interval in bold. Returns (lines, percent table, image): - the full history's chart with graph=True (if the graph preference is on), else None.""" - market_data = yahoo.fetch_detail(ticker, 600) - title = market_data.get(ticker, {}).get('profile_title', '') - ticker_link = util.finance_link(ticker, market_data.get(ticker, {}).get('profile_exchange', ''), service, days=1825, brief=False) - if ticker not in market_data or 'percent_change' not in market_data[ticker]: - return [f".history: no data found for ticker {ticker}"], None, None - table, image = yahoo.price_history(ticker) if graph else yahoo.price_history(ticker, graph=False) - lines = [webhook.bold(f"{title} ({ticker_link}) performance history", service)] - for interval in ('Max', '10Y', '5Y', '3Y', '1Y', 'YTD', '6M', '3M', '1M', '7D', '1D'): - if interval in table: - percent = table[interval] - label = webhook.bold(interval + ':', service) - if interval == highlight: - label = '● ' + label - lines.append(f"{util.get_emoji(percent)} {label} {util.signed_percent(percent)}") - return lines, table, image - -def build(kind, tickers, period, service='telegram'): - """(caption, image) for a chart kind at a button period; period None = the command's default view.""" - days = PERIODS[period][1] if period else None - if kind == 'h': - ticker = tickers[0] - lines, table, image = history_payload(ticker, service, PERIODS[period or 'x'][0], graph=not days) # default view is the full history - if table is None: - raise RuntimeError(lines[0]) - if days: - _, image = yahoo.price_history(ticker, days) - return '\n'.join(lines), image - if kind == 'p': - ticker = tickers[0] - market_data = yahoo.fetch([ticker]) - if ticker not in market_data: - raise RuntimeError(f"no data found for {ticker}") - percent, image = yahoo.price_history(ticker, days, graphCache=False) - percent = float(percent.get(days, percent.get('Max', next(iter(percent.values()), 0)))) - link = util.quote_link(ticker, market_data[ticker]['profile_exchange'], service, days=days) - return f"{util.get_emoji(percent)} {market_data[ticker]['profile_title']} ({link}) {util.signed_percent(round(percent))}", image - if kind == 'c': - args = list(tickers) + ([f'{days}d'] if days else ['max']) - caption, image, *_ = reports.prepare_compare(service, args) - if not image: - raise RuntimeError(caption[0]) - return '\n'.join(caption), image - if kind == 'f': # performance: tickers = [portfolio name or ''] - import performance - payload, image = performance.report(service, days or MAX_DAYS, tickers[0] or None, interactive=True) - if not image: - raise RuntimeError(payload[0] if payload else 'no performance data') - return '\n'.join(payload), image - if kind == 'l': # price list: tickers = [threshold, top] - import price - span = {'interday': True} if period == 'd' else {'days': days or MAX_DAYS} # 'd' is today's moves: the command's default view, not a 1-day lookback - payload, image = price.report(service, threshold=float(tickers[0]), interactive=True, top=int(tickers[1]) or None, **span) - if not image: - raise RuntimeError(payload[0] if payload else 'no price data') - return '\n'.join(payload), image - raise ValueError(f"unknown chart kind {kind}") +from lib.reports import history, performance, price + +def history_chart(service, period, days, ticker): + """.history: the change over every period, the chosen one marked, and the line of that period (or of it all).""" + lines, chart = history.history_payload(ticker, service, days) + return '\n'.join(lines), chart + +def price_chart(service, period, days, ticker): + """.price SYMBOL [period]: one stock's change over the period (all its history for Max), from Yahoo, and its + line.""" + market_data = yahoo.fetch([ticker]) + if ticker not in market_data: + raise answer.Answer(f"no data found for {ticker}") + changes, chart = history.price_line(ticker, days, market_data[ticker]) + percent = float(changes.get(days, changes.get('Max', next(iter(changes.values()), 0)))) + link = links.finance_link(ticker, market_data[ticker]['profile_exchange'], service, days=days) + title = market_data[ticker]['profile_title'] + caption = f"{formats.get_emoji(percent)} {title} ({link}) {formats.signed_percent(round(percent))}" + return caption, chart + +def compare_chart(service, period, days, *tickers): + """.compare: the stocks' changes over the period, rebased on one chart.""" + caption, chart = history.prepare_compare(service, list(tickers), days) + return '\n'.join(caption), chart + +def performance_chart(service, period, days, portfolio): + """.performance: each portfolio over the period, or the one named ('' for all).""" + payload, chart = performance.report(service, days or buttons.MAX_DAYS, portfolio or None) + return '\n'.join(payload), chart + +def price_list_chart(service, period, days, threshold, top): + """.price: the moves past `threshold`, or the top and bottom `top` ('0': not a top list), over the period; period + 'd' is today's moves, the command's default view, not a lookback of one day.""" + span = {'mode': 'interday'} if period == 'd' else {'mode': 'period', 'days': days or buttons.MAX_DAYS} + threshold, top = float(threshold), int(top) or None + found = price.moves(threshold=threshold, interactive=True, **span) + payload, chart = price.report(service, found, threshold=threshold, interactive=True, top=top, **span) + if not chart: + raise answer.Answer(*(payload or ['no price data'])) + return '\n'.join(payload), chart + +BUILDERS = {'h': history_chart, 'p': price_chart, 'c': compare_chart, 'f': performance_chart, 'l': price_list_chart} + +def build(kind, args, service, period=None, days=None): + """(caption, chart spec or None) of a chart kind from the arguments its buttons carry (buttons.keyboard): over a + button period, or over `days` (a command's own span: .compare 3m), or the command's default view.""" + if kind not in BUILDERS: + raise ValueError(f"unknown chart kind {kind}") + return BUILDERS[kind](service, period, buttons.PERIODS[period][1] if period else days, *args) diff --git a/lib/config.py b/lib/config.py index 45c2f8c1..7a22a430 100644 --- a/lib/config.py +++ b/lib/config.py @@ -1,3 +1,4 @@ +import dataclasses import difflib import json import os @@ -6,32 +7,33 @@ from pathlib import Path # TO CONFIGURE, DO NOT EDIT THIS FILE. There are two config files: -# finbot.env the startup file: DATA_DIR, ip, port, credentials, telegramChatID, scheduler. Never read by the bot: the launcher loads it -# into the environment (systemd EnvironmentFile=, docker --env-file, init.d `set -a; . finbot.env`). A change needs a restart. -# preferences.conf how the bot behaves: /preferences.conf, read by the bot itself at start and on every reload (SIGHUP). +# finbot.env the startup file: DATA_DIR, ip, port, credentials, telegramChatID, scheduler. Never read by the +# bot: the launcher loads it into the environment (systemd EnvironmentFile=, docker --env-file, +# init.d `set -a; . finbot.env`). A change needs a restart. +# preferences.conf how the bot behaves: /preferences.conf, read by the bot itself at start and on every +# reload (SIGHUP). REPO_DIR = Path(__file__).resolve().parent.parent -os.chdir(REPO_DIR) - -UNSET = object() # "argument not given": a default that follows a preference is resolved when the function runs, so a reload is seen - -STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', - 'sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', - 'slackOutgoingToken', 'slackOutgoingWebhook', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') -PREFERENCE_KEYS = ('alliterate', 'cache', 'cache_seconds', 'chunk_maxlines', 'country_code', 'debug', 'demote_leveraged', 'demote_volatile', - 'exclude_portfolios', 'future_days', 'graph', 'http_timeout', 'hyperlink', 'hyperlinkFooter', 'hyperlinkProvider', 'include_portfolios', - 'min_analysts', 'past_days', 'performance_cache_seconds', 'performance_use_sharesight', 'price_percent', 'print_headers', 'shorts_percent', 'timezone', - 'trades_use_yahoo', 'volatility_multiplier') -SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', - 'slackOutgoingToken', 'slackOutgoingWebhook', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed + +STARTUP_KEYS = ('DATA_DIR', 'ip', 'port', 'telegramChatID', 'scheduler', 'scheduler_catchup', 'sharesight_code', + 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', + 'slackChannelID', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', 'telegramBotToken', + 'telegramOutgoingToken', 'telegramOutgoingWebhook') +SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', + 'discord_webhook', 'slackBotToken', 'slackOutgoingToken', 'slackOutgoingWebhook', 'slackSigningSecret', + 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' -PASSED = 'FINBOT_PREFERENCES_PASSED' # set by the bot for its job processes: the preferences arrive in the environment, do not read the file +RETIRED = ('alliterate', 'chunk_maxlines', 'performance_use_sharesight') # preferences.conf keys that do nothing any +# more: accepted, with a note at start to delete them +# set by the bot for its job processes: the preferences arrive in the environment, do not read the file +PASSED = 'FINBOT_PREFERENCES_PASSED' OLD_ENV = REPO_DIR / '.env' # the single config file of old versions, which python-dotenv loaded; nothing reads it now class ConfigError(Exception): """One or more problems, each already naming the file and line. `errors` is the list.""" def __init__(self, errors): + """errors: one message or a list of them.""" self.errors = [errors] if isinstance(errors, str) else list(errors) super().__init__('\n'.join(self.errors)) @@ -41,7 +43,8 @@ def key_problem(key, this_keys, other_keys, other_file): if key in this_keys: return None if key in other_keys: - return f"{key} belongs in {other_file}" + (' (a change needs a restart)' if other_file == 'finbot.env' else ' (it is reloadable)') + why = ' (a change needs a restart)' if other_file == 'finbot.env' else ' (it is reloadable)' + return f"{key} belongs in {other_file}" + why if key == 'volatile_multiplier': return 'volatile_multiplier was renamed to volatility_multiplier' near = difflib.get_close_matches(key, list(this_keys) + list(other_keys), n=1) @@ -49,8 +52,8 @@ def key_problem(key, this_keys, other_keys, other_file): def parse_assignments(text, path, classify): - """KEY=value lines -> ({key: value}, [errors]). Blank and # lines are skipped, one pair of matching quotes is stripped, `classify(key)` - returns a reason to refuse the key. Every error names the file and line.""" + """KEY=value lines -> ({key: value}, [errors]). Blank and # lines are skipped, one pair of matching quotes is + stripped, `classify(key)` returns a reason to refuse the key. Every error names the file and line.""" values, errors = {}, [] for number, raw in enumerate(text.splitlines(), 1): line = raw.strip() @@ -89,11 +92,12 @@ def read_text(path): def read_preferences(path): - """{key: value} of a preferences.conf, None if the file does not exist. Raises ConfigError for anything wrong with it.""" + """{key: value} of a preferences.conf, None if there is none. Raises ConfigError for anything wrong with it.""" text = read_text(path) if text is None: return None - values, errors = parse_assignments(text, path, lambda key: key_problem(key, PREFERENCE_KEYS, STARTUP_KEYS, 'finbot.env')) + values, errors = parse_assignments(text, path, lambda key: None if key in RETIRED else + key_problem(key, PREFERENCE_KEYS, STARTUP_KEYS, 'finbot.env')) if errors: raise ConfigError(errors) return values @@ -104,26 +108,51 @@ def read_startup_file(path): text = read_text(path) if text is None: raise ConfigError(f'{path} does not exist') - values, errors = parse_assignments(text, path, lambda key: key_problem(key, STARTUP_KEYS, PREFERENCE_KEYS, PREFERENCES_FILE)) + values, errors = parse_assignments(text, path, + lambda key: key_problem(key, STARTUP_KEYS, PREFERENCE_KEYS, PREFERENCES_FILE)) if errors: raise ConfigError(errors) return values +TRUE_WORDS = ('true', '1', 't', 'yes', 'y', 'on') +FALSE_WORDS = ('false', '0', 'f', 'no', 'n', 'off') + + class _Reader: - """Typed reads from a getenv(name, default); a bad value is remembered, the default used, and all problems raised together.""" + """Typed reads from a getenv(name, default); a bad value is remembered, the default used, and all problems raised + together. A blank value means unset: the default.""" def __init__(self, getenv): + """getenv(name, default) supplies the values; the problems found collect in `errors` until check().""" self.getenv = getenv self.errors = [] + def value(self, name, default): + """The raw value, or default when it is unset or blank.""" + value = self.getenv(name, default) + return default if value == '' else value + def str(self, name, default=None): - return self.getenv(name, default) + """A text setting.""" + return self.value(name, default) + + def blankable(self, name, default): + """A text setting where a blank value means none (''), not the default.""" + value = self.getenv(name, default) + return '' if value is None else str(value).strip() def bool(self, name, default): - return str(self.getenv(name, default)).lower() in ('true', '1', 't') + """A boolean: true/false, yes/no, on/off, 1/0 or t/f in any case; anything else is an error, giving default.""" + value = self.value(name, default) + word = str(value).strip().lower() + if word in TRUE_WORDS or word in FALSE_WORDS: + return word in TRUE_WORDS + self.errors.append(f"{name}={value!r} is not true or false") + return default def number(self, kind, name, default): - value = self.getenv(name, default) + """A setting of type `kind` (int or float). A value that is not one is an error, and gives default.""" + value = self.value(name, default) try: return kind(value) except ValueError: @@ -131,94 +160,147 @@ def number(self, kind, name, default): return kind(default) def int(self, name, default): + """A whole-number setting.""" return self.number(int, name, default) def float(self, name, default): + """A number setting.""" return self.number(float, name, default) + def words(self, name, default): + """A list of words, separated by spaces: a tuple of them.""" + value = self.value(name, default) + return tuple(value.split()) if isinstance(value, str) else tuple(value) + def check(self): + """Raise ConfigError with every problem found so far.""" if self.errors: raise ConfigError(self.errors) +@dataclasses.dataclass(frozen=True) +class Env: + """The startup settings: finbot.env, which the launcher loads into the environment; fixed for the process.""" + data_dir: str # state: watchlist, seen trades, ratings, the schedule's record, preferences.conf + cache_dir: str # data_dir/cache + ip: str + port: int + scheduler: bool # run the scheduled reports (schedule.cron) inside bot.py + scheduler_catchup: int # minutes: a run missed while the bot was down still fires if it was due this recently + telegramChatID: str # '' when unset, like every token below + telegramBotToken: str + telegramOutgoingWebhook: str + telegramOutgoingToken: str + slackBotToken: str + slackChannelID: str # with slackBotToken, scheduled reports go to this channel as the bot, in place of slack_webhook + slackOutgoingWebhook: str + slackOutgoingToken: str + slackSigningSecret: str + sharesight_auth: dict + webhooks: dict # service -> outbound URL: Telegram's Bot API base, the Slack and Discord incoming webhooks + + @property + def preferences_path(self): + """Where preferences.conf is read from: the data dir.""" + return self.data_dir + '/' + PREFERENCES_FILE + + def startup_values(getenv): - """The settings fixed when the bot starts (finbot.env), computed from getenv(name, default).""" + """The settings fixed when the bot starts (finbot.env), computed from getenv(name, default). Raises ConfigError.""" r = _Reader(getenv) - data_dir = (getenv('DATA_DIR') or (getenv('STATE_DIRECTORY') or '').split(':')[0] or 'var').rstrip('/') or '/' - values = { - 'config_data_dir': data_dir, - 'config_sharesight_auth': { - "grant_type": 'client_credentials', - "code": getenv('sharesight_code'), - "client_id": getenv('sharesight_client_id'), - "client_secret": getenv('sharesight_client_secret'), - "redirect_uri": 'urn:ietf:wg:oauth:2.0:oob' - }, - 'config_scheduler': r.bool('scheduler', True), # run the scheduled reports (schedule.cron) inside bot.py - 'config_scheduler_catchup': r.int('scheduler_catchup', 5), # minutes: a run missed while the bot was down still fires if it was due this recently - 'config_ip': r.str('ip', '127.0.0.1'), - 'config_port': r.int('port', 5000), - 'config_slackBotToken': r.str('slackBotToken', False), - 'config_slackOutgoingToken': r.str('slackOutgoingToken', False), - 'config_slackOutgoingWebhook': r.str('slackOutgoingWebhook', False), - 'config_telegramBotToken': r.str('telegramBotToken', False), - 'config_telegramOutgoingWebhook': r.str('telegramOutgoingWebhook', False), - 'config_telegramChatID': r.str('telegramChatID', False), - 'config_telegramOutgoingToken': r.str('telegramOutgoingToken', False), - } + data_dir = getenv('DATA_DIR') or (getenv('STATE_DIRECTORY') or '').split(':')[0] or 'var' + data_dir = str(REPO_DIR / data_dir) # a relative one is in the checkout, wherever finbot starts hooks = {} if getenv('slack_webhook'): hooks['slack'] = getenv('slack_webhook').rstrip('/') if getenv('discord_webhook'): - hooks['discord'] = getenv('discord_webhook').rstrip('/').removesuffix('/slack') + '/slack' + # the native endpoint, not the Slack-compatible one + hooks['discord'] = getenv('discord_webhook').rstrip('/').removesuffix('/slack') if getenv('telegramBotToken'): hooks['telegram'] = 'https://api.telegram.org/bot' + getenv('telegramBotToken').rstrip('/') + '/' - values['webhooks'] = hooks + env = Env( + data_dir=data_dir, + cache_dir=data_dir + '/cache', + ip=r.str('ip', '127.0.0.1'), + port=r.int('port', 5000), + scheduler=r.bool('scheduler', True), + scheduler_catchup=r.int('scheduler_catchup', 5), + telegramChatID=r.str('telegramChatID', ''), + telegramBotToken=r.str('telegramBotToken', ''), + telegramOutgoingWebhook=r.str('telegramOutgoingWebhook', ''), + telegramOutgoingToken=r.str('telegramOutgoingToken', ''), + slackBotToken=r.str('slackBotToken', ''), + slackChannelID=r.str('slackChannelID', ''), + slackOutgoingWebhook=r.str('slackOutgoingWebhook', ''), + slackOutgoingToken=r.str('slackOutgoingToken', ''), + slackSigningSecret=r.str('slackSigningSecret', ''), + sharesight_auth={ + "grant_type": 'client_credentials', + "code": getenv('sharesight_code'), + "client_id": getenv('sharesight_client_id'), + "client_secret": getenv('sharesight_client_secret'), + "redirect_uri": 'urn:ietf:wg:oauth:2.0:oob' + }, + webhooks=hooks) r.check() - return values - - -def preference_values(getenv): - """The preferences (preferences.conf), computed from getenv(name, default). Raises ConfigError listing every bad value.""" + return env + + +@dataclasses.dataclass(frozen=True) +class Prefs: + """The preferences: preferences.conf, which the bot reads at start and on every reload, each with its default here + and read as its type says (preference_values). Read them as config.prefs. when they are used, never into a + default or a module constant, so that a reload is seen.""" + cache: bool = True + cache_seconds: int = 43200 + country_code: str = 'AU' + debug: bool = False + demote_leveraged: bool = True + demote_volatile: bool = True + exchange_suffix: str = 'AX' # Yahoo's suffix for the local exchange: a bare ticker Yahoo does not know gets it + exclude_portfolios: tuple = () # portfolio ids + future_days: int = 30 + graph: bool = True + http_timeout: int = 18 + hyperlink: bool = True + hyperlinkFooter: bool = True + hyperlinkProvider: str = 'google' + include_portfolios: tuple = () # portfolio ids + min_analysts: int = 7 # .buy, .sell and .forward pe list only stocks this many analysts cover + past_days: int = 28 + performance_cache_seconds: int = 2400 # Sharesight performance reports (40 minutes: sharesight_warm runs every 30) + price_percent: float = 5.0 + print_headers: bool = False + shorts_percent: int = 15 + timezone: str = 'Australia/Melbourne' + trades_use_yahoo: bool = True # set False if Yahoo breaks + volatility_multiplier: float = 2.0 + # the file's own lines, for job processes + source: dict = dataclasses.field(default_factory=dict, compare=False, repr=False) + + +def preference_values(getenv, source=None): + """The preferences (preferences.conf), computed from getenv(name, default): each read as its Prefs field's type + says (a tuple is a list of words), its default the field's. `source` is the file's own {key: value}. Raises + ConfigError listing every bad value.""" r = _Reader(getenv) - values = { - 'config_alliterate': r.bool('alliterate', False), - 'config_cache': r.bool('cache', True), - 'config_cache_seconds': r.int('cache_seconds', 43200), - 'config_chunk_maxlines': r.int('chunk_maxlines', 20), - 'config_country_code': r.str('country_code', 'AU'), - 'debug': r.bool('debug', False), - 'config_demote_leveraged': r.bool('demote_leveraged', True), - 'config_demote_volatile': r.bool('demote_volatile', True), - 'config_exclude_portfolios': r.str('exclude_portfolios', '').split(), - 'config_future_days': r.int('future_days', 30), - 'config_graph': r.bool('graph', True), - 'config_http_timeout': r.int('http_timeout', 18), - 'config_hyperlink': r.bool('hyperlink', True), - 'config_hyperlinkFooter': r.bool('hyperlinkFooter', True), - 'config_hyperlinkProvider': r.str('hyperlinkProvider', 'google').lower(), - 'config_include_portfolios': r.str('include_portfolios', '').split(), - 'config_min_analysts': r.int('min_analysts', 7), # .buy, .sell and .forward pe list only stocks this many analysts cover - 'config_past_days': r.int('past_days', 28), - 'config_performance_cache_seconds': r.int('performance_cache_seconds', 1800), # Sharesight performance reports (30 minutes) - 'config_performance_use_sharesight': r.bool('performance_use_sharesight', False), - 'config_price_percent': r.float('price_percent', 5), - 'config_print_headers': r.bool('print_headers', False), - 'config_shorts_percent': r.int('shorts_percent', 15), - 'config_timezone': r.str('timezone', 'Australia/Melbourne'), - 'config_trades_use_yahoo': r.bool('trades_use_yahoo', True), # set False if Yahoo breaks - 'config_volatility_multiplier': r.float('volatility_multiplier', 2), - } + readers = {bool: r.bool, int: r.int, float: r.float, str: r.str, tuple: r.words, + 'exchange_suffix': lambda name, default: r.blankable(name, default).removeprefix('.').upper(), # blank: none + 'hyperlinkProvider': lambda name, default: r.str(name, default).lower()} + values = {field.name: readers.get(field.name, readers[field.type])(field.name, field.default) + for field in dataclasses.fields(Prefs) if field.compare} r.check() - return values + return Prefs(**values, source=dict(source or {})) -PREFERENCE_NAMES = frozenset(preference_values(lambda name, default=None: default)) # the config_* names a reload can change +# preferences.conf's keys: what a reload can change +PREFERENCE_KEYS = tuple(field.name for field in dataclasses.fields(Prefs) if field.compare) def _getenv_for(preferences): """getenv(name, default): preferences.conf first (preference keys only), then the process environment.""" def getenv(name, default=None): + """A preference from preferences.conf, else the process environment's value, else default.""" if name in preferences: return preferences[name] return os.environ.get(name, default) @@ -231,289 +313,123 @@ def print_errors(e): print('ERROR config:', line, file=sys.stderr) def _fail(e): + """Log the configuration errors and exit: finbot does not start with a bad setting.""" print_errors(e) sys.exit(1) try: - globals().update(startup_values(os.environ.get)) - config_var_dir = config_data_dir - config_cache_dir = config_data_dir + '/cache' - preferences_path = config_data_dir + '/' + PREFERENCES_FILE + env = startup_values(os.environ.get) + preferences_path = env.preferences_path _preferences = {} if os.environ.get(PASSED) else (read_preferences(preferences_path) or {}) - globals().update(preference_values(_getenv_for(_preferences))) + prefs = preference_values(_getenv_for(_preferences), _preferences) except ConfigError as e: _fail(e) -for _dir in (config_var_dir, config_cache_dir): - if os.environ.get('FINBOT_CHECK_ONLY'): # util/check_config.py, possibly run as root: must not create directories the service user then cannot write +for _dir in (env.data_dir, env.cache_dir): + # util/check_config.py, possibly run as root: must not create directories the service user then cannot write + if os.environ.get('FINBOT_CHECK_ONLY'): break try: os.makedirs(_dir, exist_ok=True) except OSError as e: # e.g. read-only filesystem (Lambda) - print("can't create", _dir, e) + print("config: can't create", _dir, e, file=sys.stderr) # matplotlib needs a writable config/cache dir; without a home directory (systemd DynamicUser, containers) it fails -os.environ.setdefault('MPLCONFIGDIR', os.path.abspath(config_cache_dir + '/matplotlib')) +os.environ.setdefault('MPLCONFIGDIR', os.path.abspath(env.cache_dir + '/matplotlib')) def data_file(name): - # user copy in the data dir wins over the shipped default in defaults/ - user = config_var_dir + '/' + name + """Where a data file is read from: the user's copy in the data dir if any, else the default shipped in defaults/.""" + user = env.data_dir + '/' + name return user if os.path.isfile(user) else str(REPO_DIR / 'defaults' / name) +_data_files = {} # path -> (modification time, parsed JSON) + def data_json(name, default=None): - """A data file's JSON (the user's copy or the shipped default, as data_file has it), or `default` when there is neither.""" + """A data file's JSON (the user's copy or the shipped default, as data_file has it), or `default` when there is + neither. Parsed again only when the file changes; callers must not modify what they get.""" path = data_file(name) - if not os.path.isfile(path): + try: + mtime = os.path.getmtime(path) + except OSError: return default + if (cached := _data_files.get(path)) and cached[0] == mtime: + return cached[1] with open(path, encoding="utf-8") as f: - return json.load(f) + value = json.load(f) + _data_files[path] = (mtime, value) + return value def reload(): - """Re-read preferences.conf and apply it everywhere: this module and every module that copied a name with `from lib.config import *`. - All or nothing. Returns one line per change; raises ConfigError (nothing applied) if the file is missing, unreadable or invalid.""" - global _preferences + """Re-read preferences.conf and apply it: config.prefs is replaced in one assignment, so a request never sees half + of a reload. All or nothing. Returns one line per change; raises ConfigError (nothing applied) if the file is + missing, unreadable or invalid.""" + global prefs new = read_preferences(preferences_path) if new is None: raise ConfigError(f'reload requested but {preferences_path} does not exist; nothing reloaded') - values = preference_values(_getenv_for(new)) - changes = [] - for name, value in values.items(): - old = globals()[name] - if old == value: - continue - changes.append(f"{name.removeprefix('config_')} {old!r} -> {value!r}") - for module in list(sys.modules.values()): - namespace = getattr(module, '__dict__', None) - try: - if namespace is not None and name in namespace and namespace[name] == old: - namespace[name] = value - except Exception: - pass - _preferences = new + values = preference_values(_getenv_for(new), new) + changes = [f"{name} {getattr(prefs, name)!r} -> {getattr(values, name)!r}" for name in PREFERENCE_KEYS + if getattr(prefs, name) != getattr(values, name)] + prefs = values return changes def child_environment(): """For a job process: the preferences this process runs on, so it never sees a file edited since the last reload.""" - return dict(_preferences, **{PASSED: '1'}) + return dict(prefs.source, **{PASSED: '1'}) def check_startup_environment(): - """For bot.py at start. Preferences must come from preferences.conf, not from the environment that finbot.env fills, and a - leftover .env from before the split stops the start: its settings would otherwise be silently ignored.""" + """For bot.py at start. Preferences must come from preferences.conf, not from the environment that finbot.env fills, + and a leftover .env from before the split stops the start: its settings would otherwise be silently ignored.""" if os.environ.get(PASSED): return - problems = [f'{key} is a preference; move it from finbot.env to {PREFERENCES_FILE}' for key in PREFERENCE_KEYS if key in os.environ] + problems = [f'{key} is a preference; move it from finbot.env to {PREFERENCES_FILE}' + for key in PREFERENCE_KEYS if key in os.environ] if 'volatile_multiplier' in os.environ: # the old name, from before the split: it never worked as a preference - problems.append(f'volatile_multiplier was renamed to volatility_multiplier, a preference; move it from finbot.env to {PREFERENCES_FILE}') + problems.append('volatile_multiplier was renamed to volatility_multiplier, a preference; move it from' + f' finbot.env to {PREFERENCES_FILE}') if OLD_ENV.exists(): - problems.append(f'{OLD_ENV} is the old single config file, which finbot no longer reads; it will not start while the file is there') - problems.append(f'split it with {REPO_DIR}/util/split_env.py, which writes finbot.env.new and preferences.conf.new beside it; put finbot.env ' - f'where your launcher loads it and {PREFERENCES_FILE} in {config_data_dir}, then delete {OLD_ENV}') + problems.append(f'{OLD_ENV} is the old single config file, which finbot no longer reads; it will not start' + ' while the file is there') + problems.append(f'split it with {REPO_DIR}/util/split_env.py, which writes finbot.env.new and' + f' preferences.conf.new beside it; put finbot.env where your launcher loads it and {PREFERENCES_FILE} in' + f' {env.data_dir}, then delete {OLD_ENV}') if problems: raise ConfigError(problems) def startup_report(): - """Lines for the log at start: where the config came from (a count, never names or values; util/check_config.py names the problems).""" + """Lines for the log at start: where the config came from (counts only; check_config names the problems).""" provided = [key for key in STARTUP_KEYS if key in os.environ] lines = [f'startup environment provides {len(provided)} of {len(STARTUP_KEYS)} startup keys (values hidden)'] if os.path.isfile(preferences_path): - lines.append(f'preferences from {preferences_path} ({len(_preferences)} set; reload with SIGHUP)') + lines.append(f'preferences from {preferences_path} ({len(prefs.source)} set; reload with SIGHUP)') else: - lines.append(f'no {PREFERENCES_FILE} in {config_data_dir}, using the defaults; reload will have nothing to re-read') - return lines - - -adjectives = [ - 'absolutely', - 'amazingly', - 'awfully', - 'bitchingly', - 'bloodly', - 'comfortably', - 'darned', - 'distinctly', - 'especially', - 'ever so', - 'enjoyably', - 'exceedingly', - 'exceptionally', - 'extraordinarily', - 'extremely', - 'flatteringly', - 'fortunately', - 'fortuitously', - 'greatly', - 'honourably', - 'hugely', - 'humblingly', - 'inordinately', - 'jolly', - 'joyfully', - 'keenly', - 'kindly', - 'knowingly', - 'luckily', - 'memorably', - 'mostly', - 'needlessly', - 'overwhelmingly', - 'pleasurably', - 'questionably', - 'rewardingly', - 'serendipitously', - 'seriously', - 'significantly', - 'so', - 'super', - 'supremely', - 'surprisingly', - 'terribly', - 'terrifically', - 'thoroughly', - 'totally', - 'tremendously', - 'uber', - 'uncomprehensibly', - 'unexceptionably', - 'unquestionably', - 'unmentionably', - 'unusually', - 'discomfortingly', - 'incomprehensibly', - 'very', - 'wholesomely', - 'xtra', - 'yawningly', - 'zealously' -] - -adjectives_two = [ - 'amazing', - 'bitching', - 'comforting', - 'comprehensible', - 'dapper', - 'enjoyable', - 'exceptional', - 'extraordinary', - 'flattering', - 'fortuitous', - 'fortunate', - 'great', - 'grateful', - 'honouring', - 'huge', - 'humbling', - 'inordinate', - 'joyous', - 'keen', - 'knowing', - 'kind', - 'lucky', - 'memorable', - 'mentionable', - 'nice', - 'overwhelming', - 'pleasurable', - 'questionable', - 'rewarding', - 'serendipitous', - 'significant', - 'special', - 'super', - 'terrific', - 'tremendous', - 'useful', - 'virtuous', - 'wholesome', - 'xenial', - 'yawning', - 'zany' -] - -verb = [ - 'pretend to greet', - 'apparently share this moment with', - 'coincide in temporal reality with', - 'cross digital paths with', - 'simulate becoming acquainted with', - 'fire photons at', - 'traverse cyberspace with', - 'co-exist in spacetime with', - 'fire electrons at', - 'encode character sets with', - 'convert utf-8 to binary and then back to utf-8 with', - 'update this pixel matrix with', - 'lose money with', - 'maintain character with', - 'act like I comprehend' -] - -noTradesVerb = [ - "The money is probably resting in another account", - "Oh well. The market would have just gone down anyway", - "Heating bills don't pay themselves you know", - "You were fearful when others were also fearful", - "I bet you're regretting that extra 1¢ on your limit order now", - "This is why you can't have nice things", - "But I'm sure that's just a rounding error", - "Nothing + nothing = more nothing. Well played", - "Or maybe there was. They don't pay me for this you know", - "I guess that's right. I'm kinda busy with ChatGPT in another window", - "I like this contrarian play you're having on 'being in it to win it'", - "With your stock picking skills, this is probably for the best" -] - -primary_listing = data_json('finbot_adr.json', {}) # ADR -> its primary listing - -yahoo_country = { - 'Argentina': 'BA', - 'Austria': 'VI', - 'Australia': 'AX', - 'Belgium': 'BR', - 'Brazil': 'SA', - 'Canada': 'TO', - 'Chile': 'SN', - 'Colombia': 'CL', - 'Czech Republic': 'PR', - 'Denmark': 'CO', - 'Estonia': 'TL', - 'Finland': 'HE', - 'France': 'PA', - 'Germany': 'DE', - 'Hungary': 'BD', - 'Iceland': 'IC', - 'India': 'NS', - 'Indonesia': 'JK', - 'Ireland': 'IR', - 'Israel': 'TA', - 'Italy': 'MI', - 'Latvia': 'RG', - 'Lithuania': 'VS', - 'Mexico': 'MX', - 'Netherlands': 'AS', - 'New Zealand': 'NZ', - 'Norway': 'OL', - 'Portugal': 'LS', - 'Qatar': 'QA', - 'Saudi Arabia': 'SR', - 'South Africa': 'JO', - 'Spain': 'MC', - 'Sweden': 'ST', - 'Switzerland': 'SW', - 'Thailand': 'BK', - 'Turkey': 'IS', - 'United Kingdom': 'L', - 'Venezuela': 'CR' - # markets with numeric tickers purposefully omitted -} - -# `from lib.config import *` is how the other modules get their config_* names; keep the loader machinery out of their namespaces -_INTERNAL = {'difflib', 're', 'sys', 'ConfigError', 'key_problem', 'parse_assignments', 'read_text', 'read_preferences', 'read_startup_file', - 'startup_values', 'preference_values', 'reload', 'child_environment', 'print_errors', 'check_startup_environment', 'startup_report', 'preferences_path', - 'STARTUP_KEYS', 'PREFERENCE_KEYS', 'SECRET_KEYS', 'PREFERENCES_FILE', 'PASSED', 'PREFERENCE_NAMES', 'OLD_ENV'} -__all__ = [name for name in dir() if not name.startswith('_') and name not in _INTERNAL] + lines.append(f'no {PREFERENCES_FILE} in {env.data_dir}, using the defaults; reload will have nothing to' + ' re-read') + return lines + retired_notes(prefs.source) + + +def retired_notes(preferences): + """A line for each key of a preferences.conf ({key: value}) that does nothing any more.""" + return [f'{key} in {PREFERENCES_FILE} does nothing any more: delete the line' + for key in RETIRED if key in preferences] + + +def startup_warnings(): + """Settings in finbot.env that do not stop the start but keep part of the bot from working, a line each.""" + warnings = [] + if env.slackOutgoingWebhook and not env.slackBotToken: + warnings.append('slackOutgoingWebhook is set but slackBotToken is not: Slack commands will get no reply') + if env.slackChannelID and not env.slackBotToken: + warnings.append('slackChannelID is set but slackBotToken is not: scheduled reports cannot go there as the bot') + if env.slackOutgoingWebhook and not (env.slackSigningSecret or env.slackOutgoingToken): + warnings.append('slackOutgoingWebhook is set but neither slackSigningSecret nor slackOutgoingToken is:' + ' every Slack request will be refused') + if env.telegramOutgoingWebhook and not env.telegramOutgoingToken: + warnings.append('telegramOutgoingWebhook is set but telegramOutgoingToken is not: every Telegram update will be' + ' refused') + return warnings diff --git a/lib/delivery.py b/lib/delivery.py new file mode 100644 index 00000000..ebdf48c4 --- /dev/null +++ b/lib/delivery.py @@ -0,0 +1,127 @@ +"""Where a report goes and how: Report, the destinations (the chat that asked, or the default chats), deliver(), and the +one-line error report. A destination is a transport (lib/transports.py).""" +import dataclasses +import html +import sys +import traceback + +from lib import config +from lib import net +from lib import render +from lib import transports + + +@dataclasses.dataclass +class Report: + """What a report says: its lines; perhaps a chart, a render.Chart spec that deliver() draws only for a destination + that takes charts; and buttons, rows of buttons.Button (or, on Telegram, a keyboard markup of its own).""" + lines: list + chart: object = None + buttons: list = None + + +REPLIES = {'telegram': transports.TelegramBot, 'slack': transports.SlackBot} # a reply into the chat that asked + + +def reply_to(service, chat, message=None): + """The destination of a reply into a chat (to `message`, the one being answered, if given), or None for a network + that takes none.""" + kind = REPLIES.get(service) + return kind(chat, message) if kind else None + + +def default_chats(): + """Where a scheduled report goes, in this order: Telegram's telegramChatID; Slack's slackChannelID as the bot, with + slackBotToken (so that charts go too), else its incoming webhook; Discord's webhook. Raises RuntimeError when there + is nowhere to post.""" + env = config.env + chats = [] + if env.telegramChatID and 'telegram' in env.webhooks: + chats.append(transports.TelegramBot(env.telegramChatID)) + if env.slackChannelID and env.slackBotToken: + chats.append(transports.SlackBot(env.slackChannelID)) + elif 'slack' in env.webhooks: + chats.append(transports.SlackWebhook(env.webhooks['slack'])) + if 'discord' in env.webhooks: + chats.append(transports.DiscordWebhook(env.webhooks['discord'])) + if not chats: + raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slackChannelID, slack_webhook or' + ' discord_webhook') + return chats + + +def targets(dest): + """Where a report goes: the chat that asked for it (dest), else the default chats.""" + return [dest] if dest else default_chats() + + +def deliver(dest, report): + """Post a report: its chart, drawn here only where it can go (a destination that takes charts, with the graph + preference on), with the lines as its caption; else the lines, and the buttons under the last message unless they + were the chart's. A chart the network refuses goes as text without its buttons, unless it may have arrived after all + (a timeout). Returns the network's answers, one per message.""" + chart = report.chart + if chart is None or not (dest.takes_charts and config.prefs.graph and chart.rows): + return dest.text(report.lines, report.buttons if chart is None else None) + caption = '\n'.join(map(str, report.lines)) or chart.title + answer = dest.chart(render.draw(chart), caption, report.buttons) + if answer is transports.MAYBE_SENT: + return [] + if answer: + return [answer] + print("chart not sent, sending the report as text", file=sys.stderr) + return dest.text(caption.split('\n')) + + +def error_line(e, context=None, limit=200): + """One-line, secret-free summary of an exception (or string).""" + if isinstance(e, BaseException): + text = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ + else: + text = str(e) + text = ' '.join(net.redact(text).split()) + if len(text) > limit: + text = text[:limit - 1] + '…' + return f"⚠️ {context}: {text}" if context else f"⚠️ {text}" + + +def report_error(e, dest=None, context=None): + """Send a one-line error summary to dest, the chat that asked (its transport), else to every default chat. The line + is escaped for Telegram (sent as HTML, which refuses a stray < such as the one in ) and Slack (which asks + for the same escapes). Never raises: error reporting must not become a second failure.""" + try: + line = error_line(e, context) + print(line, file=sys.stderr) + if isinstance(e, BaseException): + trace = ''.join(traceback.format_exception(e)) + print(net.redact(trace), end='', file=sys.stderr) # a request's error carries its URL + for chat in targets(dest): + try: + chat.text([line if chat.dialect == 'discord' else html.escape(line, quote=False)]) + except Exception as send_error: + print("Failed to report error to", chat.name, ":", send_error, file=sys.stderr) + except Exception as reporting_error: + print("Failed to report error:", reporting_error, file=sys.stderr) + + +def guarded(context, func, *args, exit=True, **kwargs): + """Run a scheduled entry point; on failure report one line to the default channel, then exit non-zero (exit=True, + for the command line) or just return None (exit=False, when the bot runs it in-process). A SystemExit inside counts + as a failure.""" + try: + return func(*args, **kwargs) + except (Exception, SystemExit) as e: + report_error(e, context=context) + if exit: + sys.exit(1) + + +def cli(script, job, handler, argv=None): + """Command line entry point of a report script: parse its arguments with `job(argv) -> (label, kwargs)`, print the + usage on a bad argument, else run the handler guarded.""" + try: + label, kwargs = job(sys.argv[1:] if argv is None else argv) + except ValueError as e: + print("Usage:", script, e, file=sys.stderr) + sys.exit(1) + guarded(label, handler, **kwargs) diff --git a/lib/dialects.py b/lib/dialects.py new file mode 100644 index 00000000..c6d2f131 --- /dev/null +++ b/lib/dialects.py @@ -0,0 +1,96 @@ +"""How each chat network marks text up: bold, italic, struck-through, links, user text made safe, and message sizes. A +dialect is also the network's name (TELEGRAM == 'telegram'), so it can stand wherever the name did: reports take one as +`service` and write `service.bold(text)`. dialects.of('slack') gives the one for a name.""" +import html +import re + +from lib import config + + +class Dialect(str): + """A network's markup. Subclasses set the styles, link markup and limits.""" + __slots__ = () + styles = {} # style -> (before, after) + text_limit = 4096 # characters in one message + caption_limit = None # characters under a chart, as visible() counts them; None: no limit of its own + + def bold(self, text): + """text in bold.""" + return self._styled('bold', text) + + def italic(self, text): + """text in italics.""" + return self._styled('italic', text) + + def strike(self, text): + """text struck through.""" + return self._styled('strike', text) + + def _styled(self, style, text): + """text between the markers of a style: bold, italic or strike.""" + before, after = self.styles[style] + return before + text + after + + def link(self, url, text): + """text linking to url; just the text with the hyperlink preference off.""" + return self._link(url, text) if config.prefs.hyperlink else text + + def escape(self, text): + """Text a user wrote (a .plan entry, a name) made safe to show: Telegram's HTML and Slack's markup both take + & < >. It is unescaped first because it may arrive escaped already (inbound escapes Telegram names as + they arrive): escaping twice would show &amp;.""" + return html.escape(html.unescape(str(text)), quote=False) + + def visible(self, text): + """The length of text as the network counts it against caption_limit.""" + return len(text) + + +class Telegram(Dialect): + """Telegram's HTML parse mode.""" + __slots__ = () + styles = {'bold': ('', ''), 'italic': ('', ''), 'strike': ('', '')} + caption_limit = 1024 + + def _link(self, url, text): + """An HTML link.""" + return '' + text + '' + + def visible(self, text): + """The length without the HTML tags, which Telegram does not count.""" + return len(re.sub(r'<[^>]+>', '', text)) + + +class Slack(Dialect): + """Slack's mrkdwn.""" + __slots__ = () + styles = {'bold': ('*', '*'), 'italic': ('_', '_'), 'strike': ('~', '~')} + + def _link(self, url, text): + """A mrkdwn link: .""" + return '<' + url + '|' + text + '>' + + +class Discord(Dialect): + """Discord's markdown.""" + __slots__ = () + styles = {'bold': ('**', '**'), 'italic': ('_', '_'), 'strike': ('~~', '~~')} + text_limit = caption_limit = 2000 # markup included + + def _link(self, url, text): + """A masked link: stops a preview; brackets in the text and parentheses in the url would end it early.""" + return '[' + text.replace('[', '\\[').replace(']', '\\]') + '](<' + url.replace('(', '%28').replace(')', + '%29') + '>)' + + def escape(self, text): + """The plain text: Discord's markdown shows entities as written.""" + return html.unescape(str(text)) + + +TELEGRAM, SLACK, DISCORD = Telegram('telegram'), Slack('slack'), Discord('discord') +_BY_NAME = {dialect: dialect for dialect in (TELEGRAM, SLACK, DISCORD)} + + +def of(network): + """The dialect of a network by name ('telegram', 'slack' or 'discord'); a dialect is its own.""" + return _BY_NAME[network] diff --git a/lib/formats.py b/lib/formats.py new file mode 100644 index 00000000..bbb90c62 --- /dev/null +++ b/lib/formats.py @@ -0,0 +1,154 @@ +"""Text for people: company names trimmed, numbers in units, dates and spans in words, signed percents, ordinals.""" +import datetime +import re + +TITLE_TRIMS = [ # transform_title: (text, replacement), in order, as each sees what the ones before it made + (' - ', ' '), ('Roundhill ETF Trust ', ''), ('Class A Common Stock', ''), + ('BlackRock Institutional Trust Company N.A.', ''), ('Listed Funds Trust', ''), ('GraniteShares ETF Trust', ''), + ('ProShares Trust', ''), ('VanEck ETF Trust', ''), ('Direxion Shares ETF Trust', ''), (' of California', ''), + ('Magnificent', 'Mag'), ('First Trust NASDAQ Clean Edge Green Energy Index Fund', 'First Trust Clean Energy'), + ('First Trust Exchange-Traded Fund III First Trust Nasdaq Clean Edge Clean Energy Index Fund', + 'First Trust Green Energy'), + ('Mirae Asset Global Investments (Hong Kong) Limited', ''), ('Space Exploration Technologies', 'SpaceX'), + ('Exchange Traded Funds Series OFC', ''), ('Global X Global X', 'Global X'), ('Tema ETF Trust', ''), (' AUD', ' '), + (' USD', ' '), ('New York Shares', ''), ('China Electric Vehicle and Battery ETF', 'China EV ETF'), + ('China Electric Vehicle ETF', 'China EV ETF'), ('Atlantica Sustainable Infrastructure', 'Atlantica Sustainable'), + ('Advanced Micro Devices', 'AMD'), ('Taiwan Semiconductor Manufacturing', 'TSM'), + ('Flight Centre Travel', 'Flight Centre'), ('Global X Funds ', ''), (' of Australia', ''), + ('National Australia Bank', 'NAB'), (' PLC', ''), (' p.l.c.', ''), (' P.l.c.', ''), (' P.L.C.', ''), (' Units', '') +] + +TITLE_SHORTENINGS = [ # and these after the suffixes come off + ('Microbalifesciences', 'Microba Life Sciences'), ('Walt Disney Co (The)', 'Disney'), + ('Lisenergylimited', 'LI-S Energy'), + ('Invesco Capital Management LLC ', ''), ('QUALCOMM', 'Qualcomm'), ('Ordinary Shares Class A', ''), + ('Ordinary Shares Class C', ''), ('Lbt Innovations', 'LBT Innovations'), + ('Vanguard Information Technology', 'Vanguard Infotech'), ('The ', ''), (' Shares', ''), ('Rea ', 'REA '), + ('Csl ', 'CSL '), ('Battery ', 'Batt '), ('.com', ''), ('Etf', 'ETF'), ('N.V.', ''), (' NV ', ' '), + ('New York Re', ''), + (' Australian', ' Aus'), (' Australia', ' Aus'), (' Infrastructure', ' Infra'), (' Manufacturing Company', ' '), + (' Limited', ' '), (' Ltd', ' '), (' Holdings', ' '), (' Holding', ' '), (' (Holdings)', ''), (' Corporation', ' '), + (' Incorporated', ' '), (' incorporated', ' '), (' Technologies', ' '), (' Technology', ' '), (' Enterprises', ' '), + (' Enterprise', ' '), (' Enterpr', ' '), (' Ventures', ' '), (' Resources', ' '), (' (Intrada', ''), + (' Structured', ''), + (' Co.', ' '), (' Corp.', ' '), (' Corp', ' '), (' Tech ', ' '), (' Company', ' '), (' Group', ' '), (', Inc', ' '), + (' Inc', ' '), (' Plc', ' '), (' plc', ' '), (' Index', ' '), (' ADR', ' '), (' Daily', ' '), (' Long', ' '), + (' Bull', ' '), (' .', ' '), (' ,', ' ') +] + +def transform_title(title): + """A company or fund name as the reports show it: shorter, without fund-trust boilerplate and legal suffixes.""" + title = title.replace(' FPO', '') + if title.isupper() or title.islower(): + title = title.title() + for text, replacement in TITLE_TRIMS: + title = title.replace(text, replacement) + for suffix in (' AG', ' SE', ' Se'): + title = title.removesuffix(suffix) + title = re.sub(r'\[\w+\]$', '', title) + for text, replacement in TITLE_SHORTENINGS: + title = title.replace(text, replacement) + if title.islower(): + title = title.title() + for suffix in (' &', ' and'): + title = title.strip().removesuffix(suffix) + return ' '.join(title.replace(' Usd', ' USD').replace('Ishares', 'iShares').split()) + +def human_units(value, decimal_places=2): + """1234567 -> '1.23 M'.""" + for unit in ['', 'K', 'M', 'B', 'T', 'Q']: + if value < 1000.0 or unit == 'Q': + break + value /= 1000.0 + return f"{value:.{decimal_places}f} {unit}".rstrip() + +def sections(*blocks): + """Blocks of lines one after another, a blank line between each two; an empty block is left out.""" + lines = [] + for block in blocks: + if block and lines: + lines.append('') + lines += block or [] + return lines + +def security_line(flag, title, link): + """'🇦🇺 Title (link)': how a report line names a security (without a leading space where its market has no flag).""" + return f"{flag} {title} ({link})" if flag else f"{title} ({link})" + +def strip_url(url): + """url without its scheme and www., to show as a link's text.""" + return url.split('://', 1)[-1].removeprefix('www.') + +def make_paragraphs(walloftext): + """The text in paragraphs, each ending with the first sentence that takes it past 22 words.""" + paragraphs, buffer = [], [] + for word in walloftext.split(): + buffer.append(word) + if word.endswith(('!', '.')) and len(buffer) > 22: + paragraphs.append(' '.join(buffer)) + buffer = [] + if buffer: # the rest: it used to be dropped, and a short text with it + paragraphs.append(' '.join(buffer)) + return '\n\n'.join(paragraphs) + +def date_short(d, year=False): + """5 Oct (or 5 Oct 2026).""" + return f"{d.day} {d:%b}" + (f" {d.year}" if year else "") + +def date_range_english(days, end): + """'28 Sep - 5 Oct 2026' for the `days` ending on `end`; both ends carry their year when the range crosses one.""" + start = end - datetime.timedelta(days=days) + return f"{date_short(start, start.year != end.year)} - {date_short(end, True)}" + +def days_english(days, prefix='the past ', article=''): + """A span of days in words: 'today' for 0 or None; prefix + article + 'day', 'week', 'month' or 'year'; else prefix + and a count of weeks, months, years or days ('the past 3 weeks').""" + if not days: + return 'today' + for size, unit in ((1, 'day'), (7, 'week'), (30, 'month'), (365, 'year')): + if days == size: + return prefix + article + unit + for size, units in ((7, 'weeks'), (30, 'months'), (365, 'years')): # weeks first: 210 days are 30 weeks + if days % size == 0: + return f"{prefix}{int(days // size)} {units}" + return f"{prefix}{days} days" + +def price(value): + """A price to show: two decimals, thousands separated (1,234.50); under 1, three significant digits (0.0123).""" + return f"{value:,.2f}" if abs(value) >= 1 else f"{value:.3g}" + +def short_ticker(ticker): + """A ticker as a chart labels it: without its exchange (BHP.AX: BHP) or a crypto's dollar (BTC-USD: BTC).""" + return ticker.split('.')[0].removesuffix('-USD') + +def signed_percent(value, decimals=None): + """12 -> '+12%', -3.5 -> '-3.5%', 0 -> '0%'; thousands separated. decimals=None keeps the value's own digits.""" + if value == 0: + return '0%' + return f"{value:+,}%" if decimals is None else f"{value:+,.{decimals}f}%" + +def get_emoji(number): + """🔼 for a rise, 🔻 for a fall, ▪️ for no change.""" + if number > 0: + return '🔼' + elif number < 0: + return '🔻' + else: + return '▪️' + +def days_from_human_days(arg): + """7, 7d, 2w, 3m, 1y or ytd -> days. Raises ValueError for anything else.""" + arg = arg.upper() + if arg == 'YTD': + today = datetime.datetime.now().date() + return (today - today.replace(day=1, month=1)).days + for suffix, days in (('D', 1), ('W', 7), ('M', 30)): + try: + return int(arg.removesuffix(suffix)) * days + except ValueError: + pass + return int(arg.removesuffix('Y')) * 365 + +def ordinal(num): + """The ordinal suffix of a whole number: 'st' for 1, 'nd' for 22, 'th' for 11 and 13.""" + return 'th' if num % 100 in (11, 12, 13) else {1: 'st', 2: 'nd', 3: 'rd'}.get(num % 10, 'th') diff --git a/lib/inbound.py b/lib/inbound.py new file mode 100644 index 00000000..65f7adb5 --- /dev/null +++ b/lib/inbound.py @@ -0,0 +1,265 @@ +"""What the networks send the bot. Reading an update is apart from how it arrives: a class per network reads its updates +(Telegram, Slack), and a class per way of arriving takes them in (a webhook: the path the network posts to, how it +proves itself, the body), so that another way (a pull mode) can hand its updates to the same reader. An update becomes +a Message to answer, a Press of a button, or a Response the network gets straight back (a challenge, or nothing to +do).""" +import dataclasses +import hashlib +import hmac +import html +import json +import re +import sys +import time +from urllib.parse import parse_qs, urlparse + +from lib import config + + +@dataclasses.dataclass +class Message: + """A chat message to answer (worker.process_request).""" + service: str + chat_id: str + user: str # the sender's handle: whose watchlist add or plan it is + text: str + bot_name: str # the bot's @name, which can start a command + user_name: str # what to call them + message_id: str = '' # the message itself: Slack shows ⏳ on it while finbot works + + +@dataclasses.dataclass +class Press: + """A button pressed under one of the bot's messages (worker.process_callback).""" + service: str + press_id: str # the network's id for the press, to answer it + chat_id: str + message_id: str + data: str + user: str = '' # who pressed it, for the log + + +@dataclasses.dataclass +class Response: + """What the network gets back at once: a status and a body.""" + status: str = '200 OK' + body: bytes = b'' + + +OK = Response() +UNSUPPORTED = Response(body=b'Unsupported') # an update of a kind the bot takes no part in +UNHANDLED = Response(body=b'

Unhandled

') # a kind it might, which this code does not know +BAD_REQUEST = Response('400 Bad Request', b'

Bad Request

') +FORBIDDEN = Response('403 Forbidden', b'

Unauthorized

') +NOT_FOUND = Response('404 Not Found', b'

404

') +TOO_LARGE = Response('413 Payload Too Large', b'

Payload Too Large

') + + +SIGNATURE_AGE = 300 # seconds: Slack's limit; an older signed request may be a replay + + +def same_secret(given, expected): + """Whether a secret sent with a request is the one configured, compared in constant time. An empty configured secret + proves nothing, so it matches nothing.""" + return bool(expected) and given is not None and hmac.compare_digest(given.encode(), expected.encode()) + + +class Network: + """How a network's updates read, however they arrive.""" + service = '' + + def update_id(self, update): + """The update's id, the same when the network resends it (unanswered in time), or None if it gives none.""" + return None + + def parse(self, update): + """What the update (a dict) says: a Message, a Press, or a Response to give straight back.""" + raise NotImplementedError + + +class Telegram(Network): + """Telegram's updates: messages and button presses.""" + service = 'telegram' + bot_name = None # the bot's @username, from telegram.register() as bot.py starts + IGNORED = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', + 'venue', 'poll', 'dice'} # nothing to answer, nothing worth logging + + def update_id(self, update): + """Telegram's update_id.""" + return update.get('update_id') + + def parse(self, update): + """A message (its text, or a photo's caption) or a button press. A first name is escaped as it arrives: + replies are Telegram HTML, and a name is text, not markup.""" + if 'callback_query' in update: # chart period and job buttons + press = update['callback_query'] + message = press.get('message') + if not message or not press.get('data'): + return UNSUPPORTED + sender = press.get('from', {}) + who = '@' + sender['username'] if 'username' in sender else sender.get('first_name', 'unknown') + return Press(self.service, press['id'], str(message['chat']['id']), str(message['message_id']), + press['data'], who) + if 'message' not in update: + return UNSUPPORTED + message = update['message'] + sender = message['from'] + chat_id = str(message['chat']['id']) + first_name = html.escape(sender['first_name'], quote=False) + if 'username' in sender: + user = '@' + sender['username'] + user_name = first_name or user # the handle when the first name is blank + else: + user = user_name = '@' + first_name + if 'text' in message: + text = message['text'] + elif 'photo' in message: + text = message.get('caption', '') + elif self.IGNORED & message.keys(): + return OK + else: + print("[telegram]: unhandled: 'message' without 'text/photo'", file=sys.stderr) + return UNHANDLED + return Message(self.service, chat_id, user, text, self.bot_name, user_name, str(message.get('message_id', ''))) + + +class Slack(Network): + """Slack's Events API (messages) and its interactivity (a button pressed).""" + service = 'slack' + BY_PEOPLE = (None, 'thread_broadcast') # message subtypes a person writes; the rest are bot posts, edits, joins... + + def update_id(self, update): + """Slack's event_id (a button press has none).""" + return update.get('event_id') + + def parse(self, update): + """A message a person wrote, a button pressed, or the challenge that checks the URL. Links and mentions lose + their markup ( becomes dub.ax, <@QWERTY> @QWERTY), and Slack's escapes their meaning + (& becomes &).""" + kind = update.get('type') + if kind == 'url_verification': + response = json.dumps({'challenge': update['challenge']}) + print("replying with", response, file=sys.stderr) + return Response(body=response.encode()) + if kind == 'block_actions': + return self.press(update) + if kind != 'event_callback': + print(f"[slack]: unhandled 'type' {kind!r}", file=sys.stderr) + return UNHANDLED + event = update['event'] + bot_user = update['authorizations'][0]['user_id'] # QWERTY + if event.get('bot_id') or event.get('subtype') not in self.BY_PEOPLE or not event.get('user') or \ + event['user'] == bot_user: + return OK # not by a person: answering a bot (this one's replies and webhook reports included) could loop + if event['type'] not in ('message', 'app_mention') or 'text' not in event: + print("[slack]: unhandled event callback type", event['type'], file=sys.stderr) + return UNHANDLED + text = re.sub(r'<((?:https?|mailto):[^|>]*)(?:\|([^>]*))?>', lambda m: m.group(2) or m.group(1), event['text']) + text = html.unescape(re.sub(r'<(@[\w\.]+)>', r'\g<1>', text)) # Slack sends & < > as & < > + user = '<@' + event['user'] + '>' # a mention: Slack shows the name + return Message(self.service, event['channel'], user, text, '@' + bot_user, user, event.get('ts', '')) + + def press(self, update): + """A button pressed under one of the bot's messages (a chart's period buttons), or UNSUPPORTED for one that is + not under a message or carries no value.""" + container = update.get('container') or {} + actions = update.get('actions') or [{}] + value = actions[0].get('value') if isinstance(actions[0], dict) else None + if container.get('type') != 'message' or not container.get('message_ts') or not value: + return UNSUPPORTED + user = update.get('user') or {} + who = '@' + str(user.get('username') or user.get('id') or 'unknown') + return Press(self.service, update.get('trigger_id', ''), container.get('channel_id', ''), + container['message_ts'], value, who) + + +class Webhook: + """Updates a network posts to the bot, for its reader (a Network); subclasses give the path it posts to and how it + proves itself.""" + def __init__(self, network): + """network: the reader of its updates.""" + self.network = network + self.service = network.service + + def path(self): + """The URL path the network posts to, or '' when finbot.env sets none.""" + return '' + + def refuse_unread(self, environ): + """Why to turn the request away before its body is read (a secret header missing or wrong), else None.""" + return None + + def decode(self, body): + """The update in the request body, a dict. Raises ValueError when the body is not one.""" + update = json.loads(body) + if not isinstance(update, dict): + raise ValueError('not a JSON object') + return update + + def refuse(self, environ, body, update): + """Why to refuse the request once read (the body's signature or a secret in it missing or wrong), else None.""" + return None + + +class TelegramWebhook(Webhook): + """Telegram posting to telegramOutgoingWebhook, each update with the secret header that setWebhook gave it.""" + + def path(self): + """The path of telegramOutgoingWebhook.""" + return urlparse(config.env.telegramOutgoingWebhook).path if config.env.telegramOutgoingWebhook else '' + + def refuse_unread(self, environ): + """The header must carry telegramOutgoingToken.""" + secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') + if not same_secret(secret, config.env.telegramOutgoingToken): + return f"Telegram authorisation header {'not present' if secret is None else 'is present but incorrect'}" + return None + + +class SlackWebhook(Webhook): + """Slack posting events and button presses to slackOutgoingWebhook (the app's Events and Interactivity URL), each + request signed with the app's signing secret, and carrying its verification token.""" + + def path(self): + """The path of slackOutgoingWebhook.""" + return urlparse(config.env.slackOutgoingWebhook).path if config.env.slackOutgoingWebhook else '' + + def refuse(self, environ, body, update): + """With slackSigningSecret, the request must carry Slack's signature of its body, made in the last five minutes; + without it, the body must carry slackOutgoingToken (the older way, which Slack has deprecated).""" + if config.env.slackSigningSecret: + return self.signature_problem(environ, body, config.env.slackSigningSecret) + if 'token' not in update: + return "Slack authorisation field not present" + if not same_secret(str(update['token']), config.env.slackOutgoingToken): + return "Slack auth incorrect" + return None + + @staticmethod + def signature_problem(environ, body, secret): + """Why the request's signature fails, or None: X-Slack-Signature must be v0= and the HMAC-SHA256, under the + signing secret, of v0::, and the timestamp within five minutes of now.""" + timestamp = environ.get('HTTP_X_SLACK_REQUEST_TIMESTAMP', '') + signature = environ.get('HTTP_X_SLACK_SIGNATURE') + if signature is None or not (timestamp.isascii() and timestamp.isdigit()): + return "Slack signature not present" + if abs(time.time() - int(timestamp)) > SIGNATURE_AGE: + return "Slack signature too old (a replay, or a clock is wrong)" + signed = b'v0:' + timestamp.encode() + b':' + body + expected = 'v0=' + hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest() + return None if same_secret(signature, expected) else "Slack signature incorrect" + + def decode(self, body): + """The update: an event comes as JSON, a button press as a form whose `payload` field holds the JSON.""" + if body.startswith(b'payload='): + body = parse_qs(body.decode('utf-8', 'replace')).get('payload', [''])[0].encode() + return super().decode(body) + + +TELEGRAM, SLACK = Telegram(), Slack() # the readers +WEBHOOKS = (TelegramWebhook(TELEGRAM), SlackWebhook(SLACK)) + + +def route(path): + """The webhook whose path this is, or None.""" + return next((hook for hook in WEBHOOKS if hook.path() and path == hook.path()), None) diff --git a/lib/jobrun.py b/lib/jobrun.py index 1139f4d9..adcea41f 100644 --- a/lib/jobrun.py +++ b/lib/jobrun.py @@ -1,26 +1,34 @@ -"""Runs one scheduled command in its own process: `python -m lib.jobrun