diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5e897d5..e9a0f38b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,32 +4,57 @@ on: push: branches: [main] pull_request: + types: [opened, synchronize, reopened, edited] # edited: the base branch can change schedule: - cron: '23 19 * * 0' # weekly (Sunday, UTC): catches upstream package changes between commits workflow_dispatch: # manual run from the Actions tab jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: python-version: ['3.x', '3.13.5'] # latest release, and the production version steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + with: + fetch-depth: 0 # the base branch is needed to see what a pull request changes + - id: changes + name: Classify the change + # A pull request that only touches docs skips the steps below. The job still runs and reports success, so the + # required checks pass at once. Pushes to main, the weekly run and manual runs always test everything. + # Only a non-empty list of allowlisted paths skips the tests; any git error fails the step. The allowlist must + # never include a path a test reads. + env: + BASE_REF: ${{ github.base_ref }} + run: | + docs_only=false + if [ "$GITHUB_EVENT_NAME" = pull_request ]; then + files=$(git -c core.quotePath=false diff --name-only --no-renames "origin/$BASE_REF...HEAD") + echo "$files" + if [ -n "$files" ] && ! grep -qvE '^(README\.md|CLAUDE\.md|img/[^/]+\.png)$' <<< "$files"; then + docs_only=true + echo "::notice::Only docs changed: install, compile and tests are skipped" + fi + fi + echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT" + - if: steps.changes.outputs.docs_only != 'true' + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: pip - - run: pip install -r requirements.txt - - run: python -m compileall -q bot.py *.py lib util - - run: python -m unittest discover -s tests -v + - if: steps.changes.outputs.docs_only != 'true' + run: | + pip install -r requirements.txt + python -m compileall -q *.py lib util + python -m unittest discover -s tests -v notify: # Telegram alert when the weekly run fails. Needs repository secrets TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID. needs: test if: failure() && github.event_name == 'schedule' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Send Telegram message env: diff --git a/.gitignore b/.gitignore index cb770911..4e27a261 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .claude/settings.local.json __pycache__/ *.bak +*.new todo.txt script.zip .env diff --git a/CLAUDE.md b/CLAUDE.md index fe62dc81..19999db1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,5 +12,5 @@ The work branch named in the session instructions (`claude/...`) is the persiste - If the owner wants a fixed name such as `dev`, they will say so; only push to another branch with their explicit permission. ## Facts -- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5. +- Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, except for pull requests that change only README.md, CLAUDE.md or `img/*.png` (install, compile and tests are skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). - Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); `requirements.txt` is for pip/uv users and CI. diff --git a/README.md b/README.md index bece7a6a..79e7197c 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ All 17 keys, read by your launcher. A change needs a restart. Blank means unset. | `slack_webhook` | | Slack incoming webhook, for outbound reports | | `discord_webhook` | | Discord webhook, for outbound reports | | `telegramBotToken` | | Telegram bot token, for outbound reports | -| `slackBotToken` | | Slack bot token, for outbound messages and uploads | +| `slackBotToken` | | Slack bot token, for replies in Slack | | `telegramOutgoingWebhook` | | Telegram webhook path the bot accepts (interactive mode) | | `telegramOutgoingToken` | | Secret token Telegram must present (interactive mode) | | `slackOutgoingWebhook` | | Slack webhook path the bot accepts (interactive mode) | @@ -514,7 +514,7 @@ A key in the wrong file, or one that is not listed, stops finbot starting; on a edit `preferences.conf`, then reload if running. The service log shows what changed (`config: future_days 30 -> 45`). If the file has a mistake, the reload reports it (`ERROR config: ...`), applies nothing and leaves finbot running on its previous settings. Check an edit before reloading with `util/check_config.py /opt/finbot.env`. Edit `finbot.env`, then restart: a running bot cannot see it change. -**Upgrading from one `.env`?** Run `util/migrate_env.py --write` first if it is in the old dotenv format (`key = 'value' # comment`), then `util/split_env.py /opt/finbot.env`. It writes `finbot.env.new` and `preferences.conf.new` next to it, never overwriting, and lists any key it dropped or did not recognise. Until you split, finbot refuses to start because the one file mixes both kinds of key. +**Upgrading from one `.env`?** finbot no longer reads it, and refuses to start while a `.env` is in the checkout, so its settings cannot be silently ignored. Run `util/split_env.py` (or `util/split_env.py /path/to/old.env` for one elsewhere). It reads either format, the old dotenv one (`key = 'value' # comment`) or plain `KEY=value`, writes `finbot.env.new` and `preferences.conf.new` next to the old file, never overwriting, and lists any key it dropped or did not recognise. Put `finbot.env` where your launcher loads it and `preferences.conf` in the data directory, then delete the old file. ### Data directory All state and cache lives under `var/` in the checkout by default. Set `DATA_DIR` to relocate it (cache moves to `$DATA_DIR/cache`), e.g. for a Docker bind mount that survives rebuilds: @@ -591,7 +591,7 @@ performance_cache_seconds=1800 `performance_cache_seconds` (default 30 minutes) is the cache lifetime of Sharesight performance reports. -While `bot.py` is running (with `cache=True` and Sharesight configured), a background job keeps the Sharesight portfolio list (`cache_seconds`) and each portfolio's holdings report (1 day, `performance_cache_seconds` cache) refreshed at 90% of their cache lifetime, so chat requests do not wait on Sharesight. If a refresh fails, the previous cache keeps being served and the job retries after 5 minutes. +While `bot.py` is running with Sharesight configured and `cache=True` (a reload that switches `cache` pauses or resumes it), a background job keeps the Sharesight portfolio list (`cache_seconds`) and each portfolio's holdings report (1 day, `performance_cache_seconds` cache) refreshed at 90% of their cache lifetime, so chat requests do not wait on Sharesight. If a refresh fails, the previous cache keeps being served and the job retries after 5 minutes. ### Hyperlinks These `preferences.conf` options relate to hyperlinks: @@ -620,7 +620,7 @@ The scheduled reports are started by `bot.py`, each run in its own child process The schedulable scripts are `trades.py`, `price.py`, `reminder.py`, `cal.py`, `shorts.py`, `performance.py`, `rating.py`, `milestone.py`, `podcasts.py ""` and `cleanup.py [days]` (deletes cache files older than `days`, default 30). Other jobs, such as a DNS updater or backups, stay in cron or systemd timers. -The schedule is `defaults/schedule.cron`, written in **cron-compatible syntax**: five time fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists work) or an `@macro` such as `@daily`, then a finbot script and its arguments. Quote an argument that has spaces or special characters (a feed URL with `&`, say). Several commands separated by `;` run one after another. `MAILTO=`, `SHELL=`, `PATH=` and `HOME=` lines are ignored. Anything else a crontab allows (a user column, pipes, other programs) is not supported: such a line is reported at startup and skipped. Existing crontab lines can be pasted as they are: `> /dev/null` redirects, `{ a; b ;}` groups and a path before the script name are understood. +The schedule is `defaults/schedule.cron`, written in **cron-compatible syntax**: five time fields (minute, hour, day of month, month, day of week; `*/20`, `Mon-Fri`, ranges and lists work) or an `@macro` such as `@daily`, then a finbot script and its arguments. Quote an argument that has spaces or special characters (a feed URL with `&`, say). Several commands separated by `;` run one after another. `MAILTO=`, `SHELL=`, `PATH=` and `HOME=` lines are ignored. Anything else a crontab allows (a user column, pipes, other programs) is not supported: such a line is reported at startup and skipped. A time that cannot be or never comes (minute 61, 30 February) is different: finbot refuses to start with it, and a reload with it is refused, leaving the running schedule as it was. Existing crontab lines can be pasted as they are: `> /dev/null` redirects, `{ a; b ;}` groups and a path before the script name are understood. ``` # Every 20 minutes on weekdays */20 * * * Mon-Fri trades.py @@ -653,7 +653,7 @@ Behaviour worth knowing: ``` sudo systemd-run --pty --wait --collect -p DynamicUser=yes -p StateDirectory=finbot -p EnvironmentFile=/opt/finbot.env /opt/finbot/price.py 7 ``` -The script prints its output on your terminal and sends its report to your chat channels as usual. The transient unit gets its own dynamic user, and systemd may re-own the state directory to it for the run: check `ls -ln /var/lib/private/finbot` afterwards and `systemctl restart finbot` if the bot can no longer write there. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports inside the bot, without any of this. Outside systemd, run a script from the checkout as your own user with the settings loaded into the shell: `set -a; . ./.env; set +a; ./price.py 7` (this works while no value in the file contains spaces). +The script prints its output on your terminal and sends its report to your chat channels as usual. The transient unit gets its own dynamic user, and systemd may re-own the state directory to it for the run: check `ls -ln /var/lib/private/finbot` afterwards and `systemctl restart finbot` if the bot can no longer write there. The chat commands (`.price 7d`, `.performance 7d`, ...) run the same reports inside the bot, without any of this. Outside systemd, run a script from the checkout as your own user with the settings loaded into the shell: `set -a; . /opt/finbot.env; set +a; ./price.py 7` (this works while no value in the file contains spaces). ## Interactive bot setup The backend `bot.py` needs a frontend https server on a valid domain name with a valid x509 certifcate. @@ -695,10 +695,10 @@ Note: The utils folder contains scripts to generate `/etc/nginx/aws_subnets` and * For security, message BotFather and disable `/setjoingroups`. * Set `telegramOutgoingWebhook` in `finbot.env` to your web server (e.g. https://www.example.com:8443/telegram). * Set `telegramOutgoingToken` to a password of your choosing. Telegram will use this to authenticate with finbot. -With these options set, your bot will auto-subscribe your URL to events the bot sees, when you run `bot.py`. +With these options set, your bot will auto-subscribe your URL to events the bot sees, when you run `bot.py`. It unregisters and registers the webhook as it starts (and only then, which also wakes up a stale connection); if Telegram refuses, finbot does not start and the log says why. #### Slack -Visit https://api.slack.com/apps/ to create a new Slack app from scratch (if you already created one for a finbot webhook, you can reuse that app). +Visit https://api.slack.com/apps/ to create a new Slack app from scratch (if you already created one for a finbot webhook, you can reuse that app). Replies in Slack are text: charts go to Telegram only, since Slack retired the file upload API finbot used. * From _Basic Information > Verification Token_, copy _Verification token_ into `finbot.env` as `slackOutgoingToken`. This allows Slack to authenticate with finbot. * In `finbot.env`, put your web server URL (e.g. https://www.example.com:8443/slack) into `slackOutgoingWebhook` * Save `finbot.env` and restart `bot.py` @@ -711,7 +711,7 @@ Visit https://api.slack.com/apps/ to create a new Slack app from scratch (if you * Check the box _App Home > Allow users to send Slash commands and messages from the messages tab_ * Save Changes * Under _OAuth & Permissions_: - * Scroll down to _Scopes > Bot Token Scopes_, and add: `chat:write` and `files:write`. + * Scroll down to _Scopes > Bot Token Scopes_, and add: `chat:write`. * Scroll up to _OAuth Tokens for Your Workspace_: * If _Bot User OAuth Token_ is not visible, hit _Install to Workspace > Allow_ * Copy _Bot User OAuth Token_ into `finbot.env` as `slackBotToken`. This allows finbot to authenticate with Slack so it can send replies. diff --git a/bot.py b/bot.py index 1ff7910b..13bbc729 100755 --- a/bot.py +++ b/bot.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 +import signal +if __name__ == '__main__': + signal.signal(signal.SIGHUP, signal.SIG_IGN) # ignored until startup is done (the reload handler is installed last): its default action kills the bot import gevent.monkey gevent.monkey.patch_all() -import json, os, re, signal, sys +import json, os, re, sys, traceback from pathlib import Path from gevent import pywsgi import threading @@ -14,19 +17,29 @@ from lib import jobs from lib import charts from lib import sharesight +from lib import telegram from lib import webhook -if config_telegramBotToken: - from lib import telegram - botName = telegram.botName + +botName = None # the Telegram bot's @username, from telegram.register() at start +AUTHENTICATED = 'finbot.authenticated' # set in a request's WSGI environ once it has passed its service's check +SLACK_USER_SUBTYPES = (None, 'thread_broadcast') # Slack message subtypes a person writes; the others are bot posts, edits, joins and the like def main(environ, start_response): - """WSGI entry point. Never lets an exception escape: failures are reported as one line to the default channel.""" + """WSGI entry point. Never lets an exception escape. A failure handling an authenticated request is reported as one line to the + default channel and answered 200, so the service does not retry it into the chat again. Before authentication a failure is only + logged: a stranger must not be able to post into the chats.""" try: return handle(environ, start_response) except Exception as e: - webhook.report_error(e, context=environ.get('PATH_INFO', 'bot')) + if environ.get(AUTHENTICATED): + webhook.report_error(e, context=environ.get('PATH_INFO', 'bot')) + status = '200 OK' + else: + print(f"request to {environ.get('PATH_INFO')!r} failed:", webhook.error_line(e), file=sys.stderr) + traceback.print_exc() + status = '500 Internal Server Error' try: - start_response('500 Internal Server Error', [('Content-type', 'text/plain')], sys.exc_info()) + start_response(status, [('Content-type', 'text/plain')], sys.exc_info()) except Exception: pass return [b''] @@ -40,40 +53,50 @@ def print_body(): def print_headers(): for item in sorted(environ.items()): print(item, file=sys.stderr) - request_body = environ['wsgi.input'].read() + def reply(status, body=b''): + start_response(status, [('Content-type', 'application/json')]) + return [body] user='' userRealName='' - # prepare response - status = '200 OK' - headers = [('Content-type', 'application/json')] - start_response(status, headers) - # process request - uri = environ['PATH_INFO'] - inbound = json.loads(request_body) print_headers() if config_print_headers else None - #print_body() if debug else None + # route on the path first, and turn strangers away before reading the body + uri = environ.get('PATH_INFO', '') if config_telegramOutgoingWebhook and uri == urlparse(config_telegramOutgoingWebhook).path: service = 'telegram' - global botName - if 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' not in environ: - print_headers() - print("Fatal:", service, "authorisation header not present", file=sys.stderr) - return [b'

Unauthorized

'] - if environ['HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN'] != config_telegramOutgoingToken: + elif config_slackOutgoingWebhook and uri == urlparse(config_slackOutgoingWebhook).path: + service = 'slack' + else: + print("Unknown URI", repr(uri), file=sys.stderr) + return reply('404 Not Found', b'

404

') + if service == 'telegram': # Telegram proves itself with a header + secret = environ.get('HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN') + if secret != config_telegramOutgoingToken: print_headers() - print("Fatal: Telegram authorisation header is present but incorrect. Expected:", config_telegramOutgoingToken, file=sys.stderr) - return [b'

Unauthorized

'] + print("Fatal: Telegram authorisation header", "not present" if secret is None else "is present but incorrect", file=sys.stderr) + return reply('403 Forbidden', b'

Unauthorized

') + environ[AUTHENTICATED] = True + try: + inbound = json.loads(environ['wsgi.input'].read()) + except ValueError: + print(f"[{service}]: request body is not JSON", file=sys.stderr) + return reply('400 Bad Request', b'

Bad Request

') + if not isinstance(inbound, dict): + print(f"[{service}]: request body is not a JSON object", file=sys.stderr) + return reply('400 Bad Request', b'

Bad Request

') + #print_body() if debug else None + if service == 'telegram': + bot_name = botName if "callback_query" in inbound: # chart period buttons cq = inbound["callback_query"] cq_message = cq.get("message") if not cq_message or not cq.get("data"): - return [b'Unsupported'] + return reply('200 OK', b'Unsupported') sender = cq.get("from", {}) print("[Telegram button]:", '@' + sender["username"] if "username" in sender else sender.get("first_name", "unknown"), "chat", cq_message["chat"]["id"], cq["data"], file=sys.stderr) threading.Thread(target=worker.process_callback, args=('telegram', cq["id"], str(cq_message["chat"]["id"]), str(cq_message["message_id"]), cq["data"])).start() - return [b''] + return reply('200 OK') if "message" not in inbound: - return [b'Unsupported'] + return reply('200 OK', b'Unsupported') message_id = str(inbound["message"]["message_id"]) chat_id = str(inbound["message"]["chat"]["id"]) if "username" in inbound["message"]["from"]: @@ -97,61 +120,58 @@ def print_headers(): file_id = photo["file_id"] print("[Telegram photo]:", user, file_id, message) elif IGNORED_TELEGRAM_CONTENT & inbound["message"].keys(): - return [b''] # stickers, GIFs, voice notes and the like: nothing to answer, nothing worth logging + return reply('200 OK') # stickers, GIFs, voice notes and the like: nothing to answer, nothing worth logging else: print(f"[{service}]: unhandled: 'message' without 'text/photo'", file=sys.stderr) - return [b'

Unhandled

'] - elif config_slackOutgoingWebhook and uri == urlparse(config_slackOutgoingWebhook).path: - service = 'slack' + return reply('200 OK', b'

Unhandled

') + else: # Slack puts its token in the body if 'token' not in inbound: print("warning: Slack authorisation field not present", file=sys.stderr) print_body() - return [b'

Unauthorized

'] - if inbound['token'] == config_slackOutgoingToken: - print("Incoming Slack request authenticated") - else: - print("Slack auth incorrect. Expected:", config_slackOutgoingToken, "Got:", inbound['token'], file=sys.stderr) + return reply('403 Forbidden', b'

Unauthorized

') + if inbound['token'] != config_slackOutgoingToken: + print("Slack auth incorrect", file=sys.stderr) print_body() - return [b'

Unauthorized

'] + return reply('403 Forbidden', b'

Unauthorized

') + environ[AUTHENTICATED] = True + print("Incoming Slack request authenticated") if 'type' not in inbound: print(f"[{service}]: unhandled: no 'type'", file=sys.stderr) - return [b'Unhandled'] + return reply('200 OK', b'Unhandled') if inbound['type'] == 'url_verification': response = json.dumps({"challenge": inbound["challenge"]}) print("replying with", response, file=sys.stderr) - response = bytes(response, "utf-8") - return [response] + return reply('200 OK', bytes(response, "utf-8")) if inbound['type'] == 'event_callback': - if inbound["event"]["type"] not in ('message', 'app_mention') or "text" not in inbound['event']: - print(f"[{service}]: unhandled event callback type", inbound["event"]["type"], file=sys.stderr) - return [b'

Unhandled

'] - message_id = str(inbound["event"]["ts"]) - message = inbound['event']['text'] + event = inbound['event'] + bot_user = inbound['authorizations'][0]['user_id'] # QWERTY + if event.get('bot_id') or event.get('subtype') not in SLACK_USER_SUBTYPES or not event.get('user') or event['user'] == bot_user: + return reply('200 OK') # not written by a person: answering a bot (this one's replies and webhook reports included) could loop + if event["type"] not in ('message', 'app_mention') or "text" not in event: + print(f"[{service}]: unhandled event callback type", event["type"], file=sys.stderr) + return reply('200 OK', b'

Unhandled

') + message_id = str(event["ts"]) + message = event['text'] message = re.sub(r'', r'\g<1>', message) # becomes dub.ax message = re.sub(r'<(@[\w\.]+)>', r'\g<1>', message) # <@QWERTY> becomes @QWERTY - user = userRealName = '<@' + inbound['event']['user'] + '>' # ZXCVBN becomes <@ZXCVBN> - botName = '@' + inbound['authorizations'][0]['user_id'] # QWERTY becomes @QWERTY - chat_id = inbound['event']['channel'] + user = userRealName = '<@' + event['user'] + '>' # ZXCVBN becomes <@ZXCVBN> + bot_name = '@' + bot_user # QWERTY becomes @QWERTY + chat_id = event['channel'] print(f"[{service}]:", user, message, sys.stderr) if debug else None # this condition spawns a worker before returning else: print(f"[{service}]: unhandled 'type'", file=sys.stderr) - return [b'

Unhandled

'] - else: - print("Unknown URI", uri, file=sys.stderr) - status = "404 Not Found" - start_response(status, headers) - return [b'

404

'] + return reply('200 OK', b'

Unhandled

') def runWorker(): - worker.process_request(service, chat_id, user, message, botName, userRealName, message_id) + worker.process_request(service, chat_id, user, message, bot_name, userRealName, message_id) # process in a background thread so we don't keep the requesting client waiting t = threading.Thread(target=runWorker) t.start() # Return an empty response to the client - return [b''] + return reply('200 OK') IGNORED_TELEGRAM_CONTENT = {'sticker', 'animation', 'voice', 'video', 'video_note', 'audio', 'document', 'contact', 'location', 'venue', 'poll', 'dice'} @@ -203,23 +223,35 @@ def reload_all(*args): print('config:', line, file=sys.stderr) if os.getuid() == 0: print("Running as superuser. This is not recommended.", file=sys.stderr) + if config_scheduler: # read before anything else happens: an impossible time in it stops the start + scheduler = jobs.Scheduler(lambda: data_file('schedule.cron')) + try: + if not scheduler.load(): + scheduler = None + except jobs.ImpossibleTime as e: + for line in e.lines: + print('ERROR schedule:', line, file=sys.stderr) + print('ERROR schedule: finbot will not start until every time in the schedule can come', file=sys.stderr) + sys.exit(1) + else: + print('Scheduled reports are off (scheduler=False)', file=sys.stderr) httpd = pywsgi.WSGIServer((config_ip, config_port), main) httpd.secure_repr = False if debug else None print(f'Opening socket on http://{config_ip}:{config_port}', file=sys.stderr) + if config_telegramBotToken and config_telegramOutgoingWebhook: + try: + botName = telegram.register() # here and only here: the job processes import lib.telegram too + except Exception as e: + print('ERROR', webhook.error_line(e, 'telegram webhook'), file=sys.stderr) + sys.exit(1) if config_telegramBotToken: threading.Thread(target=charts.refresh_keyboards, daemon=True).start() # update stale DM keyboards in the background - if config_cache and config_sharesight_auth.get('client_id'): - threading.Thread(target=sharesight.keep_warm, daemon=True).start() # keep the portfolio list and holdings report cached - if config_scheduler: - scheduler = jobs.Scheduler(lambda: data_file('schedule.cron')) - if scheduler.load(): - jobs.current = scheduler # for the .jobs chat command - threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs - else: - scheduler = None - else: - print('Scheduled reports are off (scheduler=False)', file=sys.stderr) - gevent.signal_handler(signal.SIGHUP, reload_all) # systemctl kill -s HUP finbot: pick up edited preferences.conf and schedule.cron + if config_sharesight_auth.get('client_id'): + threading.Thread(target=sharesight.keep_warm, daemon=True).start() # keep the portfolio list and holdings report cached, while cache is on + if scheduler: + jobs.current = scheduler # for the .jobs chat command + threading.Thread(target=scheduler.run_forever, daemon=True).start() # the scheduled reports, formerly cron jobs + gevent.signal_handler(signal.SIGHUP, reload_all) # startup is done. systemctl reload finbot: pick up edited preferences.conf and schedule.cron try: httpd.serve_forever() except OSError as e: diff --git a/defaults/preferences.conf b/defaults/preferences.conf index fbe927aa..c954314c 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -1,5 +1,5 @@ # Preferences for finbot: how the bot behaves. The bot reads this file itself, at start and on every reload. -# Apply an edit without a restart: sudo systemctl kill -s HUP finbot (docker: docker kill -s HUP ) +# Apply an edit without a restart: sudo systemctl reload finbot (docker: docker kill -s HUP ) # Check an edit first: util/check_config.py /path/to/finbot.env # Flat KEY=value lines, one per line. This is NOT an INI file: no [sections]. No spaces around "=", no quotes, no trailing comments; # put comments on their own line. A key that is not listed here is an error and finbot refuses to start. diff --git a/finbot.service b/finbot.service index 3b5f7e97..2fff0c7a 100644 --- a/finbot.service +++ b/finbot.service @@ -6,7 +6,7 @@ Type=simple User=CHANGEME # Or, instead of User=: DynamicUser=yes plus StateDirectory=finbot (state goes to /var/lib/finbot automatically) ExecStart=/home/CHANGEME/finbot/bot.py -# systemctl reload finbot (or: systemctl kill -s HUP finbot) re-reads preferences.conf and schedule.cron without a restart +# systemctl reload finbot re-reads preferences.conf and schedule.cron without a restart ExecReload=/bin/kill -HUP $MAINPID # Refuse to start on a mistake in finbot.env or preferences.conf (the bot never reads finbot.env itself, so this is where its typos are caught). "+" runs it as root. ExecStartPre=+/home/CHANGEME/finbot/util/check_config.py /opt/finbot.env diff --git a/lib/config.py b/lib/config.py index c955605e..2cee1c8f 100644 --- a/lib/config.py +++ b/lib/config.py @@ -26,6 +26,7 @@ 'slackOutgoingToken', 'slackOutgoingWebhook', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed PREFERENCES_FILE = 'preferences.conf' PASSED = 'FINBOT_PREFERENCES_PASSED' # set by the bot for its job processes: the preferences arrive in the environment, do not read the file +OLD_ENV = REPO_DIR / '.env' # the single config file of old versions, which python-dotenv loaded; nothing reads it now class ConfigError(Exception): @@ -287,12 +288,17 @@ def child_environment(): def check_startup_environment(): - """For bot.py at start. Preferences must come from preferences.conf, not from the environment that finbot.env fills.""" + """For bot.py at start. Preferences must come from preferences.conf, not from the environment that finbot.env fills, and a + leftover .env from before the split stops the start: its settings would otherwise be silently ignored.""" if os.environ.get(PASSED): return problems = [f'{key} is a preference; move it from finbot.env to {PREFERENCES_FILE}' for key in PREFERENCE_KEYS if key in os.environ] if 'volatile_multiplier' in os.environ: # the old name, from before the split: it never worked as a preference problems.append(f'volatile_multiplier was renamed to volatility_multiplier, a preference; move it from finbot.env to {PREFERENCES_FILE}') + if OLD_ENV.exists(): + problems.append(f'{OLD_ENV} is the old single config file, which finbot no longer reads; it will not start while the file is there') + problems.append(f'split it with {REPO_DIR}/util/split_env.py, which writes finbot.env.new and preferences.conf.new beside it; put finbot.env ' + f'where your launcher loads it and {PREFERENCES_FILE} in {config_data_dir}, then delete {OLD_ENV}') if problems: raise ConfigError(problems) @@ -535,5 +541,5 @@ def startup_report(): # `from lib.config import *` is how the other modules get their config_* names; keep the loader machinery out of their namespaces _INTERNAL = {'difflib', 're', 'sys', 'ConfigError', 'key_problem', 'parse_assignments', 'read_text', 'read_preferences', 'read_startup_file', 'startup_values', 'preference_values', 'reload', 'child_environment', 'check_startup_environment', 'startup_report', 'preferences_path', - 'STARTUP_KEYS', 'PREFERENCE_KEYS', 'SECRET_KEYS', 'PREFERENCES_FILE', 'PASSED', 'PREFERENCE_NAMES'} + 'STARTUP_KEYS', 'PREFERENCE_KEYS', 'SECRET_KEYS', 'PREFERENCES_FILE', 'PASSED', 'PREFERENCE_NAMES', 'OLD_ENV'} __all__ = [name for name in dir() if not name.startswith('_') and name not in _INTERNAL] diff --git a/lib/jobrun.py b/lib/jobrun.py index fb43a8a8..1139f4d9 100644 --- a/lib/jobrun.py +++ b/lib/jobrun.py @@ -1,21 +1,26 @@ """Runs one scheduled command in its own process: `python -m lib.jobrun