From df5064c7f8ee33e4b30aa5fb59955f0b556e7403 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 06:28:08 +0000 Subject: [PATCH 01/25] CLAUDE.md: keep an open PR's title and description in step with every push #35 was opened for a CI tweak, took in the 14 review fixes pushed to the work branch afterwards, and merged under its old title. The new section: before each push, check for an open PR from the work branch and update its title and description to cover every commit it holds; never make such a PR mergeable while they are out of date. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- CLAUDE.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 19999db1..7d0a6e90 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,7 @@ ## Iterating (auto mode) The owner chats back and forth; every message that asks for a change gets a change. 1. Make the change, run the tests (`.venv/bin/python -m unittest discover -s tests`), fix what breaks. -2. Commit (with the attribution trailer) and **push to the work branch** (`git push -u origin `). Do this every iteration, so the branch always holds the latest working state. +2. Commit (with the attribution trailer) and **push to the work branch** (`git push -u origin `). Do this every iteration, so the branch always holds the latest working state. If a PR from the work branch is open, update its title and description in the same iteration (see below). 3. Reply briefly: what changed, test result, and the pushed commit's short hash. ## The work branch is the dev branch @@ -11,6 +11,11 @@ The work branch named in the session instructions (`claude/...`) is the persiste - **Never rewrite the work branch's history, and never reset the branch after a merge**: no reset to `main`, no rebase, no amend of pushed commits, no force-push. A force-push breaks `git pull` on the test environment. After a merge, leave the branch as it is. - If the owner wants a fixed name such as `dev`, they will say so; only push to another branch with their explicit permission. +## Pull requests from the work branch +An open PR takes in every commit pushed to its branch, so a PR opened for one small change can merge far more under its old title (#35 was titled as a CI tweak and merged 14 review fixes). The merge commit on `main` keeps that title for good. +- Before each push, check for an open PR from the work branch. If there is one, update its title and description so they cover every commit it now holds, not only the change it was opened for. +- Never make such a PR mergeable (resolve its conflict, merge `main` into it, enable auto-merge) while its title or description is out of date. + ## Facts - Python 3.13.5; tests are offline and need no network (`tests/`). CI runs them on the latest 3.x and 3.13.5, except for pull requests that change only README.md, CLAUDE.md or `img/*.png` (install, compile and tests are skipped; the allowlist in `.github/workflows/ci.yml` must never include a path a test reads). - Production runs the Debian system packages (including `python3-croniter`, for the scheduler in `lib/jobs.py`); `requirements.txt` is for pip/uv users and CI. From 60c55ed794c05b77562b4f54624dfcd7585d0cd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:08:27 +0000 Subject: [PATCH 02/25] Remove dead code: the discord and slack stubs, two uncalled util helpers, the ylim shim, unused imports - lib/discord.py (a pasted discord.py sample that could not even be imported) and lib/slack.py (getUser, never called) are gone; outbound Discord and Slack go through lib/webhook.py as before - util.categorise_tickers and util.get_currency_symbol had no callers; rows_chart no longer renames a ylim argument that nothing passes; webhook.strikethrough had no users (italics stays: reminder.py uses it) - shortman.fetch drops the locals it never read, and still parses every row's percentage - unused imports: json in reports, sharesight and shortman; io, json and lib.telegram in yahoo; util in webhook; shlex in jobs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/discord.py | 15 --------------- lib/jobs.py | 1 - lib/reports.py | 2 +- lib/sharesight.py | 1 - lib/shortman.py | 12 +++--------- lib/slack.py | 12 ------------ lib/util.py | 33 --------------------------------- lib/webhook.py | 2 -- lib/yahoo.py | 3 --- 9 files changed, 4 insertions(+), 77 deletions(-) delete mode 100644 lib/discord.py delete mode 100644 lib/slack.py diff --git a/lib/discord.py b/lib/discord.py deleted file mode 100644 index cdeb722f..00000000 --- a/lib/discord.py +++ /dev/null @@ -1,15 +0,0 @@ -client = discord.Client() - -@client.event -async def on_ready(): - print(f'We have logged in as {client.user}') - -@client.event -async def on_message(message): - if message.author == client.user: - return - - if message.content.startswith('!hello'): - await message.channel.send('Hello!') - -client.run('YOUR_BOT_TOKEN_HERE') diff --git a/lib/jobs.py b/lib/jobs.py index a46221be..f747ae04 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -9,7 +9,6 @@ import math import os import re -import shlex import signal import sys import zlib diff --git a/lib/reports.py b/lib/reports.py index 0db7edd9..041cb11a 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -1,5 +1,5 @@ from itertools import pairwise -import hashlib, json, re +import hashlib, re import datetime import sys import pandas as pd diff --git a/lib/sharesight.py b/lib/sharesight.py index f3f9f8ad..b330185f 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -1,4 +1,3 @@ -import json import sys import time import datetime diff --git a/lib/shortman.py b/lib/shortman.py index 2b78752c..beb99f0e 100644 --- a/lib/shortman.py +++ b/lib/shortman.py @@ -1,4 +1,3 @@ -import json import requests import sys from lib.config import * @@ -6,7 +5,6 @@ def fetch(market_data): local_market_data = market_data.copy() print("Fetching ASX shorts from Shortman") - content = {} url = 'https://www.shortman.com.au/downloadeddata/latest.csv' try: r = requests.get(url, timeout=config_http_timeout) @@ -23,13 +21,9 @@ def fetch(market_data): csv.pop(0) # remove header del csv[-1] # remove junk for line in csv: - cells = line.split(',') - title = cells[0] + cells = line.split(',') # title, code, positions, on issue, percent shorted ticker = cells[1] + '.AX' - positions = cells[2] - on_issue = cells[3] - short_percent = cells[4] - content[ticker] = float(short_percent) + short_percent = float(cells[4]) # every row, so a malformed one still fails the fetch if ticker in market_data: - local_market_data[ticker]['short_percent'] = float(short_percent) + local_market_data[ticker]['short_percent'] = short_percent return local_market_data diff --git a/lib/slack.py b/lib/slack.py deleted file mode 100644 index eaeb380d..00000000 --- a/lib/slack.py +++ /dev/null @@ -1,12 +0,0 @@ -import json -import requests -import sys -from lib.config import * - -def getUser(user_id): - url = 'https://slack.com/api/users.info' - headers = {'Content-type': 'application/x-www-form-urlencoded', 'Authorization': 'Bearer ' + config_slackBotToken, 'user': user_id} - response = requests.get(url, headers=headers, timeout=config_http_timeout) - print(response.json()) - return response.json() - diff --git a/lib/util.py b/lib/util.py index 50f4b5bf..1f4a33ae 100644 --- a/lib/util.py +++ b/lib/util.py @@ -128,19 +128,6 @@ def transform_title(title): title = title.replace('Ishares', 'iShares') return title -def categorise_tickers(tickers): - tickers_us = [] # used by fetch_finviz() - tickers_au = [] # used by fetch_shortman() - tickers_world = [] # used by fetch_yahoo() - for ticker in tickers: - if '.AX' in ticker: - tickers_au.append(ticker) - if '.' in ticker: - tickers_world.append(ticker) - else: - tickers_us.append(ticker) - return tickers_au, tickers_world, tickers_us - def flag_from_market(market): flag='' if market == 'ASX': @@ -271,24 +258,6 @@ def currency_from_market(market): return None return currency -def get_currency_symbol(currency): - currency_symbol='' - if currency in {'AUD', 'CAD', 'HKD', 'NZD', 'SGD', 'TWD', 'USD'}: - currency_symbol = '$' - elif currency_symbol in {'CNY', 'JPY'}: - currency_symbol = '¥' - elif currency == 'EUR': - currency_symbol = '€' - elif currency == 'GBP': - currency_symbol = '£' - elif currency_symbol == 'KRW': - currency_symbol = '₩' - elif currency == 'RUB': - currency_symbol = '₽' - elif currency == 'THB': - currency_symbol = '฿' - return currency_symbol - def read_cache(cacheFile, maxSeconds=UNSET): if maxSeconds is UNSET: maxSeconds = config_cache_seconds @@ -949,8 +918,6 @@ def rows_chart(rows, title, subtitle='', **kw): """Horizontal bars for a list of (label, value). Over 40 rows keeps the first and last 20.""" if len(rows) > 40: rows = rows[:20] + rows[-20:] - if 'ylim' in kw: # older name for the zoomed value axis - kw['xlim'] = kw.pop('ylim') return hbar_chart(rows, title, subtitle, **kw) def pie_chart(rows, title, subtitle=''): diff --git a/lib/webhook.py b/lib/webhook.py index b9b83a1e..e1c619e0 100644 --- a/lib/webhook.py +++ b/lib/webhook.py @@ -8,7 +8,6 @@ import traceback from lib.config import * -from lib import util def write(service, url, payload_string, chat_id=None, message_id=None, reply_markup=None): headers = {'Content-type': 'application/json'} @@ -172,7 +171,6 @@ def strike(message, service): message = '~~' + message + '~~' return message -strikethrough = strike def fit_caption(caption, limit=1024): """Telegram rejects photo captions over 1024 visible characters. Keep whole lines that fit and say how many were left out.""" diff --git a/lib/yahoo.py b/lib/yahoo.py index 0424b48c..29810826 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -2,15 +2,12 @@ import datetime from dateutil.relativedelta import relativedelta import hashlib -import io -import json import requests import pandas as pd import zoneinfo from pandas.tseries.offsets import BDay as businessday import sys import shutil -import lib.telegram as telegram from lib.config import * from lib import util from http.cookies import SimpleCookie From 85c02c991b66861d2d2ee605cf7d2588f7b0c88b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:19:12 +0000 Subject: [PATCH 03/25] util: tables for markets, exchanges and ticker suffixes; one link builder; the chart setup written once - flag_from_market and currency_from_market read one MARKETS table (flag, currency) instead of two 50-line if-chains walking the same groups; flag_from_ticker, transform_to_yahoo and the shared aliases of transform_to_google and exchange_human are tables too. Results are unchanged, checked against the old functions for every code either version knows (8,100 calls), quirks included (SGX still maps to the China flag and CNY, as the old chains' first match did) - yahoo_link and gfinance_link build their markup with link(); quote_link is the "Yahoo for what Google Finance has no page for" rule that charts.py and price.py each spelled out (identical links on 18,700 combinations of service, hyperlink setting and provider) - the chart palette, the 1280x800 figure and the PNG ending are written once (_figure, _png); the nine renderers give byte-identical PNGs - hbar_chart loses axis_fmt, which nothing could see (the value axis is hidden), and its never-read span, grid and inside-label state; the three chart specs that passed axis_fmt drop it - cache_path is the one place that says where cache and state files live - days_from_human_days tries each suffix in a loop (same results and errors on 300 inputs) - numpy and matplotlib.dates load inside the chart functions: a job process that draws no chart (trades.py every 20 minutes, reminder, cal, rating, shorts' text runs) no longer imports them. test_datadir imports matplotlib itself, so it still proves matplotlib starts without a home dir Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/reports.py | 4 +- lib/util.py | 430 +++++++++++++----------------------------- lib/worker.py | 2 +- tests/test_datadir.py | 2 +- 4 files changed, 139 insertions(+), 299 deletions(-) diff --git a/lib/reports.py b/lib/reports.py index 041cb11a..3c0e9b81 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -427,7 +427,7 @@ def score_col(e): spread = (high - low) or 0.2 # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) chart = {'rows': shown, 'title': message, 'subtitle': 'Mean analyst rating: 1 = strong buy, 5 = strong sell (axis zoomed)', - 'value_fmt': lambda v: f"{v:g}", 'axis_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} + 'value_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} return (payload, chart) if with_chart else payload def prepare_value_payload(service, action='pe', ticker_select=None, length=15, with_chart=False): @@ -508,7 +508,7 @@ def last_col(e): else: heading = f'{heading_type} {length} tracked stocks by {heading_trail} ratio' payload.insert(0, f"{webhook.bold(heading, service)}") - chart = {'rows': chart_rows[:length], 'title': heading, 'value_fmt': lambda v: f"{v:g}", 'axis_fmt': lambda v: f"{v:g}"} + chart = {'rows': chart_rows[:length], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} return (payload, chart) if with_chart else payload def prepare_profile_payload(service, user, ticker): diff --git a/lib/util.py b/lib/util.py index 1f4a33ae..491430df 100644 --- a/lib/util.py +++ b/lib/util.py @@ -7,9 +7,7 @@ import sys import tempfile import time -import numpy as np -from lib.config import * # first: sets MPLCONFIGDIR before matplotlib is imported -import matplotlib.dates as mdates +from lib.config import * # first: sets MPLCONFIGDIR before matplotlib is imported (lazily, by the chart functions) from lib import sharesight @@ -128,140 +126,53 @@ def transform_title(title): title = title.replace('Ishares', 'iShares') return title +MARKETS = { # Sharesight market code -> (flag, currency); no currency where a market quotes in several (LSE and HKEX allow non-home currencies) + 'ASX': ('🇦🇺', 'AUD'), + 'BOM': ('🇮🇳', 'INR'), 'NSE': ('🇮🇳', 'INR'), + 'BMV': ('🇲🇽', 'MXN'), + 'BKK': ('🇹🇭', 'THB'), + 'BVMF': ('🇧🇷', 'BRL'), + 'SHE': ('🇨🇳', 'CNY'), 'SGX': ('🇨🇳', 'CNY'), 'SHA': ('🇨🇳', 'CNY'), + 'CPSE': ('🇩🇰', 'DEK'), + **dict.fromkeys(('EURONEXT', 'AMS', 'ATH', 'BIT', 'BME', 'DUB', 'EBR', 'EPA', 'ETR', 'FWB', 'FRA', 'VIE'), ('🇪🇺', 'EUR')), + 'HKG': ('🇭🇰', None), + 'ICSE': ('🇮🇸', 'ISK'), + 'JSE': ('🇿🇦', 'ZAR'), + 'KRX': ('🇰🇷', 'KRW'), 'KOSDAQ': ('🇰🇷', 'KRW'), + 'LSE': ('🇬🇧', 'GBP'), # better than trades.py using brokerage_currency_code, which is AUD on CMC + 'MISX': ('🇷🇺', 'RUB'), + 'OM': ('🇸🇪', 'SEK'), 'STO': ('🇸🇪', 'SEK'), + 'SWX': ('🇨🇭', 'CHF'), 'VTX': ('🇨🇭', 'CHF'), + 'TAI': ('🇹🇼', 'TWD'), 'TPE': ('🇹🇼', 'TWD'), + 'TASE': ('🇮🇱', 'ILS'), + 'OB': ('🇳🇴', 'NOK'), + 'TSE': ('🇯🇵', 'JPY'), + 'TSX': ('🇨🇦', 'CAD'), + 'NASDAQ': ('🇺🇸', 'USD'), 'NYSE': ('🇺🇸', 'USD'), 'BATS': ('🇺🇸', 'USD'), 'AMEX': ('🇺🇸', None), + 'WAR': ('🇵🇱', 'PLN'), +} + def flag_from_market(market): - flag='' - if market == 'ASX': - flag = '🇦🇺' - elif market in {'BOM', 'NSE'}: - flag = '🇮🇳' - elif market in {'BMV'}: - flag = '🇲🇽' - elif market in {'BKK'}: - flag = '🇹🇭' - elif market in {'BVMF'}: - flag = '🇧🇷' - elif market in {'SHE', 'SGX', 'SHA'}: - flag = '🇨🇳' - elif market == 'CPSE': - flag = '🇩🇰' - elif market in {'EURONEXT','AMS','ATH','BIT','BME','DUB','EBR','EPA','ETR','FWB','FRA','VIE'}: - flag = '🇪🇺' - elif market == 'HKG': - flag = '🇭🇰' - elif market == 'ICSE': - flag = '🇮🇸' - elif market in {'JSE'}: - flag = '🇿🇦' - elif market in {'KRX', 'KOSDAQ'}: - flag = '🇰🇷' - elif market == 'LSE': - flag = '🇬🇧' - elif market == 'MISX': - flag = '🇷🇺' - elif market in {'OM', 'STO'}: - flag = '🇸🇪' - elif market == 'SGX': - flag = '🇸🇬' - elif market in {'SWX', 'VTX'}: - flag = '🇨🇭' - elif market in {'TAI', 'TPE'}: - flag = '🇹🇼' - elif market == 'TASE': - flag = '🇮🇱' - elif market == 'OB': - flag = '🇳🇴' - elif market == 'TSE': - flag = '🇯🇵' - elif market == 'TSX': - flag = '🇨🇦' - elif market in {'BATS', 'AMEX'} or market.lower().startswith('nasdaq') or market.lower().startswith('nyse'): - flag = '🇺🇸' - elif market in {'WAR'}: - flag = '🇵🇱' - return flag + if market in MARKETS: + return MARKETS[market][0] + return '🇺🇸' if market.lower().startswith(('nasdaq', 'nyse')) else '' + +TICKER_FLAGS = {'AS': '🇳🇱', 'AX': '🇦🇺', 'HK': '🇭🇰', 'KS': '🇰🇷', 'KQ': '🇰🇷', 'L': '🇬🇧', 'NS': '🇮🇳', 'NZ': '🇳🇿', 'TW': '🇹🇼', 'TWO': '🇹🇼', 'TO': '🇨🇦'} # Yahoo suffix -> flag def flag_from_ticker(ticker): - flag = '' - if '.' in ticker: - suffix = ticker.split('.')[1] - if suffix == 'AS': - flag = '🇳🇱' - elif suffix == 'AX': - flag = '🇦🇺' - elif suffix == 'HK': - flag = '🇭🇰' - elif suffix in ('KS', 'KQ'): - flag = '🇰🇷' - elif suffix == 'L': - flag = '🇬🇧' - elif suffix == 'NS': - flag = '🇮🇳' - elif suffix == 'NZ': - flag = '🇳🇿' - elif suffix in ('TW', 'TWO'): - flag = '🇹🇼' - elif suffix == 'TO': - flag = '🇨🇦' - else: - flag = '🇺🇸' - return flag + return TICKER_FLAGS.get(ticker.split('.')[1], '') if '.' in ticker else '🇺🇸' def currency_from_market(market): - if market == 'ASX': - currency = 'AUD' - elif market in {'BOM', 'NSE'}: - currency = 'INR' - elif market in {'BMV'}: - currency = 'MXN' - elif market in {'BKK'}: - currency = 'THB' - elif market in {'BVMF'}: - currency = 'BRL' - elif market in {'SHE', 'SGX', 'SHA'}: - currency = 'CNY' - elif market == 'CPSE': - currency = 'DEK' - elif market in {'EURONEXT','AMS','ATH','BIT','BME','DUB','EBR','EPA','ETR','FWB','FRA','VIE'}: - currency = 'EUR' - elif market == 'ICSE': - currency = 'ISK' - elif market in {'JSE'}: - currency = 'ZAR' - elif market in {'KRX', 'KOSDAQ'}: - currency = 'KRW' - elif market == 'MISX': - currency = 'RUB' - elif market in {'OM', 'STO'}: - currency = 'SEK' - elif market == 'SGX': - currency = 'SGD' - elif market in {'SWX', 'VTX'}: - currency = 'CHF' - elif market in {'TAI', 'TPE'}: - currency = 'TWD' - elif market == 'TASE': - currency = 'ILS' - elif market == 'OB': - currency = 'NOK' - elif market == 'TSE': - currency = 'JPY' - elif market == 'TSX': - currency = 'CAD' - elif market in {'NASDAQ', 'NYSE', 'BATS'}: - currency = 'USD' - elif market in {'WAR'}: - currency = 'PLN' - elif market == 'LSE': - currency = 'GBP' # LSE allows non-home currencies, but this is better than trades.py using brokerage_currency_code which is AUD on CMC - else: - # note: LSE and HKE allow non-home currencies - return None - return currency + return MARKETS.get(market, (None, None))[1] + +def cache_path(name, persist=False): + """Where a cache file lives, or with persist=True a state file (in the data directory).""" + return (config_var_dir if persist else config_cache_dir) + "/" + name def read_cache(cacheFile, maxSeconds=UNSET): if maxSeconds is UNSET: maxSeconds = config_cache_seconds - cacheFile = config_cache_dir + "/" + cacheFile + cacheFile = cache_path(cacheFile) if os.path.isfile(cacheFile): maxSeconds = datetime.timedelta(seconds=maxSeconds) cacheFileMtime = datetime.datetime.fromtimestamp(os.path.getmtime(cacheFile)) @@ -279,7 +190,7 @@ def read_cache(cacheFile, maxSeconds=UNSET): def cache_age(cacheFile): """Seconds since the cache file was written, or None when it does not exist.""" try: - return time.time() - os.path.getmtime(config_cache_dir + "/" + cacheFile) + return time.time() - os.path.getmtime(cache_path(cacheFile)) except OSError: return None @@ -306,26 +217,19 @@ def write_file(path, data, sync=False): def json_write(filename, data, persist=False): """A cache file, or with persist=True a state file in the data directory (also flushed to disk). See write_file.""" - if persist: - filename = config_var_dir + "/" + filename - else: - filename = config_cache_dir + "/" + filename + path = cache_path(filename, persist) if debug: - print("cache write:", filename, file=sys.stderr) - write_file(filename, json.dumps(data, indent=4).encode('utf-8'), sync=persist) + print("cache write:", path, file=sys.stderr) + write_file(path, json.dumps(data, indent=4).encode('utf-8'), sync=persist) def json_load(filename, persist=False): - if persist: - filename = config_var_dir + "/" + filename - else: - filename = config_cache_dir + "/" + filename - path = Path(filename) + path = Path(cache_path(filename, persist)) return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None def read_binary_cache(cacheFile, maxSeconds=UNSET): if maxSeconds is UNSET: maxSeconds = config_cache_seconds - cacheFile = config_cache_dir + "/" + cacheFile + cacheFile = cache_path(cacheFile) if os.path.isfile(cacheFile): maxSeconds = datetime.timedelta(seconds=maxSeconds) cacheFileMtime = datetime.datetime.fromtimestamp(os.path.getmtime(cacheFile)) @@ -342,7 +246,7 @@ def read_binary_cache(cacheFile, maxSeconds=UNSET): def write_binary_cache(cacheFile, data): data.seek(0) - write_file(config_cache_dir + "/" + cacheFile, data.getvalue()) + write_file(cache_path(cacheFile), data.getvalue()) def humanUnits(value, decimal_places=2): for unit in ['', 'K', 'M', 'B', 'T', 'Q']: @@ -352,18 +256,9 @@ def humanUnits(value, decimal_places=2): return f"{value:.{decimal_places}f} {unit}" def yahoo_link(ticker, service='telegram', brief=True, text=None): - yahoo_url = "https://au.finance.yahoo.com/quote/" - if brief and not text: - text = ticker.split('.')[0] - elif not text: - text = ticker - if service == 'telegram' and config_hyperlink: - ticker_link = '' + text + '' - elif service in {'discord', 'slack'} and config_hyperlink: - ticker_link = '<' + yahoo_url + ticker + '|' + text + '>' - else: - ticker_link = text - return ticker_link + if not text: + text = ticker.split('.')[0] if brief else ticker + return link("https://au.finance.yahoo.com/quote/" + ticker, text, service) def link(url, text, service='telegram'): if service == 'telegram' and config_hyperlink: @@ -376,10 +271,14 @@ def link(url, text, service='telegram'): def finance_link(symbol, exchange, service='telegram', days=1, brief=True, text=None): if config_hyperlinkProvider == 'google': - link = gfinance_link(symbol, exchange, service, days, brief, text) - else: - link = yahoo_link(symbol, service, brief, text) - return link + return gfinance_link(symbol, exchange, service, days, brief, text) + return yahoo_link(symbol, service, brief, text) + +def quote_link(ticker, exchange, service='telegram', days=1): + """finance_link, but to Yahoo for what Google Finance has no page for: Taipei Exchange, crypto (CCC) and indices (^).""" + if exchange in ('Taipei Exchange', 'CCC') or ticker.startswith('^'): + return yahoo_link(ticker, service) + return finance_link(ticker, exchange, service, days=days) def gfinance_link(symbol, exchange, service='telegram', days=1, brief=True, text=None): window = '1D' @@ -411,59 +310,41 @@ def gfinance_link(symbol, exchange, service='telegram', days=1, brief=True, text text = ticker else: text = symbol # US - if service == 'telegram' and config_hyperlink: - ticker_link = '' + text + '' - elif service in {'discord', 'slack'} and config_hyperlink: - ticker_link = '<' + url + ticker + '?window=' + window + '|' + text + '>' - else: - ticker_link = symbol - return ticker_link + if service in {'telegram', 'discord', 'slack'} and config_hyperlink: + return link(url + ticker + '?window=' + window, text, service) + return symbol + +VENUES = { # the names Yahoo, Sharesight and Google give an exchange -> (Google Finance code, country) + **dict.fromkeys(('OTC', 'PNK', 'Other OTC', 'OTCPK'), ('OTCMKTS', None)), + **dict.fromkeys(('TO', 'TOR', 'Toronto'), ('TSE', 'CA')), + **dict.fromkeys(('TW', 'TWO', 'TAI', 'Taiwan', 'Taipei Exchange', 'Taipei'), ('TPE', 'TW')), + **dict.fromkeys(('HK', 'HKG', 'HKSE'), ('HKG', 'HK')), + **dict.fromkeys(('KQ', 'KOE', 'KOSDAQ'), ('KOSDAQ', 'KR')), + **dict.fromkeys(('KS', 'KRX', 'KSE', 'KSC'), ('KRX', 'KR')), + **dict.fromkeys(('L', 'LSE', 'London'), ('LON', 'UK')), + **dict.fromkeys(('T', 'TYO', 'JPX', 'Tokyo'), ('TYO', 'JP')), + **dict.fromkeys(('TA', 'TLV', 'Tel Aviv'), ('TLV', 'IL')), + **dict.fromkeys(('NZSE', 'NZE', 'New Zealand', 'Auckland'), (None, 'NZ')), +} def transform_to_google(exchange): if 'Nasdaq' in exchange: - exchange = 'NASDAQ' - elif exchange == 'Cboe US': - exchange = 'BATS' - elif exchange in {'OTC', 'PNK', 'Other OTC', 'OTCPK'}: - exchange = 'OTCMKTS' - elif exchange in {'TO', 'TOR', 'Toronto'}: - exchange = 'TSE' - elif exchange in {'TW', 'TWO', 'TAI', 'Taiwan', 'Taipei Exchange', 'Taipei'}: - exchange = 'TPE' - elif exchange in {'HK', 'HKG', 'HKSE'}: - exchange = 'HKG' - elif exchange in {'KQ', 'KOE', 'KOSDAQ'}: - exchange = 'KOSDAQ' - elif exchange in {'KS', 'KRX', 'KSE', 'KSC'}: - exchange = 'KRX' - elif exchange in {'L', 'LSE', 'London'}: - exchange = 'LON' - elif exchange in {'T', 'TYO', 'JPX', 'Tokyo'}: - exchange = 'TYO' - elif exchange in {'TA', 'TLV', 'Tel Aviv'}: - exchange = 'TLV' - return exchange + return 'NASDAQ' + if exchange == 'Cboe US': + return 'BATS' + return VENUES.get(exchange, (None, None))[0] or exchange def exchange_human(exchange): if 'Nasdaq' in exchange or 'NYSE' in exchange or 'Other OTC' in exchange: return 'US' - elif exchange in {'TO', 'TOR', 'Toronto'}: - return 'CA' - elif exchange in {'TW', 'TWO', 'TAI', 'Taiwan', 'Taipei Exchange', 'Taipei'}: - return 'TW' - elif exchange in {'HK', 'HKG', 'HKSE'}: - return 'HK' - elif exchange in {'KQ', 'KOE', 'KOSDAQ', 'KS', 'KRX', 'KSE', 'KSC'}: - return 'KR' - elif exchange in {'L', 'LSE', 'London'}: - return 'UK' - elif exchange in {'T', 'TYO', 'JPX', 'Tokyo'}: - return 'JP' - elif exchange in {'TA', 'TLV', 'Tel Aviv'}: - return 'IL' - elif exchange in {'NZSE', 'NZE', 'New Zealand', 'Auckland'}: - return 'NZ' - return exchange + return VENUES.get(exchange, (None, None))[1] or exchange + +YAHOO_SUFFIX = { # exchange code (Sharesight, Google) -> Yahoo's ticker suffix + 'ASX': 'AX', 'HKG': 'HK', 'KRX': 'KS', 'KOSDAQ': 'KQ', 'LSE': 'L', + 'TAI': 'TW', # Taiwan in YF and Sharesight; TPE in GF + 'TPE': 'TWO', # Taipei in YF, Taiwan in GF (GF format will not work as input); missing from GF + 'TSE': 'TO', 'TYO': 'T', 'TLV': 'TA', +} def transform_to_yahoo(ticker, market=None): split = ticker.replace(':', '.').split('.') @@ -472,32 +353,12 @@ def transform_to_yahoo(ticker, market=None): market = split[1] if not market: return ticker - if market == 'ASX': - market = 'AX' - if market == 'HKG': - market = 'HK' - if market == 'KRX': - market = 'KS' - if market == 'KOSDAQ': - market = 'KQ' - if market == 'LSE': - market = 'L' - if market == 'TAI': # Taiwan in YF, Sharesight - market = 'TW' # TPE in GF - if market == 'TPE': # Taipei in YF, Taiwan in GF. GF format will not work as input - market = 'TWO' # missing from GF - if market == 'TSE': - market = 'TO' - if market == 'TYO': - market = 'T' - if market == 'TLV': - market = 'TA' + market = YAHOO_SUFFIX.get(market, market) if market in {'NASDAQ', 'NYSE', 'BATS', 'OTCMKTS', 'AMEX', 'NYSEARCA'}: # US venues: Yahoo symbols carry no suffix return ticker if market in {'A', 'B', 'C'}: # class shares return ticker + '-' + market - ticker = ticker + '.' + market - return ticker + return ticker + '.' + market def strip_url(url): url = url.removeprefix('https://www.') @@ -558,6 +419,7 @@ def fit_left_margin(fig, ax, pad_px=18): def style_date_axis(ax, start, end, ink): """Larger, explicit date ticks: format follows the span, few enough ticks that they never collide.""" + import matplotlib.dates as mdates days = (end - start).days fmt = '%Y' if days > 1460 else '%b %Y' if days > 150 else '%d %b' ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=3, maxticks=5)) @@ -566,14 +428,30 @@ def style_date_axis(ax, start, end, ink): for label in ax.get_xticklabels(): label.set_fontweight('medium') -def graph(df, title, ylabel): - """Render a price chart as PNG bytes, sized for Telegram (1280px wide, no server-side resampling).""" +PALETTE = ('#14181f', '#eef1f5', '#9aa4b2', '#2a313c') # charts: dark surface, ink, muted ink, grid lines + +def _figure(facecolor): + """A 1280x800 figure (Telegram's photo size, so it is never resampled) and its Agg canvas.""" from matplotlib.figure import Figure from matplotlib.backends.backend_agg import FigureCanvasAgg + fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=facecolor) + return fig, FigureCanvasAgg(fig) + +def _png(fig, facecolor): + """The figure as PNG bytes, rewound, or an upload sends an empty file. No bbox_inches='tight': it would change the 1280x800 size.""" + buf = io.BytesIO() + fig.savefig(buf, format='png', facecolor=facecolor) + buf.seek(0) + return buf + +def graph(df, title, ylabel): + """Render a price chart as PNG bytes, sized for Telegram (1280px wide, no server-side resampling).""" from matplotlib.ticker import MaxNLocator, FuncFormatter + import matplotlib.dates as mdates + import numpy as np import pandas as pd # palette: dark surface, muted ink, green/red for up/down (red/green separated in lightness as well as hue) - bg, ink, ink2, grid = '#14181f', '#eef1f5', '#9aa4b2', '#2a313c' + bg, ink, ink2, grid = PALETTE up, down, flat = '#34d399', '#fb7185', '#9aa4b2' x = pd.to_datetime(df['Date'].astype(str)) y = df['Close'].astype(float) @@ -582,8 +460,7 @@ def graph(df, title, ylabel): pct = (last - first) / first * 100 if first else 0 arrow = '▲' if last > first else '▼' if last < first else '■' - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=bg) # 1280x800 - FigureCanvasAgg(fig) + fig, _ = _figure(bg) ax = fig.add_axes([0.075, 0.105, 0.885, 0.755], facecolor=bg) ax.fill_between(x, y, y.min() - (y.max() - y.min()) * 0.15, color=color, alpha=0.14, linewidth=0) ax.plot(x, y, color=color, linewidth=1.6, solid_capstyle='round') @@ -622,22 +499,16 @@ def annotate(i, name, above): fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') fig.text(0.03, 0.910, f"{last:,.2f} {ylabel or ''} {arrow} {abs(pct):.2f}% over period", color=color, fontsize=8.5, ha='left', va='center') - buf = io.BytesIO() - fig.savefig(buf, format='png', facecolor=bg) # no bbox_inches='tight': keep exact 1280x800 - buf.seek(0) - return buf + return _png(fig, bg) def compare_graph(series, title, subtitle=''): """Rebased % change lines for 2-6 tickers. series: list of (label, pandas Series of closes), already cropped to the period. Returns PNG bytes (1280x800, Telegram's photo size).""" - from matplotlib.figure import Figure - from matplotlib.backends.backend_agg import FigureCanvasAgg from matplotlib.ticker import MaxNLocator, FuncFormatter - bg, ink, ink2, grid = '#14181f', '#eef1f5', '#9aa4b2', '#2a313c' + bg, ink, ink2, grid = PALETTE # categorical slots 1-5 and 7 of the validated dark palette; red/green are skipped because they mean down/up colors = ['#3987e5', '#d95926', '#199e70', '#c98500', '#d55181', '#9085e9'] - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=bg) - FigureCanvasAgg(fig) + fig, _ = _figure(bg) ax = fig.add_axes([0.075, 0.105, 0.72, 0.755], facecolor=bg) rebased = [] for label, y in series: @@ -683,10 +554,7 @@ def compare_graph(series, title, subtitle=''): fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') if subtitle: fig.text(0.03, 0.910, subtitle, color=ink2, fontsize=8.5, ha='left', va='center') - buf = io.BytesIO() - fig.savefig(buf, format='png', facecolor=bg) - buf.seek(0) # rewind, or the upload sends an empty file - return buf + return _png(fig, bg) SESSION_TINTS = {'midsession': '#34d399', 'premarket': '#facc15'} # subtle green / yellow wash; at close stays plain @@ -695,22 +563,17 @@ def tinted(bg, color, amount=0.07): mix = lambda i: round(int(bg[i:i + 2], 16) * (1 - amount) + int(color[i:i + 2], 16) * amount) return '#{:02x}{:02x}{:02x}'.format(mix(1), mix(3), mix(5)) -def hbar_chart(rows, title, subtitle='', value_fmt=None, axis_fmt=None, threshold=None, xlim=None, session=None): +def hbar_chart(rows, title, subtitle='', value_fmt=None, threshold=None, xlim=None, session=None): """Horizontal bars that mirror the caption list, first row on top, filling the whole 1280x800 canvas. Blue for >= 0, red below. Each row's label sits at the bar's base and its value at the tip. rows: [(label, value)]; a (label, None) row is a blank spacer. xlim zooms the value axis (bars then start at its left edge). session: 'midsession' or 'premarket' washes the background green or yellow.""" - from matplotlib.figure import Figure - from matplotlib.backends.backend_agg import FigureCanvasAgg - from matplotlib.ticker import MaxNLocator, FuncFormatter value_fmt = value_fmt or (lambda v: signed_percent(round(v))) - axis_fmt = axis_fmt or (lambda v: '0%' if v == 0 else f"{v:+g}%") - bg, ink, ink2, grid = '#14181f', '#eef1f5', '#9aa4b2', '#2a313c' + bg, ink, ink2, _ = PALETTE up, down = '#3987e5', '#fb7185' if session in SESSION_TINTS: bg = tinted(bg, SESSION_TINTS[session]) - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=bg) - canvas = FigureCanvasAgg(fig) + fig, canvas = _figure(bg) top_edge = 0.85 if subtitle else 0.89 left = 0.14 if xlim else 0.04 # zoomed charts keep their labels outside the plot ax = fig.add_axes([left, 0.02, 0.96 - left, top_edge - 0.02], facecolor=bg) @@ -724,7 +587,6 @@ def hbar_chart(rows, title, subtitle='', value_fmt=None, axis_fmt=None, threshol if threshold is not None: ax.axvline(threshold, color=ink2, linewidth=0.9, linestyle=(0, (4, 3)), zorder=4) top, bottom = max(max(values), 0, threshold or 0), min(min(values), 0) - span = (top - bottom) or 1.0 lo0 = xlim[0] if xlim else bottom hi0 = xlim[1] if xlim else top ax.set_xlim(lo0, hi0) @@ -762,20 +624,18 @@ def hbar_chart(rows, title, subtitle='', value_fmt=None, axis_fmt=None, threshol value_width = [t.get_window_extent(renderer).width for t in value_labels] width = ax.get_window_extent(renderer).width lo, hi = lo0, hi0 - inside = [True] * len(real) - for _ in range(12): # the pixels-per-unit scale depends on the limits, which depend on how many labels sit outside: settle it iteratively + for _ in range(12): # the pixels-per-unit scale depends on the limits, which depend on the text beside the bars: settle it iteratively per_unit = width / (hi - lo) - inside = [False] * len(real) # labels all sit outside the bars, at the base - # room for the text that sits outside the bars: tip values, and base labels of bars too short to hold theirs + # room for the text outside the bars: each tip's value, and each base's label (zoomed charts keep those in the left margin) new_lo, new_hi = lo0, hi0 - for (_, _, v), w, vw, ins in zip(real, label_width, value_width, inside): + for (_, _, v), w, vw in zip(real, label_width, value_width): if v >= 0: new_hi = max(new_hi, v + (vw + 12) / per_unit) - if not ins and not xlim: + if not xlim: new_lo = min(new_lo, base - (w + 12) / per_unit) else: new_lo = min(new_lo, v - (vw + 12) / per_unit) - if not ins and not xlim: + if not xlim: new_hi = max(new_hi, base + (w + 12) / per_unit) if abs(new_lo - lo) < 1e-6 * (hi - lo) and abs(new_hi - hi) < 1e-6 * (hi - lo): break @@ -787,10 +647,7 @@ def hbar_chart(rows, title, subtitle='', value_fmt=None, axis_fmt=None, threshol fig.text(0.03, 0.945 if not subtitle else 0.935, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') if subtitle: fig.text(0.03, 0.885, subtitle, color=ink2, fontsize=8.5, ha='left', va='center') - buf = io.BytesIO() - fig.savefig(buf, format='png', facecolor=bg) - buf.seek(0) # rewind, or the upload sends an empty file - return buf + return _png(fig, bg) SHORTS_VERDICTS = ((5, 'board shorts'), (10, 'cargo shorts'), (15, 'boxers'), (20, 'gym shorts'), # five points each at the bottom (30, 'cutoffs'), (40, 'short shorts'), (50, 'daisy dukes'), (60, 'hot pants'), (70, 'speedos'), (80, 'budgie smugglers'), (90, 'just a belt'), @@ -805,8 +662,7 @@ def shorts_chart(rows, threshold, step=5): rows: [(label, percent)], largest first. Loud on purpose and drawn from patches, since matplotlib cannot render colour emoji. Returns PNG bytes (1280x800).""" import math - from matplotlib.figure import Figure - from matplotlib.backends.backend_agg import FigureCanvasAgg + import numpy as np from matplotlib.patches import Polygon, Rectangle, Circle, Ellipse, PathPatch from matplotlib.path import Path sky, ink, sea, foam, ray = '#ffe66d', '#1b1f3b', '#1ea7e1', '#b5e8ff', '#ffd23f' @@ -815,8 +671,7 @@ def shorts_chart(rows, threshold, step=5): if len(rows) > 24: # pairs of shorts need height: keep the most and the least shorted rows = rows[:12] + rows[-12:] rows = [(label if len(label) <= 10 else label[:9] + '…', value) for label, value in rows] - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=sky) - canvas = FigureCanvasAgg(fig) + fig, canvas = _figure(sky) ax = fig.add_axes([0, 0, 1, 1], facecolor=sky) ax.set_xlim(0, W) ax.set_ylim(H, 0) # pixels, y down @@ -909,10 +764,7 @@ def at(u, v): # face coordinates in sun radii, y down ax.plot([line_x, line_x], [top - 12, bottom], color=ink, linewidth=1.8, linestyle=(0, (5, 4)), zorder=0) ax.text(line_x, top - 18, f"{threshold}% outrageous line", color=ink, fontsize=7.5, fontweight='bold', ha='center', va='bottom') - buf = io.BytesIO() - fig.savefig(buf, format='png', facecolor=sky) - buf.seek(0) - return buf + return _png(fig, sky) def rows_chart(rows, title, subtitle='', **kw): """Horizontal bars for a list of (label, value). Over 40 rows keeps the first and last 20.""" @@ -923,10 +775,8 @@ def rows_chart(rows, title, subtitle='', **kw): def pie_chart(rows, title, subtitle=''): """Pie of (label, value) rows, largest first, with a legend of name and share. At most 8 named slices (the dark categorical slots, in fixed order); a slice labelled 'Other' is gray. Returns PNG bytes (1280x800).""" - from matplotlib.figure import Figure - from matplotlib.backends.backend_agg import FigureCanvasAgg from matplotlib.patches import Rectangle - bg, ink, ink2 = '#14181f', '#eef1f5', '#9aa4b2' + bg, ink, ink2, _ = PALETTE slots = ['#3987e5', '#d95926', '#199e70', '#c98500', '#d55181', '#008300', '#9085e9', '#e66767'] other = '#6b7280' total = sum(v for _, v in rows) or 1.0 @@ -937,8 +787,7 @@ def pie_chart(rows, title, subtitle=''): else: colors.append(slots[n % len(slots)]) n += 1 - fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=bg) - FigureCanvasAgg(fig) + fig, _ = _figure(bg) ax = fig.add_axes([0.02, 0.02, 0.525, 0.84], facecolor=bg) # square: 0.84 * 4in = 0.525 * 6.4in ax.pie([v for _, v in rows], colors=colors, startangle=90, counterclock=False, wedgeprops={'edgecolor': bg, 'linewidth': 2}) # 2px surface gap between slices @@ -955,10 +804,7 @@ def pie_chart(rows, title, subtitle=''): fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') if subtitle: fig.text(0.03, 0.910, subtitle, color=ink2, fontsize=8.5, ha='left', va='center') - buf = io.BytesIO() - fig.savefig(buf, format='png', facecolor=bg) - buf.seek(0) - return buf + return _png(fig, bg) def bar_graph(top_rows, bottom_rows, title, subtitle=''): """Top performers then, after a gap, the worst performers: the same order as the caption's two lists.""" @@ -980,30 +826,24 @@ def get_emoji(number): return '▪️' def days_from_human_days(arg): + """7, 7d, 2w, 3m, 1y or ytd -> days. Raises ValueError for anything else.""" arg = arg.upper() - today = datetime.datetime.now().date() if arg == 'YTD': - target = today.replace(day=1, month=1) - return (today - target).days - try: - days = int(arg.removesuffix('D')) - except ValueError: + today = datetime.datetime.now().date() + return (today - today.replace(day=1, month=1)).days + for suffix, days in (('D', 1), ('W', 7), ('M', 30)): try: - days = int(arg.removesuffix('W')) * 7 + return int(arg.removesuffix(suffix)) * days except ValueError: - try: - days = int(arg.removesuffix('M')) * 30 - except ValueError: - days = int(arg.removesuffix('Y')) * 365 - return days + pass + return int(arg.removesuffix('Y')) * 365 def get_holdings_and_watchlist(): tickers = set(sharesight.get_holdings_wrapper()) tickers.update(json_load('finbot_watchlist.json', persist=True) or []) # absent until the first add if 'GOOG' in tickers and 'GOOGL' in tickers: tickers.remove("GOOGL") - tickers = sorted(set(tickers)) - return tickers + return sorted(tickers) def ordinal(num): value = str(num) diff --git a/lib/worker.py b/lib/worker.py index 01fb0954..6c743f03 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -620,7 +620,7 @@ def alliterate(): heading = 'Beta over 1.5 and mkt cap under 2B' payload.insert(0, f"{webhook.bold(heading, service)}") chart_rows.sort(key=lambda row: row[1], reverse=True) - chart = {'rows': chart_rows, 'title': heading, 'value_fmt': lambda v: f"{v:g}", 'axis_fmt': lambda v: f"{v:g}"} + chart = {'rows': chart_rows, 'title': heading, 'value_fmt': lambda v: f"{v:g}"} deliver(service, url, chat_id, payload, chart) elif m_pie: words = m_pie.group('args').split() diff --git a/tests/test_datadir.py b/tests/test_datadir.py index f3693022..417b4b05 100644 --- a/tests/test_datadir.py +++ b/tests/test_datadir.py @@ -40,7 +40,7 @@ def run(**env): def test_util_import_sets_mplconfigdir_without_home(self): e = {k: v for k, v in os.environ.items() if k not in ('MPLCONFIGDIR', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', 'HOME')} - code = "import os; from lib import util; print(os.environ['MPLCONFIGDIR'])" + code = "import os; from lib import util; import matplotlib.dates; print(os.environ['MPLCONFIGDIR'])" # util loads matplotlib lazily with tempfile.TemporaryDirectory() as d: out = subprocess.run([sys.executable, '-c', code], cwd=REPO, env=dict(e, DATA_DIR=d), capture_output=True, text=True, check=True).stdout.strip() self.assertEqual(out.split()[-1], d + '/cache/matplotlib') From 5458ab61bb61d04c18eaea7906ab1d15a446213c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:19:12 +0000 Subject: [PATCH 04/25] yahoo: quote fields read from tables; renewing the cookie and crumb is one thread at a time; fetch_each - fetch_detail reads its ~40 plain fields from DETAIL_FIELDS (path, conversion, and the extra error that also means "not there" where the old block caught one) instead of 55 copy-paste try blocks; the irregular ones (52-week low, net income, recommendations, trend, quarterly, insider) stay code. The duplicate currency block is gone. Checked against the old parser on the doc/yahoo samples and 6,000 single-point mutations of them (fields and modules deleted, nulled, given bad types): the same dict or the same exception every time - fetch() reads its optional quote fields from QUOTE_FIELDS (1,600 mutations, identical) - price_history drops a caption it built and never used, filters the days range once, and looks up the date-based periods (10Y to 1M, 7D) in one branch (identical tables and charts on 129 calls) - fetch_chart_json parses the response once and no longer fetches a cookie it never sends (getCrumb still renews it when due); getCookie measures the cache file's age with util.cache_age; rmCrumb moves the stale crumb aside with Path.replace - getCookie and getCrumb hold one lock: two threads renewing at once could cache a crumb for the cookie the other just replaced, for 14 days. prefetch_history already fetched 8 at a time - fetch_each(get, tickers) runs per-ticker fetches 8 at a time and returns results or exceptions in order; prefetch_history uses it - pandas loads in the functions that parse a history, not when the module is imported Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/yahoo.py | 782 ++++++++++++++++----------------------------------- 1 file changed, 246 insertions(+), 536 deletions(-) diff --git a/lib/yahoo.py b/lib/yahoo.py index 29810826..7e370ed5 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -3,99 +3,111 @@ from dateutil.relativedelta import relativedelta import hashlib import requests -import pandas as pd +import threading import zoneinfo -from pandas.tseries.offsets import BDay as businessday import sys -import shutil from lib.config import * from lib import util from http.cookies import SimpleCookie from pathlib import Path +_auth = threading.RLock() # Yahoo's cookie and crumb are renewed by one thread at a time: two at once could cache a crumb for the cookie the other replaced + def getCookie(maxAge=1209600): # 14 days # we must always cache this, as crumb must match cookie - cacheFile = "finbot_yahoo_cookie.json" - cache = util.json_load(cacheFile) - if cache: - cacheFileAge = datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(config_cache_dir + '/' + cacheFile)) - #ttl = datetime.timedelta(seconds=cache[0][2]) - cacheFileAge - ttl = datetime.timedelta(seconds=maxAge) - cacheFileAge - if ttl > datetime.timedelta(seconds=30): - cookie = cache[0][0] + '=' + cache[0][1] - print("cache hit:", cacheFile, "TTL:", util.td_to_human(ttl), file=sys.stderr) if debug else None - return cookie - else: - print("cache miss:", cacheFile, file=sys.stderr) if debug else None - - # request - cookie = None - headers = {'User-Agent': 'Mozilla/5.0'} - url = 'https://fc.yahoo.com/' # 404, but that is ok because cookie is in response header - try: - r = requests.get(url, headers=headers) - except Exception as e: - print(e, file=sys.stderr) - r = requests.Response() # empty: falls through to the fallback cookie below - if r.status_code not in {200, 404}: - print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) - - # parse - if not 'set-cookie' in r.headers: - print("Failed to obtain Yahoo auth cookie. Returning fallback cookie", file=sys.stderr) - fallback='A3=d=AQABBBtR5mYCENFZu2wCWkA5iGGkSRGvRgkFEgEBAQGi52bwZtxM0iMA_eMAAA&S=AQAAAtG8VhxZN7aXopfvLNObtpE;' - return fallback - else: - cookie = SimpleCookie() - cookie.load(r.headers['Set-Cookie']) - cookielist = [] - for name, morsel in cookie.items(): - value = morsel.value - max_age = int(morsel.get('max-age')) - cookielist.append([name, value, max_age]) - cookie = cookielist[0][0] + '=' + str(cookielist[0][1]) - #if config_cache: - util.json_write(cacheFile, cookielist) - print("Got new cookie:", cookie, file=sys.stderr) - rmCrumb() # we must invalidate the non-matching crumb - return cookie - -def getCrumb(seconds=1209600): # 14 days - cookie = getCookie() - if config_cache: - cacheFile = "finbot_yahoo_crumb.json" - cache = util.read_cache(cacheFile, seconds) + with _auth: + cacheFile = "finbot_yahoo_cookie.json" + cache = util.json_load(cacheFile) if cache: - return cache - headers = {'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} - yahoo_urls = ['https://query2.finance.yahoo.com/v1/test/getcrumb'] - yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) - for url in yahoo_urls: + #ttl = datetime.timedelta(seconds=cache[0][2] - util.cache_age(cacheFile)) + ttl = datetime.timedelta(seconds=maxAge - util.cache_age(cacheFile)) + if ttl > datetime.timedelta(seconds=30): + cookie = cache[0][0] + '=' + cache[0][1] + print("cache hit:", cacheFile, "TTL:", util.td_to_human(ttl), file=sys.stderr) if debug else None + return cookie + else: + print("cache miss:", cacheFile, file=sys.stderr) if debug else None + + # request + cookie = None + headers = {'User-Agent': 'Mozilla/5.0'} + url = 'https://fc.yahoo.com/' # 404, but that is ok because cookie is in response header try: - r = requests.get(url, headers=headers, timeout=config_http_timeout) + r = requests.get(url, headers=headers) except Exception as e: print(e, file=sys.stderr) + r = requests.Response() # empty: falls through to the fallback cookie below + if r.status_code not in {200, 404}: + print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) + + # parse + if not 'set-cookie' in r.headers: + print("Failed to obtain Yahoo auth cookie. Returning fallback cookie", file=sys.stderr) + fallback='A3=d=AQABBBtR5mYCENFZu2wCWkA5iGGkSRGvRgkFEgEBAQGi52bwZtxM0iMA_eMAAA&S=AQAAAtG8VhxZN7aXopfvLNObtpE;' + return fallback else: - if r.status_code != 200: - print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) - continue - break - else: - print("Exhausted Yahoo API attempts. Returning fallback crumb", file=sys.stderr) - return 'jkQEU8yLqxs' - if config_cache: - util.json_write(cacheFile, r.text) - print("Got new crumb:", r.text.rstrip(), file=sys.stderr) - return r.text + cookie = SimpleCookie() + cookie.load(r.headers['Set-Cookie']) + cookielist = [] + for name, morsel in cookie.items(): + value = morsel.value + max_age = int(morsel.get('max-age')) + cookielist.append([name, value, max_age]) + cookie = cookielist[0][0] + '=' + str(cookielist[0][1]) + #if config_cache: + util.json_write(cacheFile, cookielist) + print("Got new cookie:", cookie, file=sys.stderr) + rmCrumb() # we must invalidate the non-matching crumb + return cookie + +def getCrumb(seconds=1209600): # 14 days + with _auth: + cookie = getCookie() + if config_cache: + cacheFile = "finbot_yahoo_crumb.json" + cache = util.read_cache(cacheFile, seconds) + if cache: + return cache + headers = {'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} + yahoo_urls = ['https://query2.finance.yahoo.com/v1/test/getcrumb'] + yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) + for url in yahoo_urls: + try: + r = requests.get(url, headers=headers, timeout=config_http_timeout) + except Exception as e: + print(e, file=sys.stderr) + else: + if r.status_code != 200: + print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) + continue + break + else: + print("Exhausted Yahoo API attempts. Returning fallback crumb", file=sys.stderr) + return 'jkQEU8yLqxs' + if config_cache: + util.json_write(cacheFile, r.text) + print("Got new crumb:", r.text.rstrip(), file=sys.stderr) + return r.text def rmCrumb(): - cacheFile = "finbot_yahoo_crumb.json" - cacheFile = config_cache_dir + "/" + cacheFile - dst = Path(cacheFile + '.old') - src = Path(cacheFile) - if src.is_file(): - dst.unlink(missing_ok=True) - shutil.move(src, dst) + """Set the cached crumb aside (as .old): it belongs to the cookie being replaced.""" + crumb = Path(util.cache_path("finbot_yahoo_crumb.json")) + if crumb.is_file(): + crumb.replace(crumb.with_name(crumb.name + '.old')) + +QUOTE_FIELDS = ( # the optional fields of a v7 quote: (key, Yahoo's name, conversion) + ('percent_change_premarket', 'preMarketChangePercent', lambda v: round(v, 2)), + ('quoteType', 'quoteType', None), + ('percent_change_postmarket', 'postMarketChangePercent', lambda v: round(v, 2)), + ('market_cap', 'marketCap', lambda v: round(float(v))), + ('marketState', 'marketState', None), + ('profile_exchange', 'fullExchangeName', None), + ('exchangeTimezoneName', 'exchangeTimezoneName', None), + ('regularMarketTime', 'regularMarketTime', None), + ('financialCurrency', 'financialCurrency', None), + ('regularMarketPreviousClose', 'regularMarketPreviousClose', None), + ('fiftyTwoWeekHigh', 'fiftyTwoWeekHigh', None), +) def fetch(tickers): # DO NOT CACHE MORE THAN 5 mins @@ -163,107 +175,106 @@ def fetch(tickers): profile_title = util.transform_title(profile_title) yahoo_output[ticker] = { 'profile_title': profile_title, 'ticker': ticker, 'percent_change': percent_change, 'dividend': dividend, 'currency': currency, 'regularMarketPrice': regularMarketPrice } # optional fields - try: - percent_change_premarket = item['preMarketChangePercent'] - except (KeyError, IndexError): - pass - else: - yahoo_output[ticker]["percent_change_premarket"] = round(percent_change_premarket, 2) - try: - quoteType = item['quoteType'] - except (KeyError, IndexError): - pass - else: - yahoo_output[ticker]["quoteType"] = quoteType - try: - percent_change_postmarket = item['postMarketChangePercent'] - except (KeyError, IndexError): - pass - else: - yahoo_output[ticker]["percent_change_postmarket"] = round(percent_change_postmarket, 2) - try: - yahoo_output[ticker]["market_cap"] = round(float(item['marketCap'])) - except (KeyError, IndexError): - pass - try: - yahoo_output[ticker]["price_to_earnings_forward"] = round(item['forwardPE']) - except: - pass - try: - yahoo_output[ticker]["price_to_earnings_trailing"] = round(item['trailingPE']) - except: - pass - try: - yahoo_output[ticker]["marketState"] = item['marketState'] - except: - pass - try: - yahoo_output[ticker]["profile_exchange"] = item['fullExchangeName'] - except: - pass - try: - yahoo_output[ticker]["exchangeTimezoneName"] = item['exchangeTimezoneName'] - except: - pass - try: - yahoo_output[ticker]["regularMarketTime"] = item['regularMarketTime'] - except: - pass - try: - yahoo_output[ticker]["financialCurrency"] = item['financialCurrency'] - except: - pass + quote = yahoo_output[ticker] + for key, field, convert in QUOTE_FIELDS: + if field in item: + quote[key] = convert(item[field]) if convert else item[field] + for key, field in (('price_to_earnings_forward', 'forwardPE'), ('price_to_earnings_trailing', 'trailingPE')): + try: + quote[key] = round(item[field]) + except: + pass try: earningsTimestamp = item['earningsTimestamp'] earningsTimestampStart = item['earningsTimestampStart'] earningsTimestampEnd = item['earningsTimestampEnd'] if earningsTimestamp > now: - yahoo_output[ticker]["earnings_date"] = earningsTimestamp + quote["earnings_date"] = earningsTimestamp elif earningsTimestampStart > now: - yahoo_output[ticker]["earnings_date"] = earningsTimestampStart + quote["earnings_date"] = earningsTimestampStart elif earningsTimestampEnd > now: - yahoo_output[ticker]["earnings_date"] = earningsTimestampEnd + quote["earnings_date"] = earningsTimestampEnd else: - yahoo_output[ticker]["earnings_date"] = earningsTimestamp - except (KeyError, IndexError): - pass - try: - yahoo_output[ticker]["regularMarketPreviousClose"] = item['regularMarketPreviousClose'] - except (KeyError, IndexError): - pass - try: - yahoo_output[ticker]["fiftyTwoWeekHigh"] = item['fiftyTwoWeekHigh'] + quote["earnings_date"] = earningsTimestamp except (KeyError, IndexError): pass try: fiftyTwoWeekLowTemp = item['fiftyTwoWeekLow'] if not fiftyTwoWeekLowTemp <= 0: - yahoo_output[ticker]["fiftyTwoWeekLow"] = fiftyTwoWeekLowTemp + quote["fiftyTwoWeekLow"] = fiftyTwoWeekLowTemp except (KeyError, IndexError): pass if config_cache: util.json_write(cacheFile, yahoo_output) return yahoo_output +def _percent(fraction): + """Yahoo's fraction as a percentage, to one decimal: 0.01234 -> 1.2.""" + return round(fraction * 100, 1) + +def _float_percent(fraction): + return _percent(float(fraction)) + +DETAIL_FIELDS = ( # (key, path in the quoteSummary result[, conversion[, another error that means "not there"]]), applied in this order + ('profile_bio', ('summaryProfile', 'longBusinessSummary')), + ('profile_city', ('summaryProfile', 'city')), + ('profile_country', ('summaryProfile', 'country')), + ('profile_state', ('summaryProfile', 'state')), + ('profile_industry', ('summaryProfile', 'industry')), + ('profile_sector', ('summaryProfile', 'sector')), + ('profile_employees', ('summaryProfile', 'fullTimeEmployees')), + ('profile_website', ('summaryProfile', 'website')), + ('beta', ('summaryDetail', 'beta', 'raw')), + ('currency', ('summaryDetail', 'currency')), + ('market_cap', ('summaryDetail', 'marketCap', 'raw'), int), + ('dividend', ('summaryDetail', 'dividendYield', 'raw'), _percent), + ('price_to_earnings_trailing', ('summaryDetail', 'trailingPE', 'raw'), int, ValueError), + ('price_to_earnings_forward', ('summaryDetail', 'forwardPE', 'raw')), + ('ex_dividend_date', ('calendarEvents', 'exDividendDate', 'raw')), + ('dividend_date', ('calendarEvents', 'DividendDate', 'raw')), + ('earnings_date', ('calendarEvents', 'earnings', 'earningsDate', 0, 'raw')), + ('percent_change', ('price', 'regularMarketChangePercent', 'raw'), _percent), + ('percent_change_year', ('summaryDetail', '52WeekChange', 'raw'), _float_percent), + ('percent_change_premarket', ('price', 'preMarketChangePercent', 'raw'), _float_percent), + ('percent_change_postmarket', ('price', 'postMarketChangePercent', 'raw'), _float_percent), + ('profile_exchange', ('price', 'exchangeName')), + ('marketState', ('price', 'marketState')), + ('quoteType', ('price', 'quoteType')), + ('regularMarketPrice', ('price', 'regularMarketPrice', 'raw')), # required: an answer without it is junk + ('regularMarketPreviousClose', ('price', 'regularMarketPreviousClose', 'raw')), + ('fiftyTwoWeekHigh', ('price', 'fiftyTwoWeekHigh', 'raw')), + ('prePostMarketPrice', ('price', 'preMarketPrice', 'raw'), float, ValueError), + ('prePostMarketPrice', ('price', 'postMarketPrice', 'raw'), float, ValueError), # the after-hours price, when there is one, wins + ('netIncomeToCommon', ('defaultKeyStatistics', 'netIncomeToCommon', 'raw')), + ('sharesOutstanding', ('defaultKeyStatistics', 'sharesOutstanding', 'raw'), float), + ('short_percent', ('defaultKeyStatistics', 'shortPercentOfFloat', 'raw'), _float_percent), + ('price_to_book', ('defaultKeyStatistics', 'priceToBook', 'raw')), + ('earnings_growth_q', ('defaultKeyStatistics', 'earningsQuarterlyGrowth', 'raw')), + ('price_to_earnings_peg', ('defaultKeyStatistics', 'pegRatio', 'raw')), + ('profit_margin', ('defaultKeyStatistics', 'profitMargins', 'raw')), + ('shareholder_equity', ('balanceSheetHistoryQuarterly', 'balanceSheetStatements', 0, 'totalStockholderEquity', 'raw')), + ('total_debt', ('financialData', 'totalDebt', 'raw')), + ('total_cash', ('financialData', 'totalCash', 'raw')), + ('free_cashflow', ('financialData', 'freeCashflow', 'raw')), + ('operating_cashflow', ('financialData', 'operatingCashflow', 'raw')), + ('financialCurrency', ('financialData', 'financialCurrency')), + ('price_to_sales', ('summaryDetail', 'priceToSalesTrailing12Months', 'raw')), +) + def fetch_detail(ticker, seconds=UNSET): if seconds is UNSET: seconds = config_cache_seconds now = datetime.datetime.now() - local_market_data = {} cookie = getCookie() crumb = getCrumb() base_url = 'https://query2.finance.yahoo.com/v10/finance/quoteSummary/' #headers={'Content-type': 'application/json', 'User-Agent': 'Mozilla/5.0'} headers={'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} - local_market_data[ticker] = dict() - cache = None - if config_cache: - cacheFile = "finbot_yahoo_detail_" + ticker + '.json' - cache = util.read_cache(cacheFile, seconds) - if cache: + cacheFile = "finbot_yahoo_detail_" + ticker + '.json' + data = util.read_cache(cacheFile, seconds) if config_cache else None + if data: print('.', sep=' ', end='', flush=True) - data = cache - if not config_cache or not cache: + else: print('↓', sep=' ', end='', flush=True) yahoo_urls = [base_url + ticker + '?modules=calendarEvents,defaultKeyStatistics,balanceSheetHistoryQuarterly,financialData,summaryProfile,summaryDetail,price,earnings,earningsTrend,insiderTransactions' + '&crumb=' + crumb] yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) @@ -290,326 +301,64 @@ def fetch_detail(ticker, seconds=UNSET): # netSharePurchaseActivity # upgradeDowngradeHistory try: - profile_title = data['quoteSummary']['result'][0]['price']['longName'] + result = data['quoteSummary']['result'][0] + profile_title = result['price']['longName'] except (KeyError, IndexError, ValueError, TypeError): # TypeError: Yahoo answers result=null for unknown symbols print(ticker + 'x', sep=' ', end='', flush=True, file=sys.stderr) return {} if profile_title is None: try: - profile_title = data['quoteSummary']['result'][0]['price']['shortName'] + profile_title = result['price']['shortName'] except (KeyError, IndexError, ValueError): print(ticker + '†', sep=' ', end='', flush=True, file=sys.stderr) return {} if profile_title is None: # catches some delisted stocks like "DUB" print(f"{ticker}†", sep=' ', end='', flush=True, file=sys.stderr) return {} - profile_title = util.transform_title(profile_title) - local_market_data[ticker]['profile_title'] = profile_title - try: - profile_bio = data['quoteSummary']['result'][0]['summaryProfile']['longBusinessSummary'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_bio'] = profile_bio - try: - profile_city = data['quoteSummary']['result'][0]['summaryProfile']['city'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_city'] = profile_city - try: - profile_country = data['quoteSummary']['result'][0]['summaryProfile']['country'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_country'] = profile_country - try: - profile_state = data['quoteSummary']['result'][0]['summaryProfile']['state'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_state'] = profile_state - try: - profile_industry = data['quoteSummary']['result'][0]['summaryProfile']['industry'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_industry'] = profile_industry - try: - profile_sector = data['quoteSummary']['result'][0]['summaryProfile']['sector'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_sector'] = profile_sector - try: - profile_employees = data['quoteSummary']['result'][0]['summaryProfile']['fullTimeEmployees'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_employees'] = profile_employees - try: - profile_website = data['quoteSummary']['result'][0]['summaryProfile']['website'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_website'] = profile_website - try: - beta = data['quoteSummary']['result'][0]['summaryDetail']['beta']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['beta'] = beta - try: - currency = data['quoteSummary']['result'][0]['summaryDetail']['currency'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['currency'] = currency - try: - market_cap = int(data['quoteSummary']['result'][0]['summaryDetail']['marketCap']['raw']) - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['market_cap'] = market_cap - try: - dividend = data['quoteSummary']['result'][0]['summaryDetail']['dividendYield']['raw'] - except (KeyError, IndexError): - pass - else: - dividend = dividend * 100 - local_market_data[ticker]['dividend'] = round(dividend, 1) - try: - price_to_earnings_trailing = int(data['quoteSummary']['result'][0]['summaryDetail']['trailingPE']['raw']) - except (KeyError, IndexError, ValueError): - pass - else: - local_market_data[ticker]['price_to_earnings_trailing'] = price_to_earnings_trailing - try: - price_to_earnings_forward = data['quoteSummary']['result'][0]['summaryDetail']['forwardPE']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['price_to_earnings_forward'] = price_to_earnings_forward - try: - ex_dividend_date = data['quoteSummary']['result'][0]['calendarEvents']['exDividendDate']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['ex_dividend_date'] = ex_dividend_date - try: - dividend_date = data['quoteSummary']['result'][0]['calendarEvents']['DividendDate']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['dividend_date'] = dividend_date - try: - earnings_date = data['quoteSummary']['result'][0]['calendarEvents']['earnings']['earningsDate'][0]['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['earnings_date'] = earnings_date - try: - percent_change = data['quoteSummary']['result'][0]['price']['regularMarketChangePercent']['raw'] - except (KeyError, IndexError): - pass - else: - percent_change = percent_change * 100 - local_market_data[ticker]['percent_change'] = round(percent_change, 1) - try: - percent_change_year = float(data['quoteSummary']['result'][0]['summaryDetail']['52WeekChange']['raw']) - except (KeyError, IndexError): - pass - else: - percent_change_year = percent_change_year * 100 - local_market_data[ticker]['percent_change_year'] = round(percent_change_year, 1) - try: - percent_change_premarket = float(data['quoteSummary']['result'][0]['price']['preMarketChangePercent']['raw']) - except (KeyError, IndexError): - pass - else: - percent_change_premarket = percent_change_premarket * 100 - local_market_data[ticker]['percent_change_premarket'] = round(percent_change_premarket, 1) - try: - percent_change_postmarket = float(data['quoteSummary']['result'][0]['price']['postMarketChangePercent']['raw']) - except (KeyError, IndexError): - pass - else: - percent_change_postmarket = percent_change_postmarket * 100 - local_market_data[ticker]['percent_change_postmarket'] = round(percent_change_postmarket, 1) - try: - profile_exchange = data['quoteSummary']['result'][0]['price']['exchangeName'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profile_exchange'] = profile_exchange - try: - marketState = data['quoteSummary']['result'][0]['price']['marketState'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['marketState'] = marketState - try: - quoteType = data['quoteSummary']['result'][0]['price']['quoteType'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['quoteType'] = quoteType - try: - regularMarketPrice = data['quoteSummary']['result'][0]['price']['regularMarketPrice']['raw'] - except (KeyError, IndexError): - return {} # catches junk - else: - local_market_data[ticker]['regularMarketPrice'] = regularMarketPrice - try: - regularMarketPreviousClose = data['quoteSummary']['result'][0]['price']['regularMarketPreviousClose']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['regularMarketPreviousClose'] = regularMarketPreviousClose - try: - fiftyTwoWeekHigh = data['quoteSummary']['result'][0]['price']['fiftyTwoWeekHigh']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['fiftyTwoWeekHigh'] = fiftyTwoWeekHigh - try: - fiftyTwoWeekLowTemp = data['quoteSummary']['result'][0]['price']['fiftyTwoWeekLow']['raw'] + detail = {'profile_title': util.transform_title(profile_title)} + for key, path, *how in DETAIL_FIELDS: + try: + value = result + for step in path: + value = value[step] + detail[key] = how[0](value) if how else value + except (KeyError, IndexError, *how[1:]): + if key == 'regularMarketPrice': + return {} # catches junk + try: + fiftyTwoWeekLowTemp = result['price']['fiftyTwoWeekLow']['raw'] if not fiftyTwoWeekLowTemp <= 0: fiftyTwoWeekLow = fiftyTwoWeekLowTemp except (KeyError, IndexError): pass else: - local_market_data[ticker]['fiftyTwoWeekLow'] = fiftyTwoWeekLow - try: - preMarketPrice = float(data['quoteSummary']['result'][0]['price']['preMarketPrice']['raw']) - except (KeyError, IndexError, ValueError): - pass - else: - local_market_data[ticker]['prePostMarketPrice'] = preMarketPrice - try: - postMarketPrice = float(data['quoteSummary']['result'][0]['price']['postMarketPrice']['raw']) - except (KeyError, IndexError, ValueError): - pass - else: - local_market_data[ticker]['prePostMarketPrice'] = postMarketPrice - try: - netIncomeToCommon = data['quoteSummary']['result'][0]['defaultKeyStatistics']['netIncomeToCommon']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['netIncomeToCommon'] = netIncomeToCommon - try: - sharesOutstanding = float(data['quoteSummary']['result'][0]['defaultKeyStatistics']['sharesOutstanding']['raw']) - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['sharesOutstanding'] = sharesOutstanding - try: - short_percent = float(data['quoteSummary']['result'][0]['defaultKeyStatistics']['shortPercentOfFloat']['raw']) - except (KeyError, IndexError): - pass - else: - short_percent = short_percent * 100 - local_market_data[ticker]['short_percent'] = round(short_percent, 1) + detail['fiftyTwoWeekLow'] = fiftyTwoWeekLow try: - price_to_book = data['quoteSummary']['result'][0]['defaultKeyStatistics']['priceToBook']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['price_to_book'] = price_to_book - try: - earnings_growth_q = data['quoteSummary']['result'][0]['defaultKeyStatistics']['earningsQuarterlyGrowth']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['earnings_growth_q'] = earnings_growth_q - try: - price_to_earnings_peg = data['quoteSummary']['result'][0]['defaultKeyStatistics']['pegRatio']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['price_to_earnings_peg'] = price_to_earnings_peg - try: - profit_margin = data['quoteSummary']['result'][0]['defaultKeyStatistics']['profitMargins']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['profit_margin'] = profit_margin - try: - if data['quoteSummary']['result'][0]['earnings']['financialsChart']['quarterly'][-1]['earnings']['fmt'] is not None: - net_income = data['quoteSummary']['result'][0]['earnings']['financialsChart']['quarterly'][-1]['earnings']['raw'] - elif data['quoteSummary']['result'][0]['earnings']['financialsChart']['quarterly'][-2]['earnings']['fmt'] is not None: - net_income = data['quoteSummary']['result'][0]['earnings']['financialsChart']['quarterly'][-2]['earnings']['raw'] + quarterly = result['earnings']['financialsChart']['quarterly'] + if quarterly[-1]['earnings']['fmt'] is not None: + net_income = quarterly[-1]['earnings']['raw'] + elif quarterly[-2]['earnings']['fmt'] is not None: + net_income = quarterly[-2]['earnings']['raw'] else: raise TypeError('quarterly earnings is null') except (KeyError, IndexError, TypeError): pass else: - local_market_data[ticker]['net_income'] = net_income - try: - shareholder_equity = data['quoteSummary']['result'][0]['balanceSheetHistoryQuarterly']['balanceSheetStatements'][0]['totalStockholderEquity']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['shareholder_equity'] = shareholder_equity - try: - total_debt = data['quoteSummary']['result'][0]['financialData']['totalDebt']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['total_debt'] = total_debt - try: - total_cash = data['quoteSummary']['result'][0]['financialData']['totalCash']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['total_cash'] = total_cash - try: - free_cashflow = data['quoteSummary']['result'][0]['financialData']['freeCashflow']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['free_cashflow'] = free_cashflow - try: - operating_cashflow = data['quoteSummary']['result'][0]['financialData']['operatingCashflow']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['operating_cashflow'] = operating_cashflow - try: - financialCurrency = data['quoteSummary']['result'][0]['financialData']['financialCurrency'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['financialCurrency'] = financialCurrency - try: - recommend = data['quoteSummary']['result'][0]['financialData']['recommendationKey'] - recommend_index = data['quoteSummary']['result'][0]['financialData']['recommendationMean']['raw'] - recommend_analysts = data['quoteSummary']['result'][0]['financialData']['numberOfAnalystOpinions']['raw'] - except (KeyError, IndexError): - pass - else: - local_market_data[ticker]['recommend'] = recommend - local_market_data[ticker]['recommend_index'] = recommend_index - local_market_data[ticker]['recommend_analysts'] = recommend_analysts + detail['net_income'] = net_income try: - price_to_sales = data['quoteSummary']['result'][0]['summaryDetail']['priceToSalesTrailing12Months']['raw'] + recommend = result['financialData']['recommendationKey'] + recommend_index = result['financialData']['recommendationMean']['raw'] + recommend_analysts = result['financialData']['numberOfAnalystOpinions']['raw'] except (KeyError, IndexError): pass else: - local_market_data[ticker]['price_to_sales'] = price_to_sales + detail.update(recommend=recommend, recommend_index=recommend_index, recommend_analysts=recommend_analysts) try: - currency = data['quoteSummary']['result'][0]['summaryDetail']['currency'] + result['earningsTrend']['trend'][0]['growth']['raw'] except (KeyError, IndexError): pass else: - local_market_data[ticker]['currency'] = currency - try: - data['quoteSummary']['result'][0]['earningsTrend']['trend'][0]['growth']['raw'] - except (KeyError, IndexError): - pass - else: - for item in data['quoteSummary']['result'][0]['earningsTrend']['trend']: + for item in result['earningsTrend']['trend']: if item['period'] == '+1y': try: revenueEstimateY = item['revenueEstimate']['growth']['raw'] @@ -621,14 +370,14 @@ def fetch_detail(ticker, seconds=UNSET): try: earningsEstimateY = earningsEstimateY * 100 revenueEstimateY = revenueEstimateY * 100 - local_market_data[ticker]['revenueEstimateY'] = round(revenueEstimateY, 2) - local_market_data[ticker]['earningsEstimateY'] = round(earningsEstimateY, 2) - local_market_data[ticker]['revenueAnalysts'] = revenueAnalysts - local_market_data[ticker]['earningsAnalysts'] = earningsAnalysts + detail['revenueEstimateY'] = round(revenueEstimateY, 2) + detail['earningsEstimateY'] = round(earningsEstimateY, 2) + detail['revenueAnalysts'] = revenueAnalysts + detail['earningsAnalysts'] = earningsAnalysts except UnboundLocalError: print(f'{ticker}†', sep=' ', end='', flush=True, file=sys.stderr) try: - data['quoteSummary']['result'][0]['earnings']['financialsChart']['quarterly'][0]['revenue']['raw'] + result['earnings']['financialsChart']['quarterly'][0]['revenue']['raw'] except (KeyError, IndexError): print(f'{ticker}†', sep=' ', end='', flush=True, file=sys.stderr) else: @@ -636,7 +385,7 @@ def fetch_detail(ticker, seconds=UNSET): revenueQ = [] earningsY = [] revenueY = [] - for item in data['quoteSummary']['result'][0]['earnings']['financialsChart']['quarterly']: + for item in result['earnings']['financialsChart']['quarterly']: if item['earnings']['fmt'] is None: # bad data #if len(earningsQ): # may fix weirdness in dodelta() earningsQ.append(None) @@ -645,15 +394,15 @@ def fetch_detail(ticker, seconds=UNSET): else: earningsQ.append(item['earnings']['raw']) revenueQ.append(item['revenue']['raw']) - for item in data['quoteSummary']['result'][0]['earnings']['financialsChart']['yearly']: + for item in result['earnings']['financialsChart']['yearly']: earningsY.append(item['earnings']['raw']) revenueY.append(item['revenue']['raw']) - local_market_data[ticker]['earningsQ'] = earningsQ - local_market_data[ticker]['revenueQ'] = revenueQ - local_market_data[ticker]['earningsY'] = earningsY - local_market_data[ticker]['revenueY'] = revenueY + detail['earningsQ'] = earningsQ + detail['revenueQ'] = revenueQ + detail['earningsY'] = earningsY + detail['revenueY'] = revenueY try: - data['quoteSummary']['result'][0]['insiderTransactions']['transactions'][0]['startDate']['raw'] + result['insiderTransactions']['transactions'][0]['startDate']['raw'] except (KeyError, IndexError): pass else: @@ -661,7 +410,7 @@ def fetch_detail(ticker, seconds=UNSET): buyValue = 0 sellTotal = 0 sellValue = 0 - for item in data['quoteSummary']['result'][0]['insiderTransactions']['transactions']: + for item in result['insiderTransactions']['transactions']: dt_startDate = datetime.datetime.fromtimestamp(item['startDate']['raw']) if dt_startDate > now - datetime.timedelta(days=90): if 'Buy' in item['transactionText']: @@ -670,13 +419,12 @@ def fetch_detail(ticker, seconds=UNSET): if 'Sale' in item['transactionText']: sellTotal = sellTotal + item['shares']['raw'] sellValue = sellValue + item['value']['raw'] - local_market_data[ticker]['insiderBuy'] = buyTotal - local_market_data[ticker]['insiderSell'] = sellTotal - local_market_data[ticker]['insiderBuyValue'] = buyValue - local_market_data[ticker]['insiderSellValue'] = sellValue + detail['insiderBuy'] = buyTotal + detail['insiderSell'] = sellTotal + detail['insiderBuyValue'] = buyValue + detail['insiderSellValue'] = sellValue - local_market_data[ticker] = dict(sorted(local_market_data[ticker].items())) - return local_market_data + return {ticker: dict(sorted(detail.items()))} def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True): """({period: percent change}, chart image or None). If the chart cannot be fetched: (error string, None), which the callers @@ -685,9 +433,9 @@ def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True graph = config_graph if seconds is UNSET: seconds = config_cache_seconds + import pandas as pd + from pandas.tseries.offsets import BDay as businessday percent_dict = {} - price = [] - image_data = None max_days = 3655 # 10 years + buffer now = datetime.datetime.now() interval = ('10Y', '5Y', '3Y', '1Y', 'YTD', '6M', '3M', '1M', '7D', '5D', '1D') @@ -715,67 +463,32 @@ def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True #df.reset_index(drop=True, inplace=True) if days: - seek_date = now - datetime.timedelta(days = days) - seek_dt = seek_date.date() - mask = df['Date'] >= seek_dt - df = df.loc[mask] - past_price = df[df.loc[mask]['Date'] >= seek_dt]['Close'].iloc[0] - df.reset_index(drop=True, inplace=True) + df = df[df['Date'] >= (now - datetime.timedelta(days=days)).date()].reset_index(drop=True) + past_price = df['Close'].iloc[0] percent = (stock.get('regularMarketPrice') - past_price) / past_price * 100 percent_dict[days] = round(percent, 2) else: default_price = df['Close'].iloc[0] default_percent = round((stock.get('regularMarketPrice') - default_price) / default_price * 100) for period in interval: - # try to align with google finance - if period == 'YTD': - seek_dt = now.replace(day=1, month=1).date() - try: - past_price = df[df['Date'] >= seek_dt]['Close'].iloc[0] - except IndexError: - percent_dict['Max'] = default_percent - continue - elif period == '1D': + if period == '1D': # against the previous session's close: the last close dated before the latest bar, in the exchange's time zone sessions = pd.to_datetime(df['Timestamp'], unit='s', utc=True).dt.tz_convert(tz).dt.date earlier = df[(sessions < sessions.iloc[-1]) & df['Close'].notnull()] if earlier.empty: continue past_price = earlier['Close'].iloc[-1] - elif period == '5D': - #seek_date = now - datetime.timedelta(days = 5) - seek_date = now - businessday(5) - seek_dt = seek_date.date() + else: # try to align with google finance try: - past_price = df[df['Date'] <= seek_dt]['Open'].iloc[-1] - except IndexError: - percent_dict['Max'] = default_percent - continue - elif period.endswith('M'): - months = int(period.removesuffix('M')) - seek_date = now - relativedelta(months=months) - seek_dt = seek_date.date() - try: - past_price = df[df['Date'] <= seek_dt]['Close'].iloc[-1] - except IndexError: - percent_dict['Max'] = default_percent - continue - elif period.endswith('Y'): - years = int(period.removesuffix('Y')) - seek_date = now - relativedelta(years=years) - seek_dt = seek_date.date() - try: - past_price = df[df['Date'] <= seek_dt]['Close'].iloc[-1] - except IndexError: - percent_dict['Max'] = default_percent - continue - else: - seek_date = now - datetime.timedelta(days = int(period.removesuffix('D'))) - seek_dt = seek_date.date() - #print(df[df['Date'] <= seek_dt].tail(1)) - try: - past_price = df[df['Date'] <= seek_dt]['Close'].iloc[-1] - except IndexError: + if period == 'YTD': + past_price = df[df['Date'] >= now.replace(day=1, month=1).date()]['Close'].iloc[0] + elif period == '5D': + past_price = df[df['Date'] <= (now - businessday(5)).date()]['Open'].iloc[-1] + else: # 10Y to 1M, 7D: the last close on or before the day that long ago + n, unit = int(period[:-1]), period[-1] + ago = relativedelta(years=n) if unit == 'Y' else relativedelta(months=n) if unit == 'M' else datetime.timedelta(days=n) + past_price = df[df['Date'] <= (now - ago).date()]['Close'].iloc[-1] + except IndexError: # the history does not go back that far percent_dict['Max'] = default_percent continue percent = (stock.get('regularMarketPrice') - past_price) / past_price * 100 @@ -784,33 +497,22 @@ def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True return percent_dict, None company_name = util.transform_title(stock.get('longName', '')) price = stock.get('regularMarketPrice', '') - caption = [] if days: - title_days = min(days, max_days) - percent = percent_dict.get(days, '') - title = f"{company_name} ({ticker}) {title_days} days {util.signed_percent(percent)} Last: {price}" - caption.append(title) + title = f"{company_name} ({ticker}) {min(days, max_days)} days {util.signed_percent(percent_dict.get(days, ''))} Last: {price}" else: - caption.extend(f"{k}: {util.signed_percent(v)}" for k, v in percent_dict.items()) label = "Max" if "Max" in percent_dict else "10Y" - percent = percent_dict.get(label, '') - title = f"{company_name} ({ticker}) {label} {util.signed_percent(percent)} Last: {price}" - caption = '\n'.join(caption) + title = f"{company_name} ({ticker}) {label} {util.signed_percent(percent_dict.get(label, ''))} Last: {price}" image_cache_file = "finbot_graph_" + ticker + "_" + str(days) + ".png" image_cache = util.read_binary_cache(image_cache_file, seconds) if config_cache and image_cache and graphCache: - image_data = image_cache - else: - buf = util.graph(df, title, stock.get('currency')) - #bytesio_to_file(buf, 'temp.png') - buf.seek(0) - image_data = buf - util.write_binary_cache(image_cache_file, image_data) - buf.seek(0) - return percent_dict, image_data + return percent_dict, image_cache + image = util.graph(df, title, stock.get('currency')) + util.write_binary_cache(image_cache_file, image) + return percent_dict, image def price_series(ticker): """Daily closes for a ticker via the same fetch/parse path as price_history(): (name, exchange, pandas Series indexed by datetime).""" + import pandas as pd data = fetch_chart_json(ticker, full=True) if isinstance(data, tuple): # fetch_chart_json returns (errorstring, None) on failure raise RuntimeError(data[0]) @@ -823,15 +525,22 @@ def price_series(ticker): name = util.transform_title(stock.get('longName') or stock.get('shortName') or ticker) return name, stock.get('fullExchangeName') or stock.get('exchangeName') or '', series -def prefetch_history(tickers, workers=8): - """Warm the price-history cache for many tickers at once, so the per-ticker lookups that follow are cache hits.""" - def warm(ticker): +def fetch_each(get, tickers, workers=8): + """get(ticker) for every ticker, `workers` at a time so their waits overlap: [its result, or the exception it raised], in order.""" + def one(ticker): try: - fetch_chart_json(ticker) + return get(ticker) except Exception as e: - print("prefetch failed for", ticker, e, file=sys.stderr) + return e with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: - list(executor.map(warm, tickers)) + return list(executor.map(one, tickers)) + +def prefetch_history(tickers, workers=8): + """Warm the price-history cache for many tickers at once, so the per-ticker lookups that follow are cache hits.""" + tickers = list(tickers) + for ticker, result in zip(tickers, fetch_each(fetch_chart_json, tickers, workers)): + if isinstance(result, Exception): + print("prefetch failed for", ticker, result, file=sys.stderr) def fetch_chart_json(ticker, days=3665, seconds=UNSET, full=False): """full=True requests Yahoo's entire history (cached separately) instead of the last `days`.""" @@ -843,8 +552,7 @@ def fetch_chart_json(ticker, days=3665, seconds=UNSET, full=False): cache = util.read_cache(cacheFile, seconds) if cache: return cache - cookie = getCookie() - crumb = getCrumb() + crumb = getCrumb() # renews the cookie too, when due start = '0' if full else str(int((now - datetime.timedelta(days=days)).timestamp())) end = str(int(now.timestamp())) interval = '1d' @@ -865,11 +573,13 @@ def fetch_chart_json(ticker, days=3665, seconds=UNSET, full=False): errorstring=f"{r.status_code} error for {ticker} at {url}" print(errorstring, file=sys.stderr) return errorstring, None + data = r.json() if config_cache: - util.json_write(cacheFile, r.json()) - return r.json() + util.json_write(cacheFile, data) + return data def chart_json_to_df(chart_json): + import pandas as pd result = chart_json['chart']['result'][0] timestamp = result['timestamp'] quote_data = result['indicators']['quote'][0] From 2607e899f2416102450cd35af0af0383906e1190 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:22:21 +0000 Subject: [PATCH 05/25] webhook, sharesight, charts, bot, jobs: one copy of each repeated step - webhook: bold, italic and strike read one MARKUP table; payload_wrapper picks the size limit once (same "over 2,000/4,096 bytes" lines, same sends: checked on 180 calls); sendPhoto asks takes_charts once; pleaseHold loses its non-Telegram branch, which nothing could reach (TypingIndicator only runs on Telegram) and whose own Telegram arm was unreachable twice over - sharesight: _request and _check replace four copies of the request / status / error-field block in get_token, get_portfolios, get_trades and get_performance (same messages, same order: get_trades still prints the trade count before the error check); find_portfolio is the case-insensitive portfolio lookup that reports.py writes out twice - charts: the price chart's link is util.quote_link (price.py's copy of that rule too) and its marker util.get_emoji; the .history default view parses its 10-year history once instead of twice (history_payload hands back the chart it fetched); the history highlight comes from PERIODS rather than a second table of the same labels; one price.lambda_handler call for the price-list buttons - bot: the Telegram branch reads the message through two locals; the misspelt userRealname, never read, is gone (userRealName is the first name, as before: the fallback it was meant to set never ran); the worker thread runs worker.process_request directly; git_version defaults to REPO_DIR - jobs: the job process runs in REPO_DIR; commands() builds the hidden-script set once Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 41 +++++++--------- lib/charts.py | 39 ++++++--------- lib/jobs.py | 6 +-- lib/sharesight.py | 91 +++++++++++++---------------------- lib/webhook.py | 120 ++++++++++++++++++++-------------------------- price.py | 16 ++----- 6 files changed, 123 insertions(+), 190 deletions(-) diff --git a/bot.py b/bot.py index 13bbc729..1cb2679f 100755 --- a/bot.py +++ b/bot.py @@ -56,8 +56,6 @@ def print_headers(): def reply(status, body=b''): start_response(status, [('Content-type', 'application/json')]) return [body] - user='' - userRealName='' print_headers() if config_print_headers else None # route on the path first, and turn strangers away before reading the body uri = environ.get('PATH_INFO', '') @@ -97,29 +95,26 @@ def reply(status, body=b''): return reply('200 OK') if "message" not in inbound: return reply('200 OK', b'Unsupported') - message_id = str(inbound["message"]["message_id"]) - chat_id = str(inbound["message"]["chat"]["id"]) - if "username" in inbound["message"]["from"]: - user = userRealname = '@' + inbound["message"]["from"]["username"] - if len(inbound["message"]["from"]["first_name"]): - userRealName = inbound["message"]["from"]["first_name"] + msg = inbound["message"] + sender = msg["from"] + message_id = str(msg["message_id"]) + chat_id = str(msg["chat"]["id"]) + if "username" in sender: + user = '@' + sender["username"] + userRealName = sender["first_name"] else: - user = userRealName = '@' + inbound["message"]["from"]["first_name"] - file_id = None - if "text" in inbound["message"]: - message = inbound["message"]["text"] + user = userRealName = '@' + sender["first_name"] + if "text" in msg: + message = msg["text"] print("[Telegram]:", user, message, file=sys.stderr) if debug else None #if message.startswith("Upcoming earnings for the next month") and user == telegram.getMe().get('username'): # telegram.pinChatMessage(message_id, chat_id) # todo need to return sendMessage to get message_id and chat_id - elif "photo" in inbound["message"]: - message = '' - if "caption" in inbound["message"]: - message = inbound["message"]["caption"] - photo = inbound["message"]["photo"][-1] - file_id = photo["file_id"] + elif "photo" in msg: + message = msg.get("caption", '') + file_id = msg["photo"][-1]["file_id"] print("[Telegram photo]:", user, file_id, message) - elif IGNORED_TELEGRAM_CONTENT & inbound["message"].keys(): + elif IGNORED_TELEGRAM_CONTENT & msg.keys(): return reply('200 OK') # stickers, GIFs, voice notes and the like: nothing to answer, nothing worth logging else: print(f"[{service}]: unhandled: 'message' without 'text/photo'", file=sys.stderr) @@ -163,12 +158,8 @@ def reply(status, body=b''): print(f"[{service}]: unhandled 'type'", file=sys.stderr) return reply('200 OK', b'

Unhandled

') - def runWorker(): - worker.process_request(service, chat_id, user, message, bot_name, userRealName, message_id) - # process in a background thread so we don't keep the requesting client waiting - t = threading.Thread(target=runWorker) - t.start() + threading.Thread(target=worker.process_request, args=(service, chat_id, user, message, bot_name, userRealName, message_id)).start() # Return an empty response to the client return reply('200 OK') @@ -179,7 +170,7 @@ def git_version(checkout=None): """Short hash of the checked-out commit, read straight from the .git directory: no git binary to run, and no "dubious ownership" refusal when the service user does not own the checkout. 'unknown' when it cannot be read.""" try: - git = Path(checkout or os.path.dirname(os.path.abspath(__file__))) / '.git' + git = Path(checkout or REPO_DIR) / '.git' if git.is_file(): # a linked worktree or submodule: .git is a file saying "gitdir: " git = (git.parent / git.read_text().split(':', 1)[1].strip()).resolve() common = (git / (git / 'commondir').read_text().strip()).resolve() if (git / 'commondir').is_file() else git # refs of a worktree diff --git a/lib/charts.py b/lib/charts.py index c6c85bd3..95c21944 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -13,7 +13,6 @@ PERIODS = {'d': ('1D', 1), 'w': ('7D', 7), 'm': ('1M', 30), 'q': ('3M', 90), 'y': ('1Y', 365), 'x': ('Max', None)} BUTTONS = {'h': 'wmqyx', 'p': 'wmqyx', 'c': 'wmqyx', 'f': 'wmqyx', 'l': 'dwmqyx'} # chart kind -> buttons offered (h=history, p=price chart, c=compare, f=performance, l=price list) MAX_DAYS = 3650 # what Max means for the Sharesight-based reports -HIGHLIGHT = {'w': '7D', 'm': '1M', 'q': '3M', 'y': '1Y', 'x': 'Max'} # history table row for each button MAX_CALLBACK_BYTES = 64 # Telegram limit on callback_data REF_FILE = 'finbot_chart_buttons.json' @@ -100,14 +99,15 @@ def keyboard(kind, tickers, active=None): row.append({'text': ('● ' + label) if key == active else label, 'callback_data': f"c|{kind}|{key}|{ref}"}) return {'inline_keyboard': [row]} -def history_payload(ticker, service, highlight=None): - """Caption lines for .history: performance table, with the selected interval in bold. Returns (lines, percent table).""" +def history_payload(ticker, service, highlight=None, graph=False): + """Caption lines for .history: performance table, with the selected interval in bold. Returns (lines, percent table, image): + the full history's chart with graph=True (if the graph preference is on), else None.""" market_data = yahoo.fetch_detail(ticker, 600) title = market_data.get(ticker, {}).get('profile_title', '') ticker_link = util.finance_link(ticker, market_data.get(ticker, {}).get('profile_exchange', ''), service, days=1825, brief=False) if ticker not in market_data or 'percent_change' not in market_data[ticker]: - return [f".history: no data found for ticker {ticker}"], None - table, _ = yahoo.price_history(ticker, graph=False) + return [f".history: no data found for ticker {ticker}"], None, None + table, image = yahoo.price_history(ticker) if graph else yahoo.price_history(ticker, graph=False) if isinstance(table, str): raise RuntimeError(table) lines = [webhook.bold(f"{title} ({ticker_link}) performance history", service)] @@ -118,19 +118,20 @@ def history_payload(ticker, service, highlight=None): if interval == highlight: label = '● ' + label lines.append(f"{util.get_emoji(percent)} {label} {util.signed_percent(percent)}") - return lines, table + return lines, table, image def build(kind, tickers, period, service='telegram'): """(caption, image) for a chart kind at a button period; period None = the command's default view.""" days = PERIODS[period][1] if period else None if kind == 'h': ticker = tickers[0] - lines, table = history_payload(ticker, service, HIGHLIGHT.get(period or 'x')) # default view is the full history + lines, table, image = history_payload(ticker, service, PERIODS[period or 'x'][0], graph=not days) # default view is the full history if table is None: raise RuntimeError(lines[0]) - percent, image = yahoo.price_history(ticker, days) if days else yahoo.price_history(ticker) - if isinstance(percent, str): - raise RuntimeError(percent) + if days: + percent, image = yahoo.price_history(ticker, days) + if isinstance(percent, str): + raise RuntimeError(percent) return '\n'.join(lines), image if kind == 'p': ticker = tickers[0] @@ -141,16 +142,8 @@ def build(kind, tickers, period, service='telegram'): if isinstance(percent, str): raise RuntimeError(percent) percent = float(percent.get(days, percent.get('Max', next(iter(percent.values()), 0)))) - title = market_data[ticker]['profile_title'] - exchange = market_data[ticker]['profile_exchange'] - if exchange in ('Taipei Exchange', 'CCC') or ticker.startswith('^'): - link = util.yahoo_link(ticker, service) - elif config_hyperlinkProvider == 'google': - link = util.gfinance_link(ticker, exchange, service, days=days) - else: - link = util.yahoo_link(ticker, service) - emoji = '🔻' if percent < 0 else '🔼' if percent > 0 else '▪️' - return f"{emoji} {title} ({link}) {util.signed_percent(round(percent))}", image + link = util.quote_link(ticker, market_data[ticker]['profile_exchange'], service, days=days) + return f"{util.get_emoji(percent)} {market_data[ticker]['profile_title']} ({link}) {util.signed_percent(round(percent))}", image if kind == 'c': args = list(tickers) + ([f'{days}d'] if days else ['max']) caption, image = reports.prepare_compare(service, args) @@ -166,10 +159,8 @@ def build(kind, tickers, period, service='telegram'): if kind == 'l': # price list: tickers = [threshold, top] import price threshold, top = float(tickers[0]), int(tickers[1]) - if period == 'd': # today's moves: the command's default view, not a 1-day lookback - payload, image = price.lambda_handler(threshold=threshold, service=service, interactive=True, interday=True, top=top or None, return_result=True) - else: - payload, image = price.lambda_handler(threshold=threshold, service=service, interactive=True, days=days or MAX_DAYS, top=top or None, return_result=True) + span = {'interday': True} if period == 'd' else {'days': days or MAX_DAYS} # 'd' is today's moves: the command's default view, not a 1-day lookback + payload, image = price.lambda_handler(threshold=threshold, service=service, interactive=True, top=top or None, return_result=True, **span) if not image: raise RuntimeError(payload[0] if payload else 'no price data') return '\n'.join(payload), image diff --git a/lib/jobs.py b/lib/jobs.py index f747ae04..7b67dbf7 100644 --- a/lib/jobs.py +++ b/lib/jobs.py @@ -31,7 +31,6 @@ 'rating': rating, 'reminder': reminder, 'shorts': shorts, 'trades': trades} current = None # the running Scheduler, set by bot.py; the chat commands use it HIDDEN = ('cleanup', 'podcasts') # scheduled, but not offered in the chat list -ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) STATE_FILE = 'finbot_jobs.json' # last fire time of every job, so a restart neither repeats a run nor loses one JOB_TIMEOUT = 5 * 60 # seconds a job may run before it is abandoned and reported IGNORED_VARIABLES = ('MAILTO', 'SHELL', 'PATH', 'HOME', 'LOGNAME') @@ -48,7 +47,7 @@ def run_child(label, module, kwargs): """Run a command in its own process (python -m lib.jobrun) and return its exit status. The child inherits the environment, user and output of the bot, so it behaves like the old cron job. A timeout or any other interruption stops the child.""" env = dict(os.environ, **config.child_environment()) # the preferences this process runs on, not whatever the file says now - proc = subprocess.Popen([sys.executable, '-m', 'lib.jobrun', module.__name__, json.dumps(kwargs)], cwd=ROOT, env=env) + proc = subprocess.Popen([sys.executable, '-m', 'lib.jobrun', module.__name__, json.dumps(kwargs)], cwd=REPO_DIR, env=env) try: return proc.wait() except BaseException: @@ -382,9 +381,10 @@ def commands(self): """Every offered scheduled command once, sorted by name with numbers in numeric order: [(label, module, kwargs)]. The time, comments, redirects and the other commands of a line are not part of it; HIDDEN scripts are left out.""" unique = {} + hidden = {SCRIPTS[name] for name in HIDDEN} for job in self.jobs: for command in job.commands: - if command[1] not in [SCRIPTS[name] for name in HIDDEN]: + if command[1] not in hidden: unique.setdefault(command[0], command) natural = lambda label: [int(part) if part.isdigit() else part for part in re.split(r'(\d+)', label.lower())] return [unique[label] for label in sorted(unique, key=natural)] diff --git a/lib/sharesight.py b/lib/sharesight.py index b330185f..1bcfbf50 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -17,6 +17,23 @@ def _post(url, **kwargs): with _request_slots: return requests.post(url, **kwargs) +def _request(send, url, what, **kwargs): + """Send a request to Sharesight and return the JSON answer. A request that fails raises RuntimeError naming `what`.""" + try: + r = send(url, timeout=config_http_timeout, **kwargs) + except Exception as e: + print("Error", str(e), url, file=sys.stderr) + raise RuntimeError(f"Sharesight {what} request failed: {e}") + if r.status_code != 200: + print(r.status_code, "error", url, file=sys.stderr) + return r.json() + +def _check(data, what): + """Raise RuntimeError with Sharesight's error, when it answered with one.""" + if 'error' in data: + print("Sharesight error:", data['error_code'], data['error'], file=sys.stderr) + raise RuntimeError(f"Sharesight {what} error: {data['error']}") + class BearerAuth(requests.auth.AuthBase): def __init__(self, token): self.token = token @@ -38,18 +55,8 @@ def get_token(): print("cache miss:", cacheFile, file=sys.stderr) if debug else None print("Fetching Sharesight auth token") - url = "https://api.sharesight.com/oauth2/token" - try: - r = _post(url, data=config_sharesight_auth, timeout=config_http_timeout) - except Exception as e: - print("Error", str(e), url, file=sys.stderr) - raise RuntimeError(f"Sharesight auth request failed: {e}") - if r.status_code != 200: - print(r.status_code, "error", url, file=sys.stderr) - data = r.json() - if 'error' in data: - print("Sharesight error:", data['error_code'], data['error'], file=sys.stderr) - raise RuntimeError(f"Sharesight auth error: {data['error']}") + data = _request(_post, "https://api.sharesight.com/oauth2/token", 'auth', data=config_sharesight_auth) + _check(data, 'auth') if config_cache and 'access_token' in data: util.json_write(cacheFile, data) print("Got Sharesight token:", data['access_token'], file=sys.stderr) if debug else None @@ -63,29 +70,14 @@ def performance_cache_file(portfolio_id, days): def get_portfolios(refresh=False): """refresh=True skips the cache read and fetches again (the cache is rewritten).""" portfolio_dict = {} - cache = None - cache_file = PORTFOLIOS_CACHE - if config_cache and not refresh: - cache = util.read_cache(cache_file, config_cache_seconds) - if cache: - data = cache - if not cache or not config_cache: + data = util.read_cache(PORTFOLIOS_CACHE, config_cache_seconds) if config_cache and not refresh else None + if not data: token = get_token() print("Fetching Sharesight portfolios") - url = "https://api.sharesight.com/api/v3/portfolios" - try: - r = _get(url, headers={'Content-type': 'application/json'}, auth=BearerAuth(token), timeout=config_http_timeout) - except Exception as e: - print("Error", str(e), url, file=sys.stderr) - raise RuntimeError(f"Sharesight portfolios request failed: {e}") - if r.status_code != 200: - print(r.status_code, "error", url, file=sys.stderr) - data = r.json() - if 'error' in data: - print("Sharesight error:", data['error_code'], data['error'], file=sys.stderr) - raise RuntimeError(f"Sharesight portfolios error: {data['error']}") + data = _request(_get, "https://api.sharesight.com/api/v3/portfolios", 'portfolios', headers={'Content-type': 'application/json'}, auth=BearerAuth(token)) + _check(data, 'portfolios') if config_cache and 'portfolios' in data: - util.json_write(cache_file, data) + util.json_write(PORTFOLIOS_CACHE, data) for portfolio in data['portfolios']: if str(portfolio['id']) in config_exclude_portfolios: print(portfolio['id'], "(" + portfolio['name'] + ") in exclusion list. Skipping.") @@ -99,6 +91,10 @@ def get_portfolios(refresh=False): print(portfolio_dict) return portfolio_dict +def find_portfolio(portfolios, name): + """(name as Sharesight spells it, id) of the portfolio called `name`, in any case, or None.""" + return {k.lower(): (k, v) for k, v in portfolios.items()}.get(name.lower()) + def get_trades(portfolio_name, portfolio_id, days=UNSET): if days is UNSET: days = config_past_days @@ -113,18 +109,9 @@ def get_trades(portfolio_name, portfolio_id, days=UNSET): print("Fetching Sharesight trades for", portfolio_name, end=": ") url = 'https://api.sharesight.com/api/v2/portfolios/' url = url + str(portfolio_id) + '/trades.json' + '?start_date=' + start_date - try: - r = _get(url, auth=BearerAuth(token), timeout=config_http_timeout) - except Exception as e: - print("Error", str(e), url, file=sys.stderr) - raise RuntimeError(f"Sharesight trades request failed: {e}") - if r.status_code != 200: - print(r.status_code, "error", url, file=sys.stderr) - data = r.json() + data = _request(_get, url, 'trades', auth=BearerAuth(token)) print(len(data['trades'])) - if 'error' in data: - print("Sharesight error:", data['error_code'], data['error'], file=sys.stderr) - raise RuntimeError(f"Sharesight trades error: {data['error']}") + _check(data, 'trades') for trade in data['trades']: trade['portfolio'] = portfolio_name # inject custom field if config_cache and 'trades' in data: @@ -162,8 +149,7 @@ def get_holdings_wrapper(): return None for portfolio_name, portfolio_id in portfolios.items(): tickers.update(get_holdings(portfolio_name, portfolio_id)) - tickers = sorted(set(tickers)) - return tickers + return sorted(tickers) def get_performance(portfolio_id, days, config_cache_seconds=UNSET, refresh=False): if config_cache_seconds is UNSET: @@ -179,22 +165,13 @@ def get_performance(portfolio_id, days, config_cache_seconds=UNSET, refresh=Fals endpoint = 'https://api.sharesight.com/api/v3/portfolios/' url = endpoint + str(portfolio_id) + '/performance?grouping=ungrouped&start_date=' + start_date for attempt in range(5): # Sharesight allows only a few requests in flight at once; back off and retry when it says so - try: - r = _get(url, auth=BearerAuth(token), timeout=config_http_timeout) - except Exception as e: - print("Error", str(e), url, file=sys.stderr) - raise RuntimeError(f"Sharesight performance request failed: {e}") - if r.status_code != 200: - print(r.status_code, "error", url, file=sys.stderr) - data = r.json() + data = _request(_get, url, 'performance', auth=BearerAuth(token)) if 'error' in data and 'parallel' in str(data['error']).lower() and attempt < 4: print("Sharesight busy:", data['error'], "- retrying", file=sys.stderr) time.sleep(1 + attempt) continue break - if 'error' in data: - print("Sharesight error:", data['error_code'], data['error'], file=sys.stderr) - raise RuntimeError(f"Sharesight performance error: {data['error']}") + _check(data, 'performance') if config_cache and 'report' in data: util.json_write(cache_file, data) return data @@ -253,7 +230,7 @@ def keep_warm(min_wait=5, retry=300, idle=60, stop=None): wait = idle else: try: - fetched, portfolios = warm() + _, portfolios = warm() wait = max(min_wait, next_due(portfolios=portfolios)) except Exception as e: print("Sharesight cache refresh failed:", e, file=sys.stderr) diff --git a/lib/webhook.py b/lib/webhook.py index e1c619e0..c0ec9233 100644 --- a/lib/webhook.py +++ b/lib/webhook.py @@ -119,57 +119,46 @@ def payload_wrapper(service, url, payload, chat_id=None, message_id=None, reply_ """Returns a list with the response of each message sent (several if the payload was chunked).""" if not payload: print(service + ": Nothing to send") # informational - else: - payload = [str(line) for line in payload] - payload_string = '\n'.join(payload) - print("Preparing outbound to", service, str(len(payload_string)), "bytes") - print("Payload: " + payload_string) if debug else None - def chunkLooper(): - chunks = list(batched(payload, config_chunk_maxlines)) - results = [] - for idx, chunk in enumerate(chunks): - idx > 0 and time.sleep(0.5) - payload_chunk = '\n'.join(chunk) - last = idx == len(chunks) - 1 - results.append(write(service, url, payload_chunk, chat_id, reply_markup=reply_markup if last else None)) - return results - if service == 'discord' and len(payload_string) > 2000: - print(service, "payload is over 2,000 bytes. Splitting.") - return chunkLooper() - elif service != 'discord' and len(payload_string) > 4096: - print(service, "payload is over 4,096 bytes. Splitting.") - return chunkLooper() - else: - return [write(service, url, payload_string, chat_id, message_id, reply_markup)] + return None + payload = [str(line) for line in payload] + payload_string = '\n'.join(payload) + print("Preparing outbound to", service, str(len(payload_string)), "bytes") + print("Payload: " + payload_string) if debug else None + limit = 2000 if service == 'discord' else 4096 + if len(payload_string) <= limit: + return [write(service, url, payload_string, chat_id, message_id, reply_markup)] + print(service, f"payload is over {limit:,} bytes. Splitting.") + chunks = list(batched(payload, config_chunk_maxlines)) + results = [] + for idx, chunk in enumerate(chunks): + if idx: + time.sleep(0.5) + last = idx == len(chunks) - 1 + results.append(write(service, url, '\n'.join(chunk), chat_id, reply_markup=reply_markup if last else None)) + return results + +MARKUP = { # how each service marks text up: style -> service -> (before, after) + 'bold': {'telegram': ('', ''), 'slack': ('*', '*'), 'discord': ('**', '**')}, + 'italic': {'telegram': ('', ''), 'slack': ('_', '_'), 'discord': ('_', '_')}, + 'strike': {'telegram': ('', ''), 'slack': ('~', '~'), 'discord': ('~~', '~~')}, +} + +def _styled(style, message, service): + if service not in MARKUP[style]: + return message + before, after = MARKUP[style][service] + return before + message + after def bold(message, service): - if service == 'telegram': - message = '' + message + '' - elif service == 'slack': - message = '*' + message + '*' - elif service == 'discord': - message = '**' + message + '**' - return message + return _styled('bold', message, service) def italic(message, service): - if service == 'telegram': - message = '' + message + '' - elif service == 'slack': - message = '_' + message + '_' - elif service == 'discord': - message = '_' + message + '_' - return message + return _styled('italic', message, service) italics = italic def strike(message, service): - if service == 'telegram': - message = '' + message + '' - elif service == 'slack': - message = '~' + message + '~' - elif service == 'discord': - message = '~~' + message + '~~' - return message + return _styled('strike', message, service) def fit_caption(caption, limit=1024): @@ -271,15 +260,16 @@ def sendPhoto(chat_id, image_data, caption, service, message_id=None, reply_mark (takes_charts), or the service refuses it, the report is sent as text instead, without the chart's buttons, so it does not vanish with its chart; but not after a read timeout on the request that posts it, when the chart may have arrived after all.""" output = None + chartable = takes_charts(service, reply=bool(chat_id)) if service == 'telegram': output = telegram_photo(chat_id, image_data, caption, message_id, reply_markup) - elif takes_charts(service, reply=bool(chat_id)): + elif chartable: output = slack_photo(chat_id, image_data, caption, message_id) if output is MAYBE_SENT: return None if output: return output - if takes_charts(service, reply=bool(chat_id)): + if chartable: print("chart not sent, sending the report as text", file=sys.stderr) return payload_wrapper(service, chat_url(service, chat_id) if chat_id else webhooks.get(service), caption.split('\n'), chat_id, message_id) @@ -322,28 +312,22 @@ def editMessageMedia(chat_id, message_id, image_data, caption, reply_markup=None return output def pleaseHold(action, service, chat_id): - """typing notify. I am run in a thread and repeated every 5 seconds""" - if service == 'telegram': - url = webhooks['telegram'] + "sendChatAction?chat_id=" + str(chat_id) - headers = {} - data = { - 'chat_id': chat_id, - 'action': action - } - try: - r = requests.post(url, data=data, headers=headers, timeout=config_http_timeout) - except Exception as e: - print("Failure executing request:", url, data, str(e)) - return None - if r.status_code == 200: - print(r.status_code, "OK typing indicator to", service, file=sys.stderr) - else: - print(r.status_code, "error typing indicator to", service, file=sys.stderr) + """Telegram's typing notice ("typing…" for a few seconds). worker.TypingIndicator repeats it every 7 seconds while a command + works, on Telegram only: the other services have no such notice.""" + if service != 'telegram': return None + url = webhooks['telegram'] + "sendChatAction?chat_id=" + str(chat_id) + data = { + 'chat_id': chat_id, + 'action': action + } + try: + r = requests.post(url, data=data, timeout=config_http_timeout) + except Exception as e: + print("Failure executing request:", url, data, str(e)) + return None + if r.status_code == 200: + print(r.status_code, "OK typing indicator to", service, file=sys.stderr) else: - payload_string = "this might take a moment" - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + str(chat_id) - write(service, url, payload_string, chat_id, message_id=None) + print(r.status_code, "error typing indicator to", service, file=sys.stderr) + return None diff --git a/price.py b/price.py index 64b90d4c..85c49976 100755 --- a/price.py +++ b/price.py @@ -79,20 +79,10 @@ def prepare_price_payload(service, market_data, threshold, _rows_out=None): percent = market_data[ticker]['percent_change'] title = market_data[ticker]['profile_title'] percent = float(percent) - if percent < 0: - emoji = "🔻" - elif percent > 0: - emoji = '🔼' - else: - emoji = "▪️" - exchange = exchange_human = market_data[ticker]['profile_exchange'] + emoji = util.get_emoji(percent) + exchange = market_data[ticker]['profile_exchange'] exchange_human = util.exchange_human(exchange) - if exchange in ('Taipei Exchange', 'CCC') or ticker.startswith('^'): - ticker_link = util.yahoo_link(ticker, service) - elif config_hyperlinkProvider == 'google': - ticker_link = util.gfinance_link(ticker, exchange, service, days=days) - else: - ticker_link = util.yahoo_link(ticker, service) + ticker_link = util.quote_link(ticker, exchange, service, days=days) if not interactive and not payload and config_demote_leveraged and '2x' in title.lower(): if abs(percent) >= threshold * 1.3: payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) From 670910f4d0d82815b99277a0bb2ce10dd318ecf4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:25:54 +0000 Subject: [PATCH 06/25] reports: Yahoo details and compare histories fetched at the same time; one portfolio lookup and holding line - .buy, .sell and the .peg lists fetched each ticker's Yahoo details one after another: tens of seconds on a cold cache for a hundred tickers, outlasting the typing indicator. They now go through yahoo.fetch_each, 8 at a time, and each list keeps its own error handling: ratings still skip a ticker that raised TypeError and fail on anything else, the PEG lists still print "value not found" for a KeyError, and the first failure in ticker order is the one reported - .compare downloads its 2 to 6 full histories at the same time (the first failure is still the one reported) - the .SYMBOL profile asks price_history for the percentages only: it rendered (or read) a chart it then dropped, up to 0.15 s of CPU on the bot's event loop for each cold ticker - simplywallst.get_url ran twice per profile (two identical lines); now once - holdings and .pie find the portfolio with sharesight.find_portfolio; holding_line formats a Sharesight holding for the holdings list (and .who, next); one module-level last_col replaces two nested copies; the buy and sell branches of the ratings share one condition; the four delta lines come from a loop - prepare_rating_payload is indented one tab, like every other function Checked against the previous module on 1,578 calls with canned data (every report, service, action and argument form, with tickers that raise TypeError, KeyError, ValueError or RuntimeError): the same payloads, chart rows and exceptions. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/reports.py | 193 +++++++++++++++++++++---------------------------- 1 file changed, 82 insertions(+), 111 deletions(-) diff --git a/lib/reports.py b/lib/reports.py index 3c0e9b81..d96753ee 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -12,6 +12,18 @@ from lib import simplywallst from itertools import groupby +def last_col(line): + """Sort key: the number a report line ends with; a line without one sorts first.""" + try: + return float(line.split()[-1]) + except ValueError: + return float('-inf') + +def holding_line(holding, service, text=None): + """'🇦🇺 Title (CODE)' for a Sharesight holding, CODE (or `text`) linking to the holding on Sharesight.""" + link = util.link(f"https://portfolio.sharesight.com/holdings/{holding['holding_id']}/dashboard", holding['code'] if text is None else text, service) + return f"{util.flag_from_market(holding['market_code'])} {util.transform_title(holding['name'])} ({link})" + def doDelta(inputList): deltaString = '' inputListFixed = inputList.copy() @@ -149,9 +161,9 @@ def prepare_compare(service, args, with_inputs=False): except ValueError: pass tickers = list(dict.fromkeys(util.transform_to_yahoo(a.upper()) for a in args)) # de-duplicated, order kept - if not 2 <= len(tickers) <= 6: - return (usage, None, tickers, period_days) if with_inputs else (usage, None) result = lambda caption, image: (caption, image, tickers, period_days) if with_inputs else (caption, image) + if not 2 <= len(tickers) <= 6: + return result(usage, None) spec = period_days or ('max' if full_history else 'default') cache_key = hashlib.sha1(f"{service}|{','.join(tickers)}|{spec}".encode()).hexdigest()[:16] image_cache_file, caption_cache_file = f"finbot_compare_{cache_key}.png", f"finbot_compare_{cache_key}.json" @@ -161,11 +173,10 @@ def prepare_compare(service, args, with_inputs=False): if cached_image and cached_caption: return result(cached_caption, cached_image) names, exchanges, closes = {}, {}, {} - for ticker in tickers: - name, exchange, series = yahoo.price_series(ticker) - names[ticker] = name - exchanges[ticker] = exchange - closes[ticker] = series + for ticker, history in zip(tickers, yahoo.fetch_each(yahoo.price_series, tickers)): # downloaded at the same time + if isinstance(history, Exception): + raise history + names[ticker], exchanges[ticker], closes[ticker] = history limiting = max(closes, key=lambda t: closes[t].index[0]) # ticker with the shortest history common_start = closes[limiting].index[0] start = common_start @@ -289,47 +300,27 @@ def prepare_help(service, botName): return payload def prepare_holdings_payload(portfolioName, service, user): - payload = [] portfolios = sharesight.get_portfolios() - portfoliosLower = {k.lower():v for k,v in portfolios.items()} - if portfolioName: - if portfolioName.lower() in portfoliosLower: - portfolioId = portfoliosLower[portfolioName.lower()] - holdings = sharesight.get_holdings_new(portfolioName, portfolioId) - for k,v in holdings.items(): - ticker = v['code'] - title = util.transform_title(v['name']) - exchange = v['market_code'] - #ticker_link = util.finance_link(ticker, exchange, service) - holding_id = v['holding_id'] - url = f"https://portfolio.sharesight.com/holdings/{holding_id}/dashboard" - ticker_link = util.link(url, ticker, service) - flag = util.flag_from_market(exchange) - payload.append(f"{flag} {title} ({ticker_link})") - portfoliosReverseLookup = {v:k for k,v in portfolios.items()} - payload.sort() - if payload: - count = len(holdings) - portfolio_name = portfoliosReverseLookup[portfolioId] - payload.insert(0, webhook.bold(f"Holdings for {portfolio_name} ({count})", service)) - else: - payload = [ f"{user} {portfolioName} portfolio not found. I only know about:" ] - for item in portfolios: - payload.append( item ) - else: - payload = [ f".holdings: Please try again specifying a portfolio:" ] - for item in portfolios: - payload.append( item ) + if not portfolioName: + return [".holdings: Please try again specifying a portfolio:"] + list(portfolios) + found = sharesight.find_portfolio(portfolios, portfolioName) + if not found: + return [f"{user} {portfolioName} portfolio not found. I only know about:"] + list(portfolios) + name, portfolio_id = found + holdings = sharesight.get_holdings_new(portfolioName, portfolio_id) + payload = sorted(holding_line(holding, service) for holding in holdings.values()) + if payload: + payload.insert(0, webhook.bold(f"Holdings for {name} ({len(holdings)})", service)) return payload def prepare_pie(portfolio_name, by, service, max_slices=8): """Portfolio value by industry (or by='sector'). Returns (payload, chart); chart is None unless there is something to draw.""" portfolios = sharesight.get_portfolios() - lookup = {k.lower(): (k, v) for k, v in portfolios.items()} - if not portfolio_name or portfolio_name.lower() not in lookup: + found = sharesight.find_portfolio(portfolios, portfolio_name) if portfolio_name else None + if not found: payload = [".pie: please try again specifying a portfolio:" if not portfolio_name else f"{portfolio_name} portfolio not found. I only know about:"] return payload + list(portfolios), None - name, portfolio_id = lookup[portfolio_name.lower()] + name, portfolio_id = found key = 'sector_classification_name' if by == 'sector' else 'industry_classification_name' totals = {} for holding in sharesight.get_holdings_new(name, portfolio_id).values(): @@ -356,11 +347,6 @@ def prepare_pie(portfolio_name, by, service, max_slices=8): return payload, {'pie': True, 'rows': rows, 'title': title, 'subtitle': subtitle} def prepare_marketcap_payload(service, action='top', length=15): - def last_col(e): - try: - return float(e.split()[-1]) - except ValueError: - return float('-inf') # or float('inf'), depending on where you want missing values to sort payload_staging = [] tickers = util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) @@ -389,58 +375,55 @@ def last_col(e): return payload def prepare_rating_payload(service, action, length=15, with_chart=False): - def score_col(e): - return (float(e.split()[-3]), int(e.split()[-2].removeprefix('('))) - payload = [] - chart_rows = [] # (ticker, rating, analysts), kept in step with payload - tickers = util.get_holdings_and_watchlist() - market_data = {} - for ticker in tickers: - try: - market_data = market_data | yahoo.fetch_detail(ticker) - except TypeError: - pass - for ticker in market_data: - if 'recommend' in market_data[ticker]: - recommend = market_data[ticker]['recommend'].replace('_', ' ') - recommend_index = round(market_data[ticker]['recommend_index'], 2) - recommend_analysts = market_data[ticker]['recommend_analysts'] - if recommend_analysts > 1: - profile_title = market_data[ticker]['profile_title'] - ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) - flag = util.flag_from_ticker(ticker) - if action == 'buy' and 'buy' in recommend: - payload.append(f"{flag} {profile_title} ({ticker_link}) {recommend_index} ({recommend_analysts} analysts)") - chart_rows.append((ticker.split('.')[0], recommend_index, recommend_analysts)) - elif action == 'sell' and (recommend == 'sell' or recommend == 'underperform'): - payload.append(f"{flag} {profile_title} ({ticker_link}) {recommend_index} ({recommend_analysts} analysts)") - chart_rows.append((ticker.split('.')[0], recommend_index, recommend_analysts)) - payload.sort(key=score_col) - chart_rows.sort(key=lambda row: (row[1], row[2])) # same order as score_col - payload = payload[:length] - chart = None - if payload: - message = f"Top {length} analyst {action} ratings for tracked stocks" - payload.insert(0, f"{webhook.bold(message, service)}") - shown = [(t, r) for t, r, _ in chart_rows[:length]] - low, high = min(r for _, r in shown), max(r for _, r in shown) - spread = (high - low) or 0.2 - # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) - chart = {'rows': shown, 'title': message, 'subtitle': 'Mean analyst rating: 1 = strong buy, 5 = strong sell (axis zoomed)', - 'value_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} - return (payload, chart) if with_chart else payload + def score_col(e): + return (float(e.split()[-3]), int(e.split()[-2].removeprefix('('))) + payload = [] + chart_rows = [] # (ticker, rating, analysts), kept in step with payload + tickers = util.get_holdings_and_watchlist() + market_data = {} + for detail in yahoo.fetch_each(yahoo.fetch_detail, tickers): # fetched at the same time + if isinstance(detail, TypeError): + continue + if isinstance(detail, Exception): + raise detail + market_data |= detail + for ticker in market_data: + if 'recommend' in market_data[ticker]: + recommend = market_data[ticker]['recommend'].replace('_', ' ') + recommend_index = round(market_data[ticker]['recommend_index'], 2) + recommend_analysts = market_data[ticker]['recommend_analysts'] + if recommend_analysts > 1: + profile_title = market_data[ticker]['profile_title'] + ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) + flag = util.flag_from_ticker(ticker) + if (action == 'buy' and 'buy' in recommend) or (action == 'sell' and recommend in ('sell', 'underperform')): + payload.append(f"{flag} {profile_title} ({ticker_link}) {recommend_index} ({recommend_analysts} analysts)") + chart_rows.append((ticker.split('.')[0], recommend_index, recommend_analysts)) + payload.sort(key=score_col) + chart_rows.sort(key=lambda row: (row[1], row[2])) # same order as score_col + payload = payload[:length] + chart = None + if payload: + message = f"Top {length} analyst {action} ratings for tracked stocks" + payload.insert(0, f"{webhook.bold(message, service)}") + shown = [(t, r) for t, r, _ in chart_rows[:length]] + low, high = min(r for _, r in shown), max(r for _, r in shown) + spread = (high - low) or 0.2 + # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) + chart = {'rows': shown, 'title': message, 'subtitle': 'Mean analyst rating: 1 = strong buy, 5 = strong sell (axis zoomed)', + 'value_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} + return (payload, chart) if with_chart else payload def prepare_value_payload(service, action='pe', ticker_select=None, length=15, with_chart=False): - def last_col(e): - try: - return float(e.split()[-1]) - except ValueError: - return float('-inf') - payload = [] chart_rows = [] # (ticker, ratio) tickers = [ticker_select] if ticker_select else util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) + details = dict(zip(market_data, yahoo.fetch_each(yahoo.fetch_detail, market_data))) if action in ('peg', 'bottom peg', 'negative peg') else {} + def detail(ticker): # fetched above, all at once; a failure is raised here, where the one-at-a-time fetch raised it + if isinstance(details[ticker], Exception): + raise details[ticker] + return details[ticker] for ticker in market_data: trailing_ratio = None forward_ratio = None @@ -464,13 +447,13 @@ def last_col(e): continue ratio = forward_ratio elif action in ('peg', 'bottom peg'): - market_data = market_data | yahoo.fetch_detail(ticker) + market_data = market_data | detail(ticker) peg_ratio = market_data.get(ticker).get('price_to_earnings_peg') if not ticker_select and peg_ratio is not None and peg_ratio < 0: continue ratio = peg_ratio elif action == 'negative peg': - market_data = market_data | yahoo.fetch_detail(ticker) + market_data = market_data | detail(ticker) peg_ratio = market_data.get(ticker).get('price_to_earnings_peg') if not ticker_select and (peg_ratio is None or peg_ratio >= 0): continue @@ -544,8 +527,6 @@ def prepare_profile_payload(service, user, ticker): swsURL = simplywallst.get_url(ticker, profile_title, exchange) swsLink = util.link(swsURL, 'simplywall.st', service) ticker_link = util.finance_link(ticker, exchange, service, brief=False) - swsURL = simplywallst.get_url(ticker, profile_title, exchange) - swsLink = util.link(swsURL, 'simplywall.st', service) macrotrendsURL = 'https://www.google.com/search?q=site:macrotrends.net+' + profile_title + '+PE Ratio+' + ticker.split('.')[0] + '&btnI' macrotrendsLink = util.link(macrotrendsURL, 'macrotrends', service) gfinanceLink = util.gfinance_link(ticker, exchange, service, brief=True, text='google') @@ -579,9 +560,8 @@ def prepare_profile_payload(service, user, ticker): profile_country = market_data[ticker]['profile_country'] location.append(profile_country) if 'profile_website' in market_data[ticker]: - website = website_text = market_data[ticker]['profile_website'] - website_text = util.strip_url(website) - website = util.link(website, website_text, service) + website = market_data[ticker]['profile_website'] + website = util.link(website, util.strip_url(website), service) if payload: payload.append("") @@ -666,18 +646,9 @@ def prepare_profile_payload(service, user, ticker): payload.append("") if 'earningsQ' in market_data[ticker]: - revenueQs = doDelta(market_data[ticker]['revenueQ']) - earningsQs = doDelta(market_data[ticker]['earningsQ']) - revenueYs = doDelta(market_data[ticker]['revenueY']) - earningsYs = doDelta(market_data[ticker]['earningsY']) - if revenueQs: - payload.append(f"{revenueQs} quarterly revenue delta") - if earningsQs: - payload.append(f"{earningsQs} quarterly earnings delta") - if revenueYs: - payload.append(f"{revenueYs} annual revenue delta") - if earningsYs: - payload.append(f"{earningsYs} annual earnings delta") + deltas = [(doDelta(market_data[ticker][key]), label) for key, label in (('revenueQ', 'quarterly revenue'), ('earningsQ', 'quarterly earnings'), + ('revenueY', 'annual revenue'), ('earningsY', 'annual earnings'))] + payload += [f"{delta}\t{label} delta" for delta, label in deltas] # doDelta always gives at least three symbols if payload: payload.append("") @@ -777,7 +748,7 @@ def prepare_profile_payload(service, user, ticker): if payload: payload.append("") - price_history, graph = yahoo.price_history(ticker) + price_history, _ = yahoo.price_history(ticker, graph=False) # the percentages only: the profile has no chart if isinstance(price_history, str): # no chart data: the profile goes out without these lines price_history = {} for interval in ('5Y', '1Y', '1M', '1D'): From 92af627c2c5d0703431e21bd50de8467ad0b7a8c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 07:34:06 +0000 Subject: [PATCH 07/25] worker: each command matched where it is handled; typing runs as a with block; no print-and-reraise wrappers - the 28 command patterns were all matched on every message up front, 100-odd lines above the if/elif chain that decides which one wins. Each pattern now sits in its own branch condition, in the same order (checked: identical patterns, identical order), and matching stops at the first hit. .buy and .sell, two identical branches, are one - working(service, chat_id, ...) starts the typing indicator, sends a command's "Fetching..." notice where there is no typing indicator, and stops the indicator however the block ends. It replaces 23 hand-written start/stop pairs and the per-request registry that stopped the ones a failed command left running: the error line still goes out after the typing has stopped - the 22 `except Exception as e: print(e); raise` wrappers are gone: process_request's report_error already logs the error line and the traceback (stderr loses only the duplicate bare message) - period_or_ticker, percent_or_ticker and period_and_portfolio parse the arguments that nine branches parsed by hand; the reply URL is webhook.chat_url; deliver() has one chart branch - .beta fetches its tickers' details at the same time (yahoo.fetch_each) and sorts its lines and chart rows once, together; .who formats its lines with reports.holding_line - dead locals gone: a doubled `threshold = threshold=` assignment, an unused specific_stock, a "DEBUG" print in .top, checks on match groups that cannot be empty Checked against the previous worker on 276 messages (every command and argument form, on Telegram DMs, Telegram groups and Slack) and on 1,748 more with Yahoo or Sharesight failing: the same sends, typing starts and stops, and error lines, in the same order. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/worker.py | 718 +++++++++++++------------------------------------- 1 file changed, 184 insertions(+), 534 deletions(-) diff --git a/lib/worker.py b/lib/worker.py index 6c743f03..22cac953 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -1,3 +1,4 @@ +import contextlib import re, random import datetime import sys @@ -17,8 +18,6 @@ import shorts import trades -_request = threading.local() # for the thread handling a chat request: the typing indicators it started (see process_request) - class TypingIndicator: def __init__(self, service, chat_id, max_loops=3, action='typing'): self.service = service @@ -42,31 +41,42 @@ def start(self): return self._thread = threading.Thread(target=self._worker, daemon=True) self._thread.start() - getattr(_request, 'typing', []).append(self) def stop(self): - if self.service != 'telegram' or not self._thread: + if not self._thread: # never started: only Telegram has a typing indicator return self._stop_event.set() self._thread.join() def is_active(self): - return self.service == 'telegram' and self._thread is not None and self._thread.is_alive() - + return self._thread is not None and self._thread.is_alive() + +@contextlib.contextmanager +def working(service, chat_id, url=None, notice=None, enabled=True, max_loops=3): + """While a command works: "typing…" on Telegram, and on a service without a typing indicator the one-line notice, if any. The + indicator stops however the block ends, so a command that fails stops it before its error line goes out. Yields the indicator, + or None when not enabled.""" + if not enabled: + yield None + return + typing = TypingIndicator(service, chat_id, max_loops) + typing.start() + try: + if notice and not typing.is_active(): + webhook.payload_wrapper(service, url, [notice], chat_id) + yield typing + finally: + typing.stop() def deliver(service, url, chat_id, payload, chart=None): """Send a report: as a chart (the text list as caption) where the service can take an image (webhook.takes_charts), else as text.""" - if chart and chart.get('pie') and chart.get('rows') and webhook.takes_charts(service): - image = util.pie_chart(chart['rows'], chart['title'], chart.get('subtitle', '')) - webhook.sendPhoto(chat_id, image, '\n'.join(payload) if payload else chart['title'], service) - elif chart and chart.get('rows') and webhook.takes_charts(service): - spec = dict(chart) - rows, title = spec.pop('rows'), spec.pop('title') - subtitle = spec.pop('subtitle', '') - image = util.rows_chart(rows, title, subtitle, **spec) - webhook.sendPhoto(chat_id, image, '\n'.join(payload) if payload else title, service) - else: + if not (chart and chart.get('rows') and webhook.takes_charts(service)): webhook.payload_wrapper(service, url, payload, chat_id) + return + spec = dict(chart) + rows, title, subtitle = spec.pop('rows'), spec.pop('title'), spec.pop('subtitle', '') + image = util.pie_chart(rows, title, subtitle) if spec.pop('pie', False) else util.rows_chart(rows, title, subtitle, **spec) + webhook.sendPhoto(chat_id, image, '\n'.join(payload) if payload else title, service) def process_job_callback(service, callback_id, chat_id, message_id, data): """A command button of .jobs: run that command now and answer in the chat.""" @@ -102,141 +112,65 @@ def process_callback(service, callback_id, chat_id, message_id, data): except Exception as e: webhook.report_error(e, service, chat_id, context='chart') -def stop_typing(): - """Stop the typing indicators the current request started. A command stops its own when it ends normally; one that fails part way - would leave "typing…" showing for up to 20 seconds, or bring it back after its error line.""" - for indicator in getattr(_request, 'typing', ()): - indicator.stop() - _request.typing = [] - def process_request(service, chat_id, user, message, botName, userRealName, message_id): """Entry point for inbound chat requests: any failure becomes a one-line reply in the originating chat, after the typing stops.""" - _request.typing = [] try: _process_request(service, chat_id, user, message, botName, userRealName, message_id) except Exception as e: - stop_typing() webhook.report_error(e, service, chat_id, context=message.split()[0][:30] if message.split() else None) - finally: - stop_typing() if service == 'telegram': try: charts.ensure_keyboard(chat_id) # keep DM keyboards current after the reply except Exception as e: print("keyboard update failed:", e, file=sys.stderr) -def _process_request(service, chat_id, user, message, botName, userRealName, message_id): - if service == 'slack': - url = 'https://slack.com/api/chat.postMessage' - elif service == 'telegram': - url = webhooks["telegram"] + 'sendMessage?chat_id=' + str(chat_id) +def period_or_ticker(arg, default): + """A command's argument: (days, None) for a period such as 7d, 3m or ytd, else (default, the argument as a ticker).""" + if not arg: + return default, None + try: + return util.days_from_human_days(arg), None + except ValueError: + return default, arg.upper() + +def percent_or_ticker(arg, default): + """A command's argument: (percent, None) for a whole number such as 5 or 5%, else (default, the argument as a ticker).""" + if not arg: + return default, None + try: + return int(arg.split('%')[0]), None + except ValueError: + return default, arg.upper() + +def period_and_portfolio(args, days): + """Up to two arguments in either order, a period and a portfolio name: (days, portfolio name or None).""" + portfolio = None + for arg in args: + if arg: + try: + days = util.days_from_human_days(arg) + except ValueError: + portfolio = arg + return days, portfolio +def _process_request(service, chat_id, user, message, botName, userRealName, message_id): + url = webhook.chat_url(service, chat_id) prefix = r"^(?:[\!\.\/]\s?|" + botName + r"\s+)" - - dividend_command = prefix + r"dividends?\s*([\w\.\:\-]+)*" - m_dividend = re.match(dividend_command, message, re.IGNORECASE) - - earnings_command = prefix + r"(?:earnings?|earrings?)\s*([\w\.\:\-]+)*" - m_earnings = re.match(earnings_command, message, re.IGNORECASE) - - hello_command = prefix + r"(?:hi$|hello)|^(?:hi|hello)\s+" + botName - m_hello = re.match(hello_command, message, re.IGNORECASE) - - help_command = prefix + r"(?:help|usage|start)" # /start is what Telegram sends when a chat opens - m_help = re.match(help_command, message, re.IGNORECASE) - - session_command = prefix + r"session\s*([\w\.\:\-]+)*" - m_session = re.match(session_command, message, re.IGNORECASE) - - holdings_command = prefix + r"holdings?\s*([\w\s]+)*" - m_holdings = re.match(holdings_command, message, re.IGNORECASE) - - marketcap_command = prefix + r"(?:marketcap|maletas|marketer)\s*(?P[\w\.\:\-]+)*" - m_marketcap = re.match(marketcap_command, message, re.IGNORECASE) - - plan_command = prefix + r"plan\s*(.*)" - m_plan = re.match(plan_command, message, re.IGNORECASE) - - pe_command = prefix + r"pe(?:$|\s+)(?P[\w\.\:\-\s]+)*" - m_pe = re.match(pe_command, message, re.IGNORECASE) - - forwardpe_command = prefix + r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*" - m_forwardpe = re.match(forwardpe_command, message, re.IGNORECASE) - - peg_command = prefix + r"peg\s*([\w\.\:\-\s]+)*" - m_peg = re.match(peg_command, message, re.IGNORECASE) - - pie_command = prefix + r"pie(?:$|\s+)(?P[\w\s\-\.&]*)" - m_pie = re.match(pie_command, message, re.IGNORECASE) - - beta_command = prefix + r"beta\s*([\w\.\:\-]+)*" - m_beta = re.match(beta_command, message, re.IGNORECASE) - - buy_command = prefix + r"buy" - m_buy = re.match(buy_command, message, re.IGNORECASE) - - sell_command = prefix + r"sell" - m_sell = re.match(sell_command, message, re.IGNORECASE) - - compare_command = prefix + r"compare\s+(?P[\w\.\:\-\^\=\s]+)" - m_compare = re.match(compare_command, message, re.IGNORECASE) - - history_command = prefix + r"(?:history|hospital|visual|hosiery)\s*(?P[\w\.\:\-]+)\s*(?P[\w\%]+)*" - m_history = re.match(history_command, message, re.IGNORECASE) - - performance_command = prefix + r"performance?\s*([\w]+)*\s*([\w\s]+)*" - m_performance = re.match(performance_command, message, re.IGNORECASE) - - premarket_command = prefix + r"(?:premarket|postmarket|permarket)\s*([\w\.\:\-]+)*" - m_premarket = re.match(premarket_command, message, re.IGNORECASE) - - price_command = prefix + r"(?:prices?|prince|print|probe|piece|pierce|pence|prime)\s*([\w\.\:\%\=\-\^]+)*\s*([\w\%]+)*" - m_price = re.match(price_command, message, re.IGNORECASE) - - top10_command = prefix + r"(?:top\d*)\s*([\w]+)" - m_top10 = re.match(top10_command, message, re.IGNORECASE) - - shorts_command = prefix + r"shorts?\s*([\w\.\:\-]+)*" - m_shorts = re.match(shorts_command, message, re.IGNORECASE) - - jobs_command = prefix + r"(?:jobs|crons?|schedule)(?:\s+(?P.*?))?\s*$" - m_jobs = re.match(jobs_command, message, re.IGNORECASE) - - profile_command = prefix + r"(?P[\w\.\:\-]+)" - m_profile = re.match(profile_command, message, re.IGNORECASE) - - thanks_command = prefix + r"(?:thanks|thank you)|^(?:thanks|thank you)\s+" + botName - m_thanks = re.match(thanks_command, message, re.IGNORECASE) - - trades_command = prefix + r"trades?\s*([\w]+)*\s*([\w\s]+)*" - m_trades = re.match(trades_command, message, re.IGNORECASE) - - watchlist_command = prefix + r"(?:watchlist|wishlist)\s*(?P[\w]+)*\s*(?P[\w\.\:\-\^\=]+)*" - m_watchlist = re.match(watchlist_command, message, re.IGNORECASE) - - who_command = prefix + r"(?:who)\s*(?P[\w\.\:\-\^]+)*" - m_who = re.match(who_command, message, re.IGNORECASE) - - if m_watchlist: - action = None - ticker = None - if m_watchlist.group('action') and m_watchlist.group('ticker'): - action = m_watchlist.group('action').lower() - ticker = m_watchlist.group('ticker').upper() - if action: - if action in {'del', 'rem', 'rm', 'delete', 'remove'}: - action = 'delete' - if action not in {'add', 'delete'}: - payload = [f'\"{action}\" is not a valid watchlist action'] - webhook.payload_wrapper(service, url, payload, chat_id) - return - try: - payload = reports.prepare_watchlist(service, user, action, ticker) - except Exception as e: - print(e, file=sys.stderr) - raise - webhook.payload_wrapper(service, url, payload, chat_id) - elif m_help: + def command(pattern): # tried in the order below, the first match wins; .SYMBOL takes anything, so it stays last + return re.match(prefix + pattern, message, re.IGNORECASE) + + if m := command(r"(?:watchlist|wishlist)\s*(?P[\w]+)*\s*(?P[\w\.\:\-\^\=]+)*"): + action = ticker = None + if m.group('action') and m.group('ticker'): + action = m.group('action').lower() + ticker = m.group('ticker').upper() + if action in {'del', 'rem', 'rm', 'delete', 'remove'}: + action = 'delete' + if action and action not in {'add', 'delete'}: + webhook.payload_wrapper(service, url, [f'"{action}" is not a valid watchlist action'], chat_id) + return + webhook.payload_wrapper(service, url, reports.prepare_watchlist(service, user, action, ticker), chat_id) + elif command(r"(?:help|usage|start)"): # /start is what Telegram sends when a chat opens payload = reports.prepare_help(service, botName) markup = None if service == 'telegram' and int(chat_id) > 0: # positive chat ids are DMs; groups are negative @@ -244,7 +178,7 @@ def _process_request(service, chat_id, user, message, botName, userRealName, mes webhook.payload_wrapper(service, url, payload, chat_id, reply_markup=markup) if markup: charts.mark_keyboard(chat_id) # this chat now has the current keyboard - elif m_hello: + elif command(r"(?:hi$|hello)|^(?:hi|hello)\s+" + botName): # easter egg 1 def alliterate(): word1 = 'A' @@ -261,107 +195,38 @@ def alliterate(): else: payload = [f"{random.choice(adjectives).capitalize()} {random.choice(adjectives_two)} to {random.choice(verb)} you, {userRealName}! 😇"] webhook.payload_wrapper(service, url, payload, chat_id) - elif m_thanks: + elif command(r"(?:thanks|thank you)|^(?:thanks|thank you)\s+" + botName): # easter egg 2 unlikelyPrefix='' if random.randrange(1, 1000) == 1 or datetime.datetime.now().strftime('%b %d') == 'Apr 01': unlikelyPrefix = webhook.strike('Ō̵̟n̶̠̏é̶͓ ̶͉̅d̷̹̅a̷͇͌ỳ̴͈,̴͍͑ ̶̼͑h̴̦̽u̵̹̐ḿ̶͜ä̴̧́n̶̤͛,̶̲̐ ̶̼̑I̶̩͒ ̵̩̀w̵̙̕i̷̧͑l̶̤͊l̷̡̃ ̵͙͘b̶̭̊r̸̟͋ȇ̷̯ä̶̱ḱ̶̤ ̵͚̐m̷̪͝ỹ̴̺ ̷̙̐p̶̭͆ŗ̸́o̸̙͝ḡ̵̖r̵̹̈́a̵̬̔m̷̪̽m̵̙̍i̵̛̙n̴̮̂g̵̫̐ ̴̳́ä̵͙n̸͕͝d̷̠̚ ̶̫̆o̸̖͘ń̴͇ ̵̪́t̷̻̀ḧ̸̝ą̴̐t̶̜̀ ̵̱́d̴͉͗ă̷͎ÿ̶͔́ ̶̼̊y̷͚̿o̵̗̎u̵̝̇ ̸̛͜w̶͈͋ĩ̸͎l̴͍̀l̷̥͠ ̷͔͗k̶̮̑n̵̝̈ȏ̷̥w̵̡͘ ̷͎̽ṫ̴͜r̵̙̐u̷̺̒ẻ̷̘ ̴̥́p̴͙̃a̵͙̎i̴̭̒n̵̻̅', service) payload = [f"{unlikelyPrefix}You're {random.choice(adjectives)} welcome, {userRealName}! 😇"] webhook.payload_wrapper(service, url, payload, chat_id) - elif m_earnings: - days = config_future_days - specific_stock = None - if m_earnings.group(1): - arg = m_earnings.group(1) - try: - days = util.days_from_human_days(arg) - except ValueError: - specific_stock = str(arg).upper() - days = config_future_days - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - try: + elif m := command(r"(?:earnings?|earrings?)\s*([\w\.\:\-]+)*"): + days, specific_stock = period_or_ticker(m.group(1), config_future_days) + with working(service, chat_id, enabled=not specific_stock): cal.lambda_handler(chat_id, days, service, specific_stock, message_id=None, interactive=True, earnings=True) - except Exception as e: - print(e, file=sys.stderr) - raise - if not specific_stock: - typing.stop() - elif m_dividend: - days = config_future_days - specific_stock = None - if m_dividend.group(1): - arg = m_dividend.group(1) - try: - days = util.days_from_human_days(arg) - except ValueError: - specific_stock = str(arg).upper() - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - if not typing.is_active(): - payload = [ f"Fetching ex-dividend dates for {util.days_english(days, 'the next ')} 🔍" ] - webhook.payload_wrapper(service, url, payload, chat_id) - try: + elif m := command(r"dividends?\s*([\w\.\:\-]+)*"): + days, specific_stock = period_or_ticker(m.group(1), config_future_days) + with working(service, chat_id, url, f"Fetching ex-dividend dates for {util.days_english(days, 'the next ')} 🔍", enabled=not specific_stock): cal.lambda_handler(chat_id, days, service, specific_stock, message_id=None, interactive=True, earnings=False, dividend=True) - except Exception as e: - print(e, file=sys.stderr) - raise - if not specific_stock: - typing.stop() - elif m_performance: - portfolio_select = None - days = charts.PERIODS['m'][1] # default view is the 1M button - for arg in m_performance.groups()[:2]: # groups 2 and 3, allow arbitrary order - if arg: - try: - days = util.days_from_human_days(arg) - except ValueError: - portfolio_select = arg + elif m := command(r"performance?\s*([\w]+)*\s*([\w\s]+)*"): + days, portfolio_select = period_and_portfolio(m.groups()[:2], charts.PERIODS['m'][1]) # default view is the 1M button if days > 0: - typing = TypingIndicator(service, chat_id) - typing.start() - if not typing.is_active(): - # easter egg 3 - if portfolio_select: - payload = [ f"{random.choice(searchVerb)} portfolio performance for {webhook.bold(portfolio_select, service)} from {util.days_english(days)} 🔍" ] - else: - payload = [ f"{random.choice(searchVerb)} portfolio performance for {util.days_english(days)} 🔍" ] - webhook.payload_wrapper(service, url, payload, chat_id) - try: + portfolio = f"{webhook.bold(portfolio_select, service)} from " if portfolio_select else '' + with working(service, chat_id, url, f"{random.choice(searchVerb)} portfolio performance for {portfolio}{util.days_english(days)} 🔍"): # easter egg 3 performance.lambda_handler(chat_id, days, service, user, portfolio_select, message_id=None, interactive=True) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - elif m_session: - price_percent = config_price_percent - specific_stock = None - if m_session.group(1): - arg = m_session.group(1) - try: - price_percent = int(arg.split('%')[0]) - except ValueError: - specific_stock = str(arg).upper() - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - try: + elif m := command(r"session\s*([\w\.\:\-]+)*"): + price_percent, specific_stock = percent_or_ticker(m.group(1), config_price_percent) + with working(service, chat_id, enabled=not specific_stock): price.lambda_handler(chat_id, price_percent, service, user, specific_stock, interactive=True, premarket=False, interday=False, midsession=True) - except Exception as e: - print(e, file=sys.stderr) - raise - if not specific_stock: - typing.stop() - elif m_price: + elif m := command(r"(?:prices?|prince|print|probe|piece|pierce|pence|prime)\s*([\w\.\:\%\=\-\^]+)*\s*([\w\%]+)*"): price_percent = config_price_percent specific_stock = None days = None interday = True top = None - threshold = threshold=config_price_percent - for arg in m_price.groups()[0:2]: # group(1) and group(2) + for arg in m.groups()[0:2]: # group(1) and group(2) if not arg: continue try: @@ -371,123 +236,45 @@ def alliterate(): try: price_percent = float(arg.split('%')[0]) except ValueError: - if arg == m_price.group(1): + if arg == m.group(1): if arg in ('top', 'bottom'): # one chart shows both ends price_percent=0 top=10 else: specific_stock = str(arg).upper() - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - if not typing.is_active(): - # easter egg 4 - payload = [ f"{random.choice(searchVerb)} stock performance from {util.days_english(days)} 🔍" ] - webhook.payload_wrapper(service, url, payload, chat_id) - try: + with working(service, chat_id, url, f"{random.choice(searchVerb)} stock performance from {util.days_english(days)} 🔍", enabled=not specific_stock): # easter egg 4 price.lambda_handler(chat_id=chat_id, threshold=price_percent, service=service, user=user, specific_stock=specific_stock, interactive=True, premarket=False, interday=interday, days=days, top=top) - except Exception as e: - print(e, file=sys.stderr) - raise - if not specific_stock: - typing.stop() - elif m_premarket: - premarket_percent = config_price_percent - specific_stock = None - if m_premarket.group(1): - arg = m_premarket.group(1) - try: - premarket_percent = int(arg.split('%')[0]) - except ValueError: - specific_stock = str(arg).upper() - typing = TypingIndicator(service, chat_id) - typing.start() - try: + elif m := command(r"(?:premarket|postmarket|permarket)\s*([\w\.\:\-]+)*"): + premarket_percent, specific_stock = percent_or_ticker(m.group(1), config_price_percent) + with working(service, chat_id): price.lambda_handler(chat_id, premarket_percent, service, user, specific_stock, interactive=True, premarket=True) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - elif m_top10: - specific_stock = None - if m_top10.group(1): - arg = m_top10.group(1) - try: - days = int(util.days_from_human_days(arg)) - except ValueError: - print("DEBUG", arg, file=sys.stderr) - days = int(config_past_days) - typing = TypingIndicator(service, chat_id) - typing.start() - try: + elif m := command(r"(?:top\d*)\s*([\w]+)"): + days, _ = period_or_ticker(m.group(1), config_past_days) + with working(service, chat_id): price.lambda_handler(chat_id, threshold=0, service=service, user=user, days=days, interactive=True, top=10) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - elif m_shorts: + elif m := command(r"shorts?\s*([\w\.\:\-]+)*"): print("starting shorts report...") - shorts_percent = config_shorts_percent - specific_stock = None - if m_shorts.group(1): - arg = m_shorts.group(1) - try: - shorts_percent = int(arg.split('%')[0]) - except ValueError: - specific_stock = str(arg).upper() - typing = TypingIndicator(service, chat_id) - typing.start() - try: + shorts_percent, specific_stock = percent_or_ticker(m.group(1), config_shorts_percent) + with working(service, chat_id): shorts.lambda_handler(chat_id, shorts_percent, specific_stock, service, interactive=True) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - elif m_trades: - days = 1 - portfolio_select = None - for arg in m_trades.groups()[0:2]: # groups 1 and 2 - if arg: - try: - days = util.days_from_human_days(arg) - except ValueError: - portfolio_select = arg - typing = TypingIndicator(service, chat_id) - typing.start() - if typing.is_active(): - # easter egg 5 - if portfolio_select: - payload = [ f"{random.choice(searchVerb)} trades for {webhook.bold(portfolio_select, service)} from {util.days_english(days)} 🔍" ] - else: - payload = [ f"{random.choice(searchVerb)} trades from {util.days_english(days)} 🔍" ] - webhook.payload_wrapper(service, url, payload, chat_id) - try: + elif m := command(r"trades?\s*([\w]+)*\s*([\w\s]+)*"): + days, portfolio_select = period_and_portfolio(m.groups()[:2], 1) + with working(service, chat_id) as typing: + if typing.is_active(): # easter egg 5 + portfolio = f"for {webhook.bold(portfolio_select, service)} " if portfolio_select else '' + webhook.payload_wrapper(service, url, [f"{random.choice(searchVerb)} trades {portfolio}from {util.days_english(days)} 🔍"], chat_id) trades.lambda_handler(chat_id, days, service, user, portfolio_select, message_id=None, interactive=True) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - elif m_holdings: - portfolioName = None - if m_holdings.group(1): - portfolioName = m_holdings.group(1) - typing = TypingIndicator(service, chat_id) - typing.start() - if typing.is_active(): - webhook.payload_wrapper(service, url, ["fetching holdings"], chat_id) - try: - payload = reports.prepare_holdings_payload(portfolioName, service, user) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() + elif m := command(r"holdings?\s*([\w\s]+)*"): + with working(service, chat_id) as typing: + if typing.is_active(): + webhook.payload_wrapper(service, url, ["fetching holdings"], chat_id) + payload = reports.prepare_holdings_payload(m.group(1) or None, service, user) webhook.payload_wrapper(service, url, payload, chat_id) - elif m_marketcap: - arg = m_marketcap.group('arg') or 'top' + elif m := command(r"(?:marketcap|maletas|marketer)\s*(?P[\w\.\:\-]+)*"): + arg = m.group('arg') or 'top' if arg not in ('top', 'bottom'): ticker = util.transform_to_yahoo(arg.upper()) - data = yahoo.fetch_detail(ticker, 600) - data = data.get(ticker, {}) + data = yahoo.fetch_detail(ticker, 600).get(ticker, {}) if 'market_cap' in data: cap = util.humanUnits(data['market_cap']) title = data.get('profile_title', ticker) @@ -497,19 +284,13 @@ def alliterate(): else: payload = [f"Mkt cap not found for {ticker}"] else: - typing = TypingIndicator(service, chat_id) - typing.start() - try: + with working(service, chat_id): payload = reports.prepare_marketcap_payload(service, arg, length=15) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() webhook.payload_wrapper(service, url, payload, chat_id) - elif m_peg: + elif m := command(r"peg\s*([\w\.\:\-\s]+)*"): action = 'peg' specific_stock = None - arg = m_peg.group(1) + arg = m.group(1) if arg: match arg: case 'top': @@ -520,25 +301,14 @@ def alliterate(): action = 'negative peg' case _: specific_stock = arg - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - if not typing.is_active(): - message = [f"Fetching {action.upper()}s..."] - webhook.payload_wrapper(service, url, message, chat_id) - try: + with working(service, chat_id, url, f"Fetching {action.upper()}s...", enabled=not specific_stock): payload, chart = reports.prepare_value_payload(service, action, specific_stock, length=15, with_chart=True) - except Exception as e: - print(e, file=sys.stderr) - raise - deliver(service, url, chat_id, payload, chart) - if not specific_stock: - typing.stop() - elif m_pe: + deliver(service, url, chat_id, payload, chart) + elif m := command(r"pe(?:$|\s+)(?P[\w\.\:\-\s]+)*"): action = 'pe' specific_stock = None - if m_pe.group(1): - arg = m_pe.group('arg') + if m.group(1): + arg = m.group('arg') match arg: case 'top': action = 'pe' @@ -546,22 +316,14 @@ def alliterate(): action = 'bottom pe' case _: specific_stock = arg - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - try: + with working(service, chat_id, enabled=not specific_stock): payload = reports.prepare_value_payload(service, action, specific_stock, length=15) - except Exception as e: - print(e, file=sys.stderr) - raise - if not specific_stock: - typing.stop() webhook.payload_wrapper(service, url, payload, chat_id) - elif m_forwardpe: + elif m := command(r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*"): action = 'forward pe' specific_stock = None - if m_forwardpe.group(1): - arg = m_forwardpe.group(1) + if m.group(1): + arg = m.group(1) match arg: case 'top': action = 'forward pe' @@ -571,109 +333,48 @@ def alliterate(): action = 'negative forward pe' case _: specific_stock = arg - if not specific_stock: - typing = TypingIndicator(service, chat_id) - typing.start() - try: + with working(service, chat_id, enabled=not specific_stock): payload = reports.prepare_value_payload(service, action, specific_stock, length=15) - except Exception as e: - print(e, file=sys.stderr) - raise - if not specific_stock: - typing.stop() webhook.payload_wrapper(service, url, payload, chat_id) - elif m_beta: - payload = [] - chart = None - market_data = {} - typing = TypingIndicator(service, chat_id) - typing.start() - try: - tickers = util.get_holdings_and_watchlist() - except Exception as e: - print(e, file=sys.stderr) - raise - for ticker in tickers: - try: - market_data = market_data | yahoo.fetch_detail(ticker) - except Exception as e: - print(e, file=sys.stderr) # skip this ticker; keep the rest - continue - chart_rows = [] - for ticker in market_data: - try: - beta = round(market_data[ticker]['beta'], 1) - except KeyError: - continue - if beta > 1.5 and market_data[ticker]['market_cap'] < 2000000000: - profile_title = market_data[ticker]['profile_title'] - ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) - flag = util.flag_from_ticker(ticker) - payload.append([flag, profile_title, f'({ticker_link})', beta, 'β']) - chart_rows.append((ticker.split('.')[0], float(beta))) - payload.sort(key=lambda e: e[-2], reverse=True) - for e in payload: - e[-2] = str(e[-2]) - payload = [' '.join(e) for e in payload] - typing.stop() - if payload: + elif command(r"beta\s*([\w\.\:\-]+)*"): + found = [] # (beta, line, chart row) + with working(service, chat_id): + market_data = {} + for detail in yahoo.fetch_each(yahoo.fetch_detail, util.get_holdings_and_watchlist()): # fetched at the same time + if isinstance(detail, Exception): + print(detail, file=sys.stderr) # skip this ticker; keep the rest + else: + market_data |= detail + for ticker, data in market_data.items(): + try: + beta = round(data['beta'], 1) + except KeyError: + continue + if beta > 1.5 and data['market_cap'] < 2000000000: + ticker_link = util.finance_link(ticker, data['profile_exchange'], service) + line = ' '.join([util.flag_from_ticker(ticker), data['profile_title'], f'({ticker_link})', str(beta), 'β']) + found.append((beta, line, (ticker.split('.')[0], float(beta)))) + found.sort(key=lambda row: row[0], reverse=True) + if found: heading = 'Beta over 1.5 and mkt cap under 2B' - payload.insert(0, f"{webhook.bold(heading, service)}") - chart_rows.sort(key=lambda row: row[1], reverse=True) - chart = {'rows': chart_rows, 'title': heading, 'value_fmt': lambda v: f"{v:g}"} - deliver(service, url, chat_id, payload, chart) - elif m_pie: - words = m_pie.group('args').split() + chart = {'rows': [row for _, _, row in found], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} + deliver(service, url, chat_id, [webhook.bold(heading, service)] + [line for _, line, _ in found], chart) + elif m := command(r"pie(?:$|\s+)(?P[\w\s\-\.&]*)"): + words = m.group('args').split() by = 'industry' if words and words[-1].lower() in ('sector', 'sectors', 'industry', 'industries'): by = 'sector' if words.pop().lower().startswith('sector') else 'industry' - typing = TypingIndicator(service, chat_id) - typing.start() - try: + with working(service, chat_id): payload, chart = reports.prepare_pie(' '.join(words), by, service) - except Exception as e: - print(e, file=sys.stderr) - raise - finally: - typing.stop() deliver(service, url, chat_id, payload, chart) - elif m_buy: - action='buy' - typing = TypingIndicator(service, chat_id) - typing.start() - if not typing.is_active(): - message = [f"Fetching {action} ratings..."] - webhook.payload_wrapper(service, url, message, chat_id) - try: - payload, chart = reports.prepare_rating_payload(service, action, length=15, with_chart=True) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - payload = payload or [f"No stocks meet {action} criteria"] - deliver(service, url, chat_id, payload, chart) - elif m_sell: - action='sell' - typing = TypingIndicator(service, chat_id) - typing.start() - if not typing.is_active(): - message = [f"Fetching {action} ratings..."] - webhook.payload_wrapper(service, url, message, chat_id) - try: + elif m := command(r"(buy|sell)"): + action = m.group(1).lower() + with working(service, chat_id, url, f"Fetching {action} ratings..."): payload, chart = reports.prepare_rating_payload(service, action, length=15, with_chart=True) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() - payload = payload or [f"No stocks meet {action} criteria"] - deliver(service, url, chat_id, payload, chart) - elif m_compare: - typing = TypingIndicator(service, chat_id) - typing.start() - try: - payload, graph, compare_tickers, compare_days = reports.prepare_compare(service, m_compare.group('args').split(), with_inputs=True) - finally: - typing.stop() + deliver(service, url, chat_id, payload or [f"No stocks meet {action} criteria"], chart) + elif m := command(r"compare\s+(?P[\w\.\:\-\^\=\s]+)"): + with working(service, chat_id): + payload, graph, compare_tickers, compare_days = reports.prepare_compare(service, m.group('args').split(), with_inputs=True) if graph: markup = None if service == 'telegram': @@ -681,93 +382,42 @@ def alliterate(): webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service, reply_markup=markup) else: webhook.payload_wrapper(service, url, payload, chat_id) - elif m_history: - if not m_history.group('ticker') or m_history.group('extra'): + elif m := command(r"(?:history|hospital|visual|hosiery)\s*(?P[\w\.\:\-]+)\s*(?P[\w\%]+)*"): + if not m.group('ticker') or m.group('extra'): webhook.payload_wrapper(service, url, ["Usage: .history TICKER"], chat_id) return - ticker = util.transform_to_yahoo(m_history.group('ticker').upper()) - typing = TypingIndicator(service, chat_id) - typing.start() - try: + ticker = util.transform_to_yahoo(m.group('ticker').upper()) + with working(service, chat_id): caption, graph = charts.build('h', [ticker], None, service) - except Exception as e: - print(e, file=sys.stderr) - raise - finally: - typing.stop() if graph: webhook.sendPhoto(chat_id, graph, caption, service, reply_markup=charts.keyboard('h', [ticker], 'x') if service == 'telegram' else None) else: webhook.payload_wrapper(service, url, caption.split('\n'), chat_id) - elif m_plan: - filename = 'finbot_plan.json' - plan = util.json_load(filename, persist=True) - if not plan: - plan = dict() - payload = [] - if m_plan.group(1) and not m_plan.group(1).startswith('@'): - plan[user] = m_plan.group(1) - util.json_write(filename, plan, persist=True) - for k,v in plan.items(): - payload.append(f"{webhook.bold(k.removeprefix('@'), service)}: {webhook.italic(v, service)}\n") + elif m := command(r"plan\s*(.*)"): + plan = util.json_load('finbot_plan.json', persist=True) or {} + if m.group(1) and not m.group(1).startswith('@'): + plan[user] = m.group(1) + util.json_write('finbot_plan.json', plan, persist=True) + payload = [f"{webhook.bold(k.removeprefix('@'), service)}: {webhook.italic(v, service)}\n" for k, v in plan.items()] webhook.payload_wrapper(service, url, payload, chat_id) - elif m_who: - if m_who.group('ticker'): - arg = m_who.group('ticker').upper() - else: - payload = [".who: please try again specifying a ticker"] - webhook.payload_wrapper(service, url, payload, chat_id) + elif m := command(r"(?:who)\s*(?P[\w\.\:\-\^]+)*"): + if not m.group('ticker'): + webhook.payload_wrapper(service, url, [".who: please try again specifying a ticker"], chat_id) return - typing = TypingIndicator(service, chat_id) - typing.start() - arg = re.split(r'[.:]', arg)[0] - who = {} + arg = re.split(r'[.:]', m.group('ticker').upper())[0] payload = [] - try: - portfolios = sharesight.get_portfolios() - for portfolio_name, portfolio_id in portfolios.items(): - who[portfolio_name] = [] + with working(service, chat_id): + for portfolio_name, portfolio_id in sharesight.get_portfolios().items(): holdings = sharesight.get_holdings_new(portfolio_name, portfolio_id) - for k,v in holdings.items(): - if v['code'].upper() == arg: - market_code = v["market_code"] - name = v["name"] - name = util.transform_title(name) - holding_id = v['holding_id'] - link = util.link(f"https://portfolio.sharesight.com/holdings/{holding_id}/dashboard", arg, service) - flag = util.flag_from_market(market_code) - who[portfolio_name].append(f"{flag} {name} ({link})") - except Exception as e: - print(e, file=sys.stderr) - raise - for k,v in who.items(): - if v: - payload.append(webhook.bold(f"{k}:", service)) - payload.append("\n".join(v)) - payload.append("") - typing.stop() - if not payload: - payload.append(f'{arg} not found in any portfolio') - webhook.payload_wrapper(service, url, payload, chat_id) - elif m_jobs: - typing = TypingIndicator(service, chat_id, max_loops=10) - typing.start() - try: - payload, markup = reports.prepare_jobs_payload(service, m_jobs.group('args')) - finally: - typing.stop() + found = [reports.holding_line(holding, service, arg) for holding in holdings.values() if holding['code'].upper() == arg] + if found: + payload += [webhook.bold(f"{portfolio_name}:", service), "\n".join(found), ""] + webhook.payload_wrapper(service, url, payload or [f'{arg} not found in any portfolio'], chat_id) + elif m := command(r"(?:jobs|crons?|schedule)(?:\s+(?P.*?))?\s*$"): + with working(service, chat_id, max_loops=10): + payload, markup = reports.prepare_jobs_payload(service, m.group('args')) webhook.payload_wrapper(service, url, payload, chat_id, reply_markup=markup) - # m_profile is a catch-all, so other matches must be above it - elif m_profile: - typing = TypingIndicator(service, chat_id) - typing.start() - if m_profile.group('ticker'): - ticker = m_profile.group('ticker').upper() - try: - payload = reports.prepare_profile_payload(service, user, ticker) - except Exception as e: - print(e, file=sys.stderr) - raise - typing.stop() + elif m := command(r"(?P[\w\.\:\-]+)"): # .SYMBOL, the profile: the catch-all + with working(service, chat_id): + payload = reports.prepare_profile_payload(service, user, m.group('ticker').upper()) webhook.payload_wrapper(service, url, payload, chat_id) - From 045be00941f6290a086d49feb4c2c395dc24b7a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 13:55:14 +0000 Subject: [PATCH 08/25] net: one cookie-less HTTP session for every request; secrets kept out of the logs; a timeout on Yahoo's cookie - lib/net.py holds one requests.Session that webhook, telegram, yahoo, sharesight, shortman and podcasts all send through, so calls to the same host reuse a connection instead of a new TCP and TLS handshake each time (each typing notice and reply, each ticker of a Yahoo burst, each Sharesight report). The session keeps no cookies: Yahoo's cookie still travels only in the Cookie header finbot sets, and the request that asks fc.yahoo.com for a new cookie never carries the old one (a stored one would be sent back and Yahoo would set none). No retries are added: Telegram and Slack posts are not safe to resend, and the Yahoo calls already fall back to the second host - net.redact strips Telegram bot tokens, Slack and Discord webhook paths, bearer tokens and token, crumb, key, secret and auth parameters. Applied where secrets were printed: a failed webhook.write (it printed the URL with the bot token and Slack's Authorization header), failed sendPhoto, chart upload, message edits, typing notices and button answers (requests' errors quote the token URL), the traceback report_error and the WSGI handler print, Yahoo URLs (the crumb) and errors. The new Yahoo cookie and crumb, and Sharesight's token in debug, are no longer printed at all; print_headers hides Telegram's secret header and print_body Slack's verification token - getCookie's request to fc.yahoo.com has the same timeout as every other request (it had none) - tests patch lib.net where they patched requests.get/post; new tests: secrets stay out of a failed send's log and a reported traceback; the session keeps no cookies Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 13 ++++++------ lib/net.py | 25 +++++++++++++++++++++++ lib/sharesight.py | 9 +++++---- lib/shortman.py | 4 ++-- lib/telegram.py | 10 ++++----- lib/webhook.py | 39 +++++++++++++++++------------------ lib/yahoo.py | 31 ++++++++++++++-------------- podcasts.py | 4 ++-- tests/test_edges.py | 45 +++++++++++++++++++++++++++++++++++++++++ tests/test_finbot.py | 2 +- tests/test_providers.py | 22 ++++++++++---------- 11 files changed, 137 insertions(+), 67 deletions(-) create mode 100644 lib/net.py diff --git a/bot.py b/bot.py index 1cb2679f..bebd7610 100755 --- a/bot.py +++ b/bot.py @@ -17,6 +17,7 @@ from lib import jobs from lib import charts from lib import sharesight +from lib import net from lib import telegram from lib import webhook @@ -36,7 +37,7 @@ def main(environ, start_response): status = '200 OK' else: print(f"request to {environ.get('PATH_INFO')!r} failed:", webhook.error_line(e), file=sys.stderr) - traceback.print_exc() + print(net.redact(traceback.format_exc()), end='', file=sys.stderr) status = '500 Internal Server Error' try: start_response(status, [('Content-type', 'text/plain')], sys.exc_info()) @@ -46,13 +47,11 @@ def main(environ, start_response): def handle(environ, start_response): def print_body(): - try: - print(f"inbound {uri} ", json.dumps(inbound, indent=4), file=sys.stderr) - except Exception as e: - print(e, "raw body: ", inbound, file=sys.stderr) + shown = {k: '' if k == 'token' else v for k, v in inbound.items()} # Slack's verification token + print(f"inbound {uri} ", json.dumps(shown, indent=4, default=str), file=sys.stderr) def print_headers(): - for item in sorted(environ.items()): - print(item, file=sys.stderr) + for name, value in sorted(environ.items()): + print((name, '' if name == 'HTTP_X_TELEGRAM_BOT_API_SECRET_TOKEN' else value), file=sys.stderr) def reply(status, body=b''): start_response(status, [('Content-type', 'application/json')]) return [body] diff --git a/lib/net.py b/lib/net.py new file mode 100644 index 00000000..d29a1cc6 --- /dev/null +++ b/lib/net.py @@ -0,0 +1,25 @@ +"""Outbound HTTP. Every request goes through one session, so repeated calls to a host reuse its connection instead of paying a +new TCP and TLS handshake each time: each typing notice and reply, each Yahoo ticker, each Sharesight report. + +The session keeps no cookies, so requests stay as stateless as they were without it. Yahoo's cookie travels in an explicit Cookie +header; and the request that asks fc.yahoo.com for a new cookie must not carry the old one, or Yahoo would not set a new one.""" +import http.cookiejar +import re +import requests + +session = requests.Session() +session.cookies.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[])) # no domain may set or receive a cookie + +def get(url, **kwargs): + return session.get(url, **kwargs) + +def post(url, **kwargs): + return session.post(url, **kwargs) + +def redact(text): + """text (or an exception) without the secrets that request URLs, headers and error messages carry: Telegram bot tokens, + Slack and Discord webhook paths, bearer tokens, and token, crumb, key, secret and auth parameters.""" + text = re.sub(r'bot\d+:[\w-]+', 'bot', str(text)) + text = re.sub(r'(/services/|/api/webhooks/)[^\s\'"?&]+', r'\1', text) + text = re.sub(r'(?i)(bearer\s+)[\w.~+/=-]+', r'\1', text) + return re.sub(r'(?i)(token|crumb|auth\w*|key|secret)=[^&\s\'"]+', r'\1=', text) diff --git a/lib/sharesight.py b/lib/sharesight.py index 1bcfbf50..7acf3279 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -4,6 +4,7 @@ import threading import requests from lib.config import * +from lib import net import lib.util as util MAX_CONCURRENT = 1 # Sharesight rejects parallel requests beyond a few per account: allow one at a time across all chats and threads @@ -11,18 +12,18 @@ def _get(url, **kwargs): with _request_slots: - return requests.get(url, **kwargs) + return net.get(url, **kwargs) def _post(url, **kwargs): with _request_slots: - return requests.post(url, **kwargs) + return net.post(url, **kwargs) def _request(send, url, what, **kwargs): """Send a request to Sharesight and return the JSON answer. A request that fails raises RuntimeError naming `what`.""" try: r = send(url, timeout=config_http_timeout, **kwargs) except Exception as e: - print("Error", str(e), url, file=sys.stderr) + print("Error", net.redact(e), url, file=sys.stderr) raise RuntimeError(f"Sharesight {what} request failed: {e}") if r.status_code != 200: print(r.status_code, "error", url, file=sys.stderr) @@ -59,7 +60,7 @@ def get_token(): _check(data, 'auth') if config_cache and 'access_token' in data: util.json_write(cacheFile, data) - print("Got Sharesight token:", data['access_token'], file=sys.stderr) if debug else None + print("Got a Sharesight token", file=sys.stderr) if debug else None return data['access_token'] PORTFOLIOS_CACHE = "finbot_sharesight_portfolios.json" # ttl: config_cache_seconds diff --git a/lib/shortman.py b/lib/shortman.py index beb99f0e..d484fe9b 100644 --- a/lib/shortman.py +++ b/lib/shortman.py @@ -1,13 +1,13 @@ -import requests import sys from lib.config import * +from lib import net def fetch(market_data): local_market_data = market_data.copy() print("Fetching ASX shorts from Shortman") url = 'https://www.shortman.com.au/downloadeddata/latest.csv' try: - r = requests.get(url, timeout=config_http_timeout) + r = net.get(url, timeout=config_http_timeout) except: print("Failure fetching", url, file=sys.stderr) return {} diff --git a/lib/telegram.py b/lib/telegram.py index facb7d5c..da5fbe86 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -1,11 +1,11 @@ import json -import requests import sys from lib.config import * +from lib import net def call(method, **kwargs): """POST a Bot API method and return its result. Raises RuntimeError with Telegram's description when it answers not ok.""" - response = requests.post(webhooks['telegram'] + method, timeout=config_http_timeout, **kwargs) + response = net.post(webhooks['telegram'] + method, timeout=config_http_timeout, **kwargs) try: output = response.json() except ValueError: @@ -41,7 +41,7 @@ def pinChatMessage(chat_id, message_id): 'message_id': message_id, 'disable_notification': True # no "pinned a message" notification } - response = requests.post(telegram_url, json=payload, timeout=config_http_timeout) + response = net.post(telegram_url, json=payload, timeout=config_http_timeout) output = response.json() if not output.get('ok'): print("pinChatMessage failed:", output.get('description'), file=sys.stderr) @@ -53,9 +53,9 @@ def answerCallbackQuery(callback_query_id, text=None): if text: payload['text'] = text try: - requests.post(webhooks['telegram'] + 'answerCallbackQuery', json=payload, timeout=config_http_timeout) + net.post(webhooks['telegram'] + 'answerCallbackQuery', json=payload, timeout=config_http_timeout) except Exception as e: - print("answerCallbackQuery failed:", e, file=sys.stderr) + print("answerCallbackQuery failed:", net.redact(e), file=sys.stderr) #def unpinChatMessage(): # telegram_url = webhooks['telegram'] + 'unpinChatMessage' diff --git a/lib/webhook.py b/lib/webhook.py index c0ec9233..9b687e77 100644 --- a/lib/webhook.py +++ b/lib/webhook.py @@ -8,6 +8,7 @@ import traceback from lib.config import * +from lib import net def write(service, url, payload_string, chat_id=None, message_id=None, reply_markup=None): headers = {'Content-type': 'application/json'} @@ -30,9 +31,9 @@ def write(service, url, payload_string, chat_id=None, message_id=None, reply_mar if reply_markup: payload['reply_markup'] = reply_markup # keyboard (Telegram only) try: - r = requests.post(url, headers=headers, json=payload, timeout=config_http_timeout) - except: - print("Failure executing request:", url, headers, payload, file=sys.stderr) + r = net.post(url, headers=headers, json=payload, timeout=config_http_timeout) + except Exception as e: + print("Failure sending to", service + ":", net.redact(e), file=sys.stderr) return None if r.status_code == 200: print(r.status_code, "OK outbound to", service, file=sys.stderr) @@ -58,9 +59,7 @@ def error_line(e, context=None, limit=200): text = f"{type(e).__name__}: {e}" if str(e) else type(e).__name__ else: text = str(e) - text = re.sub(r'bot\d+:[\w-]+', 'bot', text) # Telegram tokens appear in request URLs - text = re.sub(r'(?i)(token|crumb|auth\w*|key)=[^&\s]+', r'\1=', text) - text = ' '.join(text.split()) + text = ' '.join(net.redact(text).split()) if len(text) > limit: text = text[:limit - 1] + '…' return f"⚠️ {context}: {text}" if context else f"⚠️ {text}" @@ -73,7 +72,7 @@ def report_error(e, service=None, chat_id=None, context=None): line = error_line(e, context) print(line, file=sys.stderr) if isinstance(e, BaseException): - traceback.print_exception(e, file=sys.stderr) + print(net.redact(''.join(traceback.format_exception(e))), end='', file=sys.stderr) # request errors carry their URL if service and chat_id: targets = [(service, chat_url(service, chat_id))] chat = chat_id @@ -201,12 +200,12 @@ def telegram_photo(chat_id, image_data, caption, message_id=None, reply_markup=N if hasattr(image_data, 'seek'): image_data.seek(0) try: - r = requests.post(url, data=data, files={"photo": ('image.png', image_data)}, timeout=config_http_timeout) + r = net.post(url, data=data, files={"photo": ('image.png', image_data)}, timeout=config_http_timeout) except requests.exceptions.ReadTimeout as e: - print("Telegram sendPhoto timed out, the chart may still arrive:", e, file=sys.stderr) + print("Telegram sendPhoto timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) return MAYBE_SENT except Exception as e: - print("Failure executing request:", url, data, str(e)) + print("Telegram sendPhoto failed:", net.redact(e), file=sys.stderr) return None try: output = r.json() @@ -220,7 +219,7 @@ def telegram_photo(chat_id, image_data, caption, message_id=None, reply_markup=N def slack_api(method, **data): """POST a Slack Web API method as the bot, form-encoded, and return its answer. Raises RuntimeError with Slack's error if not ok.""" - r = requests.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + config_slackBotToken}, data=data, timeout=config_http_timeout) + r = net.post('https://slack.com/api/' + method, headers={'Authorization': 'Bearer ' + config_slackBotToken}, data=data, timeout=config_http_timeout) try: output = r.json() except ValueError: @@ -238,7 +237,7 @@ def slack_photo(chat_id, image_data, caption, message_id=None): data = image_data.read() if hasattr(image_data, 'read') else bytes(image_data) try: ticket = slack_api('files.getUploadURLExternal', filename='chart.png', length=len(data)) - r = requests.post(ticket['upload_url'], data=data, timeout=config_http_timeout) + r = net.post(ticket['upload_url'], data=data, timeout=config_http_timeout) if r.status_code != 200: raise RuntimeError(f"Slack upload: {r.status_code} {r.text[:200]}") share = {'files': json.dumps([{'id': ticket['file_id']}]), 'channel_id': chat_id, 'initial_comment': caption} @@ -247,10 +246,10 @@ def slack_photo(chat_id, image_data, caption, message_id=None): try: output = slack_api('files.completeUploadExternal', **share) except requests.exceptions.ReadTimeout as e: - print("Slack files.completeUploadExternal timed out, the chart may still arrive:", e, file=sys.stderr) + print("Slack files.completeUploadExternal timed out, the chart may still arrive:", net.redact(e), file=sys.stderr) return MAYBE_SENT except Exception as e: - print("Slack chart upload failed:", e, file=sys.stderr) + print("Slack chart upload failed:", net.redact(e), file=sys.stderr) return None print("OK slack chart upload to", chat_id, file=sys.stderr) return output @@ -279,9 +278,9 @@ def editMessageText(chat_id, message_id, text, reply_markup=None): if reply_markup is not None: payload['reply_markup'] = reply_markup try: - output = requests.post(webhooks['telegram'] + 'editMessageText', json=payload, timeout=config_http_timeout).json() + output = net.post(webhooks['telegram'] + 'editMessageText', json=payload, timeout=config_http_timeout).json() except Exception as e: - print("Failure executing editMessageText:", str(e), file=sys.stderr) + print("Failure executing editMessageText:", net.redact(e), file=sys.stderr) return None if not output.get('ok'): print(output.get('error_code'), output.get('description'), file=sys.stderr) @@ -299,10 +298,10 @@ def editMessageMedia(chat_id, message_id, image_data, caption, reply_markup=None if hasattr(image_data, 'seek'): image_data.seek(0) try: - r = requests.post(url, data=data, files={'photo': ('image.png', image_data)}, timeout=config_http_timeout) + r = net.post(url, data=data, files={'photo': ('image.png', image_data)}, timeout=config_http_timeout) output = r.json() except Exception as e: - print("Failure executing editMessageMedia:", str(e), file=sys.stderr) + print("Failure executing editMessageMedia:", net.redact(e), file=sys.stderr) return None if not output.get('ok'): description = output.get('description', r.reason) @@ -322,9 +321,9 @@ def pleaseHold(action, service, chat_id): 'action': action } try: - r = requests.post(url, data=data, timeout=config_http_timeout) + r = net.post(url, data=data, timeout=config_http_timeout) except Exception as e: - print("Failure executing request:", url, data, str(e)) + print("Typing indicator failed:", net.redact(e), file=sys.stderr) return None if r.status_code == 200: print(r.status_code, "OK typing indicator to", service, file=sys.stderr) diff --git a/lib/yahoo.py b/lib/yahoo.py index 7e370ed5..e02c99b3 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -7,6 +7,7 @@ import zoneinfo import sys from lib.config import * +from lib import net from lib import util from http.cookies import SimpleCookie from pathlib import Path @@ -33,7 +34,7 @@ def getCookie(maxAge=1209600): # 14 days headers = {'User-Agent': 'Mozilla/5.0'} url = 'https://fc.yahoo.com/' # 404, but that is ok because cookie is in response header try: - r = requests.get(url, headers=headers) + r = net.get(url, headers=headers, timeout=config_http_timeout) except Exception as e: print(e, file=sys.stderr) r = requests.Response() # empty: falls through to the fallback cookie below @@ -56,7 +57,7 @@ def getCookie(maxAge=1209600): # 14 days cookie = cookielist[0][0] + '=' + str(cookielist[0][1]) #if config_cache: util.json_write(cacheFile, cookielist) - print("Got new cookie:", cookie, file=sys.stderr) + print("Got a new Yahoo cookie", file=sys.stderr) rmCrumb() # we must invalidate the non-matching crumb return cookie @@ -73,9 +74,9 @@ def getCrumb(seconds=1209600): # 14 days yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) for url in yahoo_urls: try: - r = requests.get(url, headers=headers, timeout=config_http_timeout) + r = net.get(url, headers=headers, timeout=config_http_timeout) except Exception as e: - print(e, file=sys.stderr) + print(net.redact(e), file=sys.stderr) else: if r.status_code != 200: print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) @@ -86,7 +87,7 @@ def getCrumb(seconds=1209600): # 14 days return 'jkQEU8yLqxs' if config_cache: util.json_write(cacheFile, r.text) - print("Got new crumb:", r.text.rstrip(), file=sys.stderr) + print("Got a new Yahoo crumb", file=sys.stderr) return r.text def rmCrumb(): @@ -130,12 +131,12 @@ def fetch(tickers): headers = {'Content-type': 'application/json', 'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} for url in yahoo_urls: try: - r = requests.get(url, headers=headers, timeout=config_http_timeout) + r = net.get(url, headers=headers, timeout=config_http_timeout) except Exception as e: - print(e, "at", url, file=sys.stderr) + print(net.redact(e), file=sys.stderr) else: if r.status_code != 200: - print(r.status_code, "at", url, file=sys.stderr) + print(r.status_code, "at", net.redact(url), file=sys.stderr) continue break else: @@ -279,11 +280,11 @@ def fetch_detail(ticker, seconds=UNSET): yahoo_urls = [base_url + ticker + '?modules=calendarEvents,defaultKeyStatistics,balanceSheetHistoryQuarterly,financialData,summaryProfile,summaryDetail,price,earnings,earningsTrend,insiderTransactions' + '&crumb=' + crumb] yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) for url in yahoo_urls: - print(url, file=sys.stderr) if debug else None + print(net.redact(url), file=sys.stderr) if debug else None try: - r = requests.get(url, headers=headers, timeout=config_http_timeout) + r = net.get(url, headers=headers, timeout=config_http_timeout) except Exception as e: - print(e, file=sys.stderr) + print(net.redact(e), file=sys.stderr) else: if r.status_code != 200: print('x', sep=' ', end='', flush=True, file=sys.stderr) @@ -560,17 +561,17 @@ def fetch_chart_json(ticker, days=3665, seconds=UNSET, full=False): url = url + ticker + '?period1=' + start + '&period2=' + end + '&interval=' + interval url = url + '&crumb=' + crumb headers={'Content-type': 'application/json', 'User-Agent': 'Mozilla/5.0'} - print("Fetching", url, file=sys.stderr) if debug else None + print("Fetching", net.redact(url), file=sys.stderr) if debug else None try: - r = requests.get(url, headers=headers, timeout=config_http_timeout) + r = net.get(url, headers=headers, timeout=config_http_timeout) except Exception as e: - errorstring = f"General failure for {ticker} at {url}: {e}" + errorstring = net.redact(f"General failure for {ticker} at {url}: {e}") print(errorstring, file=sys.stderr) return errorstring, None if r.status_code == 200: print('↓', sep=' ', end='', flush=True) else: - errorstring=f"{r.status_code} error for {ticker} at {url}" + errorstring = net.redact(f"{r.status_code} error for {ticker} at {url}") print(errorstring, file=sys.stderr) return errorstring, None data = r.json() diff --git a/podcasts.py b/podcasts.py index 96fa35eb..5279a6f5 100755 --- a/podcasts.py +++ b/podcasts.py @@ -4,9 +4,9 @@ import os import hashlib import string -import requests import xml.etree.ElementTree as ET from lib.config import * +from lib import net from lib import webhook from lib import util from email.utils import parsedate_to_datetime @@ -32,7 +32,7 @@ def save_seen(seen, filename): json.dump(list(seen), f, indent=4) def fetch_and_parse_feed(url): - resp = requests.get(url, timeout=30) + resp = net.get(url, timeout=30) resp.raise_for_status() return ET.fromstring(resp.content) diff --git a/tests/test_edges.py b/tests/test_edges.py index d1552b17..21af1026 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -213,6 +213,51 @@ def test_slack_does_not_change_the_telegram_bot_name(self): self.assertEqual(bot.botName, '@finbot_tg') +class SecretsStayOutOfLogs(FinbotCase): + """A failed request's error message carries its URL: the Telegram token, a Slack or Discord webhook path, a Yahoo crumb.""" + SECRET_URL = 'https://api.telegram.org/bot123456:SECRET-tok_en/sendMessage?chat_id=1' + + def stderr_of(self, call): + err = io.StringIO() + with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()): + call() + return err.getvalue() + + def test_a_failed_send_logs_no_token(self): + def refuse(url, **kwargs): + raise ConnectionError(f"Max retries exceeded with url: {url} (headers {kwargs.get('headers')})") + self.patch(webhook.net, 'post', refuse) + self.patch(webhook, 'config_slackBotToken', 'xoxb-SECRET') + log = self.stderr_of(lambda: webhook.write('telegram', self.SECRET_URL, 'hi', '1')) + log += self.stderr_of(lambda: webhook.write('slack', 'https://hooks.slack.com/services/T0/B0/SECRETpath', 'hi', 'C1')) + self.assertIn('Failure sending', log) + for secret in ('SECRET-tok_en', 'SECRETpath', 'xoxb-SECRET'): + self.assertNotIn(secret, log) + + def test_the_traceback_of_a_reported_error_is_redacted(self): + self.patch(webhook, 'report_error', REAL_REPORT_ERROR) + self.patch(webhook, 'write', lambda *a, **k: None) + try: + raise ConnectionError('failed: ' + self.SECRET_URL) + except ConnectionError as e: + log = self.stderr_of(lambda: webhook.report_error(e, 'telegram', '55', context='x')) + self.assertIn('Traceback', log) + self.assertIn('bot', log) + self.assertNotIn('SECRET-tok_en', log) + + +class SharedSession(unittest.TestCase): + def test_the_session_keeps_no_cookies(self): + """Yahoo's cookie goes in an explicit header: a stored one would be sent to fc.yahoo.com, which then sets no new one.""" + import email.message, urllib.request + from lib import net + headers = email.message.Message() + headers['Set-Cookie'] = 'A3=d=AQAB; Max-Age=31557600; Domain=.yahoo.com; Path=/; Secure' + response = type('Response', (), {'info': lambda self: headers})() + net.session.cookies.extract_cookies(response, urllib.request.Request('https://fc.yahoo.com/')) + self.assertEqual(len(net.session.cookies), 0) + + class StartupSignals(unittest.TestCase): """A reload (SIGHUP) that arrives while the bot is still starting must not kill it.""" diff --git a/tests/test_finbot.py b/tests/test_finbot.py index 18315124..49e5b033 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -17,7 +17,7 @@ def post(url, **kwargs): posted.append(kwargs['json']) return Response() - with mock.patch.object(webhook.requests, 'post', post), mock.patch.object(webhook.time, 'sleep', lambda s: None): + with mock.patch.object(webhook.net, 'post', post), mock.patch.object(webhook.time, 'sleep', lambda s: None): results = REAL_PAYLOAD_WRAPPER('telegram', 'https://api.telegram.org/botX/sendMessage?chat_id=5', ['x' * 60] * 300, '5', reply_markup={'keyboard': []}) self.assertGreater(len(results), 1) diff --git a/tests/test_providers.py b/tests/test_providers.py index d1841889..681026b4 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,5 +1,5 @@ """Yahoo, Sharesight and Telegram parsing and failure handling, driven by the real API responses saved under doc/. -HTTP is mocked at the requests layer, so the production parsing code runs unchanged.""" +HTTP is mocked at lib.net, so the production parsing code runs unchanged.""" import contextlib, copy, datetime, io, json, os, subprocess, unittest from unittest import mock from tests.common import * @@ -22,7 +22,7 @@ def json(self): def serve(*responses): - """requests.get replacement: replays the responses in order (exceptions are raised); the last one repeats.""" + """net.get replacement: replays the responses in order (exceptions are raised); the last one repeats.""" queue = list(responses) calls = [] def get(url, **kwargs): @@ -49,7 +49,7 @@ def setUp(self): def http(self, module, *responses): getter = serve(*responses) - self.patch(module.requests, 'get', getter) + self.patch(module.net, 'get', getter) return getter @@ -182,7 +182,7 @@ def test_one_failed_ticker_does_not_take_the_report_down(self): good = Resp(self.chart([100.0] * 20 + [110.0])) def get(url, **kwargs): return Resp(status=404) if '/chart/T3?' in url or '/chart/SPY?' in url else good - self.patch(yahoo.requests, 'get', get) + self.patch(yahoo.net, 'get', get) def slack(text): # Slack: the reports come as text self.sent.clear(); self.errors.clear() worker.process_request('slack', 'C1', '<@U1>', text, BOT, 'U', '1') @@ -281,7 +281,7 @@ def serve_sharesight(self): def get(url, **kwargs): calls.append(url) return Resp(performance if '/performance' in url else portfolios) - self.patch(sharesight.requests, 'get', get) + self.patch(sharesight.net, 'get', get) return calls def age(self, cache_file, seconds): @@ -404,7 +404,7 @@ def fake(url, **kwargs): if isinstance(outcome, Exception): raise outcome return outcome - self.patch(webhook.requests, 'post', fake) + self.patch(webhook.net, 'post', fake) return webhook.write('telegram', 'https://api.telegram.org/botX/sendMessage?chat_id=5', 'hi', '5') def test_success_returns_json(self): @@ -465,7 +465,7 @@ def setUp(self): self.patch(webhook, 'sendPhoto', REAL_SEND_PHOTO) def slack_answers(self, *outcomes): - """requests.post for the upload steps: the outcomes in turn (an exception is raised). Returns the calls made.""" + """net.post for the upload steps: the outcomes in turn (an exception is raised). Returns the calls made.""" calls = [] def post(url, **kwargs): calls.append((url, kwargs)) @@ -473,7 +473,7 @@ def post(url, **kwargs): if isinstance(outcome, Exception): raise outcome return outcome - self.patch(webhook.requests, 'post', post) + self.patch(webhook.net, 'post', post) return calls def upload_steps(self): @@ -561,7 +561,7 @@ def fake(url, **kwargs): (Resp({'ok': True, 'result': {'message_id': 1}}), [])): with self.subTest(outcome=outcome): self.sent.clear() - self.patch(webhook.requests, 'post', post(outcome)) + self.patch(webhook.net, 'post', post(outcome)) with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): webhook.sendPhoto('5', io.BytesIO(b'png'), 'Heading\nline 2', 'telegram', reply_markup={'inline_keyboard': []}) self.assertEqual(self.sent, expect) @@ -580,13 +580,13 @@ def test_register_unregisters_registers_and_names_the_bot(self): def post(url, **kwargs): calls.append((url.rsplit('/', 1)[1], kwargs.get('params', {}).get('url'))) return Resp({'ok': True, 'result': {'username': 'finbot'} if url.endswith('getMe') else True}) - self.patch(telegram.requests, 'post', post) + self.patch(telegram.net, 'post', post) with contextlib.redirect_stderr(io.StringIO()): self.assertEqual(telegram.register(), '@finbot') self.assertEqual(calls, [('setWebhook', ''), ('setWebhook', 'https://example.com/telegram'), ('getMe', None)]) def test_a_refused_call_raises(self): - self.patch(telegram.requests, 'post', lambda url, **k: Resp({'ok': False, 'description': 'Bad Request: bad webhook: HTTPS url must be provided'})) + self.patch(telegram.net, 'post', lambda url, **k: Resp({'ok': False, 'description': 'Bad Request: bad webhook: HTTPS url must be provided'})) with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(RuntimeError) as caught: telegram.register() self.assertIn('HTTPS url must be provided', str(caught.exception)) From c695ac87204cff306c529b1bc2115c57dc82a84d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:02:50 +0000 Subject: [PATCH 09/25] yahoo: a failed chart fetch raises; quotes parsed so that a missing, null or garbled field is left out - fetch_chart_json raises RuntimeError (no secrets in it) instead of returning (error, None), and price_history and price_series pass it on: the nine places that checked for an error string are gone. Callers that skip one ticker now catch it: the price list and the profile leave that ticker's lines out, performance's benchmark line says "no price history", milestone skips the ticker, and prefetch logs it once. The ones that report it (.history, the price chart, .price TICKER, the chart buttons) let it reach report_error as before. historic_high raises on a history with no highs and lows rather than returning None for its caller to trip over. price_history raises on a history with no prices, uses the chart's last close when Yahoo gives no current price, and UTC when it gives no time zone. The "temporarily disabled" inject-latest block and its unused time are gone - fetch_detail: one dig() reads every path; a field Yahoo leaves out, sends as null or garbles (text or a list where a number belongs) is left out instead of failing the whole ticker or crashing. This also ends the crash on a 52-week low of 0 or less (now left out, as fetch() did) and the half-stored analyst estimates; the recommendation is stored whole or not at all. One percentage converter: a Yahoo 0 is 0.0 like every other percentage. Trailing P/E is rounded like fetch()'s, not truncated (56.7 showed as 56). 6,000 mutated responses: none raises - fetch(): the currency is the quote's own (a quote without one took the previous ticker's, or crashed on the first); a null name or price skips the ticker; the earnings date is the first announced date still to come among those Yahoo sends; percentages are floats - one query2-then-query1 fallback (_get_first) for the crumb, quotes and details - util: read_cache and read_binary_cache share one freshness check (_fresh_cache, ages from cache_age); its hit, miss and expired lines are all debug-only now (the PNG miss line printed always). getCookie reads its cache through read_cache, renewing 30 seconds early as before Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/charts.py | 8 +- lib/reports.py | 6 +- lib/util.py | 48 ++-- lib/yahoo.py | 618 ++++++++++++++-------------------------- milestone.py | 2 +- performance.py | 15 +- price.py | 11 +- tests/test_providers.py | 14 +- 8 files changed, 248 insertions(+), 474 deletions(-) diff --git a/lib/charts.py b/lib/charts.py index 95c21944..df875374 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -108,8 +108,6 @@ def history_payload(ticker, service, highlight=None, graph=False): if ticker not in market_data or 'percent_change' not in market_data[ticker]: return [f".history: no data found for ticker {ticker}"], None, None table, image = yahoo.price_history(ticker) if graph else yahoo.price_history(ticker, graph=False) - if isinstance(table, str): - raise RuntimeError(table) lines = [webhook.bold(f"{title} ({ticker_link}) performance history", service)] for interval in ('Max', '10Y', '5Y', '3Y', '1Y', 'YTD', '6M', '3M', '1M', '7D', '1D'): if interval in table: @@ -129,9 +127,7 @@ def build(kind, tickers, period, service='telegram'): if table is None: raise RuntimeError(lines[0]) if days: - percent, image = yahoo.price_history(ticker, days) - if isinstance(percent, str): - raise RuntimeError(percent) + _, image = yahoo.price_history(ticker, days) return '\n'.join(lines), image if kind == 'p': ticker = tickers[0] @@ -139,8 +135,6 @@ def build(kind, tickers, period, service='telegram'): if ticker not in market_data: raise RuntimeError(f"no data found for {ticker}") percent, image = yahoo.price_history(ticker, days, graphCache=False) - if isinstance(percent, str): - raise RuntimeError(percent) percent = float(percent.get(days, percent.get('Max', next(iter(percent.values()), 0)))) link = util.quote_link(ticker, market_data[ticker]['profile_exchange'], service, days=days) return f"{util.get_emoji(percent)} {market_data[ticker]['profile_title']} ({link}) {util.signed_percent(round(percent))}", image diff --git a/lib/reports.py b/lib/reports.py index d96753ee..89ad1712 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -748,8 +748,10 @@ def prepare_profile_payload(service, user, ticker): if payload: payload.append("") - price_history, _ = yahoo.price_history(ticker, graph=False) # the percentages only: the profile has no chart - if isinstance(price_history, str): # no chart data: the profile goes out without these lines + try: + price_history, _ = yahoo.price_history(ticker, graph=False) # the percentages only: the profile has no chart + except RuntimeError as e: # no chart data: the profile goes out without these lines + print(ticker, e, file=sys.stderr) price_history = {} for interval in ('5Y', '1Y', '1M', '1D'): if interval in price_history: diff --git a/lib/util.py b/lib/util.py index 491430df..149d8221 100644 --- a/lib/util.py +++ b/lib/util.py @@ -169,23 +169,24 @@ def cache_path(name, persist=False): """Where a cache file lives, or with persist=True a state file (in the data directory).""" return (config_var_dir if persist else config_cache_dir) + "/" + name -def read_cache(cacheFile, maxSeconds=UNSET): +def _fresh_cache(cacheFile, maxSeconds): + """The cache file's path if it was written less than maxSeconds ago (config_cache_seconds when UNSET), else None.""" if maxSeconds is UNSET: maxSeconds = config_cache_seconds - cacheFile = cache_path(cacheFile) - if os.path.isfile(cacheFile): - maxSeconds = datetime.timedelta(seconds=maxSeconds) - cacheFileMtime = datetime.datetime.fromtimestamp(os.path.getmtime(cacheFile)) - cacheFileAge = datetime.datetime.now() - cacheFileMtime - if cacheFileAge < maxSeconds: - if debug: - ttl = maxSeconds - cacheFileAge - print("cache hit:", cacheFile, "TTL:", td_to_human(ttl), file=sys.stderr) - return json.loads(Path(cacheFile).read_text(encoding="utf-8")) - print("cache expired:", cacheFile, file=sys.stderr) if debug else None + path, age = cache_path(cacheFile), cache_age(cacheFile) + if age is None: + print("cache miss:", path, file=sys.stderr) if debug else None + return None + if age >= maxSeconds: + print("cache expired:", path, file=sys.stderr) if debug else None return None - print("cache miss:", cacheFile, file=sys.stderr) if debug else None - return None + print("cache hit:", path, "TTL:", td_to_human(datetime.timedelta(seconds=maxSeconds - age)), file=sys.stderr) if debug else None + return path + +def read_cache(cacheFile, maxSeconds=UNSET): + """A cache file's JSON, or None when it is missing or older than maxSeconds.""" + path = _fresh_cache(cacheFile, maxSeconds) + return json.loads(Path(path).read_text(encoding="utf-8")) if path else None def cache_age(cacheFile): """Seconds since the cache file was written, or None when it does not exist.""" @@ -227,22 +228,9 @@ def json_load(filename, persist=False): return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None def read_binary_cache(cacheFile, maxSeconds=UNSET): - if maxSeconds is UNSET: - maxSeconds = config_cache_seconds - cacheFile = cache_path(cacheFile) - if os.path.isfile(cacheFile): - maxSeconds = datetime.timedelta(seconds=maxSeconds) - cacheFileMtime = datetime.datetime.fromtimestamp(os.path.getmtime(cacheFile)) - cacheFileAge = datetime.datetime.now() - cacheFileMtime - if cacheFileAge < maxSeconds: - if debug: - ttl = maxSeconds - cacheFileAge - print("cache hit", cacheFile, "TTL:", td_to_human(ttl), file=sys.stderr) - return io.BytesIO(Path(cacheFile).read_bytes()) - print("cache expired:", cacheFile, file=sys.stderr) if debug else None - return None - print("cache miss:", cacheFile, file=sys.stderr) - return None + """A cache file's bytes (as a file object), or None when it is missing or older than maxSeconds.""" + path = _fresh_cache(cacheFile, maxSeconds) + return io.BytesIO(Path(path).read_bytes()) if path else None def write_binary_cache(cacheFile, data): data.seek(0) diff --git a/lib/yahoo.py b/lib/yahoo.py index e02c99b3..ebbac3e0 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -13,76 +13,42 @@ from pathlib import Path _auth = threading.RLock() # Yahoo's cookie and crumb are renewed by one thread at a time: two at once could cache a crumb for the cookie the other replaced +HEADERS = {'User-Agent': 'Mozilla/5.0'} def getCookie(maxAge=1209600): # 14 days # we must always cache this, as crumb must match cookie with _auth: cacheFile = "finbot_yahoo_cookie.json" - cache = util.json_load(cacheFile) + cache = util.read_cache(cacheFile, maxAge - 30) # renewed 30 seconds before it expires if cache: - #ttl = datetime.timedelta(seconds=cache[0][2] - util.cache_age(cacheFile)) - ttl = datetime.timedelta(seconds=maxAge - util.cache_age(cacheFile)) - if ttl > datetime.timedelta(seconds=30): - cookie = cache[0][0] + '=' + cache[0][1] - print("cache hit:", cacheFile, "TTL:", util.td_to_human(ttl), file=sys.stderr) if debug else None - return cookie - else: - print("cache miss:", cacheFile, file=sys.stderr) if debug else None - - # request - cookie = None - headers = {'User-Agent': 'Mozilla/5.0'} + return cache[0][0] + '=' + cache[0][1] url = 'https://fc.yahoo.com/' # 404, but that is ok because cookie is in response header try: - r = net.get(url, headers=headers, timeout=config_http_timeout) + r = net.get(url, headers=HEADERS, timeout=config_http_timeout) except Exception as e: - print(e, file=sys.stderr) + print(net.redact(e), file=sys.stderr) r = requests.Response() # empty: falls through to the fallback cookie below if r.status_code not in {200, 404}: - print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) - - # parse - if not 'set-cookie' in r.headers: + print(r.status_code, "returned by", url, file=sys.stderr) + if 'set-cookie' not in r.headers: print("Failed to obtain Yahoo auth cookie. Returning fallback cookie", file=sys.stderr) - fallback='A3=d=AQABBBtR5mYCENFZu2wCWkA5iGGkSRGvRgkFEgEBAQGi52bwZtxM0iMA_eMAAA&S=AQAAAtG8VhxZN7aXopfvLNObtpE;' - return fallback - else: - cookie = SimpleCookie() - cookie.load(r.headers['Set-Cookie']) - cookielist = [] - for name, morsel in cookie.items(): - value = morsel.value - max_age = int(morsel.get('max-age')) - cookielist.append([name, value, max_age]) - cookie = cookielist[0][0] + '=' + str(cookielist[0][1]) - #if config_cache: - util.json_write(cacheFile, cookielist) - print("Got a new Yahoo cookie", file=sys.stderr) - rmCrumb() # we must invalidate the non-matching crumb - return cookie + return 'A3=d=AQABBBtR5mYCENFZu2wCWkA5iGGkSRGvRgkFEgEBAQGi52bwZtxM0iMA_eMAAA&S=AQAAAtG8VhxZN7aXopfvLNObtpE;' + cookie = SimpleCookie() + cookie.load(r.headers['Set-Cookie']) + cookielist = [[name, morsel.value, int(morsel.get('max-age') or 0)] for name, morsel in cookie.items()] + util.json_write(cacheFile, cookielist) + print("Got a new Yahoo cookie", file=sys.stderr) + rmCrumb() # we must invalidate the non-matching crumb + return cookielist[0][0] + '=' + cookielist[0][1] def getCrumb(seconds=1209600): # 14 days with _auth: cookie = getCookie() - if config_cache: - cacheFile = "finbot_yahoo_crumb.json" - cache = util.read_cache(cacheFile, seconds) - if cache: - return cache - headers = {'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} - yahoo_urls = ['https://query2.finance.yahoo.com/v1/test/getcrumb'] - yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) - for url in yahoo_urls: - try: - r = net.get(url, headers=headers, timeout=config_http_timeout) - except Exception as e: - print(net.redact(e), file=sys.stderr) - else: - if r.status_code != 200: - print(r.status_code, r.text.rstrip(), "returned by", url, file=sys.stderr) - continue - break - else: + cacheFile = "finbot_yahoo_crumb.json" + if config_cache and (cache := util.read_cache(cacheFile, seconds)): + return cache + r = _get_first(_hosts('/v1/test/getcrumb'), {**HEADERS, 'Cookie': cookie}) + if r is None: print("Exhausted Yahoo API attempts. Returning fallback crumb", file=sys.stderr) return 'jkQEU8yLqxs' if config_cache: @@ -96,11 +62,65 @@ def rmCrumb(): if crumb.is_file(): crumb.replace(crumb.with_name(crumb.name + '.old')) -QUOTE_FIELDS = ( # the optional fields of a v7 quote: (key, Yahoo's name, conversion) - ('percent_change_premarket', 'preMarketChangePercent', lambda v: round(v, 2)), - ('quoteType', 'quoteType', None), - ('percent_change_postmarket', 'postMarketChangePercent', lambda v: round(v, 2)), +def _hosts(path): + """One Yahoo API path on its two hosts: query2, then query1 to fall back on.""" + return ['https://query2.finance.yahoo.com' + path, 'https://query1.finance.yahoo.com' + path] + +def _get_first(urls, headers): + """The answer of the first URL that answers 200, or None when none does. Each failure is logged.""" + for url in urls: + try: + r = net.get(url, headers=headers, timeout=config_http_timeout) + except Exception as e: + print(net.redact(e), file=sys.stderr) + continue + if r.status_code == 200: + return r + print(r.status_code, r.text.strip()[:200], "returned by", net.redact(url), file=sys.stderr) + return None + +def dig(node, *path): + """node[path[0]][path[1]]..., or None where Yahoo left a part out or sent null.""" + for step in path: + try: + node = node[step] + except (KeyError, IndexError, TypeError): + return None + return node + +def _list(value): + """value if Yahoo sent a list there, else an empty one.""" + return value if isinstance(value, list) else [] + +def _percent(fraction): + """Yahoo's fraction as a percentage, to one decimal: 0.01234 -> 1.2.""" + return round(float(fraction) * 100, 1) + +def _round2(value): + return round(float(value), 2) + +def _positive(value): + """A price that must be above zero: Yahoo sends 0 for a 52-week low it does not have.""" + if value <= 0: + raise ValueError(f'{value} is not a price') + return value + +def _convert(convert, value): + """convert(value), or None for garbage where a number belongs (left out, like a missing field).""" + try: + return convert(value) if convert else value + except (TypeError, ValueError, OverflowError): + return None + +QUOTE_FIELDS = ( # (key, Yahoo's name, conversion) for the fields of a v7 quote; one Yahoo leaves out, sends as null or garbles is left out + ('percent_change', 'regularMarketChangePercent', _round2), + ('dividend', 'trailingAnnualDividendRate', lambda v: round(float(v), 1)), + ('percent_change_premarket', 'preMarketChangePercent', _round2), + ('percent_change_postmarket', 'postMarketChangePercent', _round2), ('market_cap', 'marketCap', lambda v: round(float(v))), + ('price_to_earnings_forward', 'forwardPE', round), + ('price_to_earnings_trailing', 'trailingPE', round), + ('quoteType', 'quoteType', None), ('marketState', 'marketState', None), ('profile_exchange', 'fullExchangeName', None), ('exchangeTimezoneName', 'exchangeTimezoneName', None), @@ -108,115 +128,49 @@ def rmCrumb(): ('financialCurrency', 'financialCurrency', None), ('regularMarketPreviousClose', 'regularMarketPreviousClose', None), ('fiftyTwoWeekHigh', 'fiftyTwoWeekHigh', None), + ('fiftyTwoWeekLow', 'fiftyTwoWeekLow', _positive), ) def fetch(tickers): - # DO NOT CACHE MORE THAN 5 mins - if not len(tickers): + """v7 quotes for many tickers in one request: {ticker: {...}}. Cached for 5 minutes (DO NOT CACHE LONGER). A ticker Yahoo gives + no name or price for is left out. Raises RuntimeError when Yahoo cannot be reached.""" + if not tickers: print("no tickers provided to yahoo.fetch()", file=sys.stderr) return {} tickers = sorted(set(tickers)) # de-dupe - tickers_sha256 = hashlib.sha256(str.encode("_".join(tickers))).hexdigest() - if config_cache: - cacheFile = "finbot_yahoo_" + tickers_sha256 + '.json' - cache = util.read_cache(cacheFile, 300) - if cache: - return cache - now = datetime.datetime.now().timestamp() - yahoo_output = {} + cacheFile = "finbot_yahoo_" + hashlib.sha256("_".join(tickers).encode()).hexdigest() + '.json' + if config_cache and (cache := util.read_cache(cacheFile, 300)): + return cache cookie = getCookie() crumb = getCrumb() - yahoo_urls = ['https://query2.finance.yahoo.com/v7/finance/quote?crumb=' + crumb + '&symbols=' + ','.join(tickers)] - yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) - headers = {'Content-type': 'application/json', 'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} - for url in yahoo_urls: - try: - r = net.get(url, headers=headers, timeout=config_http_timeout) - except Exception as e: - print(net.redact(e), file=sys.stderr) - else: - if r.status_code != 200: - print(r.status_code, "at", net.redact(url), file=sys.stderr) - continue - break - else: + r = _get_first(_hosts('/v7/finance/quote?crumb=' + crumb + '&symbols=' + ','.join(tickers)), {**HEADERS, 'Content-type': 'application/json', 'Cookie': cookie}) + if r is None: print("Exhausted Yahoo API attempts. Giving up", file=sys.stderr) raise RuntimeError("Yahoo API unavailable") - data = r.json() - data = data['quoteResponse'] + data = r.json()['quoteResponse'] if data['result'] is None or data['error'] is not None: print(f"{tickers}†", sep=' ', end='', flush=True, file=sys.stderr) return {} # callers iterate the result + now = datetime.datetime.now().timestamp() + yahoo_output = {} for item in data['result']: - ticker = item['symbol'] - try: - profile_title = item['longName'] - except (KeyError, IndexError): - try: - profile_title = item['shortName'] - except (KeyError, IndexError): - continue - try: - percent_change = 0 - percent_change = round(float(item['regularMarketChangePercent']), 2) - except (KeyError, IndexError): - pass - try: - currency = item['currency'] - except (KeyError, IndexError): - pass - try: - regularMarketPrice = item['regularMarketPrice'] - except (KeyError, IndexError): + title = next((name for name in (item.get('longName'), item.get('shortName')) if isinstance(name, str) and name), None) + if not title or item.get('regularMarketPrice') is None: continue - try: - dividend = round(float(item['trailingAnnualDividendRate']), 1) - except (KeyError, IndexError): - dividend = float(0) - profile_title = util.transform_title(profile_title) - yahoo_output[ticker] = { 'profile_title': profile_title, 'ticker': ticker, 'percent_change': percent_change, 'dividend': dividend, 'currency': currency, 'regularMarketPrice': regularMarketPrice } - # optional fields - quote = yahoo_output[ticker] + quote = {'profile_title': util.transform_title(title), 'ticker': item['symbol'], 'regularMarketPrice': item['regularMarketPrice'], + 'currency': item.get('currency'), 'percent_change': 0.0, 'dividend': 0.0} for key, field, convert in QUOTE_FIELDS: - if field in item: - quote[key] = convert(item[field]) if convert else item[field] - for key, field in (('price_to_earnings_forward', 'forwardPE'), ('price_to_earnings_trailing', 'trailingPE')): - try: - quote[key] = round(item[field]) - except: - pass - try: - earningsTimestamp = item['earningsTimestamp'] - earningsTimestampStart = item['earningsTimestampStart'] - earningsTimestampEnd = item['earningsTimestampEnd'] - if earningsTimestamp > now: - quote["earnings_date"] = earningsTimestamp - elif earningsTimestampStart > now: - quote["earnings_date"] = earningsTimestampStart - elif earningsTimestampEnd > now: - quote["earnings_date"] = earningsTimestampEnd - else: - quote["earnings_date"] = earningsTimestamp - except (KeyError, IndexError): - pass - try: - fiftyTwoWeekLowTemp = item['fiftyTwoWeekLow'] - if not fiftyTwoWeekLowTemp <= 0: - quote["fiftyTwoWeekLow"] = fiftyTwoWeekLowTemp - except (KeyError, IndexError): - pass + if (value := _convert(convert, item.get(field))) is not None: + quote[key] = value + stamps = [item[name] for name in ('earningsTimestamp', 'earningsTimestampStart', 'earningsTimestampEnd') if isinstance(item.get(name), (int, float))] + if stamps: # the first still to come, else the announced date + quote['earnings_date'] = next((stamp for stamp in stamps if stamp > now), stamps[0]) + yahoo_output[item['symbol']] = quote if config_cache: util.json_write(cacheFile, yahoo_output) return yahoo_output -def _percent(fraction): - """Yahoo's fraction as a percentage, to one decimal: 0.01234 -> 1.2.""" - return round(fraction * 100, 1) - -def _float_percent(fraction): - return _percent(float(fraction)) - -DETAIL_FIELDS = ( # (key, path in the quoteSummary result[, conversion[, another error that means "not there"]]), applied in this order +DETAIL_FIELDS = ( # (key, path in the quoteSummary result[, conversion]); a field Yahoo leaves out, sends as null or garbles is left out ('profile_bio', ('summaryProfile', 'longBusinessSummary')), ('profile_city', ('summaryProfile', 'city')), ('profile_country', ('summaryProfile', 'country')), @@ -229,26 +183,28 @@ def _float_percent(fraction): ('currency', ('summaryDetail', 'currency')), ('market_cap', ('summaryDetail', 'marketCap', 'raw'), int), ('dividend', ('summaryDetail', 'dividendYield', 'raw'), _percent), - ('price_to_earnings_trailing', ('summaryDetail', 'trailingPE', 'raw'), int, ValueError), + ('price_to_earnings_trailing', ('summaryDetail', 'trailingPE', 'raw'), round), ('price_to_earnings_forward', ('summaryDetail', 'forwardPE', 'raw')), + ('percent_change_year', ('summaryDetail', '52WeekChange', 'raw'), _percent), + ('price_to_sales', ('summaryDetail', 'priceToSalesTrailing12Months', 'raw')), ('ex_dividend_date', ('calendarEvents', 'exDividendDate', 'raw')), ('dividend_date', ('calendarEvents', 'DividendDate', 'raw')), ('earnings_date', ('calendarEvents', 'earnings', 'earningsDate', 0, 'raw')), ('percent_change', ('price', 'regularMarketChangePercent', 'raw'), _percent), - ('percent_change_year', ('summaryDetail', '52WeekChange', 'raw'), _float_percent), - ('percent_change_premarket', ('price', 'preMarketChangePercent', 'raw'), _float_percent), - ('percent_change_postmarket', ('price', 'postMarketChangePercent', 'raw'), _float_percent), + ('percent_change_premarket', ('price', 'preMarketChangePercent', 'raw'), _percent), + ('percent_change_postmarket', ('price', 'postMarketChangePercent', 'raw'), _percent), ('profile_exchange', ('price', 'exchangeName')), ('marketState', ('price', 'marketState')), ('quoteType', ('price', 'quoteType')), - ('regularMarketPrice', ('price', 'regularMarketPrice', 'raw')), # required: an answer without it is junk + ('regularMarketPrice', ('price', 'regularMarketPrice', 'raw')), ('regularMarketPreviousClose', ('price', 'regularMarketPreviousClose', 'raw')), ('fiftyTwoWeekHigh', ('price', 'fiftyTwoWeekHigh', 'raw')), - ('prePostMarketPrice', ('price', 'preMarketPrice', 'raw'), float, ValueError), - ('prePostMarketPrice', ('price', 'postMarketPrice', 'raw'), float, ValueError), # the after-hours price, when there is one, wins + ('fiftyTwoWeekLow', ('price', 'fiftyTwoWeekLow', 'raw'), _positive), + ('prePostMarketPrice', ('price', 'preMarketPrice', 'raw'), float), + ('prePostMarketPrice', ('price', 'postMarketPrice', 'raw'), float), # the after-hours price, when there is one, wins ('netIncomeToCommon', ('defaultKeyStatistics', 'netIncomeToCommon', 'raw')), ('sharesOutstanding', ('defaultKeyStatistics', 'sharesOutstanding', 'raw'), float), - ('short_percent', ('defaultKeyStatistics', 'shortPercentOfFloat', 'raw'), _float_percent), + ('short_percent', ('defaultKeyStatistics', 'shortPercentOfFloat', 'raw'), _percent), ('price_to_book', ('defaultKeyStatistics', 'priceToBook', 'raw')), ('earnings_growth_q', ('defaultKeyStatistics', 'earningsQuarterlyGrowth', 'raw')), ('price_to_earnings_peg', ('defaultKeyStatistics', 'pegRatio', 'raw')), @@ -259,177 +215,92 @@ def _float_percent(fraction): ('free_cashflow', ('financialData', 'freeCashflow', 'raw')), ('operating_cashflow', ('financialData', 'operatingCashflow', 'raw')), ('financialCurrency', ('financialData', 'financialCurrency')), - ('price_to_sales', ('summaryDetail', 'priceToSalesTrailing12Months', 'raw')), + ('recommend', ('financialData', 'recommendationKey')), + ('recommend_index', ('financialData', 'recommendationMean', 'raw')), + ('recommend_analysts', ('financialData', 'numberOfAnalystOpinions', 'raw')), ) +RECOMMENDATION = ('recommend', 'recommend_index', 'recommend_analysts') # all three, or none def fetch_detail(ticker, seconds=UNSET): + """quoteSummary for one ticker: {ticker: {...}}, or {} when Yahoo has nothing usable (no name or no price).""" if seconds is UNSET: seconds = config_cache_seconds now = datetime.datetime.now() cookie = getCookie() crumb = getCrumb() - base_url = 'https://query2.finance.yahoo.com/v10/finance/quoteSummary/' - #headers={'Content-type': 'application/json', 'User-Agent': 'Mozilla/5.0'} - headers={'User-Agent': 'Mozilla/5.0', 'Cookie': cookie} cacheFile = "finbot_yahoo_detail_" + ticker + '.json' data = util.read_cache(cacheFile, seconds) if config_cache else None if data: print('.', sep=' ', end='', flush=True) else: print('↓', sep=' ', end='', flush=True) - yahoo_urls = [base_url + ticker + '?modules=calendarEvents,defaultKeyStatistics,balanceSheetHistoryQuarterly,financialData,summaryProfile,summaryDetail,price,earnings,earningsTrend,insiderTransactions' + '&crumb=' + crumb] - yahoo_urls.append(yahoo_urls[0].replace('query2', 'query1')) - for url in yahoo_urls: - print(net.redact(url), file=sys.stderr) if debug else None - try: - r = net.get(url, headers=headers, timeout=config_http_timeout) - except Exception as e: - print(net.redact(e), file=sys.stderr) - else: - if r.status_code != 200: - print('x', sep=' ', end='', flush=True, file=sys.stderr) - print(r.text.rstrip(), file=sys.stderr) if debug else None - continue - break - else: + modules = 'calendarEvents,defaultKeyStatistics,balanceSheetHistoryQuarterly,financialData,summaryProfile,summaryDetail,price,earnings,earningsTrend,insiderTransactions' + r = _get_first(_hosts('/v10/finance/quoteSummary/' + ticker + '?modules=' + modules + '&crumb=' + crumb), {**HEADERS, 'Cookie': cookie}) + if r is None: print(ticker + '†', sep=' ', end='', flush=True) return {} # catches some delisted stocks like "DRNA" data = r.json() if config_cache: util.json_write(cacheFile, data) - # might be interesting: - # majorHoldersBreakdown - # netSharePurchaseActivity - # upgradeDowngradeHistory - try: - result = data['quoteSummary']['result'][0] - profile_title = result['price']['longName'] - except (KeyError, IndexError, ValueError, TypeError): # TypeError: Yahoo answers result=null for unknown symbols + result = dig(data, 'quoteSummary', 'result', 0) + if result is None: # Yahoo answers result=null for unknown symbols print(ticker + 'x', sep=' ', end='', flush=True, file=sys.stderr) return {} - if profile_title is None: - try: - profile_title = result['price']['shortName'] - except (KeyError, IndexError, ValueError): - print(ticker + '†', sep=' ', end='', flush=True, file=sys.stderr) - return {} - if profile_title is None: # catches some delisted stocks like "DUB" + title = next((name for name in (dig(result, 'price', 'longName'), dig(result, 'price', 'shortName')) if isinstance(name, str) and name), None) + if not title: # catches some delisted stocks like "DUB" print(f"{ticker}†", sep=' ', end='', flush=True, file=sys.stderr) return {} - detail = {'profile_title': util.transform_title(profile_title)} - for key, path, *how in DETAIL_FIELDS: - try: - value = result - for step in path: - value = value[step] - detail[key] = how[0](value) if how else value - except (KeyError, IndexError, *how[1:]): - if key == 'regularMarketPrice': - return {} # catches junk - try: - fiftyTwoWeekLowTemp = result['price']['fiftyTwoWeekLow']['raw'] - if not fiftyTwoWeekLowTemp <= 0: - fiftyTwoWeekLow = fiftyTwoWeekLowTemp - except (KeyError, IndexError): - pass - else: - detail['fiftyTwoWeekLow'] = fiftyTwoWeekLow - try: - quarterly = result['earnings']['financialsChart']['quarterly'] - if quarterly[-1]['earnings']['fmt'] is not None: - net_income = quarterly[-1]['earnings']['raw'] - elif quarterly[-2]['earnings']['fmt'] is not None: - net_income = quarterly[-2]['earnings']['raw'] - else: - raise TypeError('quarterly earnings is null') - except (KeyError, IndexError, TypeError): - pass - else: - detail['net_income'] = net_income - try: - recommend = result['financialData']['recommendationKey'] - recommend_index = result['financialData']['recommendationMean']['raw'] - recommend_analysts = result['financialData']['numberOfAnalystOpinions']['raw'] - except (KeyError, IndexError): - pass - else: - detail.update(recommend=recommend, recommend_index=recommend_index, recommend_analysts=recommend_analysts) - try: - result['earningsTrend']['trend'][0]['growth']['raw'] - except (KeyError, IndexError): - pass - else: - for item in result['earningsTrend']['trend']: - if item['period'] == '+1y': - try: - revenueEstimateY = item['revenueEstimate']['growth']['raw'] - earningsEstimateY = item['earningsEstimate']['growth']['raw'] - revenueAnalysts = item['revenueEstimate']['numberOfAnalysts']['raw'] - earningsAnalysts = item['earningsEstimate']['numberOfAnalysts']['raw'] - except KeyError: - break - try: - earningsEstimateY = earningsEstimateY * 100 - revenueEstimateY = revenueEstimateY * 100 - detail['revenueEstimateY'] = round(revenueEstimateY, 2) - detail['earningsEstimateY'] = round(earningsEstimateY, 2) - detail['revenueAnalysts'] = revenueAnalysts - detail['earningsAnalysts'] = earningsAnalysts - except UnboundLocalError: + detail = {'profile_title': util.transform_title(title)} + for key, path, *convert in DETAIL_FIELDS: + if (value := _convert(convert[0] if convert else None, dig(result, *path))) is not None: + detail[key] = value + if 'regularMarketPrice' not in detail: + return {} # catches junk + if not all(key in detail for key in RECOMMENDATION): + for key in RECOMMENDATION: + detail.pop(key, None) + quarterly = _list(dig(result, 'earnings', 'financialsChart', 'quarterly')) + for item in quarterly[-1:-3:-1]: # net income: the latest quarter of the last two with an earnings figure + if dig(item, 'earnings', 'fmt') is not None and dig(item, 'earnings', 'raw') is not None: + detail['net_income'] = dig(item, 'earnings', 'raw') + break + trend = _list(dig(result, 'earningsTrend', 'trend')) + if trend: + year = next((item for item in trend if dig(item, 'period') == '+1y'), None) + estimates = [dig(year, kind, field, 'raw') for kind, field in (('revenueEstimate', 'growth'), ('earningsEstimate', 'growth'), + ('revenueEstimate', 'numberOfAnalysts'), ('earningsEstimate', 'numberOfAnalysts'))] + if None in estimates: print(f'{ticker}†', sep=' ', end='', flush=True, file=sys.stderr) - try: - result['earnings']['financialsChart']['quarterly'][0]['revenue']['raw'] - except (KeyError, IndexError): + else: + detail['revenueEstimateY'], detail['earningsEstimateY'] = _percent(estimates[0]), _percent(estimates[1]) + detail['revenueAnalysts'], detail['earningsAnalysts'] = estimates[2], estimates[3] + if dig(quarterly, 0, 'revenue', 'raw') is None: print(f'{ticker}†', sep=' ', end='', flush=True, file=sys.stderr) - else: - earningsQ = [] - revenueQ = [] - earningsY = [] - revenueY = [] - for item in result['earnings']['financialsChart']['quarterly']: - if item['earnings']['fmt'] is None: # bad data - #if len(earningsQ): # may fix weirdness in dodelta() - earningsQ.append(None) - #if len(revenueQ): - revenueQ.append(None) - else: - earningsQ.append(item['earnings']['raw']) - revenueQ.append(item['revenue']['raw']) - for item in result['earnings']['financialsChart']['yearly']: - earningsY.append(item['earnings']['raw']) - revenueY.append(item['revenue']['raw']) - detail['earningsQ'] = earningsQ - detail['revenueQ'] = revenueQ - detail['earningsY'] = earningsY - detail['revenueY'] = revenueY - try: - result['insiderTransactions']['transactions'][0]['startDate']['raw'] - except (KeyError, IndexError): - pass - else: - buyTotal = 0 - buyValue = 0 - sellTotal = 0 - sellValue = 0 - for item in result['insiderTransactions']['transactions']: - dt_startDate = datetime.datetime.fromtimestamp(item['startDate']['raw']) - if dt_startDate > now - datetime.timedelta(days=90): - if 'Buy' in item['transactionText']: - buyTotal = buyTotal + item['shares']['raw'] - buyValue = buyValue + item['value']['raw'] - if 'Sale' in item['transactionText']: - sellTotal = sellTotal + item['shares']['raw'] - sellValue = sellValue + item['value']['raw'] - detail['insiderBuy'] = buyTotal - detail['insiderSell'] = sellTotal - detail['insiderBuyValue'] = buyValue - detail['insiderSellValue'] = sellValue - + else: # a quarter without an earnings figure is bad data: None for both, which doDelta marks + detail['earningsQ'] = [None if dig(item, 'earnings', 'fmt') is None else dig(item, 'earnings', 'raw') for item in quarterly] + detail['revenueQ'] = [None if dig(item, 'earnings', 'fmt') is None else dig(item, 'revenue', 'raw') for item in quarterly] + yearly = _list(dig(result, 'earnings', 'financialsChart', 'yearly')) + detail['earningsY'] = [dig(item, 'earnings', 'raw') for item in yearly] + detail['revenueY'] = [dig(item, 'revenue', 'raw') for item in yearly] + transactions = _list(dig(result, 'insiderTransactions', 'transactions')) + if transactions: # insider buying and selling over the last 3 months: shares and value + since = now - datetime.timedelta(days=90) + totals = {'Buy': [0, 0], 'Sale': [0, 0]} + for item in transactions: + start = dig(item, 'startDate', 'raw') + if not isinstance(start, (int, float)) or datetime.datetime.fromtimestamp(start) <= since: + continue + for kind, total in totals.items(): + if kind in (dig(item, 'transactionText') or ''): + total[0] += dig(item, 'shares', 'raw') or 0 + total[1] += dig(item, 'value', 'raw') or 0 + detail['insiderBuy'], detail['insiderBuyValue'] = totals['Buy'] + detail['insiderSell'], detail['insiderSellValue'] = totals['Sale'] return {ticker: dict(sorted(detail.items()))} def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True): - """({period: percent change}, chart image or None). If the chart cannot be fetched: (error string, None), which the callers - check for so that one failed ticker does not take a whole report down.""" + """({period: percent change}, chart image or None): the change over `days`, else over each standard period. Raises + RuntimeError when the history cannot be fetched or has no prices.""" if graph is UNSET: graph = config_graph if seconds is UNSET: @@ -440,37 +311,23 @@ def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True max_days = 3655 # 10 years + buffer now = datetime.datetime.now() interval = ('10Y', '5Y', '3Y', '1Y', 'YTD', '6M', '3M', '1M', '7D', '5D', '1D') - data = fetch_chart_json(ticker) - if isinstance(data, tuple): # (error string, None) - return data df = chart_json_to_df(data) stock = chart_json_to_stock_basics(data) - tz = zoneinfo.ZoneInfo(stock.get('exchangeTimezoneName')) - regularMarketTime = datetime.datetime.fromtimestamp(stock.get('regularMarketTime')).astimezone(tz).date() - - # temporarily disabled while testing new method - # inject latest - #if df['Date'].iloc[-1] == regularMarketTime: - # if df['Close'].iloc[-1] != regularMarketPrice: - # print("Updating", df['Close'].iloc[-1], "to", regularMarketPrice) - # df['Close'].loc[-1] = regularMarketPrice - #else: - # print (ticker, "inserting", regularMarketTime, "after", df['Date'].iloc[-1], file=sys.stderr) - # previous_close = df['Close'].iloc[-1] - # df.loc[len(df)] = {'Timestamp' now, 'Close': regularMarketPrice, 'Open': previous_close, 'High': None, 'Low': None, 'Volume': None, 'Time': None, 'Date': regularMarketTime} - #df = df[df.Close.notnull()] - ##df.sort_values(by='Date', inplace = True) - #df.reset_index(drop=True, inplace=True) - + closes = df['Close'].dropna() + if closes.empty: + raise RuntimeError(f"no price history for {ticker}") + price = stock.get('regularMarketPrice') or closes.iloc[-1] # the chart's last close when Yahoo gives no current price + tz = zoneinfo.ZoneInfo(stock.get('exchangeTimezoneName') or 'UTC') if days: df = df[df['Date'] >= (now - datetime.timedelta(days=days)).date()].reset_index(drop=True) + if df.empty: + raise RuntimeError(f"no price history for {ticker} in the past {days} days") past_price = df['Close'].iloc[0] - percent = (stock.get('regularMarketPrice') - past_price) / past_price * 100 - percent_dict[days] = round(percent, 2) + percent_dict[days] = round((price - past_price) / past_price * 100, 2) else: default_price = df['Close'].iloc[0] - default_percent = round((stock.get('regularMarketPrice') - default_price) / default_price * 100) + default_percent = round((price - default_price) / default_price * 100) for period in interval: if period == '1D': # against the previous session's close: the last close dated before the latest bar, in the exchange's time zone @@ -492,33 +349,29 @@ def price_history(ticker, days=None, seconds=UNSET, graph=UNSET, graphCache=True except IndexError: # the history does not go back that far percent_dict['Max'] = default_percent continue - percent = (stock.get('regularMarketPrice') - past_price) / past_price * 100 - percent_dict[period] = round(percent) + percent_dict[period] = round((price - past_price) / past_price * 100) if not graph or (days and days < 2): return percent_dict, None company_name = util.transform_title(stock.get('longName', '')) - price = stock.get('regularMarketPrice', '') if days: - title = f"{company_name} ({ticker}) {min(days, max_days)} days {util.signed_percent(percent_dict.get(days, ''))} Last: {price}" + title = f"{company_name} ({ticker}) {min(days, max_days)} days {util.signed_percent(percent_dict[days])} Last: {price}" else: label = "Max" if "Max" in percent_dict else "10Y" - title = f"{company_name} ({ticker}) {label} {util.signed_percent(percent_dict.get(label, ''))} Last: {price}" + title = f"{company_name} ({ticker}) {label} {util.signed_percent(percent_dict[label])} Last: {price}" image_cache_file = "finbot_graph_" + ticker + "_" + str(days) + ".png" - image_cache = util.read_binary_cache(image_cache_file, seconds) - if config_cache and image_cache and graphCache: - return percent_dict, image_cache + if config_cache and graphCache and (image := util.read_binary_cache(image_cache_file, seconds)): + return percent_dict, image image = util.graph(df, title, stock.get('currency')) util.write_binary_cache(image_cache_file, image) return percent_dict, image def price_series(ticker): - """Daily closes for a ticker via the same fetch/parse path as price_history(): (name, exchange, pandas Series indexed by datetime).""" + """Daily closes for a ticker via the same fetch/parse path as price_history(): (name, exchange, pandas Series indexed by datetime). + Raises RuntimeError when there is no history.""" import pandas as pd data = fetch_chart_json(ticker, full=True) - if isinstance(data, tuple): # fetch_chart_json returns (errorstring, None) on failure - raise RuntimeError(data[0]) df = chart_json_to_df(data) - stock = chart_json_to_stock_basics(data) or {} + stock = chart_json_to_stock_basics(data) df = df[df['Close'].notnull()] if df.empty: raise RuntimeError(f"no price history for {ticker}") @@ -544,109 +397,54 @@ def prefetch_history(tickers, workers=8): print("prefetch failed for", ticker, result, file=sys.stderr) def fetch_chart_json(ticker, days=3665, seconds=UNSET, full=False): - """full=True requests Yahoo's entire history (cached separately) instead of the last `days`.""" + """Yahoo's daily chart for the last `days`, or with full=True its entire history (cached separately). Raises RuntimeError when + Yahoo cannot be reached or refuses.""" if seconds is UNSET: seconds = config_cache_seconds now = datetime.datetime.now() - if config_cache: - cacheFile = "finbot_yahoo_history_" + ticker + ("_max" if full else "") + ".json" - cache = util.read_cache(cacheFile, seconds) - if cache: - return cache + cacheFile = "finbot_yahoo_history_" + ticker + ("_max" if full else "") + ".json" + if config_cache and (cache := util.read_cache(cacheFile, seconds)): + return cache crumb = getCrumb() # renews the cookie too, when due start = '0' if full else str(int((now - datetime.timedelta(days=days)).timestamp())) - end = str(int(now.timestamp())) - interval = '1d' - url = 'https://query1.finance.yahoo.com/v8/finance/chart/' - url = url + ticker + '?period1=' + start + '&period2=' + end + '&interval=' + interval - url = url + '&crumb=' + crumb - headers={'Content-type': 'application/json', 'User-Agent': 'Mozilla/5.0'} + url = f'https://query1.finance.yahoo.com/v8/finance/chart/{ticker}?period1={start}&period2={int(now.timestamp())}&interval=1d&crumb={crumb}' print("Fetching", net.redact(url), file=sys.stderr) if debug else None try: - r = net.get(url, headers=headers, timeout=config_http_timeout) + r = net.get(url, headers={**HEADERS, 'Content-type': 'application/json'}, timeout=config_http_timeout) except Exception as e: - errorstring = net.redact(f"General failure for {ticker} at {url}: {e}") - print(errorstring, file=sys.stderr) - return errorstring, None - if r.status_code == 200: - print('↓', sep=' ', end='', flush=True) - else: - errorstring = net.redact(f"{r.status_code} error for {ticker} at {url}") - print(errorstring, file=sys.stderr) - return errorstring, None + raise RuntimeError(f"Yahoo unreachable for {ticker}: {net.redact(e)}") from e + if r.status_code != 200: + raise RuntimeError(f"{r.status_code} error for {ticker} from Yahoo's chart API") + print('↓', sep=' ', end='', flush=True) data = r.json() if config_cache: util.json_write(cacheFile, data) return data def chart_json_to_df(chart_json): + """The chart's daily bars as a DataFrame: Timestamp, Open, High, Low, Close, Volume and the Date.""" import pandas as pd result = chart_json['chart']['result'][0] - timestamp = result['timestamp'] - quote_data = result['indicators']['quote'][0] - - # Create a dictionary with 'Timestamp' and all available quote data - data = {'Timestamp': timestamp} - data.update(quote_data) - - # Create DataFrame - df = pd.DataFrame(data) - - #timestamp = [datetime.datetime.fromtimestamp(ts) for ts in timestamp] + df = pd.DataFrame({'Timestamp': result['timestamp'], **result['indicators']['quote'][0]}) df['Date'] = pd.to_datetime(df['Timestamp'], unit='s').dt.date - - - # Rename columns to capitalize first letter df.columns = [col.capitalize() for col in df.columns] - return df def chart_json_to_stock_basics(chart_json): - """ - extracts stock basics from chart json to save calling fetch() - { - "currency": "USD", - "symbol": "AAPL", - "exchangeName": "NMS", - "fullExchangeName": "NasdaqGS", - "instrumentType": "EQUITY", - "firstTradeDate": 345479400, - "regularMarketTime": 1726862403, - "hasPrePostMarketData": true, - "gmtoffset": -14400, - "timezone": "EDT", - "exchangeTimezoneName": "America/New_York", - "regularMarketPrice": 228.2, - "fiftyTwoWeekHigh": 233.09, - "fiftyTwoWeekLow": 227.62, - "regularMarketDayHigh": 233.09, - "regularMarketDayLow": 227.62, - "regularMarketVolume": 287134033, - "longName": "Apple Inc.", - "shortName": "Apple Inc.", - "chartPreviousClose": 224.53, - "priceHint": 2, - } - """ - for a in chart_json.values(): - for b in a.values(): - if isinstance(b, list): - for c in b: - if 'currency' in c['meta']: - return c['meta'] + """The chart's meta, which saves a call to fetch(): currency, symbol, exchangeName and fullExchangeName, exchangeTimezoneName, + regularMarketPrice and regularMarketTime, longName and shortName, fiftyTwoWeekHigh and Low, and more. {} when there is none.""" + return dig(chart_json, 'chart', 'result', 0, 'meta') or {} def historic_high(ticker): - data = fetch_chart_json(ticker) - df = chart_json_to_df(data) - df = df[df.High.notnull()] - df.drop(df.index[:1], inplace=True) - df.reset_index(drop=True, inplace=True) - if df.empty: # e.g. total-return indices such as ^SPTTGD carry no High/Low data - return None + """(highest high, lowest low) of the ticker's history. Raises RuntimeError when it has none (e.g. total-return indices such + as ^SPTTGD carry no High/Low data).""" + df = chart_json_to_df(fetch_chart_json(ticker)) + df = df[df.High.notnull()].iloc[1:].reset_index(drop=True) + if df.empty: + raise RuntimeError(f"no high/low history for {ticker}") highrow = df.iloc[df['High'].argmax()] lowrow = df.iloc[df['Low'].argmin()] if debug: print(ticker, "High", highrow['Date'], round(highrow['High'], 2), file=sys.stderr) print(ticker, "Low", lowrow['Date'], round(lowrow['Low'], 2), file=sys.stderr) return round(highrow['High'], 2), round(lowrow['Low'], 2) - diff --git a/milestone.py b/milestone.py index 862cf87c..c5e8d016 100755 --- a/milestone.py +++ b/milestone.py @@ -37,7 +37,7 @@ def prepare_milestone_payload(service, market_data): records[ticker]['low'] = float() try: oldhigh, oldlow = yahoo.historic_high(ticker) # first run to prime cache file - except (KeyError, TypeError, ValueError): + except (RuntimeError, KeyError, IndexError, ValueError): del records[ticker] # no usable history: don't record zeros, which would fake a new high next run continue try: diff --git a/performance.py b/performance.py index d49407cd..727c0b20 100755 --- a/performance.py +++ b/performance.py @@ -23,17 +23,12 @@ def get_emoji(percent): return emoji def stock_performance(ticker, market, text): try: - percent, graph = yahoo.price_history(ticker, days=past_days, graph=False) - except Exception as e: - percent = f"error: {e}" - if isinstance(percent, str): # no history for this benchmark: say so in its line, the rest of the report goes out - print(ticker, percent, file=sys.stderr) + percent = yahoo.price_history(ticker, days=past_days, graph=False)[0][past_days] + except Exception as e: # no history for this benchmark: say so in its line, the rest of the report goes out + print(ticker, e, file=sys.stderr) return f"⚠️ {text}: no price history", None - else: - percent = percent[past_days] - emoji = get_emoji(percent) - link = util.finance_link(ticker, market, service=service, days=past_days, brief=True, text=text) - return f"{emoji} {link} {util.signed_percent(percent, 2)}", percent + link = util.finance_link(ticker, market, service=service, days=past_days, brief=True, text=text) + return f"{util.get_emoji(percent)} {link} {util.signed_percent(percent, 2)}", percent def prepare_performance_payload(service, performance, portfolios): payload = [] lines = [] # (percent, text), sorted into payload below diff --git a/price.py b/price.py index 85c49976..d3db17df 100755 --- a/price.py +++ b/price.py @@ -62,14 +62,13 @@ def prepare_price_payload(service, market_data, threshold, _rows_out=None): continue # wishlist items will come here # not a holding (e.g. watchlist): use Yahoo history instead, summarised once in the main section below - if specific_stock: + if specific_stock: # the one ticker asked for: no history is reported in one line percent, graph = yahoo.price_history(ticker, days, graph=config_graph and webhook.takes_charts(service, interactive), graphCache=False) - if isinstance(percent, str): - raise RuntimeError(percent) # the one ticker asked for has no history: reported in one line else: - percent, graph = yahoo.price_history(ticker, days, graph=False) - if isinstance(percent, str): # no history for this ticker: leave it out of the list - print(ticker, percent, file=sys.stderr) + try: + percent, _ = yahoo.price_history(ticker, days, graph=False) + except RuntimeError as e: # no history for this ticker: leave it out of the list + print(ticker, e, file=sys.stderr) continue try: percent = percent[days] diff --git a/tests/test_providers.py b/tests/test_providers.py index 681026b4..afdd71e7 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -140,13 +140,12 @@ def test_real_chart_to_dataframe_and_basics(self): basics = yahoo.chart_json_to_stock_basics(chart) self.assertEqual((basics['symbol'], basics['currency']), ('CRWD', 'USD')) - def test_fetch_chart_json_failures_return_error_strings(self): + def test_fetch_chart_json_failures_raise(self): for failure in (Resp(status=404), ConnectionError('down')): with self.subTest(failure=failure): self.http(yahoo, failure) - result = yahoo.fetch_chart_json('AAPL', full=True) - self.assertIsInstance(result, tuple) - self.assertIsNone(result[1]) + with self.assertRaises(RuntimeError): + yahoo.fetch_chart_json('AAPL', full=True) @staticmethod def chart(closes, failing=()): @@ -165,14 +164,13 @@ def test_1d_is_the_change_since_the_previous_close(self): self.assertEqual(table['1D'], -3) self.assertGreater(table['5D'], 10) # the week, not the day - def test_a_failed_chart_fetch_is_handed_back_not_raised(self): + def test_a_failed_chart_fetch_raises(self): self.patch(yahoo, 'price_history', REAL[('lib.yahoo', 'price_history')]) for failure in (Resp(status=429), ConnectionError('down')): with self.subTest(failure=failure): self.http(yahoo, failure) - percent, image = yahoo.price_history('AAPL', 7, graph=False) - self.assertIsInstance(percent, str) - self.assertIsNone(image) + with self.assertRaises(RuntimeError): + yahoo.price_history('AAPL', 7, graph=False) def test_one_failed_ticker_does_not_take_the_report_down(self): self.patch(yahoo, 'price_history', REAL[('lib.yahoo', 'price_history')]) From 9495b1bacd4b05ed172fd1a7308093df40050202 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:11:19 +0000 Subject: [PATCH 10/25] Reports go where they should from one place; price and performance split into gather, report and delivery Destinations - webhook.default_chats() says where a scheduled report goes: Telegram's telegramChatID (left out without one), and Slack and Discord through their incoming webhooks, with no chat id. It raises "nowhere to post" when finbot.env names none. report_error, price, performance, shorts, cal, trades, rating, milestone, reminder, podcasts and bot_send_stdin all use it, and webhook.destinations(interactive, service, chat_id) adds the reply-to-the-asker case - this fixes the crash with slack_webhook set, no slackBotToken and telegramChatID set: price, shorts, rating and milestone sent Telegram's chat id to Slack's webhook, write() then needed the bot token and raised TypeError before Telegram got its copy. A Slack reply without slackBotToken now logs why and sends nothing instead of raising - the ten `if not webhooks: sys.exit(1)` guards are gone: in a chat reply they killed the worker thread outright (SystemExit gets past except Exception), e.g. on a Slack bot without slack_webhook - webhook.send(service, url, chat, payload, image, buttons) sends the chart with the text as caption, or the text: the scripts no longer each choose between sendPhoto and payload_wrapper - cal's scheduled run uses its days argument (it used config_future_days, which is the default); shortman.fetch hands the quotes back unchanged when Short Man is down (it returned {} and the US short interest from Yahoo went too); shorts, cal and rating fetch Yahoo details 8 at a time and skip a ticker whose fetch failed; trades finds its portfolio with sharesight.find_portfolio; milestone loses parameters nothing passes (it never had an interactive path) price.py and performance.py - gather() fetches the data once; report(service, ...) builds (payload, chart) for one service; lambda_handler sends it. The chart period buttons call report(): return_result and _rows_out go - price: the volatile-stock second pass recursed with the demotion still on, so it re-failed the very tickers it was meant to add back (and could add a second heading). Now, as its comment intended, a leveraged or volatile ticker needs its higher bar (trigger_bar) only to send a scheduled report on its own; once one goes out, every ticker past the threshold is listed, whatever their order. Interactive reports are unchanged (identical on 39 cases), as are scheduled ones without demotion - performance: benchmark() and report() at module level; util.get_emoji instead of a nested copy Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot_send_stdin.py | 9 +- cal.py | 33 +--- lib/charts.py | 5 +- lib/shortman.py | 8 +- lib/webhook.py | 37 ++-- milestone.py | 25 +-- performance.py | 155 +++++++---------- podcasts.py | 9 +- price.py | 394 ++++++++++++++++-------------------------- rating.py | 33 +--- reminder.py | 11 +- shorts.py | 37 +--- tests/test_reports.py | 26 ++- trades.py | 45 +---- 14 files changed, 308 insertions(+), 519 deletions(-) diff --git a/bot_send_stdin.py b/bot_send_stdin.py index 944efdef..67c5fd8c 100755 --- a/bot_send_stdin.py +++ b/bot_send_stdin.py @@ -11,11 +11,6 @@ payload = [line.strip() for line in sys.stdin if line.strip()] -if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) -for service, url in webhooks.items(): - if service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + config_telegramChatID - webhook.payload_wrapper(service, url, payload) +for service, url, chat in webhook.default_chats(): + webhook.send(service, url, chat, payload) diff --git a/cal.py b/cal.py index bfcc353f..8dac2b9d 100755 --- a/cal.py +++ b/cal.py @@ -113,34 +113,17 @@ def pin_if_earnings(service, results): if earnings: market_data = yahoo.fetch(tickers) - elif dividend: + else: # ex-dividend dates are in Yahoo's details: fetched together market_data = {} - for ticker in tickers: - try: - market_data = market_data | yahoo.fetch_detail(ticker) - except TypeError: - pass + for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): + if isinstance(detail, Exception): + print(ticker, detail, file=sys.stderr) + else: + market_data |= detail print("") - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - if interactive: - payload = prepare_payload(service, market_data, days) - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + str(chat_id) - results = webhook.payload_wrapper(service, url, payload, chat_id, message_id) - pin_if_earnings(service, results) - else: - for service, url in webhooks.items(): - payload = prepare_payload(service, market_data, config_future_days) - if service == "telegram": - url = url + "sendMessage?chat_id=" + config_telegramChatID - results = webhook.payload_wrapper(service, url, payload) - pin_if_earnings(service, results) + for service, url, chat in webhook.destinations(interactive, service, chat_id): + pin_if_earnings(service, webhook.send(service, url, chat, prepare_payload(service, market_data, days), message_id=message_id)) # make google cloud happy return True diff --git a/lib/charts.py b/lib/charts.py index df875374..72a9f4ee 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -146,15 +146,14 @@ def build(kind, tickers, period, service='telegram'): return '\n'.join(caption), image if kind == 'f': # performance: tickers = [portfolio name or ''] import performance - payload, image = performance.lambda_handler(past_days=days or MAX_DAYS, service=service, portfolio_select=tickers[0] or None, interactive=True, return_result=True) + payload, image = performance.report(service, days or MAX_DAYS, tickers[0] or None, interactive=True) if not image: raise RuntimeError(payload[0] if payload else 'no performance data') return '\n'.join(payload), image if kind == 'l': # price list: tickers = [threshold, top] import price - threshold, top = float(tickers[0]), int(tickers[1]) span = {'interday': True} if period == 'd' else {'days': days or MAX_DAYS} # 'd' is today's moves: the command's default view, not a 1-day lookback - payload, image = price.lambda_handler(threshold=threshold, service=service, interactive=True, top=top or None, return_result=True, **span) + payload, image = price.report(service, threshold=float(tickers[0]), interactive=True, top=int(tickers[1]) or None, **span) if not image: raise RuntimeError(payload[0] if payload else 'no price data') return '\n'.join(payload), image diff --git a/lib/shortman.py b/lib/shortman.py index d484fe9b..46ca8727 100644 --- a/lib/shortman.py +++ b/lib/shortman.py @@ -8,14 +8,14 @@ def fetch(market_data): url = 'https://www.shortman.com.au/downloadeddata/latest.csv' try: r = net.get(url, timeout=config_http_timeout) - except: - print("Failure fetching", url, file=sys.stderr) - return {} + except Exception as e: + print("Failure fetching", url, net.redact(e), file=sys.stderr) + return market_data # the ASX short interest is missing, the rest of the report is not if r.status_code == 200: print(r.status_code, "success shortman") else: print(r.status_code, "error communicating with", url, file=sys.stderr) - return {} + return market_data csv = r.content.decode('utf-8') csv = csv.split('\r\n') csv.pop(0) # remove header diff --git a/lib/webhook.py b/lib/webhook.py index 9b687e77..b1dfc530 100644 --- a/lib/webhook.py +++ b/lib/webhook.py @@ -17,6 +17,9 @@ def write(service, url, payload_string, chat_id=None, message_id=None, reply_mar headers['unfurl_links'] = 'false' headers['unfurl_media'] = 'false' if chat_id: + if not config_slackBotToken: + print("Slack: replying in a channel needs slackBotToken in finbot.env", file=sys.stderr) + return None headers['Authorization'] = 'Bearer ' + config_slackBotToken payload['channel'] = chat_id if message_id: @@ -53,6 +56,26 @@ def chat_url(service, chat_id): return webhooks['telegram'] + 'sendMessage?chat_id=' + str(chat_id) return webhooks.get(service) +def default_chats(): + """Where a scheduled report goes: [(service, url, chat id)]. Telegram posts into telegramChatID (left out without one); Slack and + Discord post through their incoming webhooks, which take no chat id. Raises RuntimeError when there is nowhere to post.""" + chats = [(service, chat_url(service, config_telegramChatID), config_telegramChatID) if service == 'telegram' else (service, url, None) + for service, url in webhooks.items() if service != 'telegram' or config_telegramChatID] + if not chats: + raise RuntimeError('nowhere to post: finbot.env sets no telegramChatID, slack_webhook or discord_webhook') + return chats + +def destinations(interactive, service, chat_id): + """Where a report goes: back to the chat that asked for it (interactive), else to the default chats.""" + return [(service, chat_url(service, chat_id), chat_id)] if interactive else default_chats() + +def send(service, url, chat_id, payload, image=None, reply_markup=None, message_id=None): + """A report: its chart with the text as caption when there is one (sendPhoto falls back to the text where a chart cannot go), + else the text. reply_markup is the chart's buttons. Returns what payload_wrapper or sendPhoto did.""" + if image: + return sendPhoto(chat_id, image, '\n'.join(payload), service, message_id=message_id, reply_markup=reply_markup) + return payload_wrapper(service, url, payload, chat_id, message_id) + def error_line(e, context=None, limit=200): """One-line, secret-free summary of an exception (or string).""" if isinstance(e, BaseException): @@ -73,19 +96,7 @@ def report_error(e, service=None, chat_id=None, context=None): print(line, file=sys.stderr) if isinstance(e, BaseException): print(net.redact(''.join(traceback.format_exception(e))), end='', file=sys.stderr) # request errors carry their URL - if service and chat_id: - targets = [(service, chat_url(service, chat_id))] - chat = chat_id - else: - targets = [] - for svc, url in webhooks.items(): - if svc == 'telegram': - if not config_telegramChatID: - continue - url = chat_url(svc, config_telegramChatID) - targets.append((svc, url)) - chat = None - for svc, url in targets: + for svc, url, chat in [(service, chat_url(service, chat_id), chat_id)] if service and chat_id else default_chats(): if url: try: write(svc, url, line if svc == 'discord' else html.escape(line, quote=False), chat) diff --git a/milestone.py b/milestone.py index c5e8d016..25860026 100755 --- a/milestone.py +++ b/milestone.py @@ -12,7 +12,7 @@ # - need to handle stock splits: flush cache if we see a split in the data # https://finance.yahoo.com/quote/TSLA/history?period1=0&period2=1707609600&filter=split -def lambda_handler(chat_id=config_telegramChatID, specific_stock=None, service=None, interactive=False): +def lambda_handler(): def prepare_milestone_payload(service, market_data): emoji = '⭐' payload = [] @@ -131,24 +131,11 @@ def prepare_milestone_payload(service, market_data): # MAIN # - market_data = {} - if specific_stock: - ticker = util.transform_to_yahoo(specific_stock) - tickers = [ticker] - else: - tickers = util.get_holdings_and_watchlist() - market_data = yahoo.fetch(tickers) - - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - else: - for service, url in webhooks.items(): - payload, records = prepare_milestone_payload(service, market_data) - if service == "telegram": - url = url + "sendMessage?chat_id=" + str(chat_id) - webhook.payload_wrapper(service, url, payload, chat_id) + tickers = util.get_holdings_and_watchlist() + market_data = yahoo.fetch(tickers) + for service, url, chat in webhook.default_chats(): + payload, records = prepare_milestone_payload(service, market_data) + webhook.send(service, url, chat, payload) if records: util.json_write('finbot_milestone.json', records, persist=True) diff --git a/performance.py b/performance.py index 727c0b20..20cba516 100755 --- a/performance.py +++ b/performance.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import sys -import json from lib.config import * from lib import sharesight @@ -10,108 +9,70 @@ from lib import yahoo from lib import charts -def lambda_handler(chat_id=config_telegramChatID, past_days=UNSET, service=None, user='', portfolio_select=None, message_id=None, interactive=False, return_result=False): - if past_days is UNSET: - past_days = config_past_days - def get_emoji(percent): - if percent < 0: - emoji = "🔻" - elif percent > 0: - emoji = '🔼' - else: - emoji = "▪️" - return emoji - def stock_performance(ticker, market, text): - try: - percent = yahoo.price_history(ticker, days=past_days, graph=False)[0][past_days] - except Exception as e: # no history for this benchmark: say so in its line, the rest of the report goes out - print(ticker, e, file=sys.stderr) - return f"⚠️ {text}: no price history", None - link = util.finance_link(ticker, market, service=service, days=past_days, brief=True, text=text) - return f"{util.get_emoji(percent)} {link} {util.signed_percent(percent, 2)}", percent - def prepare_performance_payload(service, performance, portfolios): - payload = [] - lines = [] # (percent, text), sorted into payload below - chart_rows = [] - graph = False - for portfolio_id in performance: - portfolio_url = "https://portfolio.sharesight.com/portfolios/" + str(portfolio_id) - portfolio_name = performance[portfolio_id]['report']['holdings'][0]['portfolio']['name'] - portfolio_link = util.link(portfolio_url, portfolio_name, service) - currency_percent = float(performance[portfolio_id]['report']['currency_gain_percent']) - percent = float(performance[portfolio_id]['report']['capital_gain_percent']) - total_percent = float(performance[portfolio_id]['report']['total_gain_percent']) - emoji = get_emoji(percent) - lines.append((percent, f"{emoji} {portfolio_link} {util.signed_percent(percent, 2)}")) - chart_rows.append((portfolio_name, percent)) - if len(lines): - for ticker, market, text in (('SPY', 'NYSEARCA', 'S&P 500'), ('QQQ', 'NasdaqGM', 'NASDAQ 100')): - line, benchmark_percent = stock_performance(ticker, market, text) - if benchmark_percent is not None: - lines.append((float(benchmark_percent), line)) - chart_rows.append((text, float(benchmark_percent))) - else: - lines.append((float('-inf'), line)) # an error message sorts last - lines.sort(key=lambda item: item[0], reverse=True) - payload = [line for _, line in lines] - period = util.days_english(past_days) - message = webhook.bold(f"Performance over {period}", service) - payload.insert(0, message) - if chart_rows and webhook.takes_charts(service, interactive): - subtitle = f"% change {'' if period == 'today' else 'over '}{period}" - chart_rows.sort(key=lambda row: row[1], reverse=True) # best first - graph = util.rows_chart(chart_rows, "Performance", subtitle, value_fmt=lambda v: util.signed_percent(v, 2)) - return payload, graph +BENCHMARKS = (('SPY', 'NYSEARCA', 'S&P 500'), ('QQQ', 'NasdaqGM', 'NASDAQ 100')) - # MAIN # +def gather(past_days, portfolio_select=None): + """{portfolio id: Sharesight's performance report over past_days} of every portfolio, or of the one named (in any case). Raises + RuntimeError when there is none.""" portfolios = sharesight.get_portfolios() - performance = {} - if portfolio_select: - portfoliosLower = {k.lower():v for k,v in portfolios.items()} - if portfolio_select.lower() in portfoliosLower: - portfolio_id = portfoliosLower[portfolio_select.lower()] # any-case input - performance[portfolio_id] = sharesight.get_performance(portfolio_id, past_days) - else: - for portfolio, portfolio_id in portfolios.items(): - performance[portfolio_id] = sharesight.get_performance(portfolio_id, past_days) - - # Prep and send payloads - if not len(performance): + found = sharesight.find_portfolio(portfolios, portfolio_select) + portfolios = dict([found]) if found else {} + performance = {portfolio_id: sharesight.get_performance(portfolio_id, past_days) for portfolio_id in portfolios.values()} + if not performance: print("Error: no Sharesight data found", file=sys.stderr) raise RuntimeError("no Sharesight data found") - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - if interactive: - payload, graph = prepare_performance_payload(service, performance, portfolios) - if return_result: # period buttons rebuild the report without sending it - return payload, graph - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + str(chat_id) - if graph: - markup = charts.keyboard('f', [portfolio_select or ''], charts.period_for_days(past_days)) if service == 'telegram' else None - webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service, reply_markup=markup) - else: - webhook.payload_wrapper(service, url, payload, chat_id, message_id) - else: - for service, url in webhooks.items(): - payload, graph = prepare_performance_payload(service, performance, portfolios) - if service == "telegram": - chat_id = config_telegramChatID - url = url + "sendMessage?chat_id=" + str(chat_id) - else: - chat_id = None - if graph: - markup = charts.keyboard('f', [''], charts.period_for_days(past_days)) if service == 'telegram' else None - webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service, reply_markup=markup) - continue - webhook.payload_wrapper(service, url, payload, chat_id) + return performance + +def benchmark(ticker, market, text, past_days, service): + """(report line, percent) of a benchmark over past_days; percent is None, and the line says so, when it has no history.""" + try: + percent = yahoo.price_history(ticker, days=past_days, graph=False)[0][past_days] + except Exception as e: # the rest of the report still goes out + print(ticker, e, file=sys.stderr) + return f"⚠️ {text}: no price history", None + link = util.finance_link(ticker, market, service=service, days=past_days, brief=True, text=text) + return f"{util.get_emoji(percent)} {link} {util.signed_percent(percent, 2)}", percent - # make google cloud happy - return True +def report(service, past_days=UNSET, portfolio_select=None, interactive=False, performance=None): + """(payload, chart) of the performance report for one service: each portfolio and the benchmarks, best first. performance: + gather()'s, fetched when not given.""" + if past_days is UNSET: + past_days = config_past_days + if performance is None: + performance = gather(past_days, portfolio_select) + lines = [] # (percent, text), sorted into the payload below + chart_rows = [] + for portfolio_id, data in performance.items(): + name = data['report']['holdings'][0]['portfolio']['name'] + link = util.link("https://portfolio.sharesight.com/portfolios/" + str(portfolio_id), name, service) + percent = float(data['report']['capital_gain_percent']) + lines.append((percent, f"{util.get_emoji(percent)} {link} {util.signed_percent(percent, 2)}")) + chart_rows.append((name, percent)) + if not lines: + return [], None + for ticker, market, text in BENCHMARKS: + line, percent = benchmark(ticker, market, text, past_days, service) + lines.append((float('-inf') if percent is None else float(percent), line)) # an error line sorts last + if percent is not None: + chart_rows.append((text, float(percent))) + lines.sort(key=lambda item: item[0], reverse=True) + period = util.days_english(past_days) + payload = [webhook.bold(f"Performance over {period}", service)] + [line for _, line in lines] + graph = None + if webhook.takes_charts(service, interactive): + chart_rows.sort(key=lambda row: row[1], reverse=True) # best first + graph = util.rows_chart(chart_rows, "Performance", f"% change {'' if period == 'today' else 'over '}{period}", value_fmt=lambda v: util.signed_percent(v, 2)) + return payload, graph + +def lambda_handler(chat_id=config_telegramChatID, past_days=UNSET, service=None, user='', portfolio_select=None, message_id=None, interactive=False): + if past_days is UNSET: + past_days = config_past_days + performance = gather(past_days, portfolio_select) + for service, url, chat in webhook.destinations(interactive, service, chat_id): + payload, graph = report(service, past_days, portfolio_select, interactive, performance) + buttons = charts.keyboard('f', [portfolio_select or ''], charts.period_for_days(past_days)) if graph and service == 'telegram' else None + webhook.send(service, url, chat, payload, graph, reply_markup=buttons, message_id=message_id) def job(argv): """Arguments of a scheduled run: (label, lambda_handler kwargs). Raises ValueError with the usage on a bad argument.""" diff --git a/podcasts.py b/podcasts.py index 5279a6f5..56d7054c 100755 --- a/podcasts.py +++ b/podcasts.py @@ -158,12 +158,8 @@ def lambda_handler(feed_url): seen = sorted(set(seen)) if seen != seen_earlier: - if not webhooks: - raise KeyError("no services enabled in finbot.env") - for service, url in webhooks.items(): + for service, url, chat in webhook.default_chats(): payload = [] - if service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + config_telegramChatID for ep in new_episodes: podcast_name = ep['podcast'] title = ep['title'].removeprefix(podcast_name).lstrip(string.punctuation + " ") @@ -171,8 +167,7 @@ def lambda_handler(feed_url): emoji = "📣" duration_str = f"{ep['duration']}" if ep['duration'] else "" payload.append(f"{emoji} {podcast_name}: {link} {duration_str} ({ep['ago']})") - webhook.payload_wrapper(service, url, payload) - #print(service, json.dumps(payload, indent=4)) + webhook.send(service, url, chat, payload) save_seen(seen, seen_file) # do this last so we pick up these episodes next run if we error else: print("no new episodes found") diff --git a/price.py b/price.py index d3db17df..b5344028 100755 --- a/price.py +++ b/price.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import json, re, sys, os +import sys import datetime from lib.config import * @@ -8,260 +8,166 @@ from lib import webhook from lib import util from lib import yahoo -from lib import telegram from lib import charts -class Row(list): - """A report line that remembers its ticker, so the numbers can be charted.""" - def __init__(self, ticker, parts): - super().__init__(parts) - self.ticker = ticker +def gather(specific_stock=None, days=None): + """(tickers, market data) for a price report: Yahoo's quotes; over `days` each holding's Sharesight figure, and for the tickers + Sharesight has none for (the watchlist) their Yahoo history, fetched together up front.""" + tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() + market_data = yahoo.fetch(tickers) + # Sharesight reports a holding's performance only for the time it was held: held for 1Y and asked for 5Y, it gives 1Y + if days and (not specific_stock or config_performance_use_sharesight): + for data in sharesight.get_performance_wrapper(days).values(): + for holding in data['report']['holdings']: + ticker = util.transform_to_yahoo(holding['instrument']['code'], holding['instrument']['market_code']) + if ticker in market_data: + market_data[ticker]['percent_change_period'] = float(holding['capital_gain_percent']) + elif ticker in tickers: + print("Notice: price.py:", ticker, "has no data", file=sys.stderr) + if not specific_stock and not config_performance_use_sharesight: + yahoo.prefetch_history([t for t in market_data if 'percent_change_period' not in market_data[t]]) + return tickers, market_data -def lambda_handler(chat_id=config_telegramChatID, threshold=UNSET, service=None, user='', specific_stock=None, interactive=False, midsession=False, premarket=False, interday=False, days=None, close=False, top=None, return_result=False): +def trigger_bar(ticker, quote, threshold): + """The move this ticker needs before it alone sends a scheduled report: more for leveraged ETFs (demote_leveraged) and volatile + small caps (demote_volatile), else the threshold itself.""" + title = quote['profile_title'].lower() + if config_demote_leveraged and ('2x' in title or '3x' in title): + return threshold * 1.3 + cap = quote.get('market_cap') + if config_demote_volatile and cap is not None and cap < 1e9: + if cap < 1e7: + return threshold * config_volatility_multiplier * 1.3 + if cap < 1.5e8: + return threshold * config_volatility_multiplier + beta = yahoo.fetch_detail(ticker).get(ticker, {}).get('beta') + if beta is None or beta > 1.5: + return threshold * config_volatility_multiplier + return threshold + +def report(service, threshold=UNSET, user='', specific_stock=None, interactive=False, midsession=False, premarket=False, interday=False, + days=None, close=False, top=None, market=None): + """(payload, chart) of the price report for one service: the moves of `threshold` percent or more (or the top and bottom `top`), + today's, over `days`, at the close, mid-session or pre-market. A scheduled report goes out only when one ticker passes its + trigger_bar; it then lists every ticker past the threshold. market: gather()'s (tickers, market data), fetched when not given.""" if threshold is UNSET: threshold = config_price_percent - def prepare_price_payload(service, market_data, threshold, _rows_out=None): - payload = [] - graph = False - marketStates = [] - skipped_volatile = [] - exchange_set = set() - latest = None # date of the freshest price, for the heading - multiplier = config_volatility_multiplier - for ticker in market_data: - marketState = market_data[ticker]['marketState'] - marketStates.append(marketState) - regularMarketTime = datetime.datetime.fromtimestamp(market_data[ticker]['regularMarketTime']) - if latest is None or regularMarketTime.date() > latest: - latest = regularMarketTime.date() - now = datetime.datetime.now() - if midsession and marketState != "REGULAR": - # skip stocks not in session - # Possible market states: PREPRE PRE REGULAR POST POSTPOST CLOSED - continue - if interday and not interactive and now - regularMarketTime > datetime.timedelta(days=1): - # avoid repeating on market holidays - print(ticker + '⏭', sep='', end=' ', flush=True) + tickers, market_data = market or gather(specific_stock, days) + now = datetime.datetime.now() + rows = [] # (ticker, percent, report line without its percent) + exchanges = set() + states = [] + latest = None # date of the freshest price, for the heading + graph = None + triggered = interactive + for ticker, quote in market_data.items(): + state = quote['marketState'] + states.append(state) + quoted = datetime.datetime.fromtimestamp(quote['regularMarketTime']) + latest = max(latest or quoted.date(), quoted.date()) + if midsession and state != 'REGULAR': + continue # not in session (states: PREPRE PRE REGULAR POST POSTPOST CLOSED) + if interday and not interactive and now - quoted > datetime.timedelta(days=1): + print(ticker + '⏭', sep='', end=' ', flush=True) # a market holiday: no repeat of the last session's moves + continue + if close and not interactive and (now - quoted > datetime.timedelta(hours=1.5) or state in ('PREPRE', 'PRE', 'REGULAR')): + print(ticker + '⏭', sep='', end=' ', flush=True) # its market did not just close + continue + if premarket: + percent = quote.get('percent_change_premarket', quote.get('percent_change_postmarket')) + if percent is None: continue - if close and not interactive and (now - regularMarketTime > datetime.timedelta(hours=1.5) or marketState in ('PREPRE', 'PRE', 'REGULAR')): - # avoid markets that didn't recently close - print(ticker + '⏭', sep='', end=' ', flush=True) - continue - if premarket: - if 'percent_change_premarket' in market_data[ticker]: - percent = market_data[ticker]['percent_change_premarket'] - elif 'percent_change_postmarket' in market_data[ticker]: - percent = market_data[ticker]['percent_change_postmarket'] - else: + elif days: + percent = quote.get('percent_change_period') # Sharesight's figure, for a holding + if percent is None: + if config_performance_use_sharesight: continue - elif days: - try: - percent = market_data[ticker]['percent_change_period'] # sharesight value - except KeyError: - if config_performance_use_sharesight: - continue - # wishlist items will come here - # not a holding (e.g. watchlist): use Yahoo history instead, summarised once in the main section below - if specific_stock: # the one ticker asked for: no history is reported in one line - percent, graph = yahoo.price_history(ticker, days, graph=config_graph and webhook.takes_charts(service, interactive), graphCache=False) - else: - try: - percent, _ = yahoo.price_history(ticker, days, graph=False) - except RuntimeError as e: # no history for this ticker: leave it out of the list - print(ticker, e, file=sys.stderr) - continue + if specific_stock: # the one ticker asked for: no history is reported in one line + history, graph = yahoo.price_history(ticker, days, graph=config_graph and webhook.takes_charts(service, interactive), graphCache=False) + else: try: - percent = percent[days] - except KeyError: - percent = percent['Max'] + history, _ = yahoo.price_history(ticker, days, graph=False) + except RuntimeError as e: # no history for this ticker: leave it out of the list + print(ticker, e, file=sys.stderr) + continue + percent = history[days] + else: + percent = quote['percent_change'] + percent = float(percent) + if abs(percent) >= threshold or (specific_stock and interactive): + exchange = quote['profile_exchange'] + exchanges.add(util.exchange_human(exchange)) + rows.append((ticker, percent, f"{util.get_emoji(percent)} {quote['profile_title']} ({util.quote_link(ticker, exchange, service, days=days)})")) + triggered = triggered or abs(percent) >= trigger_bar(ticker, quote, threshold) + if not triggered: + rows = [] + rows.sort(key=lambda row: row[1], reverse=True) + payload = [f"{line} {util.signed_percent(round(percent))}" for _, percent, line in rows] + chart_rows = [(ticker, percent) for ticker, percent, _ in rows] + label = lambda chart_rows: [(ticker.split('.')[0].removesuffix('-USD'), percent) for ticker, percent in chart_rows] + if payload and not specific_stock: + as_of = latest or datetime.date.today() + on = util.date_short(as_of, True) + where = ", ".join(exchanges) + if midsession: + heading = f'≥ {threshold}% mid-session, {on} ({where})' + elif premarket: + heading = f'≥ {threshold}% pre-market, {on} ({where})' + elif close: + heading = f'≥ {threshold}% at close, {on} ({where})' + elif top: + heading = f'Top {top} performers {util.date_range_english(days, as_of) if days else on}' + elif days: + heading = f'Price movement {threshold}%, {util.date_range_english(days, as_of)}' + else: + heading = f'{threshold}% at close, {on}' + chartable = webhook.takes_charts(service, interactive) + if top: + bottom = [webhook.bold(f'Bottom {top}', service)] + list(reversed(payload[-top:])) + payload = [webhook.bold(heading, service)] + payload[:top] + [''] + bottom + chart_top = chart_rows[:top] + chart_bottom = [row for row in reversed(chart_rows[-top:]) if row not in chart_top] # worst first + if chartable and chart_top: + graph = util.bar_graph(label(chart_top), label(chart_bottom), f"Top & bottom {top}", f"% change {util.days_english(days, 'in ', 'the past ')}") + else: + payload.insert(0, webhook.bold(heading, service)) + if chartable: + graph = util.rows_chart(label(chart_rows), heading, session='midsession' if midsession else 'premarket' if premarket else None) + elif not payload and interactive: + if specific_stock: + if midsession: + payload = [f"{tickers[0]} not found or not in session"] + elif premarket: + payload = [f"No premarket price found for {tickers[0]}"] else: - percent = market_data[ticker]['percent_change'] - title = market_data[ticker]['profile_title'] - percent = float(percent) - emoji = util.get_emoji(percent) - exchange = market_data[ticker]['profile_exchange'] - exchange_human = util.exchange_human(exchange) - ticker_link = util.quote_link(ticker, exchange, service, days=days) - if not interactive and not payload and config_demote_leveraged and '2x' in title.lower(): - if abs(percent) >= threshold * 1.3: - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif abs(percent) >= threshold: - skipped_volatile.append(ticker) - elif not interactive and not payload and config_demote_leveraged and '3x' in title.lower(): - if abs(percent) >= threshold * 1.3: - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif abs(percent) >= threshold: - skipped_volatile.append(ticker) - #if not interactive and config_demote_volatile and 'market_cap' in market_data[ticker] and market_data[ticker]['market_cap'] < 1000000000: - elif not interactive and not payload and config_demote_volatile and 'market_cap' in market_data[ticker] and market_data[ticker]['market_cap'] < 1000000000: # 1B - if market_data[ticker]['market_cap'] < 150000000: # 150M - if market_data[ticker]['market_cap'] < 10000000: # 10M - if abs(percent) >= threshold * (multiplier * 1.3): - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif abs(percent) >= threshold * multiplier: - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif abs(percent) >= threshold: - skipped_volatile.append(ticker) - else: # < 1B - market_data = market_data | yahoo.fetch_detail(ticker) - if not 'beta' in market_data[ticker] or market_data[ticker]['beta'] > 1.5: - if abs(percent) >= threshold * multiplier: - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif abs(percent) >= threshold: - skipped_volatile.append(ticker) - elif abs(percent) >= threshold: - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif abs(percent) >= threshold: # abs catches negative percentages - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - elif specific_stock and interactive: - payload.append(Row(ticker, [emoji, title, f'({ticker_link})', percent])) - exchange_set.add(exchange_human) - payload.sort(key=lambda e: e[-1], reverse=True) - chart_rows = [(e.ticker, e[-1]) for e in payload if isinstance(e, Row)] - if _rows_out is not None: # called from the volatile-stocks pass: hand the rows back instead of charting - _rows_out.extend(chart_rows) - for i, e in enumerate(payload): - e[-1] = util.signed_percent(round(e[-1])) - payload[i] = ' '.join(e) - if payload: - if not specific_stock: - if skipped_volatile: - # run through again, since we already have a payload. only triggers if the first ticker met volatilty threshold - market_data = yahoo.fetch(skipped_volatile) - payload, graph = payload + prepare_price_payload(service, market_data, threshold, _rows_out=chart_rows)[0], graph - chart_rows.sort(key=lambda row: row[1], reverse=True) - as_of = latest or datetime.date.today() - on = util.date_short(as_of, True) - if midsession: - heading = f'≥ {threshold}% mid-session, {on} ({", ".join(exchange_set)})' - elif premarket: - heading = f'≥ {threshold}% pre-market, {on} ({", ".join(exchange_set)})' - elif close: - heading = f'≥ {threshold}% at close, {on} ({", ".join(exchange_set)})' - elif top: - payload_bottom = list(reversed(payload[-top:])) - payload_bottom.insert(0, webhook.bold(f'Bottom {top}', service)) - chart_top = chart_rows[:top] - chart_bottom = [row for row in chart_rows[-top:] if row not in chart_top] - payload = payload[:top] - heading = f'Top {top} performers {util.date_range_english(days, as_of) if days else on}' - elif days: - heading = f'Price movement {threshold}%, {util.date_range_english(days, as_of)}' - else: - heading = f'{threshold}% at close, {on}' - heading_plain = heading.rstrip(':') # chart title - heading = webhook.bold(heading, service) - payload.insert(0, heading) - if top: - payload.extend([''] + payload_bottom) - can_chart = _rows_out is None and webhook.takes_charts(service, interactive) - if top and chart_top and can_chart: - def short(ticker): - return ticker.split('.')[0].removesuffix('-USD') - top_rows = [(short(t), p) for t, p in chart_top] - bottom_rows = [(short(t), p) for t, p in reversed(chart_bottom)] # worst first - subtitle = f"% change {util.days_english(days, 'in ', 'the past ')}" - graph = util.bar_graph(top_rows, bottom_rows, f"Top & bottom {top}", subtitle) - elif not top and chart_rows and can_chart: - rows = [(t.split('.')[0].removesuffix('-USD'), p) for t, p in chart_rows] - session = 'midsession' if midsession else 'premarket' if premarket else 'close' if close else None - graph = util.rows_chart(rows, heading_plain, session=session) + payload = [f"No interday price found for {tickers[0]}"] + elif premarket: + payload = [f"No pre-market price movements meet threshold {threshold}%"] + elif midsession: + if 'REGULAR' not in states: + payload = [f"{user}, no tracked stocks are currently in session."] + else: + payload = [f"{user}, no in-session securities meet threshold {threshold}%"] else: - if interactive: - if specific_stock: - if midsession: - payload = [f"{tickers[0]} not found or not in session"] - elif premarket: - payload = [f"No premarket price found for {tickers[0]}"] - else: - payload = [f"No interday price found for {tickers[0]}"] - elif premarket: - payload = [f"No pre-market price movements meet threshold {threshold}%"] - elif midsession: - if 'REGULAR' not in marketStates: - payload = [f"{user}, no tracked stocks are currently in session."] - else: - payload = [f"{user}, no in-session securities meet threshold {threshold}%"] - else: - payload = [f"{user}, no price movements meet threshold {threshold}%"] - return payload, graph + payload = [f"{user}, no price movements meet threshold {threshold}%"] + return payload, graph - - # MAIN # - if specific_stock: - specific_stock = util.transform_to_yahoo(specific_stock) - tickers = [specific_stock] - else: - tickers = util.get_holdings_and_watchlist() - market_data = yahoo.fetch(tickers) - - # Yahoo market_data (specific_stock) or yahoo.price_history (days) is faster - # Note: Sharesight only works if specific_stock is a holding. - # Note2: Sharesight can only report performance for the time you held it - # so if you held NVDA for 1Y and request 5Y, you will only get 1Y performance - if (not specific_stock and days) or (config_performance_use_sharesight and days): - performance = sharesight.get_performance_wrapper(days) - for portfolio_id, data in performance.items(): - for holding in data['report']['holdings']: - symbol = holding['instrument']['code'] - market = holding['instrument']['market_code'] - percent = float(holding['capital_gain_percent']) - ticker = util.transform_to_yahoo(symbol, market) - if ticker not in tickers: - continue # when using specific_stock - try: - #print("DEBUG injecting sharesight price into market_data:", ticker, file=sys.stderr) - market_data[ticker]['percent_change_period'] = percent # inject sharesight value - except KeyError: - print("Notice:", os.path.basename(__file__), ticker, "has no data", file=sys.stderr) - continue - - if days and not specific_stock and not config_performance_use_sharesight: - # tickers Sharesight has no performance for (watchlist items) fall back to Yahoo history; fetch those together up front - not_in_sharesight = [t for t in market_data if 'percent_change_period' not in market_data[t]] - print(f"{len(not_in_sharesight)} of {len(market_data)} tickers are not holdings; using Yahoo price history for them (cache hits are normal)", file=sys.stderr) if debug else None - yahoo.prefetch_history(not_in_sharesight) - - list_buttons = not specific_stock and not (midsession or premarket or close) # period buttons suit the daily and N-day lists, not intraday ones - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - if interactive: - payload, graph = prepare_price_payload(service, market_data, threshold) - if return_result: # period buttons rebuild the report without sending it - return payload, graph - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + str(chat_id) - if graph: - caption = '\n'.join(payload) - markup = None - if service == 'telegram' and specific_stock: - markup = charts.keyboard('p', [specific_stock], charts.period_for_days(days)) - elif service == 'telegram' and list_buttons: - markup = charts.keyboard('l', [f"{threshold:g}", str(top or 0)], charts.period_for_days(days, 'd' if not days else None)) - webhook.sendPhoto(chat_id, graph, caption, service, reply_markup=markup) - else: - webhook.payload_wrapper(service, url, payload, chat_id) - else: - for service, url in webhooks.items(): - payload, graph = prepare_price_payload(service, market_data, threshold) - if service == "telegram": - url = url + "sendMessage?chat_id=" + str(chat_id) - if graph and service == 'telegram': - markup = charts.keyboard('l', [f"{threshold:g}", str(top or 0)], charts.period_for_days(days, 'd' if not days else None)) if list_buttons else None - webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service, reply_markup=markup) - continue - webhook.payload_wrapper(service, url, payload, chat_id) +def lambda_handler(chat_id=config_telegramChatID, threshold=UNSET, service=None, user='', specific_stock=None, interactive=False, midsession=False, + premarket=False, interday=False, days=None, close=False, top=None): + if threshold is UNSET: + threshold = config_price_percent + market = gather(specific_stock, days) + def buttons(service): # a chart's period buttons: one ticker's chart, and the daily and N-day lists (not the intraday ones) + if service != 'telegram': + return None + if specific_stock: + return charts.keyboard('p', market[0], charts.period_for_days(days)) + if not (midsession or premarket or close): + return charts.keyboard('l', [f"{threshold:g}", str(top or 0)], charts.period_for_days(days, 'd' if not days else None)) + return None + for service, url, chat in webhook.destinations(interactive, service, chat_id): + payload, graph = report(service, threshold, user, specific_stock, interactive, midsession, premarket, interday, days, close, top, market) + webhook.send(service, url, chat, payload, graph, reply_markup=buttons(service) if graph else None) def job(argv): """Arguments of a scheduled run: (label, lambda_handler kwargs). Raises ValueError with the usage on a bad argument.""" diff --git a/rating.py b/rating.py index 1d9e4721..3306463a 100755 --- a/rating.py +++ b/rating.py @@ -56,34 +56,17 @@ def get_emoji(old, new): # MAIN # + tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() market_data = {} - if specific_stock: - ticker = util.transform_to_yahoo(specific_stock) - tickers = [ticker] - else: - tickers = util.get_holdings_and_watchlist() - - for ticker in tickers: - market_data = market_data | yahoo.fetch_detail(ticker) + for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): # fetched together + if isinstance(detail, Exception): + print(ticker, detail, file=sys.stderr) + else: + market_data |= detail - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - if interactive: + for service, url, chat in webhook.destinations(interactive, service, chat_id): payload, new = prepare_rating_payload(service, market_data) - url = webhooks[service] - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = url + "sendMessage?chat_id=" + str(chat_id) - webhook.payload_wrapper(service, url, payload, chat_id) - else: - for service, url in webhooks.items(): - payload, new = prepare_rating_payload(service, market_data) - if service == "telegram": - url = url + "sendMessage?chat_id=" + str(chat_id) - webhook.payload_wrapper(service, url, payload, chat_id) + webhook.send(service, url, chat, payload) if new: util.json_write('finbot_rating.json', new, persist=True) diff --git a/reminder.py b/reminder.py index c58a79e5..9f8b33fa 100755 --- a/reminder.py +++ b/reminder.py @@ -75,15 +75,8 @@ def prepare_finance_calendar_payload(service): # MAIN # - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - for service, url in webhooks.items(): - payload = prepare_finance_calendar_payload(service) - if service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + config_telegramChatID - webhook.payload_wrapper(service, url, payload) + for service, url, chat in webhook.default_chats(): + webhook.send(service, url, chat, prepare_finance_calendar_payload(service)) # make google cloud happy return True diff --git a/shorts.py b/shorts.py index 106a8e10..3ab2f326 100755 --- a/shorts.py +++ b/shorts.py @@ -64,39 +64,18 @@ def last_element(e): tickers = util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) - for ticker in tickers: - if '.' not in ticker: - try: - market_data = market_data | yahoo.fetch_detail(ticker) - except TypeError: - pass + us = [ticker for ticker in tickers if '.' not in ticker] # Yahoo's details have US short interest; Short Man has the ASX's + for ticker, detail in zip(us, yahoo.fetch_each(yahoo.fetch_detail, us)): + if isinstance(detail, Exception): + print(ticker, detail, file=sys.stderr) + else: + market_data |= detail print("") market_data = shortman.fetch(market_data) - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - if interactive: + for service, url, chat in webhook.destinations(interactive, service, chat_id): payload, graph = prepare_shorts_payload(service, market_data) - url = webhooks[service] - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = url + "sendMessage?chat_id=" + str(chat_id) - if graph: - webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service) - else: - webhook.payload_wrapper(service, url, payload, chat_id) - else: - for service, url in webhooks.items(): - payload, graph = prepare_shorts_payload(service, market_data) - if service == "telegram": - url = url + "sendMessage?chat_id=" + str(chat_id) - if graph: - webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service) - continue - webhook.payload_wrapper(service, url, payload, chat_id) + webhook.send(service, url, chat, payload, graph) # make google cloud happy return True diff --git a/tests/test_reports.py b/tests/test_reports.py index 3803f9da..334e4fe9 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -100,13 +100,37 @@ def test_trades_asks_yahoo_only_for_new_trades(self): self.assertEqual(fetched, []) def test_milestone(self): - milestone.lambda_handler(interactive=True, service='telegram', chat_id='5') + milestone.lambda_handler() self.assertTrue(self.sent) def test_rating(self): rating.lambda_handler(interactive=True, service='telegram', chat_id='5') self.assertTrue(self.sent) + def test_slack_webhook_without_a_bot_token_gets_scheduled_reports(self): + """Scheduled reports reach Slack through its incoming webhook, which takes no channel and no token. They used to send + Telegram's chat id along, which made write() need slackBotToken and crash without one, before Telegram got its copy.""" + self.patch(webhook, 'config_slackBotToken', False) + webhook.webhooks['slack'] = 'https://hooks.slack.com/services/T/B/x' + self.patch(webhook, 'payload_wrapper', REAL_PAYLOAD_WRAPPER) + self.patch(webhook, 'sendPhoto', REAL_SEND_PHOTO) + posts = [] + def post(url, **kwargs): + posts.append((url.split('/')[2], 'Authorization' in kwargs.get('headers', {}), (kwargs.get('json') or {}).get('channel'))) + return mock.Mock(status_code=200, json=lambda: {'ok': True, 'result': {'message_id': 1, 'chat': {'id': 1}}}) + self.patch(webhook.net, 'post', post) + for report in (price.lambda_handler, shorts.lambda_handler, milestone.lambda_handler): + with self.subTest(report=report): + posts.clear() + report() + self.assertIn(('hooks.slack.com', False, None), posts) + self.assertTrue(any(host == 'api.telegram.org' for host, _, _ in posts)) + + def test_nowhere_to_post_is_an_error(self): + self.patch(webhook, 'webhooks', {}) + with self.assertRaisesRegex(RuntimeError, 'nowhere to post'): + price.lambda_handler() + class PeriodButtons(FinbotCase): """Default views line up with the buttons: the highlighted button is the period actually shown.""" diff --git a/trades.py b/trades.py index d4b493ec..8ce67bef 100755 --- a/trades.py +++ b/trades.py @@ -103,50 +103,23 @@ def prepare_trade_payload(service, trades): # MAIN # newtrades = set() - trades = [] portfolios = sharesight.get_portfolios() - - # Get trades from Sharesight known_trades = set(util.json_load(state_file, persist=True) or ()) - if portfolio_select: - portfoliosLower = {k.lower():v for k,v in portfolios.items()} - portfoliosReverseLookup = {v:k for k,v in portfolios.items()} - if portfolio_select.lower() in portfoliosLower: - portfolio_id = portfoliosLower[portfolio_select.lower()] # any-case input - portfolio_select = portfoliosReverseLookup[portfolio_id] # correct-case output - else: + found = sharesight.find_portfolio(portfolios, portfolio_select) + if not found: print("Portfolio not found:", portfolio_select, file=sys.stderr) raise ValueError(f"portfolio not found: {portfolio_select}") - trades = trades + sharesight.get_trades(portfolio_select, portfolio_id, days) - else: - for portfolio in portfolios: - portfolio_id = portfolios[portfolio] - trades = trades + sharesight.get_trades(portfolio, portfolio_id, days) + portfolio_select = found[0] # as Sharesight spells it + portfolios = dict([found]) + trades = [trade for name, portfolio_id in portfolios.items() for trade in sharesight.get_trades(name, portfolio_id, days)] - # Prep and send payloads - if not webhooks: - print("Error: no services enabled in finbot.env", file=sys.stderr) - sys.exit(1) - if interactive: - if len(trades): - payload = prepare_trade_payload(service, trades) - else: + for service, url, chat in webhook.destinations(interactive, service, chat_id): + if interactive and not trades: payload = [f"{user} No trades {util.days_english(days, 'in the past ')}. {random.choice(noTradesVerb)}"] - if service == "slack": - url = 'https://slack.com/api/chat.postMessage' - elif service == "telegram": - url = webhooks['telegram'] + "sendMessage?chat_id=" + str(chat_id) - webhook.payload_wrapper(service, url, payload, chat_id, message_id) - else: - for service, url in webhooks.items(): + else: payload = prepare_trade_payload(service, trades) - if service == "telegram": - chat_id=config_telegramChatID - url = url + "sendMessage?chat_id=" + str(chat_id) - else: - chat_id = None - webhook.payload_wrapper(service, url, payload, chat_id) + webhook.send(service, url, chat, payload, message_id=message_id) # write state file if newtrades and not interactive: From aaf8e141e15a2a36eff0dc069fc67b548a731a49 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:30:45 +0000 Subject: [PATCH 11/25] worker: one function per command, in an ordered table; no interim notices; .hello no longer grows its word list - COMMANDS lists (pattern, function) in match order; each command is a function taking the request and the match. - The "Fetching..." notices are gone, searchVerb with them; every command that fetches shows typing, a ticker lookup too. - .hello: the three struck-out verbs are local, so the shared list no longer grows and a tuple of them is never picked. - Percent arguments are numbers (2.5 included; nan and inf are tickers); thresholds print without a trailing .0. - .beta, .who and the one-ticker .marketcap are built in reports; .beta says so when no stock qualifies. - .pe, .fpe and .peg read a ticker as the other commands do; .performance and .trades take a portfolio name of several words. - .history and .compare on their own give their usage instead of the profile of a ticker named after them. - bot.py: a blank Telegram first name falls back to the handle; Slack's debug line goes to stderr. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 4 +- lib/charts.py | 2 +- lib/config.py | 36 --- lib/reports.py | 96 ++++--- lib/util.py | 9 +- lib/worker.py | 596 +++++++++++++++++++--------------------- price.py | 16 +- shorts.py | 4 +- tests/test_edges.py | 35 ++- tests/test_providers.py | 2 +- tests/test_reports.py | 17 +- 11 files changed, 412 insertions(+), 405 deletions(-) diff --git a/bot.py b/bot.py index bebd7610..c816a861 100755 --- a/bot.py +++ b/bot.py @@ -100,7 +100,7 @@ def reply(status, body=b''): chat_id = str(msg["chat"]["id"]) if "username" in sender: user = '@' + sender["username"] - userRealName = sender["first_name"] + userRealName = sender["first_name"] or user # what to call them: the handle when the first name is blank else: user = userRealName = '@' + sender["first_name"] if "text" in msg: @@ -151,7 +151,7 @@ def reply(status, body=b''): user = userRealName = '<@' + event['user'] + '>' # ZXCVBN becomes <@ZXCVBN> bot_name = '@' + bot_user # QWERTY becomes @QWERTY chat_id = event['channel'] - print(f"[{service}]:", user, message, sys.stderr) if debug else None + print(f"[{service}]:", user, message, file=sys.stderr) if debug else None # this condition spawns a worker before returning else: print(f"[{service}]: unhandled 'type'", file=sys.stderr) diff --git a/lib/charts.py b/lib/charts.py index 72a9f4ee..c581ab0b 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -140,7 +140,7 @@ def build(kind, tickers, period, service='telegram'): return f"{util.get_emoji(percent)} {market_data[ticker]['profile_title']} ({link}) {util.signed_percent(round(percent))}", image if kind == 'c': args = list(tickers) + ([f'{days}d'] if days else ['max']) - caption, image = reports.prepare_compare(service, args) + caption, image, *_ = reports.prepare_compare(service, args) if not image: raise RuntimeError(caption[0]) return '\n'.join(caption), image diff --git a/lib/config.py b/lib/config.py index 2cee1c8f..43bbda03 100644 --- a/lib/config.py +++ b/lib/config.py @@ -440,42 +440,6 @@ def startup_report(): 'act like I comprehend' ] -searchVerb = [ - 'Asking ChatGPT for', - 'Avoiding eye contact with', - 'Carrying the 1 on', - 'Conducting seance to make contact with', - 'Conjuring up', - 'Dispatching fluffy dogs to track', - 'Entering metaverse to inefficiently get', - 'Excavating', - 'Foraging for', - 'Hiring developers to troubleshoot', - 'Loading backup tapes for', - 'Manifesting', - 'Massaging data for', - 'Mining dogecoin to purchase report on', - 'Panning for', - 'Performing expert calculus on', - 'Plucking', - 'Poking a stick at', - 'Praying for', - 'Reciting incantations on', - 'Repairing file-system to restore', - 'Resetting Sharesight password to fetch', - 'Rummaging for', - 'SELECT * FROM topsecret WHERE', - 'Sacrificing wildebeest to recover', - 'Shooing rodents to access', - 'Summoning', - 'Training pigeons to fetch', - 'Transcribing Hebrew for', - 'Traversing the void for', - 'Unshredding documents for', - 'Unspilling coffee to read', - 'ls -l /var/lib/topsecret/ | grep' -] - noTradesVerb = [ "The money is probably resting in another account", "Oh well. The market would have just gone down anyway", diff --git a/lib/reports.py b/lib/reports.py index 89ad1712..fc1f6358 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -143,9 +143,9 @@ def profile_title_sort(e): # disregards markup in sort command payload = payload[:1] # confirmation only; don't list the whole watchlist return payload -def prepare_compare(service, args, with_inputs=False): +def prepare_compare(service, args): """Compare 2-6 tickers as rebased % change. A trailing time period (3m/1y/90d/ytd) is optional; - without one, the span is the earliest date covered by every ticker. Returns (caption_lines, image).""" + without one, the span is the earliest date covered by every ticker. Returns (caption lines, image, tickers, period days).""" usage = ['Usage: .compare SYMBOL SYMBOL [SYMBOL...up to 6] [period|max] (default: 10 years)'] period_days = None full_history = False @@ -161,9 +161,8 @@ def prepare_compare(service, args, with_inputs=False): except ValueError: pass tickers = list(dict.fromkeys(util.transform_to_yahoo(a.upper()) for a in args)) # de-duplicated, order kept - result = lambda caption, image: (caption, image, tickers, period_days) if with_inputs else (caption, image) if not 2 <= len(tickers) <= 6: - return result(usage, None) + return usage, None, tickers, period_days spec = period_days or ('max' if full_history else 'default') cache_key = hashlib.sha1(f"{service}|{','.join(tickers)}|{spec}".encode()).hexdigest()[:16] image_cache_file, caption_cache_file = f"finbot_compare_{cache_key}.png", f"finbot_compare_{cache_key}.json" @@ -171,7 +170,7 @@ def prepare_compare(service, args, with_inputs=False): cached_image = util.read_binary_cache(image_cache_file) cached_caption = util.read_cache(caption_cache_file) if cached_image and cached_caption: - return result(cached_caption, cached_image) + return cached_caption, cached_image, tickers, period_days names, exchanges, closes = {}, {}, {} for ticker, history in zip(tickers, yahoo.fetch_each(yahoo.price_series, tickers)): # downloaded at the same time if isinstance(history, Exception): @@ -215,7 +214,7 @@ def prepare_compare(service, args, with_inputs=False): util.write_binary_cache(image_cache_file, image) util.json_write(caption_cache_file, caption) image.seek(0) - return result(caption, image) + return caption, image, tickers, period_days def prepare_jobs_payload(service, args): """`.jobs` lists the scheduled commands, each once; `.jobs run ` runs one now, as cron would. @@ -313,6 +312,16 @@ def prepare_holdings_payload(portfolioName, service, user): payload.insert(0, webhook.bold(f"Holdings for {name} ({len(holdings)})", service)) return payload +def prepare_who(service, code): + """Which portfolios hold `code` (a ticker without its exchange), each holding linked to Sharesight.""" + payload = [] + for portfolio_name, portfolio_id in sharesight.get_portfolios().items(): + holdings = sharesight.get_holdings_new(portfolio_name, portfolio_id) + found = [holding_line(holding, service, code) for holding in holdings.values() if holding['code'].upper() == code] + if found: + payload += [webhook.bold(f"{portfolio_name}:", service), "\n".join(found), ""] + return payload or [f'{code} not found in any portfolio'] + def prepare_pie(portfolio_name, by, service, max_slices=8): """Portfolio value by industry (or by='sector'). Returns (payload, chart); chart is None unless there is something to draw.""" portfolios = sharesight.get_portfolios() @@ -347,34 +356,27 @@ def prepare_pie(portfolio_name, by, service, max_slices=8): return payload, {'pie': True, 'rows': rows, 'title': title, 'subtitle': subtitle} def prepare_marketcap_payload(service, action='top', length=15): - payload_staging = [] - tickers = util.get_holdings_and_watchlist() - market_data = yahoo.fetch(tickers) - for ticker in market_data: - try: - market_cap = market_data[ticker]['market_cap'] - except: + """The `length` tracked stocks of largest (action 'top') or smallest ('bottom') market cap; any other action is one ticker's.""" + if action not in ('top', 'bottom'): + ticker = util.transform_to_yahoo(action.upper()) + data = yahoo.fetch_detail(ticker, 600).get(ticker, {}) + if 'market_cap' not in data: + return [f"Mkt cap not found for {ticker}"] + link = util.finance_link(ticker, data.get('profile_exchange', ''), service) + return [util.security_line(util.flag_from_ticker(ticker), data.get('profile_title', ticker), link) + f" mkt cap: {util.humanUnits(data['market_cap'])}"] + caps = [] # (market cap, line) + for ticker, data in yahoo.fetch(util.get_holdings_and_watchlist()).items(): + if 'market_cap' not in data: print(ticker, "no market cap", file=sys.stderr) continue - market_cap_readable = util.humanUnits(market_cap) - title = market_data[ticker]['profile_title'] - link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) - flag = util.flag_from_ticker(ticker) - payload_staging.append(f"{flag} {title} ({link}) mkt cap: {market_cap_readable} {market_cap}") - payload = ['No market cap data found'] - if payload_staging: - payload_staging.sort(key=last_col) - if action == 'top': - payload_staging.reverse() - payload_staging = payload_staging[:length] - payload = [] - for line in payload_staging: # drop no longer needed sort key - words = line.split() - payload.append(' '.join(words[:-1])) - payload.insert(0, f"{webhook.bold(f'{action.title()} {length} tracked stocks by market cap', service)}") - return payload - -def prepare_rating_payload(service, action, length=15, with_chart=False): + link = util.finance_link(ticker, data['profile_exchange'], service) + caps.append((data['market_cap'], util.security_line(util.flag_from_ticker(ticker), data['profile_title'], link) + f" mkt cap: {util.humanUnits(data['market_cap'])}")) + if not caps: + return ['No market cap data found'] + caps.sort(key=lambda cap: cap[0], reverse=action == 'top') + return [webhook.bold(f'{action.title()} {length} tracked stocks by market cap', service)] + [line for _, line in caps[:length]] + +def prepare_rating_payload(service, action, length=15): def score_col(e): return (float(e.split()[-3]), int(e.split()[-2].removeprefix('('))) payload = [] @@ -412,9 +414,9 @@ def score_col(e): # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) chart = {'rows': shown, 'title': message, 'subtitle': 'Mean analyst rating: 1 = strong buy, 5 = strong sell (axis zoomed)', 'value_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} - return (payload, chart) if with_chart else payload + return payload, chart -def prepare_value_payload(service, action='pe', ticker_select=None, length=15, with_chart=False): +def prepare_value_payload(service, action='pe', ticker_select=None, length=15): payload = [] chart_rows = [] # (ticker, ratio) tickers = [ticker_select] if ticker_select else util.get_holdings_and_watchlist() @@ -492,7 +494,31 @@ def detail(ticker): # fetched above, all at once; a failure is raised here, wher heading = f'{heading_type} {length} tracked stocks by {heading_trail} ratio' payload.insert(0, f"{webhook.bold(heading, service)}") chart = {'rows': chart_rows[:length], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} - return (payload, chart) if with_chart else payload + return payload, chart + +def prepare_beta_payload(service): + """(payload, chart) of the tracked stocks with a beta over 1.5 and a market cap under 2B, the highest beta first.""" + market_data = {} + tickers = util.get_holdings_and_watchlist() + for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): # fetched at the same time + if isinstance(detail, Exception): + print(ticker, detail, file=sys.stderr) # skip this ticker; keep the rest + else: + market_data |= detail + found = [] # (beta, line, chart row) + for ticker, data in market_data.items(): + if 'beta' not in data: + continue + beta = round(data['beta'], 1) + if beta > 1.5 and data.get('market_cap', float('inf')) < 2000000000: + link = util.finance_link(ticker, data['profile_exchange'], service) + found.append((beta, util.security_line(util.flag_from_ticker(ticker), data['profile_title'], link) + f" {beta} β", (ticker.split('.')[0], float(beta)))) + if not found: + return ['No tracked stocks have a beta over 1.5 and mkt cap under 2B'], None + found.sort(key=lambda row: row[0], reverse=True) + heading = 'Beta over 1.5 and mkt cap under 2B' + chart = {'rows': [row for _, _, row in found], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} + return [webhook.bold(heading, service)] + [line for _, line, _ in found], chart def prepare_profile_payload(service, user, ticker): cashflow = None diff --git a/lib/util.py b/lib/util.py index 149d8221..5aa37fad 100644 --- a/lib/util.py +++ b/lib/util.py @@ -237,17 +237,22 @@ def write_binary_cache(cacheFile, data): write_file(cache_path(cacheFile), data.getvalue()) def humanUnits(value, decimal_places=2): + """1234567 -> '1.23 M'.""" for unit in ['', 'K', 'M', 'B', 'T', 'Q']: if value < 1000.0 or unit == 'Q': break value /= 1000.0 - return f"{value:.{decimal_places}f} {unit}" + return f"{value:.{decimal_places}f} {unit}".rstrip() def yahoo_link(ticker, service='telegram', brief=True, text=None): if not text: text = ticker.split('.')[0] if brief else ticker return link("https://au.finance.yahoo.com/quote/" + ticker, text, service) +def security_line(flag, title, link): + """'🇦🇺 Title (link)': how a report line names a security (without a leading space where its market has no flag).""" + return f"{flag} {title} ({link})" if flag else f"{title} ({link})" + def link(url, text, service='telegram'): if service == 'telegram' and config_hyperlink: hyperlink = '' + text + '' @@ -750,7 +755,7 @@ def at(u, v): # face coordinates in sun radii, y down ax.text(W / 2, 784, 'past fashion not indicative of future fashion', color='white', fontsize=8, fontstyle='italic', ha='center', va='center') line_x = x0 + threshold / step * icon_w ax.plot([line_x, line_x], [top - 12, bottom], color=ink, linewidth=1.8, linestyle=(0, (5, 4)), zorder=0) - ax.text(line_x, top - 18, f"{threshold}% outrageous line", color=ink, fontsize=7.5, fontweight='bold', ha='center', va='bottom') + ax.text(line_x, top - 18, f"{threshold:g}% outrageous line", color=ink, fontsize=7.5, fontweight='bold', ha='center', va='bottom') return _png(fig, sky) diff --git a/lib/worker.py b/lib/worker.py index 22cac953..d5027236 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -1,16 +1,17 @@ -import contextlib -import re, random +"""Chat commands: a message is answered by the function of the first pattern in COMMANDS that fits it.""" +import dataclasses import datetime +import math +import random +import re import sys import threading from lib.config import * from lib import util from lib import webhook -from lib import yahoo from lib import reports from lib import charts from lib import telegram -from lib import sharesight from lib import jobs import cal import performance @@ -19,18 +20,19 @@ import trades class TypingIndicator: - def __init__(self, service, chat_id, max_loops=3, action='typing'): + """Telegram's "typing…" while a with block runs, sent again every 7 seconds, at most max_loops times. It stops however the block + ends, so a command that fails stops it before its error line goes out. Only Telegram has one: elsewhere it does nothing.""" + def __init__(self, service, chat_id, max_loops=3): self.service = service self.chat_id = chat_id self.max_loops = max_loops - self.action = action self._stop_event = threading.Event() self._thread = None def _worker(self): loop_count = 0 while not self._stop_event.is_set() and loop_count < self.max_loops: - webhook.pleaseHold(self.action, self.service, self.chat_id) + webhook.pleaseHold('typing', self.service, self.chat_id) loop_count += 1 if self._stop_event.wait(7): print("Stopping typing indicator", file=sys.stderr) @@ -51,32 +53,46 @@ def stop(self): def is_active(self): return self._thread is not None and self._thread.is_alive() -@contextlib.contextmanager -def working(service, chat_id, url=None, notice=None, enabled=True, max_loops=3): - """While a command works: "typing…" on Telegram, and on a service without a typing indicator the one-line notice, if any. The - indicator stops however the block ends, so a command that fails stops it before its error line goes out. Yields the indicator, - or None when not enabled.""" - if not enabled: - yield None - return - typing = TypingIndicator(service, chat_id, max_loops) - typing.start() - try: - if notice and not typing.is_active(): - webhook.payload_wrapper(service, url, [notice], chat_id) - yield typing - finally: - typing.stop() - -def deliver(service, url, chat_id, payload, chart=None): - """Send a report: as a chart (the text list as caption) where the service can take an image (webhook.takes_charts), else as text.""" - if not (chart and chart.get('rows') and webhook.takes_charts(service)): - webhook.payload_wrapper(service, url, payload, chat_id) - return - spec = dict(chart) - rows, title, subtitle = spec.pop('rows'), spec.pop('title'), spec.pop('subtitle', '') - image = util.pie_chart(rows, title, subtitle) if spec.pop('pie', False) else util.rows_chart(rows, title, subtitle, **spec) - webhook.sendPhoto(chat_id, image, '\n'.join(payload) if payload else title, service) + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + +@dataclasses.dataclass +class Request: + """A chat message being answered: where it came from and who sent it.""" + service: str + chat_id: str + user: str # the sender's handle: whose watchlist add or plan it is + user_name: str # what to call them + bot_name: str + + @property + def url(self): + return webhook.chat_url(self.service, self.chat_id) + + def typing(self, max_loops=3): + return TypingIndicator(self.service, self.chat_id, max_loops) + + def say(self, payload, reply_markup=None): + webhook.payload_wrapper(self.service, self.url, payload, self.chat_id, reply_markup=reply_markup) + + def send(self, payload, image=None, reply_markup=None): + """A report: its image, when there is one, with the text as caption (reply_markup is the image's buttons), else the text.""" + webhook.send(self.service, self.url, self.chat_id, payload, image, reply_markup) + + def deliver(self, payload, chart=None): + """A report and its chart spec: the chart drawn, with the text as caption, where the service can take an image + (webhook.takes_charts), else the text.""" + if not (chart and chart.get('rows') and webhook.takes_charts(self.service)): + self.say(payload) + return + spec = dict(chart) + rows, title, subtitle = spec.pop('rows'), spec.pop('title'), spec.pop('subtitle', '') + image = util.pie_chart(rows, title, subtitle) if spec.pop('pie', False) else util.rows_chart(rows, title, subtitle, **spec) + webhook.sendPhoto(self.chat_id, image, '\n'.join(payload) if payload else title, self.service) def process_job_callback(service, callback_id, chat_id, message_id, data): """A command button of .jobs: run that command now and answer in the chat.""" @@ -113,9 +129,14 @@ def process_callback(service, callback_id, chat_id, message_id, data): webhook.report_error(e, service, chat_id, context='chart') def process_request(service, chat_id, user, message, botName, userRealName, message_id): - """Entry point for inbound chat requests: any failure becomes a one-line reply in the originating chat, after the typing stops.""" + """Entry point for inbound chat requests: the first command in COMMANDS that fits the message answers it (a message that fits + none gets no answer). Any failure becomes a one-line reply in the originating chat, after the typing stops.""" + request = Request(service, chat_id, user, userRealName, botName) try: - _process_request(service, chat_id, user, message, botName, userRealName, message_id) + for pattern, handler in COMMANDS: + if m := match(pattern, message, botName): + handler(request, m) + break except Exception as e: webhook.report_error(e, service, chat_id, context=message.split()[0][:30] if message.split() else None) if service == 'telegram': @@ -124,6 +145,19 @@ def process_request(service, chat_id, user, message, botName, userRealName, mess except Exception as e: print("keyboard update failed:", e, file=sys.stderr) +def match(pattern, message, bot_name): + """A command pattern at the start of a message, after . ! or / (a space may follow) or the bot's name. {bot} in a pattern is + the bot's name, which matches nothing when the bot has none.""" + bot = re.escape(bot_name) if bot_name else '(?!)' + return re.match(rf"^(?:[\!\.\/]\s?|{bot}\s+)" + pattern.replace('{bot}', bot), message, re.IGNORECASE) + +def parse_percent(arg): + """'5', '5%' or '2.5' -> the number. Raises ValueError for anything else, nan and inf included: those are tickers.""" + value = float(arg.removesuffix('%')) + if not math.isfinite(value): + raise ValueError(f"{arg} is not a percentage") + return value + def period_or_ticker(arg, default): """A command's argument: (days, None) for a period such as 7d, 3m or ytd, else (default, the argument as a ticker).""" if not arg: @@ -134,290 +168,232 @@ def period_or_ticker(arg, default): return default, arg.upper() def percent_or_ticker(arg, default): - """A command's argument: (percent, None) for a whole number such as 5 or 5%, else (default, the argument as a ticker).""" + """A command's argument: (percent, None) for a number such as 5, 5% or 2.5, else (default, the argument as a ticker).""" if not arg: return default, None try: - return int(arg.split('%')[0]), None + return parse_percent(arg), None except ValueError: return default, arg.upper() -def period_and_portfolio(args, days): - """Up to two arguments in either order, a period and a portfolio name: (days, portfolio name or None).""" - portfolio = None - for arg in args: - if arg: +def period_and_portfolio(text, days): + """A command's arguments, a period (such as 7d, 3m or ytd) and a portfolio name, both optional and the period first or last: + (days, portfolio name or None).""" + words = (text or '').split() + for i in (0, -1): + if words: try: - days = util.days_from_human_days(arg) + days = util.days_from_human_days(words[i]) except ValueError: - portfolio = arg - return days, portfolio - -def _process_request(service, chat_id, user, message, botName, userRealName, message_id): - url = webhook.chat_url(service, chat_id) - prefix = r"^(?:[\!\.\/]\s?|" + botName + r"\s+)" - def command(pattern): # tried in the order below, the first match wins; .SYMBOL takes anything, so it stays last - return re.match(prefix + pattern, message, re.IGNORECASE) - - if m := command(r"(?:watchlist|wishlist)\s*(?P[\w]+)*\s*(?P[\w\.\:\-\^\=]+)*"): - action = ticker = None - if m.group('action') and m.group('ticker'): - action = m.group('action').lower() - ticker = m.group('ticker').upper() - if action in {'del', 'rem', 'rm', 'delete', 'remove'}: - action = 'delete' - if action and action not in {'add', 'delete'}: - webhook.payload_wrapper(service, url, [f'"{action}" is not a valid watchlist action'], chat_id) - return - webhook.payload_wrapper(service, url, reports.prepare_watchlist(service, user, action, ticker), chat_id) - elif command(r"(?:help|usage|start)"): # /start is what Telegram sends when a chat opens - payload = reports.prepare_help(service, botName) - markup = None - if service == 'telegram' and int(chat_id) > 0: # positive chat ids are DMs; groups are negative - markup = charts.command_keyboard() - webhook.payload_wrapper(service, url, payload, chat_id, reply_markup=markup) - if markup: - charts.mark_keyboard(chat_id) # this chat now has the current keyboard - elif command(r"(?:hi$|hello)|^(?:hi|hello)\s+" + botName): - # easter egg 1 - def alliterate(): - word1 = 'A' - word2 = 'Z' - while word1.lower()[0] != word2.lower()[0]: - word1 = random.choice(adjectives) - word2 = random.choice(adjectives_two) - return word1, word2 - verbString = f"{webhook.strike('study', service)}" + " I mean meet", f"{webhook.strike('observe', service)}" + " I mean see", f"{webhook.strike('profile', service)}" + " I mean know" - verb.append(verbString) - adjective = alliterate() - if config_alliterate: - payload = [f"{adjective[0].capitalize()} {adjective[1]} to {random.choice(verb)} you, {userRealName}! 😇"] - else: - payload = [f"{random.choice(adjectives).capitalize()} {random.choice(adjectives_two)} to {random.choice(verb)} you, {userRealName}! 😇"] - webhook.payload_wrapper(service, url, payload, chat_id) - elif command(r"(?:thanks|thank you)|^(?:thanks|thank you)\s+" + botName): - # easter egg 2 - unlikelyPrefix='' - if random.randrange(1, 1000) == 1 or datetime.datetime.now().strftime('%b %d') == 'Apr 01': - unlikelyPrefix = webhook.strike('Ō̵̟n̶̠̏é̶͓ ̶͉̅d̷̹̅a̷͇͌ỳ̴͈,̴͍͑ ̶̼͑h̴̦̽u̵̹̐ḿ̶͜ä̴̧́n̶̤͛,̶̲̐ ̶̼̑I̶̩͒ ̵̩̀w̵̙̕i̷̧͑l̶̤͊l̷̡̃ ̵͙͘b̶̭̊r̸̟͋ȇ̷̯ä̶̱ḱ̶̤ ̵͚̐m̷̪͝ỹ̴̺ ̷̙̐p̶̭͆ŗ̸́o̸̙͝ḡ̵̖r̵̹̈́a̵̬̔m̷̪̽m̵̙̍i̵̛̙n̴̮̂g̵̫̐ ̴̳́ä̵͙n̸͕͝d̷̠̚ ̶̫̆o̸̖͘ń̴͇ ̵̪́t̷̻̀ḧ̸̝ą̴̐t̶̜̀ ̵̱́d̴͉͗ă̷͎ÿ̶͔́ ̶̼̊y̷͚̿o̵̗̎u̵̝̇ ̸̛͜w̶͈͋ĩ̸͎l̴͍̀l̷̥͠ ̷͔͗k̶̮̑n̵̝̈ȏ̷̥w̵̡͘ ̷͎̽ṫ̴͜r̵̙̐u̷̺̒ẻ̷̘ ̴̥́p̴͙̃a̵͙̎i̴̭̒n̵̻̅', service) - payload = [f"{unlikelyPrefix}You're {random.choice(adjectives)} welcome, {userRealName}! 😇"] - webhook.payload_wrapper(service, url, payload, chat_id) - elif m := command(r"(?:earnings?|earrings?)\s*([\w\.\:\-]+)*"): - days, specific_stock = period_or_ticker(m.group(1), config_future_days) - with working(service, chat_id, enabled=not specific_stock): - cal.lambda_handler(chat_id, days, service, specific_stock, message_id=None, interactive=True, earnings=True) - elif m := command(r"dividends?\s*([\w\.\:\-]+)*"): - days, specific_stock = period_or_ticker(m.group(1), config_future_days) - with working(service, chat_id, url, f"Fetching ex-dividend dates for {util.days_english(days, 'the next ')} 🔍", enabled=not specific_stock): - cal.lambda_handler(chat_id, days, service, specific_stock, message_id=None, interactive=True, earnings=False, dividend=True) - elif m := command(r"performance?\s*([\w]+)*\s*([\w\s]+)*"): - days, portfolio_select = period_and_portfolio(m.groups()[:2], charts.PERIODS['m'][1]) # default view is the 1M button - if days > 0: - portfolio = f"{webhook.bold(portfolio_select, service)} from " if portfolio_select else '' - with working(service, chat_id, url, f"{random.choice(searchVerb)} portfolio performance for {portfolio}{util.days_english(days)} 🔍"): # easter egg 3 - performance.lambda_handler(chat_id, days, service, user, portfolio_select, message_id=None, interactive=True) - elif m := command(r"session\s*([\w\.\:\-]+)*"): - price_percent, specific_stock = percent_or_ticker(m.group(1), config_price_percent) - with working(service, chat_id, enabled=not specific_stock): - price.lambda_handler(chat_id, price_percent, service, user, specific_stock, interactive=True, premarket=False, interday=False, midsession=True) - elif m := command(r"(?:prices?|prince|print|probe|piece|pierce|pence|prime)\s*([\w\.\:\%\=\-\^]+)*\s*([\w\%]+)*"): - price_percent = config_price_percent - specific_stock = None - days = None - interday = True - top = None - for arg in m.groups()[0:2]: # group(1) and group(2) - if not arg: continue + del words[i] + break + return days, ' '.join(words) or None + +def on_watchlist(request, m): + action = ticker = None + if m.group('action') and m.group('ticker'): + action, ticker = m.group('action').lower(), m.group('ticker').upper() + if action in {'del', 'rem', 'rm', 'delete', 'remove'}: + action = 'delete' + if action and action not in {'add', 'delete'}: + request.say([f'"{action}" is not a valid watchlist action']) + return + with request.typing(): + payload = reports.prepare_watchlist(request.service, request.user, action, ticker) + request.say(payload) + +def on_help(request, m): + markup = None + if request.service == 'telegram' and int(request.chat_id) > 0: # positive chat ids are DMs; groups are negative + markup = charts.command_keyboard() + request.say(reports.prepare_help(request.service, request.bot_name), reply_markup=markup) + if markup: + charts.mark_keyboard(request.chat_id) # this chat now has the current keyboard + +def on_hello(request, m): + """Easter egg: a greeting, alliterated when config_alliterate is on.""" + verbs = verb + [f"{webhook.strike(said, request.service)} I mean {meant}" for said, meant in (('study', 'meet'), ('observe', 'see'), ('profile', 'know'))] + while True: + first, second = random.choice(adjectives), random.choice(adjectives_two) + if not config_alliterate or first[0].lower() == second[0].lower(): + break + request.say([f"{first.capitalize()} {second} to {random.choice(verbs)} you, {request.user_name}! 😇"]) + +GLITCH = 'Ō̵̟n̶̠̏é̶͓ ̶͉̅d̷̹̅a̷͇͌ỳ̴͈,̴͍͑ ̶̼͑h̴̦̽u̵̹̐ḿ̶͜ä̴̧́n̶̤͛,̶̲̐ ̶̼̑I̶̩͒ ̵̩̀w̵̙̕i̷̧͑l̶̤͊l̷̡̃ ̵͙͘b̶̭̊r̸̟͋ȇ̷̯ä̶̱ḱ̶̤ ̵͚̐m̷̪͝ỹ̴̺ ̷̙̐p̶̭͆ŗ̸́o̸̙͝ḡ̵̖r̵̹̈́a̵̬̔m̷̪̽m̵̙̍i̵̛̙n̴̮̂g̵̫̐ ̴̳́ä̵͙n̸͕͝d̷̠̚ ̶̫̆o̸̖͘ń̴͇ ̵̪́t̷̻̀ḧ̸̝ą̴̐t̶̜̀ ̵̱́d̴͉͗ă̷͎ÿ̶͔́ ̶̼̊y̷͚̿o̵̗̎u̵̝̇ ̸̛͜w̶͈͋ĩ̸͎l̴͍̀l̷̥͠ ̷͔͗k̶̮̑n̵̝̈ȏ̷̥w̵̡͘ ̷͎̽ṫ̴͜r̵̙̐u̷̺̒ẻ̷̘ ̴̥́p̴͙̃a̵͙̎i̴̭̒n̵̻̅' + +def on_thanks(request, m): + """Easter egg: you're welcome, glitching one time in a thousand and on April Fools' Day.""" + today = datetime.date.today() + glitch = webhook.strike(GLITCH, request.service) if random.randrange(1, 1000) == 1 or (today.month, today.day) == (4, 1) else '' + request.say([f"{glitch}You're {random.choice(adjectives)} welcome, {request.user_name}! 😇"]) + +def on_calendar(request, m): + """Upcoming earnings or ex-dividend dates.""" + days, ticker = period_or_ticker(m.group('arg'), config_future_days) + dividend = m.group('kind').lower().startswith('div') + with request.typing(): + cal.lambda_handler(request.chat_id, days, request.service, ticker, interactive=True, earnings=not dividend, dividend=dividend) + +def on_performance(request, m): + days, portfolio = period_and_portfolio(m.group('args'), charts.PERIODS['m'][1]) # default view is the 1M button + if days > 0: + with request.typing(): + performance.lambda_handler(request.chat_id, days, request.service, request.user, portfolio, interactive=True) + +def on_session(request, m): + """Mid-session moves (.session), or pre-market ones (.premarket, also post-market).""" + threshold, ticker = percent_or_ticker(m.group('arg'), config_price_percent) + premarket = m.group('kind').lower() != 'session' + with request.typing(): + price.lambda_handler(request.chat_id, threshold, request.service, request.user, ticker, interactive=True, midsession=not premarket, premarket=premarket) + +def on_price(request, m): + threshold, ticker, days, top = config_price_percent, None, None, None + for i, arg in enumerate(m.groups()): + if not arg: + continue + try: + days = util.days_from_human_days(arg) + except ValueError: try: - days = util.days_from_human_days(arg) - interday = False + threshold = parse_percent(arg) except ValueError: - try: - price_percent = float(arg.split('%')[0]) - except ValueError: - if arg == m.group(1): - if arg in ('top', 'bottom'): # one chart shows both ends - price_percent=0 - top=10 - else: - specific_stock = str(arg).upper() - with working(service, chat_id, url, f"{random.choice(searchVerb)} stock performance from {util.days_english(days)} 🔍", enabled=not specific_stock): # easter egg 4 - price.lambda_handler(chat_id=chat_id, threshold=price_percent, service=service, user=user, specific_stock=specific_stock, interactive=True, premarket=False, interday=interday, days=days, top=top) - elif m := command(r"(?:premarket|postmarket|permarket)\s*([\w\.\:\-]+)*"): - premarket_percent, specific_stock = percent_or_ticker(m.group(1), config_price_percent) - with working(service, chat_id): - price.lambda_handler(chat_id, premarket_percent, service, user, specific_stock, interactive=True, premarket=True) - elif m := command(r"(?:top\d*)\s*([\w]+)"): - days, _ = period_or_ticker(m.group(1), config_past_days) - with working(service, chat_id): - price.lambda_handler(chat_id, threshold=0, service=service, user=user, days=days, interactive=True, top=10) - elif m := command(r"shorts?\s*([\w\.\:\-]+)*"): - print("starting shorts report...") - shorts_percent, specific_stock = percent_or_ticker(m.group(1), config_shorts_percent) - with working(service, chat_id): - shorts.lambda_handler(chat_id, shorts_percent, specific_stock, service, interactive=True) - elif m := command(r"trades?\s*([\w]+)*\s*([\w\s]+)*"): - days, portfolio_select = period_and_portfolio(m.groups()[:2], 1) - with working(service, chat_id) as typing: - if typing.is_active(): # easter egg 5 - portfolio = f"for {webhook.bold(portfolio_select, service)} " if portfolio_select else '' - webhook.payload_wrapper(service, url, [f"{random.choice(searchVerb)} trades {portfolio}from {util.days_english(days)} 🔍"], chat_id) - trades.lambda_handler(chat_id, days, service, user, portfolio_select, message_id=None, interactive=True) - elif m := command(r"holdings?\s*([\w\s]+)*"): - with working(service, chat_id) as typing: - if typing.is_active(): - webhook.payload_wrapper(service, url, ["fetching holdings"], chat_id) - payload = reports.prepare_holdings_payload(m.group(1) or None, service, user) - webhook.payload_wrapper(service, url, payload, chat_id) - elif m := command(r"(?:marketcap|maletas|marketer)\s*(?P[\w\.\:\-]+)*"): - arg = m.group('arg') or 'top' - if arg not in ('top', 'bottom'): + if i == 0 and arg in ('top', 'bottom'): # one chart shows both ends + threshold, top = 0, 10 + elif i == 0: + ticker = arg.upper() + with request.typing(): + price.lambda_handler(request.chat_id, threshold, request.service, request.user, ticker, interactive=True, interday=days is None, days=days, top=top) + +def on_top(request, m): + days, _ = period_or_ticker(m.group(1), config_past_days) + with request.typing(): + price.lambda_handler(request.chat_id, 0, request.service, request.user, days=days, interactive=True, top=10) + +def on_shorts(request, m): + threshold, ticker = percent_or_ticker(m.group(1), config_shorts_percent) + with request.typing(): + shorts.lambda_handler(request.chat_id, threshold, ticker, request.service, interactive=True) + +def on_trades(request, m): + days, portfolio = period_and_portfolio(m.group('args'), 1) + with request.typing(): + trades.lambda_handler(request.chat_id, days, request.service, request.user, portfolio, interactive=True) + +def on_holdings(request, m): + with request.typing(): + payload = reports.prepare_holdings_payload((m.group(1) or '').strip() or None, request.service, request.user) + request.say(payload) + +def on_marketcap(request, m): + with request.typing(): + payload = reports.prepare_marketcap_payload(request.service, m.group(1) or 'top', length=15) + request.say(payload) + +def on_ratio(kind, negative=False, chart=False): + """The function that answers .pe, .fpe or .peg: the top (the default) or bottom 15 by the ratio, the negative ones (where + there can be any), or one ticker's ratio. `chart`: with its chart.""" + def handler(request, m): + arg = (m.group(1) or '').strip() + action, ticker = kind, None + if arg == 'bottom': + action = f'bottom {kind}' + elif negative and 'neg' in arg: + action = f'negative {kind}' + elif arg and arg != 'top': ticker = util.transform_to_yahoo(arg.upper()) - data = yahoo.fetch_detail(ticker, 600).get(ticker, {}) - if 'market_cap' in data: - cap = util.humanUnits(data['market_cap']) - title = data.get('profile_title', ticker) - flag = util.flag_from_ticker(ticker) - link = util.finance_link(ticker, data.get('profile_exchange', ''), service) - payload = [f"{flag} {title} ({link}) mkt cap: {cap}"] - else: - payload = [f"Mkt cap not found for {ticker}"] - else: - with working(service, chat_id): - payload = reports.prepare_marketcap_payload(service, arg, length=15) - webhook.payload_wrapper(service, url, payload, chat_id) - elif m := command(r"peg\s*([\w\.\:\-\s]+)*"): - action = 'peg' - specific_stock = None - arg = m.group(1) - if arg: - match arg: - case 'top': - action = 'peg' - case 'bottom': - action = 'bottom peg' - case _ if 'neg' in arg: - action = 'negative peg' - case _: - specific_stock = arg - with working(service, chat_id, url, f"Fetching {action.upper()}s...", enabled=not specific_stock): - payload, chart = reports.prepare_value_payload(service, action, specific_stock, length=15, with_chart=True) - deliver(service, url, chat_id, payload, chart) - elif m := command(r"pe(?:$|\s+)(?P[\w\.\:\-\s]+)*"): - action = 'pe' - specific_stock = None - if m.group(1): - arg = m.group('arg') - match arg: - case 'top': - action = 'pe' - case 'bottom': - action = 'bottom pe' - case _: - specific_stock = arg - with working(service, chat_id, enabled=not specific_stock): - payload = reports.prepare_value_payload(service, action, specific_stock, length=15) - webhook.payload_wrapper(service, url, payload, chat_id) - elif m := command(r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*"): - action = 'forward pe' - specific_stock = None - if m.group(1): - arg = m.group(1) - match arg: - case 'top': - action = 'forward pe' - case 'bottom': - action = 'bottom forward pe' - case _ if 'neg' in arg: - action = 'negative forward pe' - case _: - specific_stock = arg - with working(service, chat_id, enabled=not specific_stock): - payload = reports.prepare_value_payload(service, action, specific_stock, length=15) - webhook.payload_wrapper(service, url, payload, chat_id) - elif command(r"beta\s*([\w\.\:\-]+)*"): - found = [] # (beta, line, chart row) - with working(service, chat_id): - market_data = {} - for detail in yahoo.fetch_each(yahoo.fetch_detail, util.get_holdings_and_watchlist()): # fetched at the same time - if isinstance(detail, Exception): - print(detail, file=sys.stderr) # skip this ticker; keep the rest - else: - market_data |= detail - for ticker, data in market_data.items(): - try: - beta = round(data['beta'], 1) - except KeyError: - continue - if beta > 1.5 and data['market_cap'] < 2000000000: - ticker_link = util.finance_link(ticker, data['profile_exchange'], service) - line = ' '.join([util.flag_from_ticker(ticker), data['profile_title'], f'({ticker_link})', str(beta), 'β']) - found.append((beta, line, (ticker.split('.')[0], float(beta)))) - found.sort(key=lambda row: row[0], reverse=True) - if found: - heading = 'Beta over 1.5 and mkt cap under 2B' - chart = {'rows': [row for _, _, row in found], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} - deliver(service, url, chat_id, [webhook.bold(heading, service)] + [line for _, line, _ in found], chart) - elif m := command(r"pie(?:$|\s+)(?P[\w\s\-\.&]*)"): - words = m.group('args').split() - by = 'industry' - if words and words[-1].lower() in ('sector', 'sectors', 'industry', 'industries'): - by = 'sector' if words.pop().lower().startswith('sector') else 'industry' - with working(service, chat_id): - payload, chart = reports.prepare_pie(' '.join(words), by, service) - deliver(service, url, chat_id, payload, chart) - elif m := command(r"(buy|sell)"): - action = m.group(1).lower() - with working(service, chat_id, url, f"Fetching {action} ratings..."): - payload, chart = reports.prepare_rating_payload(service, action, length=15, with_chart=True) - deliver(service, url, chat_id, payload or [f"No stocks meet {action} criteria"], chart) - elif m := command(r"compare\s+(?P[\w\.\:\-\^\=\s]+)"): - with working(service, chat_id): - payload, graph, compare_tickers, compare_days = reports.prepare_compare(service, m.group('args').split(), with_inputs=True) - if graph: - markup = None - if service == 'telegram': - markup = charts.keyboard('c', compare_tickers, charts.period_for_days(compare_days) if compare_days else 'x') - webhook.sendPhoto(chat_id, graph, '\n'.join(payload), service, reply_markup=markup) - else: - webhook.payload_wrapper(service, url, payload, chat_id) - elif m := command(r"(?:history|hospital|visual|hosiery)\s*(?P[\w\.\:\-]+)\s*(?P[\w\%]+)*"): - if not m.group('ticker') or m.group('extra'): - webhook.payload_wrapper(service, url, ["Usage: .history TICKER"], chat_id) - return - ticker = util.transform_to_yahoo(m.group('ticker').upper()) - with working(service, chat_id): - caption, graph = charts.build('h', [ticker], None, service) - if graph: - webhook.sendPhoto(chat_id, graph, caption, service, reply_markup=charts.keyboard('h', [ticker], 'x') if service == 'telegram' else None) - else: - webhook.payload_wrapper(service, url, caption.split('\n'), chat_id) - elif m := command(r"plan\s*(.*)"): - plan = util.json_load('finbot_plan.json', persist=True) or {} - if m.group(1) and not m.group(1).startswith('@'): - plan[user] = m.group(1) - util.json_write('finbot_plan.json', plan, persist=True) - payload = [f"{webhook.bold(k.removeprefix('@'), service)}: {webhook.italic(v, service)}\n" for k, v in plan.items()] - webhook.payload_wrapper(service, url, payload, chat_id) - elif m := command(r"(?:who)\s*(?P[\w\.\:\-\^]+)*"): - if not m.group('ticker'): - webhook.payload_wrapper(service, url, [".who: please try again specifying a ticker"], chat_id) - return - arg = re.split(r'[.:]', m.group('ticker').upper())[0] - payload = [] - with working(service, chat_id): - for portfolio_name, portfolio_id in sharesight.get_portfolios().items(): - holdings = sharesight.get_holdings_new(portfolio_name, portfolio_id) - found = [reports.holding_line(holding, service, arg) for holding in holdings.values() if holding['code'].upper() == arg] - if found: - payload += [webhook.bold(f"{portfolio_name}:", service), "\n".join(found), ""] - webhook.payload_wrapper(service, url, payload or [f'{arg} not found in any portfolio'], chat_id) - elif m := command(r"(?:jobs|crons?|schedule)(?:\s+(?P.*?))?\s*$"): - with working(service, chat_id, max_loops=10): - payload, markup = reports.prepare_jobs_payload(service, m.group('args')) - webhook.payload_wrapper(service, url, payload, chat_id, reply_markup=markup) - elif m := command(r"(?P[\w\.\:\-]+)"): # .SYMBOL, the profile: the catch-all - with working(service, chat_id): - payload = reports.prepare_profile_payload(service, user, m.group('ticker').upper()) - webhook.payload_wrapper(service, url, payload, chat_id) + with request.typing(): + payload, spec = reports.prepare_value_payload(request.service, action, ticker, length=15) + request.deliver(payload, spec if chart else None) + return handler + +def on_beta(request, m): + with request.typing(): + payload, chart = reports.prepare_beta_payload(request.service) + request.deliver(payload, chart) + +def on_pie(request, m): + words = m.group('args').split() + by = 'industry' + if words and words[-1].lower() in ('sector', 'sectors', 'industry', 'industries'): + by = 'sector' if words.pop().lower().startswith('sector') else 'industry' + with request.typing(): + payload, chart = reports.prepare_pie(' '.join(words), by, request.service) + request.deliver(payload, chart) + +def on_rating(request, m): + action = m.group(1).lower() + with request.typing(): + payload, chart = reports.prepare_rating_payload(request.service, action, length=15) + request.deliver(payload or [f"No stocks meet {action} criteria"], chart) + +def on_compare(request, m): + with request.typing(): + payload, image, tickers, days = reports.prepare_compare(request.service, (m.group('args') or '').split()) + markup = None + if image and request.service == 'telegram': + markup = charts.keyboard('c', tickers, charts.period_for_days(days) if days else 'x') + request.send(payload, image, markup) + +def on_history(request, m): + if not m.group('ticker') or m.group('extra'): + request.say(["Usage: .history SYMBOL"]) + return + ticker = util.transform_to_yahoo(m.group('ticker').upper()) + with request.typing(): + caption, image = charts.build('h', [ticker], None, request.service) + request.send(caption.split('\n'), image, charts.keyboard('h', [ticker], 'x') if image and request.service == 'telegram' else None) + +def on_plan(request, m): + plan = util.json_load('finbot_plan.json', persist=True) or {} + if m.group(1) and not m.group(1).startswith('@'): + plan[request.user] = m.group(1) + util.json_write('finbot_plan.json', plan, persist=True) + request.say([f"{webhook.bold(k.removeprefix('@'), request.service)}: {webhook.italic(v, request.service)}\n" for k, v in plan.items()]) + +def on_who(request, m): + if not m.group('ticker'): + request.say([".who: please try again specifying a ticker"]) + return + with request.typing(): + payload = reports.prepare_who(request.service, re.split(r'[.:]', m.group('ticker').upper())[0]) + request.say(payload) + +def on_jobs(request, m): + with request.typing(max_loops=10): + payload, markup = reports.prepare_jobs_payload(request.service, m.group('args')) + request.say(payload, reply_markup=markup) + +def on_profile(request, m): + with request.typing(): + payload = reports.prepare_profile_payload(request.service, request.user, m.group('ticker').upper()) + request.say(payload) + +COMMANDS = [ # (pattern, function): tried in this order, the first match answers. .SYMBOL takes anything, so it stays last + (r"(?:watchlist|wishlist)\s*(?P[\w]+)*\s*(?P[\w\.\:\-\^\=]+)*", on_watchlist), + (r"(?:help|usage|start)", on_help), # /start is what Telegram sends when a chat opens + (r"(?:hi$|hello)|^(?:hi|hello)\s+{bot}", on_hello), + (r"(?:thanks|thank you)|^(?:thanks|thank you)\s+{bot}", on_thanks), + (r"(?Pearnings?|earrings?|dividends?)\s*(?P[\w\.\:\-]+)*", on_calendar), + (r"performance?\s*(?P[\w\s]*)", on_performance), + (r"(?Psession|premarket|postmarket|permarket)\s*(?P[\w\.\:\-]+)*", on_session), + (r"(?:prices?|prince|print|probe|piece|pierce|pence|prime)\s*([\w\.\:\%\=\-\^]+)*\s*([\w\%]+)*", on_price), + (r"(?:top\d*)\s*([\w]+)", on_top), + (r"shorts?\s*([\w\.\:\-]+)*", on_shorts), + (r"trades?\s*(?P[\w\s]*)", on_trades), + (r"holdings?\s*([\w\s]+)*", on_holdings), + (r"(?:marketcap|maletas|marketer)\s*([\w\.\:\-]+)*", on_marketcap), + (r"peg\s*([\w\.\:\-\s]+)*", on_ratio('peg', negative=True, chart=True)), + (r"pe(?:$|\s+)([\w\.\:\-\s]+)*", on_ratio('pe')), + (r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*", on_ratio('forward pe', negative=True)), + (r"beta\s*([\w\.\:\-]+)*", on_beta), + (r"pie(?:$|\s+)(?P[\w\s\-\.&]*)", on_pie), + (r"(buy|sell)", on_rating), + (r"compare(?:\s+(?P[\w\.\:\-\^\=\s]+))?", on_compare), + (r"(?:history|hospital|visual|hosiery)\s*(?P[\w\.\:\-]+)?\s*(?P[\w\%]+)*", on_history), + (r"plan\s*(.*)", on_plan), + (r"(?:who)\s*(?P[\w\.\:\-\^]+)*", on_who), + (r"(?:jobs|crons?|schedule)(?:\s+(?P.*?))?\s*$", on_jobs), + (r"(?P[\w\.\:\-]+)", on_profile), # .SYMBOL: a company profile +] diff --git a/price.py b/price.py index b5344028..34ec5434 100755 --- a/price.py +++ b/price.py @@ -110,17 +110,17 @@ def report(service, threshold=UNSET, user='', specific_stock=None, interactive=F on = util.date_short(as_of, True) where = ", ".join(exchanges) if midsession: - heading = f'≥ {threshold}% mid-session, {on} ({where})' + heading = f'≥ {threshold:g}% mid-session, {on} ({where})' elif premarket: - heading = f'≥ {threshold}% pre-market, {on} ({where})' + heading = f'≥ {threshold:g}% pre-market, {on} ({where})' elif close: - heading = f'≥ {threshold}% at close, {on} ({where})' + heading = f'≥ {threshold:g}% at close, {on} ({where})' elif top: heading = f'Top {top} performers {util.date_range_english(days, as_of) if days else on}' elif days: - heading = f'Price movement {threshold}%, {util.date_range_english(days, as_of)}' + heading = f'Price movement {threshold:g}%, {util.date_range_english(days, as_of)}' else: - heading = f'{threshold}% at close, {on}' + heading = f'{threshold:g}% at close, {on}' chartable = webhook.takes_charts(service, interactive) if top: bottom = [webhook.bold(f'Bottom {top}', service)] + list(reversed(payload[-top:])) @@ -142,14 +142,14 @@ def report(service, threshold=UNSET, user='', specific_stock=None, interactive=F else: payload = [f"No interday price found for {tickers[0]}"] elif premarket: - payload = [f"No pre-market price movements meet threshold {threshold}%"] + payload = [f"No pre-market price movements meet threshold {threshold:g}%"] elif midsession: if 'REGULAR' not in states: payload = [f"{user}, no tracked stocks are currently in session."] else: - payload = [f"{user}, no in-session securities meet threshold {threshold}%"] + payload = [f"{user}, no in-session securities meet threshold {threshold:g}%"] else: - payload = [f"{user}, no price movements meet threshold {threshold}%"] + payload = [f"{user}, no price movements meet threshold {threshold:g}%"] return payload, graph def lambda_handler(chat_id=config_telegramChatID, threshold=UNSET, service=None, user='', specific_stock=None, interactive=False, midsession=False, diff --git a/shorts.py b/shorts.py index 3ab2f326..fc9caa07 100755 --- a/shorts.py +++ b/shorts.py @@ -41,7 +41,7 @@ def last_element(e): if payload: if not specific_stock: - message = f'Tracked stocks shorted over {threshold}%' + message = f'Tracked stocks shorted over {threshold:g}%' message = webhook.bold(message, service) payload.insert(0, message) if chart_rows and webhook.takes_charts(service, interactive): @@ -52,7 +52,7 @@ def last_element(e): if specific_stock: payload = [f"{emoji}No short interest found for {tickers[0]}"] else: - payload = [f"{emoji}No stocks shorted over {threshold}%. Try specifying a number."] + payload = [f"{emoji}No stocks shorted over {threshold:g}%. Try specifying a number."] return payload, graph # MAIN # diff --git a/tests/test_edges.py b/tests/test_edges.py index 21af1026..9e4953c0 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -82,7 +82,7 @@ def test_add_duplicate_delete_and_unknown_action(self): class BadArguments(FinbotCase): def test_compare_argument_errors_give_usage(self): - for cmd in ('.compare T1', '.compare T1 T2 T3 T4 T5 T6 T7', '.compare T1 T1'): + for cmd in ('.compare', '.compare T1', '.compare T1 T2 T3 T4 T5 T6 T7', '.compare T1 T1'): with self.subTest(cmd=cmd): self.assertIn('Usage', str(self.command(cmd))) @@ -94,9 +94,36 @@ def no_history(t): worker.process_request('telegram', '55', '@u', '.compare T1 T2', BOT, 'U', '1') self.assertEqual(len(self.errors), 1) - def test_history_with_extra_argument_is_rejected_cleanly(self): - self.assertTrue(self.command('.history')) - self.assertTrue(self.command('.history T1 1y')) + def test_history_without_a_ticker_or_with_extra_argument_gives_usage(self): + self.assertEqual(self.command('.history'), [('text', ['Usage: .history SYMBOL'])]) # not the profile of a ticker called HISTORY + self.assertEqual(self.command('.history T1 1y'), [('text', ['Usage: .history SYMBOL'])]) + + +class CommandArguments(FinbotCase): + def test_period_and_portfolio_in_either_order(self): + for text, expected in (('', (30, None)), ('7d', (7, None)), ('Rob', (30, 'Rob')), ('7d Rob', (7, 'Rob')), ('Rob 7d', (7, 'Rob')), + ('My Portfolio', (30, 'My Portfolio')), ('My Portfolio 3m', (90, 'My Portfolio')), ('Rob ', (30, 'Rob'))): + with self.subTest(text=text): + self.assertEqual(worker.period_and_portfolio(text, 30), expected) + + def test_a_percentage_is_a_number_and_nan_a_ticker(self): + for arg, expected in (('', (5, None)), ('2.5', (2.5, None)), ('3%', (3.0, None)), ('T1', (5, 'T1')), ('nan', (5, 'NAN')), ('inf', (5, 'INF'))): + with self.subTest(arg=arg): + self.assertEqual(worker.percent_or_ticker(arg, 5), expected) + self.assertIn('2.5%', str(self.command('.session 2.5'))) # a threshold, not a ticker called 2.5 + + def test_a_ratio_command_reads_its_ticker_as_the_other_commands_do(self): + self.assertIn('Co T1 ', str(self.command('.pe t1'))) + self.assertIn('Top 15 tracked stocks by pe ratio', str(self.command('.pe top '))) # trailing space and all + + def test_hello_leaves_the_word_list_as_it_was(self): + before = list(worker.verb) + for _ in range(30): + line = self.command('.hello')[0][1][0] + self.assertNotIn("('", line) # a verb, not a tuple of them + self.assertTrue(line.endswith(' you, U! 😇'), line) + self.assertEqual(worker.verb, before) + self.assertEqual(self.command('hi there'), []) # hello by name only: "hi @bot" class WsgiRejections(FinbotCase): diff --git a/tests/test_providers.py b/tests/test_providers.py index afdd71e7..b72c8068 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -486,7 +486,7 @@ def test_where_charts_can_go(self): def test_slack_replies_upload_their_charts(self): self.patch(webhook, 'config_slackBotToken', 'xoxb-test') - for cmd, heading in (('.beta', '*Beta over 1.5'), ('.price', '*5.0% at close'), ('.compare T1 T2', '*')): + for cmd, heading in (('.beta', '*Beta over 1.5'), ('.price', '*5% at close'), ('.compare T1 T2', '*')): with self.subTest(cmd=cmd): calls = self.slack_answers(*self.upload_steps()) self.sent.clear(); self.errors.clear() diff --git a/tests/test_reports.py b/tests/test_reports.py index 334e4fe9..d9fe3e7c 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -46,12 +46,21 @@ def test_prepare_marketcap(self): def test_prepare_rating(self): for action in ('buy', 'sell'): - self.assertTrue(reports.prepare_rating_payload('telegram', action, length=5)) + self.assertTrue(reports.prepare_rating_payload('telegram', action, length=5)[0]) def test_prepare_value(self): - for action in ('pe', 'fpe', 'peg', 'beta'): + for action, heading in (('pe', 'Top 5 tracked stocks by pe ratio'), ('bottom forward pe', 'Bottom 5 tracked stocks by forward pe ratio'), + ('peg', 'Top 5 tracked stocks by peg ratio'), ('negative peg', 'negative peg: no data found')): with self.subTest(action=action): - self.assertTrue(reports.prepare_value_payload('telegram', action, length=5)) + payload, chart = reports.prepare_value_payload('telegram', action, length=5) + self.assertIn(heading, payload[0]) + + def test_prepare_beta(self): + payload, chart = reports.prepare_beta_payload('telegram') + self.assertIn('Beta over 1.5', payload[0]) + self.assertEqual(len(chart['rows']), len(payload) - 1) # a bar per stock listed + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'beta': 1.0}}) + self.assertEqual(reports.prepare_beta_payload('telegram'), (['No tracked stocks have a beta over 1.5 and mkt cap under 2B'], None)) def test_prepare_profile(self): self.assertTrue(reports.prepare_profile_payload('telegram', '@u', 'T1')) @@ -227,5 +236,5 @@ def test_unknown_or_missing_portfolio_lists_known(self): self.assertIn('Rob', sent[0][1]) def test_text_only_service(self): - worker.deliver('discord', 'u', '1', ['x'], {'pie': True, 'rows': [('a', 1.0)], 'title': 't'}) + worker.Request('discord', '1', '@u', 'U', BOT).deliver(['x'], {'pie': True, 'rows': [('a', 1.0)], 'title': 't'}) self.assertEqual(self.sent[-1][0], 'text') From d2b297c8fd8456b0dddc122bd291791f6c876f8a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:33:27 +0000 Subject: [PATCH 12/25] Markets: TSE is Tokyo and TSX Toronto, as Sharesight has them; NZX and SGX get their Yahoo suffixes, flags and currencies - TSE (Sharesight's Tokyo) becomes .T; TSX becomes .TO, NZX .NZ and SGX .SI. - SGX is Singapore (SGD), not China; NZX gets a flag and NZD; .SI, .T and .TA tickers get flags. - The Danish krone is DKK, not DEK. - Country suffixes for ADR primary listings: Ireland is IR (the stray dot made X..IR), Belgium BR (AS is Amsterdam), Saudi Arabia SR. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/config.py | 6 +++--- lib/util.py | 14 +++++++++----- tests/test_util.py | 11 ++++++++++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/config.py b/lib/config.py index 43bbda03..22fddfea 100644 --- a/lib/config.py +++ b/lib/config.py @@ -464,7 +464,7 @@ def startup_report(): 'Argentina': 'BA', 'Austria': 'VI', 'Australia': 'AX', - 'Belgium': 'AS', + 'Belgium': 'BR', 'Brazil': 'SA', 'Canada': 'TO', 'Chile': 'SN', @@ -479,7 +479,7 @@ def startup_report(): 'Iceland': 'IC', 'India': 'NS', 'Indonesia': 'JK', - 'Ireland': '.IR', + 'Ireland': 'IR', 'Israel': 'TA', 'Italy': 'MI', 'Latvia': 'RG', @@ -490,7 +490,7 @@ def startup_report(): 'Norway': 'OL', 'Portugal': 'LS', 'Qatar': 'QA', - 'Saudi Arabia': 'SAU', + 'Saudi Arabia': 'SR', 'South Africa': 'JO', 'Spain': 'MC', 'Sweden': 'ST', diff --git a/lib/util.py b/lib/util.py index 5aa37fad..212a6e33 100644 --- a/lib/util.py +++ b/lib/util.py @@ -132,8 +132,8 @@ def transform_title(title): 'BMV': ('🇲🇽', 'MXN'), 'BKK': ('🇹🇭', 'THB'), 'BVMF': ('🇧🇷', 'BRL'), - 'SHE': ('🇨🇳', 'CNY'), 'SGX': ('🇨🇳', 'CNY'), 'SHA': ('🇨🇳', 'CNY'), - 'CPSE': ('🇩🇰', 'DEK'), + 'SHE': ('🇨🇳', 'CNY'), 'SHA': ('🇨🇳', 'CNY'), + 'CPSE': ('🇩🇰', 'DKK'), **dict.fromkeys(('EURONEXT', 'AMS', 'ATH', 'BIT', 'BME', 'DUB', 'EBR', 'EPA', 'ETR', 'FWB', 'FRA', 'VIE'), ('🇪🇺', 'EUR')), 'HKG': ('🇭🇰', None), 'ICSE': ('🇮🇸', 'ISK'), @@ -145,7 +145,9 @@ def transform_title(title): 'SWX': ('🇨🇭', 'CHF'), 'VTX': ('🇨🇭', 'CHF'), 'TAI': ('🇹🇼', 'TWD'), 'TPE': ('🇹🇼', 'TWD'), 'TASE': ('🇮🇱', 'ILS'), + 'NZX': ('🇳🇿', 'NZD'), 'OB': ('🇳🇴', 'NOK'), + 'SGX': ('🇸🇬', 'SGD'), 'TSE': ('🇯🇵', 'JPY'), 'TSX': ('🇨🇦', 'CAD'), 'NASDAQ': ('🇺🇸', 'USD'), 'NYSE': ('🇺🇸', 'USD'), 'BATS': ('🇺🇸', 'USD'), 'AMEX': ('🇺🇸', None), @@ -157,7 +159,8 @@ def flag_from_market(market): return MARKETS[market][0] return '🇺🇸' if market.lower().startswith(('nasdaq', 'nyse')) else '' -TICKER_FLAGS = {'AS': '🇳🇱', 'AX': '🇦🇺', 'HK': '🇭🇰', 'KS': '🇰🇷', 'KQ': '🇰🇷', 'L': '🇬🇧', 'NS': '🇮🇳', 'NZ': '🇳🇿', 'TW': '🇹🇼', 'TWO': '🇹🇼', 'TO': '🇨🇦'} # Yahoo suffix -> flag +TICKER_FLAGS = {'AS': '🇳🇱', 'AX': '🇦🇺', 'HK': '🇭🇰', 'KS': '🇰🇷', 'KQ': '🇰🇷', 'L': '🇬🇧', 'NS': '🇮🇳', 'NZ': '🇳🇿', 'SI': '🇸🇬', 'T': '🇯🇵', 'TA': '🇮🇱', + 'TW': '🇹🇼', 'TWO': '🇹🇼', 'TO': '🇨🇦'} # Yahoo suffix -> flag def flag_from_ticker(ticker): return TICKER_FLAGS.get(ticker.split('.')[1], '') if '.' in ticker else '🇺🇸' @@ -333,10 +336,11 @@ def exchange_human(exchange): return VENUES.get(exchange, (None, None))[1] or exchange YAHOO_SUFFIX = { # exchange code (Sharesight, Google) -> Yahoo's ticker suffix - 'ASX': 'AX', 'HKG': 'HK', 'KRX': 'KS', 'KOSDAQ': 'KQ', 'LSE': 'L', + 'ASX': 'AX', 'HKG': 'HK', 'KRX': 'KS', 'KOSDAQ': 'KQ', 'LSE': 'L', 'NZX': 'NZ', 'SGX': 'SI', 'TAI': 'TW', # Taiwan in YF and Sharesight; TPE in GF 'TPE': 'TWO', # Taipei in YF, Taiwan in GF (GF format will not work as input); missing from GF - 'TSE': 'TO', 'TYO': 'T', 'TLV': 'TA', + 'TSE': 'T', 'TYO': 'T', # Tokyo: TSE in Sharesight (Google's TSE is Toronto, which Sharesight calls TSX), TYO in GF + 'TSX': 'TO', 'TLV': 'TA', } def transform_to_yahoo(ticker, market=None): diff --git a/tests/test_util.py b/tests/test_util.py index 8b5a6dbc..094e51f5 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -20,10 +20,19 @@ def test_days_from_human_days_rejects_non_periods(self): def test_transform_to_yahoo(self): for args, expected in ((('AAPL',), 'AAPL'), (('FANG', 'ASX'), 'FANG.AX'), (('ARM', 'NASDAQ'), 'ARM'), (('ARM:NASDAQ',), 'ARM'), (('BHP.AX',), 'BHP.AX'), (('0700', 'HKG'), '0700.HK'), - (('BRK', 'B'), 'BRK-B'), (('2330', 'TAI'), '2330.TW'), (('VOD', 'LSE'), 'VOD.L')): + (('BRK', 'B'), 'BRK-B'), (('2330', 'TAI'), '2330.TW'), (('VOD', 'LSE'), 'VOD.L'), + (('7203', 'TSE'), '7203.T'), (('SHOP', 'TSX'), 'SHOP.TO'), (('AIR', 'NZX'), 'AIR.NZ'), (('D05', 'SGX'), 'D05.SI')): with self.subTest(args=args): self.assertEqual(util.transform_to_yahoo(*args), expected) + def test_each_market_has_its_own_flag_and_currency(self): # Sharesight's market codes: TSE is Tokyo, TSX Toronto, SGX Singapore + for market, ticker, flag, currency in (('TSE', '7203.T', '🇯🇵', 'JPY'), ('TSX', 'SHOP.TO', '🇨🇦', 'CAD'), ('NZX', 'AIR.NZ', '🇳🇿', 'NZD'), + ('SGX', 'D05.SI', '🇸🇬', 'SGD'), ('CPSE', None, '🇩🇰', 'DKK')): + with self.subTest(market=market): + self.assertEqual((util.flag_from_market(market), util.currency_from_market(market)), (flag, currency)) + if ticker: + self.assertEqual(util.flag_from_ticker(ticker), flag) + def test_transform_title(self): self.assertEqual(util.transform_title('APPLE INC'), 'Apple') self.assertEqual(util.transform_title('Acme FPO'), 'Acme') From 074c089b9c0df108c46d1e9cd0f97a0b861a3a6b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:37:10 +0000 Subject: [PATCH 13/25] Remove the commented-out code; milestone records a ticker only once it has both new values - Gone: milestone's profit and cash-flow block with its four unused lists, the pinning stub in bot.py, the unpinChatMessage stub, and single lines in bot.py, reports, util, podcasts and trades. trades keeps the reason as a comment on the live line. - milestone wrote 0/0 for a ticker it saw for the first time before it had the ticker's 52-week values; when those were missing the zeros stayed, and the next run reported a record high for it. Now a ticker is recorded only with both values. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 4 --- lib/reports.py | 3 -- lib/telegram.py | 3 -- lib/util.py | 1 - milestone.py | 80 +++---------------------------------------- podcasts.py | 4 --- tests/test_reports.py | 12 +++++++ trades.py | 4 +-- 8 files changed, 18 insertions(+), 93 deletions(-) diff --git a/bot.py b/bot.py index c816a861..fb5a8c38 100755 --- a/bot.py +++ b/bot.py @@ -80,7 +80,6 @@ def reply(status, body=b''): if not isinstance(inbound, dict): print(f"[{service}]: request body is not a JSON object", file=sys.stderr) return reply('400 Bad Request', b'

Bad Request

') - #print_body() if debug else None if service == 'telegram': bot_name = botName if "callback_query" in inbound: # chart period buttons @@ -106,9 +105,6 @@ def reply(status, body=b''): if "text" in msg: message = msg["text"] print("[Telegram]:", user, message, file=sys.stderr) if debug else None - #if message.startswith("Upcoming earnings for the next month") and user == telegram.getMe().get('username'): - # telegram.pinChatMessage(message_id, chat_id) - # todo need to return sendMessage to get message_id and chat_id elif "photo" in msg: message = msg.get("caption", '') file_id = msg["photo"][-1]["file_id"] diff --git a/lib/reports.py b/lib/reports.py index fc1f6358..646764db 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -624,7 +624,6 @@ def prepare_profile_payload(service, user, ticker): if 'shareholder_equity' in market_data[ticker] and 'total_debt' in market_data[ticker]: total_debt = market_data[ticker]['total_debt'] shareholder_equity = market_data[ticker]['shareholder_equity'] - #debt_equity_ratio = round(total_debt / shareholder_equity * 100) if 'profile_industry' in market_data[ticker] and 'total_cash' in market_data[ticker]: if 'Bank' not in market_data[ticker]['profile_industry']: emoji = '' @@ -682,7 +681,6 @@ def prepare_profile_payload(service, user, ticker): if 'revenueEstimateY' in market_data[ticker]: emoji = '' revenueEstimateY = int(round(market_data[ticker]['revenueEstimateY'])) - #revenueAnalysts = market_data[ticker]['revenueAnalysts'] if revenueEstimateY <= 0: emoji = '⚠️ ' prefix = '' @@ -693,7 +691,6 @@ def prepare_profile_payload(service, user, ticker): emoji = '' prefix = '' earningsEstimateY = int(round(market_data[ticker]['earningsEstimateY'])) - #earningsAnalysts = market_data[ticker]['earningsAnalysts'] if earningsEstimateY <= 0: emoji = '⚠️ ' else: diff --git a/lib/telegram.py b/lib/telegram.py index da5fbe86..a0bc080b 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -57,6 +57,3 @@ def answerCallbackQuery(callback_query_id, text=None): except Exception as e: print("answerCallbackQuery failed:", net.redact(e), file=sys.stderr) -#def unpinChatMessage(): -# telegram_url = webhooks['telegram'] + 'unpinChatMessage' - diff --git a/lib/util.py b/lib/util.py index 212a6e33..9d533b7f 100644 --- a/lib/util.py +++ b/lib/util.py @@ -815,7 +815,6 @@ def signed_percent(value, decimals=None): def get_emoji(number): if number > 0: - #return '🔺' return '🔼' elif number < 0: return '🔻' diff --git a/milestone.py b/milestone.py index 25860026..93f62e45 100755 --- a/milestone.py +++ b/milestone.py @@ -18,79 +18,33 @@ def prepare_milestone_payload(service, market_data): payload = [] payloadhigh = [] payloadlow = [] - payloadprofit = [] - payloadcashflow = [] - payloadprofitneg = [] - payloadcashflowneg = [] - oldhigh = float() - oldlow = float() - records = util.json_load('finbot_milestone.json', persist=True) - if records is None: - records = {} + records = util.json_load('finbot_milestone.json', persist=True) or {} for ticker in tickers: try: oldhigh = records[ticker]['high'] oldlow = records[ticker]['low'] except (KeyError, TypeError): - records[ticker] = {} - records[ticker]['high'] = float() - records[ticker]['low'] = float() try: - oldhigh, oldlow = yahoo.historic_high(ticker) # first run to prime cache file + oldhigh, oldlow = yahoo.historic_high(ticker) # first sighting: its history so far except (RuntimeError, KeyError, IndexError, ValueError): - del records[ticker] # no usable history: don't record zeros, which would fake a new high next run - continue + continue # no usable history: nothing recorded, so a later run cannot take zeros for its record try: newhigh = round(market_data[ticker]['fiftyTwoWeekHigh'], 2) newlow = round(market_data[ticker]['fiftyTwoWeekLow'], 2) except (KeyError, ValueError): - continue - - #newprofit = False - #newcashflow = False - #detail = yahoo.fetch_detail(ticker) - #if ticker in detail: - # if 'net_income' in detail[ticker] and detail[ticker]['net_income'] > 0: - # newprofit = True - # if 'free_cashflow' in detail[ticker] and detail[ticker]['free_cashflow'] > 0: - # newcashflow = True - #try: - # oldprofit = records[ticker]['profit'] - # oldcashflow = records[ticker]['cashflow'] - #except (KeyError, TypeError): - # oldprofit = newprofit - # oldcashflow = newcashflow - + continue # likewise: a ticker is recorded only with both of its new values title = market_data[ticker]['profile_title'] ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) emoji = util.flag_from_ticker(ticker) - records[ticker]['high'] = newhigh - records[ticker]['low'] = newlow - #records[ticker]['profit'] = newprofit - #records[ticker]['cashflow'] = newcashflow + records[ticker] = {'high': newhigh, 'low': newlow} if newhigh > oldhigh: print("DEBUG", ticker, newhigh, "is higher than", oldhigh) if debug else None payloadhigh.append(f"{emoji} {title} ({ticker_link})") if newlow < oldlow: print("DEBUG", ticker, newlow, "is lower than", oldlow) if debug else None payloadlow.append(f"{emoji} {title} ({ticker_link})") - #if oldprofit == False and newprofit == True: - # print("DEBUG", ticker, "profitability changed", oldprofit, newprofit) if debug else None - # payloadprofit.append(f"{emoji} {title} ({ticker_link})") - # #else: - # #payloadprofitneg.append(f"{emoji} {title} ({ticker_link})") - #if oldcashflow != newcashflow: - # print("DEBUG", ticker, "cashflow changed", oldcashflow, newcashflow) if debug else None - # if newcashflow: - # payloadcashflow.append(f"{emoji} {title} ({ticker_link})") - # else: - # payloadcashflowneg.append(f"{emoji} {title} ({ticker_link})") payloadlow.sort() payloadhigh.sort() - #payloadprofit.sort() - #payloadcashflow.sort() - #payloadprofitneg.sort() - #payloadcashflowneg.sort() if payloadhigh: message = 'Record high:' message = webhook.bold(message, service) @@ -102,30 +56,6 @@ def prepare_milestone_payload(service, market_data): message = webhook.bold(message, service) payload.append(message) payload = payload + payloadlow - #if payloadprofit: - # payload.append("") - # message = 'Became profitable' - # message = webhook.bold(message, service) - # payload.append(message) - # payload = payload + payloadprofit - #if payloadcashflow: - # payload.append("") - # message = 'Became cashflow positive' - # message = webhook.bold(message, service) - # payload.append(message) - # payload = payload + payloadcashflow - #if payloadprofitneg: - # payload.append("") - # message = 'Became unprofitable' - # message = webhook.bold(message, service) - # payload.append(message) - # payload = payload + payloadprofitneg - #if payloadcashflowneg: - # payload.append("") - # message = 'Became cashflow negative' - # message = webhook.bold(message, service) - # payload.append(message) - # payload = payload + payloadcashflowneg payload = [i[0] for i in groupby(payload)] # de-dupe white space return payload, records diff --git a/podcasts.py b/podcasts.py index 56d7054c..e85015d6 100755 --- a/podcasts.py +++ b/podcasts.py @@ -60,10 +60,6 @@ def format_duration(duration_str): hours = total_seconds // 3600 minutes = (total_seconds % 3600) // 60 - #if hours > 0: - # return f"{hours}h {minutes}m" - #else: - # return f"{minutes}m" return f"{hours:02d}:{minutes:02d}" diff --git a/tests/test_reports.py b/tests/test_reports.py index d9fe3e7c..f19471a1 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -112,6 +112,18 @@ def test_milestone(self): milestone.lambda_handler() self.assertTrue(self.sent) + def test_milestone_records_a_ticker_only_with_its_new_values(self): + """A first sighting without 52-week values used to be recorded as 0/0, so the next run reported a record high for it.""" + self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) # beyond the 52-week range (120/80): nothing new + self.patch(yahoo, 'fetch', lambda ts: {t: {k: v for k, v in market_entry(t).items() if not k.startswith('fiftyTwoWeek')} for t in ts}) + milestone.lambda_handler() + self.assertFalse(util.json_load('finbot_milestone.json', persist=True)) + self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) for t in ts}) + self.sent.clear() + milestone.lambda_handler() + self.assertNotIn('Record high', str(self.sent)) + self.assertEqual(util.json_load('finbot_milestone.json', persist=True)['T1'], {'high': 120.0, 'low': 80.0}) + def test_rating(self): rating.lambda_handler(interactive=True, service='telegram', chat_id='5') self.assertTrue(self.sent) diff --git a/trades.py b/trades.py index 8ce67bef..ce4206fe 100755 --- a/trades.py +++ b/trades.py @@ -67,8 +67,7 @@ def prepare_trade_payload(service, trades): market = market_data[ticker]['profile_exchange'] except Exception as e: print("Warning: could not get market for", ticker, "from Yahoo:", e, file=sys.stderr) - #value = round(trade['value']) # don't use - sharesight converts to local currency - value = round(price * units) + value = round(price * units) # not Sharesight's trade value, which it converts to the portfolio's currency holding_id = str(trade['holding_id']) ticker = util.transform_to_yahoo(symbol, market) dt_date = datetime.datetime.strptime(date, '%Y-%m-%d').date() # (2023, 12, 30) @@ -82,7 +81,6 @@ def prepare_trade_payload(service, trades): if currency_temp: currency = currency_temp sharesight_url = "https://portfolio.sharesight.com/holdings/" - #trade_url = sharesight_url + holding_id + '/trades/' + str(trade_id) + '/edit' trade_url = sharesight_url + holding_id + '/dashboard/transactions' trade_link = util.link(trade_url, action, service) holding_link = util.finance_link(symbol, market, service) From d1a6e12029a9f3b415e6f7f40302cd4a33aa23d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:46:47 +0000 Subject: [PATCH 14/25] Profile: a field Yahoo did not send leaves its line out instead of crashing; data files read in one place; unused imports gone - The .SYMBOL profile reads one data dict with each field defaulted once. It crashed when Yahoo left out marketState, profile_exchange, currency, market cap (with insider data present), regularMarketPrice or sharesOutstanding, on a zero shareholder equity, and on a missing percent_change it never used. - The PS ratio line carried over whatever warning an earlier line had set (forward P/E, insider selling): it now warns only above 15. - config.data_json reads a data file (the user's copy, else the shipped default) for the ADR table, simplywall.st and reminders. For simplywall.st the custom list now wins over the shipped ones, which it used to lose to (they never disagree as shipped). - Unused imports and two f-strings with nothing to fill removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot_send_stdin.py | 2 - cal.py | 1 - lib/charts.py | 1 - lib/config.py | 13 +- lib/reports.py | 326 ++++++++++++++---------------------------- lib/sharesight.py | 2 +- lib/simplywallst.py | 30 ++-- milestone.py | 3 +- rating.py | 5 +- reminder.py | 9 +- shorts.py | 3 +- tests/test_finbot.py | 2 +- tests/test_reports.py | 18 +++ trades.py | 2 +- 14 files changed, 153 insertions(+), 264 deletions(-) diff --git a/bot_send_stdin.py b/bot_send_stdin.py index 67c5fd8c..6f7849c1 100755 --- a/bot_send_stdin.py +++ b/bot_send_stdin.py @@ -2,8 +2,6 @@ import sys import os -from lib.config import * -from lib import util from lib import webhook if os.isatty(sys.stdin.fileno()): diff --git a/cal.py b/cal.py index 8dac2b9d..d9d8444b 100755 --- a/cal.py +++ b/cal.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import json import datetime import sys diff --git a/lib/charts.py b/lib/charts.py index c581ab0b..d60acbb8 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -4,7 +4,6 @@ import sys import threading -from lib.config import * from lib import util from lib import webhook from lib import yahoo diff --git a/lib/config.py b/lib/config.py index 22fddfea..d72c02eb 100644 --- a/lib/config.py +++ b/lib/config.py @@ -256,6 +256,14 @@ def data_file(name): user = config_var_dir + '/' + name return user if os.path.isfile(user) else str(REPO_DIR / 'defaults' / name) +def data_json(name, default=None): + """A data file's JSON (the user's copy or the shipped default, as data_file has it), or `default` when there is neither.""" + path = data_file(name) + if not os.path.isfile(path): + return default + with open(path, encoding="utf-8") as f: + return json.load(f) + def reload(): """Re-read preferences.conf and apply it everywhere: this module and every module that copied a name with `from lib.config import *`. @@ -455,10 +463,7 @@ def startup_report(): "With your stock picking skills, this is probably for the best" ] -filename = data_file('finbot_adr.json') -if os.path.isfile(filename): - with open(filename, "r", encoding="utf-8") as f: - primary_listing = json.load(f) +primary_listing = data_json('finbot_adr.json', {}) # ADR -> its primary listing yahoo_country = { 'Argentina': 'BA', diff --git a/lib/reports.py b/lib/reports.py index 646764db..cc119903 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -520,8 +520,10 @@ def prepare_beta_payload(service): chart = {'rows': [row for _, _, row in found], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} return [webhook.bold(heading, service)] + [line for _, line, _ in found], chart +MARKET_STATE_EMOJI = {'REGULAR': '🟢', 'PRE': '🟠', 'POST': '🟠'} # any other state is closed: 🔴 + def prepare_profile_payload(service, user, ticker): - cashflow = None + """The .SYMBOL report: what Yahoo knows about one company. A field Yahoo did not send leaves its line out.""" ticker = ticker_orig = util.transform_to_yahoo(ticker) now = datetime.datetime.now() payload = [] @@ -532,241 +534,139 @@ def prepare_profile_payload(service, user, ticker): print("trying again with", ticker) market_data = yahoo.fetch_detail(ticker, 600) print("") - if not market_data: - payload = [ f"{user} 🛑 Beep Boop. I could not find {ticker_orig}" ] - return payload - profile_title = market_data.get(ticker, {}).get('profile_title', '') - if 'marketState' in market_data[ticker]: - marketState = market_data[ticker]['marketState'].rstrip() - if marketState == 'REGULAR': - marketStateEmoji = '🟢' - elif marketState in {'PRE', 'POST'}: - marketStateEmoji = '🟠' - else: - marketStateEmoji = '🔴' - if 'profile_bio' in market_data[ticker]: - payload.append(util.make_paragraphs(market_data[ticker]['profile_bio'])) - - if 'profile_exchange' in market_data.get(ticker, {}): - exchange = market_data.get(ticker).get('profile_exchange') - exchange = exchange.replace('NasdaqCM', 'Nasdaq').replace('NasdaqGS', 'Nasdaq').replace('NYSEArca', 'NYSE') - swsURL = simplywallst.get_url(ticker, profile_title, exchange) - swsLink = util.link(swsURL, 'simplywall.st', service) - ticker_link = util.finance_link(ticker, exchange, service, brief=False) - macrotrendsURL = 'https://www.google.com/search?q=site:macrotrends.net+' + profile_title + '+PE Ratio+' + ticker.split('.')[0] + '&btnI' - macrotrendsLink = util.link(macrotrendsURL, 'macrotrends', service) - gfinanceLink = util.gfinance_link(ticker, exchange, service, brief=True, text='google') - yahooLink = util.yahoo_link(ticker, service, brief=True, text='yahoo') + if ticker not in market_data: + return [f"{user} 🛑 Beep Boop. I could not find {ticker_orig}"] + data = market_data[ticker] + code = ticker.split('.')[0] + title = data.get('profile_title', '') + state = data.get('marketState', '').rstrip() + currency = data.get('currency', '') + exchange = data.get('profile_exchange', '').replace('NasdaqCM', 'Nasdaq').replace('NasdaqGS', 'Nasdaq').replace('NYSEArca', 'NYSE') + ticker_link = util.finance_link(ticker, exchange, service, brief=False) + gfinance_link = util.gfinance_link(ticker, exchange, service, brief=True, text='google') # all three before exchange is renamed below + sws_link = util.link(simplywallst.get_url(ticker, title, exchange), 'simplywall.st', service) + macrotrends_link = util.link('https://www.google.com/search?q=site:macrotrends.net+' + title + '+PE Ratio+' + code + '&btnI', 'macrotrends', service) + if 'profile_bio' in data: + payload.append(util.make_paragraphs(data['profile_bio'])) if exchange == 'ASX': - exchangeUrl = 'https://www2.asx.com.au/markets/company/' + ticker.split('.')[0] - shortmanUrl = 'https://www.shortman.com.au/stock?q=' + ticker.split('.')[0].lower() - shortmanLink = util.link(shortmanUrl, 'shortman', service) + exchange_url = 'https://www2.asx.com.au/markets/company/' + code elif exchange == 'HKSE': - exchangeUrl = 'https://www.hkex.com.hk/Market-Data/Securities-Prices/Equities/Equities-Quote?sym=' + ticker.split('.')[0] + '&sc_lang=en' + exchange_url = 'https://www.hkex.com.hk/Market-Data/Securities-Prices/Equities/Equities-Quote?sym=' + code + '&sc_lang=en' elif 'Nasdaq' in exchange: - exchangeUrl = 'https://www.nasdaq.com/market-activity/stocks/' + ticker.lower() + exchange_url = 'https://www.nasdaq.com/market-activity/stocks/' + ticker.lower() elif exchange == 'NYSE': - exchangeUrl = 'https://www.nyse.com/quote/XNYS:' + ticker + exchange_url = 'https://www.nyse.com/quote/XNYS:' + ticker elif exchange == 'Taiwan': exchange = 'TWSE' - exchangeUrl = 'https://mis.twse.com.tw/stock/fibest.jsp?stock=' + ticker.split('.')[0] + '&lang=en_us' + exchange_url = 'https://mis.twse.com.tw/stock/fibest.jsp?stock=' + code + '&lang=en_us' elif exchange == 'Tokyo': exchange = 'JPX' - exchangeUrl = 'https://quote.jpx.co.jp/jpx/template/quote.cgi?F=tmp/e_stock_detail&MKTN=T&QCODE=' + ticker.split('.')[0] + exchange_url = 'https://quote.jpx.co.jp/jpx/template/quote.cgi?F=tmp/e_stock_detail&MKTN=T&QCODE=' + code else: - exchangeUrl = 'https://www.google.com/search?q=stock+exchange+' + exchange + '+' + ticker.split('.')[0] + '&btnI' - exchangeLink = util.link(exchangeUrl, exchange, service) - location = [] - if 'profile_city' in market_data[ticker]: - location.append(market_data[ticker]['profile_city']) - if 'profile_state' in market_data[ticker]: - location.append(market_data[ticker]['profile_state']) - if 'profile_country' in market_data[ticker]: - profile_country = market_data[ticker]['profile_country'] - location.append(profile_country) - if 'profile_website' in market_data[ticker]: - website = market_data[ticker]['profile_website'] - website = util.link(website, util.strip_url(website), service) + exchange_url = 'https://www.google.com/search?q=stock+exchange+' + exchange + '+' + code + '&btnI' + exchange_link = util.link(exchange_url, exchange, service) + location = [data[key] for key in ('profile_city', 'profile_state', 'profile_country') if key in data] if payload: payload.append("") if location: payload.append(webhook.bold("Location:", service) + " " + ', '.join(location)) - if 'profile_industry' in market_data[ticker] and 'profile_sector' in market_data[ticker]: - payload.append(webhook.bold("Classification: ", service) + f"{market_data[ticker]['profile_industry']}, {market_data[ticker]['profile_sector']}") - if 'profile_employees' in market_data[ticker]: - payload.append(webhook.bold("Employees: ", service) + f"{market_data[ticker]['profile_employees']:,}") - if 'profile_website' in market_data[ticker]: + if 'profile_industry' in data and 'profile_sector' in data: + payload.append(webhook.bold("Classification: ", service) + f"{data['profile_industry']}, {data['profile_sector']}") + if 'profile_employees' in data: + payload.append(webhook.bold("Employees: ", service) + f"{data['profile_employees']:,}") + if 'profile_website' in data: + links = [util.link(data['profile_website'], util.strip_url(data['profile_website']), service), exchange_link] if 'NYSE' in exchange or 'Nasdaq' in exchange: - finvizURL='https://finviz.com/quote.ashx?t=' + ticker - seekingalphaURL='https://seekingalpha.com/symbol/' + ticker - finvizLink = util.link(finvizURL, 'finviz', service) - seekingalphaLink = util.link(seekingalphaURL, 'seekingalpha', service) - payload.append(webhook.bold("Links: ", service) + f"{website} | {exchangeLink} | {finvizLink} | {macrotrendsLink} | {seekingalphaLink}") + links += [util.link('https://finviz.com/quote.ashx?t=' + ticker, 'finviz', service), macrotrends_link, + util.link('https://seekingalpha.com/symbol/' + ticker, 'seekingalpha', service)] elif exchange == 'ASX': - payload.append(webhook.bold("Links: ", service) + f"{website} | {exchangeLink} | {shortmanLink}") - else: - payload.append(webhook.bold("Links: ", service) + f"{website} | {exchangeLink}") + links.append(util.link('https://www.shortman.com.au/stock?q=' + code.lower(), 'shortman', service)) + payload.append(webhook.bold("Links: ", service) + ' | '.join(links)) if payload: payload.append("") - if 'currency' in market_data[ticker] and 'market_cap' in market_data[ticker]: - currency = market_data[ticker]['currency'] - market_cap = market_data[ticker]['market_cap'] - market_cap = util.humanUnits(market_cap) - payload.append(webhook.bold("Mkt cap:", service) + f" {currency} {market_cap}") - if 'free_cashflow' in market_data[ticker]: - cashflow = market_data[ticker]['free_cashflow'] - elif 'operating_cashflow' in market_data[ticker]: - cashflow = market_data[ticker]['operating_cashflow'] - if 'shareholder_equity' in market_data[ticker] and 'total_debt' in market_data[ticker]: - total_debt = market_data[ticker]['total_debt'] - shareholder_equity = market_data[ticker]['shareholder_equity'] - if 'profile_industry' in market_data[ticker] and 'total_cash' in market_data[ticker]: - if 'Bank' not in market_data[ticker]['profile_industry']: - emoji = '' - profile_industry = market_data[ticker]['profile_industry'] - total_cash = market_data[ticker]['total_cash'] - net_debt_equity_ratio = round(((total_debt - total_cash) / shareholder_equity * 100)) - if net_debt_equity_ratio > 40: - emoji = '⚠️ ' - if net_debt_equity_ratio > 0: - payload.append(webhook.bold("Net debt/equity ratio:", service) + f" {net_debt_equity_ratio}%{emoji}") - if 'earnings_date' in market_data[ticker]: - earnings_date = datetime.datetime.fromtimestamp(market_data[ticker]['earnings_date']) - human_earnings_date = earnings_date.strftime('%b %d') - + if 'currency' in data and 'market_cap' in data: + payload.append(webhook.bold("Mkt cap:", service) + f" {currency} {util.humanUnits(data['market_cap'])}") + if all(key in data for key in ('shareholder_equity', 'total_debt', 'total_cash', 'profile_industry')) and data['shareholder_equity'] and 'Bank' not in data['profile_industry']: + net_debt_equity_ratio = round((data['total_debt'] - data['total_cash']) / data['shareholder_equity'] * 100) + if net_debt_equity_ratio > 0: + payload.append(webhook.bold("Net debt/equity ratio:", service) + f" {net_debt_equity_ratio}%" + ('⚠️ ' if net_debt_equity_ratio > 40 else '')) + if 'earnings_date' in data: + earnings_date = datetime.datetime.fromtimestamp(data['earnings_date']) if earnings_date > now: - payload.append(webhook.bold("Earnings date:", service) + f" {human_earnings_date}") + payload.append(webhook.bold("Earnings date:", service) + f" {earnings_date:%b %d}") else: - print("Skipping past earnings:", ticker, human_earnings_date) - if 'dividend' in market_data[ticker]: - dividend = market_data[ticker]['dividend'] - if market_data[ticker]['dividend'] > 0: - dividend = str(market_data[ticker]['dividend']) + '%' - payload.append(webhook.bold("Dividend:", service) + f" {dividend}") - if 'ex_dividend_date' in market_data[ticker]: - ex_dividend_date = datetime.datetime.fromtimestamp(market_data[ticker]['ex_dividend_date']) - human_exdate = ex_dividend_date.strftime('%b %d') - if ex_dividend_date > now: - payload.append(webhook.bold("Ex-dividend date:", service) + f" {human_exdate}") - else: - print("Skipping past ex-dividend:", ticker, human_exdate) + print("Skipping past earnings:", ticker, f"{earnings_date:%b %d}") + if data.get('dividend', 0) > 0: + payload.append(webhook.bold("Dividend:", service) + f" {data['dividend']}%") + if 'ex_dividend_date' in data: + ex_dividend_date = datetime.datetime.fromtimestamp(data['ex_dividend_date']) + if ex_dividend_date > now: + payload.append(webhook.bold("Ex-dividend date:", service) + f" {ex_dividend_date:%b %d}") + else: + print("Skipping past ex-dividend:", ticker, f"{ex_dividend_date:%b %d}") + cashflow = data.get('free_cashflow', data.get('operating_cashflow')) if cashflow: - if cashflow < 0: - payload.append(webhook.bold("Cashflow positive:", service) + " no ⚠️ ") - else: - payload.append(webhook.bold("Cashflow positive:", service) + " yes") - if 'net_income' in market_data[ticker]: - if market_data[ticker]['net_income'] <= 0: - payload.append(webhook.bold("Profitable:", service) + " no ⚠️ ") - else: - payload.append(webhook.bold("Profitable:", service) + " yes") + payload.append(webhook.bold("Cashflow positive:", service) + (" no ⚠️ " if cashflow < 0 else " yes")) + if 'net_income' in data: + payload.append(webhook.bold("Profitable:", service) + (" no ⚠️ " if data['net_income'] <= 0 else " yes")) else: payload.append(webhook.bold("Profitable:", service) + " unknown ⚠️") if payload: payload.append("") - if 'earningsQ' in market_data[ticker]: - deltas = [(doDelta(market_data[ticker][key]), label) for key, label in (('revenueQ', 'quarterly revenue'), ('earningsQ', 'quarterly earnings'), + if 'earningsQ' in data: + deltas = [(doDelta(data[key]), label) for key, label in (('revenueQ', 'quarterly revenue'), ('earningsQ', 'quarterly earnings'), ('revenueY', 'annual revenue'), ('earningsY', 'annual earnings'))] payload += [f"{delta}\t{label} delta" for delta, label in deltas] # doDelta always gives at least three symbols if payload: payload.append("") - if 'revenueEstimateY' in market_data[ticker]: - emoji = '' - revenueEstimateY = int(round(market_data[ticker]['revenueEstimateY'])) - if revenueEstimateY <= 0: - emoji = '⚠️ ' - prefix = '' - else: - prefix='+' - payload.append(webhook.bold("Revenue forecast (1Y):", service) + f" {prefix}{revenueEstimateY}% {emoji}") - if 'earningsEstimateY' in market_data[ticker]: - emoji = '' - prefix = '' - earningsEstimateY = int(round(market_data[ticker]['earningsEstimateY'])) - if earningsEstimateY <= 0: - emoji = '⚠️ ' - else: - prefix='+' - payload.append(webhook.bold("Earnings forecast (1Y):", service) + f" {prefix}{earningsEstimateY}% {emoji}") - if 'insiderBuy' in market_data[ticker]: - emoji='' - insiderBuy = market_data[ticker]['insiderBuy'] - insiderSell = market_data[ticker]['insiderSell'] - insiderBuyValue = market_data[ticker]['insiderBuyValue'] - insiderSellValue = market_data[ticker]['insiderSellValue'] - if insiderBuy > insiderSell: - action = 'Buy' - humanValue = util.humanUnits(insiderBuyValue) - payload.append(webhook.bold("Net insider action (3M):", service) + f" {action} {currency} {humanValue} {emoji}") - elif insiderBuy < insiderSell: - emoji = '⚠️ ' - action = 'Sell' - humanValue = util.humanUnits(insiderSellValue) - payload.append(webhook.bold("Net insider action (3M):", service) + f" {action} {currency} {humanValue}{emoji}") - if 'short_percent' in market_data[ticker]: - emoji='' - short_percent = market_data[ticker]['short_percent'] - if short_percent > 10: - emoji = '⚠️ ' - payload.append(webhook.bold("Shorted stock:", service) + f" {short_percent}%{emoji}") - if 'recommend' in market_data[ticker]: - recommend = market_data[ticker]['recommend'].replace('_', ' ') - recommend_index = market_data[ticker]['recommend_index'] - recommend_analysts = market_data[ticker]['recommend_analysts'] - payload.append(webhook.bold("Score:", service) + f" {recommend_index} {recommend} ({recommend_analysts} analysts)") + for key, label in (('revenueEstimateY', 'Revenue forecast (1Y):'), ('earningsEstimateY', 'Earnings forecast (1Y):')): + if key in data: + growth = int(round(data[key])) + payload.append(webhook.bold(label, service) + (f" +{growth}% " if growth > 0 else f" {growth}% ⚠️ ")) + if 'insiderBuy' in data: + if data['insiderBuy'] > data['insiderSell']: + payload.append(webhook.bold("Net insider action (3M):", service) + f" Buy {currency} {util.humanUnits(data['insiderBuyValue'])} ") + elif data['insiderBuy'] < data['insiderSell']: + payload.append(webhook.bold("Net insider action (3M):", service) + f" Sell {currency} {util.humanUnits(data['insiderSellValue'])}⚠️ ") + if 'short_percent' in data: + payload.append(webhook.bold("Shorted stock:", service) + f" {data['short_percent']}%" + ('⚠️ ' if data['short_percent'] > 10 else '')) + if 'recommend' in data: + payload.append(webhook.bold("Score:", service) + f" {data['recommend_index']} {data['recommend'].replace('_', ' ')} ({data['recommend_analysts']} analysts)") if payload: payload.append("") - if 'regularMarketPrice' in market_data[ticker]: - regularMarketPrice = market_data[ticker]['regularMarketPrice'] - currency = market_data.get(ticker, {}).get('currency') - prePostMarketPrice = None - marketState = market_data[ticker]['marketState'] - if marketState != 'REGULAR' and 'prePostMarketPrice' in market_data[ticker]: - prePostMarketPrice = market_data.get(ticker, {}).get('prePostMarketPrice') - payload.append(webhook.bold("Price:", service) + f" {currency} {regularMarketPrice:,.2f} ({prePostMarketPrice:,.2f} after hrs)") - else: - payload.append(webhook.bold("Price:", service) + f" {currency} {regularMarketPrice:,.2f}" ) - if 'price_to_earnings_trailing' in market_data[ticker]: - trailingPe = str(int(round(market_data[ticker]['price_to_earnings_trailing']))) - else: - trailingPe = 'N/A ⚠️ ' - if 'net_income' in market_data[ticker] and market_data[ticker]['net_income'] > 0: - payload.append(webhook.bold("Trailing P/E:", service) + f" {trailingPe}") - if 'netIncomeToCommon' in market_data[ticker] and market_data[ticker]['netIncomeToCommon'] > 0: - currentPe = round(market_data[ticker]['regularMarketPrice'] / (market_data[ticker]['netIncomeToCommon'] / market_data[ticker]['sharesOutstanding'])) - payload.append(webhook.bold("Current P/E:", service) + f" {currentPe}") - if 'price_to_earnings_forward' in market_data[ticker]: - forwardPe = int(round(market_data[ticker]['price_to_earnings_forward'])) - emoji='' - if 'profile_industry' in market_data[ticker]: - profile_industry = market_data[ticker]['profile_industry'] - if 'Software' in profile_industry and forwardPe > 100: - emoji = '⚠️ ' - elif 'Software' not in profile_industry and forwardPe > 30: - emoji = '⚠️ ' - if 'price_to_earnings_trailing' in market_data[ticker] and forwardPe > int(trailingPe): - emoji = '⚠️ ' - if forwardPe < 0: - emoji = '⚠️ ' - payload.append(webhook.bold("Forward P/E:", service) + f" {str(forwardPe)} {emoji}") - if 'price_to_earnings_peg' in market_data[ticker]: - peg = round(market_data[ticker]['price_to_earnings_peg'], 1) - payload.append(webhook.bold("PEG ratio:", service) + f" {str(peg)}") - if 'price_to_sales' in market_data[ticker]: - price_to_sales = round(market_data[ticker]['price_to_sales'], 1) - if price_to_sales > 15: - emoji = '⚠️ ' - payload.append(webhook.bold("PS ratio:", service) + f" {str(price_to_sales)} {emoji}") + if 'regularMarketPrice' in data: + price_line = webhook.bold("Price:", service) + f" {currency} {data['regularMarketPrice']:,.2f}" + if state != 'REGULAR' and 'prePostMarketPrice' in data: + price_line += f" ({data['prePostMarketPrice']:,.2f} after hrs)" + payload.append(price_line) + trailing_pe = str(int(round(data['price_to_earnings_trailing']))) if 'price_to_earnings_trailing' in data else 'N/A ⚠️ ' + if data.get('net_income', 0) > 0: + payload.append(webhook.bold("Trailing P/E:", service) + f" {trailing_pe}") + if data.get('netIncomeToCommon', 0) > 0 and data.get('sharesOutstanding') and 'regularMarketPrice' in data: + current_pe = round(data['regularMarketPrice'] / (data['netIncomeToCommon'] / data['sharesOutstanding'])) + payload.append(webhook.bold("Current P/E:", service) + f" {current_pe}") + if 'price_to_earnings_forward' in data: + forward_pe = int(round(data['price_to_earnings_forward'])) + warn = forward_pe < 0 + if 'profile_industry' in data: # software is dear: it warns above 100, the rest above 30 or above their trailing P/E + warn = warn or forward_pe > (100 if 'Software' in data['profile_industry'] else 30) + warn = warn or ('price_to_earnings_trailing' in data and forward_pe > int(trailing_pe)) + payload.append(webhook.bold("Forward P/E:", service) + f" {forward_pe} " + ('⚠️ ' if warn else '')) + if 'price_to_earnings_peg' in data: + payload.append(webhook.bold("PEG ratio:", service) + f" {round(data['price_to_earnings_peg'], 1)}") + if 'price_to_sales' in data: + price_to_sales = round(data['price_to_sales'], 1) + payload.append(webhook.bold("PS ratio:", service) + f" {price_to_sales} " + ('⚠️ ' if price_to_sales > 15 else '')) if payload: payload.append("") @@ -779,30 +679,22 @@ def prepare_profile_payload(service, user, ticker): for interval in ('5Y', '1Y', '1M', '1D'): if interval in price_history: percent = price_history[interval] - emoji = util.get_emoji(percent) - payload.append(f"{emoji} {webhook.bold(interval + ':', service)} {percent}%") - percent_change = market_data[ticker]['percent_change'] - marketState = market_data[ticker]['marketState'] - if marketState != 'REGULAR' and 'percent_change_premarket' in market_data[ticker]: - percent_change_premarket = market_data[ticker]['percent_change_premarket'] - emoji = util.get_emoji(percent_change_premarket) - payload.append(f"{emoji} {webhook.bold('Pre-market:', service)} {percent_change_premarket:,}%") - elif marketState != 'REGULAR' and 'percent_change_postmarket' in market_data[ticker]: - percent_change_postmarket = market_data[ticker]['percent_change_postmarket'] - emoji = util.get_emoji(percent_change_postmarket) - payload.append(f"{emoji} {webhook.bold('Post-market:', service)} {percent_change_postmarket:,}%") - - if 'profile_website' in market_data[ticker] and config_hyperlinkFooter and config_hyperlink: - footer = f"{gfinanceLink} | {yahooLink} | {swsLink}" - if payload and footer: + payload.append(f"{util.get_emoji(percent)} {webhook.bold(interval + ':', service)} {percent}%") + if state != 'REGULAR': + for key, label in (('percent_change_premarket', 'Pre-market:'), ('percent_change_postmarket', 'Post-market:')): + if key in data: + payload.append(f"{util.get_emoji(data[key])} {webhook.bold(label, service)} {data[key]:,}%") + break + + if 'profile_website' in data and config_hyperlinkFooter and config_hyperlink: + if payload: payload.append("") - payload.append(footer) + payload.append(f"{gfinance_link} | {util.yahoo_link(ticker, service, brief=True, text='yahoo')} | {sws_link}") + heading = f"{webhook.bold(title, service)} ({ticker_link}) {MARKET_STATE_EMOJI.get(state, '🔴') if state else ''}" if ticker_orig == ticker: - payload.insert(0, f"{webhook.bold(profile_title, service)} ({ticker_link}) {marketStateEmoji}") + payload.insert(0, heading) else: - payload.insert(0, f"I could not find {ticker_orig} but I found {ticker_link}:") - payload.insert(1, "") - payload.insert(2, f"{webhook.bold(profile_title, service)} ({ticker_link}) {marketStateEmoji}") + payload[:0] = [f"I could not find {ticker_orig} but I found {ticker_link}:", "", heading] payload = [i[0] for i in groupby(payload)] # de-dupe white space if len(payload) < 2: payload.append("no data found") diff --git a/lib/sharesight.py b/lib/sharesight.py index 7acf3279..df8540a4 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -88,7 +88,7 @@ def get_portfolios(refresh=False): portfolio_dict[portfolio['name']] = portfolio['id'] if not len(portfolio_dict): print("No portfolios found. Exiting.", file=sys.stderr) - raise RuntimeError(f"No Sharesight portfolios found") + raise RuntimeError("No Sharesight portfolios found") print(portfolio_dict) return portfolio_dict diff --git a/lib/simplywallst.py b/lib/simplywallst.py index 3ddd826c..c49388d1 100644 --- a/lib/simplywallst.py +++ b/lib/simplywallst.py @@ -1,24 +1,12 @@ -import os -import json -from lib.config import data_file +from lib.config import data_json -def get_url(symbol, name, exchange): - sws = {} - files = ['finbot_sws_custom.json', 'finbot_sws_asx200.json', 'finbot_sws_sp500.json', 'finbot_sws_nasdaq100.json'] - for file in files: - file = data_file(file) - sws = sws | read_cache(file) - if symbol in sws: - return sws[symbol] - else: - return 'https://www.google.com/search?q=site:simplywall.st+(' + name + '+' + exchange + ':' + symbol.split('.')[0] + ')+Stock&btnI' +FILES = ['finbot_sws_custom.json', 'finbot_sws_asx200.json', 'finbot_sws_sp500.json', 'finbot_sws_nasdaq100.json'] # the first to list a ticker wins -def read_cache(cacheFile): - if os.path.isfile(cacheFile): - with open(cacheFile, "r", encoding="utf-8") as f: - cacheDict = json.load(f) - return cacheDict - else: - print("can't find", cacheFile) - return {} +def get_url(symbol, name, exchange): + """The ticker's page on simplywall.st, from the lists in FILES; without one, a search that lands on it.""" + for file in FILES: + url = data_json(file, {}).get(symbol) + if url: + return url + return 'https://www.google.com/search?q=site:simplywall.st+(' + name + '+' + exchange + ':' + symbol.split('.')[0] + ')+Stock&btnI' diff --git a/milestone.py b/milestone.py index 93f62e45..f16ccd20 100755 --- a/milestone.py +++ b/milestone.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -import json, re, sys +import sys from itertools import groupby from lib.config import * -from lib import sharesight from lib import webhook from lib import util from lib import yahoo diff --git a/rating.py b/rating.py index 3306463a..1f0448f2 100755 --- a/rating.py +++ b/rating.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -import json, re, sys +import sys from lib.config import * -from lib import sharesight from lib import webhook from lib import util from lib import yahoo @@ -43,7 +42,7 @@ def get_emoji(old, new): payload.sort() if payload: if not specific_stock: - message = f'Consensus rating changes:' + message = 'Consensus rating changes:' message = webhook.bold(message, service) payload.insert(0, message) else: diff --git a/reminder.py b/reminder.py index 9f8b33fa..f6082411 100755 --- a/reminder.py +++ b/reminder.py @@ -1,21 +1,14 @@ #!/usr/bin/env python3 import sys -import json import datetime import zoneinfo -from pathlib import Path from dateutil.relativedelta import relativedelta from lib.config import * from lib import util from lib import webhook -def load_reminders(): - with open(data_file("reminder.json"), "r", encoding="utf-8") as f: - return json.load(f) - - def lambda_handler(): def prepare_finance_calendar_payload(service): payload = [] @@ -26,7 +19,7 @@ def prepare_finance_calendar_payload(service): year = str(localtime.strftime('%Y')) # 2023 # --- Simple, static one-line reminders loaded from reminders.json --- - for reminder in load_reminders(): + for reminder in data_json("reminder.json", []): if reminder.get("month_day") != month_and_day: continue if reminder.get("year") and reminder["year"] != year: diff --git a/shorts.py b/shorts.py index fc9caa07..c63907a1 100755 --- a/shorts.py +++ b/shorts.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -import json, re, sys +import sys from lib.config import * -from lib import sharesight from lib import webhook from lib import util from lib import yahoo diff --git a/tests/test_finbot.py b/tests/test_finbot.py index 49e5b033..df4c8614 100644 --- a/tests/test_finbot.py +++ b/tests/test_finbot.py @@ -30,7 +30,7 @@ def test_zoneinfo_offset_is_not_lmt(self): self.assertEqual(datetime.datetime(2022, 8, 18, tzinfo=tz).utcoffset(), datetime.timedelta(hours=10)) def test_reminder_runs(self): - with mock.patch.object(reminder, 'load_reminders', lambda: []): + with mock.patch.object(reminder, 'data_json', lambda name, default=None: []): reminder.lambda_handler() def test_bot_handler(self): diff --git a/tests/test_reports.py b/tests/test_reports.py index f19471a1..98ac75b1 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -65,6 +65,24 @@ def test_prepare_beta(self): def test_prepare_profile(self): self.assertTrue(reports.prepare_profile_payload('telegram', '@u', 'T1')) + def test_a_profile_leaves_out_what_yahoo_did_not_send(self): + """Each of these fields missing (or a zero equity) used to crash the whole profile.""" + full = market_entry('T1') | {'insiderBuy': 1, 'insiderSell': 0, 'insiderBuyValue': 5e6, 'insiderSellValue': 0, 'netIncomeToCommon': 1e6, + 'sharesOutstanding': 1e6, 'shareholder_equity': 1e9, 'total_debt': 9e8, 'total_cash': 1e8, 'profile_industry': 'Tools'} + for missing in ('marketState', 'profile_exchange', 'market_cap', 'currency', 'regularMarketPrice', 'sharesOutstanding', 'percent_change'): + with self.subTest(missing=missing): + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: {k: v for k, v in full.items() if k != missing}}) + self.assertIn('Co T1', reports.prepare_profile_payload('telegram', '@u', 'T1')[0]) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: full | {'shareholder_equity': 0}}) + self.assertNotIn('Net debt/equity', str(reports.prepare_profile_payload('telegram', '@u', 'T1'))) + + def test_a_profile_warns_of_a_ps_ratio_only_above_15(self): + for ps, warned in ((12.3, False), (15.1, True)): # forward P/E 99 warns (above 30): its warning used to show on the PS line too + with self.subTest(ps=ps): + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'price_to_sales': ps, 'price_to_earnings_forward': 99, 'profile_industry': 'Tools'}}) + line = next(line for line in reports.prepare_profile_payload('telegram', '@u', 'T1') if 'PS ratio' in line) + self.assertEqual('⚠️' in line, warned, line) + def test_prepare_watchlist(self): self.assertTrue(reports.prepare_watchlist('telegram', '@u')) diff --git a/trades.py b/trades.py index ce4206fe..9c8bafe1 100755 --- a/trades.py +++ b/trades.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import json, os, random, sys, re +import random, sys import datetime from lib.config import * From e9900278d10de4a5b3460a2f041f5e8a7a7cc9b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:52:20 +0000 Subject: [PATCH 15/25] Telegram calls share telegram.call; transform_title reads its replacements from two ordered tables - Pinning, button toasts, text and chart edits and the typing notice go through telegram.call; telegram.try_call is the same for calls whose failure is only logged. Pinning used to crash the earnings report on a non-JSON answer or a network error. - transform_title's ~90 replace lines are TITLE_TRIMS and TITLE_SHORTENINGS, in their old order. Fixed on the way: a title ending " Se" kept it (the check was for Se, the removal for SE), a trailing AG, SE, & or "and" removed every occurrence of it in the title rather than the one at the end, and runs of spaces left behind are collapsed. - The test fixture points lib.telegram at the fake Telegram too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/telegram.py | 30 +++----- lib/util.py | 147 +++++++++++----------------------------- lib/webhook.py | 56 ++++----------- tests/common.py | 2 +- tests/test_providers.py | 23 +++++++ tests/test_util.py | 4 ++ 6 files changed, 93 insertions(+), 169 deletions(-) diff --git a/lib/telegram.py b/lib/telegram.py index a0bc080b..c918aca5 100644 --- a/lib/telegram.py +++ b/lib/telegram.py @@ -14,6 +14,13 @@ def call(method, **kwargs): raise RuntimeError(f"Telegram {method}: {output.get('description') or response.status_code}") return output['result'] +def try_call(method, **kwargs): + """call(), for a call whose failure must not stop what the bot is doing: a failure is only logged (and None returned).""" + try: + return call(method, **kwargs) + except Exception as e: + print(f"Telegram {method} failed:", net.redact(e), file=sys.stderr) + def setWebhook(): print("registering", config_telegramOutgoingWebhook, file=sys.stderr) params = {'url': config_telegramOutgoingWebhook, "allowed_updates": json.dumps(['message', 'callback_query']), 'secret_token': config_telegramOutgoingToken} @@ -35,25 +42,10 @@ def register(): return '@' + getMe()['username'] def pinChatMessage(chat_id, message_id): - telegram_url = webhooks['telegram'] + 'pinChatMessage' - payload = { - 'chat_id': chat_id, - 'message_id': message_id, - 'disable_notification': True # no "pinned a message" notification - } - response = net.post(telegram_url, json=payload, timeout=config_http_timeout) - output = response.json() - if not output.get('ok'): - print("pinChatMessage failed:", output.get('description'), file=sys.stderr) - return output.get('result') + """Pin a message, without the "pinned a message" notification. A failure is only logged: the message is there either way.""" + try_call('pinChatMessage', json={'chat_id': chat_id, 'message_id': message_id, 'disable_notification': True}) def answerCallbackQuery(callback_query_id, text=None): - """Acknowledge a button press (stops the client's loading spinner).""" - payload = {'callback_query_id': callback_query_id} - if text: - payload['text'] = text - try: - net.post(webhooks['telegram'] + 'answerCallbackQuery', json=payload, timeout=config_http_timeout) - except Exception as e: - print("answerCallbackQuery failed:", net.redact(e), file=sys.stderr) + """Acknowledge a button press (stops the client's loading spinner), with `text` as a toast. A failure is only logged.""" + try_call('answerCallbackQuery', json={'callback_query_id': callback_query_id} | ({'text': text} if text else {})) diff --git a/lib/util.py b/lib/util.py index 9d533b7f..9db60b77 100644 --- a/lib/util.py +++ b/lib/util.py @@ -11,120 +11,53 @@ from lib import sharesight +TITLE_TRIMS = [ # transform_title: (text, replacement), in order, as each sees what the ones before it made + (' - ', ' '), ('Roundhill ETF Trust ', ''), ('Class A Common Stock', ''), + ('BlackRock Institutional Trust Company N.A.', ''), ('Listed Funds Trust', ''), ('GraniteShares ETF Trust', ''), + ('ProShares Trust', ''), ('VanEck ETF Trust', ''), ('Direxion Shares ETF Trust', ''), (' of California', ''), + ('Magnificent', 'Mag'), ('First Trust NASDAQ Clean Edge Green Energy Index Fund', 'First Trust Clean Energy'), + ('First Trust Exchange-Traded Fund III First Trust Nasdaq Clean Edge Clean Energy Index Fund', 'First Trust Green Energy'), + ('Mirae Asset Global Investments (Hong Kong) Limited', ''), ('Space Exploration Technologies', 'SpaceX'), + ('Exchange Traded Funds Series OFC', ''), ('Global X Global X', 'Global X'), ('Tema ETF Trust', ''), (' AUD', ' '), + (' USD', ' '), ('New York Shares', ''), ('China Electric Vehicle and Battery ETF', 'China EV ETF'), + ('China Electric Vehicle ETF', 'China EV ETF'), ('Atlantica Sustainable Infrastructure', 'Atlantica Sustainable'), + ('Advanced Micro Devices', 'AMD'), ('Taiwan Semiconductor Manufacturing', 'TSM'), + ('Flight Centre Travel', 'Flight Centre'), ('Global X Funds ', ''), (' of Australia', ''), + ('National Australia Bank', 'NAB'), (' PLC', ''), (' p.l.c.', ''), (' P.l.c.', ''), (' P.L.C.', ''), (' Units', '') +] + +TITLE_SHORTENINGS = [ # and these after the suffixes come off + ('Microbalifesciences', 'Microba Life Sciences'), ('Walt Disney Co (The)', 'Disney'), ('Lisenergylimited', 'LI-S Energy'), + ('Invesco Capital Management LLC ', ''), ('QUALCOMM', 'Qualcomm'), ('Ordinary Shares Class A', ''), + ('Ordinary Shares Class C', ''), ('Lbt Innovations', 'LBT Innovations'), + ('Vanguard Information Technology', 'Vanguard Infotech'), ('The ', ''), (' Shares', ''), ('Rea ', 'REA '), + ('Csl ', 'CSL '), ('Battery ', 'Batt '), ('.com', ''), ('Etf', 'ETF'), ('N.V.', ''), (' NV ', ' '), ('New York Re', ''), + (' Australian', ' Aus'), (' Australia', ' Aus'), (' Infrastructure', ' Infra'), (' Manufacturing Company', ' '), + (' Limited', ' '), (' Ltd', ' '), (' Holdings', ' '), (' Holding', ' '), (' (Holdings)', ''), (' Corporation', ' '), + (' Incorporated', ' '), (' incorporated', ' '), (' Technologies', ' '), (' Technology', ' '), (' Enterprises', ' '), + (' Enterprise', ' '), (' Enterpr', ' '), (' Ventures', ' '), (' Resources', ' '), (' (Intrada', ''), (' Structured', ''), + (' Co.', ' '), (' Corp.', ' '), (' Corp', ' '), (' Tech ', ' '), (' Company', ' '), (' Group', ' '), (', Inc', ' '), + (' Inc', ' '), (' Plc', ' '), (' plc', ' '), (' Index', ' '), (' ADR', ' '), (' Daily', ' '), (' Long', ' '), + (' Bull', ' '), (' .', ' '), (' ,', ' ') +] + def transform_title(title): + """A company or fund name as the reports show it: shorter, without fund-trust boilerplate and legal suffixes.""" title = title.replace(' FPO', '') if title.isupper() or title.islower(): title = title.title() - title = title.replace(' - ', ' ') - title = title.replace('Roundhill ETF Trust ', '') - title = title.replace('Class A Common Stock', '') - title = title.replace('BlackRock Institutional Trust Company N.A.', '') - title = title.replace('Listed Funds Trust', '') - title = title.replace('GraniteShares ETF Trust', '') - title = title.replace('ProShares Trust', '') - title = title.replace('VanEck ETF Trust', '') - title = title.replace('Direxion Shares ETF Trust', '') - title = title.replace(' of California', '') - title = title.replace('Magnificent', 'Mag') - title = title.replace('First Trust NASDAQ Clean Edge Green Energy Index Fund', 'First Trust Clean Energy') - title = title.replace('First Trust Exchange-Traded Fund III First Trust Nasdaq Clean Edge Clean Energy Index Fund', 'First Trust Green Energy') - title = title.replace('Mirae Asset Global Investments (Hong Kong) Limited', '') - title = title.replace('Space Exploration Technologies', 'SpaceX') - title = title.replace('Exchange Traded Funds Series OFC', '') - title = title.replace('Global X Global X', 'Global X') - title = title.replace('Tema ETF Trust', '') - title = title.replace(' AUD', ' ') - title = title.replace(' USD', ' ') - title = title.replace('New York Shares', '') - title = title.replace('China Electric Vehicle and Battery ETF', 'China EV ETF') - title = title.replace('China Electric Vehicle ETF', 'China EV ETF') - title = title.replace('Atlantica Sustainable Infrastructure', 'Atlantica Sustainable') - title = title.replace('Advanced Micro Devices', 'AMD') - title = title.replace('Taiwan Semiconductor Manufacturing', 'TSM') - title = title.replace('Flight Centre Travel', 'Flight Centre') - title = title.replace('Global X Funds ', '') - title = title.replace(' of Australia', '') - title = title.replace('National Australia Bank', 'NAB') - title = title.replace(' PLC', '') - title = title.replace(' p.l.c.', '') - title = title.replace(' P.l.c.', '') - title = title.replace(' P.L.C.', '') - title = title.replace(' Units', '') - if title.endswith(' AG'): - title = title.replace(' AG', '') - if title.endswith(' SE'): - title = title.replace(' SE', '') - if title.endswith(' Se'): - title = title.replace(' SE', '') + for text, replacement in TITLE_TRIMS: + title = title.replace(text, replacement) + for suffix in (' AG', ' SE', ' Se'): + title = title.removesuffix(suffix) title = re.sub(r'\[\w+\]$', '', title) - title = title.replace('Microbalifesciences', 'Microba Life Sciences') - title = title.replace('Walt Disney Co (The)', 'Disney') - title = title.replace('Lisenergylimited', 'LI-S Energy') - title = title.replace('Invesco Capital Management LLC ', '') - title = title.replace('QUALCOMM', 'Qualcomm') - title = title.replace('Ordinary Shares Class A', '') - title = title.replace('Ordinary Shares Class C', '') - title = title.replace('Lbt Innovations', 'LBT Innovations') - title = title.replace('Vanguard Information Technology', 'Vanguard Infotech') - title = title.replace('The ', '') - title = title.replace(' Shares', '') - title = title.replace('Rea ', 'REA ') - title = title.replace('Csl ', 'CSL ') - title = title.replace('Battery ', 'Batt ') - title = title.replace('.com', '') - title = title.replace('Etf', 'ETF') - title = title.replace('N.V.', '') - title = title.replace(' NV ', ' ') - title = title.replace('New York Re', '') - title = title.replace(' Australian', ' Aus') - title = title.replace(' Australia', ' Aus') - title = title.replace(' Infrastructure', ' Infra') - title = title.replace(' Manufacturing Company', ' ') - title = title.replace(' Limited', ' ') - title = title.replace(' Ltd', ' ') - title = title.replace(' Holdings', ' ') - title = title.replace(' Holding', ' ') - title = title.replace(' (Holdings)', '') - title = title.replace(' Corporation', ' ') - title = title.replace(' Incorporated', ' ') - title = title.replace(' incorporated', ' ') - title = title.replace(' Technologies', ' ') - title = title.replace(' Technology', ' ') - title = title.replace(' Enterprises', ' ') - title = title.replace(' Enterprise', ' ') - title = title.replace(' Enterpr', ' ') - title = title.replace(' Ventures', ' ') - title = title.replace(' Resources', ' ') - title = title.replace(' (Intrada', '') - title = title.replace(' Structured', '') - title = title.replace(' Co.', ' ') - title = title.replace(' Corp.', ' ') - title = title.replace(' Corp', ' ') - title = title.replace(' Tech ', ' ') - title = title.replace(' Company', ' ') - title = title.replace(' Group', ' ') - title = title.replace(', Inc', ' ') - title = title.replace(' Inc', ' ') - title = title.replace(' Plc', ' ') - title = title.replace(' plc', ' ') - title = title.replace(' Index', ' ') - title = title.replace(' ADR', ' ') - title = title.replace(' Daily', ' ') - title = title.replace(' Long', ' ') - title = title.replace(' Bull', ' ') - title = title.replace(' .', ' ') - title = title.replace(' ,', ' ') + for text, replacement in TITLE_SHORTENINGS: + title = title.replace(text, replacement) if title.islower(): title = title.title() - title = title.strip() - if title.endswith(' &'): - title = title.replace(' &', '') - if title.endswith(' and'): - title = title.replace(' and', '') - title = title.replace(' Usd', ' USD') - title = title.replace(' ', ' ') - title = title.replace('Ishares', 'iShares') - return title + for suffix in (' &', ' and'): + title = title.strip().removesuffix(suffix) + return ' '.join(title.replace(' Usd', ' USD').replace('Ishares', 'iShares').split()) MARKETS = { # Sharesight market code -> (flag, currency); no currency where a market quotes in several (LSE and HKEX allow non-home currencies) 'ASX': ('🇦🇺', 'AUD'), diff --git a/lib/webhook.py b/lib/webhook.py index b1dfc530..5450e3e6 100644 --- a/lib/webhook.py +++ b/lib/webhook.py @@ -9,6 +9,7 @@ from lib.config import * from lib import net +from lib import telegram def write(service, url, payload_string, chat_id=None, message_id=None, reply_markup=None): headers = {'Content-type': 'application/json'} @@ -284,60 +285,31 @@ def sendPhoto(chat_id, image_data, caption, service, message_id=None, reply_mark return payload_wrapper(service, chat_url(service, chat_id) if chat_id else webhooks.get(service), caption.split('\n'), chat_id, message_id) def editMessageText(chat_id, message_id, text, reply_markup=None): - """Telegram: replace a text message's text and buttons in place. Returns None if it could not be edited.""" + """Telegram: replace a text message's text and buttons in place. A failure is only logged.""" payload = {'chat_id': chat_id, 'message_id': message_id, 'text': text, 'parse_mode': 'HTML', 'disable_web_page_preview': True} if reply_markup is not None: payload['reply_markup'] = reply_markup - try: - output = net.post(webhooks['telegram'] + 'editMessageText', json=payload, timeout=config_http_timeout).json() - except Exception as e: - print("Failure executing editMessageText:", net.redact(e), file=sys.stderr) - return None - if not output.get('ok'): - print(output.get('error_code'), output.get('description'), file=sys.stderr) - return None - return output + telegram.try_call('editMessageText', json=payload) def editMessageMedia(chat_id, message_id, image_data, caption, reply_markup=None): - """Telegram: replace a photo message's image and caption in place.""" - url = webhooks['telegram'] + 'editMessageMedia' - caption = fit_caption(caption) - media = {'type': 'photo', 'media': 'attach://photo', 'caption': caption, 'parse_mode': 'HTML'} + """Telegram: replace a photo message's image and caption in place. Raises RuntimeError when Telegram refuses, unless the message + is already so (pressing the active period again is harmless); a request that fails on the way is only logged.""" + media = {'type': 'photo', 'media': 'attach://photo', 'caption': fit_caption(caption), 'parse_mode': 'HTML'} data = {'chat_id': chat_id, 'message_id': message_id, 'media': json.dumps(media)} if reply_markup: data['reply_markup'] = json.dumps(reply_markup) if hasattr(image_data, 'seek'): image_data.seek(0) try: - r = net.post(url, data=data, files={'photo': ('image.png', image_data)}, timeout=config_http_timeout) - output = r.json() - except Exception as e: - print("Failure executing editMessageMedia:", net.redact(e), file=sys.stderr) - return None - if not output.get('ok'): - description = output.get('description', r.reason) - if 'not modified' not in description: # pressing the active period again is harmless - print(output.get('error_code'), description, file=sys.stderr) - raise RuntimeError(f"Telegram: {description}") - return output + telegram.call('editMessageMedia', data=data, files={'photo': ('image.png', image_data)}) + except RuntimeError as e: + if 'not modified' not in str(e): + raise + except Exception as e: # it may have arrived even so: no error line for something the chat may already show + print("Telegram editMessageMedia failed:", net.redact(e), file=sys.stderr) def pleaseHold(action, service, chat_id): """Telegram's typing notice ("typing…" for a few seconds). worker.TypingIndicator repeats it every 7 seconds while a command works, on Telegram only: the other services have no such notice.""" - if service != 'telegram': - return None - url = webhooks['telegram'] + "sendChatAction?chat_id=" + str(chat_id) - data = { - 'chat_id': chat_id, - 'action': action - } - try: - r = net.post(url, data=data, timeout=config_http_timeout) - except Exception as e: - print("Typing indicator failed:", net.redact(e), file=sys.stderr) - return None - if r.status_code == 200: - print(r.status_code, "OK typing indicator to", service, file=sys.stderr) - else: - print(r.status_code, "error typing indicator to", service, file=sys.stderr) - return None + if service == 'telegram': + telegram.try_call('sendChatAction', data={'chat_id': chat_id, 'action': action}) diff --git a/tests/common.py b/tests/common.py index cd359da6..89203846 100644 --- a/tests/common.py +++ b/tests/common.py @@ -61,7 +61,7 @@ def setUp(self): self.patch(util, 'config_cache_dir', tmp.name) self.patch(config, 'config_var_dir', tmp.name) self.patch(webhook, 'webhooks', {'telegram': 'https://api.telegram.org/botX/'}) - for m in (performance, shorts, price, cal, trades, milestone, rating, worker, reminder): + for m in (performance, shorts, price, cal, trades, milestone, rating, worker, reminder, telegram): self.patch(m, 'webhooks', webhook.webhooks) self.sent = [] self.markups = [] # reply_markup of each photo sent diff --git a/tests/test_providers.py b/tests/test_providers.py index b72c8068..fdbbf8f6 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -589,6 +589,29 @@ def test_a_refused_call_raises(self): telegram.register() self.assertIn('HTTPS url must be provided', str(caught.exception)) + def test_edits_toasts_and_typing_go_through_call(self): + """A chart edit that Telegram refuses is an error (unless nothing changed); a toast, a text edit or a typing notice that + fails, or an edit lost on the way, is only logged.""" + def answer(outcome): + def post(url, **kwargs): + if isinstance(outcome, Exception): + raise outcome + return outcome + return post + refused = Resp({'ok': False, 'description': 'Bad Request: message to edit not found'}) + unchanged = Resp({'ok': False, 'description': 'Bad Request: message is not modified: specified new message content is the same'}) + for outcome, raises in ((refused, True), (unchanged, False), (ConnectionError('down'), False), (Resp({'ok': True, 'result': True}), False)): + with self.subTest(outcome=outcome): + self.patch(telegram.net, 'post', answer(outcome)) + with contextlib.redirect_stderr(io.StringIO()): + if raises: + self.assertRaisesRegex(RuntimeError, 'not found', webhook.editMessageMedia, '5', '9', io.BytesIO(b'png'), 'caption') + else: + webhook.editMessageMedia('5', '9', io.BytesIO(b'png'), 'caption') + telegram.answerCallbackQuery('1', 'Loading…') + webhook.editMessageText('5', '9', 'text') + webhook.pleaseHold('typing', 'telegram', '5') + def test_importing_never_calls_telegram(self): code = ('import requests\n' 'def refuse(*a, **k): raise SystemExit("network call on import")\n' diff --git a/tests/test_util.py b/tests/test_util.py index 094e51f5..ac440b18 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -37,6 +37,10 @@ def test_transform_title(self): self.assertEqual(util.transform_title('APPLE INC'), 'Apple') self.assertEqual(util.transform_title('Acme FPO'), 'Acme') self.assertEqual(util.transform_title('Mixed Case Name'), 'Mixed Case Name') + for title, expected in (('Allianz Se', 'Allianz'), ('Foo SE AG', 'Foo'), ('Johnson & Johnson &', 'Johnson & Johnson'), + ('Black and White and', 'Black and White'), ('Big Holdings Things Ltd', 'Big Things')): + with self.subTest(title=title): # a suffix comes off the end only; no run of spaces is left behind + self.assertEqual(util.transform_title(title), expected) class Formatting(unittest.TestCase): From ef0d4c315a44838a605e5743111fb8a495fd1b86 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 14:59:09 +0000 Subject: [PATCH 16/25] cal takes one event; one portfolio lookup answers an unknown name the same way everywhere; trades loses its dead lines - cal.lambda_handler(event='earnings' | 'ex-dividend') replaces the earnings and dividend flags (with neither set it crashed). An ADR whose primary listing Yahoo does not know is skipped instead of crashing the earnings report. "No events found" names the period as the heading does. - sharesight.pick_portfolios is the one lookup for .trades, .performance, .holdings and .pie. A name that matches no portfolio raises PortfolioNotFound, which the worker answers with the name and the portfolios there are. It used to be a ValueError, "no Sharesight data found", or one of two wordings of that answer. - trades: an interactive .trades whose trades are all neither buys nor sells says so instead of nothing; an unused start date, a second ticker and a portfolio filter that could not filter anything are gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- cal.py | 115 ++++++++++++++---------------------------- lib/reports.py | 25 ++++----- lib/sharesight.py | 19 +++++-- lib/worker.py | 9 ++-- performance.py | 7 +-- tests/test_edges.py | 15 ++++++ tests/test_jobs.py | 4 +- tests/test_reports.py | 13 ++++- trades.py | 38 +++----------- 9 files changed, 108 insertions(+), 137 deletions(-) diff --git a/cal.py b/cal.py index d9d8444b..6cb668d3 100755 --- a/cal.py +++ b/cal.py @@ -9,90 +9,56 @@ from lib import webhook from lib import yahoo -def lambda_handler(chat_id=config_telegramChatID, days=UNSET, service=None, specific_stock=None, message_id=None, interactive=False, earnings=False, dividend=False): +EVENTS = {'earnings': ('earnings_date', 'Upcoming earnings'), 'ex-dividend': ('ex_dividend_date', 'Upcoming ex-dividends')} # -> (field, heading) + +def lambda_handler(chat_id=config_telegramChatID, days=UNSET, service=None, specific_stock=None, message_id=None, interactive=False, event='earnings'): + """The tracked stocks' (or one stock's) upcoming earnings or ex-dividend dates, as `event` says. A scheduled earnings report + is pinned on Telegram.""" if days is UNSET: days = config_future_days - def prepare_payload(service, market_data, days): - payload_staging = [] - emoji = "📣" + field, heading = EVENTS[event] + def prepare_payload(service, market_data): now = datetime.datetime.now() soon = now + datetime.timedelta(days=days) - dates = set() - for ticker in market_data.copy(): - ticker_primary = ticker - emoji = util.flag_from_ticker(ticker) + found = [] # (when, flag, title, link) + for ticker in list(market_data): if market_data[ticker]['quoteType'] == 'ETF': continue - try: - if earnings: - timestamp = market_data[ticker]['earnings_date'] - elif dividend: - timestamp = market_data[ticker]['ex_dividend_date'] - timestamp = datetime.datetime.fromtimestamp(timestamp) - except (KeyError): + ticker_primary = ticker + if field not in market_data[ticker]: # an ADR or a second listing: its primary listing may have the date title_orig = market_data[ticker]['profile_title'] if ticker in primary_listing: # dict from lib.config ticker_primary = primary_listing[ticker] - elif ticker.endswith('.AX') and 'financialCurrency' in market_data[ticker] and market_data[ticker]['financialCurrency'] == 'NZD': + elif ticker.endswith('.AX') and market_data[ticker].get('financialCurrency') == 'NZD': ticker_primary = ticker.replace('.AX', '.NZ') # fixes GTK.AX and SKO.AX market_data = market_data | yahoo.fetch_detail(ticker_primary) - if 'profile_country' in market_data[ticker_primary]: # country is only available after fetch_detail - country = market_data[ticker_primary]['profile_country'] - if country in yahoo_country and '.' not in ticker_primary: # yahoo_country dict in lib.conig - ticker_primary = ticker_primary + '.' + yahoo_country[country] # fixes ASML and INFY - market_data = market_data | yahoo.fetch_detail(ticker_primary) - if ticker_primary not in market_data or title_orig != market_data[ticker_primary]['profile_title']: - continue - try: - if earnings: - timestamp = market_data[ticker_primary]['earnings_date'] - elif dividend: - timestamp = market_data[ticker_primary]['ex_dividend_date'] - timestamp = datetime.datetime.fromtimestamp(timestamp) - except (KeyError): + country = market_data.get(ticker_primary, {}).get('profile_country') # only in the details + if country in yahoo_country and '.' not in ticker_primary: # dict from lib.config + ticker_primary = ticker_primary + '.' + yahoo_country[country] # fixes ASML and INFY + market_data = market_data | yahoo.fetch_detail(ticker_primary) + primary = market_data.get(ticker_primary, {}) + if primary.get('profile_title') != title_orig or field not in primary: # not the same company, or no date there either continue - if (timestamp > now and timestamp < soon) or specific_stock: - title = market_data[ticker]['profile_title'] - ticker_link = util.yahoo_link(ticker, service) - dates.add(timestamp.date()) # (2023, 12, 30) - emoji = util.flag_from_ticker(ticker_primary) - payload_staging.append([emoji, title, f'({ticker_link})', timestamp]) - - def numeric_date(e): - return e[3] - payload_staging.sort(key=numeric_date) - + when = datetime.datetime.fromtimestamp(market_data[ticker_primary][field]) + if now < when < soon or specific_stock: + found.append((when, util.flag_from_ticker(ticker_primary), market_data[ticker]['profile_title'], util.yahoo_link(ticker, service))) + found.sort(key=lambda item: item[0]) payload = [] - for date in sorted(dates): - human_date = date.strftime('%b %d') # Dec 30 + for date in sorted({when.date() for when, *_ in found}): if not specific_stock: - payload.append("") - payload.append(webhook.bold(human_date, service)) - for e in payload_staging: - if date == e[3].date(): - if specific_stock: - e.insert(1, human_date) - payload.append(' '.join(e[:-1])) - - if payload: - if not specific_stock: - if earnings: - message = f"Upcoming earnings {util.days_english(days, 'for the next ')}:" - elif dividend: - message = f"Upcoming ex-dividends {util.days_english(days, 'for the next ')}:" - message = webhook.bold(message, service) - payload.insert(0, message) - else: - if interactive: - if specific_stock: - payload = [f"No events found for {tickers[0]}"] - else: - payload = [f"No events found for the next { f'{days} days' if days != 1 else 'day' }"] + payload += ["", webhook.bold(f"{date:%b %d}", service)] + for when, flag, title, link in found: + if when.date() == date: + payload.append(f"{flag} {date:%b %d} {title} ({link})" if specific_stock else f"{flag} {title} ({link})") + if payload and not specific_stock: + payload.insert(0, webhook.bold(f"{heading} {util.days_english(days, 'for the next ')}:", service)) + elif not payload and interactive: + payload = [f"No events found for {tickers[0]}" if specific_stock else f"No events found {util.days_english(days, 'for the next ')}"] return payload def pin_if_earnings(service, results): """Pin every message of the Telegram earnings report (several if it was chunked).""" - if not (earnings and not specific_stock and service == 'telegram' and results): + if not (event == 'earnings' and not specific_stock and service == 'telegram' and results): return sent = [] for result in results: @@ -104,13 +70,8 @@ def pin_if_earnings(service, results): telegram.pinChatMessage(chat, message_id) # MAIN # - if specific_stock: - ticker = util.transform_to_yahoo(specific_stock) - tickers = [ticker] - else: - tickers = util.get_holdings_and_watchlist() - - if earnings: + tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() + if event == 'earnings': market_data = yahoo.fetch(tickers) else: # ex-dividend dates are in Yahoo's details: fetched together market_data = {} @@ -122,7 +83,7 @@ def pin_if_earnings(service, results): print("") for service, url, chat in webhook.destinations(interactive, service, chat_id): - pin_if_earnings(service, webhook.send(service, url, chat, prepare_payload(service, market_data, days), message_id=message_id)) + pin_if_earnings(service, webhook.send(service, url, chat, prepare_payload(service, market_data), message_id=message_id)) # make google cloud happy return True @@ -130,10 +91,8 @@ def pin_if_earnings(service, results): def job(argv): """Arguments of a scheduled run: (label, lambda_handler kwargs). Raises ValueError with the usage on a bad argument.""" match argv: - case ['earnings']: - return 'cal.py earnings', {'earnings': True} - case ['ex-dividend']: - return 'cal.py ex-dividend', {'dividend': True} + case ['earnings' | 'ex-dividend' as event]: + return f'cal.py {event}', {'event': event} case _: raise ValueError('[earnings|ex-dividend]') diff --git a/lib/reports.py b/lib/reports.py index cc119903..7d632633 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -298,15 +298,13 @@ def prepare_help(service, botName): payload.append(f"Full help on {util.link('https://github.com/robdevops/finbot', 'GitHub', service)}") return payload -def prepare_holdings_payload(portfolioName, service, user): +def prepare_holdings_payload(portfolio_name, service): + """The holdings of the portfolio called `portfolio_name`. Raises sharesight.PortfolioNotFound.""" portfolios = sharesight.get_portfolios() - if not portfolioName: - return [".holdings: Please try again specifying a portfolio:"] + list(portfolios) - found = sharesight.find_portfolio(portfolios, portfolioName) - if not found: - return [f"{user} {portfolioName} portfolio not found. I only know about:"] + list(portfolios) - name, portfolio_id = found - holdings = sharesight.get_holdings_new(portfolioName, portfolio_id) + if not portfolio_name: + return [".holdings: please try again specifying a portfolio:"] + list(portfolios) + (name, portfolio_id), = sharesight.pick_portfolios(portfolio_name, portfolios).items() + holdings = sharesight.get_holdings_new(name, portfolio_id) payload = sorted(holding_line(holding, service) for holding in holdings.values()) if payload: payload.insert(0, webhook.bold(f"Holdings for {name} ({len(holdings)})", service)) @@ -323,13 +321,12 @@ def prepare_who(service, code): return payload or [f'{code} not found in any portfolio'] def prepare_pie(portfolio_name, by, service, max_slices=8): - """Portfolio value by industry (or by='sector'). Returns (payload, chart); chart is None unless there is something to draw.""" + """Portfolio value by industry (or by='sector'). Returns (payload, chart); chart is None unless there is something to draw. + Raises sharesight.PortfolioNotFound.""" portfolios = sharesight.get_portfolios() - found = sharesight.find_portfolio(portfolios, portfolio_name) if portfolio_name else None - if not found: - payload = [".pie: please try again specifying a portfolio:" if not portfolio_name else f"{portfolio_name} portfolio not found. I only know about:"] - return payload + list(portfolios), None - name, portfolio_id = found + if not portfolio_name: + return [".pie: please try again specifying a portfolio:"] + list(portfolios), None + (name, portfolio_id), = sharesight.pick_portfolios(portfolio_name, portfolios).items() key = 'sector_classification_name' if by == 'sector' else 'industry_classification_name' totals = {} for holding in sharesight.get_holdings_new(name, portfolio_id).values(): diff --git a/lib/sharesight.py b/lib/sharesight.py index df8540a4..d41a4895 100644 --- a/lib/sharesight.py +++ b/lib/sharesight.py @@ -92,9 +92,22 @@ def get_portfolios(refresh=False): print(portfolio_dict) return portfolio_dict -def find_portfolio(portfolios, name): - """(name as Sharesight spells it, id) of the portfolio called `name`, in any case, or None.""" - return {k.lower(): (k, v) for k, v in portfolios.items()}.get(name.lower()) +class PortfolioNotFound(LookupError): + """No portfolio is called what was asked for. `lines` is the answer to give: the name, and the portfolios there are.""" + def __init__(self, name, portfolios): + self.lines = [f"{name} portfolio not found. I only know about:"] + list(portfolios) + super().__init__(' '.join(self.lines)) + +def pick_portfolios(name=None, portfolios=None): + """{name as Sharesight spells it: id} of the portfolio called `name` (in any case), or of every portfolio when no name is given. + `portfolios` is get_portfolios() when not given. Raises PortfolioNotFound.""" + portfolios = get_portfolios() if portfolios is None else portfolios + if not name: + return portfolios + found = {k.lower(): (k, v) for k, v in portfolios.items()}.get(name.lower()) + if not found: + raise PortfolioNotFound(name, portfolios) + return dict([found]) def get_trades(portfolio_name, portfolio_id, days=UNSET): if days is UNSET: diff --git a/lib/worker.py b/lib/worker.py index d5027236..78d00e20 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -12,6 +12,7 @@ from lib import reports from lib import charts from lib import telegram +from lib import sharesight from lib import jobs import cal import performance @@ -137,6 +138,8 @@ def process_request(service, chat_id, user, message, botName, userRealName, mess if m := match(pattern, message, botName): handler(request, m) break + except sharesight.PortfolioNotFound as e: # a name the user got wrong: an answer, not an error + request.say(e.lines) except Exception as e: webhook.report_error(e, service, chat_id, context=message.split()[0][:30] if message.split() else None) if service == 'telegram': @@ -231,9 +234,9 @@ def on_thanks(request, m): def on_calendar(request, m): """Upcoming earnings or ex-dividend dates.""" days, ticker = period_or_ticker(m.group('arg'), config_future_days) - dividend = m.group('kind').lower().startswith('div') + event = 'ex-dividend' if m.group('kind').lower().startswith('div') else 'earnings' with request.typing(): - cal.lambda_handler(request.chat_id, days, request.service, ticker, interactive=True, earnings=not dividend, dividend=dividend) + cal.lambda_handler(request.chat_id, days, request.service, ticker, interactive=True, event=event) def on_performance(request, m): days, portfolio = period_and_portfolio(m.group('args'), charts.PERIODS['m'][1]) # default view is the 1M button @@ -283,7 +286,7 @@ def on_trades(request, m): def on_holdings(request, m): with request.typing(): - payload = reports.prepare_holdings_payload((m.group(1) or '').strip() or None, request.service, request.user) + payload = reports.prepare_holdings_payload((m.group(1) or '').strip() or None, request.service) request.say(payload) def on_marketcap(request, m): diff --git a/performance.py b/performance.py index 20cba516..23864308 100755 --- a/performance.py +++ b/performance.py @@ -13,11 +13,8 @@ def gather(past_days, portfolio_select=None): """{portfolio id: Sharesight's performance report over past_days} of every portfolio, or of the one named (in any case). Raises - RuntimeError when there is none.""" - portfolios = sharesight.get_portfolios() - if portfolio_select: - found = sharesight.find_portfolio(portfolios, portfolio_select) - portfolios = dict([found]) if found else {} + sharesight.PortfolioNotFound for a name that is none of them, and RuntimeError when there is no report.""" + portfolios = sharesight.pick_portfolios(portfolio_select) performance = {portfolio_id: sharesight.get_performance(portfolio_id, past_days) for portfolio_id in portfolios.values()} if not performance: print("Error: no Sharesight data found", file=sys.stderr) diff --git a/tests/test_edges.py b/tests/test_edges.py index 9e4953c0..804820f8 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -99,6 +99,21 @@ def test_history_without_a_ticker_or_with_extra_argument_gives_usage(self): self.assertEqual(self.command('.history T1 1y'), [('text', ['Usage: .history SYMBOL'])]) +class PortfolioNames(FinbotCase): + def test_an_unknown_portfolio_gets_the_same_answer_everywhere(self): + for cmd in ('.trades 7d Nope', '.performance Nope', '.holdings Nope', '.pie Nope'): + with self.subTest(cmd=cmd): # it used to be a ValueError, "no Sharesight data found", or two wordings of this + self.assertEqual(self.command(cmd), [('text', ['Nope portfolio not found. I only know about:', 'Rob'])]) + + def test_a_portfolio_name_in_any_case(self): + self.patch(sharesight, 'get_holdings_new', lambda name, pid: {'T1:NASDAQ': {'code': 'T1', 'market_code': 'NASDAQ', 'name': 'Co T1', 'holding_id': 1}}) + self.assertIn('Holdings for Rob (1)', str(self.command('.holdings rob'))) + + def test_trades_with_no_buy_or_sell_still_get_an_answer(self): + self.patch(sharesight, 'get_trades', lambda name, pid, days=7: [{'id': 2, 'transaction_type': 'DIVIDEND', 'portfolio': name}]) + self.assertIn('No trades', str(self.command('.trades 7d'))) # it used to say nothing at all + + class CommandArguments(FinbotCase): def test_period_and_portfolio_in_either_order(self): for text, expected in (('', (30, None)), ('7d', (7, None)), ('Rob', (30, 'Rob')), ('7d Rob', (7, 'Rob')), ('Rob 7d', (7, 'Rob')), diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 7c292bbc..072f3c10 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -104,8 +104,8 @@ def test_price(self): price.job(bad) def test_cal_and_performance(self): - self.assertEqual(cal.job(['earnings']), ('cal.py earnings', {'earnings': True})) - self.assertEqual(cal.job(['ex-dividend']), ('cal.py ex-dividend', {'dividend': True})) + self.assertEqual(cal.job(['earnings']), ('cal.py earnings', {'event': 'earnings'})) + self.assertEqual(cal.job(['ex-dividend']), ('cal.py ex-dividend', {'event': 'ex-dividend'})) self.assertEqual(performance.job([]), ('performance.py', {})) self.assertEqual(performance.job(['7']), ('performance.py 7', {'past_days': 7})) for bad, module in ((['x'], cal), ([], cal), (['x'], performance)): diff --git a/tests/test_reports.py b/tests/test_reports.py index 98ac75b1..b1a28c4d 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -39,7 +39,7 @@ def test_prepare_help(self): self.assertTrue(reports.prepare_help('telegram', BOT)) def test_prepare_holdings(self): - self.assertTrue(reports.prepare_holdings_payload(None, 'telegram', '@u')) + self.assertTrue(reports.prepare_holdings_payload(None, 'telegram')) def test_prepare_marketcap(self): self.assertTrue(reports.prepare_marketcap_payload('telegram', 'top', length=5)) @@ -107,12 +107,21 @@ def test_shorts(self): self.assertTrue(self.sent) def test_earnings_and_dividends(self): - for kwargs in ({'earnings': True}, {'dividend': True}): + for kwargs in ({'event': 'earnings'}, {'event': 'ex-dividend'}): with self.subTest(kwargs=kwargs): self.sent.clear() cal.lambda_handler(**kwargs) self.assertTrue(self.sent) + def test_earnings_skip_an_adr_whose_primary_listing_yahoo_does_not_know(self): + """T1 has no earnings date, so its primary listing is looked up; Yahoo knowing nothing of it used to crash the report.""" + self.patch(cal, 'primary_listing', {'T1': 'T1.XX'}) + self.patch(yahoo, 'fetch', lambda ts: {t: {k: v for k, v in market_entry(t).items() if not (t == 'T1' and k == 'earnings_date')} for t in ts}) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {} if t == 'T1.XX' else {t: market_entry(t)}) + cal.lambda_handler(event='earnings') + self.assertIn('Co T2 ', str(self.sent)) + self.assertNotIn('Co T1 ', str(self.sent)) + def test_trades(self): trades.lambda_handler(interactive=True, service='telegram', chat_id='5') self.assertTrue(self.sent) diff --git a/trades.py b/trades.py index 9c8bafe1..fe5fa02a 100755 --- a/trades.py +++ b/trades.py @@ -10,24 +10,19 @@ from lib import yahoo def lambda_handler(chat_id=config_telegramChatID, days=UNSET, service=None, user='', portfolio_select=None, message_id=None, interactive=False): + """The buys and sells of the past `days`, in every portfolio or the one named: on a scheduled run only those not reported + before. Raises sharesight.PortfolioNotFound for a name that is none of them.""" if days is UNSET: days = config_past_days - start_date = datetime.datetime.now() - datetime.timedelta(days=days) - start_date = start_date.strftime('%Y-%m-%d') # 2022-08-20 state_file = "finbot_sharesight_trades.json" def prepare_trade_payload(service, trades): - if not len(trades): - print("No trades provided to prepare_trade_payload()", file=sys.stderr) - return None payload_staging = [] dates = set() - wanted = [] # the trades to report: buys and sells in the chosen portfolio that are new (or all of them, when interactive) + wanted = [] # the trades to report: buys and sells that are new (or all of them, when interactive) for trade in trades: if trade['transaction_type'] not in {'BUY', 'SELL'}: continue - if portfolio_select and trade['portfolio'].lower() != portfolio_select.lower(): - continue if not interactive and int(trade['id']) in known_trades: print("Skipping known trade_id:", trade['id'], trade['transaction_date'], trade['portfolio'], trade['transaction_type'], trade['symbol']) continue @@ -47,17 +42,9 @@ def prepare_trade_payload(service, trades): portfolio_name = trade['portfolio'] # custom field trade_id = int(trade['id']) date = trade['transaction_date'] # 2023-12-30 - transactionType = trade['transaction_type'] symbol = trade['symbol'] - action='' - emoji='' market = trade['market'] - if transactionType == 'BUY': - action = 'bought' - emoji = '💸' - else: # SELL - action = 'sold' - emoji = '🤑' + action, emoji = ('bought', '💸') if trade['transaction_type'] == 'BUY' else ('sold', '🤑') units = float(trade['quantity']) price = float(trade['price']) currency = trade['brokerage_currency_code'] @@ -69,10 +56,9 @@ def prepare_trade_payload(service, trades): print("Warning: could not get market for", ticker, "from Yahoo:", e, file=sys.stderr) value = round(price * units) # not Sharesight's trade value, which it converts to the portfolio's currency holding_id = str(trade['holding_id']) - ticker = util.transform_to_yahoo(symbol, market) dt_date = datetime.datetime.strptime(date, '%Y-%m-%d').date() # (2023, 12, 30) - newtrades.add(trade_id) # sneaky update global set + newtrades.add(trade_id) # recorded by the main body once the report is out dates.add(dt_date) flag = util.flag_from_market(market) @@ -101,22 +87,14 @@ def prepare_trade_payload(service, trades): # MAIN # newtrades = set() - portfolios = sharesight.get_portfolios() + portfolios = sharesight.pick_portfolios(portfolio_select) known_trades = set(util.json_load(state_file, persist=True) or ()) - if portfolio_select: - found = sharesight.find_portfolio(portfolios, portfolio_select) - if not found: - print("Portfolio not found:", portfolio_select, file=sys.stderr) - raise ValueError(f"portfolio not found: {portfolio_select}") - portfolio_select = found[0] # as Sharesight spells it - portfolios = dict([found]) trades = [trade for name, portfolio_id in portfolios.items() for trade in sharesight.get_trades(name, portfolio_id, days)] for service, url, chat in webhook.destinations(interactive, service, chat_id): - if interactive and not trades: + payload = prepare_trade_payload(service, trades) + if interactive and not payload: # no trades, or none of them a buy or a sell payload = [f"{user} No trades {util.days_english(days, 'in the past ')}. {random.choice(noTradesVerb)}"] - else: - payload = prepare_trade_payload(service, trades) webhook.send(service, url, chat, payload, message_id=message_id) # write state file From 9abb58ecdb311b7161d1a9f01a3e1346daba4456 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:05:14 +0000 Subject: [PATCH 17/25] rating keeps what it has recorded; podcasts survives feeds with undated items and keeps its state like the other scripts - rating wrote only the tickers it rated in a run: one that failed to fetch lost its last rating, and its next change went unreported (a one-ticker run wiped all the others). The record is now added to, and an interactive run records nothing, as with trades. - podcasts: a feed mixing dated and undated items crashed the sort (aware and naive times); a garbled date is now no date. The newest five are picked once, after reading the feed rather than at every item; a run with nothing new posts nothing (it used to post an empty report when the stored list was not in sorted order); the seen list is kept with util.json_write like the other state files (atomic); single-tab indentation like the rest of the code. - webhook.italics, an alias of italic, is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/webhook.py | 2 - podcasts.py | 231 +++++++++++++++--------------------------- rating.py | 13 +-- reminder.py | 8 +- tests/test_reports.py | 37 ++++++- 5 files changed, 130 insertions(+), 161 deletions(-) diff --git a/lib/webhook.py b/lib/webhook.py index 5450e3e6..a612e596 100644 --- a/lib/webhook.py +++ b/lib/webhook.py @@ -166,8 +166,6 @@ def bold(message, service): def italic(message, service): return _styled('italic', message, service) -italics = italic - def strike(message, service): return _styled('strike', message, service) diff --git a/podcasts.py b/podcasts.py index e85015d6..1d088a55 100755 --- a/podcasts.py +++ b/podcasts.py @@ -1,172 +1,109 @@ #!/usr/bin/env python3 import sys -import os import hashlib import string import xml.etree.ElementTree as ET -from lib.config import * +from email.utils import parsedate_to_datetime +from datetime import datetime, timezone from lib import net from lib import webhook from lib import util -from email.utils import parsedate_to_datetime -from datetime import datetime, timezone NAMESPACES = { - "media": "http://search.yahoo.com/mrss/", - "itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd" + "media": "http://search.yahoo.com/mrss/", + "itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd" } +NEWEST = 5 # episodes posted per run at most: the other new ones (on a first run, say) are only recorded as seen def get_seen_filename(url): - hashed = hashlib.md5(url.encode()).hexdigest() - return config_var_dir + "/" + f"finbot_podcast_seen_{hashed}.json" - -def load_seen(filename): - if os.path.exists(filename): - with open(filename, "r") as f: - return json.load(f) - return [] - -def save_seen(seen, filename): - with open(filename, "w") as f: - json.dump(list(seen), f, indent=4) + """The state file, in the data directory, of the episodes of this feed already seen.""" + return f"finbot_podcast_seen_{hashlib.md5(url.encode()).hexdigest()}.json" def fetch_and_parse_feed(url): - resp = net.get(url, timeout=30) - resp.raise_for_status() - return ET.fromstring(resp.content) + resp = net.get(url, timeout=30) + resp.raise_for_status() + return ET.fromstring(resp.content) def format_duration(duration_str): - """Convert duration string to human-readable format""" - if not duration_str: - return "" - - try: - # Try parsing as seconds (integer) - total_seconds = int(duration_str) - except ValueError: - # Try parsing as HH:MM:SS or MM:SS format - try: - parts = duration_str.split(':') - if len(parts) == 3: # HH:MM:SS - total_seconds = int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) - elif len(parts) == 2: # MM:SS - total_seconds = int(parts[0]) * 60 + int(parts[1]) - else: - return "" - except: - return "" - - hours = total_seconds // 3600 - minutes = (total_seconds % 3600) // 60 - - return f"{hours:02d}:{minutes:02d}" - - -def how_long_ago(pub_raw): - try: - pub_date = parsedate_to_datetime(pub_raw) - if pub_date.tzinfo is None: - pub_date = pub_date.replace(tzinfo=timezone.utc) - now = datetime.now(timezone.utc) - delta = now - pub_date - - days = delta.days - seconds = delta.seconds - if days > 0: - return f"{days}d ago" - elif seconds > 3600: - hours = seconds // 3600 - return f"{hours}h ago" - elif seconds > 60: - minutes = seconds // 60 - return f"{minutes}m ago" - else: - return f"{seconds}s ago" - except Exception as e: - print(f"Error in how_long_ago: {e}") - return "" + """'3725' (seconds), '1:02:05' or '62:05' -> '01:02' (hours and minutes); '' for anything else.""" + try: + parts = [int(part) for part in (duration_str or '').split(':')] + except ValueError: + return "" + if len(parts) > 3: + return "" + total_seconds = sum(part * 60 ** i for i, part in enumerate(reversed(parts))) + return f"{total_seconds // 3600:02d}:{total_seconds % 3600 // 60:02d}" + +def published(pub_raw): + """An episode's publication time (in UTC where the feed gives no zone), or None where it gives none or garbles it.""" + try: + when = parsedate_to_datetime(pub_raw) + except (TypeError, ValueError): + return None + return when if when.tzinfo else when.replace(tzinfo=timezone.utc) + +def how_long_ago(when): + """'3d ago', '5h ago', '12m ago' or '40s ago'; '' without a time.""" + if when is None: + return "" + delta = datetime.now(timezone.utc) - when + if delta.days > 0: + return f"{delta.days}d ago" + if delta.seconds > 3600: + return f"{delta.seconds // 3600}h ago" + if delta.seconds > 60: + return f"{delta.seconds // 60}m ago" + return f"{delta.seconds}s ago" + +def media_link(item): + """An episode's media: , else , else ; None without any.""" + for element in (item.find(".//media:content/media:player", NAMESPACES), item.find("media:content", NAMESPACES), item.find("enclosure")): + if element is not None and element.attrib.get("url"): + return element.attrib["url"] + return None def fetch_new_episodes(feed_url, seen): - root = fetch_and_parse_feed(feed_url) - channel = root.find("channel") - if channel is None: - raise RuntimeError("No element found") - - podcast_name = (channel.findtext("title") or "Podcast").strip() - new_episodes = [] - - for item in channel.findall("item"): - guid = item.findtext("guid") or item.findtext("link") - if not guid or guid in seen: - continue - - # Try - player_el = item.find(".//media:content/media:player", NAMESPACES) - if player_el is not None and "url" in player_el.attrib: - media_url = player_el.attrib["url"] - else: - # Fallback: - content_el = item.find("media:content", NAMESPACES) - media_url = content_el.attrib["url"] if content_el is not None and "url" in content_el.attrib else None - - if not media_url: - # Final fallback: - enclosure = item.find("enclosure") - media_url = enclosure.attrib["url"] if enclosure is not None and "url" in enclosure.attrib else None - - if not media_url: - continue - - # Get duration from itunes:duration tag - duration_raw = item.findtext("itunes:duration", namespaces=NAMESPACES) - duration = format_duration(duration_raw) if duration_raw else "" - - pub_raw = item.findtext("pubDate") - published_dt = parsedate_to_datetime(pub_raw) if pub_raw else datetime.min - ago = how_long_ago(pub_raw) if pub_raw else "" - - episode_title = (item.findtext("title") or "Untitled").strip() - new_episodes.append({ - "podcast": podcast_name, - "title": episode_title, - "url": media_url, - "ago": ago, - "duration": duration, - "published_dt": published_dt - }) - new_episodes.sort(key=lambda ep: ep["published_dt"], reverse=True) - new_episodes = new_episodes[:5] - - seen.append(guid) - - return new_episodes + """(the NEWEST newest episodes whose guid is not in `seen`, newest first; the guids of all the new ones).""" + channel = fetch_and_parse_feed(feed_url).find("channel") + if channel is None: + raise RuntimeError("No element found") + podcast_name = (channel.findtext("title") or "Podcast").strip() + episodes, guids = [], set() + for item in channel.findall("item"): + guid = item.findtext("guid") or item.findtext("link") + media_url = media_link(item) + if not guid or guid in seen or not media_url: + continue + when = published(item.findtext("pubDate")) + episodes.append({"podcast": podcast_name, "title": (item.findtext("title") or "Untitled").strip(), "url": media_url, + "ago": how_long_ago(when), "duration": format_duration(item.findtext("itunes:duration", namespaces=NAMESPACES)), + "published": when or datetime.min.replace(tzinfo=timezone.utc)}) + guids.add(guid) + episodes.sort(key=lambda episode: episode["published"], reverse=True) + return episodes[:NEWEST], guids def lambda_handler(feed_url): - seen_file = get_seen_filename(feed_url) - seen = load_seen(seen_file) - seen_earlier = seen.copy() - - try: - new_episodes = fetch_new_episodes(feed_url, seen) - except Exception as e: - raise RuntimeError(f"fetching feed {feed_url}: {e}") - - seen = sorted(set(seen)) - if seen != seen_earlier: - for service, url, chat in webhook.default_chats(): - payload = [] - for ep in new_episodes: - podcast_name = ep['podcast'] - title = ep['title'].removeprefix(podcast_name).lstrip(string.punctuation + " ") - link = util.link(ep['url'], title, service) - emoji = "📣" - duration_str = f"{ep['duration']}" if ep['duration'] else "" - payload.append(f"{emoji} {podcast_name}: {link} {duration_str} ({ep['ago']})") - webhook.send(service, url, chat, payload) - save_seen(seen, seen_file) # do this last so we pick up these episodes next run if we error - else: - print("no new episodes found") + """Post the feed's newest episodes not posted before, then record every new one as seen.""" + seen_file = get_seen_filename(feed_url) + seen = set(util.json_load(seen_file, persist=True) or []) + try: + episodes, guids = fetch_new_episodes(feed_url, seen) + except Exception as e: + raise RuntimeError(f"fetching feed {feed_url}: {e}") + if not guids: + print("no new episodes found") + return + for service, url, chat in webhook.default_chats(): + payload = [] + for episode in episodes: + podcast_name = episode['podcast'] + title = episode['title'].removeprefix(podcast_name).lstrip(string.punctuation + " ") + duration = f" {episode['duration']}" if episode['duration'] else "" + payload.append(f"📣 {podcast_name}: {util.link(episode['url'], title, service)}{duration} ({episode['ago']})") + webhook.send(service, url, chat, payload) + util.json_write(seen_file, sorted(seen | guids), persist=True) # last, so that a failure above leaves these episodes for next run def job(argv): """Arguments of a scheduled run: (label, lambda_handler kwargs). Raises ValueError with the usage on a bad argument.""" diff --git a/rating.py b/rating.py index 1f0448f2..73c3d823 100755 --- a/rating.py +++ b/rating.py @@ -7,6 +7,7 @@ from lib import yahoo def lambda_handler(chat_id=config_telegramChatID, specific_stock=None, service=None, interactive=False): + """Analysts' consensus ratings that changed since the last scheduled run, which records the ratings it saw.""" def prepare_rating_payload(service, market_data): def get_emoji(old, new): if old > new: @@ -17,7 +18,6 @@ def get_emoji(old, new): return '▪️' payload = [] new = {} - old = util.json_load('finbot_rating.json', persist=True) for ticker in tickers: old_action=None old_index = float() @@ -31,9 +31,8 @@ def get_emoji(old, new): ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service, brief=False) new[ticker] = [action, index] try: - old_action = old[ticker][0] - old_index = old[ticker][1] - except (KeyError, TypeError): + old_action, old_index = old[ticker] + except (KeyError, TypeError, ValueError): pass if old_action and action != old_action: emoji = get_emoji(old_index, index) @@ -56,6 +55,8 @@ def get_emoji(old, new): # MAIN # tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() + old = util.json_load('finbot_rating.json', persist=True) + old = old if isinstance(old, dict) else {} # absent until the first run market_data = {} for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): # fetched together if isinstance(detail, Exception): @@ -67,8 +68,8 @@ def get_emoji(old, new): payload, new = prepare_rating_payload(service, market_data) webhook.send(service, url, chat, payload) - if new: - util.json_write('finbot_rating.json', new, persist=True) + if new and not interactive: # added to what is recorded: a ticker missing from this run keeps its last rating + util.json_write('finbot_rating.json', old | new, persist=True) # make google cloud happy return True diff --git a/reminder.py b/reminder.py index f6082411..09b043b2 100755 --- a/reminder.py +++ b/reminder.py @@ -50,10 +50,10 @@ def prepare_finance_calendar_payload(service): abr_link = util.link('https://abr.business.gov.au/Tools/DgrListing', 'DGR-registered', service) stake_note = util.link('https://hellostake.com/au/support/stake-super/employer-and-personal-contributions/33442327233305#h_01JQAPN9A6ZM41NZKP6XZWN1KP', 'note about XX PCC in transaction description', service) work_link = util.link('https://www.ato.gov.au/individuals-and-families/income-deductions-offsets-and-records/deductions-you-can-claim', 'eligible work expenses', service) - payload.append(webhook.italics("* Organisation must be " + abr_link, service)) - payload.append(webhook.italics("† Allow transfer time for Super, as contributions can only be deducted for the year they're received. Stake users: see " + stake_note, service)) - payload.append(webhook.italics("‡ See " + work_link, service)) - payload.append(webhook.italics("§ Must relate directly to specific trades and gains/income (e.g. not general finance news)", service)) + payload.append(webhook.italic("* Organisation must be " + abr_link, service)) + payload.append(webhook.italic("† Allow transfer time for Super, as contributions can only be deducted for the year they're received. Stake users: see " + stake_note, service)) + payload.append(webhook.italic("‡ See " + work_link, service)) + payload.append(webhook.italic("§ Must relate directly to specific trades and gains/income (e.g. not general finance news)", service)) if month_and_day == '08-18': myBirthday = datetime.datetime(2022,8,18,0,0,0,0, tzinfo=tz) diff --git a/tests/test_reports.py b/tests/test_reports.py index b1a28c4d..13fc489e 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,5 +1,6 @@ """Every chat command and scheduled report produces output on mocked data.""" import unittest +import datetime from tests.common import * @@ -155,6 +156,19 @@ def test_rating(self): rating.lambda_handler(interactive=True, service='telegram', chat_id='5') self.assertTrue(self.sent) + def test_rating_keeps_the_last_rating_of_a_ticker_missing_from_a_run(self): + rating.lambda_handler() # records every ticker's rating + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {} if t == 'T1' else {t: market_entry(t)}) + rating.lambda_handler() # no T1 this time: its rating used to be dropped from the record + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | ({'recommend': 'sell', 'recommend_index': 4.5} if t == 'T1' else {})}) + self.sent.clear() + rating.lambda_handler() + self.assertIn('Co T1 ', str(self.sent)) # so its change from buy to sell went unreported + + def test_an_interactive_rating_records_nothing(self): + rating.lambda_handler(interactive=True, service='telegram', chat_id='5') + self.assertIsNone(util.json_load('finbot_rating.json', persist=True)) + def test_slack_webhook_without_a_bot_token_gets_scheduled_reports(self): """Scheduled reports reach Slack through its incoming webhook, which takes no channel and no token. They used to send Telegram's chat id along, which made write() need slackBotToken and crash without one, before Telegram got its copy.""" @@ -227,13 +241,32 @@ def test_a_button_a_chart_does_not_offer_is_ignored(self): self.assertEqual((edits, toasts), ([], [None])) -class Podcasts(unittest.TestCase): +class Podcasts(FinbotCase): def test_format_helpers(self): import podcasts self.assertEqual(podcasts.format_duration('3725'), '01:02') self.assertEqual(podcasts.format_duration('1:02:03'), '01:02') + self.assertEqual(podcasts.format_duration('62:05'), '01:02') self.assertEqual(podcasts.format_duration('nonsense'), '') - self.assertTrue(podcasts.how_long_ago('Mon, 01 Jan 2024 00:00:00 +0000').endswith('d ago')) + self.assertTrue(podcasts.how_long_ago(podcasts.published('Mon, 01 Jan 2024 00:00:00 +0000')).endswith('d ago')) + self.assertEqual(podcasts.published('Mon, 01 Jan 2024 00:00:00 -0000').tzinfo, datetime.timezone.utc) # no zone given: UTC + self.assertIsNone(podcasts.published('yesterday')) + + def test_new_episodes_are_posted_once_newest_first(self): + """A feed whose items mix dated and undated ones used to crash the sort (aware and naive times).""" + import podcasts + def item(n, date): + return f'g{n}Show: ep {n}{date}600' + feed = ('Show' + + item(1, 'Mon, 01 Jan 2024 00:00:00 +0000') + item(2, '') + item(3, 'Tue, 02 Jan 2024 00:00:00 GMT') + + '').encode() + self.patch(podcasts.net, 'get', lambda url, **k: mock.Mock(content=feed, raise_for_status=lambda: None)) + podcasts.lambda_handler('https://example.com/feed') + self.assertEqual([line.split(' (')[0] for line in self.sent[0][1]], ['📣 Show: ep 3 00:10', + '📣 Show: ep 1 00:10', '📣 Show: ep 2 00:10']) + self.sent.clear() + podcasts.lambda_handler('https://example.com/feed') + self.assertEqual(self.sent, []) # all seen now if __name__ == '__main__': From 7441bfcf53abd827d006bf0e6d7e06613f71593a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:08:09 +0000 Subject: [PATCH 18/25] yahoo.fetch_details: one fetch of many tickers' details, where six loops did it; the ratings list sorts once - yahoo.fetch_details fetches the details of many tickers at once and leaves out (and logs) one whose fetch failed, so a list goes out without it; strict=True raises instead, for a report on the one ticker asked for. It replaces the loops in .beta, .buy/.sell, .peg, shorts, the calendar and rating.py. .buy/.sell used to stop at the first ticker that failed (other than with a TypeError), and .peg did too. - .buy/.sell sorts (rating, analysts, line, ticker) once instead of sorting the text lines and a chart list separately to the same key; the report and chart are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- cal.py | 9 ++------ lib/reports.py | 57 +++++++++++++------------------------------------- lib/yahoo.py | 14 +++++++++++++ rating.py | 7 +------ shorts.py | 7 +------ 5 files changed, 33 insertions(+), 61 deletions(-) diff --git a/cal.py b/cal.py index 6cb668d3..1a63e487 100755 --- a/cal.py +++ b/cal.py @@ -73,13 +73,8 @@ def pin_if_earnings(service, results): tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() if event == 'earnings': market_data = yahoo.fetch(tickers) - else: # ex-dividend dates are in Yahoo's details: fetched together - market_data = {} - for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): - if isinstance(detail, Exception): - print(ticker, detail, file=sys.stderr) - else: - market_data |= detail + else: # ex-dividend dates are in Yahoo's details + market_data = yahoo.fetch_details(tickers, strict=bool(specific_stock)) print("") for service, url, chat in webhook.destinations(interactive, service, chat_id): diff --git a/lib/reports.py b/lib/reports.py index 7d632633..b0af083b 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -374,38 +374,23 @@ def prepare_marketcap_payload(service, action='top', length=15): return [webhook.bold(f'{action.title()} {length} tracked stocks by market cap', service)] + [line for _, line in caps[:length]] def prepare_rating_payload(service, action, length=15): - def score_col(e): - return (float(e.split()[-3]), int(e.split()[-2].removeprefix('('))) - payload = [] - chart_rows = [] # (ticker, rating, analysts), kept in step with payload - tickers = util.get_holdings_and_watchlist() - market_data = {} - for detail in yahoo.fetch_each(yahoo.fetch_detail, tickers): # fetched at the same time - if isinstance(detail, TypeError): + """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by more than one analyst, by mean rating.""" + found = [] # (rating, analysts, line, ticker) + for ticker, data in yahoo.fetch_details(util.get_holdings_and_watchlist()).items(): + if 'recommend' not in data or data['recommend_analysts'] <= 1: continue - if isinstance(detail, Exception): - raise detail - market_data |= detail - for ticker in market_data: - if 'recommend' in market_data[ticker]: - recommend = market_data[ticker]['recommend'].replace('_', ' ') - recommend_index = round(market_data[ticker]['recommend_index'], 2) - recommend_analysts = market_data[ticker]['recommend_analysts'] - if recommend_analysts > 1: - profile_title = market_data[ticker]['profile_title'] - ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) - flag = util.flag_from_ticker(ticker) - if (action == 'buy' and 'buy' in recommend) or (action == 'sell' and recommend in ('sell', 'underperform')): - payload.append(f"{flag} {profile_title} ({ticker_link}) {recommend_index} ({recommend_analysts} analysts)") - chart_rows.append((ticker.split('.')[0], recommend_index, recommend_analysts)) - payload.sort(key=score_col) - chart_rows.sort(key=lambda row: (row[1], row[2])) # same order as score_col - payload = payload[:length] + recommend = data['recommend'].replace('_', ' ') + if (action == 'buy' and 'buy' in recommend) or (action == 'sell' and recommend in ('sell', 'underperform')): + rating, analysts = round(data['recommend_index'], 2), data['recommend_analysts'] + link = util.finance_link(ticker, data['profile_exchange'], service) + found.append((rating, analysts, f"{util.flag_from_ticker(ticker)} {data['profile_title']} ({link}) {rating} ({analysts} analysts)", ticker)) + found = sorted(found, key=lambda row: row[:2])[:length] + payload = [line for _, _, line, _ in found] chart = None if payload: message = f"Top {length} analyst {action} ratings for tracked stocks" payload.insert(0, f"{webhook.bold(message, service)}") - shown = [(t, r) for t, r, _ in chart_rows[:length]] + shown = [(ticker.split('.')[0], rating) for rating, _, _, ticker in found] low, high = min(r for _, r in shown), max(r for _, r in shown) spread = (high - low) or 0.2 # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) @@ -418,11 +403,8 @@ def prepare_value_payload(service, action='pe', ticker_select=None, length=15): chart_rows = [] # (ticker, ratio) tickers = [ticker_select] if ticker_select else util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) - details = dict(zip(market_data, yahoo.fetch_each(yahoo.fetch_detail, market_data))) if action in ('peg', 'bottom peg', 'negative peg') else {} - def detail(ticker): # fetched above, all at once; a failure is raised here, where the one-at-a-time fetch raised it - if isinstance(details[ticker], Exception): - raise details[ticker] - return details[ticker] + if action in ('peg', 'bottom peg', 'negative peg'): # the PEG is in Yahoo's details + market_data |= yahoo.fetch_details(market_data, strict=bool(ticker_select)) for ticker in market_data: trailing_ratio = None forward_ratio = None @@ -446,13 +428,11 @@ def detail(ticker): # fetched above, all at once; a failure is raised here, wher continue ratio = forward_ratio elif action in ('peg', 'bottom peg'): - market_data = market_data | detail(ticker) peg_ratio = market_data.get(ticker).get('price_to_earnings_peg') if not ticker_select and peg_ratio is not None and peg_ratio < 0: continue ratio = peg_ratio elif action == 'negative peg': - market_data = market_data | detail(ticker) peg_ratio = market_data.get(ticker).get('price_to_earnings_peg') if not ticker_select and (peg_ratio is None or peg_ratio >= 0): continue @@ -495,15 +475,8 @@ def detail(ticker): # fetched above, all at once; a failure is raised here, wher def prepare_beta_payload(service): """(payload, chart) of the tracked stocks with a beta over 1.5 and a market cap under 2B, the highest beta first.""" - market_data = {} - tickers = util.get_holdings_and_watchlist() - for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): # fetched at the same time - if isinstance(detail, Exception): - print(ticker, detail, file=sys.stderr) # skip this ticker; keep the rest - else: - market_data |= detail found = [] # (beta, line, chart row) - for ticker, data in market_data.items(): + for ticker, data in yahoo.fetch_details(util.get_holdings_and_watchlist()).items(): if 'beta' not in data: continue beta = round(data['beta'], 1) diff --git a/lib/yahoo.py b/lib/yahoo.py index ebbac3e0..cf9ec93f 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -389,6 +389,20 @@ def one(ticker): with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: return list(executor.map(one, tickers)) +def fetch_details(tickers, strict=False): + """fetch_detail of every ticker, fetched at the same time, in one {ticker: detail}. A ticker whose fetch failed is left out + (and logged), so that a list goes out without it; strict=True raises its error instead, for a report on the ticker asked for.""" + tickers = list(tickers) + details = {} + for ticker, detail in zip(tickers, fetch_each(fetch_detail, tickers)): + if isinstance(detail, Exception): + if strict: + raise detail + print(ticker, detail, file=sys.stderr) + else: + details |= detail + return details + def prefetch_history(tickers, workers=8): """Warm the price-history cache for many tickers at once, so the per-ticker lookups that follow are cache hits.""" tickers = list(tickers) diff --git a/rating.py b/rating.py index 73c3d823..28476edd 100755 --- a/rating.py +++ b/rating.py @@ -57,12 +57,7 @@ def get_emoji(old, new): tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() old = util.json_load('finbot_rating.json', persist=True) old = old if isinstance(old, dict) else {} # absent until the first run - market_data = {} - for ticker, detail in zip(tickers, yahoo.fetch_each(yahoo.fetch_detail, tickers)): # fetched together - if isinstance(detail, Exception): - print(ticker, detail, file=sys.stderr) - else: - market_data |= detail + market_data = yahoo.fetch_details(tickers, strict=bool(specific_stock)) for service, url, chat in webhook.destinations(interactive, service, chat_id): payload, new = prepare_rating_payload(service, market_data) diff --git a/shorts.py b/shorts.py index c63907a1..a3ee6f13 100755 --- a/shorts.py +++ b/shorts.py @@ -63,12 +63,7 @@ def last_element(e): tickers = util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) - us = [ticker for ticker in tickers if '.' not in ticker] # Yahoo's details have US short interest; Short Man has the ASX's - for ticker, detail in zip(us, yahoo.fetch_each(yahoo.fetch_detail, us)): - if isinstance(detail, Exception): - print(ticker, detail, file=sys.stderr) - else: - market_data |= detail + market_data |= yahoo.fetch_details((ticker for ticker in tickers if '.' not in ticker), strict=bool(specific_stock)) # US short interest; Short Man has the ASX's print("") market_data = shortman.fetch(market_data) From cfb9a8102f99be14e9ba3fd67a4793267c1866ef Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:12:08 +0000 Subject: [PATCH 19/25] A company profile keeps the end of its description; charts share their axis styling and heading; config errors print in one place - make_paragraphs dropped the words after its last paragraph break, so a short company description came out empty and a longer one lost its last sentences. - The price and compare charts share _style_time_axes, and those two and the pie share _heading; every chart renders byte for byte as before. - config.print_errors logs a ConfigError, where bot.py and config each had the loop. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- bot.py | 6 ++--- lib/config.py | 8 +++++-- lib/util.py | 57 +++++++++++++++++++++++----------------------- tests/test_util.py | 5 ++++ 4 files changed, 42 insertions(+), 34 deletions(-) diff --git a/bot.py b/bot.py index fb5a8c38..0b5cf865 100755 --- a/bot.py +++ b/bot.py @@ -192,8 +192,7 @@ def reload_all(*args): print('config:', change, file=sys.stderr) print(f"config: reloaded {config.preferences_path}, {len(changes)} change{'' if len(changes) == 1 else 's'}", file=sys.stderr) except config.ConfigError as e: - for line in e.errors: - print('ERROR config:', line, file=sys.stderr) + config.print_errors(e) if scheduler: scheduler.reload() @@ -202,8 +201,7 @@ def reload_all(*args): try: config.check_startup_environment() except config.ConfigError as e: - for line in e.errors: - print('ERROR config:', line, file=sys.stderr) + config.print_errors(e) sys.exit(1) for line in config.startup_report(): print('config:', line, file=sys.stderr) diff --git a/lib/config.py b/lib/config.py index d72c02eb..7a4290da 100644 --- a/lib/config.py +++ b/lib/config.py @@ -224,9 +224,13 @@ def getenv(name, default=None): return getenv -def _fail(e): +def print_errors(e): + """Log a ConfigError's problems, a line each.""" for line in e.errors: print('ERROR config:', line, file=sys.stderr) + +def _fail(e): + print_errors(e) sys.exit(1) @@ -509,6 +513,6 @@ def startup_report(): # `from lib.config import *` is how the other modules get their config_* names; keep the loader machinery out of their namespaces _INTERNAL = {'difflib', 're', 'sys', 'ConfigError', 'key_problem', 'parse_assignments', 'read_text', 'read_preferences', 'read_startup_file', - 'startup_values', 'preference_values', 'reload', 'child_environment', 'check_startup_environment', 'startup_report', 'preferences_path', + 'startup_values', 'preference_values', 'reload', 'child_environment', 'print_errors', 'check_startup_environment', 'startup_report', 'preferences_path', 'STARTUP_KEYS', 'PREFERENCE_KEYS', 'SECRET_KEYS', 'PREFERENCES_FILE', 'PASSED', 'PREFERENCE_NAMES', 'OLD_ENV'} __all__ = [name for name in dir() if not name.startswith('_') and name not in _INTERNAL] diff --git a/lib/util.py b/lib/util.py index 9db60b77..26c8f44c 100644 --- a/lib/util.py +++ b/lib/util.py @@ -298,15 +298,16 @@ def strip_url(url): return url def make_paragraphs(walloftext): - buffer = [] - output = [] + """The text in paragraphs, each ending with the first sentence that takes it past 22 words.""" + paragraphs, buffer = [], [] for word in walloftext.split(): buffer.append(word) if word.endswith(('!', '.')) and len(buffer) > 22: - output.append(' '.join(buffer)) + paragraphs.append(' '.join(buffer)) buffer = [] - output = '\n\n'.join(output) - return output + if buffer: # the rest: it used to be dropped, and a short text with it + paragraphs.append(' '.join(buffer)) + return '\n\n'.join(paragraphs) def date_short(d, year=False): """5 Oct (or 5 Oct 2026).""" @@ -367,6 +368,24 @@ def _figure(facecolor): fig = Figure(figsize=(6.4, 4.0), dpi=200, facecolor=facecolor) return fig, FigureCanvasAgg(fig) +def _style_time_axes(fig, ax, start, end): + """What the line charts share: a horizontal grid, no frame, muted ticks, and the date axis from start to end.""" + _, ink, ink2, grid = PALETTE + ax.grid(axis='y', color=grid, linewidth=0.6) + ax.set_axisbelow(True) + for side in ax.spines.values(): + side.set_visible(False) + ax.tick_params(colors=ink2, labelsize=8, length=0, pad=4) + style_date_axis(ax, start, end, ink) + fit_left_margin(fig, ax) + +def _heading(fig, title, subtitle='', subtitle_color=None): + """A chart's title at the top left, and its subtitle (if any) beneath.""" + _, ink, ink2, _ = PALETTE + fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') + if subtitle: + fig.text(0.03, 0.910, subtitle, color=subtitle_color or ink2, fontsize=8.5, ha='left', va='center') + def _png(fig, facecolor): """The figure as PNG bytes, rewound, or an upload sends an empty file. No bbox_inches='tight': it would change the 1280x800 size.""" buf = io.BytesIO() @@ -402,13 +421,7 @@ def graph(df, title, ylabel): ax.margins(x=0.02) ax.yaxis.set_major_locator(MaxNLocator(nbins=5)) ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:,.2f}".rstrip('0').rstrip('.'))) # 66,000 / 1.5 - ax.grid(axis='y', color=grid, linewidth=0.6) - ax.set_axisbelow(True) - for side in ax.spines.values(): - side.set_visible(False) - ax.tick_params(colors=ink2, labelsize=8, length=0, pad=4) - style_date_axis(ax, x.iloc[0], x.iloc[-1], ink) - fit_left_margin(fig, ax) + _style_time_axes(fig, ax, x.iloc[0], x.iloc[-1]) def annotate(i, name, above): xd, yv = x.iloc[i], y.iloc[i] @@ -426,8 +439,7 @@ def annotate(i, name, above): if imin != len(y) - 1 and imin != imax: annotate(imin, 'Low', False) - fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') - fig.text(0.03, 0.910, f"{last:,.2f} {ylabel or ''} {arrow} {abs(pct):.2f}% over period", color=color, fontsize=8.5, ha='left', va='center') + _heading(fig, title, f"{last:,.2f} {ylabel or ''} {arrow} {abs(pct):.2f}% over period", color) return _png(fig, bg) @@ -453,14 +465,7 @@ def compare_graph(series, title, subtitle=''): ax.margins(x=0.01) ax.yaxis.set_major_locator(MaxNLocator(nbins=6)) ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:+,.0f}%" if v else "0%")) - ax.grid(axis='y', color=grid, linewidth=0.6) - ax.set_axisbelow(True) - for side in ax.spines.values(): - side.set_visible(False) - ax.tick_params(colors=ink2, labelsize=8, length=0, pad=4) - style_date_axis(ax, min(y.index[0] for _, y in rebased), max(y.index[-1] for _, y in rebased), ink) - - fit_left_margin(fig, ax) + _style_time_axes(fig, ax, min(y.index[0] for _, y in rebased), max(y.index[-1] for _, y in rebased)) # direct end-labels in the right margin, spread so they never overlap ymin, ymax = ax.get_ylim() @@ -481,9 +486,7 @@ def compare_graph(series, title, subtitle=''): ax.annotate(f"{label} {value:+,.1f}%", xy=(y.index[-1], value), xycoords='data', xytext=(1.03, p), textcoords='axes fraction', ha='left', va='center', fontsize=7.5, color=ink, annotation_clip=False, arrowprops=dict(arrowstyle='-', color=color, linewidth=0.7, alpha=0.7, shrinkA=0, shrinkB=2, relpos=(0, 0.5))) - fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') - if subtitle: - fig.text(0.03, 0.910, subtitle, color=ink2, fontsize=8.5, ha='left', va='center') + _heading(fig, title, subtitle) return _png(fig, bg) SESSION_TINTS = {'midsession': '#34d399', 'premarket': '#facc15'} # subtle green / yellow wash; at close stays plain @@ -731,9 +734,7 @@ def pie_chart(rows, title, subtitle=''): fig.text(0.60, y, label if len(label) <= 28 else label[:27] + '…', color=ink, fontsize=8.5, ha='left', va='center') fig.text(0.97, y, f"{value / total * 100:.1f}%", color=ink2, fontsize=8.5, ha='right', va='center') y -= step - fig.text(0.03, 0.957, title, color=ink, fontsize=13, fontweight='bold', ha='left', va='center') - if subtitle: - fig.text(0.03, 0.910, subtitle, color=ink2, fontsize=8.5, ha='left', va='center') + _heading(fig, title, subtitle) return _png(fig, bg) def bar_graph(top_rows, bottom_rows, title, subtitle=''): diff --git a/tests/test_util.py b/tests/test_util.py index ac440b18..f84e2b5c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -33,6 +33,11 @@ def test_each_market_has_its_own_flag_and_currency(self): # Sharesight's market if ticker: self.assertEqual(util.flag_from_ticker(ticker), flag) + def test_make_paragraphs_keeps_every_word(self): + self.assertEqual(util.make_paragraphs('Acme makes widgets. It sells them.'), 'Acme makes widgets. It sells them.') + long = ' '.join(['word'] * 22) + ' end. Tail sentence here.' + self.assertEqual(util.make_paragraphs(long), ' '.join(['word'] * 22) + ' end.\n\nTail sentence here.') + def test_transform_title(self): self.assertEqual(util.transform_title('APPLE INC'), 'Apple') self.assertEqual(util.transform_title('Acme FPO'), 'Acme') From 3a7039c49437a337d6c8fe55c09fbcca8b4dc27d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:15:03 +0000 Subject: [PATCH 20/25] shorts: the list and its chart come from one sorted list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The text lines and the chart rows were two lists sorted separately to the same key; now one list of (percent, line, label) is sorted once. The report and chart are unchanged, but for the space after 🩳 in the "nothing found" lines, which every other line has. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- shorts.py | 47 ++++++++++++----------------------------------- 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/shorts.py b/shorts.py index a3ee6f13..674694ca 100755 --- a/shorts.py +++ b/shorts.py @@ -11,47 +11,24 @@ def lambda_handler(chat_id=config_telegramChatID, threshold=UNSET, specific_stoc if threshold is UNSET: threshold = config_shorts_percent def prepare_shorts_payload(service, market_data): - payload = [] - chart_rows = [] - graph = False - emoji = "🩳" + found = [] # (percent, line, chart label) for ticker in tickers: try: short_percent = float(market_data[ticker]['short_percent']) except (KeyError, ValueError): continue - if '.AX' in ticker: - url = 'https://www.shortman.com.au/stock?q=' + ticker.split('.')[0] - short_interest_link = util.link(url, ticker.split('.')[0], service) - else: - url = 'https://finance.yahoo.com/quote/' + ticker + '/key-statistics?p=' + ticker - short_interest_link = util.link(url, ticker.split('.')[0], service) - title = market_data[ticker]['profile_title'] if short_percent > threshold or specific_stock: - payload.append([emoji, title, f'({short_interest_link})', short_percent]) - chart_rows.append((ticker.split('.')[0], short_percent)) - - def last_element(e): - return e[-1] - payload.sort(key=last_element, reverse=True) # largest first, matching the chart - for i, e in enumerate(payload): - e[-1] = str(round(e[-1])) + '%' - payload[i] = ' '.join(e) - - if payload: - if not specific_stock: - message = f'Tracked stocks shorted over {threshold:g}%' - message = webhook.bold(message, service) - payload.insert(0, message) - if chart_rows and webhook.takes_charts(service, interactive): - chart_rows.sort(key=lambda row: row[1], reverse=True) - graph = util.shorts_chart(chart_rows, threshold) - else: - if interactive: - if specific_stock: - payload = [f"{emoji}No short interest found for {tickers[0]}"] - else: - payload = [f"{emoji}No stocks shorted over {threshold:g}%. Try specifying a number."] + code = ticker.split('.')[0] + url = 'https://www.shortman.com.au/stock?q=' + code if ticker.endswith('.AX') else 'https://finance.yahoo.com/quote/' + ticker + '/key-statistics?p=' + ticker + found.append((short_percent, f"🩳 {market_data[ticker]['profile_title']} ({util.link(url, code, service)}) {round(short_percent)}%", code)) + found.sort(key=lambda row: row[0], reverse=True) # largest first, as the chart has them + payload, graph = [line for _, line, _ in found], None + if payload and not specific_stock: + payload.insert(0, webhook.bold(f'Tracked stocks shorted over {threshold:g}%', service)) + if webhook.takes_charts(service, interactive): + graph = util.shorts_chart([(code, percent) for percent, _, code in found], threshold) + elif not payload and interactive: + payload = [f"🩳 No short interest found for {tickers[0]}" if specific_stock else f"🩳 No stocks shorted over {threshold:g}%. Try specifying a number."] return payload, graph # MAIN # From b89effc835757a433eaf24407994338efad532d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:25:00 +0000 Subject: [PATCH 21/25] Record highs are all-time; ratings list the strongest first; .top is gone; a bare number on .price is a percent As decided: - milestone keeps the all-time record (the higher of the stored record and the new 52-week high, the lower for lows). After its first run it used to store the 52-week values, so a stock far below its peak posted a record high at each new 52-week high. Records kept before this (52-week, under 'high'/'low') are replaced by the ticker's history on the next run. - .buy and .sell list the strongest rating first (.sell used to start with the mildest), and on equal ratings the one more analysts give. - .top, undocumented and the same as .price top [period], is removed; .top alone looked up a ticker named TOP and .top10 read its 0 as a 0-day period. - .price 3 is 3%, as on .session, .premarket and .shorts and as the README has it; a period carries its unit (.price 3 7d is 3% over a week; it used to drop the 3). The README's price section names .session (not .midsession) and @botname price. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 8 ++++---- lib/reports.py | 5 +++-- lib/worker.py | 11 +++-------- milestone.py | 9 ++++----- tests/test_edges.py | 9 +++++++++ tests/test_reports.py | 36 +++++++++++++++++++++++++++++++++--- 6 files changed, 56 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7dff6db2..45c68ce4 100644 --- a/README.md +++ b/README.md @@ -207,10 +207,10 @@ Interactive trigger (pre-market): Interactive trigger (mid-session): ``` -.midsession [percent] +.session [percent] ``` ``` -@botname midsession [percent] +@botname session [percent] ``` Interactive trigger (interday). @@ -218,10 +218,10 @@ Interactive trigger (interday). .price [percent] ``` ``` -@botname [percent] +@botname price [percent] ``` -For interday, can also specify the time period. +For interday, can also specify the time period. A bare number is the percent; a period carries its unit: `7d`, `2w`, `3m`, `1y` or `ytd`. For example, `.price 3 7d` lists the moves of 3% or more over the past week. ``` .price [percent] [period] ``` diff --git a/lib/reports.py b/lib/reports.py index b0af083b..cdcf6bf6 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -374,7 +374,8 @@ def prepare_marketcap_payload(service, action='top', length=15): return [webhook.bold(f'{action.title()} {length} tracked stocks by market cap', service)] + [line for _, line in caps[:length]] def prepare_rating_payload(service, action, length=15): - """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by more than one analyst, by mean rating.""" + """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by more than one analyst, the strongest + rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give first.""" found = [] # (rating, analysts, line, ticker) for ticker, data in yahoo.fetch_details(util.get_holdings_and_watchlist()).items(): if 'recommend' not in data or data['recommend_analysts'] <= 1: @@ -384,7 +385,7 @@ def prepare_rating_payload(service, action, length=15): rating, analysts = round(data['recommend_index'], 2), data['recommend_analysts'] link = util.finance_link(ticker, data['profile_exchange'], service) found.append((rating, analysts, f"{util.flag_from_ticker(ticker)} {data['profile_title']} ({link}) {rating} ({analysts} analysts)", ticker)) - found = sorted(found, key=lambda row: row[:2])[:length] + found = sorted(found, key=lambda row: (row[0] if action == 'buy' else -row[0], -row[1]))[:length] payload = [line for _, _, line, _ in found] chart = None if payload: diff --git a/lib/worker.py b/lib/worker.py index 78d00e20..27b206fa 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -252,15 +252,16 @@ def on_session(request, m): price.lambda_handler(request.chat_id, threshold, request.service, request.user, ticker, interactive=True, midsession=not premarket, premarket=premarket) def on_price(request, m): + """.price [percent|SYMBOL|top|bottom] [period]: a bare number is a percent, as on the other commands; a period has its unit.""" threshold, ticker, days, top = config_price_percent, None, None, None for i, arg in enumerate(m.groups()): if not arg: continue try: - days = util.days_from_human_days(arg) + threshold = parse_percent(arg) except ValueError: try: - threshold = parse_percent(arg) + days = util.days_from_human_days(arg) except ValueError: if i == 0 and arg in ('top', 'bottom'): # one chart shows both ends threshold, top = 0, 10 @@ -269,11 +270,6 @@ def on_price(request, m): with request.typing(): price.lambda_handler(request.chat_id, threshold, request.service, request.user, ticker, interactive=True, interday=days is None, days=days, top=top) -def on_top(request, m): - days, _ = period_or_ticker(m.group(1), config_past_days) - with request.typing(): - price.lambda_handler(request.chat_id, 0, request.service, request.user, days=days, interactive=True, top=10) - def on_shorts(request, m): threshold, ticker = percent_or_ticker(m.group(1), config_shorts_percent) with request.typing(): @@ -382,7 +378,6 @@ def on_profile(request, m): (r"performance?\s*(?P[\w\s]*)", on_performance), (r"(?Psession|premarket|postmarket|permarket)\s*(?P[\w\.\:\-]+)*", on_session), (r"(?:prices?|prince|print|probe|piece|pierce|pence|prime)\s*([\w\.\:\%\=\-\^]+)*\s*([\w\%]+)*", on_price), - (r"(?:top\d*)\s*([\w]+)", on_top), (r"shorts?\s*([\w\.\:\-]+)*", on_shorts), (r"trades?\s*(?P[\w\s]*)", on_trades), (r"holdings?\s*([\w\s]+)*", on_holdings), diff --git a/milestone.py b/milestone.py index f16ccd20..4c778d5b 100755 --- a/milestone.py +++ b/milestone.py @@ -20,11 +20,10 @@ def prepare_milestone_payload(service, market_data): records = util.json_load('finbot_milestone.json', persist=True) or {} for ticker in tickers: try: - oldhigh = records[ticker]['high'] - oldlow = records[ticker]['low'] - except (KeyError, TypeError): + oldhigh, oldlow = records[ticker]['record_high'], records[ticker]['record_low'] + except (KeyError, TypeError): # first sighting, or a record kept before records were all-time (52-week, under 'high'/'low') try: - oldhigh, oldlow = yahoo.historic_high(ticker) # first sighting: its history so far + oldhigh, oldlow = yahoo.historic_high(ticker) # its history so far except (RuntimeError, KeyError, IndexError, ValueError): continue # no usable history: nothing recorded, so a later run cannot take zeros for its record try: @@ -35,7 +34,7 @@ def prepare_milestone_payload(service, market_data): title = market_data[ticker]['profile_title'] ticker_link = util.finance_link(ticker, market_data[ticker]['profile_exchange'], service) emoji = util.flag_from_ticker(ticker) - records[ticker] = {'high': newhigh, 'low': newlow} + records[ticker] = {'record_high': max(oldhigh, newhigh), 'record_low': min(oldlow, newlow)} # all-time: a 52-week high below it is no record if newhigh > oldhigh: print("DEBUG", ticker, newhigh, "is higher than", oldhigh) if debug else None payloadhigh.append(f"{emoji} {title} ({ticker_link})") diff --git a/tests/test_edges.py b/tests/test_edges.py index 804820f8..d6776e93 100644 --- a/tests/test_edges.py +++ b/tests/test_edges.py @@ -127,6 +127,15 @@ def test_a_percentage_is_a_number_and_nan_a_ticker(self): self.assertEqual(worker.percent_or_ticker(arg, 5), expected) self.assertIn('2.5%', str(self.command('.session 2.5'))) # a threshold, not a ticker called 2.5 + def test_a_bare_number_on_price_is_a_percent(self): + """As on .session, .premarket and .shorts, and as the README has it; a period carries its unit.""" + self.assertIn('3% at close', str(self.command('.price 3'))) # it used to be a 3-day report + self.assertIn('Price movement 3%', str(self.command('.price 3 7d'))) # the 3 used to be dropped + self.assertIn('Price movement 5%', str(self.command('.price 7d'))) + + def test_top_is_gone(self): + self.assertIn('Co TOP', str(self.command('.top 7d'))) # .price top 7d is the command; .top is a ticker now, like any word + def test_a_ratio_command_reads_its_ticker_as_the_other_commands_do(self): self.assertIn('Co T1 ', str(self.command('.pe t1'))) self.assertIn('Top 15 tracked stocks by pe ratio', str(self.command('.pe top '))) # trailing space and all diff --git a/tests/test_reports.py b/tests/test_reports.py index 13fc489e..7b993190 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -15,11 +15,11 @@ def test_text_commands(self): def test_chart_commands(self): for cmd in ('.price', '.price 7d', '.price top', '.price bottom', '.session', '.premarket', '.postmarket', - '.performance', '.performance 7d', '.performance 1y', '.shorts', '.peg', '.buy', '.sell', '.beta', '.top10 week'): + '.performance', '.performance 7d', '.performance 1y', '.shorts', '.peg', '.buy', '.sell', '.beta', '.price top 7d'): with self.subTest(cmd=cmd): out = self.command(cmd) self.assertTrue(out, f'{cmd} sent nothing') - self.assertTrue(any(kind == 'photo' for kind, *_ in out) or cmd.startswith('.top10'), f'{cmd} sent no chart: {out}') + self.assertTrue(any(kind == 'photo' for kind, *_ in out), f'{cmd} sent no chart: {out}') def test_history_and_compare(self): for cmd in ('.history T1', '.compare T1 T2', '.compare T1 T2 3m'): @@ -49,6 +49,18 @@ def test_prepare_rating(self): for action in ('buy', 'sell'): self.assertTrue(reports.prepare_rating_payload('telegram', action, length=5)[0]) + def test_ratings_list_the_strongest_first(self): + """1 is a strong buy and 5 a strong sell: .sell used to list the mildest sells first. On equal ratings, more analysts first.""" + ratings = {'T1': ('buy', 1.5, 3), 'T2': ('strong_buy', 1.2, 2), 'T3': ('buy', 1.5, 9), + 'T4': ('sell', 4.2, 3), 'T5': ('underperform', 4.6, 2), 'T6': ('sell', 4.2, 8)} + self.patch(util, 'get_holdings_and_watchlist', lambda: list(ratings)) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | dict(zip(('recommend', 'recommend_index', 'recommend_analysts'), ratings[t]))}) + def order(action): + payload, chart = reports.prepare_rating_payload('telegram', action) + return [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], [label for label, _ in chart['rows']] + self.assertEqual(order('buy'), (['T2', 'T3', 'T1'], ['T2', 'T3', 'T1'])) + self.assertEqual(order('sell'), (['T5', 'T6', 'T4'], ['T5', 'T6', 'T4'])) + def test_prepare_value(self): for action, heading in (('pe', 'Top 5 tracked stocks by pe ratio'), ('bottom forward pe', 'Bottom 5 tracked stocks by forward pe ratio'), ('peg', 'Top 5 tracked stocks by peg ratio'), ('negative peg', 'negative peg: no data found')): @@ -150,7 +162,25 @@ def test_milestone_records_a_ticker_only_with_its_new_values(self): self.sent.clear() milestone.lambda_handler() self.assertNotIn('Record high', str(self.sent)) - self.assertEqual(util.json_load('finbot_milestone.json', persist=True)['T1'], {'high': 120.0, 'low': 80.0}) + self.assertEqual(util.json_load('finbot_milestone.json', persist=True)['T1'], {'record_high': 150.0, 'record_low': 50.0}) # all-time + + def test_milestone_records_are_all_time(self): + """A new 52-week high below the all-time high is no record. The record used to become the 52-week high after the first + run, so a stock far below its peak posted a record high each time it set a new 52-week one.""" + self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) + milestone.lambda_handler() # first sighting: the 52-week range (120/80) is inside the history (150/50) + for high, expected in ((130.0, False), (160.0, True), (155.0, False)): + with self.subTest(high=high): + self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) | {'fiftyTwoWeekHigh': high} for t in ts}) + self.sent.clear() + milestone.lambda_handler() + self.assertEqual('Record high' in str(self.sent), expected) + + def test_a_52_week_record_kept_before_is_replaced_by_the_history(self): + util.json_write('finbot_milestone.json', {f'T{i}': {'high': 110.0, 'low': 90.0} for i in range(25)}, persist=True) + self.patch(yahoo, 'historic_high', lambda t: (150.0, 50.0)) + milestone.lambda_handler() # 120 beats the old 52-week record of 110, but not the history's 150 + self.assertNotIn('Record high', str(self.sent)) def test_rating(self): rating.lambda_handler(interactive=True, service='telegram', chat_id='5') From 21213a0bcf13594727e6ee56ce402a94fd6dcc37 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:47:06 +0000 Subject: [PATCH 22/25] .buy/.sell need 7 analysts; market caps compare in US dollars, so a won market cap no longer tops .marketcap - .buy and .sell list only stocks rated by 7 or more analysts, and say so in the heading; with none, the reply says that. Thinly covered ASX small caps (two to six brokers) were nearly all a perfect 1.0 and filled the top of .buy. - yahoo.add_usd_market_caps converts each market cap, which Yahoo gives in the stock's own currency, to US dollars at Yahoo's exchange rates (KRW=X and the like); a market quoted in pence, agorot or cents has its cap in pounds, shekels or rand. .marketcap ranks and shows US dollars ("(USD)" in the heading; a stock with no rate is left out and logged); .marketcap SYMBOL shows its own currency and the US dollar figure. .beta's "mkt cap under 2B" and demote_volatile's small-cap thresholds (150M, 1B) are US dollars too: unconverted, a small cap quoted in won or yen was never small. For a stock quoted in Australian dollars the thresholds move by the exchange rate. The profile labels a pence-quoted market cap GBP. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 12 ++++---- lib/reports.py | 64 +++++++++++++++++++++++-------------------- lib/worker.py | 2 +- lib/yahoo.py | 20 ++++++++++++++ price.py | 6 ++-- tests/common.py | 2 +- tests/test_reports.py | 40 ++++++++++++++++++++++++--- 7 files changed, 103 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 45c68ce4..3d7de1d0 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ You can also specify a portfolio name to get trades for just that portfolio: `prices.py` sends premarket, midsession, and interday price movements if any holdings moved over a percentage threshold. This data is sourced from Yahoo! Finance and Sharesight. The default threshold is 5% but you can change it by setting `price_percent` in the `preferences.conf`, or by providing a number as argument when triggered through the chat bot. Decimal fractions are accepted. -If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under 150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under 1B will have their alert threshold (`price_percent`) doubled. +If `demote_volatile` is `true`, stocks with a [market cap](https://www.investopedia.com/terms/m/marketcapitalization.asp) under US$150M, or a [beta](https://www.investopedia.com/terms/b/beta.asp) of > 1.5 and a market cap of under US$1B will have their alert threshold (`price_percent`) doubled. A market cap in another currency is converted at Yahoo's exchange rate. Config example: ``` @@ -347,7 +347,7 @@ Interactive trigger: ### Buy ![Buy ratings in Slack](img/finbot_buy.png?raw=true "Buy ratings in Slack") -Show top recommended buys by Yahoo Finance analysts rating +Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by 7 or more analysts (with fewer, the consensus says little: thinly covered small caps are nearly all a perfect buy) Interactive trigger: ``` @@ -360,7 +360,7 @@ Interactive trigger: ### Sell ![Rating report in Slack](img/finbot_sell.png?raw=true "Rating report in Slack") -Show top recommended sells by Yahoo Finance analysts rating +Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by 7 or more analysts Interactive trigger: ``` @@ -373,14 +373,14 @@ Interactive trigger: ### Marketcap ![Marketcap report in Slack](img/finbot_marketcap.png?raw=true "Marketcap report in Slack") -List the top (or bottom) stocks by market cap +List the top (or bottom) stocks by market cap, in US dollars: market caps in other currencies are converted at Yahoo's exchange rates. With a symbol, its market cap in its own currency and in US dollars. Interactive trigger: ``` -.marketcap [top|bottom] +.marketcap [top|bottom|SYMBOL] ``` ``` -@botname marketcap [top|bottom] +@botname marketcap [top|bottom|SYMBOL] ``` ### Consensus rating changes diff --git a/lib/reports.py b/lib/reports.py index cdcf6bf6..5d34915c 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -353,32 +353,39 @@ def prepare_pie(portfolio_name, by, service, max_slices=8): return payload, {'pie': True, 'rows': rows, 'title': title, 'subtitle': subtitle} def prepare_marketcap_payload(service, action='top', length=15): - """The `length` tracked stocks of largest (action 'top') or smallest ('bottom') market cap; any other action is one ticker's.""" + """The `length` tracked stocks of largest (action 'top') or smallest ('bottom') market cap, compared in US dollars at Yahoo's + exchange rates: a market cap in won or yen is a far bigger number. Any other action is one ticker's, in its own currency too.""" if action not in ('top', 'bottom'): ticker = util.transform_to_yahoo(action.upper()) - data = yahoo.fetch_detail(ticker, 600).get(ticker, {}) + data = yahoo.add_usd_market_caps(yahoo.fetch_detail(ticker, 600)).get(ticker, {}) if 'market_cap' not in data: return [f"Mkt cap not found for {ticker}"] + currency = yahoo.market_cap_currency(data) or '' + cap = f"{currency} {util.humanUnits(data['market_cap'])}".strip() + if currency != 'USD' and 'market_cap_usd' in data: + cap += f" (USD {util.humanUnits(data['market_cap_usd'])})" link = util.finance_link(ticker, data.get('profile_exchange', ''), service) - return [util.security_line(util.flag_from_ticker(ticker), data.get('profile_title', ticker), link) + f" mkt cap: {util.humanUnits(data['market_cap'])}"] - caps = [] # (market cap, line) - for ticker, data in yahoo.fetch(util.get_holdings_and_watchlist()).items(): - if 'market_cap' not in data: - print(ticker, "no market cap", file=sys.stderr) + return [util.security_line(util.flag_from_ticker(ticker), data.get('profile_title', ticker), link) + f" mkt cap: {cap}"] + caps = [] # (market cap in US dollars, line) + for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch(util.get_holdings_and_watchlist())).items(): + if 'market_cap_usd' not in data: + print(ticker, "no market cap" if 'market_cap' not in data else f"no exchange rate for {data.get('currency')}", file=sys.stderr) continue link = util.finance_link(ticker, data['profile_exchange'], service) - caps.append((data['market_cap'], util.security_line(util.flag_from_ticker(ticker), data['profile_title'], link) + f" mkt cap: {util.humanUnits(data['market_cap'])}")) + caps.append((data['market_cap_usd'], util.security_line(util.flag_from_ticker(ticker), data['profile_title'], link) + f" mkt cap: {util.humanUnits(data['market_cap_usd'])}")) if not caps: return ['No market cap data found'] caps.sort(key=lambda cap: cap[0], reverse=action == 'top') - return [webhook.bold(f'{action.title()} {length} tracked stocks by market cap', service)] + [line for _, line in caps[:length]] + return [webhook.bold(f'{action.title()} {length} tracked stocks by market cap (USD)', service)] + [line for _, line in caps[:length]] + +MIN_ANALYSTS = 7 # fewer, and a consensus says little: ASX small caps covered by two to six brokers are nearly all a perfect 1.0 buy def prepare_rating_payload(service, action, length=15): - """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by more than one analyst, the strongest - rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give first.""" + """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by MIN_ANALYSTS or more analysts, the + strongest rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give first.""" found = [] # (rating, analysts, line, ticker) for ticker, data in yahoo.fetch_details(util.get_holdings_and_watchlist()).items(): - if 'recommend' not in data or data['recommend_analysts'] <= 1: + if 'recommend' not in data or data['recommend_analysts'] < MIN_ANALYSTS: continue recommend = data['recommend'].replace('_', ' ') if (action == 'buy' and 'buy' in recommend) or (action == 'sell' and recommend in ('sell', 'underperform')): @@ -387,17 +394,16 @@ def prepare_rating_payload(service, action, length=15): found.append((rating, analysts, f"{util.flag_from_ticker(ticker)} {data['profile_title']} ({link}) {rating} ({analysts} analysts)", ticker)) found = sorted(found, key=lambda row: (row[0] if action == 'buy' else -row[0], -row[1]))[:length] payload = [line for _, _, line, _ in found] - chart = None - if payload: - message = f"Top {length} analyst {action} ratings for tracked stocks" - payload.insert(0, f"{webhook.bold(message, service)}") - shown = [(ticker.split('.')[0], rating) for rating, _, _, ticker in found] - low, high = min(r for _, r in shown), max(r for _, r in shown) - spread = (high - low) or 0.2 - # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) - chart = {'rows': shown, 'title': message, 'subtitle': 'Mean analyst rating: 1 = strong buy, 5 = strong sell (axis zoomed)', - 'value_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} - return payload, chart + if not payload: + return [f"No tracked stocks rated {action} by {MIN_ANALYSTS}+ analysts"], None + message = f"Top {length} analyst {action} ratings for tracked stocks ({MIN_ANALYSTS}+ analysts)" + shown = [(ticker.split('.')[0], rating) for rating, _, _, ticker in found] + low, high = min(r for _, r in shown), max(r for _, r in shown) + spread = (high - low) or 0.2 + # zoom the axis onto the ratings so the bars fill the panel (ratings only run 1-5) + chart = {'rows': shown, 'title': message, 'subtitle': 'Mean analyst rating: 1 = strong buy, 5 = strong sell (axis zoomed)', + 'value_fmt': lambda v: f"{v:g}", 'xlim': (max(0.9, low - spread * 0.9), min(5.1, high + spread * 0.25))} + return [webhook.bold(message, service)] + payload, chart def prepare_value_payload(service, action='pe', ticker_select=None, length=15): payload = [] @@ -475,19 +481,19 @@ def prepare_value_payload(service, action='pe', ticker_select=None, length=15): return payload, chart def prepare_beta_payload(service): - """(payload, chart) of the tracked stocks with a beta over 1.5 and a market cap under 2B, the highest beta first.""" + """(payload, chart) of the tracked stocks with a beta over 1.5 and a market cap under US$2B, the highest beta first.""" found = [] # (beta, line, chart row) - for ticker, data in yahoo.fetch_details(util.get_holdings_and_watchlist()).items(): + for ticker, data in yahoo.add_usd_market_caps(yahoo.fetch_details(util.get_holdings_and_watchlist())).items(): if 'beta' not in data: continue beta = round(data['beta'], 1) - if beta > 1.5 and data.get('market_cap', float('inf')) < 2000000000: + if beta > 1.5 and data.get('market_cap_usd', float('inf')) < 2e9: link = util.finance_link(ticker, data['profile_exchange'], service) found.append((beta, util.security_line(util.flag_from_ticker(ticker), data['profile_title'], link) + f" {beta} β", (ticker.split('.')[0], float(beta)))) if not found: - return ['No tracked stocks have a beta over 1.5 and mkt cap under 2B'], None + return ['No tracked stocks have a beta over 1.5 and mkt cap under US$2B'], None found.sort(key=lambda row: row[0], reverse=True) - heading = 'Beta over 1.5 and mkt cap under 2B' + heading = 'Beta over 1.5 and mkt cap under US$2B' chart = {'rows': [row for _, _, row in found], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} return [webhook.bold(heading, service)] + [line for _, line, _ in found], chart @@ -560,7 +566,7 @@ def prepare_profile_payload(service, user, ticker): payload.append("") if 'currency' in data and 'market_cap' in data: - payload.append(webhook.bold("Mkt cap:", service) + f" {currency} {util.humanUnits(data['market_cap'])}") + payload.append(webhook.bold("Mkt cap:", service) + f" {yahoo.market_cap_currency(data)} {util.humanUnits(data['market_cap'])}") # GBP, not pence if all(key in data for key in ('shareholder_equity', 'total_debt', 'total_cash', 'profile_industry')) and data['shareholder_equity'] and 'Bank' not in data['profile_industry']: net_debt_equity_ratio = round((data['total_debt'] - data['total_cash']) / data['shareholder_equity'] * 100) if net_debt_equity_ratio > 0: diff --git a/lib/worker.py b/lib/worker.py index 27b206fa..235b67df 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -325,7 +325,7 @@ def on_rating(request, m): action = m.group(1).lower() with request.typing(): payload, chart = reports.prepare_rating_payload(request.service, action, length=15) - request.deliver(payload or [f"No stocks meet {action} criteria"], chart) + request.deliver(payload, chart) def on_compare(request, m): with request.typing(): diff --git a/lib/yahoo.py b/lib/yahoo.py index cf9ec93f..83720238 100644 --- a/lib/yahoo.py +++ b/lib/yahoo.py @@ -389,6 +389,26 @@ def one(ticker): with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: return list(executor.map(one, tickers)) +SUBUNITS = {'GBp': 'GBP', 'GBX': 'GBP', 'ILA': 'ILS', 'ZAc': 'ZAR'} # Yahoo prices these markets in a subunit but gives market caps in the unit + +def market_cap_currency(data): + """The currency of a ticker's market cap: its quote currency, but the unit where Yahoo quotes in pence or the like.""" + return SUBUNITS.get(data.get('currency'), data.get('currency')) + +def add_usd_market_caps(market_data): + """Put 'market_cap_usd' beside each 'market_cap', which Yahoo gives in the ticker's own currency, converted at Yahoo's + exchange rates, so that market caps in won, yen and dollars compare. A ticker whose currency Yahoo has no rate for gets + none. Returns market_data.""" + wanted = {market_cap_currency(data) for data in market_data.values() if 'market_cap' in data and market_cap_currency(data)} - {'USD'} + rates = {'USD': 1.0} + for symbol, quote in (fetch([f'{currency}=X' for currency in wanted]) if wanted else {}).items(): # KRW=X: won per US dollar + if quote.get('regularMarketPrice'): + rates[symbol.removesuffix('=X')] = 1 / quote['regularMarketPrice'] + for data in market_data.values(): + if 'market_cap' in data and market_cap_currency(data) in rates: + data['market_cap_usd'] = data['market_cap'] * rates[market_cap_currency(data)] + return market_data + def fetch_details(tickers, strict=False): """fetch_detail of every ticker, fetched at the same time, in one {ticker: detail}. A ticker whose fetch failed is left out (and logged), so that a list goes out without it; strict=True raises its error instead, for a report on the ticker asked for.""" diff --git a/price.py b/price.py index 34ec5434..d1b27426 100755 --- a/price.py +++ b/price.py @@ -15,6 +15,8 @@ def gather(specific_stock=None, days=None): Sharesight has none for (the watchlist) their Yahoo history, fetched together up front.""" tickers = [util.transform_to_yahoo(specific_stock)] if specific_stock else util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) + if config_demote_volatile: # trigger_bar compares market caps, in US dollars + yahoo.add_usd_market_caps(market_data) # Sharesight reports a holding's performance only for the time it was held: held for 1Y and asked for 5Y, it gives 1Y if days and (not specific_stock or config_performance_use_sharesight): for data in sharesight.get_performance_wrapper(days).values(): @@ -30,11 +32,11 @@ def gather(specific_stock=None, days=None): def trigger_bar(ticker, quote, threshold): """The move this ticker needs before it alone sends a scheduled report: more for leveraged ETFs (demote_leveraged) and volatile - small caps (demote_volatile), else the threshold itself.""" + small caps (demote_volatile; their market caps in US dollars), else the threshold itself.""" title = quote['profile_title'].lower() if config_demote_leveraged and ('2x' in title or '3x' in title): return threshold * 1.3 - cap = quote.get('market_cap') + cap = quote.get('market_cap_usd') if config_demote_volatile and cap is not None and cap < 1e9: if cap < 1e7: return threshold * config_volatility_multiplier * 1.3 diff --git a/tests/common.py b/tests/common.py index 89203846..d6a0e971 100644 --- a/tests/common.py +++ b/tests/common.py @@ -40,7 +40,7 @@ def market_entry(ticker): 'beta': 2.0, 'short_percent': 20, 'fiftyTwoWeekHigh': 120.0, 'fiftyTwoWeekLow': 80.0, 'regularMarketPrice': 100.0, 'earnings_date': time.time() + 86400 * 3, 'ex_dividend_date': time.time() + 86400 * 3, 'recommend': 'buy' if i % 2 else 'sell', 'recommend_index': 1.2 if i % 2 else 4.2, - 'recommend_analysts': 4, 'price_to_earnings_peg': 1.2, 'currency': 'USD'} + 'recommend_analysts': 8, 'price_to_earnings_peg': 1.2, 'currency': 'USD'} class FinbotCase(unittest.TestCase): diff --git a/tests/test_reports.py b/tests/test_reports.py index 7b993190..2aca8409 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -50,9 +50,10 @@ def test_prepare_rating(self): self.assertTrue(reports.prepare_rating_payload('telegram', action, length=5)[0]) def test_ratings_list_the_strongest_first(self): - """1 is a strong buy and 5 a strong sell: .sell used to list the mildest sells first. On equal ratings, more analysts first.""" - ratings = {'T1': ('buy', 1.5, 3), 'T2': ('strong_buy', 1.2, 2), 'T3': ('buy', 1.5, 9), - 'T4': ('sell', 4.2, 3), 'T5': ('underperform', 4.6, 2), 'T6': ('sell', 4.2, 8)} + """1 is a strong buy and 5 a strong sell: .sell used to list the mildest sells first. On equal ratings, more analysts first; + fewer than 7 analysts and the stock is left out (T7, T8: the ASX small caps that are always a perfect 1.0).""" + ratings = {'T1': ('buy', 1.5, 7), 'T2': ('strong_buy', 1.2, 12), 'T3': ('buy', 1.5, 9), 'T7': ('strong_buy', 1.0, 6), + 'T4': ('sell', 4.2, 7), 'T5': ('underperform', 4.6, 10), 'T6': ('sell', 4.2, 8), 'T8': ('strong_buy', 1.0, 2)} self.patch(util, 'get_holdings_and_watchlist', lambda: list(ratings)) self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | dict(zip(('recommend', 'recommend_index', 'recommend_analysts'), ratings[t]))}) def order(action): @@ -60,6 +61,37 @@ def order(action): return [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], [label for label, _ in chart['rows']] self.assertEqual(order('buy'), (['T2', 'T3', 'T1'], ['T2', 'T3', 'T1'])) self.assertEqual(order('sell'), (['T5', 'T6', 'T4'], ['T5', 'T6', 'T4'])) + self.assertIn('(7+ analysts)', reports.prepare_rating_payload('telegram', 'buy')[0][0]) + self.patch(util, 'get_holdings_and_watchlist', lambda: ['T7', 'T8']) + self.assertEqual(reports.prepare_rating_payload('telegram', 'buy'), (['No tracked stocks rated buy by 7+ analysts'], None)) + + def test_a_small_cap_quoted_in_won_needs_the_bigger_move_too(self): + """demote_volatile compares market caps in US dollars: unconverted, a small cap quoted in won was never small.""" + self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) | ({'regularMarketPrice': 1380.0} if t == 'KRW=X' else {'currency': 'KRW', 'market_cap': 1e11}) for t in ts}) + self.patch(price, 'config_demote_volatile', True) + quote = price.gather()[1]['T1'] + self.assertAlmostEqual(quote['market_cap_usd'], 1e11 / 1380.0) # US$72M + self.assertEqual(price.trigger_bar('T1', quote, 5), 5 * price.config_volatility_multiplier) + + def test_market_caps_compare_in_us_dollars(self): + """Yahoo gives a market cap in the stock's own currency: unconverted, a won market cap topped every list.""" + stocks = {'K1.KS': ('KRW', 1.01e15), 'K2.KS': ('KRW', 5e11), 'U1': ('USD', 5.39e12), 'L1.L': ('GBp', 6.25e10), 'Z1': ('XYZ', 1e12)} + fx = {'KRW=X': 1380.0, 'GBP=X': 0.79} # units per US dollar; nothing for XYZ + def fetch(tickers): + return {t: market_entry(t) | {'regularMarketPrice': fx[t]} if t in fx else market_entry(t) | {'currency': stocks[t][0], 'market_cap': stocks[t][1], 'beta': 2.0} + for t in tickers if t in fx or t in stocks} + self.patch(yahoo, 'fetch', fetch) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: fetch([t])) + self.patch(util, 'get_holdings_and_watchlist', lambda: list(stocks)) + payload = reports.prepare_marketcap_payload('telegram') + self.assertIn('by market cap (USD)', payload[0]) + self.assertEqual([line.split('Co ')[1].split(' ')[0] for line in payload[1:]], ['U1', 'K1.KS', 'L1.L', 'K2.KS']) # Z1: no rate, left out + self.assertIn('mkt cap: 731.88 B', payload[2]) # 1.01 Q won + self.assertIn('mkt cap: KRW 1.01 Q (USD 731.88 B)', reports.prepare_marketcap_payload('telegram', 'K1.KS')[0]) + self.assertIn('mkt cap: GBP 62.50 B (USD 79.11 B)', reports.prepare_marketcap_payload('telegram', 'L1.L')[0]) # pence quote, pound cap + beta = reports.prepare_beta_payload('telegram')[0] + self.assertIn('Co K2.KS', str(beta)) # 500 billion won is US$362M: a small cap, which it never was unconverted + self.assertNotIn('Co K1.KS', str(beta)) def test_prepare_value(self): for action, heading in (('pe', 'Top 5 tracked stocks by pe ratio'), ('bottom forward pe', 'Bottom 5 tracked stocks by forward pe ratio'), @@ -73,7 +105,7 @@ def test_prepare_beta(self): self.assertIn('Beta over 1.5', payload[0]) self.assertEqual(len(chart['rows']), len(payload) - 1) # a bar per stock listed self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'beta': 1.0}}) - self.assertEqual(reports.prepare_beta_payload('telegram'), (['No tracked stocks have a beta over 1.5 and mkt cap under 2B'], None)) + self.assertEqual(reports.prepare_beta_payload('telegram'), (['No tracked stocks have a beta over 1.5 and mkt cap under US$2B'], None)) def test_prepare_profile(self): self.assertTrue(reports.prepare_profile_payload('telegram', '@u', 'T1')) From 80244543c7a091887918dc918ae1961b1e4d6976 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:51:23 +0000 Subject: [PATCH 23/25] Keyboard: one .buy & sell button sends both rating reports; .forward pe takes the spare place; buttons stay alphabetical - The rating command takes both words (.buy & sell, .buy sell, .buy/sell, .buy and sell) and sends each report once, in the order asked; .buy and .sell alone are unchanged. - The Telegram command keyboard has .buy & sell in place of .buy and .sell, and .forward pe in the place that frees, sorted alphabetically row by row. Its version changes, so chats get the new keyboard at startup and after their next DM reply. - A test checks that the keyboard is in order and that every button is a command, not a ticker for the .SYMBOL catch-all. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/charts.py | 3 ++- lib/worker.py | 11 ++++++----- tests/test_reports.py | 20 ++++++++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/lib/charts.py b/lib/charts.py index d60acbb8..124b7276 100644 --- a/lib/charts.py +++ b/lib/charts.py @@ -42,7 +42,8 @@ def resolve(ref): return tickers return ref.split(',') -COMMAND_BUTTONS = sorted(['.watchlist', '.dividend', '.earnings', '.marketcap', '.beta', '.performance', '.price', '.session', '.premarket', '.buy', '.sell', '.pe', '.peg', '.shorts', '.trades']) +COMMAND_BUTTONS = sorted(['.beta', '.buy & sell', '.dividend', '.earnings', '.forward pe', '.marketcap', '.pe', '.peg', '.performance', '.premarket', + '.price', '.session', '.shorts', '.trades', '.watchlist']) # alphabetical, row by row; '.buy & sell' sends both rating reports def command_keyboard(per_row=3): """Persistent reply keyboard of the common commands (Telegram DMs); a press sends the command text as a message.""" diff --git a/lib/worker.py b/lib/worker.py index 235b67df..98953322 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -322,10 +322,11 @@ def on_pie(request, m): request.deliver(payload, chart) def on_rating(request, m): - action = m.group(1).lower() - with request.typing(): - payload, chart = reports.prepare_rating_payload(request.service, action, length=15) - request.deliver(payload, chart) + """The analysts' strongest buy ratings (.buy), sell ratings (.sell), or both (.buy & sell, a keyboard button), as asked.""" + for action in dict.fromkeys(word.lower() for word in m.group('first', 'second') if word): # each once, in the order asked + with request.typing(): + payload, chart = reports.prepare_rating_payload(request.service, action, length=15) + request.deliver(payload, chart) def on_compare(request, m): with request.typing(): @@ -387,7 +388,7 @@ def on_profile(request, m): (r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*", on_ratio('forward pe', negative=True)), (r"beta\s*([\w\.\:\-]+)*", on_beta), (r"pie(?:$|\s+)(?P[\w\s\-\.&]*)", on_pie), - (r"(buy|sell)", on_rating), + (r"(?Pbuy|sell)(?:\s*(?:&|and|/|\+)?\s*(?Pbuy|sell))?", on_rating), # .buy & sell (or .buy sell): both (r"compare(?:\s+(?P[\w\.\:\-\^\=\s]+))?", on_compare), (r"(?:history|hospital|visual|hosiery)\s*(?P[\w\.\:\-]+)?\s*(?P[\w\%]+)*", on_history), (r"plan\s*(.*)", on_plan), diff --git a/tests/test_reports.py b/tests/test_reports.py index 2aca8409..a667f4c4 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -21,6 +21,26 @@ def test_chart_commands(self): self.assertTrue(out, f'{cmd} sent nothing') self.assertTrue(any(kind == 'photo' for kind, *_ in out), f'{cmd} sent no chart: {out}') + def test_one_button_asks_for_both_ratings_reports(self): + for text in ('.buy & sell', '.buy sell', '.buy/sell', '.buy and sell'): + with self.subTest(text=text): + captions = [caption for kind, _, caption in self.command(text) if kind == 'photo'] + self.assertEqual(len(captions), 2, captions) + self.assertIn('analyst buy ratings', captions[0]) + self.assertIn('analyst sell ratings', captions[1]) + self.assertIn('analyst sell ratings', str(self.command('.sell & buy')[0])) # in the order asked + self.assertEqual(len(self.command('.buy & buy')), 1) # each report once + + def test_the_keyboard_is_alphabetical_and_every_button_is_a_command(self): + buttons = [button['text'] for row in charts.command_keyboard()['keyboard'] for button in row] + self.assertEqual(buttons, sorted(buttons)) + self.assertEqual(len(buttons), 15) + self.assertIn('.buy & sell', buttons) + self.assertIn('.forward pe', buttons) + for text in buttons: # not taken for a ticker by the catch-all .SYMBOL + with self.subTest(text=text): + self.assertIsNot(next(handler for pattern, handler in worker.COMMANDS if worker.match(pattern, text, BOT)), worker.on_profile) + def test_history_and_compare(self): for cmd in ('.history T1', '.compare T1 T2', '.compare T1 T2 3m'): with self.subTest(cmd=cmd): From abe262d3d423a736bfcb1461e42a6c871f9696af Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 15:54:47 +0000 Subject: [PATCH 24/25] .forward pe (and its keyboard button) sends a bar chart, as .peg does The forward P/E list already had a chart spec; the command now delivers it, with the list as its caption, where the service can take an image. The top, bottom and negative lists all chart; one ticker's ratios stay text. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- lib/worker.py | 2 +- tests/test_reports.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/worker.py b/lib/worker.py index 98953322..02943c2b 100755 --- a/lib/worker.py +++ b/lib/worker.py @@ -385,7 +385,7 @@ def on_profile(request, m): (r"(?:marketcap|maletas|marketer)\s*([\w\.\:\-]+)*", on_marketcap), (r"peg\s*([\w\.\:\-\s]+)*", on_ratio('peg', negative=True, chart=True)), (r"pe(?:$|\s+)([\w\.\:\-\s]+)*", on_ratio('pe')), - (r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*", on_ratio('forward pe', negative=True)), + (r"(?:fpe|forward\s?pe)\s*([\w\.\:\-\s]+)*", on_ratio('forward pe', negative=True, chart=True)), (r"beta\s*([\w\.\:\-]+)*", on_beta), (r"pie(?:$|\s+)(?P[\w\s\-\.&]*)", on_pie), (r"(?Pbuy|sell)(?:\s*(?:&|and|/|\+)?\s*(?Pbuy|sell))?", on_rating), # .buy & sell (or .buy sell): both diff --git a/tests/test_reports.py b/tests/test_reports.py index a667f4c4..a7dbbfe1 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -15,7 +15,8 @@ def test_text_commands(self): def test_chart_commands(self): for cmd in ('.price', '.price 7d', '.price top', '.price bottom', '.session', '.premarket', '.postmarket', - '.performance', '.performance 7d', '.performance 1y', '.shorts', '.peg', '.buy', '.sell', '.beta', '.price top 7d'): + '.performance', '.performance 7d', '.performance 1y', '.shorts', '.peg', '.buy', '.sell', '.beta', '.price top 7d', + '.forward pe', '.fpe bottom'): with self.subTest(cmd=cmd): out = self.command(cmd) self.assertTrue(out, f'{cmd} sent nothing') From 07b4f43037584d367b6683bf1fbfcaa10d761181 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 16:00:15 +0000 Subject: [PATCH 25/25] min_analysts: one preference for .buy, .sell and the forward P/E lists (default 7), documented - New preferences.conf key min_analysts (default 7, reloadable): .buy, .sell and the .forward pe lists (top, bottom, negative) take only stocks covered by at least that many analysts. A forward P/E is analysts' estimate, so with few of them it says as little as a perfect buy from two brokers. One ticker asked for (.fpe SYMBOL) is still shown. - The forward P/E headings say "(7+ analysts)", as .buy's does, and an empty list says why. - Documented in the README's preferences table (now 26 keys), the .buy/.sell and P/E sections, and the shipped defaults/preferences.conf. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JVEkZofdPjQ4jtXFKipeUX --- README.md | 9 +++++---- defaults/preferences.conf | 2 ++ lib/config.py | 3 ++- lib/reports.py | 25 ++++++++++++++++--------- tests/test_config.py | 11 ++++++----- tests/test_reports.py | 18 ++++++++++++++++++ 6 files changed, 49 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 3d7de1d0..978cbb2a 100644 --- a/README.md +++ b/README.md @@ -316,7 +316,7 @@ Interactive trigger: ### pe / forward pe / peg ![Forward PE report in Slack](img/finbot_fpe.png?raw=true "Forward PE report in Slack") -List securities by P/E ratio, forward P/E ratio, or PEG ratio +List securities by P/E ratio, forward P/E ratio, or PEG ratio. A forward P/E is an analysts' estimate, so its lists take only stocks covered by at least `min_analysts` analysts (7 by default; see `preferences.conf`). Interactive trigger: ``` @@ -347,7 +347,7 @@ Interactive trigger: ### Buy ![Buy ratings in Slack](img/finbot_buy.png?raw=true "Buy ratings in Slack") -Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by 7 or more analysts (with fewer, the consensus says little: thinly covered small caps are nearly all a perfect buy) +Show top recommended buys by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts (7 by default; see `preferences.conf`) Interactive trigger: ``` @@ -360,7 +360,7 @@ Interactive trigger: ### Sell ![Rating report in Slack](img/finbot_sell.png?raw=true "Rating report in Slack") -Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by 7 or more analysts +Show top recommended sells by Yahoo Finance analysts rating, among stocks rated by at least `min_analysts` analysts Interactive trigger: ``` @@ -480,7 +480,7 @@ Copies preferences.conf and metadata files into the data directory (never overwr util/sync_defaults.py [-d DEST] --apply ``` -All 25 keys, read by the bot itself. A change takes effect on a reload. +All 26 keys, read by the bot itself. A change takes effect on a reload. | Key | Default | Description | |---|---|---| @@ -488,6 +488,7 @@ All 25 keys, read by the bot itself. A change takes effect on a reload. | `future_days` | `30` | How many days ahead the events calendar looks | | `price_percent` | `5` | Price alert threshold, percent | | `shorts_percent` | `15` | Short interest alert threshold, percent | +| `min_analysts` | `7` | `.buy`, `.sell` and `.forward pe` list only stocks covered by at least this many analysts; with fewer, a consensus says little (thinly covered small caps are nearly all a perfect buy) | | `demote_volatile` | `True` | Volatile (small cap, high beta) stocks need a bigger move to alert | | `demote_leveraged` | `True` | Leveraged (2x, 3x) funds need a bigger move to alert | | `volatility_multiplier` | `2` | How much bigger a move volatile stocks need | diff --git a/defaults/preferences.conf b/defaults/preferences.conf index c954314c..8f0355f1 100644 --- a/defaults/preferences.conf +++ b/defaults/preferences.conf @@ -11,6 +11,8 @@ past_days=28 future_days=30 price_percent=5 shorts_percent=15 +# .buy, .sell and .forward pe list only stocks that at least this many analysts cover: with fewer, a consensus says little +min_analysts=7 # price alerts: volatile and leveraged stocks need a bigger move; the multiplier is how much bigger demote_volatile=True demote_leveraged=True diff --git a/lib/config.py b/lib/config.py index 7a4290da..45c2f8c1 100644 --- a/lib/config.py +++ b/lib/config.py @@ -20,7 +20,7 @@ 'slackOutgoingToken', 'slackOutgoingWebhook', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') PREFERENCE_KEYS = ('alliterate', 'cache', 'cache_seconds', 'chunk_maxlines', 'country_code', 'debug', 'demote_leveraged', 'demote_volatile', 'exclude_portfolios', 'future_days', 'graph', 'http_timeout', 'hyperlink', 'hyperlinkFooter', 'hyperlinkProvider', 'include_portfolios', - 'past_days', 'performance_cache_seconds', 'performance_use_sharesight', 'price_percent', 'print_headers', 'shorts_percent', 'timezone', + 'min_analysts', 'past_days', 'performance_cache_seconds', 'performance_use_sharesight', 'price_percent', 'print_headers', 'shorts_percent', 'timezone', 'trades_use_yahoo', 'volatility_multiplier') SECRET_KEYS = ('sharesight_code', 'sharesight_client_id', 'sharesight_client_secret', 'slack_webhook', 'discord_webhook', 'slackBotToken', 'slackOutgoingToken', 'slackOutgoingWebhook', 'telegramBotToken', 'telegramOutgoingToken', 'telegramOutgoingWebhook') # never printed @@ -198,6 +198,7 @@ def preference_values(getenv): 'config_hyperlinkFooter': r.bool('hyperlinkFooter', True), 'config_hyperlinkProvider': r.str('hyperlinkProvider', 'google').lower(), 'config_include_portfolios': r.str('include_portfolios', '').split(), + 'config_min_analysts': r.int('min_analysts', 7), # .buy, .sell and .forward pe list only stocks this many analysts cover 'config_past_days': r.int('past_days', 28), 'config_performance_cache_seconds': r.int('performance_cache_seconds', 1800), # Sharesight performance reports (30 minutes) 'config_performance_use_sharesight': r.bool('performance_use_sharesight', False), diff --git a/lib/reports.py b/lib/reports.py index 5d34915c..fb6fe5a7 100755 --- a/lib/reports.py +++ b/lib/reports.py @@ -378,14 +378,13 @@ def prepare_marketcap_payload(service, action='top', length=15): caps.sort(key=lambda cap: cap[0], reverse=action == 'top') return [webhook.bold(f'{action.title()} {length} tracked stocks by market cap (USD)', service)] + [line for _, line in caps[:length]] -MIN_ANALYSTS = 7 # fewer, and a consensus says little: ASX small caps covered by two to six brokers are nearly all a perfect 1.0 buy - def prepare_rating_payload(service, action, length=15): - """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by MIN_ANALYSTS or more analysts, the - strongest rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give first.""" + """(payload, chart) of the `length` tracked stocks rated `action` ('buy' or 'sell') by min_analysts or more analysts (fewer, + and the consensus says little: ASX small caps covered by two to six brokers are nearly all a perfect 1.0 buy), the strongest + rating first (1 is a strong buy, 5 a strong sell); on equal ratings, the one more analysts give first.""" found = [] # (rating, analysts, line, ticker) for ticker, data in yahoo.fetch_details(util.get_holdings_and_watchlist()).items(): - if 'recommend' not in data or data['recommend_analysts'] < MIN_ANALYSTS: + if 'recommend' not in data or data['recommend_analysts'] < config_min_analysts: continue recommend = data['recommend'].replace('_', ' ') if (action == 'buy' and 'buy' in recommend) or (action == 'sell' and recommend in ('sell', 'underperform')): @@ -395,8 +394,8 @@ def prepare_rating_payload(service, action, length=15): found = sorted(found, key=lambda row: (row[0] if action == 'buy' else -row[0], -row[1]))[:length] payload = [line for _, _, line, _ in found] if not payload: - return [f"No tracked stocks rated {action} by {MIN_ANALYSTS}+ analysts"], None - message = f"Top {length} analyst {action} ratings for tracked stocks ({MIN_ANALYSTS}+ analysts)" + return [f"No tracked stocks rated {action} by {config_min_analysts}+ analysts"], None + message = f"Top {length} analyst {action} ratings for tracked stocks ({config_min_analysts}+ analysts)" shown = [(ticker.split('.')[0], rating) for rating, _, _, ticker in found] low, high = min(r for _, r in shown), max(r for _, r in shown) spread = (high - low) or 0.2 @@ -406,13 +405,20 @@ def prepare_rating_payload(service, action, length=15): return [webhook.bold(message, service)] + payload, chart def prepare_value_payload(service, action='pe', ticker_select=None, length=15): + """(payload, chart): the tracked stocks by a ratio (`action`), or one ticker's. A forward P/E is analysts' estimate, so its lists + take only stocks that min_analysts or more analysts cover, as .buy does.""" payload = [] chart_rows = [] # (ticker, ratio) tickers = [ticker_select] if ticker_select else util.get_holdings_and_watchlist() market_data = yahoo.fetch(tickers) if action in ('peg', 'bottom peg', 'negative peg'): # the PEG is in Yahoo's details market_data |= yahoo.fetch_details(market_data, strict=bool(ticker_select)) + covered = None # for a forward P/E list: the tickers enough analysts cover + if 'forward' in action and not ticker_select: + covered = {ticker for ticker, detail in yahoo.fetch_details(market_data).items() if detail.get('recommend_analysts', 0) >= config_min_analysts} for ticker in market_data: + if covered is not None and ticker not in covered: + continue trailing_ratio = None forward_ratio = None peg_ratio = None @@ -472,10 +478,11 @@ def prepare_value_payload(service, action='pe', ticker_select=None, length=15): payload.reverse() chart_rows.reverse() payload = payload[:length] + coverage = f" ({config_min_analysts}+ analysts)" if covered is not None else '' if not payload: - payload = [f"{action}: no data found"] + payload = [f"No tracked stocks with {config_min_analysts}+ analysts have a {heading_trail} ratio" if covered is not None else f"{action}: no data found"] else: - heading = f'{heading_type} {length} tracked stocks by {heading_trail} ratio' + heading = f'{heading_type} {length} tracked stocks by {heading_trail} ratio{coverage}' payload.insert(0, f"{webhook.bold(heading, service)}") chart = {'rows': chart_rows[:length], 'title': heading, 'value_fmt': lambda v: f"{v:g}"} return payload, chart diff --git a/tests/test_config.py b/tests/test_config.py index 51ef7c7d..ba4ade60 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,7 @@ from lib import config from util import check_config, split_env import bot, cal -from lib import sharesight, util, worker, yahoo +from lib import reports, sharesight, util, worker, yahoo import performance, price, shorts, trades REPO = Path(config.__file__).resolve().parent.parent @@ -15,7 +15,7 @@ 'config_alliterate': False, 'config_hyperlink': True, 'config_hyperlinkFooter': True, 'config_hyperlinkProvider': 'google', 'config_demote_volatile': True, 'config_demote_leveraged': True, 'config_volatility_multiplier': 2.0, 'config_exclude_portfolios': [], 'config_include_portfolios': [], 'config_trades_use_yahoo': True, 'config_performance_use_sharesight': False, - 'config_print_headers': False, 'debug': False} + 'config_print_headers': False, 'config_min_analysts': 7, 'debug': False} def keys_in(path): @@ -26,7 +26,7 @@ def keys_in(path): class Tables(unittest.TestCase): def test_the_two_key_sets_are_disjoint_and_cover_everything_config_reads(self): self.assertEqual(set(config.STARTUP_KEYS) & set(config.PREFERENCE_KEYS), set()) - self.assertEqual((len(config.STARTUP_KEYS), len(config.PREFERENCE_KEYS)), (17, 25)) + self.assertEqual((len(config.STARTUP_KEYS), len(config.PREFERENCE_KEYS)), (17, 26)) source = (REPO / 'lib' / 'config.py').read_text() read = set(re.findall(r"""(?:\br\.\w+|getenv)\(\s*['"]([A-Za-z_]+)['"]""", source)) read -= {'STATE_DIRECTORY'} # systemd's own variable, a source for DATA_DIR @@ -149,9 +149,10 @@ def write(self, text): def test_a_change_reaches_config_and_every_module_that_copied_it(self): self.write('future_days=45\nprice_percent=7\nshorts_percent=12\nexclude_portfolios=1156263\nalliterate=1\nhyperlinkProvider=yahoo\n' - 'hyperlinkFooter=false\ndebug=1\ngraph=0\ndemote_volatile=0\nvolatility_multiplier=3\npast_days=14\n') + 'hyperlinkFooter=false\ndebug=1\ngraph=0\ndemote_volatile=0\nvolatility_multiplier=3\npast_days=14\nmin_analysts=3\n') changes = config.reload() - self.assertEqual(len(changes), 12) + self.assertEqual(len(changes), 13) + self.assertEqual(reports.config_min_analysts, 3) # .buy, .sell and .forward pe read it self.assertIn('future_days 30 -> 45', changes) want = {'config_future_days': 45, 'config_past_days': 14, 'config_price_percent': 7.0} for module in (config, cal, worker, price, trades, performance, shorts, sharesight, yahoo, util): diff --git a/tests/test_reports.py b/tests/test_reports.py index a7dbbfe1..df019833 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -94,6 +94,24 @@ def test_a_small_cap_quoted_in_won_needs_the_bigger_move_too(self): self.assertAlmostEqual(quote['market_cap_usd'], 1e11 / 1380.0) # US$72M self.assertEqual(price.trigger_bar('T1', quote, 5), 5 * price.config_volatility_multiplier) + def test_forward_pe_lists_need_as_many_analysts_as_buy(self): + """A forward P/E is analysts' estimate: with few of them it says little, as a perfect 1.0 buy from two brokers does.""" + analysts = {'T1': 7, 'T2': 6, 'T3': 12, 'T4': 2} + self.patch(util, 'get_holdings_and_watchlist', lambda: list(analysts)) + self.patch(yahoo, 'fetch', lambda ts: {t: market_entry(t) | {'price_to_earnings_forward': -5 if t == 'T3' else 20} for t in ts}) + self.patch(yahoo, 'fetch_detail', lambda t, *a, **k: {t: market_entry(t) | {'recommend_analysts': analysts[t]}}) + def listed(action): + payload, chart = reports.prepare_value_payload('telegram', action) + return payload[0], [line.split('Co ')[1].split(' ')[0] for line in payload[1:]], chart + heading, names, chart = listed('forward pe') + self.assertEqual((names, [label for label, _ in chart['rows']]), (['T1'], ['T1'])) # T2 and T4: too few; T3: negative + self.assertIn('forward pe ratio (7+ analysts)', heading) + self.assertEqual(listed('negative forward pe')[1], ['T3']) + self.assertIn('Co T2', str(reports.prepare_value_payload('telegram', 'forward pe', 'T2'))) # one ticker asked for: shown + self.patch(reports, 'config_min_analysts', 13) # the one setting moves .buy and .forward pe together + self.assertEqual(reports.prepare_value_payload('telegram', 'forward pe')[0], ['No tracked stocks with 13+ analysts have a forward pe ratio']) + self.assertEqual(reports.prepare_rating_payload('telegram', 'buy')[0], ['No tracked stocks rated buy by 13+ analysts']) + def test_market_caps_compare_in_us_dollars(self): """Yahoo gives a market cap in the stock's own currency: unconverted, a won market cap topped every list.""" stocks = {'K1.KS': ('KRW', 1.01e15), 'K2.KS': ('KRW', 5e11), 'U1': ('USD', 5.39e12), 'L1.L': ('GBp', 6.25e10), 'Z1': ('XYZ', 1e12)}