diff --git a/.gitignore b/.gitignore index 2b46247..dccd7a8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ mcp_ecowitt.json chart-diffs/ !tests/data/*.sqlite .claude/settings.local.json +matplotlib/ diff --git a/README.md b/README.md index f7c4609..d0a5ed2 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ A Telegram bot for a personal Ecowitt weather station and AirGradient air-quality sensor. Ask it about the weather or air quality (text or charts); it also sends silent alerts to every -chat it is in: rain starting/stopping, rain likely soon, gusts over 40 km/h, UV index of 10 or more, indoor/outdoor temperatures crossing (held back for the cooldown, then announced if it still stands), -and unhealthy air (with when it is safe again). Every alert carries Subscribe / Unsubscribe buttons that expand into the alert types (rain, rain predicted, gusts, UV, temperature crossing, particulates, pollen & asthma, forecast changes): Subscribe lists the types that are off, Unsubscribe the ones that are on; `/alerts` opens the same settings (in a private chat, a βš™οΈ Settings button there and under each alert has Enable and Disable notification sounds sub-menus that list the alert types; alerts are silent unless you do; and `/alerts on|off` still subscribes or unsubscribes every type); `/usage` is the Help & Alerts button. In a group anyone can open `/alerts`, but only admins can change anything (`lib/alerts/menu.py`). +chat it is in: rain starting/stopping, gusts over 40 km/h, UV index of 10 or more, indoor/outdoor temperatures crossing (held back for the cooldown, then announced if it still stands), +and unhealthy air (with when it is safe again). Every alert carries Subscribe / Unsubscribe buttons that expand into the alert types (rain, gusts, UV, temperature crossing, particulates, pollen & asthma, forecast changes): Subscribe lists the types that are off, Unsubscribe the ones that are on; `/alerts` opens the same settings (in a private chat, a βš™οΈ Settings button there and under each alert has Enable and Disable notification sounds sub-menus that list the alert types; alerts are silent unless you do; and `/alerts on|off` still subscribes or unsubscribes every type); `/usage` is the Help & Alerts button. In a group anyone can open `/alerts`, but only admins can change anything (`lib/alerts/menu.py`). ## Run @@ -24,6 +24,7 @@ Under systemd see `ecobot.service` (it loads the environment file; the clone's l | `TZ` | e.g. `Australia/Melbourne` | | `ECOWITT_API_KEY`, `ECOWITT_APP_KEY` | enables Ecowitt | | `AIRGRADIENT_API_TOKEN`, `AIRGRADIENT_LOCATION_ID` | enables AirGradient | +| `DATA_DIR` | optional: the folder that holds the bot's state (`bot_state.json`) and its SQLite caches (`ecowitt_cache.sqlite`, `airgradient_cache.sqlite`, `conditions_cache.sqlite`), kept apart from the code so a container can mount it (e.g. `/data`). A relative path is also accepted and is taken from the working directory; the folder is created if missing. Unset, a systemd service with `StateDirectory=` uses that folder (`STATE_DIRECTORY`, e.g. `/var/lib/ecobot`), else the app's own folder. Matplotlib's font cache goes in a `matplotlib` sub-folder (unless `MPLCONFIGDIR` is set). Default: the app's own folder, where the files have always been | | `AIRGRADIENT_DASHBOARD_URL` | optional "live chart" link under air-quality replies and alerts | | `POLLEN=on` | optional: Melbourne grass pollen and thunderstorm asthma risk (scraped from melbournepollen.com.au) in the report, a `pollen_asthma` tool and an alert when High. Off by default; `POLLEN_DISTRICT` picks the district (default `Central`, never printed) | | `PLACE` | the name shown after the Pollen & asthma and Forecast headings in the report (default `Melbourne`) | @@ -31,7 +32,7 @@ Under systemd see `ecobot.service` (it loads the environment file; the clone's l | `ADMIN_ONLY` | on by default: the bot answers nobody (private chats and groups alike) except the admins and creators of the groups it is in; anyone else gets "Sorry, only group admins can use this bot." when they address it (ordinary group talk is ignored). The one exception is `/alerts` in a group, which anyone may open (its buttons and `/alerts on|off` stay admin-only). `ADMIN_ONLY=off` answers everyone, as before. The admins are asked of Telegram and remembered for 10 minutes; a group counts once the bot is added to it or someone speaks there | | `ADMIN_CHAT_IDS` | optional, with `ADMIN_ONLY`: more ids allowed besides those admins (added to them, not instead of them), separated by commas or spaces. A user id (the same number as their private chat) lets someone in before the bot is in any group; a group id (negative) adds that group's admins and lets its anonymous admins (those who post "as the group") in | | `ALERT_COOLDOWN_MINUTES` | optional: the rain and temperature-crossing alerts: how long it must stay dry before "the rain has stopped" is sent, and the least time between alerts of the same kind (default 30, from 5 to 150). A change that comes inside the cooldown is held and announced when it ends, if the state still differs from the last one alerted; if it is back where it was, nothing is sent | -| `RAIN_QUIET_HOURS` | optional: local hours with no rain alerts (started, stopped, predicted), as `0-6` (the default; `22-6` wraps midnight; `off` for none). Rain that fell in them is summed up in one message once they end | +| `RAIN_QUIET_HOURS` | optional: local hours with no rain alerts (started, stopped), as `0-6` (the default; `22-6` wraps midnight; `off` for none). Rain that fell in them is summed up in one message once they end | | `CHART_ALL_FEELS_LIKE=on` | optional: "weather all week" (every reading, a panel each, sent with just its title; UV is left out as it has the same shape as solar) also draws the feels-like panel; off by default, as it nearly repeats temperature. Naming either ("plot feels like", "plot solar and uv") always works | | `TUYA_DEVICE_ID`, `TUYA_CLIENT_ID`, `TUYA_CLIENT_SECRET` | enables the irrigation controller (a Tuya smart irrigation timer, through the Tuya cloud API): an Irrigation button in private chats (it replaces Wind) with its status and buttons to water for 5 to 30 minutes or switch the valve off, pause for 24 to 72 hours (the Pause button becomes Unpause while paused), refresh, and an Alerts menu to turn the chat's irrigation battery and irrigation pause alerts on or off (or type irrigation, water, tap or sprinkler); a daily check at 18:00 that delays it 24 hours when more than 1 mm fell in the last 24 hours or Open-Meteo gives today or tomorrow a 50% or better chance of 1 mm (needs Ecowitt and `FORECAST=on` for each half); an "irrigation pause" alert when it does, and an "irrigation battery" alert below 5%. `TUYA_BASE_URL` (default `https://openapi.tuyaeu.com`, the EU data centre) and `IRRIGATION_CHECK_HOUR` (default `18`) are optional | | `CHART_ALL_VPD=on` | optional: "weather all week" also draws the vapour pressure deficit panel; off by default, as it is temperature and humidity combined (the station reports it for the outdoor sensor only). Naming it ("plot vpd") always works | @@ -56,14 +57,14 @@ lib/panels.py how a reading becomes a panel (title, unit, colour key, lib/charts.py the one renderer: panels on a shared time axis, rain behind the lines, one colour per reading lib/captions.py when a chart is drawn, and the hint that makes the model's reply its caption lib/ecowitt/ api, store (SQLite + memory), fetch, extremes, query (one history question), link (weather_link), - days (rank/count days), outlook (raining / likely soon), station, nightly archive + days (rank/count days), rain (rain times and amounts for the alerts), station, nightly archive lib/airgradient/ metrics, store (SQLite), source -lib/alerts/ notify (chats, silent send), weather, air, pollen, irrigation (the daily check) +lib/alerts/ notify (chats, silent send), weather, particulates, pollen_and_asthma, irrigation (battery and rain delay) lib/irrigation/ the optional Tuya irrigation controller: tuya (signed API client), menu (status text and buttons), watchdog (closes the valve after a run) lib/pollen/, forecast/ the optional website sources: pollen + thunderstorm asthma, Open-Meteo forecast (fetched only 6 am-6 pm, see warm.py SYNC_HOURS) tests/ pytest, against fake Ecowitt/AirGradient/Telegram; tests/evals holds the routing cases, tests/charts the chart reference images scripts/ ecowitt_metrics.py (which metrics the station reports), cache_status.py (is everything cached?), eval_prompts.py, show_request.py (exactly what is - sent to the model for a question), check_rain.py (are rain totals trustworthy?), + sent to the model for a question), check_rain.py (are rain totals trustworthy?), migrate_cache.py (one-off: converts an Ecowitt cache from before the series layout), dump_specs.py (draw the charts from the caches, offline, to compare before/after a change) ``` @@ -88,7 +89,7 @@ Both are off by default. They are websites, not APIs (the pollen page is parsed) data starts. Questions only go to the APIs for data the cache is missing. ## The report -"report" (and "status", "sitrep") and "weather now" are written in code, with no model call (`lib/report.py`): the bot fetches the tools' results together and lays them out, so they are quick and always the same. Sun, UV and wind are left out when zero (night, calm). The report is: Weather station, Air quality, and Pollen & asthma and Forecast when those sources are on (pollen and forecast always come from the cache). +"report" (and "status", "sitrep") and "weather now" are written in code, with no model call (`lib/report.py`): the bot fetches the tools' results together and lays them out, so they are quick and always the same. Sun, UV and wind are left out when zero (night, calm). The report is: Weather station, Air quality, and Pollen & asthma and Forecast when those sources are on (pollen and forecast always come from the cache). The report carries a πŸ”„ Refresh button that reads everything again and redraws it in the same message (`lib/report_menu.py`). A plain chart request ("rain chart 7d", "plot temperature and humidity", "air quality 30d", the chart buttons) is also drawn and written up in code, with no model call (the chart goes out with just its title as the caption; the write-up is kept in the chat's memory, not sent) (`intent.weather_chart`, `report.chart_caption`): the period, then low and high per reading (the average first if one was asked), the peak and rating for air quality, and for a rain chart the least and most rain and whether more is expected. Anything that asks for correlation, analysis, a forecast, thinking or a description, or compares readings or names a particular date or time, still goes to the model. diff --git a/RECODE.md b/RECODE.md index 744e8e6..e65371e 100644 --- a/RECODE.md +++ b/RECODE.md @@ -12,7 +12,7 @@ 267 lib/alerts/weather.py 258 lib/correlate.py 227 lib/ecowitt/store.py - 203 lib/ecowitt/station.py + 203 lib/ecowitt/source.py 200 lib/prompt.py 190 lib/ecowitt/days.py 160 ecobot.py @@ -24,7 +24,7 @@ 88 lib/airgradient/metrics.py 83 lib/ecowitt/direction.py 71 lib/airgradient/store.py - 59 lib/alerts/air.py + 59 lib/alerts/particulates.py 58 lib/config.py 54 lib/tools.py 51 lib/warm.py @@ -49,7 +49,7 @@ What did change: `analysis/pairs.py` and the rain helpers in `rain.py`; `correlate.py` is `analysis/scan.py`; - one weather-series table (`series.py`) feeds the stack chart, the composer and the question recogniser; - `intent.read` returns one `Reading` used by the bot, the evals and `show_request`; -- the station no longer imports the alert module (the rain outlook lives in `ecowitt/outlook.py`); +- the station no longer imports the alert module (the rain helpers live in `ecowitt/rain.py`); - the renderer draws bands and lines in one place. Left as they were, on purpose: typed chart specs (dict specs are read by many tests and the tools), the alert rules diff --git a/TELEGRAM_UX.md b/TELEGRAM_UX.md index 1b69217..94c23f0 100644 --- a/TELEGRAM_UX.md +++ b/TELEGRAM_UX.md @@ -33,7 +33,7 @@ A command or a button never does work of its own. It is turned into the plain se | `/reset` | as today | (unchanged) | | `/help` | the command list | (unchanged) | -Defaults: `/weather` and `/air` with no period are "now"; `/chart` with no reading is temperature; a chart with no period is 7 days. `/status` and `/report` use the sentence the bot already recognises (`intent.wants_report`), which fetches the weather and the air quality in one go and lists everything. +Defaults: `/weather` and `/air` with no period are "now"; `/chart` with no reading is temperature; a chart with no period is 7 days. `/status` and `/report` use the sentence the bot already recognises (`intent.wants_report`), which fetches the weather and the air quality in one go and lists everything. The report has a πŸ”„ Refresh button (`rp:refresh`) that fetches it again and edits the same message. **Implementation.** A pure function `command_to_text(name, args) -> str` holds the table above. One generic `CommandHandler` for the new names calls it and then `Bot.respond(update, context, text)`. In groups, Telegram sends `/weather@ecobot`; `python-telegram-bot` strips the suffix, and a command reaches a bot in a group even with privacy mode on, so no mention is needed. Replies stay silent. diff --git a/ecobot.py b/ecobot.py index 778b7b0..903eda0 100644 --- a/ecobot.py +++ b/ecobot.py @@ -6,20 +6,26 @@ import asyncio import contextlib import logging +import os import signal -import subprocess import time +from lib.config import ROOT, Config, data_dir + +# Matplotlib keeps its font cache here: a service user without a home directory would otherwise get a throw-away /tmp one on every start. +# Set before anything imports Matplotlib; an MPLCONFIGDIR in the environment wins. +os.environ.setdefault("MPLCONFIGDIR", str(data_dir() / "matplotlib")) + from openai import AsyncOpenAI from telegram import Update from telegram.ext import Application, Defaults from lib import intent from lib.airgradient import AirGradient -from lib.alerts import AIR_CHECK_SECONDS, AirMonitor, AlertState, ForecastMonitor, IrrigationMonitor, Notifier, PollenMonitor, WeatherMonitor -from lib.alerts.menu import available_kinds +from lib.alerts import (PARTICULATES_CHECK_SECONDS, AlertState, ForecastMonitor, IrrigationMonitor, Notifier, ParticulatesMonitor, + PollenAndAsthmaMonitor, WeatherMonitor) +from lib.alerts.menu import LABELS, kinds_for from lib.bot import Bot, polling_error -from lib.config import ROOT, Config from lib.ecowitt import Archive, Ecowitt from lib.forecast import Forecast from lib.irrigation import Tuya @@ -30,7 +36,18 @@ from lib.tools import Tools from lib.warm import every, safely -logging.basicConfig(format="%(levelname)s %(name)s: %(message)s", level=logging.INFO) + +class LogFormat(logging.Formatter): + """"INFO message" for this file's own lines, "INFO lib.bot: message" for the rest: the module name says who spoke.""" + + def format(self, record): + record.where = "" if record.name == "ecobot" else f"{record.name}: " + return super().format(record) + + +_handler = logging.StreamHandler() +_handler.setFormatter(LogFormat("%(levelname)s %(where)s%(message)s")) +logging.basicConfig(level=logging.INFO, handlers=[_handler]) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("telegram.ext.Application").setLevel(logging.WARNING) # "Application started" etc. log = logging.getLogger("ecobot") @@ -60,13 +77,42 @@ async def start(source): def version() -> str: """The checked-out commit, so the log shows which code is running.""" + return commit_in(ROOT / ".git") + + +def commit_in(git_dir) -> str: + """The short hash HEAD points at, read from the repository's files rather than by running git (which refuses a folder owned by + another user, as when the service runs as its own user). "unknown" when it cannot be read.""" try: - return subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True, - timeout=5, check=True).stdout.strip() or "unknown" + head = (git_dir / "HEAD").read_text().strip() + if head.startswith("ref:"): + ref = head.split(None, 1)[1] + try: + head = (git_dir / ref).read_text().strip() + except FileNotFoundError: # after `git pack-refs` or gc the branch is a line in packed-refs + head = next(line.split()[0] for line in (git_dir / "packed-refs").read_text().splitlines() + if line.endswith(f" {ref}")) + return head[:7] if len(head) >= 7 and all(c in "0123456789abcdef" for c in head.lower()) else "unknown" except Exception: return "unknown" +INDENT = 13 # the continuation lines of a wrapped log line + + +def wrapped(head: str, items: list[str], tail: str = "", width: int = 90) -> str: + """head + the items joined by ", " + tail, over several lines when it is longer than `width`: a line breaks only between items, + and the next one is indented.""" + lines, line = [], head + for i, item in enumerate(items): + piece = item + (", " if i < len(items) - 1 else tail) + if line.strip() != head.strip() and len(line + piece.rstrip()) > width: + lines.append(line.rstrip()) + line = " " * INDENT + line += piece + return "\n".join([*lines, line]) + + async def main(): log.info("Starting ecobot %s", version()) cfg = Config.from_env() @@ -85,8 +131,7 @@ async def main(): agent = Agent(AsyncOpenAI(api_key=cfg.xai_api_key, base_url=cfg.xai_base_url), cfg.xai_model, tools) state = AlertState(cfg.state_path) if cfg.admin_only: - log.info("Admin only: the bot answers the admins of the groups it is in%s", - f" and {len(cfg.admin_chat_ids)} listed id(s)" if cfg.admin_chat_ids else "") + log.info("ADMIN_ONLY=on the bot only answers admins of known groups; set ADMIN_CHAT_IDS for more") if not cfg.admin_chat_ids and not any(chat < 0 for chat in state.chats): log.warning("Admin only is on but no group is known yet and ADMIN_CHAT_IDS is empty, so nobody can use the bot: " "add it to a group, list your user id in ADMIN_CHAT_IDS, or set ADMIN_ONLY=off") @@ -114,30 +159,29 @@ async def main(): try: # any startup failure below still runs the shutdown steps (and shows the real error) log.info("@%s + %s (sources: %s)", app.bot.username, cfg.xai_model, ", ".join(s.name for s in sources)) - notify = Notifier(app.bot, state, available_kinds({s.name for s in sources} | ({"Irrigation"} if irrigation else set()))) + kinds = kinds_for((s.name for s in sources), irrigation is not None) + notify = Notifier(app.bot, state, kinds) await bot.refresh_keyboards(app.bot) # chats whose buttons are out of date are told, with the new ones # Alerts - kinds = [] + running: list[str] = [] # the alert types whose monitors are hooked up (the weather ones once they have started) if eco: monitor = WeatherMonitor(eco, state, notify, cfg.alert_cooldown_minutes * 60, cfg.rain_quiet_hours) - kinds += ["rain", "rain likely", "temp. crossing"] if air: - air_monitor = AirMonitor(air, state, notify) - tasks.append(asyncio.create_task(every(AIR_CHECK_SECONDS, air_monitor.check))) - kinds.append("air") + particulates_monitor = ParticulatesMonitor(air, state, notify) + tasks.append(asyncio.create_task(every(PARTICULATES_CHECK_SECONDS, particulates_monitor.check))) + running += ParticulatesMonitor.KINDS if forecast: forecast.warmer.after.append(ForecastMonitor(forecast, state, notify).check) # after each refresh (in the day) - kinds.append("forecast changes") + running += ForecastMonitor.KINDS if pollen: - pollen.warmer.after.append(PollenMonitor(pollen, state, notify).check) # after each refresh (in the day) - kinds += ["pollen", "asthma"] + pollen.warmer.after.append(PollenAndAsthmaMonitor(pollen, state, notify).check) # after each refresh (in the day) + running += PollenAndAsthmaMonitor.KINDS if irrigation: # the daily check (rain delay, battery) and the watchdog that closes the valve after a run tasks.append(asyncio.create_task(IrrigationMonitor(irrigation, state, notify, cfg.tz, cfg.irrigation_check_hour, - eco, forecast).run())) + eco, forecast).run())) tasks.append(asyncio.create_task(every(WATCHDOG_SECONDS, bot.watchdog.check))) - kinds.append("irrigation battery") - log.info("Alerts: %s: %d chats", ", ".join(kinds) or "none", len(state.alert_chats())) + running += IrrigationMonitor.KINDS # Keeping warm: everything questions need, refreshed before they arrive tasks.extend(asyncio.create_task(s.warmer.run()) for s in sources) @@ -155,6 +199,7 @@ async def ecowitt_part(): try: await monitor.init() eco.warmer.after.append(monitor.check) # checked on every fresh refresh + running[:0] = WeatherMonitor.KINDS await monitor.check() except Exception: log.exception("Weather alerts couldn't start") @@ -178,6 +223,10 @@ async def air_backfill(): log.exception("Startup warm-up failed (the regular refreshes will catch up)") log.info("Startup warm-up: %s, %.1fs", ", ".join(parts) or "nothing to fetch", time.monotonic() - started) + on = [k for k in LABELS if k in running] # the line says what started, not what is configured + log.info("%s", wrapped("Alerts: ", [LABELS[k] for k in on] or ["none"], f": {len(state.alert_chats())} chats")) + if missing := [LABELS[k] for k in kinds if k not in running]: + log.warning("Alerts NOT running: %s (see the error above)", ", ".join(missing)) tasks.append(asyncio.create_task(startup_warmup())) await stop.wait() diff --git a/lib/airgradient/source.py b/lib/airgradient/source.py index b90bd29..9aa1a51 100644 --- a/lib/airgradient/source.py +++ b/lib/airgradient/source.py @@ -27,7 +27,7 @@ from ..lines import Plotted, build_line from ..panels import panel_for from ..specs import Chart, Line -from ..timeutil import local_date, now_local, to_local +from ..timeutil import local_date, now_local, to_local, when from ..tools import Tool, Turn from ..warm import Warmer from .days import DESCRIPTION as DAYS_DESCRIPTION, PARAMETERS as DAYS_PARAMETERS, air_days_tool @@ -125,10 +125,6 @@ def _lock(self, key) -> asyncio.Lock: def _now(self) -> datetime: return now_local(self.tz) - def _when(self, ts: int) -> str: - dt = to_local(ts, self.tz) - return f"{dt:%a} {dt.day} {dt:%b %Y} {dt:%I:%M%p}".replace(" 0", " ").replace("AM", "am").replace("PM", "pm") - # ---------- cached data ---------- async def _current_row(self, refresh: bool = False) -> dict: async with self._lock("current"): @@ -278,7 +274,7 @@ async def current(self) -> dict: row = await self._current_row() previous = await self._previous_row(epoch(row["timestamp"])) if row.get("timestamp") else None out = {"sensor": row.get("locationName"), "sensor_type": row.get("locationType"), - "time": self._when(epoch(row["timestamp"])) if row.get("timestamp") else None} + "time": when(epoch(row["timestamp"]), self.tz) if row.get("timestamp") else None} if row.get("timestamp"): # for the air alerts' staleness check (removed before the model sees it) out["_time_utc"] = datetime.fromisoformat(row["timestamp"].replace("Z", "+00:00")) for name, (_, unit) in METRICS.items(): @@ -337,8 +333,8 @@ async def history(self, start: str | None, end: str | None, chart: bool = False, if not pts: continue lo, hi = min(pts, key=lambda p: p[1]), max(pts, key=lambda p: p[1]) - entry = {"unit": unit, "low": lo[1], "low_time": self._when(lo[0]), - "high": hi[1], "high_time": self._when(hi[0]), + entry = {"unit": unit, "low": lo[1], "low_time": when(lo[0], self.tz), + "high": hi[1], "high_time": when(hi[0], self.tz), "average": round(statistics.fmean(v for _, v in pts), 1)} entry["high_rating"], entry["average_rating"] = rating(name, hi[1]), rating(name, entry["average"]) if name == "pm2_5": diff --git a/lib/alerts/__init__.py b/lib/alerts/__init__.py index 1f19430..c8405d4 100644 --- a/lib/alerts/__init__.py +++ b/lib/alerts/__init__.py @@ -1,8 +1,8 @@ -from .air import CHECK_SECONDS as AIR_CHECK_SECONDS, AirMonitor +from .particulates import CHECK_SECONDS as PARTICULATES_CHECK_SECONDS, ParticulatesMonitor from .forecast import ForecastMonitor from .irrigation import IrrigationMonitor from .notify import AlertState, Notifier, with_footer -from .pollen import PollenMonitor +from .pollen_and_asthma import PollenAndAsthmaMonitor from .weather import WeatherMonitor -__all__ = ["AIR_CHECK_SECONDS", "AirMonitor", "AlertState", "ForecastMonitor", "IrrigationMonitor", "Notifier", "PollenMonitor", "WeatherMonitor", "with_footer"] +__all__ = ["PARTICULATES_CHECK_SECONDS", "ParticulatesMonitor", "AlertState", "ForecastMonitor", "IrrigationMonitor", "Notifier", "PollenAndAsthmaMonitor", "WeatherMonitor", "with_footer"] diff --git a/lib/alerts/forecast.py b/lib/alerts/forecast.py index dd8f27b..958f6a7 100644 --- a/lib/alerts/forecast.py +++ b/lib/alerts/forecast.py @@ -33,6 +33,8 @@ def revised(old: dict, new: dict) -> bool: class ForecastMonitor: + KINDS = ("forecast",) + def __init__(self, forecast, state, notify): """notify: the Notifier (its to_chat sends to one chat).""" self.forecast, self.state, self.notify = forecast, state, notify diff --git a/lib/alerts/irrigation.py b/lib/alerts/irrigation.py index bcddb6c..ac3237c 100644 --- a/lib/alerts/irrigation.py +++ b/lib/alerts/irrigation.py @@ -1,6 +1,6 @@ """The irrigation controller, checked once a day (18:00 local by default). - - a battery below 5% is told to the chats subscribed to "irrigation battery" (at most once a day, until it is charged); + - a battery below 5% is told to the chats subscribed to "irrigation battery" (kind irrigation_battery) (at most once a day, until it is charged); - if more than 1 mm of rain fell in the last 24 hours, or Open-Meteo gives today or tomorrow a 50% or better chance of at least 1 mm of rain, the controller is put on a 24 hour weather delay (it skips its own schedule) and the chats subscribed to "irrigation pause" are told why. A longer delay already set is left alone (nothing changes, so nothing is sent). @@ -13,7 +13,7 @@ import logging from datetime import datetime, timedelta -from ..ecowitt.outlook import rain_amount +from ..ecowitt.rain import rain_amount from ..timeutil import now_local from .forecast import RAIN_FROM_PCT @@ -26,13 +26,15 @@ class IrrigationMonitor: + KINDS = ("irrigation_battery", "irrigation_pause") + def __init__(self, device, state, notify, tz, hour: int = 18, station=None, forecast=None): """station: the Ecowitt source (its recent() gives the measured rain); forecast: the Forecast source. Either may be None.""" self.device, self.state, self.notify, self.tz, self.hour = device, state, notify, tz, hour self.station, self.forecast = station, forecast def _memory(self) -> dict: - return self.state.monitor.setdefault("irrigation", {}) + return self.state.monitor.setdefault("irrigation_monitor", {}) def due(self, now: datetime) -> bool: """Is today's check still to be done (it is past the check time and none is recorded for today)?""" @@ -71,7 +73,7 @@ async def check(self, now: datetime | None = None): memory["battery_warned"] = today self.state.save() await self.notify(f"πŸͺ« The irrigation battery is at {battery}%. Replace or recharge it soon, or watering will stop.", - kind="irrigation") + kind="irrigation_battery") reason = await self.rain_reason() if reason: current = status.get("weather_delay") diff --git a/lib/alerts/menu.py b/lib/alerts/menu.py index b5b7605..1825e14 100644 --- a/lib/alerts/menu.py +++ b/lib/alerts/menu.py @@ -17,12 +17,12 @@ from ..buttons import menu_text # kind -> the name on the button, in the order listed -LABELS = {"rain": "rain", "rain_likely": "rain predicted", "gusts": "gusts", "uv": "UV", "temps": "temperature crossing", - "air": "particulates", "pollen": "pollen & asthma", "forecast": "forecast changes", - "irrigation": "irrigation battery", "irrigation_pause": "irrigation pause"} +LABELS = {"rain": "rain", "gusts": "gusts", "uv": "UV", "temps": "temperature crossing", + "particulates": "particulates", "pollen_and_asthma": "pollen & asthma", "forecast": "forecast changes", + "irrigation_battery": "irrigation battery", "irrigation_pause": "irrigation pause"} # the emoji on each type's button: the one its own alerts start with (every type has one: a test holds the two lists together) -EMOJI = {"rain": "\U0001f327️", "rain_likely": "\U0001f326️", "gusts": "\U0001f4a8", "uv": "\U0001f9f4", "temps": "\U0001f321️", - "air": "\U0001f637", "pollen": "\U0001f33c", "forecast": "\U0001f504", "irrigation": "\U0001faab", "irrigation_pause": "β˜”"} +EMOJI = {"rain": "\U0001f327️", "gusts": "\U0001f4a8", "uv": "\U0001f9f4", "temps": "\U0001f321️", + "particulates": "\U0001f637", "pollen_and_asthma": "\U0001f33c", "forecast": "\U0001f504", "irrigation_battery": "\U0001faab", "irrigation_pause": "β˜”"} ALL = "all" TITLE = "πŸ”” Alerts in this chat" PER_ROW = 2 @@ -31,12 +31,17 @@ def available_kinds(sources: set[str]) -> list[str]: """The alert types the bot can send, from the names of its sources.""" - wanted = (["rain", "rain_likely", "gusts", "uv", "temps"] if "Ecowitt" in sources else []) + ( - ["air"] if "AirGradient" in sources else []) + (["pollen"] if "Pollen" in sources else []) + ( - ["forecast"] if "Forecast" in sources else []) + (["irrigation", "irrigation_pause"] if "Irrigation" in sources else []) + wanted = (["rain", "gusts", "uv", "temps"] if "Ecowitt" in sources else []) + ( + ["particulates"] if "AirGradient" in sources else []) + (["pollen_and_asthma"] if "Pollen" in sources else []) + ( + ["forecast"] if "Forecast" in sources else []) + (["irrigation_battery", "irrigation_pause"] if "Irrigation" in sources else []) return wanted +def kinds_for(source_names, irrigation: bool) -> list[str]: + """The alert types the bot sends, from the names of its sources and whether an irrigation controller is configured.""" + return available_kinds(set(source_names) | ({"Irrigation"} if irrigation else set())) + + def label(kind: str) -> str: """The name of an alert type on its button: its emoji, then its name.""" return f"{EMOJI[kind]} {LABELS[kind]}" diff --git a/lib/alerts/air.py b/lib/alerts/particulates.py similarity index 84% rename from lib/alerts/air.py rename to lib/alerts/particulates.py index 5025625..db021c8 100644 --- a/lib/alerts/air.py +++ b/lib/alerts/particulates.py @@ -1,4 +1,4 @@ -"""Air-quality alerts from the AirGradient sensor, checked every 30 minutes. +"""Particulates alerts (PM2.5 and PM10) from the AirGradient sensor, checked every 30 minutes. "Wear a mask outside" when PM2.5 or PM10 reaches US AQI 151+ ("unhealthy", the official level at which masks are advised for everyone) for two checks in a row; "back to safe" when both are at @@ -18,17 +18,19 @@ STALE_SECONDS = 2 * 3600 # ignore readings older than this (sensor offline) -class AirMonitor: +class ParticulatesMonitor: + KINDS = ("particulates",) + def __init__(self, air, state, notify): self.air, self.state, self.notify = air, state, notify async def check(self, reading: dict | None = None, now: datetime | None = None): reading = reading if reading is not None else await self.air.current() now = now or datetime.now(self.air.tz) - m = self.state.monitor.setdefault("air", {"unsafe": False, "above": 0, "below": 0}) + m = self.state.monitor.setdefault("particulates", {"unsafe": False, "above": 0, "below": 0}) when = reading.get("_time_utc") if when and (now - when).total_seconds() > STALE_SECONDS: - log.info("Air quality: latest reading is stale (%s); skipping", when) + log.info("Particulates: latest reading is stale (%s); skipping", when) return pm25 = (reading.get("pm2_5") or {}).get("value") pm10 = (reading.get("pm10") or {}).get("value") @@ -41,10 +43,10 @@ async def check(self, reading: dict | None = None, now: datetime | None = None): if not m["unsafe"] and m["above"] >= CONFIRM_CHECKS: m["unsafe"] = True await self.notify("\U0001f637 Unhealthy air outside β€” wear a P2/N95 mask today. " + self._levels(reading), - link=self.air.link, kind="air") + link=self.air.link, kind="particulates") elif m["unsafe"] and m["below"] >= CONFIRM_CHECKS: m["unsafe"] = False - await self.notify("βœ… Outdoor air is safe again. " + self._levels(reading), link=self.air.link, kind="air") + await self.notify("βœ… Outdoor air is safe again. " + self._levels(reading), link=self.air.link, kind="particulates") self.state.save() @staticmethod diff --git a/lib/alerts/pollen.py b/lib/alerts/pollen_and_asthma.py similarity index 92% rename from lib/alerts/pollen.py rename to lib/alerts/pollen_and_asthma.py index 5cef778..ab5f81a 100644 --- a/lib/alerts/pollen.py +++ b/lib/alerts/pollen_and_asthma.py @@ -20,7 +20,9 @@ LINK = ("melbournepollen.com.au", "https://www.melbournepollen.com.au/") -class PollenMonitor: +class PollenAndAsthmaMonitor: + KINDS = ("pollen_and_asthma",) + def __init__(self, pollen, state, notify): self.pollen, self.state, self.notify = pollen, state, notify @@ -30,7 +32,7 @@ async def check(self): cur = self.pollen.current() if not cur["fetched_at"] or time.time() - cur["fetched_at"] > STALE_SECONDS: return - m = self.state.monitor.setdefault("pollen", {}) + m = self.state.monitor.setdefault("pollen_and_asthma", {}) today = self.pollen.now().date() for key in ("grass", "asthma"): reading = cur[key] @@ -45,5 +47,5 @@ async def check(self): m[key] = {"rank": rank, "day": day} for_day = reading.get("date") when = f" (forecast for {for_day:%a %d %b})" if for_day and for_day != today else "" - await self.notify(f"{LEVEL_EMOJI[reading['level']]} {LABELS[key]} is {reading['level']}{when}.{ADVICE[key]}", link=LINK, kind="pollen") + await self.notify(f"{LEVEL_EMOJI[reading['level']]} {LABELS[key]} is {reading['level']}{when}.{ADVICE[key]}", link=LINK, kind="pollen_and_asthma") self.state.save() diff --git a/lib/alerts/weather.py b/lib/alerts/weather.py index f6895b6..dee3ee7 100644 --- a/lib/alerts/weather.py +++ b/lib/alerts/weather.py @@ -4,8 +4,6 @@ - Rain: "started" (one tip of the gauge is enough) and "stopped" (dry for ALERT_COOLDOWN_MINUTES), with how much fell. Not during quiet hours (RAIN_QUIET_HOURS, default midnight to 6am): those are kept quiet and one summary of the overnight rain is sent after they end. - - Rain likely soon: pressure falling over 3 hours plus arriving moisture, scored, tuned for - Melbourne (see assess_rain). At most once every 6 hours. - Strong gusts: one alert when a gust goes over 40 km/h, and no more until the gusts have stayed at or under it for an hour, so a blustery afternoon is one message, not twenty. - Strong sun: one alert when the UV index reaches 10, and no more until it has stayed below 10 for an hour. @@ -19,13 +17,12 @@ from ..ecowitt.glance import UVI_ALERT from .backoff import due, remember -from ..ecowitt.outlook import PREDICT_MIN_SCORE, Rows, assess_rain, duration, rain_amount, wet_flags +from ..ecowitt.rain import Rows, duration, rain_amount, wet_flags from ..timeutil import to_local log = logging.getLogger(__name__) ALERT_COOLDOWN_SECONDS = 30 * 60 # the default; ALERT_COOLDOWN_MINUTES sets it (the readings looked at go back 3 hours, so 150 minutes is the most) -PREDICT_EVERY_SECONDS = 6 * 3600 CROSS_MARGIN = 0.3 GUST_ALERT_KMH = 40 GUST_REARM_SECONDS = 3600 @@ -41,6 +38,8 @@ def side(r: dict) -> str | None: class WeatherMonitor: + KINDS = ("rain", "gusts", "uv", "temps") + def __init__(self, station, state, notify, cooldown_seconds: int = ALERT_COOLDOWN_SECONDS, quiet: tuple[int, int] | None = None): self.station, self.state, self.notify = station, state, notify self.cooldown = cooldown_seconds @@ -85,7 +84,6 @@ async def check(self): live = await self.station.live_rain() # newer than the 5-minute history by up to 5 minutes await self._rain(rows + [live] if live and live[0] > rows[-1][0] else rows) await self._night_summary(rows) - await self._rain_likely(rows) await self._gusts(rows) await self._uv(rows) await self._cross(rows) @@ -156,27 +154,6 @@ async def _night_summary(self, rows: Rows): still = " It's still raining." if self.state.monitor.get("rain", {}).get("alerted_state") == "raining" else "" await self.notify(f"\U0001f327️ Overnight rain: {amount}, from about {clock(night['first'])} to {clock(night['last'] + 300)}.{still}", kind="rain") - async def _rain_likely(self, rows: Rows): - """Not while raining or within an hour of rain; at most every 6 hours.""" - m = self.state.monitor.setdefault("predict", {"last": 0}) - latest_ts = rows[-1][0] - if self._quiet(latest_ts): - return - if self.state.monitor.get("rain", {}).get("alerted_state") == "raining" or any( - w for ts, w, _ in wet_flags(rows) if latest_ts - ts < 3600): - return - if latest_ts - m.get("last", 0) < PREDICT_EVERY_SECONDS: - return - outlook = assess_rain(rows, self.tz, self.station.longitude) - if outlook is None: - return - log.info("Rain-likely check: pressure fall %.1f hPa (%.1f beyond the tide), %s, score %d", - outlook.raw_drop, outlook.drop, "night" if outlook.night else "day", outlook.score) - if outlook.score >= PREDICT_MIN_SCORE: - m["last"] = latest_ts - await self.notify("\U0001f326️ Rain looks likely soon: " + ", ".join(outlook.reasons[:3]) + - ". (An estimate from the station's readings, not an official forecast.)", kind="rain_likely") - async def _gusts(self, rows: Rows): """Only readings since the last check count, so a restart doesn't re-announce an old gust.""" m = self.state.monitor.setdefault("gust", {"active": False, "calm_since": None}) diff --git a/lib/bot.py b/lib/bot.py index f3ad98d..259008a 100644 --- a/lib/bot.py +++ b/lib/bot.py @@ -20,7 +20,7 @@ from telegram.error import BadRequest, ChatMigrated, Forbidden, NetworkError, TelegramError from telegram.ext import CallbackQueryHandler, ChatMemberHandler, CommandHandler, ContextTypes, MessageHandler, filters -from . import intent, periods, prompt, report, templates +from . import intent, periods, prompt, report, report_menu, templates from .admins import Admins from .alerts import menu from .alerts import AlertState, with_footer @@ -183,11 +183,12 @@ async def run(self, stop: asyncio.Event): async def deliver(msg: Message, text: str, photos: list[bytes], link: tuple[str, str] | None = None, markup=None, - titles: list[str] | None = None, period_row=None, sent_photo: list | None = None) -> bool: + titles: list[str] | None = None, period_row=None, sent_photo: list | None = None, bold_headings: bool = False) -> bool: """Send the answer with any charts. A chart carries only its title as its caption; any answer text is its own message, sent first. link = (label, url) adds a small italic link line at the end of the text. markup (the button keyboard) rides on the last text, or on the single photo when there is no text; returns whether it was sent (a group of photos can't carry one). - period_row (the period buttons) rides on a single photo instead, and then the keyboard waits; the photo's Message goes into sent_photo.""" + period_row (the period buttons) rides on a single photo instead, and then the keyboard waits; the photo's Message goes into sent_photo. + bold_headings: the text is the report (or a pollen / forecast lookup), whose heading names are set in bold.""" text = text.strip() if photos: text = strip_chart_talk(text) @@ -198,6 +199,7 @@ async def deliver(msg: Message, text: str, photos: list[bytes], link: tuple[str, for i, chunk in enumerate(chunks): last = i == len(chunks) - 1 body, ents = with_footer(chunk, link) if last else (chunk, []) + ents = [*(report.heading_entities(chunk) if bold_headings else []), *ents] await msg.reply_text(body, entities=ents or None, disable_web_page_preview=True, **({"reply_markup": markup} if markup and last and not photos else {})) carried = bool(markup) and not photos @@ -298,7 +300,7 @@ async def _press(self, query) -> bool: def buttons(self) -> dict: """Every family of buttons the bot draws: the start of its data -> what handles it. The only way into any of them is on_button.""" return {"al:": self.on_alert_button, **({"ir:": self.on_irrigation_button} if self.irrigation else {}), - periods.PREFIX: self.on_period_button} + periods.PREFIX: self.on_period_button, report_menu.PREFIX: self.on_report_button} async def on_button(self, update: Update, context: ContextTypes.DEFAULT_TYPE): """The one place every button press comes in: believed (_press), a known family (a prefix nobody handles is refused, so a @@ -344,7 +346,7 @@ async def on_membership(self, update: Update, context: ContextTypes.DEFAULT_TYPE self.state.remove_chat(change.chat.id, f"bot {status}") def _alert_kinds(self) -> list[str]: - return menu.available_kinds(set(self.by_name) | ({"Irrigation"} if self.irrigation else set())) + return menu.kinds_for(self.by_name, bool(self.irrigation)) async def on_alerts(self, update: Update, context: ContextTypes.DEFAULT_TYPE): """/alerts: the alert settings, as buttons (Subscribe | Unsubscribe). /alerts on and /alerts off still subscribe or @@ -443,8 +445,8 @@ async def on_alert_button(self, update: Update, context: ContextTypes.DEFAULT_TY raise # ---------- irrigation ---------- - def _irrigation_alerts(self, chat_id: int, kind: str = "irrigation") -> bool: - """Does this chat get the irrigation battery alerts (kind "irrigation") or the pause alerts ("irrigation_pause")?""" + def _irrigation_alerts(self, chat_id: int, kind: str = "irrigation_battery") -> bool: + """Does this chat get the irrigation battery alerts (kind "irrigation_battery") or the pause alerts ("irrigation_pause")?""" return bool(self.state and self.state.subscribed(chat_id, kind)) def _irrigation_keyboard(self, chat_id: int, opened: str | None = None, paused: bool = False, running: bool = False): @@ -479,7 +481,7 @@ async def _irrigation_act(self, action: str, arg: str, chat_id: int) -> str | No self.watchdog.disarm() return "Switched off" if action in ("batt", "pause") and arg in ("on", "off") and self.state and chat_id in self.state.chats: - self.state.set_kind(chat_id, "irrigation" if action == "batt" else "irrigation_pause", arg == "on") + self.state.set_kind(chat_id, "irrigation_battery" if action == "batt" else "irrigation_pause", arg == "on") return f"Irrigation {'battery' if action == 'batt' else 'pause'} alerts {arg} in this chat" return None @@ -554,6 +556,24 @@ async def _edit_irrigation(query, text: str | None, markup, entities=None): if "not modified" not in str(e).lower(): raise + async def on_report_button(self, update: Update, context: ContextTypes.DEFAULT_TYPE): + """A press on Refresh under the report: read everything again and redraw the report in the same message. The chat's + conversation is not touched (this is not a new question).""" + query = update.callback_query + if query.data != report_menu.REFRESH: + await query.answer(STALE_BUTTON) + return + read = intent.read("Report", now_local(self.cfg.tz), "Ecowitt" in self.by_name, "AirGradient" in self.by_name, + "Pollen" in self.by_name, "Forecast" in self.by_name) + try: + text = await asyncio.wait_for(self._written_in_code(read, Turn(text="Report")), TURN_SECONDS) + except Exception as e: # the message stays as it was + log.warning("Report refresh failed: %s", e) + await query.answer("Couldn't refresh, please try again", show_alert=True) + return + await query.answer() # no toast: the times in the headings show what changed + await self._edit_irrigation(query, text, report_menu.keyboard(), report.heading_entities(text)) + async def on_period_button(self, update: Update, context: ContextTypes.DEFAULT_TYPE): """A press on 1D / 7D / 1M / 3M / 1Y under a chart: ask the chart's question again for that period and put the new chart in the same message.""" @@ -751,8 +771,7 @@ async def _chart_in_code(self, read, turn: Turn) -> str | None: question) when there is nothing to caption.""" calls = [read.fast[:2], *read.more] results = await asyncio.gather(*(self.agent.tools.call(name, json.dumps(args), turn) for name, args in calls)) - caption = report.chart_caption(read.fast[0], read.fast[1], results[0], turn, self.cfg.tz, - results[1] if len(results) > 1 else None) + caption = report.chart_caption(read.fast[0], read.fast[1], results[0], turn, self.cfg.tz) if caption is None: turn.charts.clear() turn.forecast_shown.clear() @@ -870,15 +889,18 @@ async def respond(self, update: Update, context: ContextTypes.DEFAULT_TYPE, text for m in working[new_from:] for tc in m.get("tool_calls") or []) air = self.by_name.get("AirGradient") markup = templates.keyboard(self.irrigation is not None) if self._keyboard_stale(msg) else None - row = periods.keyboard(intent.period_days(text, now)) if len(photos) == 1 else None # period buttons under a single chart + refresh = ok and read.report and not photos and 0 < len(reply) <= TG_LIMIT and not markup # the report, in one message, to be refreshed in place + row =periods.keyboard(intent.period_days(text, now)) if len(photos) == 1 else None # period buttons under a single chart sent_photo = [] if row and ok and not reply.strip(): # a chart and nothing else: keep it for the period buttons self.images.put(self.images.key(text, intent.period_days(text, now), now), photos[0], titles[0]) try: if redraw is not None and row and await self._redraw(redraw, reply, photos[0], titles[0], row, text): reply, photos, row = "", [], None - carried_keyboard = await deliver(msg, reply, photos, link=air.link if air and used_air else None, markup=markup, - titles=titles, period_row=row, sent_photo=sent_photo) + carried_keyboard = await deliver(msg, reply, photos, link=air.link if air and used_air else None, + markup=markup or (report_menu.keyboard() if refresh else None), + titles=titles, period_row=row, sent_photo=sent_photo, + bold_headings=ok and bool(read.report or read.lookup in ("pollen", "forecast"))) and bool(markup) # the refresh button is not the chat's keyboard if sent_photo: self.charted.remember(msg.chat_id, sent_photo[0].message_id, text) if self.state and ok and turn.forecast_shown: # it was sent: a later revision of these days is worth telling this chat diff --git a/lib/config.py b/lib/config.py index 1422751..a8784ec 100644 --- a/lib/config.py +++ b/lib/config.py @@ -12,6 +12,15 @@ log = logging.getLogger(__name__) +def data_dir() -> Path: + """Where the state file and the SQLite caches live: DATA_DIR (a relative path is taken from the working directory), else the + folder systemd gives a service that has StateDirectory= (STATE_DIRECTORY), else the app's own folder, where they have always been. + Kept apart from the code so a container can mount it. Symlinks are left as written: /var/lib/ecobot, not the closed + /var/lib/private/ecobot behind it.""" + value = os.getenv("DATA_DIR", "").strip() or os.getenv("STATE_DIRECTORY", "").split(":")[0].strip() + return Path(os.path.abspath(Path(value).expanduser())) if value else ROOT + + def _tz() -> tzinfo: name = os.getenv("TZ", "").lstrip(":") return ZoneInfo(name) if name else datetime.now().astimezone().tzinfo @@ -112,6 +121,15 @@ def airgradient(self) -> bool: @classmethod def from_env(cls) -> "Config": env = lambda k, default="": os.getenv(k, default).strip() + data = data_dir() + try: + data.mkdir(parents=True, exist_ok=True) + writable = os.access(data, os.W_OK | os.X_OK) + except OSError: + writable = False + if not writable: + raise SystemExit(f"{data} is not writable by this user: set DATA_DIR to a writable folder " + "(under systemd with StateDirectory=ecobot that is /var/lib/ecobot)") cfg = cls( telegram_token=os.environ["TELEGRAM_BOT_TOKEN"], xai_api_key=os.environ["XAI_API_KEY"], @@ -140,6 +158,10 @@ def from_env(cls) -> "Config": admin_only=_flag(env("ADMIN_ONLY"), True), admin_chat_ids=_ids(env("ADMIN_CHAT_IDS")), alert_cooldown_minutes=int(min(150, max(5, _number(env("ALERT_COOLDOWN_MINUTES")) or 30))), + state_path=data / "bot_state.json", + cache_path=data / "ecowitt_cache.sqlite", + air_cache_path=data / "airgradient_cache.sqlite", + conditions_cache_path=data / "conditions_cache.sqlite", ) if not (cfg.ecowitt or cfg.airgradient): raise SystemExit("Set ECOWITT_API_KEY + ECOWITT_APP_KEY and/or AIRGRADIENT_API_TOKEN + AIRGRADIENT_LOCATION_ID") diff --git a/lib/ecowitt/__init__.py b/lib/ecowitt/__init__.py index 8c242fc..f225d3e 100644 --- a/lib/ecowitt/__init__.py +++ b/lib/ecowitt/__init__.py @@ -1,4 +1,4 @@ from .archive import Archive -from .station import Ecowitt +from .source import Ecowitt __all__ = ["Archive", "Ecowitt"] diff --git a/lib/ecowitt/archive.py b/lib/ecowitt/archive.py index 22d1aa2..8602968 100644 --- a/lib/ecowitt/archive.py +++ b/lib/ecowitt/archive.py @@ -75,8 +75,9 @@ async def run_once(self) -> tuple[int, int]: if todo: # each range takes the pause after it plus about a second for the request itself seconds = len(todo) * (max(self.pace, MIN_GAP_SECONDS) + 1) - log.info("Ecowitt archive: %d req to fetch (about %s)", len(todo), - f"{seconds:.0f} s" if seconds < 90 else f"{seconds / 60:.0f} min") + if seconds >= 60: # a short run only gets the line when it finishes + log.info("Ecowitt archive: %d req to fetch (about %s)", len(todo), + f"{seconds:.0f} s" if seconds < 90 else f"{seconds / 60:.0f} min") fetched = failed = 0 for i, (cycle, start, end) in enumerate(todo, 1): fetcher = self.station.fetcher(self.groups) diff --git a/lib/ecowitt/outlook.py b/lib/ecowitt/outlook.py deleted file mode 100644 index 304243a..0000000 --- a/lib/ecowitt/outlook.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Is it raining, or likely to soon? Read from the station's recent 5-minute readings, by one set of rules for the alerts -and the status report.""" - -import math -from datetime import datetime, UTC -from statistics import fmean -from typing import NamedTuple - -PREDICT_MIN_SCORE = 4 -# Air pressure has a twice-daily "tide" (highs ~10am/10pm, lows ~4am/4pm local solar time). -# Around 37 degrees south its swing is ~0.7 hPa either side; it's subtracted before judging a fall. -TIDE_AMPLITUDE_HPA = 0.7 -NIGHT_HOURS = (20, 8) # 8pm-8am: cooling alone brings the air close to its dew point - -type Rows = list[tuple[int, dict]] # [(epoch, {"group.field": value})], oldest first - - -def tide(ts: int, longitude: float) -> float: - """Expected pressure offset from the atmospheric tide at this moment (hPa).""" - utc = datetime.fromtimestamp(ts, UTC) - solar_hour = (utc.hour + utc.minute / 60 + longitude / 15) % 24 - return TIDE_AMPLITUDE_HPA * math.cos(2 * math.pi * (solar_hour - 10) / 12) - - -def wet_flags(rows: Rows) -> list[tuple[int, bool, float]]: - """[(ts, rained in this reading, rain rate)]: rate above zero, or the daily total rising.""" - out, prev_daily = [], None - for ts, r in rows: - rate = r.get("rainfall.rain_rate", 0.0) - daily = r.get("rainfall.daily") - rising = daily is not None and prev_daily is not None and daily > prev_daily # ignores the midnight reset - if daily is not None: - prev_daily = daily - out.append((ts, rate > 0 or rising, rate)) - return out - - -def rain_amount(rows: Rows, since: int, last_wet: int) -> float | None: - """Rain that fell between since and last_wet: the rise in the daily total (allowing for its - midnight reset). Ecowitt's 'event' counter isn't used, as it spans several showers.""" - total, prev = 0.0, None - for ts, r in rows: - daily = r.get("rainfall.daily") - if daily is None or ts > last_wet: - continue - if ts >= since and prev is not None: - total += daily - prev if daily >= prev else daily # a drop means the midnight reset - prev = daily - return round(total, 1) if prev is not None else None - - -def duration(seconds: float) -> str: - minutes = int(round(seconds / 60)) - if minutes < 60: - return f"{minutes} min" - hours, minutes = divmod(minutes, 60) - return f"{hours}h {minutes:02d}m" if minutes else f"{hours}h" - - -class Outlook(NamedTuple): - score: int - reasons: list[str] - raw_drop: float # hPa fallen in 3 hours - drop: float # the same with the tide's expected change removed - night: bool - - -def assess_rain(rows: Rows, tz, longitude: float) -> Outlook | None: - """Score how likely rain is soon; None when there is no real pressure fall to go on. - - Two normal night-time effects are ignored, as they aren't signs of rain: the pressure tide's - overnight dip (subtracted from the fall), and the air cooling to near its dew point (at night - only a *rising* dew point counts, i.e. moister air moving in).""" - latest_ts, latest = rows[-1] - within = lambda secs: next((r for ts, r in rows if latest_ts - ts <= secs), None) - old_ts, old = next(((ts, r) for ts, r in rows if latest_ts - ts <= 3 * 3600 - and r.get("pressure.relative") is not None), (None, None)) - p_now = latest.get("pressure.relative") - if p_now is None or old is None: - return None - raw_drop = old["pressure.relative"] - p_now - drop = raw_drop + (tide(latest_ts, longitude) - tide(old_ts, longitude)) - if drop < 1.0: # a real (beyond-tide) pressure fall is required - return None - score = 3 if drop >= 3 else 2 if drop >= 2 else 1 - reasons = [f"pressure down {raw_drop:.1f} hPa in 3 hours"] - hour = datetime.fromtimestamp(latest_ts, UTC).astimezone(tz).hour - night = hour >= NIGHT_HOURS[0] or hour < NIGHT_HOURS[1] - temp, dew = latest.get("outdoor.temperature"), latest.get("outdoor.dew_point") - if night: - dew_old = (within(2 * 3600) or {}).get("outdoor.dew_point") - if dew is not None and dew_old is not None and dew - dew_old >= 1: - score += 2 if dew - dew_old >= 2 else 1 - reasons.append(f"dew point up {dew - dew_old:.1f}Β°C in 2 hours") - else: - hum = latest.get("outdoor.humidity") - if hum is not None: - hum_old = (within(3600) or {}).get("outdoor.humidity") - rising = hum_old is not None and hum - hum_old >= 5 - if hum >= 90 or rising: - score += (2 if hum >= 95 else 1 if hum >= 90 else 0) + (1 if rising else 0) - reasons.append(f"humidity {hum:.0f}%" + (" and rising" if rising else "")) - if temp is not None and dew is not None and temp - dew <= 2: - score += 2 if temp - dew <= 1 else 1 - reasons.append(f"only {temp - dew:.1f}Β°C above the dew point") - gusts = [(ts, r["wind.wind_gust"]) for ts, r in rows if r.get("wind.wind_gust") is not None] - now_gusts = [g for ts, g in gusts if latest_ts - ts <= 1800] - earlier = [g for ts, g in gusts if latest_ts - ts > 1800] - if now_gusts and earlier and max(now_gusts) - fmean(earlier) >= 10: - score += 1 - reasons.append(f"gusts picking up ({max(now_gusts):.0f} km/h)") - return Outlook(score, reasons, raw_drop, drop, night) - - -def rain_outlook(rows: Rows, tz, longitude: float) -> str | None: - """"Raining now" or "rain likely soon", in words for a status report, by the same rules as the alerts; None when neither.""" - wet = wet_flags(rows) - if any(w for _, w, _ in wet[-2:]): - rate = max(r for _, w, r in wet[-2:] if w) - return "raining now" + (f" ({rate:g} mm/h)" if rate > 0 else "") - outlook = assess_rain(rows, tz, longitude) - if outlook and outlook.score >= PREDICT_MIN_SCORE: - return ("rain looks likely soon: " + ", ".join(outlook.reasons[:3]) + - " (an estimate from the station's readings, not an official forecast)") - return None diff --git a/lib/ecowitt/query.py b/lib/ecowitt/query.py index 0cbef41..b072459 100644 --- a/lib/ecowitt/query.py +++ b/lib/ecowitt/query.py @@ -99,6 +99,7 @@ async def run(self) -> str: self.rain_bars = await self._rain_bars() await self._summarise_direction() if not self.store and not self.direction: + await self._log_empty() return "Error: no history data returned." + ( " Details: " + "; ".join(self.f.errors[:5]) if self.f.errors else "") self.overall = {k: ex for k, s in self.store.items() if len(ex := series_extremes(s)) == 2} @@ -106,6 +107,19 @@ async def run(self) -> str: refined = await self._refine() return self._answer(refined) + async def _log_empty(self): + """Say why a query found nothing: what was asked, where it was answered from, and what the cache holds for the window.""" + f = self.f + lo, hi = f.epoch(self.start), f.epoch(self.end) + try: + held = {c: sum([await asyncio.to_thread(f.cache.count, f.mac, c, g, lo, hi) for g in f.groups]) for c in ("5min", "30min", "1day")} + except Exception as e: + held = f"unreadable ({e})" + log.warning("History query empty: %s to %s (tz %s, epoch %d-%d), groups %s, %d ranges, %d from cache, %d from memory, %d requests, " + "%d failed%s, cache %s, readings held for the window %s", f"{self.start:%Y-%m-%d %H:%M}", f"{self.end:%Y-%m-%d %H:%M}", + getattr(self.tz, "key", self.tz), lo, hi, ",".join(f.groups), f.ranges, f.from_cache, f.from_memory, f.calls, + len(f.errors), f" ({f.errors[0]})" if f.errors else "", getattr(f.cache, "path", "?"), held) + async def _fine_ranges(self): """30-minute readings with no low and high of their own get them from the cached 5-minute readings (store.slot_ranges), so their bands are real ranges. Wind keeps its own band (the speed up to the gusts).""" diff --git a/lib/ecowitt/rain.py b/lib/ecowitt/rain.py new file mode 100644 index 0000000..383ea71 --- /dev/null +++ b/lib/ecowitt/rain.py @@ -0,0 +1,38 @@ +"""Rain from the station's recent 5-minute readings, for the rain alerts: when it rained, how much fell, and how long it took.""" + +type Rows = list[tuple[int, dict]] # [(epoch, {"group.field": value})], oldest first + + +def wet_flags(rows: Rows) -> list[tuple[int, bool, float]]: + """[(ts, rained in this reading, rain rate)]: rate above zero, or the daily total rising.""" + out, prev_daily = [], None + for ts, r in rows: + rate = r.get("rainfall.rain_rate", 0.0) + daily = r.get("rainfall.daily") + rising = daily is not None and prev_daily is not None and daily > prev_daily # ignores the midnight reset + if daily is not None: + prev_daily = daily + out.append((ts, rate > 0 or rising, rate)) + return out + + +def rain_amount(rows: Rows, since: int, last_wet: int) -> float | None: + """Rain that fell between since and last_wet: the rise in the daily total (allowing for its + midnight reset). Ecowitt's 'event' counter isn't used, as it spans several showers.""" + total, prev = 0.0, None + for ts, r in rows: + daily = r.get("rainfall.daily") + if daily is None or ts > last_wet: + continue + if ts >= since and prev is not None: + total += daily - prev if daily >= prev else daily # a drop means the midnight reset + prev = daily + return round(total, 1) if prev is not None else None + + +def duration(seconds: float) -> str: + minutes = int(round(seconds / 60)) + if minutes < 60: + return f"{minutes} min" + hours, minutes = divmod(minutes, 60) + return f"{hours}h {minutes:02d}m" if minutes else f"{hours}h" diff --git a/lib/ecowitt/station.py b/lib/ecowitt/source.py similarity index 90% rename from lib/ecowitt/station.py rename to lib/ecowitt/source.py index e8e1809..16dd52e 100644 --- a/lib/ecowitt/station.py +++ b/lib/ecowitt/source.py @@ -5,13 +5,13 @@ import asyncio import json import logging +import os import time from datetime import datetime, timedelta, UTC from ..config import Config from ..intent import NOW_GROUPS from ..timeutil import now_local -from .outlook import rain_outlook from ..tools import Tool, Turn from ..warm import FAST_REFRESH_SECONDS, Warmer from .api import EcowittAPI, GROUPS, UNITS @@ -84,8 +84,7 @@ def readings(data: dict) -> list[tuple[int, dict]]: "with times, for any range up to 4 years. Resolution and units are handled automatically.") REALTIME_PARAMS = { "type": "object", - "properties": {"groups": {"type": "string", "description": "Comma-separated group names, e.g. 'outdoor,indoor'. " - "For rain outlooks: 'outdoor,pressure,rainfall,rainfall_piezo,wind'."}}, + "properties": {"groups": {"type": "string", "description": "Comma-separated group names, e.g. 'outdoor,indoor'."}}, } REALTIME_DESCRIPTION = "Current readings from the owner's Ecowitt weather station." @@ -94,6 +93,22 @@ def readings(data: dict) -> list[tuple[int, dict]]: LIVE_FRESH_SECONDS = 90 # a question is answered from that reading while it is this young +def live_summary(data: dict) -> str: + """A live reading in a few words for the log: rain first, then outdoor temperature and humidity, wind and pressure. A reading the + response lacks is left out.""" + def get(group: str, field: str) -> str | None: + obj = (data.get(group) or {}).get(field) + if not isinstance(obj, dict) or "value" not in obj: + return None + return f"{obj['value']} {obj.get('unit', '')}".replace("℃", "Β°C").replace("mm/hr", "mm/h").strip() + wind = get("wind", "wind_speed") + parts = [f"rain {rate}" if (rate := get("rainfall", "rain_rate")) else None, + f"{daily} today" if (daily := get("rainfall", "daily")) else None, + get("outdoor", "temperature"), get("outdoor", "humidity"), + f"wind {wind}" if wind else None, get("pressure", "relative")] + return ", ".join(p for p in parts if p) or "no readings" + + class Ecowitt: name = "Ecowitt" @@ -139,6 +154,11 @@ async def start(self): log.warning("Public holiday questions won't work: pip install holidays (then restart)") log.info("Weather station: Ecowitt '%s' (%s)%s", self.station_name, self.mac, f", devices found: {len(devices)}" if len(devices) > 1 else "") + try: + size = os.path.getsize(self.cache.path) if os.path.exists(self.cache.path) else 0 + log.info("History cache: %s (%.1f MB)", self.cache.path, size / 1e6) + except Exception as e: # only a log line + log.warning("History cache not readable: %s", e) def describe(self) -> str: created = f", created {self.created:%Y-%m-%d %H:%M}" if self.created else "" @@ -184,8 +204,7 @@ async def _realtime(self, args: dict, turn: Turn | None = None) -> str: except (TypeError, ValueError): pass when = datetime.fromtimestamp(newest, UTC).astimezone(self.tz).strftime("%a %d %b %Y %H:%M") if newest else None - outlook = await self._rain_outlook() if "rainfall" in groups else None - return json.dumps({"time": when, **out, **({"rain_outlook": outlook} if outlook else {}), **({"emoji": emoji} if emoji else {})}, + return json.dumps({"time": when, **out, **({"emoji": emoji} if emoji else {})}, ensure_ascii=False, separators=(",", ":")) async def _live_data(self, groups: list[str]) -> dict: @@ -209,24 +228,16 @@ async def live_rain(self) -> tuple[int, dict] | None: raise ValueError("no rain reading in the response") except Exception as e: # the alert still works from the history if self._live_ok is not False: # once per outage, not every minute - log.warning("Live rain reading failed (%s); the rain alert uses the 5-minute history", e) + log.warning("Live readings failed (%s); the rain alert uses the 5-minute history", e) self._live_ok = False return None if self._live_ok is None: - log.info("Live rain reading working (%.1f mm/h, %.1f mm today)", row.get("rainfall.rain_rate", 0), row.get("rainfall.daily", 0)) + log.info("Working (%s)", live_summary(data)) elif not self._live_ok: - log.info("Live rain reading recovered") + log.info("Recovered (%s)", live_summary(data)) self._live_ok = True return ts, row - async def _rain_outlook(self) -> str | None: - """Raining now, or likely soon (the same rules as the alerts), from the last 3 hours of readings.""" - try: - return rain_outlook(await self.recent(3), self.tz, self.longitude) - except Exception as e: # the current reading is still worth sending without it - log.warning("Rain outlook unavailable: %s", e) - return None - # ---------- keeping warm ---------- async def warm(self, fresh: bool = True) -> str: """Every resolution's newest readings for every group: the last 7 days at 30 minutes, yesterday and today at diff --git a/lib/ecowitt/store.py b/lib/ecowitt/store.py index 38b9637..80262d6 100644 --- a/lib/ecowitt/store.py +++ b/lib/ecowitt/store.py @@ -1,8 +1,8 @@ """Where Ecowitt readings are kept between questions. -HistoryCache (SQLite, survives restarts) holds settled readings per -(mac, cycle, group, field, timestamp), plus which time ranges have been fetched per -(mac, cycle, group). Coverage is recorded even when Ecowitt returns nothing (e.g. before +HistoryCache (SQLite, survives restarts) holds settled readings per series and timestamp (a series is one +(mac, cycle, group, field), kept once in the `series` table with its unit and a small integer id, so a reading's key is just that id and +its time), plus which time ranges have been fetched per (mac, cycle, group). Coverage is recorded even when Ecowitt returns nothing (e.g. before the station existed), so those ranges aren't asked for again. The newest readings may still change (late uploads), so only buckets older than a per-cycle horizon are stored. @@ -75,16 +75,17 @@ def merge(intervals: list[Interval]) -> list[Interval]: class HistoryCache: def __init__(self, path, units: dict): self._lock = threading.Lock() + self.path = path self.db = sqlite3.connect(path, check_same_thread=False) self.db.execute("PRAGMA journal_mode=WAL") self.db.executescript(""" CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE IF NOT EXISTS series ( + id INTEGER PRIMARY KEY, mac TEXT, cycle TEXT, grp TEXT, field TEXT, unit TEXT, + UNIQUE (mac, cycle, grp, field)); CREATE TABLE IF NOT EXISTS points ( - mac TEXT, cycle TEXT, grp TEXT, field TEXT, ts INTEGER, value TEXT, - PRIMARY KEY (mac, cycle, grp, field, ts)) WITHOUT ROWID; - CREATE TABLE IF NOT EXISTS fields ( - mac TEXT, cycle TEXT, grp TEXT, field TEXT, unit TEXT, - PRIMARY KEY (mac, cycle, grp, field)) WITHOUT ROWID; + series INTEGER, ts INTEGER, value TEXT, + PRIMARY KEY (series, ts)) WITHOUT ROWID; CREATE TABLE IF NOT EXISTS coverage (mac TEXT, cycle TEXT, grp TEXT, start INTEGER, end INTEGER); CREATE INDEX IF NOT EXISTS coverage_key ON coverage (mac, cycle, grp); """) @@ -93,7 +94,7 @@ def __init__(self, path, units: dict): row = self.db.execute("SELECT value FROM meta WHERE key = 'signature'").fetchone() if row and row[0] != signature: log.warning("History cache settings changed - clearing %s", path) - self.db.executescript("DELETE FROM points; DELETE FROM fields; DELETE FROM coverage;") + self.db.executescript("DELETE FROM points; DELETE FROM series; DELETE FROM coverage;") self.db.execute("INSERT OR REPLACE INTO meta VALUES ('signature', ?)", (signature,)) self._convert_units() self.db.commit() @@ -105,11 +106,10 @@ def _convert_units(self): if (row := self.db.execute("SELECT value FROM meta WHERE key = 'unit_fixes'").fetchone()) and row[0] == done: return for (prefix, old), (unit, factor) in UNIT_FIXES.items(): - for key in self.db.execute("SELECT mac, cycle, grp, field FROM fields WHERE unit = ? AND (field = ? OR substr(field, 1, ?) = ?)", - (old, prefix, len(prefix) + 1, prefix + "_")).fetchall(): - self.db.execute("UPDATE points SET value = printf('%.3f', CAST(value AS REAL) * ?) " - "WHERE mac=? AND cycle=? AND grp=? AND field=?", (factor, *key)) - self.db.execute("UPDATE fields SET unit = ? WHERE mac=? AND cycle=? AND grp=? AND field=?", (unit, *key)) + for (key,) in self.db.execute("SELECT id FROM series WHERE unit = ? AND (field = ? OR substr(field, 1, ?) = ?)", + (old, prefix, len(prefix) + 1, prefix + "_")).fetchall(): + self.db.execute("UPDATE points SET value = printf('%.3f', CAST(value AS REAL) * ?) WHERE series=?", (factor, key)) + self.db.execute("UPDATE series SET unit = ? WHERE id=?", (unit, key)) self.db.execute("INSERT OR REPLACE INTO meta VALUES ('unit_fixes', ?)", (done,)) def close(self): @@ -120,6 +120,15 @@ def _query(self, sql: str, *args) -> list: with self._lock: return self.db.execute(sql, args).fetchall() + def count(self, mac: str, cycle: str, grp: str, start: int, end: int) -> int: + """How many readings are held for this group at this resolution between two times (for diagnosing an empty answer).""" + return self._query("SELECT COUNT(*) FROM points p JOIN series s ON s.id = p.series " + "WHERE s.mac=? AND s.cycle=? AND s.grp=? AND p.ts BETWEEN ? AND ?", mac, cycle, grp, start, end)[0][0] + + def newest(self, mac: str, cycle: str) -> int | None: + """The time of the newest reading held at this resolution (None if none).""" + return self._query("SELECT MAX(p.ts) FROM points p JOIN series s ON s.id = p.series WHERE s.mac=? AND s.cycle=?", mac, cycle)[0][0] + def coverage(self, mac: str, cycle: str, grp: str) -> list[Interval]: """Every time range held for this group at this resolution (merged, oldest first).""" return merge(self._query("SELECT start, end FROM coverage WHERE mac=? AND cycle=? AND grp=?", mac, cycle, grp)) @@ -129,12 +138,13 @@ def missing(self, mac: str, cycle: str, grp: str, start: int, end: int) -> list[ def load_fields(self, mac: str, cycle: str, grp: str, fields: list[str] | None, start: int, end: int) -> dict: """A group's fields (all of them if `fields` is None) in Ecowitt's response shape: {field: {unit, list}}.""" - pick = f" AND field IN ({','.join('?' * len(fields))})" if fields is not None else "" + pick = f" AND s.field IN ({','.join('?' * len(fields))})" if fields is not None else "" args = (mac, cycle, grp, *(fields or ())) - units = dict(self._query(f"SELECT field, unit FROM fields WHERE mac=? AND cycle=? AND grp=?{pick}", *args)) + units = dict(self._query(f"SELECT s.field, s.unit FROM series s WHERE s.mac=? AND s.cycle=? AND s.grp=?{pick}", *args)) out: dict = {} - for field, ts, value in self._query(f"SELECT field, ts, value FROM points WHERE mac=? AND cycle=? AND grp=?{pick} " - "AND ts BETWEEN ? AND ? ORDER BY ts", *args, start, end): + for field, ts, value in self._query("SELECT s.field, p.ts, p.value FROM series s JOIN points p ON p.series = s.id " + f"WHERE s.mac=? AND s.cycle=? AND s.grp=?{pick} AND p.ts BETWEEN ? AND ? ORDER BY p.ts", + *args, start, end): out.setdefault(field, {"unit": units.get(field, ""), "list": {}})["list"][str(ts)] = value return out @@ -147,8 +157,9 @@ def slot_ranges(self, mac: str, grp: str, field: str, values: dict[int, float], """(lows, highs) for 30-minute readings that have no range of their own (dew point, feels-like, VPD, solar, UV, pressure ...): the lowest and highest of the cached 5-minute readings inside each slot (and the slot's own value). Only what the cache holds; slots with fewer than two 5-minute readings get none.""" - rows = self._query("SELECT ts, CAST(value AS REAL) FROM points WHERE mac=? AND cycle='5min' AND grp=? AND field=? " - "AND ts BETWEEN ? AND ? ORDER BY ts", mac, grp, field, start, end) + rows = self._query("SELECT p.ts, CAST(p.value AS REAL) FROM points p JOIN series s ON s.id = p.series " + "WHERE s.mac=? AND s.cycle='5min' AND s.grp=? AND s.field=? AND p.ts BETWEEN ? AND ? ORDER BY p.ts", + mac, grp, field, start, end) times, fine = [ts for ts, _ in rows], [v for _, v in rows] lows, highs = {}, {} for t, v in values.items(): @@ -197,13 +208,17 @@ def store(self, mac: str, cycle: str, groups: list[str], data: dict, start: int, with self._lock: grown = set() # groups that now carry a reading the cache has never had (Ecowitt added a metric) for _, _, grp, field, _ in units: - known = {r[0] for r in self.db.execute("SELECT field FROM fields WHERE mac=? AND cycle=? AND grp=?", (mac, cycle, grp))} + known = {r[0] for r in self.db.execute("SELECT field FROM series WHERE mac=? AND cycle=? AND grp=?", (mac, cycle, grp))} if known and field not in known: grown.add(grp) if grown: log.info("Ecowitt added a metric to %s (%s): its history is fetched again to fill it in", ", ".join(sorted(grown)), cycle) - self.db.executemany("INSERT OR REPLACE INTO points VALUES (?,?,?,?,?,?)", rows) - self.db.executemany("INSERT OR REPLACE INTO fields VALUES (?,?,?,?,?)", units) + ids = {} + for key in units: # a series is created the first time it is seen; its unit follows what Ecowitt says now + self.db.execute("INSERT OR IGNORE INTO series (mac, cycle, grp, field, unit) VALUES (?,?,?,?,?)", key) + self.db.execute("UPDATE series SET unit=? WHERE mac=? AND cycle=? AND grp=? AND field=?", (key[4], *key[:4])) + ids[key[:4]] = self.db.execute("SELECT id FROM series WHERE mac=? AND cycle=? AND grp=? AND field=?", key[:4]).fetchone()[0] + self.db.executemany("INSERT OR REPLACE INTO points VALUES (?,?,?)", [(ids[r[:4]], r[4], r[5]) for r in rows]) for grp, cov_end in covered: existing = [] if grp in grown else self.db.execute( # the days held so far lack it: they are asked for again "SELECT start, end FROM coverage WHERE mac=? AND cycle=? AND grp=?", (mac, cycle, grp)).fetchall() diff --git a/lib/forecast/source.py b/lib/forecast/source.py index 8d1a5d3..9ad1787 100644 --- a/lib/forecast/source.py +++ b/lib/forecast/source.py @@ -13,7 +13,7 @@ from .. import intent from ..config import Config from ..snapshots import Snapshots -from ..timeutil import now_local +from ..timeutil import now_local, when from ..tools import Tool, Turn from ..warm import Warmer, in_sync_hours @@ -227,4 +227,5 @@ async def handle(self, args: dict, turn: Turn | None = None) -> str: return json.dumps({"error": "No forecast is available right now."}) if turn: turn.forecast_shown.extend(self.upcoming(count)) # remembered for the chat once the answer is sent - return json.dumps({"lines": lines, "place": self.place, "source": self.source}, ensure_ascii=False) + stamped = {"time": when(int(self.fetched_at), self.tz)} if self.fetched_at else {} # when it was last fetched + return json.dumps({"lines": lines, "place": self.place, "source": self.source, **stamped}, ensure_ascii=False) diff --git a/lib/intent.py b/lib/intent.py index f8e1a46..1927c90 100644 --- a/lib/intent.py +++ b/lib/intent.py @@ -393,7 +393,7 @@ def chart_field(text: str) -> str | None: return WEATHER_READINGS[found[0]].field if len(found) == 1 and found[0] != "temperature" else None -# "rain chart 7d": a rain chart whose caption is just the least and most rain and whether more is expected +# "rain chart 7d": a rain chart whose caption is just the least and most rain RAIN_CHART = re.compile(r"^\s*rain\s+(chart|graph|plot)\b[^,&+]*$", I) @@ -781,7 +781,7 @@ class Reading: average_asked: bool = False more: list[tuple[str, dict]] = field(default_factory=list) # the rest of the report's calls (fast is the first) weather_now: bool = False # "weather now": every reading the station has, in the report's layout - rain_caption: bool = False # "rain chart 7d": the caption is the least and most rain and whether rain is expected + rain_caption: bool = False # "rain chart 7d": the caption is just the least and most rain chart_in_code: bool = False # a chart asked for plainly: fetched and captioned in code, no model chart_all: bool = False # "weather all week": every reading charted, sent with no caption readings: list[str] = field(default_factory=list) # the readings the words name @@ -830,8 +830,6 @@ def read(text: str, now: datetime, ecowitt: bool = True, air: bool = True, polle and not any(p.search(text) for p in JUDGEMENT)) rain_caption = ecowitt and wants_rain_caption(text) lookup, named = plain_lookup(text, fast) if not (report or in_code) else ("", []) - if in_code and rain_caption: # the caption also says whether rain is expected - calls = [("", {}), ("weather_now", {"groups": "rainfall"})] return Reading(reasoning_effort(text), needs_data(text), about_the_bot(text), report, period_hints(text, now), fast, bool(GRAPH.search(text)), chart_field(text), chart_fields(text), bool(AVERAGE.search(text)), more=calls[1:], weather_now=ecowitt and wants_weather_now(text), rain_caption=rain_caption, chart_in_code=in_code, chart_all=in_code and wants_chart_all(text), diff --git a/lib/irrigation/watchdog.py b/lib/irrigation/watchdog.py index 041be3c..92ff8e0 100644 --- a/lib/irrigation/watchdog.py +++ b/lib/irrigation/watchdog.py @@ -21,7 +21,7 @@ def __init__(self, device: Tuya, state, clock=time.time): self.device, self.state, self.clock = device, state, clock def _memory(self) -> dict: - return self.state.monitor.setdefault("irrigation", {}) + return self.state.monitor.setdefault("irrigation_watchdog", {}) def arm(self, minutes: int): """A run of this many minutes has started.""" diff --git a/lib/pollen/source.py b/lib/pollen/source.py index e591dee..0022131 100644 --- a/lib/pollen/source.py +++ b/lib/pollen/source.py @@ -14,7 +14,7 @@ from .. import intent from ..config import Config from ..snapshots import Snapshots -from ..timeutil import now_local +from ..timeutil import now_local, when from ..tools import Tool, Turn from ..warm import Warmer, in_sync_hours from .parse import LEVEL_EMOJI, parse_melbourne_pollen @@ -151,6 +151,8 @@ async def handle(self, args: dict, turn: Turn | None = None) -> str: cur = self.current() out = {"lines": lines, "place": self.place, "source": "melbournepollen.com.au (grass pollen from pollen-trap data; asthma risk from the " "Victorian Department of Health and BOM, 1 Oct - 31 Dec)"} + if cur["fetched_at"]: + out["time"] = when(int(cur["fetched_at"]), self.tz) # when the page was last fetched if cur["asthma"] and cur["asthma"]["updated"]: out["asthma_updated"] = cur["asthma"]["updated"] if not cur["asthma"]: diff --git a/lib/prompt.py b/lib/prompt.py index 356fbed..7d1cb2c 100644 --- a/lib/prompt.py +++ b/lib/prompt.py @@ -190,7 +190,7 @@ def capabilities(sources: list[str], brief: bool = False) -> str: lines.append("- Charts: any one reading, or several readings together on one time axis, one panel each (" + ", ".join(WEATHER_NAMES) + "; \"weather all week\" draws every reading); wind as average speed with gusts beside a compass rose of directions; " "air quality with ratings.") - lines.append("- Alerts, sent to chats automatically: rain starting or stopping, rain likely soon, wind gusts over 40 km/h, UV index of 10 or more, " + lines.append("- Alerts, sent to chats automatically: rain starting or stopping, wind gusts over 40 km/h, UV index of 10 or more, " "indoor/outdoor temperature crossing, air-quality mask alerts" + (", pollen or thunderstorm asthma risk reaching High" if "melbournepollen" in have else "") + ( ", a forecast it sent being revised (rain, or max temperature over 2 degrees)" if "Open-Meteo" in have else "") + ". Each alert has buttons to subscribe or unsubscribe by type, and /alerts opens the same settings. Custom alerts " "(\"tell me when winds reach 100\", another limit) can't be added: say so.") @@ -204,9 +204,8 @@ def capabilities(sources: list[str], brief: bool = False) -> str: RAIN_CAPTION_SECTION = """ -THIS IS A RAIN CHART: its caption is ONLY the least and the most rain in the period (daily totals, mm, with the dated days), then one line -saying whether rain is expected: call weather_now with groups "rainfall" and use its "rain_outlook" (raining now, or rain likely -soon); with no rain_outlook, say no rain is expected soon. Nothing else: no averages, no other readings. +THIS IS A RAIN CHART: its caption is ONLY the least and the most rain in the period (daily totals, mm, with the dated days). +Nothing else: no averages, no other readings, no outlook. """ diff --git a/lib/report.py b/lib/report.py index 5a25113..9d9b45d 100644 --- a/lib/report.py +++ b/lib/report.py @@ -5,11 +5,12 @@ import re from datetime import datetime +from telegram import MessageEntity + from .ecowitt.glance import solar_band from .series import WEATHER, find AIR_ROWS = (("pm2_5", "PM2.5"), ("pm10", "PM10"), ("pm1", "PM1"), ("co2", "COβ‚‚"), ("voc_index", "VOC index"), ("nox_index", "NOx index")) -OUTLOOK_NOTE = re.compile(r"\s*\(an estimate from the station's readings, not an official forecast\)", re.I) def _load(result: str | None) -> dict | None: @@ -46,9 +47,33 @@ def _join(parts: list[str | None], sep: str = ", ") -> str: return sep.join(p for p in parts if p) +HEADINGS = ("Weather station", "Air quality", "Pollen & asthma", "Forecast") # the names the report's sections (and the pollen and forecast lookups) start with +HEADING_LINE = re.compile(rf"^({'|'.join(map(re.escape, HEADINGS))})(?= \(|$)", re.M) + + +def heading_entities(text: str) -> list[MessageEntity]: + """Bold entities over the name of each heading line in `text` (not over its brackets). Telegram counts positions in UTF-16 code + units, where an emoji is two.""" + units = lambda s: len(s.encode("utf-16-le")) // 2 + return [MessageEntity(MessageEntity.BOLD, units(text[:m.start()]), units(m.group(1))) for m in HEADING_LINE.finditer(text)] + + +def _stamp(result: dict) -> str | None: + """The time of day of the data, "11:45pm", from the result's "time" ("Sun 11 Oct 2026 23:50" from the station, "Sun 11 Oct 2026 11:45pm" + from the rest); None when it is missing or unreadable.""" + for form in ("%a %d %b %Y %I:%M%p", "%a %d %b %Y %H:%M"): + try: + at = datetime.strptime(result.get("time") or "", form) + except ValueError: + continue + return f"{at.hour % 12 or 12}:{at:%M}{'am' if at.hour < 12 else 'pm'}" + return None + + def _heading(name: str, result: dict) -> str: - """"Forecast (Melbourne)": the place the result names, if it does.""" - return f"{name} ({result['place']})" if result.get("place") else name + """"Forecast (Melbourne, 6:00pm)": the place the result names and the time of its data, whichever it has.""" + note = ", ".join(x for x in (result.get("place"), _stamp(result)) if x) + return f"{name} ({note})" if note else name def _getters(now: dict): @@ -68,13 +93,9 @@ def weather_lines(now: dict) -> list[str]: """The weather station's bullets from a weather_now result: an emoji sits right before the reading it is keyed to.""" get, tag = _getters(now) rain_rate = (now.get("rainfall") or {}).get("rain_rate", "") - outlook = OUTLOOK_NOTE.sub(" (an estimate)", now["rain_outlook"]) if now.get("rain_outlook") else None - if outlook and outlook.startswith("raining"): - outlook = tag("rainfall", "rain_rate", outlook) - elif not outlook and rain_rate.split()[:1] not in ([], ["0"], ["0.0"]): - outlook = tag("rainfall", "rain_rate", f"raining ({get('rainfall', 'rain_rate')})") + raining = tag("rainfall", "rain_rate", f"raining ({get('rainfall', 'rain_rate')})") if rain_rate.split()[:1] not in ([], ["0"], ["0.0"]) else None month = get("rainfall", "monthly") - rain = _join([_join([tag("rainfall", "daily"), f"(month total {month})" if month else None], " "), outlook], "; ") + rain = _join([_join([tag("rainfall", "daily"), f"(month total {month})" if month else None], " "), raining], "; ") degrees = (now.get("wind") or {}).get("wind_direction", "") calm = _zero(get("wind", "wind_speed")) and _zero(get("wind", "wind_gust")) # no wind: no line @@ -185,10 +206,10 @@ def report(results: dict[str, str]) -> str: sections = [] if "weather_now" in results: now = _load(results["weather_now"]) - sections.append(["Weather station", *(weather_lines(now) if now else ["β€’ not available right now"])]) + sections.append([_heading("Weather station", now or {}), *(weather_lines(now) if now else ["β€’ not available right now"])]) if "air_quality" in results: air = _load(results["air_quality"]) - sections.append(["Air quality", *(air_lines(air) if air else ["β€’ not available right now"])]) + sections.append([_heading("Air quality", air or {}), *(air_lines(air) if air else ["β€’ not available right now"])]) if pollen := _load(results.get("pollen_asthma")): sections.append([_heading("Pollen & asthma", pollen), *(f"β€’ {line}" for line in pollen.get("lines", []))]) if forecast := _load(results.get("weather_forecast")): @@ -225,7 +246,7 @@ def _range(label: str, entry: dict, average: bool) -> str: def _rain_line(turn, tz) -> list[str]: - """Least and most rain in a period (the chart's own bars), then whether more is expected.""" + """Least and most rain in a period (the chart's own bars).""" bars = next((p.bars for c in turn.charts for p in c.panels if p.bars and p.bars.x), None) if not bars: return [] @@ -236,18 +257,15 @@ def day(i: int) -> str: return [f"β˜” Rain{per}: least {bars.y[lo]:g} mm ({day(lo)}) Β· most {bars.y[hi]:g} mm ({day(hi)}) Β· {sum(bars.y):.1f} mm in all"] -def _weather_caption(result: dict, args: dict, turn, tz, now_result: dict | None) -> list[str]: +def _weather_caption(result: dict, args: dict, turn, tz) -> list[str]: series = result.get("series") or {} names = [n for n in dict.fromkeys(turn.chart_fields or [turn.chart_field or "temperature"]) if find(n)] lines: list[str] = [] for name in names: reading = find(name) keys = [k for k in series if k.split(".", 1)[1] == reading.field] - if reading.field == "daily": # rain: the amounts the chart draws, and what is expected + if reading.field == "daily": # rain: the amounts the chart draws lines += _rain_line(turn, tz) - if now_result is not None: - outlook = OUTLOOK_NOTE.sub(" (an estimate)", now_result.get("rain_outlook") or "") - lines.append("β˜” " + (outlook or "No rain is expected soon.").lstrip("β˜” ")) continue for key in keys: group = key.split(".", 1)[0] @@ -275,14 +293,14 @@ def _air_caption(result: dict, args: dict, turn) -> list[str]: return lines -def chart_caption(tool: str, args: dict, result: str | None, turn, tz, extra: str | None = None) -> str | None: +def chart_caption(tool: str, args: dict, result: str | None, turn, tz) -> str | None: """The caption for a chart, from the tool's result: the period, then a line per reading. None when there is nothing to say (the model then takes the question).""" data = _load(result) if not data or not turn.charts: return None lines = (_air_caption(data, args, turn) if tool == "air_quality" else - _weather_caption(data, args, turn, tz, _load(extra) if extra is not None else None)) + _weather_caption(data, args, turn, tz)) if not lines: return None notes = (["(Older days are still downloading, so they are left out.)"] if data.get("note_missing") else diff --git a/lib/report_menu.py b/lib/report_menu.py new file mode 100644 index 0000000..48da949 --- /dev/null +++ b/lib/report_menu.py @@ -0,0 +1,11 @@ +"""The button under the full report ("status"): Refresh, which reads everything again and redraws the report in place +(Bot.on_report_button).""" + +from telegram import InlineKeyboardButton, InlineKeyboardMarkup + +PREFIX = "rp:" +REFRESH = f"{PREFIX}refresh" + + +def keyboard() -> InlineKeyboardMarkup: + return InlineKeyboardMarkup([[InlineKeyboardButton("πŸ”„ Refresh", callback_data=REFRESH)]]) diff --git a/lib/timeutil.py b/lib/timeutil.py index 8dde16c..671a33f 100644 --- a/lib/timeutil.py +++ b/lib/timeutil.py @@ -17,6 +17,12 @@ def to_local(ts: int, tz: tzinfo) -> datetime: return datetime.fromtimestamp(ts, UTC).astimezone(tz) +def when(ts: int, tz: tzinfo) -> str: + """An epoch as "Sun 11 Oct 2026 11:45pm" in tz (the form the tools put in their "time" field).""" + dt = to_local(ts, tz) + return f"{dt:%a} {dt.day} {dt:%b %Y} {dt:%I:%M%p}".replace(" 0", " ").replace("AM", "am").replace("PM", "pm") + + def local_date(ts: int, tz: tzinfo) -> date: return to_local(ts, tz).date() diff --git a/lib/tools.py b/lib/tools.py index 83f4390..123f19e 100644 --- a/lib/tools.py +++ b/lib/tools.py @@ -69,7 +69,7 @@ async def call(self, name: str, raw_args: str, turn: Turn | None = None) -> str: if len(out) > MAX_OUTPUT_CHARS: log.warning("Tool %s output truncated: %d chars", name, len(out)) out = out[:MAX_OUTPUT_CHARS] + "\n...[truncated - request a shorter range]" - log.info("Tool result %s: %s", name, out if len(out) < 300 else f"{len(out)} chars") + log.info("Tool result %s: %s", name, out if len(out) < 90 else f"{len(out)} chars") if turn: turn.result_chars += len(out) return out diff --git a/scripts/cache_status.py b/scripts/cache_status.py index 0c2699a..453fd36 100644 --- a/scripts/cache_status.py +++ b/scripts/cache_status.py @@ -12,7 +12,8 @@ from datetime import date, datetime, timedelta, UTC from pathlib import Path -from _common import ROOT, parser # noqa: E402 (also puts the repo on sys.path) +from _common import parser # noqa: E402 (also puts the repo on sys.path) +from lib.config import data_dir # noqa: E402 from lib.ecowitt.api import GROUPS, RETENTION # noqa: E402 from lib.ecowitt.store import horizon, subtract # noqa: E402 @@ -25,8 +26,8 @@ def day(ts: int) -> str: def ecowitt(path: Path): db = sqlite3.connect(f"file:{path}?mode=ro", uri=True) for (mac,) in db.execute("SELECT DISTINCT mac FROM coverage").fetchall(): - first = db.execute("SELECT MIN(ts) FROM points WHERE mac=?", (mac,)).fetchone()[0] - rows = db.execute("SELECT COUNT(*) FROM points WHERE mac=?", (mac,)).fetchone()[0] + first = db.execute("SELECT MIN(p.ts) FROM points p JOIN series s ON s.id = p.series WHERE s.mac=?", (mac,)).fetchone()[0] + rows = db.execute("SELECT COUNT(*) FROM points p JOIN series s ON s.id = p.series WHERE s.mac=?", (mac,)).fetchone()[0] print(f"Ecowitt {mac}: {rows:,} readings, station data from {day(first) if first else 'n/a'}") now = time.time() for cycle in ("5min", "30min", "4hour", "1day"): @@ -41,7 +42,7 @@ def ecowitt(path: Path): gaps = subtract((int(start), int(end)), cov) if gaps and gaps[-1][1] >= end - 1 and gaps[-1][0] >= end - 2 * 86400: gaps.pop() # the newest day or two: still settling, filled in by the next archive run - stored = db.execute("SELECT COUNT(*) FROM points WHERE mac=? AND cycle=? AND grp=?", + stored = db.execute("SELECT COUNT(*) FROM points p JOIN series s ON s.id = p.series WHERE s.mac=? AND s.cycle=? AND s.grp=?", (mac, cycle, group)).fetchone()[0] span = f"{day(min(c[0] for c in cov))} to {day(max(c[1] for c in cov))}" if not gaps: @@ -70,8 +71,8 @@ def airgradient(path: Path): if __name__ == "__main__": ap = parser(__doc__) - ap.add_argument("--ecowitt", type=Path, default=ROOT / "ecowitt_cache.sqlite") - ap.add_argument("--airgradient", type=Path, default=ROOT / "airgradient_cache.sqlite") + ap.add_argument("--ecowitt", type=Path, default=data_dir() / "ecowitt_cache.sqlite") + ap.add_argument("--airgradient", type=Path, default=data_dir() / "airgradient_cache.sqlite") args = ap.parse_args() for path, report in ((args.ecowitt, ecowitt), (args.airgradient, airgradient)): if path.exists(): diff --git a/scripts/check_rain.py b/scripts/check_rain.py index 4ec6562..c4e5e97 100644 --- a/scripts/check_rain.py +++ b/scripts/check_rain.py @@ -20,7 +20,8 @@ from pathlib import Path from zoneinfo import ZoneInfo -from _common import ROOT, parser # noqa: E402 (also puts the repo on sys.path) +from _common import parser # noqa: E402 (also puts the repo on sys.path) +from lib.config import data_dir # noqa: E402 from lib.ecowitt import days as day_tool # noqa: E402 from lib.ecowitt.api import UNITS # noqa: E402 @@ -34,7 +35,7 @@ def fields_held(cache: HistoryCache, mac: str) -> dict[str, list[str]]: """Which rainfall fields are stored at each resolution.""" with cache._lock: - rows = cache.db.execute("SELECT cycle, field FROM fields WHERE mac=? AND grp='rainfall' ORDER BY cycle, field", (mac,)).fetchall() + rows = cache.db.execute("SELECT cycle, field FROM series WHERE mac=? AND grp='rainfall' ORDER BY cycle, field", (mac,)).fetchall() out: dict[str, list[str]] = {} for cycle, field in rows: out.setdefault(cycle, []).append(field) @@ -107,7 +108,7 @@ def report(res: dict): def main(): ap = parser(__doc__) ap.add_argument("--days", type=int, default=88) - ap.add_argument("--cache", type=Path, default=ROOT / "ecowitt_cache.sqlite") + ap.add_argument("--cache", type=Path, default=data_dir() / "ecowitt_cache.sqlite") args = ap.parse_args() tz = ZoneInfo(os.getenv("TZ") or "Australia/Melbourne") macs = [m for (m,) in sqlite3.connect(f"file:{args.cache}?mode=ro", uri=True).execute("SELECT DISTINCT mac FROM coverage")] diff --git a/scripts/dump_specs.py b/scripts/dump_specs.py index ec0c1b6..4c10f32 100644 --- a/scripts/dump_specs.py +++ b/scripts/dump_specs.py @@ -18,7 +18,8 @@ from zoneinfo import ZoneInfo import httpx -from _common import ROOT, parser # noqa: E402 (also puts the repo on sys.path) +from _common import parser # noqa: E402 (also puts the repo on sys.path) +from lib.config import data_dir # noqa: E402 from lib.airgradient import AirGradient # noqa: E402 from lib.charts import render # noqa: E402 @@ -103,7 +104,7 @@ def copy_db(src: Path, dest: Path): async def main(): logging.disable(logging.WARNING) # the fetchers log every refused request ap = parser(__doc__) - ap.add_argument("--data-dir", default=str(ROOT), help="folder holding ecowitt_cache.sqlite and airgradient_cache.sqlite") + ap.add_argument("--data-dir", default=str(data_dir()), help="folder (default DATA_DIR) holding ecowitt_cache.sqlite and airgradient_cache.sqlite") ap.add_argument("--tz", default="Australia/Melbourne") ap.add_argument("--out", default="chart_dump") args = ap.parse_args() diff --git a/scripts/migrate_cache.py b/scripts/migrate_cache.py new file mode 100644 index 0000000..8c55b67 --- /dev/null +++ b/scripts/migrate_cache.py @@ -0,0 +1,73 @@ +"""Convert the Ecowitt history cache to the smaller layout, in place. Run it ONCE, with the bot stopped: + + python scripts/migrate_cache.py /var/lib/ecobot/ecowitt_cache.sqlite [--no-backup] + +The old layout repeated (mac, cycle, group, field) as text in the key of every reading; the new one keeps each series once in a `series` +table and keys a reading by that small id and its time. Every value string and unit is carried over as it is, so nothing is fetched again +(Ecowitt only keeps 90 days of 5-minute readings, so the cache cannot be rebuilt). The conversion is one transaction; the original is kept as +.old unless --no-backup, and the file is compacted at the end. A file already in the new layout is left alone. +""" + +import shutil +import sqlite3 +from pathlib import Path + +from _common import parser + + +def converted(db: sqlite3.Connection) -> bool: + return "series" in {name for (name,) in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + + +def migrate(path: Path, backup: bool = True) -> tuple[int, int, int, int]: + """(rows, series, bytes before, bytes after); rows is -1 when the file was already converted.""" + before = path.stat().st_size + db = sqlite3.connect(path, isolation_level=None) + try: + if converted(db): + return -1, 0, before, before + db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + if backup: + if shutil.disk_usage(path.parent).free < 2 * before: + raise SystemExit(f"Not enough free disk for the backup and the conversion (need about {2 * before / 1e6:.0f} MB)") + shutil.copy2(path, path.with_name(path.name + ".old")) + db.execute("BEGIN") # one statement at a time: executescript would commit first + for statement in ( + "ALTER TABLE points RENAME TO points_old", + "ALTER TABLE fields RENAME TO fields_old", + """CREATE TABLE series (id INTEGER PRIMARY KEY, mac TEXT, cycle TEXT, grp TEXT, field TEXT, unit TEXT, + UNIQUE (mac, cycle, grp, field))""", + "CREATE TABLE points (series INTEGER, ts INTEGER, value TEXT, PRIMARY KEY (series, ts)) WITHOUT ROWID", + "INSERT INTO series (mac, cycle, grp, field, unit) SELECT mac, cycle, grp, field, unit FROM fields_old ORDER BY mac, cycle, grp, field", + "INSERT OR IGNORE INTO series (mac, cycle, grp, field, unit) SELECT DISTINCT mac, cycle, grp, field, '' FROM points_old", + """INSERT INTO points SELECT s.id, p.ts, p.value FROM points_old p + JOIN series s ON s.mac = p.mac AND s.cycle = p.cycle AND s.grp = p.grp AND s.field = p.field ORDER BY s.id, p.ts"""): + db.execute(statement) + rows = db.execute("SELECT COUNT(*) FROM points").fetchone()[0] + if rows != db.execute("SELECT COUNT(*) FROM points_old").fetchone()[0]: + raise SystemExit("The converted row count differs from the original; nothing was changed") + series = db.execute("SELECT COUNT(*) FROM series").fetchone()[0] + db.execute("DROP TABLE points_old") + db.execute("DROP TABLE fields_old") + db.execute("COMMIT") + db.execute("VACUUM") + return rows, series, before, path.stat().st_size + except BaseException: + if db.in_transaction: + db.execute("ROLLBACK") + raise + finally: + db.close() + + +if __name__ == "__main__": + ap = parser(__doc__) + ap.add_argument("cache", type=Path, help="the Ecowitt history cache (ecowitt_cache.sqlite)") + ap.add_argument("--no-backup", action="store_true", help="do not keep .old") + args = ap.parse_args() + rows, series, before, after = migrate(args.cache, backup=not args.no_backup) + if rows < 0: + print(f"{args.cache} is already in the new layout; nothing to do.") + else: + print(f"Converted {rows:,} readings in {series} series: {before / 1e6:.1f} MB -> {after / 1e6:.1f} MB" + + ("" if args.no_backup else f" (the original is {args.cache.name}.old)")) diff --git a/scripts/show_request.py b/scripts/show_request.py index deb1def..8ca76d7 100644 --- a/scripts/show_request.py +++ b/scripts/show_request.py @@ -20,11 +20,12 @@ from types import SimpleNamespace as NS from zoneinfo import ZoneInfo -from _common import ROOT, parser # noqa: E402 (also puts the repo on sys.path) +from _common import parser # noqa: E402 (also puts the repo on sys.path) +from lib.config import data_dir # noqa: E402 from lib import intent, prompt # noqa: E402 from lib.airgradient import source as air # noqa: E402 -from lib.ecowitt import days, station # noqa: E402 +from lib.ecowitt import days, source as station # noqa: E402 from lib.ecowitt.api import UNITS # noqa: E402 from lib.ecowitt.store import HistoryCache # noqa: E402 from lib.llm import Agent # noqa: E402 @@ -61,7 +62,7 @@ def banner(text): async def main(): ap = parser(__doc__) ap.add_argument("question") - ap.add_argument("--cache", type=Path, default=ROOT / "ecowitt_cache.sqlite") + ap.add_argument("--cache", type=Path, default=data_dir() / "ecowitt_cache.sqlite") ap.add_argument("--call", help="JSON tool call the stand-in model makes first") args = ap.parse_args() diff --git a/tests/data/ecowitt_cache.sqlite b/tests/data/ecowitt_cache.sqlite index 4095c31..5adfda6 100644 Binary files a/tests/data/ecowitt_cache.sqlite and b/tests/data/ecowitt_cache.sqlite differ diff --git a/tests/data/make_fixture.py b/tests/data/make_fixture.py index 8c52419..df54928 100644 --- a/tests/data/make_fixture.py +++ b/tests/data/make_fixture.py @@ -1,5 +1,6 @@ """Writes the two small cache files in this folder. They were generated ONCE by the pre-recode code; the tests load them -to prove that caches written by older versions still open and read the same. Do not regenerate them from new code.""" +to prove that a recorded cache still opens and reads the same. Do not regenerate them from new code. (The Ecowitt one was later +converted, with its data untouched, to the series layout by scripts/migrate_cache.py.)""" import asyncio import sys import tempfile diff --git a/tests/evalkit.py b/tests/evalkit.py index c2efacf..5fdb060 100644 --- a/tests/evalkit.py +++ b/tests/evalkit.py @@ -14,7 +14,7 @@ from lib import compose, intent, prompt from lib.airgradient import days as air_days, source as air -from lib.ecowitt import days, link, station +from lib.ecowitt import days, link, source as station from lib.forecast import source as forecast from lib.pollen import source as pollen from lib.llm import Agent diff --git a/tests/test_admin_only.py b/tests/test_admin_only.py index 4a59e3e..7806e5c 100644 --- a/tests/test_admin_only.py +++ b/tests/test_admin_only.py @@ -7,7 +7,7 @@ from telegram import CallbackQuery, InlineKeyboardButton, InlineKeyboardMarkup from telegram.ext import CallbackQueryHandler -from lib import periods, templates +from lib import periods, report_menu, templates from lib.admins import Admins from lib.alerts import AlertState, menu from lib.bot import MAX_CALLBACK_BYTES, NOT_ADMIN, NOT_ADMIN_TOAST, STALE_BUTTON, Bot @@ -319,7 +319,7 @@ async def test_alert_buttons_in_a_group_stay_admin_only_even_when_admin_only_is_ def every_keyboard(bot): """Every inline keyboard the bot can draw, in every state.""" kinds = menu.available_kinds({"Ecowitt", "AirGradient", "Pollen", "Forecast", "Irrigation"}) - muted_sets = [set(), {"rain"}, {"rain", "uv", "air"}, set(menu.LABELS)] + muted_sets = [set(), {"rain"}, {"rain", "uv", "particulates"}, set(menu.LABELS)] for muted in muted_sets: for opened in (None, "sub", "unsub"): for parent in (None, *kinds): @@ -331,6 +331,7 @@ def every_keyboard(bot): yield irrigation_menu.keyboard(battery, pause, opened, paused) for current in (None, *periods.PERIODS): yield periods.keyboard(current) + yield report_menu.keyboard() def test_every_button_the_bot_draws_is_short_plain_and_has_exactly_one_handler(tmp_path): @@ -344,7 +345,7 @@ def test_every_button_the_bot_draws_is_short_plain_and_has_exactly_one_handler(t assert 0 < len(data.encode()) <= MAX_CALLBACK_BYTES and data.isprintable(), data assert sum(data.startswith(prefix) for prefix in table) == 1, data assert bot._on_keyboard(NS(reply_markup=markup), data) - assert len(seen) > 60 and {p for p in table} == {"al:", "ir:", periods.PREFIX} + assert len(seen) > 60 and {p for p in table} == {"al:", "ir:", periods.PREFIX, report_menu.PREFIX} assert "ir:" not in make(tmp_path, irrigation=None)[0].buttons() @@ -432,13 +433,13 @@ async def test_random_data_never_gets_past_the_entry_point(tmp_path): rng = random.Random(7) spy = Spy() bot, state, tg = make(tmp_path, irrigation=spy) - boards = [alert_keyboard("sub"), irrigation_menu.keyboard(True, True), periods.keyboard(1)] + boards = [alert_keyboard("sub"), irrigation_menu.keyboard(True, True), periods.keyboard(1), report_menu.keyboard()] known = {b.callback_data for board_ in boards for row in board_.inline_keyboard for b in row} pieces = ["ir", "al", "pd", "on", "off", "water", "delay", "sw", "batt", "pause", "refresh", "open", "close", "all", "sub", "unsub", "rain", "uv", "24h", "cancel", "5", "30", "99999", "-1", "", " ", "\n", "Γ©", "πŸ’§", "*", "%s", "{}", "\\", ";", "$(x)", "None"] before = (dict(state.chats), dict(state.charts)) for _ in range(3000): - data = rng.choice(["ir", "al", "pd", ""]) + ":" + ":".join(rng.choice(pieces) for _ in range(rng.randint(0, 4))) + data = rng.choice(["ir", "al", "pd", "rp", ""]) + ":" + ":".join(rng.choice(pieces) for _ in range(rng.randint(0, 4))) if data in known: continue press = await press_through(bot, tg, data, rng.choice(boards), user=rng.choice([ADMIN, STRANGER])) diff --git a/tests/test_agent.py b/tests/test_agent.py index 237a46f..fedbe25 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -238,12 +238,12 @@ async def test_streamed_answers_give_the_same_result_and_report_the_text_so_far( assert msgs[1]["tool_calls"][0]["function"] == {"name": "weather_now", "arguments": '{"groups": "outdoor"}'} -def test_a_rain_chart_caption_is_only_the_least_and_most_rain_and_whether_more_is_expected(): +def test_a_rain_chart_caption_is_only_the_least_and_most_rain(): from datetime import datetime from lib import prompt now = datetime(2026, 9, 29, 14, 5) text = prompt.build(now, ["Ecowitt weather station"], rain_caption=True) - assert "THIS IS A RAIN CHART" in text and "rain_outlook" in text + assert "THIS IS A RAIN CHART" in text and "rain_outlook" not in text assert "THIS IS A RAIN CHART" not in prompt.build(now, ["Ecowitt weather station"]) diff --git a/tests/test_alert_menu.py b/tests/test_alert_menu.py index 3c8872a..894c771 100644 --- a/tests/test_alert_menu.py +++ b/tests/test_alert_menu.py @@ -8,7 +8,7 @@ from lib.bot import Bot from tests.fakes import TZ -KINDS = ["rain", "rain_likely", "gusts", "uv", "temps", "air", "pollen", "forecast"] +KINDS = ["rain", "gusts", "uv", "temps", "particulates", "pollen_and_asthma", "forecast"] def rows(markup): @@ -20,30 +20,30 @@ def test_the_collapsed_menu_is_subscribe_and_unsubscribe(): def test_unsubscribe_opens_the_types_that_are_on_and_subscribe_the_types_that_are_off(): - got = rows(keyboard({"pollen", "forecast"}, KINDS, "unsub")) + got = rows(keyboard({"pollen_and_asthma", "forecast"}, KINDS, "unsub")) assert got[0] == [("βž• Sub β–Έ", "al:open:sub"), ("βž– Unsub β–Ύ", "al:close")] - assert got[1] == [("🌧️ rain", "al:off:rain:unsub"), ("🌦️ rain predicted", "al:off:rain_likely:unsub")] - assert [text for row in got[1:-1] for text, _ in row] == ["🌧️ rain", "🌦️ rain predicted", "πŸ’¨ gusts", "🧴 UV", "🌑️ temperature crossing", "😷 particulates"] + assert got[1] == [("🌧️ rain", "al:off:rain:unsub"), ("πŸ’¨ gusts", "al:off:gusts:unsub")] + assert [text for row in got[1:-1] for text, _ in row] == ["🌧️ rain", "πŸ’¨ gusts", "🧴 UV", "🌑️ temperature crossing", "😷 particulates"] assert got[-1] == [("πŸ”• Unsubscribe from all", "al:off:all:unsub")] - got = rows(keyboard({"pollen", "forecast"}, KINDS, "sub")) + got = rows(keyboard({"pollen_and_asthma", "forecast"}, KINDS, "sub")) assert got[0][0] == ("βž• Sub β–Ύ", "al:close") - assert got[1:] == [[("🌼 pollen & asthma", "al:on:pollen:sub"), ("πŸ”„ forecast changes", "al:on:forecast:sub")], [("πŸ”” Subscribe to all", "al:on:all:sub")]] + assert got[1:] == [[("🌼 pollen & asthma", "al:on:pollen_and_asthma:sub"), ("πŸ”„ forecast changes", "al:on:forecast:sub")], [("πŸ”” Subscribe to all", "al:on:all:sub")]] assert not any(data == "al:noop" for row in got for _, data in row) # no button that does nothing - assert options({"pollen"}, KINDS, "sub") == ["pollen"] and options({"pollen"}, ["rain", "pollen"], "unsub") == ["rain"] + assert options({"pollen_and_asthma"}, KINDS, "sub") == ["pollen_and_asthma"] and options({"pollen_and_asthma"}, ["rain", "pollen_and_asthma"], "unsub") == ["rain"] assert len(rows(keyboard(set(), KINDS, "sub"))) == 1 # nothing off: nothing listed def test_only_the_types_the_bot_has_are_listed_and_every_callback_is_short(): - assert available_kinds({"Ecowitt"}) == ["rain", "rain_likely", "gusts", "uv", "temps"] - assert available_kinds({"Ecowitt", "AirGradient", "Pollen", "Forecast"}) == KINDS and available_kinds({"AirGradient"}) == ["air"] + assert available_kinds({"Ecowitt"}) == ["rain", "gusts", "uv", "temps"] + assert available_kinds({"Ecowitt", "AirGradient", "Pollen", "Forecast"}) == KINDS and available_kinds({"AirGradient"}) == ["particulates"] for section in (None, "sub", "unsub"): for text, data in (b for row in rows(keyboard({"uv"}, KINDS, section)) for b in row): assert len(data.encode()) <= 64 def test_the_title_lists_what_is_on_and_off(): - assert title({"gusts", "uv"}, KINDS[:5]) == f"{TITLE}\nβœ… On: rain, rain predicted, temperature crossing\nπŸ”• Off: gusts, UV" - assert title(set(), ["air"]).endswith("πŸ”• Off: nothing") and title({"air"}, ["air"]).endswith("βœ… On: nothing\nπŸ”• Off: particulates") + assert title({"gusts", "uv"}, KINDS[:4]) == f"{TITLE}\nβœ… On: rain, temperature crossing\nπŸ”• Off: gusts, UV" + assert title(set(), ["particulates"]).endswith("πŸ”• Off: nothing") and title({"particulates"}, ["particulates"]).endswith("βœ… On: nothing\nπŸ”• Off: particulates") def test_state_mutes_one_type_or_all_and_old_files_load(tmp_path): @@ -204,7 +204,7 @@ async def reply_text(body, **kw): chat = NS(type="private", id=1, title=None) update = NS(effective_message=NS(reply_text=reply_text, chat_id=1), effective_chat=chat, effective_user=NS(full_name="Rob", username="rob")) await bot.on_alerts(update, NS(args=[])) - assert replies[0][0].startswith(TITLE) and "βœ… On: rain, rain predicted, gusts, UV, temperature crossing, particulates, pollen & asthma, forecast changes" in replies[0][0] + assert replies[0][0].startswith(TITLE) and "βœ… On: rain, gusts, UV, temperature crossing, particulates, pollen & asthma, forecast changes" in replies[0][0] assert rows(replies[0][1]["reply_markup"])[0] == [("βž• Sub β–Έ", "al:open:sub"), ("βž– Unsub β–Έ", "al:open:unsub"), ("βš™οΈ Settings β–Έ", "al:open:set")] await bot.on_alerts(update, NS(args=["off"])) assert not state.alerts_on(1) and "βœ… On: nothing" in replies[1][0] @@ -213,7 +213,7 @@ async def reply_text(body, **kw): async def test_every_monitors_alert_names_its_type(tmp_path): - from lib.alerts import AirMonitor, ForecastMonitor, PollenMonitor, WeatherMonitor + from lib.alerts import ParticulatesMonitor, ForecastMonitor, PollenAndAsthmaMonitor, WeatherMonitor from tests.test_alerts import FakePollen, FakeStation, air_reading, gusts, rain, uv from tests.test_forecast_alert import FakeForecast, day state = AlertState(tmp_path / "s.json") @@ -234,15 +234,15 @@ async def weather(first, then): await weather(uv(3, 5), uv(3, 5, 10)) assert kinds == ["rain", "rain", "gusts", "uv"] kinds.clear() - mask = AirMonitor(NS(tz=TZ, link=None), state, notify) + mask = ParticulatesMonitor(NS(tz=TZ, link=None), state, notify) for pm in (80, 80, 20, 20): await mask.check(air_reading(pm)) - assert kinds == ["air", "air"] + assert kinds == ["particulates", "particulates"] kinds.clear() pollen = FakePollen() pollen.grass = "High" - await PollenMonitor(pollen, state, notify).check() - assert kinds == ["pollen"] + await PollenAndAsthmaMonitor(pollen, state, notify).check() + assert kinds == ["pollen_and_asthma"] seen = [] notifier = Notifier(Sent(), state) real = notifier.to_chat @@ -257,10 +257,10 @@ async def spy(chat_id, text, link=None, kind=None): def test_under_an_alert_the_unsubscribe_list_is_its_type_then_other_which_holds_the_rest_and_all(): - got = rows(keyboard({"pollen", "forecast"}, KINDS, "unsub", parent="uv")) + got = rows(keyboard({"pollen_and_asthma", "forecast"}, KINDS, "unsub", parent="uv")) assert got == [[("βž• Sub β–Έ", "al:open:sub:uv"), ("βž– Unsub β–Ύ", "al:close:uv")], [("● 🧴 UV", "al:off:uv:unsub:uv")], [("Other β–Έ", "al:open:other:uv")]] - other = rows(keyboard({"pollen", "forecast"}, KINDS, "other", parent="uv")) + other = rows(keyboard({"pollen_and_asthma", "forecast"}, KINDS, "other", parent="uv")) assert other[1:3] == [[("● 🧴 UV", "al:off:uv:other:uv")], [("Other β–Ύ", "al:open:unsub:uv")]] listed = [d for r in other[3:-1] for _, d in r] assert listed and all(d.startswith("al:off:") and d.endswith(":other:uv") and ":uv:" not in d for d in listed) # the rest, not its own type @@ -271,13 +271,13 @@ def test_under_an_alert_the_unsubscribe_list_is_its_type_then_other_which_holds_ gone = rows(keyboard({"uv", "gusts"}, KINDS, "unsub", parent="uv")) # its type is already off: a plain list of the rest, with "all" assert "Other" not in str(gone) and gone[-1] == [("πŸ”• Unsubscribe from all", "al:off:all:unsub:uv")] assert len(rows(keyboard(set(KINDS), KINDS, "unsub", parent="uv"))) == 1 # nothing is on at all: nothing listed - sub = rows(keyboard({"pollen", "forecast"}, KINDS, "sub", parent="uv")) # Subscribe is the types that are off, as ever + sub = rows(keyboard({"pollen_and_asthma", "forecast"}, KINDS, "sub", parent="uv")) # Subscribe is the types that are off, as ever assert [t for row in sub[1:-1] for t, _ in row] == ["🌼 pollen & asthma", "πŸ”„ forecast changes"] and sub[-1][0][0] == "πŸ”” Subscribe to all" assert rows(keyboard(set(), KINDS, "unsub"))[1][0][0] == "🌧️ rain" # the /alerts message has no parent: every type that is on def test_the_all_button_is_only_there_when_the_list_has_more_than_one_item(): - assert not any("all" in data for row in rows(keyboard({"uv", "gusts", "rain", "rain_likely", "temps", "air", "pollen"}, KINDS, "unsub")) + assert not any("all" in data for row in rows(keyboard({"uv", "gusts", "rain", "temps", "particulates", "pollen_and_asthma"}, KINDS, "unsub")) for _, data in row) # one type on (forecast changes): no "Unsubscribe from all" assert not any("all" in data for row in rows(keyboard({"uv"}, KINDS, "sub")) for _, data in row) # one type off: no "Subscribe to all" assert rows(keyboard({"uv", "gusts"}, KINDS, "sub"))[-1][0][0] == "πŸ”” Subscribe to all" @@ -320,7 +320,7 @@ def test_every_alert_type_has_its_emoji_and_every_button_in_the_lists_starts_wit from lib.alerts.menu import EMOJI, label assert set(EMOJI) == set(LABELS) # a new type cannot be added without one assert len(set(EMOJI.values())) == len(EMOJI) and all(not e.isascii() for e in EMOJI.values()) # all different, all emoji - assert label("rain") == "🌧️ rain" and label("irrigation") == "πŸͺ« irrigation battery" and label("irrigation_pause") == "β˜” irrigation pause" + assert label("rain") == "🌧️ rain" and label("irrigation_battery") == "πŸͺ« irrigation battery" and label("irrigation_pause") == "β˜” irrigation pause" everything = list(LABELS) for section in ("sub", "unsub"): for parent in (None, "uv"): @@ -455,3 +455,14 @@ async def test_the_sound_lists_have_an_all_button(tmp_path): assert rows(q.edits[0][1])[-1] == [("πŸ”” Enable all", "al:snd:all:on:uv")] await press(bot, Query("al:snd:all:on:uv", text="β˜€οΈ UV 10")) assert "uv" in state.loud(1) and "rain" in state.loud(1) and "gusts" not in state.loud(1) + + +def test_every_alert_type_sent_is_registered_and_each_monitor_declares_what_it_sends(): + import re + from pathlib import Path + from lib import alerts + monitors = [alerts.WeatherMonitor, alerts.ParticulatesMonitor, alerts.ForecastMonitor, alerts.PollenAndAsthmaMonitor, alerts.IrrigationMonitor] + assert sorted(k for m in monitors for k in m.KINDS) == sorted(LABELS) # every type has exactly one monitor + for m in monitors: + sent = set(re.findall(r'kind="(\w+)"', Path(__import__(m.__module__, fromlist=["x"]).__file__).read_text())) + assert sent == set(m.KINDS), m.__name__ diff --git a/tests/test_alerts.py b/tests/test_alerts.py index e45e809..1093e3e 100644 --- a/tests/test_alerts.py +++ b/tests/test_alerts.py @@ -1,8 +1,7 @@ import time from datetime import datetime, timedelta, timezone -from lib.alerts import AirMonitor, AlertState, Notifier, WeatherMonitor, with_footer -from lib.ecowitt import outlook +from lib.alerts import ParticulatesMonitor, AlertState, Notifier, WeatherMonitor, with_footer from tests.fakes import TZ T0 = int(datetime(2026, 9, 29, 13, 0, tzinfo=TZ).timestamp()) # 1pm Melbourne, a settled day @@ -96,14 +95,6 @@ async def test_a_live_rate_alone_starts_it_and_a_failed_live_call_is_ignored(tmp assert len(sent) == 1 and "3 mm/h" in sent[0] -def test_the_status_outlook_says_raining_now_or_likely_soon_or_nothing(): - assert outlook.rain_outlook(rain(0, 0, 1), TZ, 145.0) == "raining now (1.2 mm/h)" - assert outlook.rain_outlook(rain(0, 0, 0), TZ, 145.0) is None - likely = outlook.rain_outlook(pressure_rows(4.0, 14), TZ, 145.0) - assert likely.startswith("rain looks likely soon: pressure down") and "not an official forecast" in likely - assert outlook.rain_outlook(pressure_rows(0.2, 14), TZ, 145.0) is None - - def gusts(*values): return [(T0 + i * 300, {"wind.wind_gust": v}) for i, v in enumerate(values)] @@ -138,48 +129,6 @@ async def test_old_gusts_are_not_announced_after_a_restart(tmp_path): assert sent == [] -def pressure_rows(drop, hour_local, hum=93.0, dew_rise=0.0, temp=15.0): - """3 hours of readings ending at the given local hour, pressure falling `drop` hPa in total.""" - end = int(datetime(2026, 9, 29, hour_local, 0, tzinfo=TZ).timestamp()) - n = 37 - return rows(n, start=end - 36 * 300, **{ - "pressure.relative": lambda i: 1015 - drop * i / 36, "outdoor.humidity": hum, - "outdoor.temperature": temp, "outdoor.dew_point": lambda i: temp - 1 + dew_rise * i / 36, - "rainfall.rain_rate": 0.0, "wind.wind_gust": 10.0}) - - -def test_daytime_fall_with_humid_air_predicts_rain(): - out = outlook.assess_rain(pressure_rows(3.2, 15), TZ, 145.0) - assert out and out.score >= outlook.PREDICT_MIN_SCORE and not out.night - - -def test_the_overnight_pressure_dip_and_cooling_are_not_rain(): - # ~4am: pressure sags with the daily tide and the air sits near its dew point, as every night - for drop in (0.5, 1.2, 1.6): - out = outlook.assess_rain(pressure_rows(drop, 4), TZ, 145.0) - assert out is None or out.score < outlook.PREDICT_MIN_SCORE, (drop, out) - - -def test_a_real_night_front_still_counts(): - out = outlook.assess_rain(pressure_rows(4.5, 3, dew_rise=2.5), TZ, 145.0) - assert out and out.night and out.score >= outlook.PREDICT_MIN_SCORE - - -async def test_rain_likely_alerts_at_most_every_6_hours(tmp_path): - m, state, sent = monitor(tmp_path, pressure_rows(3.5, 15)) - await m.check() - await m.check() - assert [s for s in sent if "likely" in s].__len__() == 1 - - -async def test_no_rain_prediction_while_raining(tmp_path): - data = pressure_rows(3.5, 15) - data[-1][1]["rainfall.rain_rate"] = 2.0 - m, state, sent = monitor(tmp_path, data) - await m.check() - assert not [s for s in sent if "likely" in s] - - def temps(outdoor, indoor, ts=T0): return [(ts, {"outdoor.temperature": outdoor, "indoor.temperature": indoor})] @@ -269,7 +218,7 @@ async def test_mask_alert_needs_two_bad_checks_and_pairs_with_all_clear(tmp_path async def notify(text, link=None, **kw): sent.append((text, link)) - mon = AirMonitor(FakeAir(), state, notify) + mon = ParticulatesMonitor(FakeAir(), state, notify) await mon.check(air_reading(80)) assert not sent # one puff of smoke isn't enough await mon.check(air_reading(20)) @@ -290,7 +239,7 @@ async def test_mask_threshold_is_official_aqi_151(tmp_path): async def notify(text, link=None, **kw): sent.append(text) - mon = AirMonitor(FakeAir(), state, notify) + mon = ParticulatesMonitor(FakeAir(), state, notify) for _ in range(3): await mon.check(air_reading(55.4)) # AQI 150: unhealthy for sensitive groups only assert not sent @@ -304,7 +253,7 @@ async def test_stale_air_readings_are_ignored(tmp_path): async def notify(text, link=None, **kw): sent.append(text) - mon = AirMonitor(FakeAir(), state, notify) + mon = ParticulatesMonitor(FakeAir(), state, notify) for _ in range(3): await mon.check(air_reading(200, age=timedelta(hours=3))) assert not sent @@ -401,13 +350,13 @@ def current(self): def pollen_monitor(tmp_path): - from lib.alerts import PollenMonitor + from lib.alerts import PollenAndAsthmaMonitor state, sent = AlertState(tmp_path / "s.json"), [] async def notify(text, link=None, **kw): sent.append(text) pollen = FakePollen() - return PollenMonitor(pollen, state, notify), pollen, sent + return PollenAndAsthmaMonitor(pollen, state, notify), pollen, sent async def test_high_pollen_and_asthma_alert_once_per_metric_and_day(tmp_path): @@ -435,12 +384,12 @@ async def test_high_pollen_and_asthma_alert_once_per_metric_and_day(tmp_path): async def test_a_restart_does_not_repeat_an_alert_and_a_missing_forecast_or_stale_page_is_ignored(tmp_path): - from lib.alerts import PollenMonitor + from lib.alerts import PollenAndAsthmaMonitor mon, pollen, sent = pollen_monitor(tmp_path) pollen.grass = "High" await mon.check() assert len(sent) == 1 - again = PollenMonitor(pollen, mon.state, mon.notify) # the state file keeps what was sent + again = PollenAndAsthmaMonitor(pollen, mon.state, mon.notify) # the state file keeps what was sent await again.check() assert len(sent) == 1 pollen.asthma = None # off-season: nothing to say diff --git a/tests/test_bot.py b/tests/test_bot.py index f62b3a4..f655542 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -510,3 +510,50 @@ async def send_chat_action(*a, **k): pass await Bot(NS(tz=TZ), Agent(), sources, None).respond(update, NS(bot=NS(send_chat_action=send_chat_action, id=99)), text) assert sent == ["t"], text + + +async def test_the_report_carries_a_refresh_button_that_redraws_it_in_place(): + import json + from lib import report_menu + + class Tools: + reading = "12.3 ℃" + + async def call(self, name, raw, turn=None): + return {"weather_now": json.dumps({"outdoor": {"temperature": self.reading}}), + "air_quality": json.dumps({"pm10": {"value": 3.0, "unit": "Β΅g/mΒ³", "rating": "🟒 good"}})}[name] + + class Agent: + tools = Tools() + + class Query: + data, edits, toast = report_menu.REFRESH, [], None + + async def answer(self, text=None, **kw): + self.toast = text + + async def edit_message_text(self, text, reply_markup=None, **kw): + self.edits.append((text, reply_markup)) + self.entities = kw.get("entities") + sources = [NS(name=n, wants=lambda t: True, poke=lambda: None, describe=lambda n=n: n) for n in ("Ecowitt", "AirGradient")] + sent = [] + + async def reply_text(body, **kw): + sent.append((body, kw.get("reply_markup"), kw.get("entities"))) + msg = NS(chat_id=1, message_thread_id=None, is_topic_message=False, reply_text=reply_text, + chat=NS(type="private", title=None), from_user=NS(full_name="Rob"), reply_to_message=None) + update = NS(effective_message=msg, effective_chat=msg.chat, effective_user=NS(username="rob", full_name="Rob")) + + async def send_chat_action(*a, **k): + pass + context = NS(bot=NS(send_chat_action=send_chat_action, id=99)) + bot = Bot(NS(tz=TZ), Agent(), sources, None) + await bot.respond(update, context, "report") + await bot.respond(update, context, "weather now") + assert [b.callback_data for b in sent[0][1].inline_keyboard[0]] == [report_menu.REFRESH] and sent[1][1] is None + assert [e.type for e in sent[0][2]] == ["bold", "bold"] and sent[1][2] is None # the report's headings are bold; "weather now" has none + Tools.reading = "13.0 ℃" + query = Query() + await bot.on_report_button(NS(callback_query=query), context) + assert query.toast is None and "13.0 Β°C" in query.edits[0][0] and query.edits[0][1].inline_keyboard[0][0].callback_data == report_menu.REFRESH + assert [e.type for e in query.entities] == ["bold", "bold"] diff --git a/tests/test_check_rain.py b/tests/test_check_rain.py index 776f984..1f0f8d1 100644 --- a/tests/test_check_rain.py +++ b/tests/test_check_rain.py @@ -37,9 +37,9 @@ async def test_healthy_data_is_reported_as_matching(station, capsys): def average_the_thirty_minute_rain(eco, factor): """What averages look like: no daily_high field, and values a fraction of the real total.""" with eco.cache._lock: - eco.cache.db.execute("DELETE FROM points WHERE cycle='30min' AND grp='rainfall' AND field IN ('daily_high','daily_low')") - eco.cache.db.execute("UPDATE points SET value = CAST(value AS REAL) * ? WHERE cycle='30min' AND grp='rainfall' AND field='daily'", - (factor,)) + ids = "(SELECT id FROM series WHERE cycle='30min' AND grp='rainfall' AND field {})" + eco.cache.db.execute("DELETE FROM points WHERE series IN " + ids.format("IN ('daily_high','daily_low')")) + eco.cache.db.execute("UPDATE points SET value = CAST(value AS REAL) * ? WHERE series IN " + ids.format("= 'daily'"), (factor,)) eco.cache.db.commit() diff --git a/tests/test_contract.py b/tests/test_contract.py index b18b99b..239cbd7 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -23,22 +23,25 @@ def tables(path) -> dict[str, list[str]]: return {n: [r[1] for r in db.execute(f"PRAGMA table_info({n})")] for n in names} -def test_the_cache_schemas_are_unchanged(): - assert tables(DATA / "ecowitt_cache.sqlite") == { +def test_the_cache_schemas_are_unchanged(tmp_path): + layout = { "coverage": ["mac", "cycle", "grp", "start", "end"], - "fields": ["mac", "cycle", "grp", "field", "unit"], "meta": ["key", "value"], - "points": ["mac", "cycle", "grp", "field", "ts", "value"]} + "points": ["series", "ts", "value"], + "series": ["id", "mac", "cycle", "grp", "field", "unit"]} + assert tables(DATA / "ecowitt_cache.sqlite") == layout + HistoryCache(tmp_path / "fresh.sqlite", api.UNITS).close() # a new file gets the same tables + assert tables(tmp_path / "fresh.sqlite") == layout assert tables(DATA / "airgradient_cache.sqlite") == { "days": ["loc", "day", "n"], "readings": ["loc", "ts", "pm2_5", "pm10", "pm1", "co2", "voc_index", "nox_index"]} -def test_a_cache_written_by_an_earlier_version_still_opens_and_reads(tmp_path): +def test_a_recorded_cache_still_opens_and_reads(tmp_path): shutil.copy(DATA / "ecowitt_cache.sqlite", tmp_path / "eco.sqlite") shutil.copy(DATA / "airgradient_cache.sqlite", tmp_path / "air.sqlite") cache = HistoryCache(tmp_path / "eco.sqlite", api.UNITS) # the same units: nothing is cleared - now = cache._query("SELECT MAX(ts) FROM points WHERE cycle = '30min'")[0][0] # the fixture's own newest reading, so the test does not age + now = cache._query("SELECT MAX(p.ts) FROM points p JOIN series s ON s.id = p.series WHERE s.cycle = '30min'")[0][0] # the fixture's own newest reading, so the test does not age for cycle, at_least in (("5min", 100), ("30min", 100), ("1day", 1)): temperature, = cache.slots(MAC, cycle, "outdoor", ["temperature"], now - 5 * 86400, now) assert len(temperature) >= at_least, cycle @@ -58,7 +61,7 @@ async def test_every_tool_result_keeps_its_shape(tmp_path, archived_cache): span = {"start_date": day(9), "end_date": f"{today - timedelta(days=2)} 23:59:59"} call = lambda i, args: eco.tools[i].handler(args) now = json.loads(await call(0, {"groups": "outdoor,indoor,pressure,wind,rainfall"})) - assert set(now) - {"rain_outlook"} == {"time", "outdoor", "emoji"} # the outlook line only appears when the data suggests rain + assert set(now) == {"time", "outdoor", "emoji"} history = json.loads(await call(1, {"groups": "outdoor", **span})) assert set(history) - {"chart"} == {"period", "series"} # "chart": a hint that appears when one was drawn assert set(history["series"]["outdoor.temperature"]) == { diff --git a/tests/test_data_dir.py b/tests/test_data_dir.py new file mode 100644 index 0000000..a1abe68 --- /dev/null +++ b/tests/test_data_dir.py @@ -0,0 +1,96 @@ +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from lib.config import ROOT, Config, data_dir + +NAMES = ("bot_state.json", "ecowitt_cache.sqlite", "airgradient_cache.sqlite", "conditions_cache.sqlite") + + +def configured(monkeypatch, **env): + for key, value in {"TELEGRAM_BOT_TOKEN": "t", "XAI_API_KEY": "x", "ECOWITT_API_KEY": "a", "ECOWITT_APP_KEY": "b", **env}.items(): + monkeypatch.setenv(key, value) + return Config.from_env() + + +def paths(cfg): + return (cfg.state_path, cfg.cache_path, cfg.air_cache_path, cfg.conditions_cache_path) + + +def test_without_data_dir_the_files_stay_where_they_have_always_been(monkeypatch): + monkeypatch.delenv("DATA_DIR", raising=False) + monkeypatch.delenv("STATE_DIRECTORY", raising=False) + assert data_dir() == ROOT + assert paths(configured(monkeypatch)) == tuple(ROOT / n for n in NAMES) + monkeypatch.setenv("DATA_DIR", " ") + assert data_dir() == ROOT # blank is the same as unset + + +def test_data_dir_moves_the_state_and_all_three_caches_and_is_created(monkeypatch, tmp_path): + target = tmp_path / "a" / "data" + cfg = configured(monkeypatch, DATA_DIR=str(target)) + assert paths(cfg) == tuple(target / n for n in NAMES) and target.is_dir() + + +def test_a_relative_data_dir_is_taken_from_the_working_directory(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + cfg = configured(monkeypatch, DATA_DIR="state") + assert cfg.state_path == tmp_path / "state" / "bot_state.json" and (tmp_path / "state").is_dir() + assert data_dir() == Path(tmp_path / "state").resolve() + + +def test_systemd_state_directory_is_the_default_and_data_dir_wins(monkeypatch, tmp_path): + first, second = tmp_path / "one", tmp_path / "two" + monkeypatch.delenv("DATA_DIR", raising=False) + monkeypatch.setenv("STATE_DIRECTORY", f"{first}:{second}") # systemd may list several: the first is used + assert data_dir() == first and paths(configured(monkeypatch))[0] == first / "bot_state.json" and first.is_dir() + monkeypatch.setenv("DATA_DIR", str(second)) + assert data_dir() == second + + +def test_a_symlinked_data_dir_is_used_as_written(monkeypatch, tmp_path): + real = tmp_path / "private" + real.mkdir() + link = tmp_path / "ecobot" + link.symlink_to(real) + monkeypatch.setenv("DATA_DIR", str(link)) + assert data_dir() == link and configured(monkeypatch).state_path == link / "bot_state.json" + + +def test_a_folder_that_cannot_be_made_stops_the_start_with_a_plain_message(monkeypatch, tmp_path): + blocker = tmp_path / "file" + blocker.write_text("not a folder") + with pytest.raises(SystemExit) as stop: + configured(monkeypatch, DATA_DIR=str(blocker / "data")) + assert "not writable" in str(stop.value) and "DATA_DIR" in str(stop.value) + + +def test_matplotlib_keeps_its_cache_in_the_data_folder_unless_told_otherwise(tmp_path): + code = "import os, ecobot; print(os.environ['MPLCONFIGDIR'])" + env = {k: v for k, v in os.environ.items() if k != "MPLCONFIGDIR"} + out = subprocess.run([sys.executable, "-c", code], cwd=ROOT, capture_output=True, text=True, env={**env, "DATA_DIR": str(tmp_path)}) + assert out.stdout.strip() == str(tmp_path / "matplotlib"), out.stderr[-500:] + out = subprocess.run([sys.executable, "-c", code], cwd=ROOT, capture_output=True, text=True, + env={**env, "DATA_DIR": str(tmp_path), "MPLCONFIGDIR": str(tmp_path / "mine")}) + assert out.stdout.strip() == str(tmp_path / "mine") + + +def test_the_running_commit_is_read_without_running_git(tmp_path): + from ecobot import commit_in + git = lambda *a: subprocess.run(["git", *a], cwd=tmp_path, check=True, capture_output=True, text=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t"}).stdout.strip() + assert commit_in(tmp_path / ".git") == "unknown" # no repository + git("init", "-q", "-b", "main") + git("commit", "-q", "--allow-empty", "-m", "one") + want = git("rev-parse", "--short=7", "HEAD") + assert commit_in(tmp_path / ".git") == want # a loose branch ref + git("pack-refs", "--all", "--prune") + assert commit_in(tmp_path / ".git") == want # a packed ref + git("checkout", "-q", "--detach") + assert commit_in(tmp_path / ".git") == want # a detached HEAD + (tmp_path / ".git" / "HEAD").write_text("garbage") + assert commit_in(tmp_path / ".git") == "unknown" diff --git a/tests/test_ecowitt.py b/tests/test_ecowitt.py index 883bb07..ba1f84c 100644 --- a/tests/test_ecowitt.py +++ b/tests/test_ecowitt.py @@ -8,7 +8,7 @@ from lib.tools import Turn from lib.ecowitt import Ecowitt from lib.ecowitt import api as ecowitt_api -from tests.fakes import MAC, archived_station, config, ecowitt_transport +from tests.fakes import MAC, FakeEcowitt, archived_station, config, ecowitt_transport @pytest.fixture @@ -568,27 +568,6 @@ async def wind_chart(days, **extra): await eco.close() -async def test_weather_now_carries_the_rain_outlook_when_it_is_raining_or_likely(tmp_path, monkeypatch): - transport, _ = ecowitt_transport() - eco = Ecowitt(config(tmp_path), transport=transport) - await eco.start() - now = int(datetime.now(eco.tz).timestamp()) - wet = [(now - 300 * i, {"rainfall.rain_rate": 1.5, "rainfall.daily": 2.0}) for i in range(2, -1, -1)] - dry = [(now - 300 * i, {"rainfall.rain_rate": 0.0, "rainfall.daily": 2.0}) for i in range(2, -1, -1)] - rows = {"value": wet} - - async def recent(hours): - return rows["value"] - monkeypatch.setattr(eco, "recent", recent) - out = json.loads(await eco.tools[0].handler({"groups": "outdoor,rainfall"})) - assert out["rain_outlook"] == "raining now (1.5 mm/h)" - rows["value"] = dry - assert "rain_outlook" not in json.loads(await eco.tools[0].handler({"groups": "outdoor,rainfall"})) - rows["value"] = wet - assert "rain_outlook" not in json.loads(await eco.tools[0].handler({"groups": "outdoor"})) # no rain group asked for - await eco.close() - - def test_the_rolling_mean_smooths_a_staircase_without_moving_timestamps_or_crossing_gaps(): from lib.timeutil import rolling_mean assert rolling_mean({0: 5.0, 300: 5.0, 600: 5.0}, 450) == {0: 5.0, 300: 5.0, 600: 5.0} @@ -994,7 +973,7 @@ async def test_weather_all_week_is_a_stack_of_every_reading_the_station_has(tmp_ def test_a_reading_named_as_a_group_becomes_its_group_and_an_unknown_name_is_dropped(): - from lib.ecowitt.station import parse_groups + from lib.ecowitt.source import parse_groups assert parse_groups("pressure,humidity") == ["pressure", "outdoor"] assert parse_groups("rain, dew_point, wind, outdoor") == ["rainfall", "outdoor", "wind"] assert parse_groups("outdoor.temperature,bogus") == ["outdoor"] and parse_groups("bogus") == ["outdoor", "indoor"] @@ -1104,11 +1083,11 @@ async def test_a_failing_live_rain_reading_warns_once_then_says_when_it_recovers async def realtime(mac, groups): return answers.pop(0) monkeypatch.setattr(station.api, "realtime", realtime) - with caplog.at_level("INFO", logger="lib.ecowitt.station"): + with caplog.at_level("INFO", logger="lib.ecowitt.source"): assert await station.live_rain() is None and await station.live_rain() is None assert await station.live_rain() == (100, {"rainfall.rain_rate": 1.5, "rainfall.daily": 2.0}) await station.live_rain() - assert [r.levelname for r in caplog.records] == ["WARNING", "INFO"] and "recovered" in caplog.records[1].message + assert [r.levelname for r in caplog.records] == ["WARNING", "INFO"] and "Recovered" in caplog.records[1].message async def test_a_waiting_person_goes_before_the_background_requests_queued_ahead_of_them(): @@ -1213,3 +1192,33 @@ async def test_rain_older_than_the_30_minute_readings_is_drawn_from_the_daily_re q.turn = Turn(text="rain by month for 2 months") monthly = await q._rain_bars() assert monthly.per == "month" and monthly.values and round(sum(monthly.y), 1) == 120.0 and len(monthly.x) in (2, 3) + + +async def test_an_empty_history_answer_is_explained_in_the_log_and_the_start_says_which_cache_it_uses(tmp_path, caplog): + import logging + + class Silent(FakeEcowitt): + @staticmethod + def _history(p): + return {} + + transport, _ = ecowitt_transport(Silent()) + eco = Ecowitt(config(tmp_path), transport=transport) + with caplog.at_level(logging.INFO): + await eco.start() + today = datetime.now(eco.tz).date() + reply = await eco.tools[1].handler({"groups": "outdoor", "start_date": f"{today - timedelta(days=1)} 00:00:00", + "end_date": f"{today} 23:59:59"}) + assert reply.startswith("Error: no history data returned") + text = caplog.text + assert "History cache: " in text and str(eco.cache.path) in text + assert "History query empty" in text and "readings held for the window {'5min': 0, '30min': 0, '1day': 0}" in text + + +def test_the_live_summary_gives_rain_first_then_a_few_readings_and_skips_what_is_missing(): + from lib.ecowitt.source import live_summary + cell = lambda v, u: {"value": v, "unit": u} + data = {"rainfall": {"rain_rate": cell("0.0", "mm/hr"), "daily": cell("0.8", "mm")}, "outdoor": {"temperature": cell("20.3", "℃"), "humidity": cell("62", "%")}, + "wind": {"wind_speed": cell("12", "km/h")}, "pressure": {"relative": cell("1013.2", "hPa")}} + assert live_summary(data) == "rain 0.0 mm/h, 0.8 mm today, 20.3 Β°C, 62 %, wind 12 km/h, 1013.2 hPa" + assert live_summary({"rainfall": {"daily": cell("0.8", "mm")}}) == "0.8 mm today" and live_summary({}) == "no readings" diff --git a/tests/test_forecast.py b/tests/test_forecast.py index 86ea375..e0f9da8 100644 --- a/tests/test_forecast.py +++ b/tests/test_forecast.py @@ -95,6 +95,7 @@ async def test_the_place_is_the_configured_name(tmp_path): forecast, _ = make(tmp_path, place="Geelong") await forecast.start() assert json.loads(await forecast.handle({}))["place"] == "Geelong" + assert json.loads(await forecast.handle({}))["time"] # when it was fetched, for the report's heading await forecast.close() diff --git a/tests/test_irrigation.py b/tests/test_irrigation.py index c4b94a8..0acd6c7 100644 --- a/tests/test_irrigation.py +++ b/tests/test_irrigation.py @@ -200,12 +200,12 @@ def test_the_irrigation_button_takes_wind_s_place_only_when_configured_and_chang def test_the_alert_list_gets_irrigation_battery_only_when_there_is_a_controller(): - assert available_kinds({"Ecowitt", "Irrigation"})[-2:] == ["irrigation", "irrigation_pause"] - assert "irrigation" not in available_kinds({"Ecowitt"}) and "irrigation_pause" not in available_kinds({"Ecowitt"}) + assert available_kinds({"Ecowitt", "Irrigation"})[-2:] == ["irrigation_battery", "irrigation_pause"] + assert "irrigation_battery" not in available_kinds({"Ecowitt"}) and "irrigation_pause" not in available_kinds({"Ecowitt"}) from lib.alerts.menu import LABELS, keyboard - assert LABELS["irrigation"] == "irrigation battery" and LABELS["irrigation_pause"] == "irrigation pause" - unsub = rows(keyboard(set(), ["rain", "irrigation"], "unsub")) - assert ("πŸͺ« irrigation battery", "al:off:irrigation:unsub") in [b for row in unsub for b in row] + assert LABELS["irrigation_battery"] == "irrigation battery" and LABELS["irrigation_pause"] == "irrigation pause" + unsub = rows(keyboard(set(), ["rain", "irrigation_battery"], "unsub")) + assert ("πŸͺ« irrigation battery", "al:off:irrigation_battery:unsub") in [b for row in unsub for b in row] # ---------- the daily check ---------- @@ -294,7 +294,7 @@ async def test_the_check_delays_a_day_when_more_than_1_mm_fell_or_is_likely(tmp_ assert device.commands == ([("delay", "24h")] if delayed else []) assert [(text.startswith("β˜” Irrigation paused for 24 hours: ") and text.endswith("."), kw) for text, kw in sent] == ( [(True, {"kind": "irrigation_pause"})] if delayed else []) # told once when it paused, never otherwise - assert state.monitor["irrigation"]["checked"] == "2026-10-08" + assert state.monitor["irrigation_monitor"]["checked"] == "2026-10-08" async def test_a_delay_already_set_is_never_shortened_or_repeated(tmp_path): @@ -309,7 +309,7 @@ async def test_a_battery_under_5_percent_is_told_once_a_day_to_the_irrigation_al m, state, sent = daily(tmp_path, device) await m.check(NOW) await m.check(NOW) - assert len(sent) == 1 and sent[0][1] == {"kind": "irrigation"} and "4%" in sent[0][0] and "πŸͺ«" in sent[0][0] + assert len(sent) == 1 and sent[0][1] == {"kind": "irrigation_battery"} and "4%" in sent[0][0] and "πŸͺ«" in sent[0][0] await m.check(datetime(2026, 10, 9, 18, 0)) # the next day: again, until it is charged assert len(sent) == 2 device.status_now["battery_percentage"] = 5 # 5% is not below 5% @@ -324,16 +324,16 @@ async def delay(self, duration): m, state, sent = daily(tmp_path, Failing(battery_percentage=2), None, Forecast(90, 90)) with pytest.raises(IrrigationError): await m.check(NOW) - assert len(sent) == 1 and "checked" not in state.monitor["irrigation"] # not recorded: the next try repeats the check + assert len(sent) == 1 and "checked" not in state.monitor["irrigation_monitor"] # not recorded: the next try repeats the check with pytest.raises(IrrigationError): await m.check(NOW) - assert len(sent) == 1 and sent[0][1] == {"kind": "irrigation"} # no "paused" alert for a pause that was not made + assert len(sent) == 1 and sent[0][1] == {"kind": "irrigation_battery"} # no "paused" alert for a pause that was not made def test_a_check_is_due_once_a_day_after_the_hour_and_the_next_is_the_following_18_00(tmp_path): m, state, _ = daily(tmp_path, Device()) assert not m.due(datetime(2026, 10, 8, 17, 59)) and m.due(datetime(2026, 10, 8, 18, 0)) and m.due(datetime(2026, 10, 8, 23, 0)) - state.monitor["irrigation"] = {"checked": "2026-10-08"} + state.monitor["irrigation_monitor"] = {"checked": "2026-10-08"} assert not m.due(datetime(2026, 10, 8, 23, 0)) and not m.due(datetime(2026, 10, 9, 3, 0)) and m.due(datetime(2026, 10, 9, 18, 0)) assert m.seconds_until_next(datetime(2026, 10, 8, 17, 0)) == 3600 and m.seconds_until_next(datetime(2026, 10, 8, 18, 0)) == 86400 assert m.seconds_until_next(datetime(2026, 10, 8, 23, 30)) == 18.5 * 3600 @@ -356,7 +356,7 @@ async def fake_sleep(seconds): with pytest.raises(asyncio.CancelledError): await m.run() assert sleeps[0] == 1800 and device.commands == [("delay", "24h")] # tried, failed, waited 30 minutes, tried, did it - assert state.monitor["irrigation"]["checked"] == "2026-10-08" + assert state.monitor["irrigation_monitor"]["checked"] == "2026-10-08" assert sleeps[1] == 86400 # then until 18:00 tomorrow (now is exactly 18:00 here) @@ -492,19 +492,19 @@ async def test_the_alerts_menu_toggles_each_alert_type_for_this_chat_alone_and_s q = await press(bot, "ir:open:alerts") assert q.edits[0][0] == "markup" and [r[0][0] for r in rows(q.edits[0][1])[2:]] == ["πŸ”• Disable battery alerts", "πŸ”• Disable pause alerts"] q = await press(bot, "ir:batt:off") - assert state.muted(1) == {"irrigation"} and q.toast == "Irrigation battery alerts off in this chat" + assert state.muted(1) == {"irrigation_battery"} and q.toast == "Irrigation battery alerts off in this chat" assert rows(q.edits[0][2])[2:] == [[("πŸ”” Enable battery alerts", "ir:batt:on")], [("πŸ”• Disable pause alerts", "ir:pause:off")]] # still open assert q.edits[0][1].endswith("pause: not paused (alerts: on)\nbattery: 100% πŸ”‹ (alerts: off)") # the status text follows q = await press(bot, "ir:pause:off") - assert state.muted(1) == {"irrigation", "irrigation_pause"} and q.toast == "Irrigation pause alerts off in this chat" + assert state.muted(1) == {"irrigation_battery", "irrigation_pause"} and q.toast == "Irrigation pause alerts off in this chat" assert q.edits[0][1].endswith("pause: not paused (alerts: off)\nbattery: 100% πŸ”‹ (alerts: off)") and rows(q.edits[0][2])[3] == [("πŸ”” Enable pause alerts", "ir:pause:on")] q = await press(bot, "ir:batt:on") assert state.muted(1) == {"irrigation_pause"} and q.toast == "Irrigation battery alerts on in this chat" # the other type is untouched q = await press(bot, "ir:pause:on") - assert state.muted(1) == set() and state.alert_chats("irrigation") == [1] == state.alert_chats("irrigation_pause") - for kind, prefix in (("irrigation", "batt"), ("irrigation_pause", "pause")): # whatever a button says, pressing it does that + assert state.muted(1) == set() and state.alert_chats("irrigation_battery") == [1] == state.alert_chats("irrigation_pause") + for kind, prefix in (("irrigation_battery", "batt"), ("irrigation_pause", "pause")): # whatever a button says, pressing it does that for _ in range(2): - alerts = rows(menu.keyboard(state.subscribed(1, "irrigation"), state.subscribed(1, "irrigation_pause"), "alerts")) + alerts = rows(menu.keyboard(state.subscribed(1, "irrigation_battery"), state.subscribed(1, "irrigation_pause"), "alerts")) text, data = next(b for row in alerts[2:] for b in row if b[1].startswith(f"ir:{prefix}:")) await press(bot, data) assert state.subscribed(1, kind) is text.startswith("πŸ”” Enable"), (text, data) @@ -541,7 +541,7 @@ async def test_the_handler_is_registered_and_the_alert_list_includes_irrigation_ plain.register(NS(add_handler=lambda h, group=0: added_plain.append(h), add_error_handler=lambda h: None)) assert len(added) == len(added_plain) # one entry point for every button, with or without a controller assert "ir:" in bot.buttons() and "ir:" not in plain.buttons() and {"al:", "pd:"} <= set(plain.buttons()) - assert "irrigation" in bot._alert_kinds() and "irrigation" not in plain._alert_kinds() + assert "irrigation_battery" in bot._alert_kinds() and "irrigation_battery" not in plain._alert_kinds() assert bot.keyboard_version == templates.version(True) != plain.keyboard_version == templates.VERSION @@ -687,9 +687,9 @@ async def delay(self, duration): m, state, sent = daily(tmp_path, device, None, Forecast(90, 90)) with pytest.raises(IrrigationError): await m.check(NOW) - assert sent == [] and "checked" not in state.monitor["irrigation"] # nothing told for a pause that was not made + assert sent == [] and "checked" not in state.monitor["irrigation_monitor"] # nothing told for a pause that was not made await m.check(NOW) - assert device.commands == [("delay", "24h")] and len(sent) == 1 and state.monitor["irrigation"]["checked"] == "2026-10-08" + assert device.commands == [("delay", "24h")] and len(sent) == 1 and state.monitor["irrigation_monitor"]["checked"] == "2026-10-08" async def test_the_pause_alert_reaches_every_subscribed_chat_with_the_usual_menu(tmp_path): @@ -698,7 +698,7 @@ async def test_the_pause_alert_reaches_every_subscribed_chat_with_the_usual_menu for chat in (1, 2, 3, -7): state.add_chat(chat, str(chat)) state.set_kind(2, "irrigation_pause", False) # this chat turned the pause alerts off - state.set_kind(3, "irrigation", False) # ... and this one only the battery alerts: it still gets the pause + state.set_kind(3, "irrigation_battery", False) # ... and this one only the battery alerts: it still gets the pause class Telegram: async def send_message(self, chat_id, text, **kw): @@ -713,7 +713,7 @@ async def send_message(self, chat_id, text, **kw): from lib.alerts.menu import keyboard kinds = available_kinds({"Ecowitt", "AirGradient", "Pollen", "Forecast", "Irrigation"}) for section in (None, "sub", "unsub"): # every button's data fits Telegram's 64 bytes, for the new type too - for parent in (None, "irrigation_pause", "irrigation"): + for parent in (None, "irrigation_pause", "irrigation_battery"): assert all(len(data.encode()) <= 64 for row in rows(keyboard(set(), kinds, section, parent)) for _, data in row) @@ -757,7 +757,7 @@ async def test_every_member_of_a_group_may_use_the_irrigation_buttons(tmp_path): q = await press(bot, "ir:refresh", **group) assert q.toast == "Refreshed" and q.edits q = await press(bot, "ir:batt:off", **group) - assert state.muted(-5) == {"irrigation"} and q.toast == "Irrigation battery alerts off in this chat" + assert state.muted(-5) == {"irrigation_battery"} and q.toast == "Irrigation battery alerts off in this chat" def test_the_error_footer_is_bold_and_positioned_in_utf16_units(): diff --git a/tests/test_migrate_cache.py b/tests/test_migrate_cache.py new file mode 100644 index 0000000..8196fdd --- /dev/null +++ b/tests/test_migrate_cache.py @@ -0,0 +1,87 @@ +"""scripts/migrate_cache.py: the old Ecowitt cache layout (readings keyed by mac, cycle, group, field) becomes the series layout.""" + +import sqlite3 +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +import migrate_cache # noqa: E402 +from lib.ecowitt import api # noqa: E402 +from lib.ecowitt.store import HistoryCache # noqa: E402 + +ROWS = [("M", "5min", "outdoor", "temperature", 100 + 300 * i, f"{10 + i / 10:.1f}") for i in range(5)] + [ + ("M", "5min", "outdoor", "humidity", 100, "94"), ("M", "1day", "rainfall", "daily", 86400, "0.0"), + ("M", "5min", "wind", "gust", 100, "-")] # a field with no `fields` row, and a value that is not a number + + +def old_cache(path: Path): + db = sqlite3.connect(path) + db.executescript(""" + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE points (mac TEXT, cycle TEXT, grp TEXT, field TEXT, ts INTEGER, value TEXT, + PRIMARY KEY (mac, cycle, grp, field, ts)) WITHOUT ROWID; + CREATE TABLE fields (mac TEXT, cycle TEXT, grp TEXT, field TEXT, unit TEXT, PRIMARY KEY (mac, cycle, grp, field)) WITHOUT ROWID; + CREATE TABLE coverage (mac TEXT, cycle TEXT, grp TEXT, start INTEGER, end INTEGER);""") + db.executemany("INSERT INTO points VALUES (?,?,?,?,?,?)", ROWS) + db.executemany("INSERT INTO fields VALUES (?,?,?,?,?)", [("M", "5min", "outdoor", "temperature", "℃"), ("M", "5min", "outdoor", "humidity", "%"), + ("M", "1day", "rainfall", "daily", "mm")]) + db.executemany("INSERT INTO coverage VALUES (?,?,?,?,?)", [("M", "5min", "outdoor", 0, 1000), ("M", "1day", "rainfall", 0, 200000)]) + db.commit() + db.close() + + +def contents(path: Path): + db = sqlite3.connect(path) + points = sorted(db.execute("SELECT s.mac, s.cycle, s.grp, s.field, p.ts, p.value FROM points p JOIN series s ON s.id = p.series").fetchall()) + return points, sorted(db.execute("SELECT mac, cycle, grp, field, unit FROM series").fetchall()), sorted(db.execute("SELECT * FROM coverage").fetchall()) + + +def test_every_value_string_unit_and_coverage_row_survives_and_the_original_is_kept(tmp_path): + path = tmp_path / "eco.sqlite" + old_cache(path) + rows, series, before, after = migrate_cache.migrate(path) + assert rows == len(ROWS) and series == 4 + points, units, coverage = contents(path) + assert points == sorted(ROWS) and coverage == [("M", "1day", "rainfall", 0, 200000), ("M", "5min", "outdoor", 0, 1000)] + assert ("M", "5min", "outdoor", "temperature", "℃") in units and ("M", "5min", "wind", "gust", "") in units # the unmatched point gets a series + assert (tmp_path / "eco.sqlite.old").exists() + assert {r[0] for r in sqlite3.connect(tmp_path / "eco.sqlite.old").execute("SELECT name FROM sqlite_master WHERE type='table'")} >= {"points", "fields"} + cache = HistoryCache(path, {}) # and the bot reads the result + assert cache.load("M", "5min", ["outdoor"], 0, 10_000)["outdoor"]["humidity"] == {"unit": "%", "list": {"100": "94"}} + cache.close() + + +def test_a_second_run_changes_nothing_and_no_backup_leaves_none(tmp_path): + path = tmp_path / "eco.sqlite" + old_cache(path) + migrate_cache.migrate(path, backup=False) + assert not (tmp_path / "eco.sqlite.old").exists() + first = contents(path) + assert migrate_cache.migrate(path)[0] == -1 and contents(path) == first and not (tmp_path / "eco.sqlite.old").exists() + + +def test_a_failure_part_way_leaves_the_original_tables(tmp_path, monkeypatch): + path = tmp_path / "eco.sqlite" + old_cache(path) + real = sqlite3.connect + + class Failing(sqlite3.Connection): + def execute(self, sql, *args): + if sql.startswith("DROP TABLE points_old"): + raise sqlite3.OperationalError("disk full") + return super().execute(sql, *args) + monkeypatch.setattr(migrate_cache.sqlite3, "connect", lambda *a, **k: real(*a, factory=Failing, **k)) + with pytest.raises(sqlite3.OperationalError): + migrate_cache.migrate(path, backup=False) + db = sqlite3.connect(path) + assert {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} >= {"points", "fields"} + assert db.execute("SELECT COUNT(*) FROM points").fetchone()[0] == len(ROWS) and "mac" in [r[1] for r in db.execute("PRAGMA table_info(points)")] + + +def test_the_stored_unit_matches_what_the_bot_expects(tmp_path): + path = tmp_path / "eco.sqlite" + old_cache(path) + migrate_cache.migrate(path, backup=False) + HistoryCache(path, api.UNITS).close() # the signature step runs on the converted file without error diff --git a/tests/test_pollen.py b/tests/test_pollen.py index d24a54f..0201a2c 100644 --- a/tests/test_pollen.py +++ b/tests/test_pollen.py @@ -149,6 +149,7 @@ async def test_last_seasons_page_is_not_shown_and_the_report_leaves_the_block_ou pollen, _ = make_on(tmp_path, date(2026, 12, 20), html=lambda: page(grass="High", day=date(2026, 12, 20))) await pollen.start() assert pollen.lines()[0].startswith("Grass pollen: πŸ”΄ High") + assert json.loads(await pollen.handle({"cached": True}))["time"] # when the page was fetched, for the report's heading await pollen.close() january, calls = make_on(tmp_path, date(2027, 1, 5)) # the saved page is still in the database await january.start() diff --git a/tests/test_report.py b/tests/test_report.py index 8bbdc64..42c553c 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -8,7 +8,7 @@ '"app_tempin":"18.7 ℃"},"solar_and_uvi":{"solar":"0.0 W/mΒ²","uvi":"0"},"rainfall":{"rain_rate":"1.8 mm/hr","daily":"6.3 mm",' '"event":"49.8 mm","1_hour":"1.3 mm","weekly":"49.8 mm","monthly":"49.8 mm","yearly":"5105.7 mm"},"wind":{"wind_speed":"0.0 km/h",' '"wind_gust":"0.0 km/h","wind_direction":"131 ΒΊ"},"pressure":{"relative":"1025.0 hPa","absolute":"1009.5 hPa"},' - '"rain_outlook":"raining now (4.8 mm/h)","emoji":{"outdoor.temperature":"πŸ§₯","outdoor.feels_like":"πŸ§₯","outdoor.app_temp":"πŸ§₯",' + '"emoji":{"outdoor.temperature":"πŸ§₯","outdoor.feels_like":"πŸ§₯","outdoor.app_temp":"πŸ§₯",' '"outdoor.humidity":"πŸ’¦","indoor.temperature":"πŸ§₯","indoor.feels_like":"πŸ§₯","indoor.app_tempin":"πŸ§₯","rainfall.rain_rate":"🌧️",' '"rainfall.daily":"β˜”","pressure.relative":"πŸ—œοΈ"}}') AIR = ('{"sensor": "Fairleigh", "sensor_type": "outdoor", "time": "Fri 2 Oct 2026 2:22am", "pm2_5": {"value": 1.4, "unit": "Β΅g/mΒ³", ' @@ -16,23 +16,23 @@ '"unit": "Β΅g/mΒ³", "rating": "🟒 good"}, "co2": {"value": 433.0, "unit": "ppm", "rating": "🟒 good"}, "voc_index": {"value": 88.0, ' '"unit": "relative index (100 = this sensor\'s recent average)", "rating": "🟒 good"}, "nox_index": {"value": 1.0, ' '"unit": "relative index (1 = baseline)", "rating": "🟒 good"}}') -POLLEN = '{"lines": ["Grass pollen: 🟒 Low", "Thunderstorm asthma risk: 🟒 Low"], "place": "Melbourne", "source": "melbournepollen.com.au"}' +POLLEN = '{"lines": ["Grass pollen: 🟒 Low", "Thunderstorm asthma risk: 🟒 Low"], "place": "Melbourne", "source": "melbournepollen.com.au", "time": "Fri 2 Oct 2026 6:00pm"}' FORECAST = ('{"lines": ["Today: 🌦️ Showers. 11–16Β°C, 97% chance of rain", "Saturday: β›ˆοΈ Thunderstorm. 12–20Β°C, 95% chance of rain"], ' - '"place": "Melbourne", "source": "Open-Meteo"}') + '"place": "Melbourne", "source": "Open-Meteo", "time": "Fri 2 Oct 2026 6:05pm"}') WEATHER = """\ β€’ Outdoor: πŸ§₯ 12.3 Β°C, πŸ’¦ 94 %, dew point 11.3 Β°C, VPD 0.085 kPa β€’ Indoor: πŸ§₯ 18.3 Β°C, 63 % β€’ Pressure: πŸ—œοΈ 1025.0 hPa -β€’ Rain today: β˜” 6.3 mm (month total 49.8 mm); 🌧️ raining now (4.8 mm/h)""" # night and calm: no Sun or Wind line +β€’ Rain today: β˜” 6.3 mm (month total 49.8 mm); 🌧️ raining (1.8 mm/h)""" # night and calm: no Sun or Wind line def test_the_report_is_laid_out_in_code_with_emoji_before_the_readings_they_belong_to(): got = report.report({"weather_now": NOW, "air_quality": AIR, "pollen_asthma": POLLEN, "weather_forecast": FORECAST}) - assert got == f"""Weather station + assert got == f"""Weather station (2:21am) {WEATHER} -Air quality +Air quality (2:22am) β€’ PM2.5: 1.4 Β΅g/mΒ³ 🟒 good (AQI 8) β€’ PM10: 3.0 Β΅g/mΒ³ 🟒 good β€’ PM1: 1.0 Β΅g/mΒ³ 🟒 good @@ -40,11 +40,11 @@ def test_the_report_is_laid_out_in_code_with_emoji_before_the_readings_they_belo β€’ VOC index: 88 🟒 good β€’ NOx index: 1.0 🟒 good -Pollen & asthma (Melbourne) +Pollen & asthma (Melbourne, 6:00pm) β€’ Grass pollen: 🟒 Low β€’ Thunderstorm asthma risk: 🟒 Low -Forecast (Melbourne) +Forecast (Melbourne, 6:05pm) β€’ Today: 🌦️ Showers. 11–16Β°C, 97% chance of rain β€’ Saturday: β›ˆοΈ Thunderstorm. 12–20Β°C, 95% chance of rain""" assert not got.startswith("Current report") @@ -55,14 +55,11 @@ def test_weather_now_is_just_the_weather_stations_bullets(): assert report.weather_now("Error calling tool: timed out") is None and report.weather_now(None) is None -def test_a_dry_calm_day_has_no_rain_line_noise_and_a_likely_rain_outlook_is_shortened(): +def test_a_dry_calm_day_has_no_rain_line_noise(): dry = ('{"outdoor":{"temperature":"21.0 ℃","humidity":"50 %"},"rainfall":{"rain_rate":"0.0 mm/hr","daily":"0.0 mm","monthly":"12.0 mm"},' '"wind":{"wind_speed":"12.0 km/h","wind_gust":"20.0 km/h","wind_direction":"270 ΒΊ"},"emoji":{}}') lines = report.weather_now(dry).splitlines() assert lines == ["β€’ Outdoor: 21.0 Β°C, 50 %", "β€’ Rain today: 0.0 mm (month total 12.0 mm)", "β€’ Wind: 12.0 km/h from 270Β°, gust 20.0 km/h"] - likely = dry.replace('"emoji"', '"rain_outlook":"rain looks likely soon: pressure down 3.0 hPa in 3 hours (an estimate from the station\'s ' - 'readings, not an official forecast)","emoji"') - assert "β€’ Rain today: 0.0 mm (month total 12.0 mm); rain looks likely soon: pressure down 3.0 hPa in 3 hours (an estimate)" in report.weather_now(likely) drizzle = dry.replace('"0.0 mm/hr"', '"0.6 mm/hr"').replace('"emoji":{}', '"emoji":{"rainfall.rain_rate":"🌧️"}') assert "; 🌧️ raining (0.6 mm/h)" in report.weather_now(drizzle) @@ -71,7 +68,7 @@ def test_a_failed_or_missing_source_is_left_out_or_said_so_and_the_forecast_with got = report.report({"weather_now": "Error calling tool: boom", "air_quality": AIR, "pollen_asthma": '{"error": "No pollen levels are available right now."}', "weather_forecast": '{"lines": ["Today: β˜€οΈ Sunny. 20–25Β°C"]}'}) - assert got.startswith("Weather station\nβ€’ not available right now\n\nAir quality\nβ€’ PM2.5:") + assert got.startswith("Weather station\nβ€’ not available right now\n\nAir quality (2:22am)\nβ€’ PM2.5:") assert "Pollen" not in got and got.endswith("Forecast\nβ€’ Today: β˜€οΈ Sunny. 20–25Β°C") assert report.report({}) == "No readings are available right now." @@ -150,17 +147,14 @@ def test_an_average_chart_caption_leads_with_the_average(): "Temperature: average 15.2 Β°C (low 8.1, high 24.3)" -def test_a_rain_chart_caption_is_the_least_and_most_rain_then_whether_more_is_expected(): +def test_a_rain_chart_caption_is_the_least_and_most_rain(): from zoneinfo import ZoneInfo turn = _turn() turn.chart_fields = ["rain"] result = json.dumps({"period": "Tue 29 Sep 2026 - Tue 06 Oct 2026", "series": {"rainfall.daily": {"unit": "mm", "low": "0", "high": "7.2"}}}) - now = json.dumps({"rain_outlook": "Rain looks likely soon: pressure falling. (An estimate from the station's readings, not an official forecast)"}) - lines = report.chart_caption("weather_history", {}, result, turn, ZoneInfo("UTC"), now).splitlines() + lines = report.chart_caption("weather_history", {}, result, turn, ZoneInfo("UTC")).splitlines() assert lines[1].startswith("β˜” Rain per day: least 0 mm (") and "most 7.2 mm (" in lines[1] and "8.6 mm in all" in lines[1] - assert lines[2] == "β˜” Rain looks likely soon: pressure falling. (an estimate)" - dry = report.chart_caption("weather_history", {}, result, turn, ZoneInfo("UTC"), "{}") - assert dry.splitlines()[2] == "β˜” No rain is expected soon." + assert len(lines) == 2 def test_an_air_chart_caption_gives_each_metrics_peak_rating_and_average(): @@ -188,14 +182,14 @@ def test_a_reading_lookup_prints_only_what_was_asked(): assert report.reading_lines(now, ["humidity"], ["outdoor"]) == ["β€’ Outdoor: πŸ’¦ 94 %"] assert report.reading_lines(now, ["wind"], []) == ["β€’ Wind: calm"] assert report.reading_lines(now, ["pressure"], []) == ["β€’ Pressure: πŸ—œοΈ 1025.0 hPa"] - assert report.reading_lines(now, ["rain"], [])[0].startswith("β€’ Rain today: β˜” 6.3 mm") and "raining now" in report.reading_lines(now, ["rain"], [])[0] + assert report.reading_lines(now, ["rain"], [])[0].startswith("β€’ Rain today: β˜” 6.3 mm") and "raining (1.8 mm/h)" in report.reading_lines(now, ["rain"], [])[0] def test_air_pollen_and_forecast_lookups_print_the_tools_own_lines(): assert report.lookup("air", AIR, ["pm10"], []) == "β€’ PM10: 3.0 Β΅g/mΒ³ 🟒 good" assert report.lookup("air", AIR, [], []).splitlines()[0] == "β€’ PM2.5: 1.4 Β΅g/mΒ³ 🟒 good (AQI 8)" - assert report.lookup("pollen", POLLEN, [], []) == "Pollen & asthma (Melbourne)\nβ€’ Grass pollen: 🟒 Low\nβ€’ Thunderstorm asthma risk: 🟒 Low" - assert report.lookup("forecast", FORECAST, [], []).startswith("Forecast (Melbourne)\nβ€’ Today: ") + assert report.lookup("pollen", POLLEN, [], []) == "Pollen & asthma (Melbourne, 6:00pm)\nβ€’ Grass pollen: 🟒 Low\nβ€’ Thunderstorm asthma risk: 🟒 Low" + assert report.lookup("forecast", FORECAST, [], []).startswith("Forecast (Melbourne, 6:05pm)\nβ€’ Today: ") assert report.lookup("pollen", json.dumps({"error": "off season"}), [], []) is None @@ -232,3 +226,34 @@ def test_a_figure_per_period_lookup_gives_the_figure_over_the_whole_period(): assert report.lookup("days", result, [], []) == "Total rain: 1234.5 mm over 731 days (Fri 4 Oct 2024 – Sat 3 Oct 2026)." hot = json.dumps({"period": "a 2025 - b 2026", "days_checked": 3, "matching_days": 3, "stat": {"what": "Highest temperature", "of": "temp_max", "unit": "℃"}, "value": 41.2}) assert report.lookup("days", hot, [], []).startswith("Highest temperature: 41.2 Β°C over 3 days") + + +def test_a_heading_carries_the_place_and_the_time_of_day_of_its_data(): + assert report._heading("Forecast", {"place": "Melbourne", "time": "Fri 2 Oct 2026 6:05pm"}) == "Forecast (Melbourne, 6:05pm)" + assert report._heading("Weather station", {"time": "Fri 02 Oct 2026 00:07"}) == "Weather station (12:07am)" # the station's 24-hour form + assert report._heading("Weather station", {"time": "Fri 02 Oct 2026 13:30"}) == "Weather station (1:30pm)" + assert report._heading("Air quality", {"time": "Fri 2 Oct 2026 12:00pm"}) == "Air quality (12:00pm)" + assert report._heading("Forecast", {"place": "Melbourne"}) == "Forecast (Melbourne)" # no time: as before + assert report._heading("Air quality", {"time": "yesterday"}) == "Air quality" and report._heading("Air quality", {"time": None}) == "Air quality" + + +def test_when_gives_the_form_the_tools_use_in_their_time_field(): + from datetime import datetime + from lib.timeutil import when + tz = ZoneInfo("Australia/Melbourne") + assert when(int(datetime(2026, 10, 2, 18, 5, tzinfo=tz).timestamp()), tz) == "Fri 2 Oct 2026 6:05pm" + assert when(int(datetime(2026, 10, 2, 0, 7, tzinfo=tz).timestamp()), tz) == "Fri 2 Oct 2026 12:07am" + + +from zoneinfo import ZoneInfo # noqa: E402 + + +def test_only_the_name_of_each_heading_is_bold_not_its_brackets_or_time(): + text = report.report({"weather_now": NOW, "air_quality": AIR, "pollen_asthma": POLLEN, "weather_forecast": FORECAST}) + units = lambda s: len(s.encode("utf-16-le")) // 2 + names = [text.encode("utf-16-le")[2 * e.offset:2 * (e.offset + e.length)].decode("utf-16-le") for e in report.heading_entities(text)] + assert names == ["Weather station", "Air quality", "Pollen & asthma", "Forecast"] + assert all(e.type == "bold" for e in report.heading_entities(text)) + after_emoji = report.heading_entities("β€’ Rain today: β˜” 6.3 mm 🌧️\nForecast (Melbourne, 6:05pm)\nβ€’ Today") # emoji count as two units + assert [(e.offset, e.length) for e in after_emoji] == [(units("β€’ Rain today: β˜” 6.3 mm 🌧️\n"), len("Forecast"))] + assert report.heading_entities("Forecast\nβ€’ x") and report.heading_entities("β€’ Forecast: sunny\nForecasts") == [] diff --git a/tests/test_startup.py b/tests/test_startup.py index 25d03bb..5ef1f26 100644 --- a/tests/test_startup.py +++ b/tests/test_startup.py @@ -1,5 +1,6 @@ """Runs ecobot.main() end to end against fake sources and a fake Telegram.""" +import logging import asyncio import os import signal @@ -77,6 +78,39 @@ async def stop_when_archived(): # the background archives finish, then we shut final = next(line for line in log.splitlines() if "Ecowitt archive:" in line and " req, held " in line) assert "failed" not in final and "days (5min/30min/4h/1d)" in final assert len(app.handlers) == 10 # start+help, reset, keyboard, alerts, usage, the one button handler, other updates, membership, messages, errors + assert "Alerts: rain, gusts, UV, temperature crossing, particulates: 0 chats" in log and "Alerts NOT running" not in log + + +async def test_the_alerts_line_leaves_out_the_weather_alerts_when_they_could_not_start(tmp_path, monkeypatch, caplog): + caplog.set_level("INFO") + cfg = config(tmp_path) + eco_t, _ = ecowitt_transport(history_days=45) + air_t, _ = air_transport(oldest=datetime.now(timezone.utc) - timedelta(days=5)) + app = FakeApp() + monkeypatch.setattr(ecobot.Config, "from_env", classmethod(lambda cls: cfg)) + for cls, transport in ((ecobot.Ecowitt, eco_t), (ecobot.AirGradient, air_t)): + monkeypatch.setattr(cls, "__init__", lambda self, c, transport=None, orig=cls.__init__, t=transport: orig(self, c, t)) + monkeypatch.setattr(ecobot.Application, "builder", staticmethod(lambda: FakeBuilder(app))) + monkeypatch.setattr(archive, "PACE_SECONDS", 0) + monkeypatch.setattr(air_source, "BACKFILL_PACE", 0) + monkeypatch.setattr(air_source, "BACKFILL_EMPTY_STOP", 3) + + async def broken(self): + raise RuntimeError("no readings") + monkeypatch.setattr(ecobot.WeatherMonitor, "init", broken) + + async def stop_when_archived(): + for _ in range(300): + if " req, held " in caplog.text and "day(s) cached" in caplog.text: + break + await asyncio.sleep(0.1) + os.kill(os.getpid(), signal.SIGTERM) + stopper = asyncio.create_task(stop_when_archived()) + await ecobot.main() + await stopper + log = caplog.text + assert "Weather alerts couldn't start" in log and "Alerts: particulates: 0 chats" in log + assert "Alerts NOT running: rain, gusts, UV, temperature crossing (see the error above)" in log async def test_a_failed_first_refresh_does_not_stop_the_alerts_or_the_archives(tmp_path, monkeypatch, caplog): @@ -214,3 +248,16 @@ async def stop_when_archived(): await stopper assert "irrigation battery" in caplog.text # in the list of alert types assert len(app.handlers) == 10 and closed == [1] # the same handlers (buttons share one entry point), and the client closed at shutdown + + +def test_a_long_alerts_line_continues_indented_at_90_characters(): + from lib.alerts.menu import LABELS + lines = ecobot.wrapped("Alerts: ", list(LABELS.values()), ": 3 chats").splitlines() + assert len(lines) > 1 and all(len(line) <= 90 for line in lines) + assert all(line.startswith(" " * 13) and not line.startswith(" " * 14) for line in lines[1:]) + + +def test_log_lines_name_their_module_except_the_main_script(): + fmt = ecobot.LogFormat("%(levelname)s %(where)s%(message)s") + record = lambda name: logging.LogRecord(name, logging.INFO, __file__, 1, "hello", None, None) + assert fmt.format(record("ecobot")) == "INFO hello" and fmt.format(record("lib.bot")) == "INFO lib.bot: hello" diff --git a/tests/test_store.py b/tests/test_store.py index 92d6045..7872a5b 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -70,8 +70,8 @@ def test_vpd_reported_in_inhg_is_stored_and_read_in_kpa_and_old_rows_are_convert assert fixed["outdoor"]["temperature"]["unit"] == "C" cache, now = make(tmp_path), int(time.time()) old = now - 5 * 86400 - cache.db.execute("INSERT INTO fields VALUES ('M','30min','outdoor','vpd','inHg')") - cache.db.execute("INSERT INTO points VALUES ('M','30min','outdoor','vpd',?, '0.261')", (old,)) + cache.db.execute("INSERT INTO series (id, mac, cycle, grp, field, unit) VALUES (1,'M','30min','outdoor','vpd','inHg')") + cache.db.execute("INSERT INTO points VALUES (1, ?, '0.261')", (old,)) cache.db.execute("DELETE FROM meta WHERE key = 'unit_fixes'") # as a cache from before the fix cache.db.commit() cache.close() @@ -90,3 +90,13 @@ def test_slot_ranges_come_from_the_cached_five_minute_readings(tmp_path): lows, highs = cache.slot_ranges("M", "outdoor", "dew_point", {base: 12.0, base + 1800: 18.0, base + 3600: 30.0}, base, base + 7200) assert (lows[base], highs[base]) == (10.0, 15.0) and (lows[base + 1800], highs[base + 1800]) == (16.0, 21.0) assert base + 3600 not in lows # no 5-minute readings there: no range + + +def test_the_cache_says_where_it_is_and_how_much_it_holds(tmp_path): + cache, now = make(tmp_path), int(time.time()) + old = now - 5 * 86400 + cache.store("M", "5min", ["outdoor"], data(old, old + 300, old + 600), old - 300, now) + assert cache.path == tmp_path / "c.sqlite" + assert cache.count("M", "5min", "outdoor", old, old + 300) == 2 and cache.count("M", "5min", "outdoor", old + 1000, now) == 0 + assert cache.count("M", "30min", "outdoor", 0, now) == 0 + assert cache.newest("M", "5min") == old + 600 and cache.newest("M", "30min") is None