From 625f373acb6a51639fcc5b7bc068f9b25f586e21 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:21:12 +0000 Subject: [PATCH 01/39] feat: send email content from code with posthog.emails.send Squash of Silthus/posthog-python#2 at e5fdedf, re-signed for required signatures. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_strict_types.sh | 33 +- .sampo/changesets/emails-code-form.md | 9 + posthog/async_client.py | 11 +- posthog/client.py | 13 +- posthog/emails.py | 651 +++++++++++++++++++++++--- posthog/test/test_emails.py | 510 +++++++++++++++++++- references/public_api_snapshot.txt | 39 +- 7 files changed, 1174 insertions(+), 92 deletions(-) create mode 100644 .sampo/changesets/emails-code-form.md diff --git a/.github/scripts/check_strict_types.sh b/.github/scripts/check_strict_types.sh index 7b46c0af0..67da012e8 100755 --- a/.github/scripts/check_strict_types.sh +++ b/.github/scripts/check_strict_types.sh @@ -15,6 +15,7 @@ import atexit import posthog from posthog import FeatureFlagEvaluations, FlagValue, Posthog +from posthog.emails import AsyncEmails, Emails, SendEmailResult from posthog.ai.evaluations import ( AsyncOfflineEvaluations, BooleanScorerConfig, @@ -40,6 +41,18 @@ span.end() _ = (flag_value, all_flags, enabled, payload, evaluations, active) +emails = Emails("phs_example") +template_send: SendEmailResult = emails.send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", template_data={"name": "Ada"}) +rendered_send: SendEmailResult = emails.send(to="user@example.com", template_key="password-reset", subject="Reset", html="

Reset

", from_="Acme ", reply_to=["help@example.com"], capture_engagement=False, max_retries=0) +local_template_send: SendEmailResult = emails.send(to="user@example.com", template_key="welcome", subject="Hi {{ variables.name }}", text="Hi", template_data={"name": "Ada"}) +invite_send: SendEmailResult = emails.send(to="user@example.com", template_key="invite", subject="Invite", html="

Invite

", cc="team@example.com", bcc=["audit@example.com"], headers={"X-Entity-Ref-ID": "order-123"}, tags=[{"name": "category", "value": "invite"}], attachments=[{"filename": "invite.ics", "content": b"BEGIN:VCALENDAR", "content_type": "text/calendar"}, {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}]) +_ = (template_send, rendered_send, local_template_send, invite_send) + + +async def async_email_types() -> None: + sent: SendEmailResult = await AsyncEmails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

") + _ = sent + async def offline_evaluation_types( sync_client: OfflineEvaluations, async_client: AsyncOfflineEvaluations @@ -64,15 +77,25 @@ async def offline_evaluation_types( _ = config PY -cat > "$tmp/invalid_offline_types.py" <<'PY' +cat > "$tmp/invalid_types.py" <<'PY' # pyright: strict from posthog.ai.evaluations import BooleanScorerConfig, CategoricalScorerConfig, NumericPassingRule, NumericScorerConfig, OfflineEvaluations +from posthog.emails import AsyncEmails, Emails client = OfflineEvaluations(project_id=123, secret_key="phx_example") NumericPassingRule(operator="gt", threshold=0.8) # expected-type-error NumericScorerConfig(unknown_setting=True) # expected-type-error CategoricalScorerConfig(selection_mode="single") # expected-type-error client.scorers.create(name="Bad config", kind="numeric", config=BooleanScorerConfig(true_is_failure=True)) # expected-type-error +Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", template_key="welcome", subject="Hi", html="

Hi

") # expected-type-error +Emails("phs_example").send(to="user@example.com", template_key="welcome", html="

Hi

") # expected-type-error +Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", cc="team@example.com") # expected-type-error +Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", capture_engagement=False) # expected-type-error +Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", attachments=[{"filename": "a.txt", "content": [104, 105]}]) # expected-type-error + + +async def invalid_async_email() -> None: + await AsyncEmails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", subject="Hi") # expected-type-error PY "$tmp/.venv/bin/python" - <<'PY' > "$tmp/public_api_access.py" @@ -116,8 +139,8 @@ JSON cd "$tmp" "$tmp/.venv/bin/python" -m pyright strict_posthog_types.py public_api_access.py -if "$tmp/.venv/bin/python" -m pyright --outputjson invalid_offline_types.py > invalid_types.json; then - echo "Invalid offline evaluation configurations unexpectedly passed type checking." +if "$tmp/.venv/bin/python" -m pyright --outputjson invalid_types.py > invalid_types.json; then + echo "Invalid offline evaluation configurations and email sends unexpectedly passed type checking." exit 1 fi "$tmp/.venv/bin/python" - <<'PY' @@ -125,11 +148,11 @@ import json from pathlib import Path expected = { - index for index, line in enumerate(Path("invalid_offline_types.py").read_text().splitlines()) + index for index, line in enumerate(Path("invalid_types.py").read_text().splitlines()) if "expected-type-error" in line } diagnostics = json.loads(Path("invalid_types.json").read_text())["generalDiagnostics"] actual = {item["range"]["start"]["line"] for item in diagnostics if item["severity"] == "error"} assert actual == expected, diagnostics -print("Offline evaluation configuration types reject invalid inputs.") +print("Offline evaluation configuration and email send types reject invalid inputs.") PY diff --git a/.sampo/changesets/emails-code-form.md b/.sampo/changesets/emails-code-form.md new file mode 100644 index 000000000..df27efc68 --- /dev/null +++ b/.sampo/changesets/emails-code-form.md @@ -0,0 +1,9 @@ +--- +pypi/posthog: minor +--- + +Send transactional email from content in your code with `posthog.emails.send(template_key=..., subject=..., html=...)`. Without `template_data`, the content is sent as is. With `template_data`, the content is a template: PostHog fills in its variables, and the client uploads it by key whenever PostHog doesn't hold its content yet, then sends again with the same idempotency key. A local template needs `html`. A send that mixes `template_id` and `template_key` fails type checks and raises `TypeError`. + +A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `max_retries` like a template send. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. + +A code send also takes `cc`, `bcc`, `headers`, `tags` and `attachments`. An attachment's `content` can be `bytes`, `bytearray` or `memoryview`, which the client encodes as Base64, or a Base64 string. An optional field of a code send passed as `None` counts as left out. diff --git a/posthog/async_client.py b/posthog/async_client.py index 57ed13e29..787705f96 100644 --- a/posthog/async_client.py +++ b/posthog/async_client.py @@ -133,6 +133,7 @@ def __init__( app_host: Optional[str] = None, email_request_timeout_seconds: float = 15, email_request_max_retries: int = 2, + migrated_from: Optional[str] = None, ) -> None: if flush_at <= 0: raise ValueError("flush_at must be greater than zero") @@ -177,10 +178,13 @@ def __init__( else resolved_secret_key ) or None self.personal_api_key = self.secret_key - _validate_options(email_request_timeout_seconds, email_request_max_retries) + _validate_options( + email_request_timeout_seconds, email_request_max_retries, migrated_from + ) self.app_host = app_host self.email_request_timeout_seconds = email_request_timeout_seconds self.email_request_max_retries = email_request_max_retries + self.migrated_from = migrated_from self.feature_flags_request_timeout_seconds = ( feature_flags_request_timeout_seconds ) @@ -260,8 +264,8 @@ def __init__( @property def emails(self) -> AsyncEmails: """ - Send transactional email from PostHog email templates, and read the - status of each email sent with ``get``. + Send transactional email from PostHog email templates or from content + in your code, and read the status of each email sent with ``get``. Needs ``secret_key``, a project secret API key with the ``hog_flow:write`` scope, and ``posthog[async]``. Sends go to ``app_host``, or to the PostHog @@ -288,6 +292,7 @@ def emails(self) -> AsyncEmails: app_host=self.app_host, timeout=self.email_request_timeout_seconds, max_retries=self.email_request_max_retries, + migrated_from=self.migrated_from, ) return self._emails diff --git a/posthog/client.py b/posthog/client.py index 388e20b2d..fda66cf2a 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -733,6 +733,7 @@ def __init__( app_host: Optional[str] = None, email_request_timeout_seconds: float = 15, email_request_max_retries: int = 2, + migrated_from: Optional[str] = None, ): """ Initialize a new PostHog client instance. @@ -793,6 +794,8 @@ def __init__( email_request_max_retries: Number of retries for an email send after a network error, a rate limit, or an unavailable service. Defaults to 2. Set to 0 when a job runner already retries the send. + migrated_from: The email provider you moved from, such as + ``"resend"``. ``emails`` sends it with every request. super_properties: Properties merged into every captured event. enable_exception_autocapture: Automatically capture uncaught exceptions. @@ -1068,10 +1071,13 @@ def __init__( else resolved_secret_key ) or None self.personal_api_key = self.secret_key - _validate_options(email_request_timeout_seconds, email_request_max_retries) + _validate_options( + email_request_timeout_seconds, email_request_max_retries, migrated_from + ) self.app_host = app_host self.email_request_timeout_seconds = email_request_timeout_seconds self.email_request_max_retries = email_request_max_retries + self.migrated_from = migrated_from if debug: # Ensures that debug level messages are logged when debug mode is on. # Otherwise, defaults to WARNING level. See https://docs.python.org/3/howto/logging.html#what-happens-if-no-configuration-is-provided @@ -1250,8 +1256,8 @@ def _request_identity_kwargs(self) -> Dict[str, str]: @property def emails(self) -> Emails: """ - Send transactional email from PostHog email templates, and read the - status of each email sent with ``get``. + Send transactional email from PostHog email templates or from content + in your code, and read the status of each email sent with ``get``. Needs ``secret_key``, a project secret API key with the ``hog_flow:write`` scope. Sends go to ``app_host``, or to the PostHog Cloud app host of @@ -1279,6 +1285,7 @@ def emails(self) -> Emails: app_host=self.app_host, timeout=self.email_request_timeout_seconds, max_retries=self.email_request_max_retries, + migrated_from=self.migrated_from, ) return self._emails diff --git a/posthog/emails.py b/posthog/emails.py index 72b1ca3ef..c65af278b 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -1,4 +1,4 @@ -"""Transactional email, sent from an email template in PostHog. +"""Transactional email, sent from an email template in PostHog or from content in your code. Setup: @@ -22,6 +22,26 @@ idempotency_key="welcome:user_123", ) print(result.id) # the queued email + +# Content you rendered for this recipient, named by a key +posthog.emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=rendered_html, + distinct_id="user_123", +) + +# A template in your code. PostHog fills in the variables, and the client +# uploads the template whenever its content changes. +posthog.emails.send( + to="user@example.com", + template_key="welcome", + subject="Welcome, {{ variables.name }}", + html=welcome_html, + template_data={"name": "Ada"}, + distinct_id="user_123", +) ``` ``posthog.emails`` above is the client's property. For async code, use @@ -44,6 +64,10 @@ :class:`Emails`, when a job runner already retries the send, and pass an ``idempotency_key`` that names the email, such as ``welcome:user_123``. +Moving from another email provider: set ``migrated_from``, such as +``"resend"``, on the client or on :class:`Emails`, so PostHog knows where your +sends came from. + Errors: a send that is not queued raises :class:`EmailError`. Branch on ``code``. The ones a setup usually hits: @@ -74,6 +98,8 @@ from __future__ import annotations import asyncio +import base64 +import hashlib import importlib import json import logging @@ -85,10 +111,21 @@ from dataclasses import dataclass from datetime import datetime from types import TracebackType -from typing import Any, Callable, Mapping, Optional, TypeVar, Union +from typing import ( + Any, + Callable, + Mapping, + Optional, + Sequence, + TypedDict, + TypeVar, + Union, + overload, +) from uuid import UUID, uuid4 import requests +from typing_extensions import NotRequired, Unpack from .capture_v1 import _parse_retry_after from .request import ( @@ -102,21 +139,30 @@ __all__ = [ "AsyncEmails", + "CodeEmailArgs", + "EmailAttachment", "EmailError", "EmailStatus", + "EmailTag", "EmailWarning", "Emails", "SendEmailResult", + "TemplateEmailArgs", ] log = logging.getLogger("posthog") _SEND_PATH = "/api/projects/@current/messaging/send/email/" _STATUS_PATH = "/api/projects/@current/messaging/emails/{message_id}/" +_UPLOAD_PATH = "/api/projects/@current/messaging_templates/by_key/{template_key}/" +_TEMPLATE_KEY = re.compile(r"[a-zA-Z0-9_-]{1,400}") +_CONTENT_HASH_HEADER = b"posthog-email-template-v1\n" _MAX_RETRY_DELAY = 30.0 _IDEMPOTENCY_KEY = re.compile(r"^[\x21-\x7e]{1,255}$") +_HEADER_TOKEN = re.compile(r"[\x21-\x7e]+") _IN_FLIGHT_CODES = frozenset({"idempotency_key_in_flight"}) _QUOTA_CODES = frozenset({"quota_limited"}) +_TEMPLATE_CONFLICTS = frozenset({"template_outdated", "template_edited_in_posthog"}) class EmailError(Exception): @@ -168,6 +214,142 @@ class SendEmailResult: warnings: tuple[EmailWarning, ...] = () +class EmailTag(TypedDict): + """A name and value that PostHog adds to the email's engagement events as ``$email_tags``.""" + + name: str + value: str + + +class EmailAttachment(TypedDict): + """A file sent with the email. + + Args: + filename: The file name the recipient sees, such as ``invite.ics``. + content: The file as ``bytes``, ``bytearray`` or ``memoryview``, or + already Base64 encoded as a string. + content_type: The MIME type. Without it, the type follows the file name. + content_id: Shows the file inline: reference it in ``html`` as + ```` with ``content_id`` ``logo``. + """ + + filename: str + content: Union[bytes, bytearray, memoryview, str] + content_type: NotRequired[str] + content_id: NotRequired[str] + + +class _RecipientArgs(TypedDict): + to: str + distinct_id: NotRequired[Optional[str]] + idempotency_key: NotRequired[Optional[str]] + max_retries: NotRequired[Optional[int]] + + +class TemplateEmailArgs(_RecipientArgs): + """A send of an email template made in PostHog, named by its ID. + + Args: + to: One recipient email address. + template_id: The UUID of a template made transactional in PostHog. + template_data: Values for the template's ``{{ variables.* }}`` + placeholders. + distinct_id: The recipient's distinct ID in your product. + idempotency_key: 1 to 255 visible ASCII characters. + max_retries: Overrides the client's maximum number of retries for this + send. ``0`` turns retries off. + """ + + template_id: Union[str, UUID] + template_data: NotRequired[Optional[Mapping[str, Any]]] + + +class CodeEmailArgs(_RecipientArgs): + """A send of email content that lives in your code, named by a key. + + Without ``template_data``, the content is rendered for this one recipient + and sent as is. With it, the content is a template: PostHog fills in its + ``{{ variables.* }}`` placeholders, and the client uploads the template + whenever PostHog doesn't hold its content yet. + + An optional field passed as ``None`` counts as left out. + + A PostHog version that doesn't take a field yet, such as ``cc``, refuses + the send with ``invalid_request`` and names the field in + ``EmailError.field``. It never drops the field and sends the rest. + + Args: + to: One recipient email address. + template_key: Names the email, so its metrics, logs and status group + under it. Letters, digits, ``-`` and ``_``, at most 400. + subject: The subject line. + html: The HTML body. Give ``html``, ``text`` or both. A local template + needs ``html``. + text: The plain-text body. Without it, PostHog derives it from + ``html``. HTML with no text in it, such as a single image, goes out + without a plain-text part. + from_: A verified sender, such as ``"Acme "``. + reply_to: One address or a list of addresses. + cc: One address or a list of addresses that get a visible copy. + bcc: One address or a list of addresses that get a hidden copy. + headers: Extra email headers by name, such as ``X-Entity-Ref-ID``. + tags: Names and values that PostHog adds to the email's engagement + events. + attachments: Files sent with the email. + capture_engagement: Capture the email's opens and clicks as events of + the recipient. Defaults to ``True``. A project whose admin turned + engagement capture off captures nothing, whatever the send asks. A + template made in PostHog follows the project's setting, so the + template form doesn't take it. + template_data: Values for the ``{{ variables.* }}`` placeholders. + distinct_id: The recipient's distinct ID in your product. + idempotency_key: 1 to 255 visible ASCII characters. Template sends + and code sends share one set of keys: a key already used for + another email, or for a ``template_id`` send, raises + ``idempotency_key_reused``. A retry with the same key returns the + first send's result, also when its workflow was paused since. + max_retries: Overrides the client's maximum number of retries for this + send. ``0`` turns retries off. + """ + + template_key: str + subject: str + html: NotRequired[Optional[str]] + text: NotRequired[Optional[str]] + from_: NotRequired[Optional[str]] + reply_to: NotRequired[Optional[Union[str, Sequence[str]]]] + cc: NotRequired[Optional[Union[str, Sequence[str]]]] + bcc: NotRequired[Optional[Union[str, Sequence[str]]]] + headers: NotRequired[Optional[Mapping[str, str]]] + tags: NotRequired[Optional[Sequence[EmailTag]]] + attachments: NotRequired[Optional[Sequence[EmailAttachment]]] + capture_engagement: NotRequired[Optional[bool]] + template_data: NotRequired[Optional[Mapping[str, Any]]] + + +_OPTIONAL_CODE_FIELDS = frozenset( + { + "html", + "text", + "from_", + "reply_to", + "cc", + "bcc", + "headers", + "tags", + "attachments", + "capture_engagement", + "template_data", + } +) + + +@dataclass(frozen=True) +class _PreparedUpload: + path: str + body: bytes + + @dataclass(frozen=True) class EmailStatus: """What happened to an email that ``send()`` queued. @@ -200,6 +382,219 @@ class _PreparedSend: body: bytes idempotency_key: str max_retries: Optional[int] = None + upload: Optional[_PreparedUpload] = None + + +def _prepare(args: Mapping[str, Any]) -> _PreparedSend: + is_template_form = "template_id" in args + if is_template_form == ("template_key" in args): + raise TypeError("Pass either template_id or template_key.") + form = TemplateEmailArgs if is_template_form else CodeEmailArgs + unexpected = sorted(set(args) - set(form.__annotations__)) + if unexpected: + raise TypeError( + f"send() got arguments that the {'template' if is_template_form else 'code'} " + f"form doesn't take: {', '.join(unexpected)}." + ) + if "to" not in args: + raise TypeError("send() is missing the argument to.") + if is_template_form: + return _prepare_send( + args["to"], + args["template_id"], + args.get("template_data"), + args.get("distinct_id"), + args.get("idempotency_key"), + args.get("max_retries"), + ) + return _prepare_code_send(args) + + +def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: + args = { + name: value + for name, value in args.items() + if value is not None or name not in _OPTIONAL_CODE_FIELDS + } + if "subject" not in args: + raise TypeError("send() is missing the argument subject.") + to, distinct_id, idempotency_key = _recipient( + args["to"], args.get("distinct_id"), args.get("idempotency_key") + ) + max_retries = _retry_limit(args.get("max_retries")) + template_key = args["template_key"] + if not isinstance(template_key, str) or not _TEMPLATE_KEY.fullmatch(template_key): + raise ValueError( + "template_key can only have letters, digits, hyphens (-) and underscores (_), at most 400." + ) + content = {"subject": _nonempty_text(args, "subject")} + content.update( + {field: _string(args, field) for field in ("html", "text") if field in args} + ) + if "html" not in content and "text" not in content: + raise TypeError("send() needs html, text or both.") + if not (content.get("html") or content.get("text")): + raise ValueError("html or text must be a nonempty string.") + + payload: dict[str, Any] = {"to": to} + upload = None + if "template_data" in args: + template_data = args["template_data"] + if not isinstance(template_data, Mapping): + raise TypeError("template_data must be a mapping.") + if not content.get("html"): + raise ValueError( + "A local template needs html. Pass the HTML body with template_data." + ) + payload["template"] = {"key": template_key, "hash": _content_hash(content)} + payload["message_data"] = clean(dict(template_data)) + if distinct_id is not None: + payload["identifiers"] = {"distinct_id": distinct_id} + upload = _PreparedUpload( + path=_UPLOAD_PATH.format(template_key=template_key), + body=_encode( + { + "name": template_key, + **content, + "if_not_superseded": True, + "if_code_managed": True, + } + ), + ) + else: + payload.update({"template_key": template_key, **content}) + if distinct_id is not None: + payload["distinct_id"] = distinct_id + payload.update(_sender(args)) + return _PreparedSend( + path=_SEND_PATH, + body=_encode(payload), + idempotency_key=idempotency_key, + max_retries=max_retries, + upload=upload, + ) + + +def _content_hash(content: Mapping[str, str]) -> str: + """PostHog's content hash v1 of a template, so a send can say which content it holds.""" + encoded = bytearray(_CONTENT_HASH_HEADER) + for field in ("subject", "preheader", "html", "text"): + value = content.get(field, "").replace("\r\n", "\n").replace("\r", "\n") + data = value.encode("utf-8") + encoded += f"{len(data)}:".encode() + data + b"\n" + return "v1:" + hashlib.sha256(encoded).hexdigest() + + +def _sender(args: Mapping[str, Any]) -> dict[str, Any]: + sender: dict[str, Any] = {} + if "from_" in args: + sender["from"] = _nonempty_text(args, "from_") + for field in ("reply_to", "cc", "bcc"): + if field in args: + sender[field] = _addresses(field, args[field]) + if "headers" in args: + sender["headers"] = _headers_field(args["headers"]) + if "tags" in args: + sender["tags"] = _tags(args["tags"]) + if "attachments" in args: + sender["attachments"] = _attachments(args["attachments"]) + capture_engagement = args.get("capture_engagement", True) + if not isinstance(capture_engagement, bool): + raise TypeError("capture_engagement must be True or False.") + sender["capture_engagement"] = capture_engagement + return sender + + +def _addresses(field: str, value: Any) -> list[str]: + addresses = [value] if isinstance(value, str) else value + if not isinstance(addresses, Sequence) or not all( + isinstance(address, str) for address in addresses + ): + raise TypeError(f"{field} must be one address or a list of addresses.") + if not all(address.strip() for address in addresses): + raise ValueError(f"{field} must not hold a blank address.") + return list(addresses) + + +def _headers_field(headers: Any) -> dict[str, str]: + if not isinstance(headers, Mapping) or not all( + isinstance(name, str) and isinstance(value, str) + for name, value in headers.items() + ): + raise TypeError("headers must map header names to string values.") + return dict(headers) + + +def _tags(tags: Any) -> list[dict[str, str]]: + if ( + isinstance(tags, str) + or not isinstance(tags, Sequence) + or not all(_is_tag(tag) for tag in tags) + ): + raise TypeError("tags must be a list of {'name': str, 'value': str}.") + return [{"name": tag["name"], "value": tag["value"]} for tag in tags] + + +def _attachments(attachments: Any) -> list[dict[str, str]]: + if isinstance(attachments, (str, bytes)) or not isinstance(attachments, Sequence): + raise TypeError("attachments must be a list of files.") + return [_attachment(index, file) for index, file in enumerate(attachments)] + + +def _attachment(index: int, file: Any) -> dict[str, str]: + name = f"attachments[{index}]" + if not isinstance(file, Mapping) or not {"filename", "content"} <= set(file): + raise TypeError(f"{name} needs a filename and its content.") + unexpected = set(file) - set(EmailAttachment.__annotations__) + if unexpected: + raise TypeError( + f"{name} has fields an attachment doesn't take: {', '.join(sorted(unexpected))}." + ) + if not all(isinstance(file[field], str) for field in file if field != "content"): + raise TypeError( + f"{name} needs its filename, content_type and content_id as strings." + ) + return {**file, "content": _base64_content(name, file["content"])} + + +def _base64_content(name: str, content: Any) -> str: + if isinstance(content, (bytes, bytearray, memoryview)): + return base64.b64encode(bytes(content)).decode("ascii") + if isinstance(content, str): + return content + raise TypeError( + f"{name}.content must be bytes or a Base64 string. " + "Convert a list of byte values with bytes(content)." + ) + + +def _is_tag(tag: Any) -> bool: + return ( + isinstance(tag, Mapping) + and set(tag) == {"name", "value"} + and isinstance(tag["name"], str) + and isinstance(tag["value"], str) + ) + + +def _string(args: Mapping[str, Any], name: str) -> str: + value = args[name] + if not isinstance(value, str): + raise TypeError(f"{name} must be a string.") + return value + + +def _nonempty_text(args: Mapping[str, Any], name: str) -> str: + value = _string(args, name) + if not value: + raise ValueError(f"{name} must not be empty.") + return value + + +def _encode(payload: Mapping[str, Any]) -> bytes: + return json.dumps( + payload, cls=DatetimeSerializer, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") def _prepare_send( @@ -210,10 +605,8 @@ def _prepare_send( idempotency_key: Optional[str], max_retries: Optional[int] = None, ) -> _PreparedSend: - if not isinstance(to, str): - raise TypeError("to must be one email address as a string.") - if not to.strip(): - raise ValueError("to must not be empty.") + to, distinct_id, idempotency_key = _recipient(to, distinct_id, idempotency_key) + max_retries = _retry_limit(max_retries) try: template_uuid = ( template_id if isinstance(template_id, UUID) else UUID(template_id) @@ -222,6 +615,29 @@ def _prepare_send( raise ValueError("template_id must be the template's UUID.") from None if template_data is not None and not isinstance(template_data, Mapping): raise TypeError("template_data must be a mapping.") + + payload: dict[str, Any] = { + "template": {"id": str(template_uuid)}, + "to": to, + "message_data": clean(dict(template_data or {})), + } + if distinct_id is not None: + payload["identifiers"] = {"distinct_id": distinct_id} + return _PreparedSend( + path=_SEND_PATH, + body=json.dumps(payload, cls=DatetimeSerializer).encode("utf-8"), + idempotency_key=idempotency_key, + max_retries=max_retries, + ) + + +def _recipient( + to: Any, distinct_id: Any, idempotency_key: Any +) -> tuple[str, Optional[str], str]: + if not isinstance(to, str): + raise TypeError("to must be one email address as a string.") + if not to.strip(): + raise ValueError("to must not be empty.") if distinct_id is not None and ( not isinstance(distinct_id, str) or not distinct_id ): @@ -234,26 +650,17 @@ def _prepare_send( raise ValueError( "idempotency_key must be 1 to 255 visible ASCII characters without spaces." ) + return to, distinct_id, idempotency_key + + +def _retry_limit(max_retries: Any) -> Optional[int]: if max_retries is not None and ( isinstance(max_retries, bool) or not isinstance(max_retries, int) or max_retries < 0 ): raise ValueError("max_retries must be a nonnegative integer.") - - payload: dict[str, Any] = { - "template": {"id": str(template_uuid)}, - "to": to, - "message_data": clean(dict(template_data or {})), - } - if distinct_id is not None: - payload["identifiers"] = {"distinct_id": distinct_id} - return _PreparedSend( - path=_SEND_PATH, - body=json.dumps(payload, cls=DatetimeSerializer).encode("utf-8"), - idempotency_key=idempotency_key, - max_retries=max_retries, - ) + return max_retries def _status_path(message_id: Union[str, UUID]) -> str: @@ -266,7 +673,9 @@ def _status_path(message_id: Union[str, UUID]) -> str: return _STATUS_PATH.format(message_id=message_uuid) -def _validate_options(timeout: float, max_retries: int) -> None: +def _validate_options( + timeout: float, max_retries: int, migrated_from: Optional[str] +) -> None: if ( isinstance(timeout, bool) or not isinstance(timeout, (int, float)) @@ -280,6 +689,12 @@ def _validate_options(timeout: float, max_retries: int) -> None: or max_retries < 0 ): raise ValueError("max_retries must be a nonnegative integer.") + if migrated_from is not None and not ( + isinstance(migrated_from, str) and _HEADER_TOKEN.fullmatch(migrated_from) + ): + raise ValueError( + "migrated_from must be visible ASCII characters, such as 'resend'." + ) def _text(value: Any) -> Optional[str]: @@ -300,7 +715,7 @@ def _error_name(status: Optional[int], code: str) -> str: if status == 409: return ( "validation_error" - if code == "template_outdated" + if code in _TEMPLATE_CONFLICTS else "concurrent_idempotent_requests" ) if status == 422: @@ -402,6 +817,12 @@ def _parse_status_response( ) +def _parse_upload(status: int, payload: Any, retry_after_header: Optional[str]) -> None: + if not 200 <= status < 300: + fields = payload if isinstance(payload, dict) else {} + raise _response_error(fields, status, retry_after_header) + + def _response_error( fields: Mapping[str, Any], status: int, retry_after_header: Optional[str] ) -> EmailError: @@ -444,20 +865,33 @@ def _no_op() -> None: pass +def _is_outdated(error: EmailError) -> bool: + return error.status_code == 409 and error.code == "template_outdated" + + def _log_warnings(result: SendEmailResult) -> None: for warning in result.warnings: log.warning("PostHog email warning %s: %s", warning.code, warning.message) -def _headers(secret_key: str, idempotency_key: Optional[str]) -> dict[str, str]: +def _headers( + secret_key: str, + migrated_from: Optional[str], + *, + has_body: bool = False, + idempotency_key: Optional[str] = None, +) -> dict[str, str]: headers = { "Authorization": f"Bearer {secret_key}", "Accept": "application/json", "User-Agent": f"posthog-python/{VERSION}", } - if idempotency_key is not None: + if has_body: headers["Content-Type"] = "application/json" + if idempotency_key is not None: headers["Idempotency-Key"] = idempotency_key + if migrated_from is not None: + headers["X-PostHog-Migrated-From"] = migrated_from return headers @@ -491,13 +925,15 @@ def __init__( timeout: float, max_retries: int, disabled: bool, + migrated_from: Optional[str], ) -> None: - _validate_options(timeout, max_retries) + _validate_options(timeout, max_retries, migrated_from) self.secret_key = secret_key.strip() if isinstance(secret_key, str) else None self.host = determine_app_host(host) self.timeout = timeout self.max_retries = max_retries self.disabled = disabled + self.migrated_from = migrated_from self._parent_disabled: Callable[[], bool] = lambda: False self._admit: Callable[[], Optional[Callable[[], None]]] = lambda: _no_op self._host_error: Optional[str] = None @@ -526,7 +962,7 @@ def _secret_key(self, key_needed: str) -> str: class Emails(_EmailsBase): - """Sends transactional email from PostHog email templates. + """Sends transactional email from PostHog email templates or from content in your code. ``host`` is your PostHog app host. PostHog Cloud ingestion hosts map to the app host, so ``us.i.posthog.com`` sends to ``us.posthog.com``. Any other @@ -536,7 +972,8 @@ class Emails(_EmailsBase): to ``max_retries`` times, with the same idempotency key. Set ``max_retries=0`` when a job runner already retries the send. ``timeout`` is in seconds, per request. Pass ``session`` to send through your own - ``requests.Session``, for example a test double. + ``requests.Session``, for example a test double. ``migrated_from``, such + as ``"resend"``, tells PostHog which provider your sends came from. """ def __init__( @@ -548,6 +985,7 @@ def __init__( max_retries: int = 2, disabled: bool = False, session: Optional[requests.Session] = None, + migrated_from: Optional[str] = None, ) -> None: super().__init__( secret_key, @@ -555,22 +993,25 @@ def __init__( timeout=timeout, max_retries=max_retries, disabled=disabled, + migrated_from=migrated_from, ) self._session: Optional[requests.Session] = session self._owns_session = session is None self._session_lock = threading.Lock() - def send( - self, - *, - to: str, - template_id: Union[str, UUID], - template_data: Optional[Mapping[str, Any]] = None, - distinct_id: Optional[str] = None, - idempotency_key: Optional[str] = None, - max_retries: Optional[int] = None, - ) -> SendEmailResult: - """Queue one email from a transactional email template. + @overload + def send(self, **kwargs: Unpack[TemplateEmailArgs]) -> SendEmailResult: ... + + @overload + def send(self, **kwargs: Unpack[CodeEmailArgs]) -> SendEmailResult: ... + + def send(self, **kwargs: Any) -> SendEmailResult: + """Queue one email, from a template in PostHog or from content in your code. + + Pass exactly one form. The template form names a template made in + PostHog with ``template_id``. The code form names the email with + ``template_key`` and passes ``subject`` and ``html``, ``text`` or both: + see :class:`CodeEmailArgs`. Args: to: One recipient email address. @@ -608,17 +1049,33 @@ def send( distinct_id="user_123", idempotency_key="welcome:user_123", ) + emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=rendered_html, + distinct_id="user_123", + ) ``` """ - prepared = _prepare_send( - to, template_id, template_data, distinct_id, idempotency_key, max_retries - ) + prepared = _prepare(kwargs) finish = self._start(_SEND_NOT_DONE) try: - headers = _headers( - self._secret_key(_SEND_KEY_NEEDED), prepared.idempotency_key - ) - result = self._call( + result = self._deliver(prepared, self._secret_key(_SEND_KEY_NEEDED)) + finally: + finish() + _log_warnings(result) + return result + + def _deliver(self, prepared: _PreparedSend, secret_key: str) -> SendEmailResult: + headers = _headers( + secret_key, + self.migrated_from, + has_body=True, + idempotency_key=prepared.idempotency_key, + ) + try: + return self._call( "POST", prepared.path, prepared.body, @@ -626,10 +1083,25 @@ def send( _parse_response, prepared.max_retries, ) - finally: - finish() - _log_warnings(result) - return result + except EmailError as error: + if prepared.upload is None or not _is_outdated(error): + raise + self._call( + "PUT", + prepared.upload.path, + prepared.upload.body, + _headers(secret_key, self.migrated_from, has_body=True), + _parse_upload, + prepared.max_retries, + ) + return self._call( + "POST", + prepared.path, + prepared.body, + headers, + _parse_response, + prepared.max_retries, + ) def get(self, message_id: Union[str, UUID]) -> EmailStatus: """Read what happened to an email that :meth:`send` queued. @@ -658,7 +1130,7 @@ def get(self, message_id: Union[str, UUID]) -> EmailStatus: path = _status_path(message_id) finish = self._start(_GET_NOT_DONE) try: - headers = _headers(self._secret_key(_GET_KEY_NEEDED), None) + headers = _headers(self._secret_key(_GET_KEY_NEEDED), self.migrated_from) return self._call("GET", path, None, headers, _parse_status_response) finally: finish() @@ -753,6 +1225,7 @@ def __init__( max_retries: int = 2, disabled: bool = False, http_client: Any = None, + migrated_from: Optional[str] = None, ) -> None: super().__init__( secret_key, @@ -760,21 +1233,19 @@ def __init__( timeout=timeout, max_retries=max_retries, disabled=disabled, + migrated_from=migrated_from, ) self._client: Any = http_client self._owns_client = http_client is None - async def send( - self, - *, - to: str, - template_id: Union[str, UUID], - template_data: Optional[Mapping[str, Any]] = None, - distinct_id: Optional[str] = None, - idempotency_key: Optional[str] = None, - max_retries: Optional[int] = None, - ) -> SendEmailResult: - """Queue one email from a transactional email template. + @overload + async def send(self, **kwargs: Unpack[TemplateEmailArgs]) -> SendEmailResult: ... + + @overload + async def send(self, **kwargs: Unpack[CodeEmailArgs]) -> SendEmailResult: ... + + async def send(self, **kwargs: Any) -> SendEmailResult: + """Queue one email, from a template in PostHog or from content in your code. Takes the same arguments and raises the same errors as :meth:`Emails.send`. @@ -789,15 +1260,26 @@ async def send( ) ``` """ - prepared = _prepare_send( - to, template_id, template_data, distinct_id, idempotency_key, max_retries - ) + prepared = _prepare(kwargs) finish = self._start(_SEND_NOT_DONE) try: - headers = _headers( - self._secret_key(_SEND_KEY_NEEDED), prepared.idempotency_key - ) - result = await self._call( + result = await self._deliver(prepared, self._secret_key(_SEND_KEY_NEEDED)) + finally: + finish() + _log_warnings(result) + return result + + async def _deliver( + self, prepared: _PreparedSend, secret_key: str + ) -> SendEmailResult: + headers = _headers( + secret_key, + self.migrated_from, + has_body=True, + idempotency_key=prepared.idempotency_key, + ) + try: + return await self._call( "POST", prepared.path, prepared.body, @@ -805,10 +1287,25 @@ async def send( _parse_response, prepared.max_retries, ) - finally: - finish() - _log_warnings(result) - return result + except EmailError as error: + if prepared.upload is None or not _is_outdated(error): + raise + await self._call( + "PUT", + prepared.upload.path, + prepared.upload.body, + _headers(secret_key, self.migrated_from, has_body=True), + _parse_upload, + prepared.max_retries, + ) + return await self._call( + "POST", + prepared.path, + prepared.body, + headers, + _parse_response, + prepared.max_retries, + ) async def get(self, message_id: Union[str, UUID]) -> EmailStatus: """Read what happened to an email that :meth:`send` queued. @@ -819,7 +1316,7 @@ async def get(self, message_id: Union[str, UUID]) -> EmailStatus: path = _status_path(message_id) finish = self._start(_GET_NOT_DONE) try: - headers = _headers(self._secret_key(_GET_KEY_NEEDED), None) + headers = _headers(self._secret_key(_GET_KEY_NEEDED), self.migrated_from) return await self._call("GET", path, None, headers, _parse_status_response) finally: finish() @@ -915,10 +1412,15 @@ def _client_emails( app_host: Optional[str], timeout: float, max_retries: int, + migrated_from: Optional[str], ) -> _EmailsT: if app_host: emails = factory( - secret_key, host=app_host, timeout=timeout, max_retries=max_retries + secret_key, + host=app_host, + timeout=timeout, + max_retries=max_retries, + migrated_from=migrated_from, ) emails._parent_disabled = is_disabled emails._admit = admit @@ -930,6 +1432,7 @@ def _client_emails( host=inferred_host if is_cloud else None, timeout=timeout, max_retries=max_retries, + migrated_from=migrated_from, ) if not is_cloud: # A custom host is often an ingestion proxy, which must not receive the secret key. diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 0d2099a86..c7fc3cd82 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -176,6 +176,7 @@ def test_client_email_options_set_the_host_timeout_and_retries(): app_host="https://posthog.example.com/", email_request_timeout_seconds=3, email_request_max_retries=0, + migrated_from="resend", ) unavailable = response(503, {"code": "delivery_unavailable"}) @@ -188,6 +189,35 @@ def test_client_email_options_set_the_host_timeout_and_retries(): request.assert_called_once() assert request.call_args.args[1].startswith("https://posthog.example.com/api/") assert request.call_args.kwargs["timeout"] == 3 + sent = request.call_args.kwargs["auth"]( + requests.Request("POST", "https://posthog.example.com/").prepare() + ) + assert sent.headers["X-PostHog-Migrated-From"] == "resend" + + +def test_names_the_provider_the_client_moved_from_on_every_request(): + emails, adapter = emails_with([accepted(), status_reply()], migrated_from="resend") + + send(emails) + emails.get(MESSAGE_ID) + + assert [ + request.headers["X-PostHog-Migrated-From"] for request in adapter.requests + ] == [ + "resend", + "resend", + ] + + +@parameterized.expand([("space", "re send"), ("not_text", 1), ("empty", "")]) +def test_rejects_a_provider_name_that_is_not_a_header_value(_name, migrated_from): + with pytest.raises(ValueError): + Posthog( + "phc_test", + secret_key=SECRET_KEY, + sync_mode=True, + migrated_from=migrated_from, + ) @parameterized.expand( @@ -282,6 +312,18 @@ def test_waits_for_retry_after_and_gives_up_on_a_longer_one(no_sleep): ("client_option", {"max_retries": 0}, {}, 1), ("per_send_override", {}, {"max_retries": 0}, 1), ("per_send_override_above_client", {"max_retries": 0}, {"max_retries": 1}, 2), + ( + "code_send_override", + {}, + { + "template_id": None, + "template_key": "password-reset", + "subject": "Reset", + "html": "

Reset

", + "max_retries": 0, + }, + 1, + ), ] ) def test_max_retries_zero_leaves_retries_to_the_caller( @@ -290,9 +332,12 @@ def test_max_retries_zero_leaves_retries_to_the_caller( emails, adapter = emails_with( [rejected(503, "delivery_unavailable")] * 3, **client_options ) + args = {"to": "user@example.com", "template_id": TEMPLATE_ID, **send_options} with pytest.raises(EmailError): - send(emails, **send_options) + emails.send( + **{name: value for name, value in args.items() if value is not None} + ) assert len(adapter.requests) == requests_made @@ -572,3 +617,466 @@ def handler(request: httpx.Request) -> httpx.Response: assert [(request.method, str(request.url)) for request in seen] == [ ("GET", STATUS_URL) ] * 2 + + +HTML = "

Reset it here.

" + + +def send_code(emails: Emails, **overrides: Any): + return emails.send( + **{ + "to": "user@example.com", + "template_key": "password-reset", + "subject": "Reset your password", + **overrides, + } + ) + + +@parameterized.expand( + [ + ("html_only", {"html": HTML}, {"html": HTML}), + ("text_only", {"text": "Reset it here."}, {"text": "Reset it here."}), + ( + "every_field", + { + "html": HTML, + "text": "Reset it here.", + "from_": "Acme ", + "reply_to": "help@example.com", + "capture_engagement": False, + "distinct_id": "user_123", + }, + { + "html": HTML, + "text": "Reset it here.", + "from": "Acme ", + "reply_to": ["help@example.com"], + "capture_engagement": False, + "distinct_id": "user_123", + }, + ), + ( + "several_reply_to", + {"html": HTML, "reply_to": ["a@example.com", "b@example.com"]}, + {"html": HTML, "reply_to": ["a@example.com", "b@example.com"]}, + ), + ( + "empty_address_lists", + {"html": HTML, "reply_to": [], "cc": [], "bcc": []}, + {"html": HTML, "reply_to": [], "cc": [], "bcc": []}, + ), + ( + "none_as_left_out", + { + "html": HTML, + "text": None, + "from_": None, + "reply_to": None, + "cc": None, + "bcc": None, + "headers": None, + "tags": None, + "attachments": None, + "capture_engagement": None, + "template_data": None, + "distinct_id": None, + }, + {"html": HTML}, + ), + ] +) +def test_rendered_content_is_posted_as_is(_name, overrides, extra_body): + emails, adapter = emails_with([accepted("msg_1")]) + + result = send_code(emails, idempotency_key="reset:user_123", **overrides) + + assert result.id == "msg_1" + [request] = adapter.requests + assert (request.method, request.url) == ("POST", SEND_URL) + assert request.headers["Idempotency-Key"] == "reset:user_123" + assert json.loads(request.body) == { + "to": "user@example.com", + "template_key": "password-reset", + "subject": "Reset your password", + "capture_engagement": True, + **extra_body, + } + + +WELCOME = { + "template_key": "welcome", + "subject": "Welcome, {{ variables.first_name }}", + "html": "

Hi {{ variables.first_name }}

", +} +# The server's content_hash v1 test vectors, shared with posthog-node. +WELCOME_HASH = "v1:f74cd913f3aa2e5667a588f2865634e56693df77793ca594d18e9b5f89e3d2e6" +WELCOME_CRLF_TEXT_HASH = ( + "v1:8112876ee1b2dd7327f571b04b546ed057bfff5a904d9f0ae507b6a449ab6453" +) +UPLOAD_URL = ( + "https://us.posthog.com/api/projects/@current/messaging_templates/by_key/welcome/" +) + + +def send_local_template(emails: Emails, **overrides: Any): + return emails.send( + **{ + "to": "user@example.com", + **WELCOME, + "template_data": {"first_name": "Jane"}, + **overrides, + } + ) + + +@parameterized.expand( + [ + ("html_only", {}, WELCOME_HASH), + ("crlf_text", {"text": "\r\n\r"}, WELCOME_CRLF_TEXT_HASH), + ("empty_text", {"text": ""}, WELCOME_HASH), + ] +) +def test_a_local_template_is_sent_by_key_and_content_hash(_name, overrides, hash): + emails, adapter = emails_with([accepted("msg_1")]) + + result = send_local_template( + emails, from_="Acme ", distinct_id="user_123", **overrides + ) + + assert result.id == "msg_1" + [request] = adapter.requests + assert (request.method, request.url) == ("POST", SEND_URL) + assert json.loads(request.body) == { + "to": "user@example.com", + "template": {"key": "welcome", "hash": hash}, + "message_data": {"first_name": "Jane"}, + "from": "Acme ", + "capture_engagement": True, + "identifiers": {"distinct_id": "user_123"}, + } + + +def uploaded() -> Reply: + return (201, {"id": "tpl_1", "key": "welcome", "result": "created"}, {}) + + +def test_an_outdated_local_template_is_uploaded_then_sent_again_with_the_same_key(): + emails, adapter = emails_with( + [rejected(409, "template_outdated"), uploaded(), accepted("msg_1")] + ) + + result = send_local_template(emails, text="Hi {{ variables.first_name }}") + + assert result.id == "msg_1" + first_send, upload, second_send = adapter.requests + assert (upload.method, upload.url) == ("PUT", UPLOAD_URL) + assert json.loads(upload.body) == { + "name": "welcome", + "subject": "Welcome, {{ variables.first_name }}", + "html": "

Hi {{ variables.first_name }}

", + "text": "Hi {{ variables.first_name }}", + "if_not_superseded": True, + "if_code_managed": True, + } + assert "Idempotency-Key" not in upload.headers + assert upload.headers["Authorization"] == f"Bearer {SECRET_KEY}" + assert second_send.body == first_send.body + assert ( + second_send.headers["Idempotency-Key"] == first_send.headers["Idempotency-Key"] + ) + + +@parameterized.expand( + [ + ( + "refused_upload", + rejected(409, "template_edited_in_posthog"), + "template_edited_in_posthog", + 2, + ), + ("outdated_again", None, "template_outdated", 3), + ] +) +def test_a_local_template_uploads_at_most_once_per_send( + _name, upload_reply, code, requests_made +): + replies = ( + [rejected(409, "template_outdated"), upload_reply] + if upload_reply + else [ + rejected(409, "template_outdated"), + uploaded(), + rejected(409, "template_outdated"), + ] + ) + emails, adapter = emails_with(replies) + + with pytest.raises(EmailError) as raised: + send_local_template(emails) + + assert (raised.value.status_code, raised.value.code, raised.value.name) == ( + 409, + code, + "validation_error", + ) + assert len(adapter.requests) == requests_made + + +def test_rendered_content_is_never_uploaded(): + emails, adapter = emails_with([rejected(409, "template_outdated")]) + + with pytest.raises(EmailError): + send_code(emails, html=HTML) + + assert [request.method for request in adapter.requests] == ["POST"] + + +@pytest.mark.asyncio +async def test_async_uploads_an_outdated_local_template_then_sends_again(): + seen: list[httpx.Request] = [] + replies = [ + httpx.Response(409, json={"code": "template_outdated"}), + httpx.Response(201, json={"id": "tpl_1", "result": "created"}), + httpx.Response(202, json={"message_id": "msg_1"}), + ] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return replies.pop(0) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + result = await AsyncEmails(SECRET_KEY, http_client=client).send( + to="user@example.com", **WELCOME, template_data={"first_name": "Jane"} + ) + + assert result.id == "msg_1" + assert [(request.method, str(request.url)) for request in seen] == [ + ("POST", SEND_URL), + ("PUT", UPLOAD_URL), + ("POST", SEND_URL), + ] + first_send, upload, second_send = seen + assert json.loads(first_send.content)["template"] == { + "key": "welcome", + "hash": WELCOME_HASH, + } + assert "Idempotency-Key" not in upload.headers + assert second_send.content == first_send.content + assert ( + second_send.headers["Idempotency-Key"] == first_send.headers["Idempotency-Key"] + ) + + +@parameterized.expand( + [ + ("both_forms", {"template_id": TEMPLATE_ID, "html": HTML}, TypeError), + ("no_form", {"template_key": None}, TypeError), + ("no_content", {}, TypeError), + ("no_subject", {"subject": None, "html": HTML}, TypeError), + ("subject_not_text", {"subject": 1, "html": HTML}, TypeError), + ("from_not_text", {"html": HTML, "from_": 1}, TypeError), + ("from_keyword", {"html": HTML, "from": "a@example.com"}, TypeError), + ( + "key_with_space", + {"template_key": "reset password", "html": HTML}, + ValueError, + ), + ( + "key_with_trailing_newline", + {"template_key": "welcome\n", "html": HTML}, + ValueError, + ), + ("key_too_long", {"template_key": "k" * 401, "html": HTML}, ValueError), + ("empty_html", {"html": ""}, ValueError), + ("empty_html_and_text", {"html": "", "text": ""}, ValueError), + ("blank_cc", {"html": HTML, "cc": " "}, ValueError), + ("cc_not_text", {"html": HTML, "cc": 123}, TypeError), + ("template_data_list", {"html": HTML, "template_data": [1]}, TypeError), + ( + "local_template_without_html", + {"text": "Hi", "template_data": {"first_name": "Jane"}}, + ValueError, + ), + ( + "local_template_with_empty_html", + {"html": "", "text": "Hi", "template_data": {"first_name": "Jane"}}, + ValueError, + ), + ( + "capture_engagement_on_template_form", + { + "template_id": TEMPLATE_ID, + "template_key": None, + "subject": None, + "capture_engagement": False, + }, + TypeError, + ), + ( + "cc_on_template_form", + { + "template_id": TEMPLATE_ID, + "template_key": None, + "subject": None, + "cc": "a@example.com", + }, + TypeError, + ), + ] +) +def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error): + emails, adapter = emails_with([]) + args = { + "to": "user@example.com", + "template_key": "password-reset", + "subject": "Reset your password", + **overrides, + } + + with pytest.raises(error): + emails.send( + **{name: value for name, value in args.items() if value is not None} + ) + + assert adapter.requests == [] + + +COPIES_HEADERS_AND_TAGS = { + "cc": "manager@example.com", + "bcc": ["audit@example.com", "archive@example.com"], + "headers": {"X-Entity-Ref-ID": "order-123"}, + "tags": [{"name": "category", "value": "password_reset"}], +} +COPIES_HEADERS_AND_TAGS_BODY = { + "cc": ["manager@example.com"], + "bcc": ["audit@example.com", "archive@example.com"], + "headers": {"X-Entity-Ref-ID": "order-123"}, + "tags": [{"name": "category", "value": "password_reset"}], +} + + +def test_rendered_content_carries_copies_headers_and_tags(): + emails, adapter = emails_with([accepted()]) + + send_code(emails, html=HTML, **COPIES_HEADERS_AND_TAGS) + + [request] = adapter.requests + assert json.loads(request.body) == { + "to": "user@example.com", + "template_key": "password-reset", + "subject": "Reset your password", + "html": HTML, + "capture_engagement": True, + **COPIES_HEADERS_AND_TAGS_BODY, + } + + +def test_a_server_that_does_not_take_a_field_refuses_the_send_and_names_it(): + refused = { + "type": "invalid_request", + "code": "invalid_request", + "detail": "cc: PostHog doesn't take this field on this route.", + "attr": "cc", + } + emails, adapter = emails_with([(400, refused, {}), accepted()]) + + with pytest.raises(EmailError) as raised: + send_code(emails, html=HTML, cc="manager@example.com") + + error = raised.value + assert (error.name, error.code, error.field) == ( + "validation_error", + "invalid_request", + "cc", + ) + assert len(adapter.requests) == 1 + + +def test_a_local_template_sends_copies_headers_and_tags_but_never_uploads_them(): + emails, adapter = emails_with( + [rejected(409, "template_outdated"), uploaded(), accepted()] + ) + + send_local_template(emails, **COPIES_HEADERS_AND_TAGS) + + first_send, upload, _ = adapter.requests + sent = json.loads(first_send.body) + assert {field: sent[field] for field in COPIES_HEADERS_AND_TAGS_BODY} == ( + COPIES_HEADERS_AND_TAGS_BODY + ) + assert not set(COPIES_HEADERS_AND_TAGS_BODY) & set(json.loads(upload.body)) + + +INVITE = b"BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n" + + +def test_attachments_are_sent_base64_encoded_once_across_retries(): + emails, adapter = emails_with([rejected(503, "delivery_unavailable"), accepted()]) + + send_code( + emails, + html=HTML, + attachments=[ + { + "filename": "invite.ics", + "content": INVITE, + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, + ], + ) + + first, retry = adapter.requests + assert json.loads(first.body)["attachments"] == [ + { + "filename": "invite.ics", + "content": "QkVHSU46VkNBTEVOREFSDQpFTkQ6VkNBTEVOREFSDQo=", + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, + ] + assert retry.body == first.body + + +@parameterized.expand( + [ + ("bytearray", bytearray(INVITE)), + ("memoryview", memoryview(INVITE)), + ( + "strided_memoryview", + memoryview(bytes(b for byte in INVITE for b in (byte, 0)))[::2], + ), + ] +) +def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, content): + emails, adapter = emails_with([accepted()]) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": content}] + ) + + [request] = adapter.requests + assert json.loads(request.body)["attachments"] == [ + { + "filename": "invite.ics", + "content": "QkVHSU46VkNBTEVOREFSDQpFTkQ6VkNBTEVOREFSDQo=", + } + ] + + +@parameterized.expand( + [ + ("list_of_ints", {"filename": "a.txt", "content": [104, 105]}), + ("no_filename", {"content": b"104"}), + ("unknown_field", {"filename": "a.txt", "content": b"104", "path": "/tmp/a"}), + ] +) +def test_rejects_an_attachment_that_is_not_a_file(_name, attachment): + emails, adapter = emails_with([]) + + with pytest.raises(TypeError) as raised: + send_code(emails, html=HTML, attachments=[attachment]) + + assert "104" not in str(raised.value) + assert adapter.requests == [] diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 3eadb81bd..2eb20a585 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -758,6 +758,7 @@ attribute posthog.async_client.AsyncClient.is_server = is_server attribute posthog.async_client.AsyncClient.log = logging.getLogger('posthog') attribute posthog.async_client.AsyncClient.log_captured_exceptions = log_captured_exceptions attribute posthog.async_client.AsyncClient.max_retries = max(0, max_retries) +attribute posthog.async_client.AsyncClient.migrated_from = migrated_from attribute posthog.async_client.AsyncClient.on_error = on_error attribute posthog.async_client.AsyncClient.personal_api_key = self.secret_key attribute posthog.async_client.AsyncClient.project_root = os.getcwd() @@ -828,6 +829,7 @@ attribute posthog.client.Client.log = logging.getLogger('posthog') attribute posthog.client.Client.log_captured_exceptions = log_captured_exceptions attribute posthog.client.Client.max_retries = max(0, max_retries) attribute posthog.client.Client.metrics: PostHogMetrics +attribute posthog.client.Client.migrated_from = migrated_from attribute posthog.client.Client.on_error = on_error attribute posthog.client.Client.personal_api_key = self.secret_key attribute posthog.client.Client.poll_interval = poll_interval @@ -884,6 +886,23 @@ attribute posthog.debug = False attribute posthog.default_client = None attribute posthog.disable_geoip = True attribute posthog.disabled = False +attribute posthog.emails.CodeEmailArgs.attachments: NotRequired[Optional[Sequence[EmailAttachment]]] +attribute posthog.emails.CodeEmailArgs.bcc: NotRequired[Optional[Union[str, Sequence[str]]]] +attribute posthog.emails.CodeEmailArgs.capture_engagement: NotRequired[Optional[bool]] +attribute posthog.emails.CodeEmailArgs.cc: NotRequired[Optional[Union[str, Sequence[str]]]] +attribute posthog.emails.CodeEmailArgs.from_: NotRequired[Optional[str]] +attribute posthog.emails.CodeEmailArgs.headers: NotRequired[Optional[Mapping[str, str]]] +attribute posthog.emails.CodeEmailArgs.html: NotRequired[Optional[str]] +attribute posthog.emails.CodeEmailArgs.reply_to: NotRequired[Optional[Union[str, Sequence[str]]]] +attribute posthog.emails.CodeEmailArgs.subject: str +attribute posthog.emails.CodeEmailArgs.tags: NotRequired[Optional[Sequence[EmailTag]]] +attribute posthog.emails.CodeEmailArgs.template_data: NotRequired[Optional[Mapping[str, Any]]] +attribute posthog.emails.CodeEmailArgs.template_key: str +attribute posthog.emails.CodeEmailArgs.text: NotRequired[Optional[str]] +attribute posthog.emails.EmailAttachment.content: Union[bytes, bytearray, memoryview, str] +attribute posthog.emails.EmailAttachment.content_id: NotRequired[str] +attribute posthog.emails.EmailAttachment.content_type: NotRequired[str] +attribute posthog.emails.EmailAttachment.filename: str attribute posthog.emails.EmailError.code = code attribute posthog.emails.EmailError.docs_url = docs_url attribute posthog.emails.EmailError.field = field @@ -898,12 +917,16 @@ attribute posthog.emails.EmailStatus.id: str attribute posthog.emails.EmailStatus.reason: Optional[str] = None attribute posthog.emails.EmailStatus.status: str attribute posthog.emails.EmailStatus.status_at: datetime +attribute posthog.emails.EmailTag.name: str +attribute posthog.emails.EmailTag.value: str attribute posthog.emails.EmailWarning.code: str attribute posthog.emails.EmailWarning.fix: Optional[str] = None attribute posthog.emails.EmailWarning.message: str attribute posthog.emails.SendEmailResult.id: str attribute posthog.emails.SendEmailResult.replayed: bool = False attribute posthog.emails.SendEmailResult.warnings: tuple[EmailWarning, ...] = () +attribute posthog.emails.TemplateEmailArgs.template_data: NotRequired[Optional[Mapping[str, Any]]] +attribute posthog.emails.TemplateEmailArgs.template_id: Union[str, UUID] attribute posthog.enable_exception_autocapture = False attribute posthog.enable_exception_autocapture_rate_limiting = False attribute posthog.enable_full_ai_capture = False @@ -1320,21 +1343,25 @@ class posthog.ai.types.TokenUsage class posthog.ai.types.ToolInProgress class posthog.args.OptionalCaptureArgs class posthog.args.OptionalSetArgs -class posthog.async_client.AsyncClient(project_api_key: str, host: Optional[str] = None, debug: bool = False, max_queue_size: int = 10000, send: bool = True, on_error=None, flush_at: int = 100, flush_interval: float = 5.0, gzip: bool = False, max_retries: int = 3, timeout: float = 15, thread: int = 1, disabled: bool = False, disable_geoip: bool = True, is_server: bool = True, historical_migration: bool = False, super_properties: Optional[dict[str, Any]] = None, before_send=None, log_captured_exceptions: bool = False, project_root: Optional[str] = None, capture_exception_code_variables: bool = False, code_variables_mask_patterns=None, code_variables_ignore_patterns=None, code_variables_mask_url_credentials=None, code_variables_detect_secrets=None, in_app_modules: Optional[list[str]] = None, capture_mode: Optional[Union[CaptureMode, str]] = None, capture_compression: Optional[Union[CaptureCompression, str]] = None, capture_trace_context: bool = False, secret_key: Optional[str] = None, personal_api_key: Optional[str] = None, feature_flags_request_timeout_seconds: float = 3, feature_flags_request_max_retries: int = 1, app_host: Optional[str] = None, email_request_timeout_seconds: float = 15, email_request_max_retries: int = 2) +class posthog.async_client.AsyncClient(project_api_key: str, host: Optional[str] = None, debug: bool = False, max_queue_size: int = 10000, send: bool = True, on_error=None, flush_at: int = 100, flush_interval: float = 5.0, gzip: bool = False, max_retries: int = 3, timeout: float = 15, thread: int = 1, disabled: bool = False, disable_geoip: bool = True, is_server: bool = True, historical_migration: bool = False, super_properties: Optional[dict[str, Any]] = None, before_send=None, log_captured_exceptions: bool = False, project_root: Optional[str] = None, capture_exception_code_variables: bool = False, code_variables_mask_patterns=None, code_variables_ignore_patterns=None, code_variables_mask_url_credentials=None, code_variables_detect_secrets=None, in_app_modules: Optional[list[str]] = None, capture_mode: Optional[Union[CaptureMode, str]] = None, capture_compression: Optional[Union[CaptureCompression, str]] = None, capture_trace_context: bool = False, secret_key: Optional[str] = None, personal_api_key: Optional[str] = None, feature_flags_request_timeout_seconds: float = 3, feature_flags_request_max_retries: int = 1, app_host: Optional[str] = None, email_request_timeout_seconds: float = 15, email_request_max_retries: int = 2, migrated_from: Optional[str] = None) class posthog.async_client.AsyncPosthog class posthog.bucketed_rate_limiter.BucketedRateLimiter(bucket_size: Number, refill_rate: Number, refill_interval_seconds: Number, on_bucket_rate_limited: Optional[Callable[[Hashable], None]] = None, clock: Callable[[], float] = time.monotonic) class posthog.capture_compression.CaptureCompression class posthog.capture_mode.CaptureMode class posthog.capture_v1.CaptureV1Error(status: int | str, message: str, *, retry_after: Optional[float] = None, request_id: Optional[str] = None, attempts: Optional[int] = None, retry_exhausted: Optional[list[str]] = None, drops: Optional[list[tuple[str, Optional[str]]]] = None) -class posthog.client.Client(project_api_key: str, host=None, debug=False, max_queue_size=10000, send=True, on_error=None, flush_at=100, flush_interval=5.0, gzip=False, max_retries=3, sync_mode=False, timeout: float = 15, thread=1, poll_interval=30, personal_api_key=None, disabled=False, disable_geoip=True, is_server=True, historical_migration=False, feature_flags_request_timeout_seconds: float = 3, feature_flags_request_max_retries=1, super_properties=None, enable_exception_autocapture=False, log_captured_exceptions=False, project_root=None, privacy_mode=False, before_send=None, flag_fallback_cache_url=None, enable_local_evaluation=True, flag_definition_cache_provider: Optional[FlagDefinitionCacheProvider] = None, capture_exception_code_variables=False, code_variables_mask_patterns=None, code_variables_ignore_patterns=None, code_variables_mask_url_credentials=None, code_variables_detect_secrets=None, in_app_modules: list[str] | None = None, enable_exception_autocapture_rate_limiting=False, exception_autocapture_bucket_size=ExceptionCapture.DEFAULT_BUCKET_SIZE, exception_autocapture_refill_rate=ExceptionCapture.DEFAULT_REFILL_RATE, exception_autocapture_refill_interval_seconds=ExceptionCapture.DEFAULT_REFILL_INTERVAL_SECONDS, capture_mode: Optional[Union[CaptureMode, str]] = None, capture_compression: Optional[Union[CaptureCompression, str]] = None, secret_key=None, metrics: Optional[dict] = None, enable_full_ai_capture=False, capture_trace_context=False, _use_ai_lane=False, _enable_multimodal_capture=False, traces: Optional[dict] = None, app_host: Optional[str] = None, email_request_timeout_seconds: float = 15, email_request_max_retries: int = 2) +class posthog.client.Client(project_api_key: str, host=None, debug=False, max_queue_size=10000, send=True, on_error=None, flush_at=100, flush_interval=5.0, gzip=False, max_retries=3, sync_mode=False, timeout: float = 15, thread=1, poll_interval=30, personal_api_key=None, disabled=False, disable_geoip=True, is_server=True, historical_migration=False, feature_flags_request_timeout_seconds: float = 3, feature_flags_request_max_retries=1, super_properties=None, enable_exception_autocapture=False, log_captured_exceptions=False, project_root=None, privacy_mode=False, before_send=None, flag_fallback_cache_url=None, enable_local_evaluation=True, flag_definition_cache_provider: Optional[FlagDefinitionCacheProvider] = None, capture_exception_code_variables=False, code_variables_mask_patterns=None, code_variables_ignore_patterns=None, code_variables_mask_url_credentials=None, code_variables_detect_secrets=None, in_app_modules: list[str] | None = None, enable_exception_autocapture_rate_limiting=False, exception_autocapture_bucket_size=ExceptionCapture.DEFAULT_BUCKET_SIZE, exception_autocapture_refill_rate=ExceptionCapture.DEFAULT_REFILL_RATE, exception_autocapture_refill_interval_seconds=ExceptionCapture.DEFAULT_REFILL_INTERVAL_SECONDS, capture_mode: Optional[Union[CaptureMode, str]] = None, capture_compression: Optional[Union[CaptureCompression, str]] = None, secret_key=None, metrics: Optional[dict] = None, enable_full_ai_capture=False, capture_trace_context=False, _use_ai_lane=False, _enable_multimodal_capture=False, traces: Optional[dict] = None, app_host: Optional[str] = None, email_request_timeout_seconds: float = 15, email_request_max_retries: int = 2, migrated_from: Optional[str] = None) class posthog.consumer.Consumer(queue, api_key, flush_at=100, host=None, on_error=None, flush_interval=5.0, gzip=False, retries=10, timeout: float = 15, historical_migration=False, endpoint=EVENTS_ENDPOINT, max_msg_size=MAX_MSG_SIZE, capture_mode=CaptureMode.V0, capture_compression=CaptureCompression.NONE) class posthog.contexts.ContextScope(parent=None, fresh: bool = False, capture_exceptions: bool = True, client: Optional[Client] = None) -class posthog.emails.AsyncEmails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, http_client: Any = None) +class posthog.emails.AsyncEmails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, http_client: Any = None, migrated_from: Optional[str] = None) +class posthog.emails.CodeEmailArgs +class posthog.emails.EmailAttachment class posthog.emails.EmailError(*, name: str, code: str, message: str, status_code: Optional[int] = None, why: Optional[str] = None, fix: Optional[str] = None, docs_url: Optional[str] = None, field: Optional[str] = None, retry_after: Optional[float] = None) class posthog.emails.EmailStatus(id: str, status: str, created_at: datetime, status_at: datetime, reason: Optional[str] = None) +class posthog.emails.EmailTag class posthog.emails.EmailWarning(code: str, message: str, fix: Optional[str] = None) -class posthog.emails.Emails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, session: Optional[requests.Session] = None) +class posthog.emails.Emails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, session: Optional[requests.Session] = None, migrated_from: Optional[str] = None) class posthog.emails.SendEmailResult(id: str, replayed: bool = False, warnings: tuple[EmailWarning, ...] = ()) +class posthog.emails.TemplateEmailArgs class posthog.exception_capture.ExceptionCapture(client: Client, rate_limiting_enabled=False, bucket_size=DEFAULT_BUCKET_SIZE, refill_rate=DEFAULT_REFILL_RATE, refill_interval_seconds=DEFAULT_REFILL_INTERVAL_SECONDS) class posthog.exception_utils.AnnotatedValue(value, metadata) class posthog.exception_utils.VariableSizeLimiter(max_size=DEFAULT_TOTAL_VARIABLES_SIZE_LIMIT) @@ -1833,10 +1860,10 @@ method posthog.contexts.ContextScope.set_distinct_id(distinct_id: str) method posthog.contexts.ContextScope.set_session_id(session_id: str) method posthog.emails.AsyncEmails.aclose() -> None method posthog.emails.AsyncEmails.get(message_id: Union[str, UUID]) -> EmailStatus -method posthog.emails.AsyncEmails.send(*, to: str, template_id: Union[str, UUID], template_data: Optional[Mapping[str, Any]] = None, distinct_id: Optional[str] = None, idempotency_key: Optional[str] = None, max_retries: Optional[int] = None) -> SendEmailResult +method posthog.emails.AsyncEmails.send(**kwargs: Any) -> SendEmailResult method posthog.emails.Emails.close() -> None method posthog.emails.Emails.get(message_id: Union[str, UUID]) -> EmailStatus -method posthog.emails.Emails.send(*, to: str, template_id: Union[str, UUID], template_data: Optional[Mapping[str, Any]] = None, distinct_id: Optional[str] = None, idempotency_key: Optional[str] = None, max_retries: Optional[int] = None) -> SendEmailResult +method posthog.emails.Emails.send(**kwargs: Any) -> SendEmailResult method posthog.exception_capture.ExceptionCapture.capture_exception(exception, metadata=None) method posthog.exception_capture.ExceptionCapture.close() method posthog.exception_capture.ExceptionCapture.exception_handler(exc_type, exc_value, exc_traceback) From 20fad8c767d0be1c45047ca8b4123e23e53710d8 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:09:26 +0000 Subject: [PATCH 02/39] feat: cut emails.send to the template and rendered code forms Parks the local-template form (template_data on a code send, the content hash, the upload by key and its retry) and the cc, bcc, headers, tags and attachments fields, to match posthog-node. A code send now takes rendered html and/or text only. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_strict_types.sh | 7 +- .sampo/changesets/emails-code-form.md | 6 +- posthog/emails.py | 245 +----------------- posthog/test/test_emails.py | 341 +------------------------- references/public_api_snapshot.txt | 14 -- 5 files changed, 21 insertions(+), 592 deletions(-) diff --git a/.github/scripts/check_strict_types.sh b/.github/scripts/check_strict_types.sh index 67da012e8..4e41c9e8f 100755 --- a/.github/scripts/check_strict_types.sh +++ b/.github/scripts/check_strict_types.sh @@ -44,9 +44,7 @@ _ = (flag_value, all_flags, enabled, payload, evaluations, active) emails = Emails("phs_example") template_send: SendEmailResult = emails.send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", template_data={"name": "Ada"}) rendered_send: SendEmailResult = emails.send(to="user@example.com", template_key="password-reset", subject="Reset", html="

Reset

", from_="Acme ", reply_to=["help@example.com"], capture_engagement=False, max_retries=0) -local_template_send: SendEmailResult = emails.send(to="user@example.com", template_key="welcome", subject="Hi {{ variables.name }}", text="Hi", template_data={"name": "Ada"}) -invite_send: SendEmailResult = emails.send(to="user@example.com", template_key="invite", subject="Invite", html="

Invite

", cc="team@example.com", bcc=["audit@example.com"], headers={"X-Entity-Ref-ID": "order-123"}, tags=[{"name": "category", "value": "invite"}], attachments=[{"filename": "invite.ics", "content": b"BEGIN:VCALENDAR", "content_type": "text/calendar"}, {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}]) -_ = (template_send, rendered_send, local_template_send, invite_send) +_ = (template_send, rendered_send) async def async_email_types() -> None: @@ -91,7 +89,8 @@ Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3 Emails("phs_example").send(to="user@example.com", template_key="welcome", html="

Hi

") # expected-type-error Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", cc="team@example.com") # expected-type-error Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", capture_engagement=False) # expected-type-error -Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", attachments=[{"filename": "a.txt", "content": [104, 105]}]) # expected-type-error +Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", template_data={"name": "Ada"}) # expected-type-error +Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", cc="team@example.com") # expected-type-error async def invalid_async_email() -> None: diff --git a/.sampo/changesets/emails-code-form.md b/.sampo/changesets/emails-code-form.md index df27efc68..f02c6d9bd 100644 --- a/.sampo/changesets/emails-code-form.md +++ b/.sampo/changesets/emails-code-form.md @@ -2,8 +2,6 @@ pypi/posthog: minor --- -Send transactional email from content in your code with `posthog.emails.send(template_key=..., subject=..., html=...)`. Without `template_data`, the content is sent as is. With `template_data`, the content is a template: PostHog fills in its variables, and the client uploads it by key whenever PostHog doesn't hold its content yet, then sends again with the same idempotency key. A local template needs `html`. A send that mixes `template_id` and `template_key` fails type checks and raises `TypeError`. +Send transactional email from content in your code with `posthog.emails.send(template_key=..., subject=..., html=...)`. The client sends the content as is, so render a React email to HTML in your app first. A send that mixes `template_id` and `template_key` fails type checks and raises `TypeError`. -A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `max_retries` like a template send. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. - -A code send also takes `cc`, `bcc`, `headers`, `tags` and `attachments`. An attachment's `content` can be `bytes`, `bytearray` or `memoryview`, which the client encodes as Base64, or a Base64 string. An optional field of a code send passed as `None` counts as left out. +A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `from_`, `reply_to` and `max_retries`. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. An optional field of a code send passed as `None` counts as left out. diff --git a/posthog/emails.py b/posthog/emails.py index 39e8262de..8237e2b47 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -31,17 +31,6 @@ html=rendered_html, distinct_id="user_123", ) - -# A template in your code. PostHog fills in the variables, and the client -# uploads the template whenever its content changes. -posthog.emails.send( - to="user@example.com", - template_key="welcome", - subject="Welcome, {{ variables.name }}", - html=welcome_html, - template_data={"name": "Ada"}, - distinct_id="user_123", -) ``` ``posthog.emails`` above is the client's property. For async code, use @@ -98,8 +87,6 @@ from __future__ import annotations import asyncio -import base64 -import hashlib import importlib import json import logging @@ -140,10 +127,8 @@ __all__ = [ "AsyncEmails", "CodeEmailArgs", - "EmailAttachment", "EmailError", "EmailStatus", - "EmailTag", "EmailWarning", "Emails", "SendEmailResult", @@ -154,15 +139,12 @@ _SEND_PATH = "/api/projects/@current/messaging/send/email/" _STATUS_PATH = "/api/projects/@current/messaging/emails/{message_id}/" -_UPLOAD_PATH = "/api/projects/@current/messaging_templates/by_key/{template_key}/" _TEMPLATE_KEY = re.compile(r"[a-zA-Z0-9_-]{1,400}") -_CONTENT_HASH_HEADER = b"posthog-email-template-v1\n" _MAX_RETRY_DELAY = 30.0 _IDEMPOTENCY_KEY = re.compile(r"^[\x21-\x7e]{1,255}$") _HEADER_TOKEN = re.compile(r"[\x21-\x7e]+") _IN_FLIGHT_CODES = frozenset({"idempotency_key_in_flight"}) _QUOTA_CODES = frozenset({"quota_limited"}) -_TEMPLATE_CONFLICTS = frozenset({"template_outdated", "template_edited_in_posthog"}) class EmailError(Exception): @@ -214,31 +196,6 @@ class SendEmailResult: warnings: tuple[EmailWarning, ...] = () -class EmailTag(TypedDict): - """A name and value that PostHog adds to the email's engagement events as ``$email_tags``.""" - - name: str - value: str - - -class EmailAttachment(TypedDict): - """A file sent with the email. - - Args: - filename: The file name the recipient sees, such as ``invite.ics``. - content: The file as ``bytes``, ``bytearray`` or ``memoryview``, or - already Base64 encoded as a string. - content_type: The MIME type. Without it, the type follows the file name. - content_id: Shows the file inline: reference it in ``html`` as - ```` with ``content_id`` ``logo``. - """ - - filename: str - content: Union[bytes, bytearray, memoryview, str] - content_type: NotRequired[str] - content_id: NotRequired[str] - - class _RecipientArgs(TypedDict): to: str distinct_id: NotRequired[Optional[str]] @@ -265,43 +222,29 @@ class TemplateEmailArgs(_RecipientArgs): class CodeEmailArgs(_RecipientArgs): - """A send of email content that lives in your code, named by a key. + """A send of email content that your code rendered for one recipient, named by a key. - Without ``template_data``, the content is rendered for this one recipient - and sent as is. With it, the content is a template: PostHog fills in its - ``{{ variables.* }}`` placeholders, and the client uploads the template - whenever PostHog doesn't hold its content yet. + The content goes out as is. To send a React email, render it to HTML in + your app first. An optional field passed as ``None`` counts as left out. - A PostHog version that doesn't take a field yet, such as ``cc``, refuses - the send with ``invalid_request`` and names the field in - ``EmailError.field``. It never drops the field and sends the rest. - Args: to: One recipient email address. template_key: Names the email, so its metrics, logs and status group under it. Letters, digits, ``-`` and ``_``, at most 400. subject: The subject line. - html: The HTML body. Give ``html``, ``text`` or both. A local template - needs ``html``. + html: The HTML body. Give ``html``, ``text`` or both. text: The plain-text body. Without it, PostHog derives it from ``html``. HTML with no text in it, such as a single image, goes out without a plain-text part. from_: A verified sender, such as ``"Acme "``. reply_to: One address or a list of addresses. - cc: One address or a list of addresses that get a visible copy. - bcc: One address or a list of addresses that get a hidden copy. - headers: Extra email headers by name, such as ``X-Entity-Ref-ID``. - tags: Names and values that PostHog adds to the email's engagement - events. - attachments: Files sent with the email. capture_engagement: Capture the email's opens and clicks as events of the recipient. Defaults to ``True``. A project whose admin turned engagement capture off captures nothing, whatever the send asks. A template made in PostHog follows the project's setting, so the template form doesn't take it. - template_data: Values for the ``{{ variables.* }}`` placeholders. distinct_id: The recipient's distinct ID in your product. idempotency_key: 1 to 255 visible ASCII characters. Template sends and code sends share one set of keys: a key already used for @@ -318,13 +261,7 @@ class CodeEmailArgs(_RecipientArgs): text: NotRequired[Optional[str]] from_: NotRequired[Optional[str]] reply_to: NotRequired[Optional[Union[str, Sequence[str]]]] - cc: NotRequired[Optional[Union[str, Sequence[str]]]] - bcc: NotRequired[Optional[Union[str, Sequence[str]]]] - headers: NotRequired[Optional[Mapping[str, str]]] - tags: NotRequired[Optional[Sequence[EmailTag]]] - attachments: NotRequired[Optional[Sequence[EmailAttachment]]] capture_engagement: NotRequired[Optional[bool]] - template_data: NotRequired[Optional[Mapping[str, Any]]] _OPTIONAL_CODE_FIELDS = frozenset( @@ -333,23 +270,11 @@ class CodeEmailArgs(_RecipientArgs): "text", "from_", "reply_to", - "cc", - "bcc", - "headers", - "tags", - "attachments", "capture_engagement", - "template_data", } ) -@dataclass(frozen=True) -class _PreparedUpload: - path: str - body: bytes - - @dataclass(frozen=True) class EmailStatus: """What happened to an email that ``send()`` queued. @@ -382,7 +307,6 @@ class _PreparedSend: body: bytes idempotency_key: str max_retries: Optional[int] = None - upload: Optional[_PreparedUpload] = None def _prepare(args: Mapping[str, Any]) -> _PreparedSend: @@ -436,31 +360,7 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: if not (content.get("html") or content.get("text")): raise ValueError("html or text must be a nonempty string.") - payload: dict[str, Any] = {"to": to} - upload = None - if "template_data" in args: - template_data = args["template_data"] - if not isinstance(template_data, Mapping): - raise TypeError("template_data must be a mapping.") - if not content.get("html"): - raise ValueError( - "A local template needs html. Pass the HTML body with template_data." - ) - payload["template"] = {"key": template_key, "hash": _content_hash(content)} - payload["message_data"] = clean(dict(template_data)) - upload = _PreparedUpload( - path=_UPLOAD_PATH.format(template_key=template_key), - body=_encode( - { - "name": template_key, - **content, - "if_not_superseded": True, - "if_code_managed": True, - } - ), - ) - else: - payload.update({"template_key": template_key, **content}) + payload: dict[str, Any] = {"to": to, "template_key": template_key, **content} if distinct_id is not None: payload["distinct_id"] = distinct_id payload.update(_sender(args)) @@ -469,33 +369,15 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: body=_encode(payload), idempotency_key=idempotency_key, max_retries=max_retries, - upload=upload, ) -def _content_hash(content: Mapping[str, str]) -> str: - """PostHog's content hash v1 of a template, so a send can say which content it holds.""" - encoded = bytearray(_CONTENT_HASH_HEADER) - for field in ("subject", "preheader", "html", "text"): - value = content.get(field, "").replace("\r\n", "\n").replace("\r", "\n") - data = value.encode("utf-8") - encoded += f"{len(data)}:".encode() + data + b"\n" - return "v1:" + hashlib.sha256(encoded).hexdigest() - - def _sender(args: Mapping[str, Any]) -> dict[str, Any]: sender: dict[str, Any] = {} if "from_" in args: sender["from"] = _nonempty_text(args, "from_") - for field in ("reply_to", "cc", "bcc"): - if field in args: - sender[field] = _addresses(field, args[field]) - if "headers" in args: - sender["headers"] = _headers_field(args["headers"]) - if "tags" in args: - sender["tags"] = _tags(args["tags"]) - if "attachments" in args: - sender["attachments"] = _attachments(args["attachments"]) + if "reply_to" in args: + sender["reply_to"] = _addresses("reply_to", args["reply_to"]) capture_engagement = args.get("capture_engagement", True) if not isinstance(capture_engagement, bool): raise TypeError("capture_engagement must be True or False.") @@ -514,67 +396,6 @@ def _addresses(field: str, value: Any) -> list[str]: return list(addresses) -def _headers_field(headers: Any) -> dict[str, str]: - if not isinstance(headers, Mapping) or not all( - isinstance(name, str) and isinstance(value, str) - for name, value in headers.items() - ): - raise TypeError("headers must map header names to string values.") - return dict(headers) - - -def _tags(tags: Any) -> list[dict[str, str]]: - if ( - isinstance(tags, str) - or not isinstance(tags, Sequence) - or not all(_is_tag(tag) for tag in tags) - ): - raise TypeError("tags must be a list of {'name': str, 'value': str}.") - return [{"name": tag["name"], "value": tag["value"]} for tag in tags] - - -def _attachments(attachments: Any) -> list[dict[str, str]]: - if isinstance(attachments, (str, bytes)) or not isinstance(attachments, Sequence): - raise TypeError("attachments must be a list of files.") - return [_attachment(index, file) for index, file in enumerate(attachments)] - - -def _attachment(index: int, file: Any) -> dict[str, str]: - name = f"attachments[{index}]" - if not isinstance(file, Mapping) or not {"filename", "content"} <= set(file): - raise TypeError(f"{name} needs a filename and its content.") - unexpected = set(file) - set(EmailAttachment.__annotations__) - if unexpected: - raise TypeError( - f"{name} has fields an attachment doesn't take: {', '.join(sorted(unexpected))}." - ) - if not all(isinstance(file[field], str) for field in file if field != "content"): - raise TypeError( - f"{name} needs its filename, content_type and content_id as strings." - ) - return {**file, "content": _base64_content(name, file["content"])} - - -def _base64_content(name: str, content: Any) -> str: - if isinstance(content, (bytes, bytearray, memoryview)): - return base64.b64encode(bytes(content)).decode("ascii") - if isinstance(content, str): - return content - raise TypeError( - f"{name}.content must be bytes or a Base64 string. " - "Convert a list of byte values with bytes(content)." - ) - - -def _is_tag(tag: Any) -> bool: - return ( - isinstance(tag, Mapping) - and set(tag) == {"name", "value"} - and isinstance(tag["name"], str) - and isinstance(tag["value"], str) - ) - - def _string(args: Mapping[str, Any], name: str) -> str: value = args[name] if not isinstance(value, str): @@ -713,7 +534,7 @@ def _error_name(status: Optional[int], code: str) -> str: if status == 409: return ( "validation_error" - if code in _TEMPLATE_CONFLICTS + if code == "template_outdated" else "concurrent_idempotent_requests" ) if status == 422: @@ -815,12 +636,6 @@ def _parse_status_response( ) -def _parse_upload(status: int, payload: Any, retry_after_header: Optional[str]) -> None: - if not 200 <= status < 300: - fields = payload if isinstance(payload, dict) else {} - raise _response_error(fields, status, retry_after_header) - - def _response_error( fields: Mapping[str, Any], status: int, retry_after_header: Optional[str] ) -> EmailError: @@ -863,10 +678,6 @@ def _no_op() -> None: pass -def _is_outdated(error: EmailError) -> bool: - return error.status_code == 409 and error.code == "template_outdated" - - def _log_warnings(result: SendEmailResult) -> None: for warning in result.warnings: log.warning("PostHog email warning %s: %s", warning.code, warning.message) @@ -1072,26 +883,6 @@ def _deliver(self, prepared: _PreparedSend, secret_key: str) -> SendEmailResult: has_body=True, idempotency_key=prepared.idempotency_key, ) - try: - return self._call( - "POST", - prepared.path, - prepared.body, - headers, - _parse_response, - prepared.max_retries, - ) - except EmailError as error: - if prepared.upload is None or not _is_outdated(error): - raise - self._call( - "PUT", - prepared.upload.path, - prepared.upload.body, - _headers(secret_key, self.migrated_from, has_body=True), - _parse_upload, - prepared.max_retries, - ) return self._call( "POST", prepared.path, @@ -1276,26 +1067,6 @@ async def _deliver( has_body=True, idempotency_key=prepared.idempotency_key, ) - try: - return await self._call( - "POST", - prepared.path, - prepared.body, - headers, - _parse_response, - prepared.max_retries, - ) - except EmailError as error: - if prepared.upload is None or not _is_outdated(error): - raise - await self._call( - "PUT", - prepared.upload.path, - prepared.upload.body, - _headers(secret_key, self.migrated_from, has_body=True), - _parse_upload, - prepared.max_retries, - ) return await self._call( "POST", prepared.path, diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 8ba561324..96fff6dac 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -663,8 +663,8 @@ def send_code(emails: Emails, **overrides: Any): ), ( "empty_address_lists", - {"html": HTML, "reply_to": [], "cc": [], "bcc": []}, - {"html": HTML, "reply_to": [], "cc": [], "bcc": []}, + {"html": HTML, "reply_to": []}, + {"html": HTML, "reply_to": []}, ), ( "none_as_left_out", @@ -673,13 +673,7 @@ def send_code(emails: Emails, **overrides: Any): "text": None, "from_": None, "reply_to": None, - "cc": None, - "bcc": None, - "headers": None, - "tags": None, - "attachments": None, "capture_engagement": None, - "template_data": None, "distinct_id": None, }, {"html": HTML}, @@ -704,170 +698,6 @@ def test_rendered_content_is_posted_as_is(_name, overrides, extra_body): } -WELCOME = { - "template_key": "welcome", - "subject": "Welcome, {{ variables.first_name }}", - "html": "

Hi {{ variables.first_name }}

", -} -# The server's content_hash v1 test vectors, shared with posthog-node. -WELCOME_HASH = "v1:f74cd913f3aa2e5667a588f2865634e56693df77793ca594d18e9b5f89e3d2e6" -WELCOME_CRLF_TEXT_HASH = ( - "v1:8112876ee1b2dd7327f571b04b546ed057bfff5a904d9f0ae507b6a449ab6453" -) -UPLOAD_URL = ( - "https://us.posthog.com/api/projects/@current/messaging_templates/by_key/welcome/" -) - - -def send_local_template(emails: Emails, **overrides: Any): - return emails.send( - **{ - "to": "user@example.com", - **WELCOME, - "template_data": {"first_name": "Jane"}, - **overrides, - } - ) - - -@parameterized.expand( - [ - ("html_only", {}, WELCOME_HASH), - ("crlf_text", {"text": "\r\n\r"}, WELCOME_CRLF_TEXT_HASH), - ("empty_text", {"text": ""}, WELCOME_HASH), - ] -) -def test_a_local_template_is_sent_by_key_and_content_hash(_name, overrides, hash): - emails, adapter = emails_with([accepted("msg_1")]) - - result = send_local_template( - emails, from_="Acme ", distinct_id="user_123", **overrides - ) - - assert result.id == "msg_1" - [request] = adapter.requests - assert (request.method, request.url) == ("POST", SEND_URL) - assert json.loads(request.body) == { - "to": "user@example.com", - "template": {"key": "welcome", "hash": hash}, - "message_data": {"first_name": "Jane"}, - "from": "Acme ", - "capture_engagement": True, - "distinct_id": "user_123", - } - - -def uploaded() -> Reply: - return (201, {"id": "tpl_1", "key": "welcome", "result": "created"}, {}) - - -def test_an_outdated_local_template_is_uploaded_then_sent_again_with_the_same_key(): - emails, adapter = emails_with( - [rejected(409, "template_outdated"), uploaded(), accepted("msg_1")] - ) - - result = send_local_template(emails, text="Hi {{ variables.first_name }}") - - assert result.id == "msg_1" - first_send, upload, second_send = adapter.requests - assert (upload.method, upload.url) == ("PUT", UPLOAD_URL) - assert json.loads(upload.body) == { - "name": "welcome", - "subject": "Welcome, {{ variables.first_name }}", - "html": "

Hi {{ variables.first_name }}

", - "text": "Hi {{ variables.first_name }}", - "if_not_superseded": True, - "if_code_managed": True, - } - assert "Idempotency-Key" not in upload.headers - assert upload.headers["Authorization"] == f"Bearer {SECRET_KEY}" - assert second_send.body == first_send.body - assert ( - second_send.headers["Idempotency-Key"] == first_send.headers["Idempotency-Key"] - ) - - -@parameterized.expand( - [ - ( - "refused_upload", - rejected(409, "template_edited_in_posthog"), - "template_edited_in_posthog", - 2, - ), - ("outdated_again", None, "template_outdated", 3), - ] -) -def test_a_local_template_uploads_at_most_once_per_send( - _name, upload_reply, code, requests_made -): - replies = ( - [rejected(409, "template_outdated"), upload_reply] - if upload_reply - else [ - rejected(409, "template_outdated"), - uploaded(), - rejected(409, "template_outdated"), - ] - ) - emails, adapter = emails_with(replies) - - with pytest.raises(EmailError) as raised: - send_local_template(emails) - - assert (raised.value.status_code, raised.value.code, raised.value.name) == ( - 409, - code, - "validation_error", - ) - assert len(adapter.requests) == requests_made - - -def test_rendered_content_is_never_uploaded(): - emails, adapter = emails_with([rejected(409, "template_outdated")]) - - with pytest.raises(EmailError): - send_code(emails, html=HTML) - - assert [request.method for request in adapter.requests] == ["POST"] - - -@pytest.mark.asyncio -async def test_async_uploads_an_outdated_local_template_then_sends_again(): - seen: list[httpx.Request] = [] - replies = [ - httpx.Response(409, json={"code": "template_outdated"}), - httpx.Response(201, json={"id": "tpl_1", "result": "created"}), - httpx.Response(202, json={"message_id": "msg_1"}), - ] - - def handler(request: httpx.Request) -> httpx.Response: - seen.append(request) - return replies.pop(0) - - async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: - result = await AsyncEmails(SECRET_KEY, http_client=client).send( - to="user@example.com", **WELCOME, template_data={"first_name": "Jane"} - ) - - assert result.id == "msg_1" - assert [(request.method, str(request.url)) for request in seen] == [ - ("POST", SEND_URL), - ("PUT", UPLOAD_URL), - ("POST", SEND_URL), - ] - first_send, upload, second_send = seen - assert json.loads(first_send.content)["template"] == { - "key": "welcome", - "hash": WELCOME_HASH, - } - assert "Idempotency-Key" not in upload.headers - assert second_send.content == first_send.content - assert ( - second_send.headers["Idempotency-Key"] == first_send.headers["Idempotency-Key"] - ) - - @parameterized.expand( [ ("both_forms", {"template_id": TEMPLATE_ID, "html": HTML}, TypeError), @@ -890,19 +720,14 @@ def handler(request: httpx.Request) -> httpx.Response: ("key_too_long", {"template_key": "k" * 401, "html": HTML}, ValueError), ("empty_html", {"html": ""}, ValueError), ("empty_html_and_text", {"html": "", "text": ""}, ValueError), - ("blank_cc", {"html": HTML, "cc": " "}, ValueError), - ("cc_not_text", {"html": HTML, "cc": 123}, TypeError), - ("template_data_list", {"html": HTML, "template_data": [1]}, TypeError), - ( - "local_template_without_html", - {"text": "Hi", "template_data": {"first_name": "Jane"}}, - ValueError, - ), + ("blank_reply_to", {"html": HTML, "reply_to": " "}, ValueError), + ("reply_to_not_text", {"html": HTML, "reply_to": 123}, TypeError), ( - "local_template_with_empty_html", - {"html": "", "text": "Hi", "template_data": {"first_name": "Jane"}}, - ValueError, + "template_data_on_code_form", + {"html": HTML, "template_data": {"first_name": "Jane"}}, + TypeError, ), + ("cc_on_code_form", {"html": HTML, "cc": "a@example.com"}, TypeError), ( "capture_engagement_on_template_form", { @@ -913,16 +738,6 @@ def handler(request: httpx.Request) -> httpx.Response: }, TypeError, ), - ( - "cc_on_template_form", - { - "template_id": TEMPLATE_ID, - "template_key": None, - "subject": None, - "cc": "a@example.com", - }, - TypeError, - ), ] ) def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error): @@ -940,143 +755,3 @@ def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error ) assert adapter.requests == [] - - -COPIES_HEADERS_AND_TAGS = { - "cc": "manager@example.com", - "bcc": ["audit@example.com", "archive@example.com"], - "headers": {"X-Entity-Ref-ID": "order-123"}, - "tags": [{"name": "category", "value": "password_reset"}], -} -COPIES_HEADERS_AND_TAGS_BODY = { - "cc": ["manager@example.com"], - "bcc": ["audit@example.com", "archive@example.com"], - "headers": {"X-Entity-Ref-ID": "order-123"}, - "tags": [{"name": "category", "value": "password_reset"}], -} - - -def test_rendered_content_carries_copies_headers_and_tags(): - emails, adapter = emails_with([accepted()]) - - send_code(emails, html=HTML, **COPIES_HEADERS_AND_TAGS) - - [request] = adapter.requests - assert json.loads(request.body) == { - "to": "user@example.com", - "template_key": "password-reset", - "subject": "Reset your password", - "html": HTML, - "capture_engagement": True, - **COPIES_HEADERS_AND_TAGS_BODY, - } - - -def test_a_server_that_does_not_take_a_field_refuses_the_send_and_names_it(): - refused = { - "type": "invalid_request", - "code": "invalid_request", - "detail": "cc: PostHog doesn't take this field on this route.", - "attr": "cc", - } - emails, adapter = emails_with([(400, refused, {}), accepted()]) - - with pytest.raises(EmailError) as raised: - send_code(emails, html=HTML, cc="manager@example.com") - - error = raised.value - assert (error.name, error.code, error.field) == ( - "validation_error", - "invalid_request", - "cc", - ) - assert len(adapter.requests) == 1 - - -def test_a_local_template_sends_copies_headers_and_tags_but_never_uploads_them(): - emails, adapter = emails_with( - [rejected(409, "template_outdated"), uploaded(), accepted()] - ) - - send_local_template(emails, **COPIES_HEADERS_AND_TAGS) - - first_send, upload, _ = adapter.requests - sent = json.loads(first_send.body) - assert {field: sent[field] for field in COPIES_HEADERS_AND_TAGS_BODY} == ( - COPIES_HEADERS_AND_TAGS_BODY - ) - assert not set(COPIES_HEADERS_AND_TAGS_BODY) & set(json.loads(upload.body)) - - -INVITE = b"BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n" - - -def test_attachments_are_sent_base64_encoded_once_across_retries(): - emails, adapter = emails_with([rejected(503, "delivery_unavailable"), accepted()]) - - send_code( - emails, - html=HTML, - attachments=[ - { - "filename": "invite.ics", - "content": INVITE, - "content_type": "text/calendar; method=REQUEST", - }, - {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, - ], - ) - - first, retry = adapter.requests - assert json.loads(first.body)["attachments"] == [ - { - "filename": "invite.ics", - "content": "QkVHSU46VkNBTEVOREFSDQpFTkQ6VkNBTEVOREFSDQo=", - "content_type": "text/calendar; method=REQUEST", - }, - {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, - ] - assert retry.body == first.body - - -@parameterized.expand( - [ - ("bytearray", bytearray(INVITE)), - ("memoryview", memoryview(INVITE)), - ( - "strided_memoryview", - memoryview(bytes(b for byte in INVITE for b in (byte, 0)))[::2], - ), - ] -) -def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, content): - emails, adapter = emails_with([accepted()]) - - send_code( - emails, html=HTML, attachments=[{"filename": "invite.ics", "content": content}] - ) - - [request] = adapter.requests - assert json.loads(request.body)["attachments"] == [ - { - "filename": "invite.ics", - "content": "QkVHSU46VkNBTEVOREFSDQpFTkQ6VkNBTEVOREFSDQo=", - } - ] - - -@parameterized.expand( - [ - ("list_of_ints", {"filename": "a.txt", "content": [104, 105]}), - ("no_filename", {"content": b"104"}), - ("unknown_field", {"filename": "a.txt", "content": b"104", "path": "/tmp/a"}), - ] -) -def test_rejects_an_attachment_that_is_not_a_file(_name, attachment): - emails, adapter = emails_with([]) - - with pytest.raises(TypeError) as raised: - send_code(emails, html=HTML, attachments=[attachment]) - - assert "104" not in str(raised.value) - assert adapter.requests == [] diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index e3a4ae521..86395957e 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -886,23 +886,13 @@ attribute posthog.debug = False attribute posthog.default_client = None attribute posthog.disable_geoip = True attribute posthog.disabled = False -attribute posthog.emails.CodeEmailArgs.attachments: NotRequired[Optional[Sequence[EmailAttachment]]] -attribute posthog.emails.CodeEmailArgs.bcc: NotRequired[Optional[Union[str, Sequence[str]]]] attribute posthog.emails.CodeEmailArgs.capture_engagement: NotRequired[Optional[bool]] -attribute posthog.emails.CodeEmailArgs.cc: NotRequired[Optional[Union[str, Sequence[str]]]] attribute posthog.emails.CodeEmailArgs.from_: NotRequired[Optional[str]] -attribute posthog.emails.CodeEmailArgs.headers: NotRequired[Optional[Mapping[str, str]]] attribute posthog.emails.CodeEmailArgs.html: NotRequired[Optional[str]] attribute posthog.emails.CodeEmailArgs.reply_to: NotRequired[Optional[Union[str, Sequence[str]]]] attribute posthog.emails.CodeEmailArgs.subject: str -attribute posthog.emails.CodeEmailArgs.tags: NotRequired[Optional[Sequence[EmailTag]]] -attribute posthog.emails.CodeEmailArgs.template_data: NotRequired[Optional[Mapping[str, Any]]] attribute posthog.emails.CodeEmailArgs.template_key: str attribute posthog.emails.CodeEmailArgs.text: NotRequired[Optional[str]] -attribute posthog.emails.EmailAttachment.content: Union[bytes, bytearray, memoryview, str] -attribute posthog.emails.EmailAttachment.content_id: NotRequired[str] -attribute posthog.emails.EmailAttachment.content_type: NotRequired[str] -attribute posthog.emails.EmailAttachment.filename: str attribute posthog.emails.EmailError.code = code attribute posthog.emails.EmailError.docs_url = docs_url attribute posthog.emails.EmailError.field = field @@ -917,8 +907,6 @@ attribute posthog.emails.EmailStatus.id: str attribute posthog.emails.EmailStatus.reason: Optional[str] = None attribute posthog.emails.EmailStatus.status: str attribute posthog.emails.EmailStatus.status_at: datetime -attribute posthog.emails.EmailTag.name: str -attribute posthog.emails.EmailTag.value: str attribute posthog.emails.EmailWarning.code: str attribute posthog.emails.EmailWarning.fix: Optional[str] = None attribute posthog.emails.EmailWarning.message: str @@ -1354,10 +1342,8 @@ class posthog.consumer.Consumer(queue, api_key, flush_at=100, host=None, on_erro class posthog.contexts.ContextScope(parent=None, fresh: bool = False, capture_exceptions: bool = True, client: Optional[Client] = None) class posthog.emails.AsyncEmails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, http_client: Any = None, migrated_from: Optional[str] = None) class posthog.emails.CodeEmailArgs -class posthog.emails.EmailAttachment class posthog.emails.EmailError(*, name: str, code: str, message: str, status_code: Optional[int] = None, why: Optional[str] = None, fix: Optional[str] = None, docs_url: Optional[str] = None, field: Optional[str] = None, retry_after: Optional[float] = None) class posthog.emails.EmailStatus(id: str, status: str, created_at: datetime, status_at: datetime, reason: Optional[str] = None) -class posthog.emails.EmailTag class posthog.emails.EmailWarning(code: str, message: str, fix: Optional[str] = None) class posthog.emails.Emails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, session: Optional[requests.Session] = None, migrated_from: Optional[str] = None) class posthog.emails.SendEmailResult(id: str, replayed: bool = False, warnings: tuple[EmailWarning, ...] = ()) From 9a4dd48c905b458ba299a8940350f6b1e2e4f572 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:23:14 +0000 Subject: [PATCH 03/39] test(emails): cover the async code-form send and migrated_from Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 96fff6dac..f84a68ee6 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -698,6 +698,50 @@ def test_rendered_content_is_posted_as_is(_name, overrides, extra_body): } +@pytest.mark.asyncio +async def test_async_client_posts_rendered_content_and_names_the_provider(): + seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(202, json={"message_id": "msg_1", "replayed": True}) + + client = AsyncPosthog("phc_test", secret_key=SECRET_KEY, migrated_from="resend") + client.emails._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + result = await client.emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=HTML, + text="Reset it here.", + from_="Acme ", + reply_to="help@example.com", + capture_engagement=False, + distinct_id="user_123", + idempotency_key="reset:user_123", + ) + await client.shutdown() + + assert (result.id, result.replayed) == ("msg_1", True) + [request] = seen + assert (request.method, str(request.url)) == ("POST", SEND_URL) + assert request.headers["Authorization"] == f"Bearer {SECRET_KEY}" + assert request.headers["Idempotency-Key"] == "reset:user_123" + assert request.headers["X-PostHog-Migrated-From"] == "resend" + assert json.loads(request.content) == { + "to": "user@example.com", + "template_key": "password-reset", + "subject": "Reset your password", + "html": HTML, + "text": "Reset it here.", + "from": "Acme ", + "reply_to": ["help@example.com"], + "capture_engagement": False, + "distinct_id": "user_123", + } + + @parameterized.expand( [ ("both_forms", {"template_id": TEMPLATE_ID, "html": HTML}, TypeError), From 685367c4587798c179e8281a6ed1e5ca1c2ed084 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:07:28 +0000 Subject: [PATCH 04/39] feat: send cc, bcc and attachments with emails.send A code send takes cc and bcc as one address or a list, and attachments as files with bytes or Base64 content. The client encodes bytes once, before any retry. A file it can't send raises EmailError with the code invalid_attachment and a field such as attachments__0__content, and the error never repeats the file's content. A template_id send refuses all three fields before any request. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_strict_types.sh | 7 +- .sampo/changesets/emails-code-form.md | 4 +- posthog/emails.py | 117 +++++++++++++++- posthog/test/test_emails.py | 188 +++++++++++++++++++++++++- references/public_api_snapshot.txt | 8 ++ 5 files changed, 316 insertions(+), 8 deletions(-) diff --git a/.github/scripts/check_strict_types.sh b/.github/scripts/check_strict_types.sh index 4e41c9e8f..36935a0b7 100755 --- a/.github/scripts/check_strict_types.sh +++ b/.github/scripts/check_strict_types.sh @@ -44,7 +44,8 @@ _ = (flag_value, all_flags, enabled, payload, evaluations, active) emails = Emails("phs_example") template_send: SendEmailResult = emails.send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", template_data={"name": "Ada"}) rendered_send: SendEmailResult = emails.send(to="user@example.com", template_key="password-reset", subject="Reset", html="

Reset

", from_="Acme ", reply_to=["help@example.com"], capture_engagement=False, max_retries=0) -_ = (template_send, rendered_send) +invite_send: SendEmailResult = emails.send(to="user@example.com", template_key="invite", subject="Invite", html="

Invite

", cc="team@example.com", bcc=["audit@example.com"], attachments=[{"filename": "invite.ics", "content": b"BEGIN:VCALENDAR", "content_type": "text/calendar"}, {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}]) +_ = (template_send, rendered_send, invite_send) async def async_email_types() -> None: @@ -90,7 +91,9 @@ Emails("phs_example").send(to="user@example.com", template_key="welcome", html=" Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", cc="team@example.com") # expected-type-error Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", capture_engagement=False) # expected-type-error Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", template_data={"name": "Ada"}) # expected-type-error -Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", cc="team@example.com") # expected-type-error +Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", attachments=[{"filename": "a.txt", "content": [104, 105]}]) # expected-type-error +Emails("phs_example").send(to="user@example.com", template_key="welcome", subject="Hi", html="

Hi

", attachments=[{"filename": "a.txt", "content": b"hi", "path": "/tmp/a.txt"}]) # expected-type-error +Emails("phs_example").send(to="user@example.com", template_id="0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b", attachments=[{"filename": "a.txt", "content": b"hi"}]) # expected-type-error async def invalid_async_email() -> None: diff --git a/.sampo/changesets/emails-code-form.md b/.sampo/changesets/emails-code-form.md index f02c6d9bd..ea2dadc9d 100644 --- a/.sampo/changesets/emails-code-form.md +++ b/.sampo/changesets/emails-code-form.md @@ -4,4 +4,6 @@ pypi/posthog: minor Send transactional email from content in your code with `posthog.emails.send(template_key=..., subject=..., html=...)`. The client sends the content as is, so render a React email to HTML in your app first. A send that mixes `template_id` and `template_key` fails type checks and raises `TypeError`. -A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `from_`, `reply_to` and `max_retries`. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. An optional field of a code send passed as `None` counts as left out. +A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `from_`, `reply_to`, `cc`, `bcc`, `attachments` and `max_retries`. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. An optional field of a code send passed as `None` counts as left out. + +An attachment is a `filename` and its `content`, with an optional `content_type` and `content_id`. Its `content` can be `bytes`, `bytearray` or `memoryview`, which the client encodes as Base64 once per send, or a Base64 string, which it sends as is. PostHog can't fetch an attachment from a path or URL. An attachment the client can't send raises `EmailError` with the code `invalid_attachment` before any request, and the error never repeats the file's content. A `template_id` send doesn't take `cc`, `bcc` or `attachments`. diff --git a/posthog/emails.py b/posthog/emails.py index 8237e2b47..dc338d21e 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -87,6 +87,7 @@ from __future__ import annotations import asyncio +import base64 import importlib import json import logging @@ -127,6 +128,7 @@ __all__ = [ "AsyncEmails", "CodeEmailArgs", + "EmailAttachment", "EmailError", "EmailStatus", "EmailWarning", @@ -196,6 +198,25 @@ class SendEmailResult: warnings: tuple[EmailWarning, ...] = () +class EmailAttachment(TypedDict): + """A file sent with the email. + + Args: + filename: The file name the recipient sees, such as ``invite.ics``. + content: The file as ``bytes``, ``bytearray`` or ``memoryview``, which + the client encodes as Base64, or already Base64 encoded as a string. + PostHog can't fetch a file from a path or URL. + content_type: The MIME type. Without it, the type follows the file name. + content_id: Shows the file inline: reference it in ``html`` as + ```` with ``content_id`` ``logo``. + """ + + filename: str + content: Union[bytes, bytearray, memoryview, str] + content_type: NotRequired[str] + content_id: NotRequired[str] + + class _RecipientArgs(TypedDict): to: str distinct_id: NotRequired[Optional[str]] @@ -229,6 +250,10 @@ class CodeEmailArgs(_RecipientArgs): An optional field passed as ``None`` counts as left out. + A PostHog version that doesn't take a field yet, such as ``cc``, refuses + the send with ``invalid_request`` and names the field in + ``EmailError.field``. It never drops the field and sends the rest. + Args: to: One recipient email address. template_key: Names the email, so its metrics, logs and status group @@ -240,6 +265,13 @@ class CodeEmailArgs(_RecipientArgs): without a plain-text part. from_: A verified sender, such as ``"Acme "``. reply_to: One address or a list of addresses. + cc: One address or a list of addresses that get a visible copy, at + most 10. + bcc: One address or a list of addresses that get a hidden copy, at + most 10. + attachments: Files sent with the email, at most 10 and 512 KB in + total. A file the client can't send raises :class:`EmailError` + with ``code`` ``invalid_attachment``. capture_engagement: Capture the email's opens and clicks as events of the recipient. Defaults to ``True``. A project whose admin turned engagement capture off captures nothing, whatever the send asks. A @@ -261,15 +293,22 @@ class CodeEmailArgs(_RecipientArgs): text: NotRequired[Optional[str]] from_: NotRequired[Optional[str]] reply_to: NotRequired[Optional[Union[str, Sequence[str]]]] + cc: NotRequired[Optional[Union[str, Sequence[str]]]] + bcc: NotRequired[Optional[Union[str, Sequence[str]]]] + attachments: NotRequired[Optional[Sequence[EmailAttachment]]] capture_engagement: NotRequired[Optional[bool]] +_ATTACHMENT_FIELDS = frozenset(EmailAttachment.__annotations__) _OPTIONAL_CODE_FIELDS = frozenset( { "html", "text", "from_", "reply_to", + "cc", + "bcc", + "attachments", "capture_engagement", } ) @@ -376,8 +415,11 @@ def _sender(args: Mapping[str, Any]) -> dict[str, Any]: sender: dict[str, Any] = {} if "from_" in args: sender["from"] = _nonempty_text(args, "from_") - if "reply_to" in args: - sender["reply_to"] = _addresses("reply_to", args["reply_to"]) + for field in ("reply_to", "cc", "bcc"): + if field in args: + sender[field] = _addresses(field, args[field]) + if "attachments" in args: + sender["attachments"] = _attachments(args["attachments"]) capture_engagement = args.get("capture_engagement", True) if not isinstance(capture_engagement, bool): raise TypeError("capture_engagement must be True or False.") @@ -396,6 +438,75 @@ def _addresses(field: str, value: Any) -> list[str]: return list(addresses) +def _invalid_attachment(message: str, field: str) -> EmailError: + # The message never holds the file's content, so the error is safe to log. + return EmailError( + name="validation_error", + code="invalid_attachment", + message=message, + field=field, + ) + + +def _attachments(attachments: Any) -> list[dict[str, str]]: + if isinstance( + attachments, (str, bytes, bytearray, memoryview, Mapping) + ) or not isinstance(attachments, Sequence): + raise _invalid_attachment("attachments must be a list of files.", "attachments") + return [_attachment(index, file) for index, file in enumerate(attachments)] + + +def _attachment(index: int, file: Any) -> dict[str, str]: + field = f"attachments__{index}" + if not isinstance(file, Mapping): + raise _invalid_attachment( + "An attachment is a mapping with filename and content. " + "Pass the file's bytes as its content.", + field, + ) + if "path" in file: + raise _invalid_attachment( + "PostHog can't fetch an attachment from a path or URL. " + "Read the file and pass its bytes as content instead.", + f"{field}__path", + ) + unknown = next((name for name in file if name not in _ATTACHMENT_FIELDS), None) + if unknown is not None: + raise _invalid_attachment( + "An attachment takes filename, content, content_type and content_id, " + f"not {unknown}.", + f"{field}__{unknown}", + ) + if not isinstance(file.get("filename"), str): + raise _invalid_attachment( + "An attachment needs its filename as a string.", f"{field}__filename" + ) + attachment = { + "filename": file["filename"], + "content": _base64_content(field, file.get("content")), + } + for name in ("content_type", "content_id"): + if name in file: + if not isinstance(file[name], str): + raise _invalid_attachment( + f"An attachment's {name} must be a string.", f"{field}__{name}" + ) + attachment[name] = file[name] + return attachment + + +def _base64_content(field: str, content: Any) -> str: + if isinstance(content, (bytes, bytearray, memoryview)): + return base64.b64encode(bytes(content)).decode("ascii") + if isinstance(content, str): + return content + raise _invalid_attachment( + "An attachment needs content: bytes or a Base64 string. " + "Convert a list of byte values with bytes(content).", + f"{field}__content", + ) + + def _string(args: Mapping[str, Any], name: str) -> str: value = args[name] if not isinstance(value, str): @@ -847,6 +958,8 @@ def send(self, **kwargs: Any) -> SendEmailResult: Raises: EmailError: PostHog did not queue the email, or the client could not tell whether it did. + ``code`` is ``invalid_attachment`` for an attachment the + client can't send, before any request. TypeError, ValueError: An argument has the wrong type or format. Examples: diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index f84a68ee6..37dc98576 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -620,6 +620,8 @@ def handler(request: httpx.Request) -> httpx.Response: HTML = "

Reset it here.

" +INVITE = b"BEGIN:VCALENDAR\r\nEND:VCALENDAR\r\n" +INVITE_BASE64 = "QkVHSU46VkNBTEVOREFSDQpFTkQ6VkNBTEVOREFSDQo=" def send_code(emails: Emails, **overrides: Any): @@ -644,6 +646,8 @@ def send_code(emails: Emails, **overrides: Any): "text": "Reset it here.", "from_": "Acme ", "reply_to": "help@example.com", + "cc": "manager@example.com", + "bcc": ["audit@example.com", "archive@example.com"], "capture_engagement": False, "distinct_id": "user_123", }, @@ -652,6 +656,8 @@ def send_code(emails: Emails, **overrides: Any): "text": "Reset it here.", "from": "Acme ", "reply_to": ["help@example.com"], + "cc": ["manager@example.com"], + "bcc": ["audit@example.com", "archive@example.com"], "capture_engagement": False, "distinct_id": "user_123", }, @@ -663,8 +669,8 @@ def send_code(emails: Emails, **overrides: Any): ), ( "empty_address_lists", - {"html": HTML, "reply_to": []}, - {"html": HTML, "reply_to": []}, + {"html": HTML, "reply_to": [], "cc": [], "bcc": [], "attachments": []}, + {"html": HTML, "reply_to": [], "cc": [], "bcc": [], "attachments": []}, ), ( "none_as_left_out", @@ -673,6 +679,9 @@ def send_code(emails: Emails, **overrides: Any): "text": None, "from_": None, "reply_to": None, + "cc": None, + "bcc": None, + "attachments": None, "capture_engagement": None, "distinct_id": None, }, @@ -717,6 +726,9 @@ def handler(request: httpx.Request) -> httpx.Response: text="Reset it here.", from_="Acme ", reply_to="help@example.com", + cc="manager@example.com", + bcc=["audit@example.com"], + attachments=[{"filename": "invite.ics", "content": INVITE}], capture_engagement=False, distinct_id="user_123", idempotency_key="reset:user_123", @@ -737,6 +749,9 @@ def handler(request: httpx.Request) -> httpx.Response: "text": "Reset it here.", "from": "Acme ", "reply_to": ["help@example.com"], + "cc": ["manager@example.com"], + "bcc": ["audit@example.com"], + "attachments": [{"filename": "invite.ics", "content": INVITE_BASE64}], "capture_engagement": False, "distinct_id": "user_123", } @@ -766,12 +781,13 @@ def handler(request: httpx.Request) -> httpx.Response: ("empty_html_and_text", {"html": "", "text": ""}, ValueError), ("blank_reply_to", {"html": HTML, "reply_to": " "}, ValueError), ("reply_to_not_text", {"html": HTML, "reply_to": 123}, TypeError), + ("blank_cc", {"html": HTML, "cc": " "}, ValueError), + ("bcc_not_text", {"html": HTML, "bcc": [123]}, TypeError), ( "template_data_on_code_form", {"html": HTML, "template_data": {"first_name": "Jane"}}, TypeError, ), - ("cc_on_code_form", {"html": HTML, "cc": "a@example.com"}, TypeError), ( "capture_engagement_on_template_form", { @@ -782,6 +798,36 @@ def handler(request: httpx.Request) -> httpx.Response: }, TypeError, ), + ( + "cc_on_template_form", + { + "template_id": TEMPLATE_ID, + "template_key": None, + "subject": None, + "cc": "a@example.com", + }, + TypeError, + ), + ( + "bcc_on_template_form", + { + "template_id": TEMPLATE_ID, + "template_key": None, + "subject": None, + "bcc": ["a@example.com"], + }, + TypeError, + ), + ( + "attachments_on_template_form", + { + "template_id": TEMPLATE_ID, + "template_key": None, + "subject": None, + "attachments": [{"filename": "a.txt", "content": b"hi"}], + }, + TypeError, + ), ] ) def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error): @@ -799,3 +845,139 @@ def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error ) assert adapter.requests == [] + + +def test_a_server_that_does_not_take_a_field_refuses_the_send_and_names_it(): + refused = { + "type": "invalid_request", + "code": "invalid_request", + "detail": "cc: PostHog doesn't take this field on this route.", + "attr": "cc", + } + emails, adapter = emails_with([(400, refused, {}), accepted()]) + + with pytest.raises(EmailError) as raised: + send_code(emails, html=HTML, cc="manager@example.com") + + error = raised.value + assert (error.name, error.code, error.field) == ( + "validation_error", + "invalid_request", + "cc", + ) + assert len(adapter.requests) == 1 + + +def test_attachments_are_sent_base64_encoded_once_across_retries(): + emails, adapter = emails_with([rejected(503, "delivery_unavailable"), accepted()]) + + send_code( + emails, + html=HTML, + attachments=[ + { + "filename": "invite.ics", + "content": INVITE, + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, + ], + ) + + first, retry = adapter.requests + assert json.loads(first.body)["attachments"] == [ + { + "filename": "invite.ics", + "content": INVITE_BASE64, + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, + ] + assert retry.body == first.body + + +@parameterized.expand( + [ + ("bytearray", bytearray(INVITE)), + ("memoryview", memoryview(INVITE)), + ( + "strided_memoryview", + memoryview(bytes(b for byte in INVITE for b in (byte, 0)))[::2], + ), + ] +) +def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, content): + emails, adapter = emails_with([accepted()]) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": content}] + ) + + [request] = adapter.requests + assert json.loads(request.body)["attachments"] == [ + {"filename": "invite.ics", "content": INVITE_BASE64} + ] + + +SECRET_CONTENT = b"secret-file-content" + + +@parameterized.expand( + [ + ( + "one_file_not_a_list", + {"filename": "a.txt", "content": SECRET_CONTENT}, + "attachments", + ), + ("bytes_not_a_list", SECRET_CONTENT, "attachments"), + ("file_not_a_mapping", [SECRET_CONTENT], "attachments__0"), + ( + "list_of_ints", + [{"filename": "a.txt", "content": list(SECRET_CONTENT)}], + "attachments__0__content", + ), + ("no_content", [{"filename": "a.txt"}], "attachments__0__content"), + ("no_filename", [{"content": SECRET_CONTENT}], "attachments__0__filename"), + ( + "path", + [{"filename": "a.txt", "content": SECRET_CONTENT, "path": "/tmp/a.txt"}], + "attachments__0__path", + ), + ( + "unknown_field", + [ + { + "filename": "a.txt", + "content": SECRET_CONTENT, + "contentType": "text/plain", + } + ], + "attachments__0__contentType", + ), + ( + "content_type_not_text", + [ + {"filename": "a.txt", "content": "aGk="}, + {"filename": "b.txt", "content": SECRET_CONTENT, "content_type": 1}, + ], + "attachments__1__content_type", + ), + ] +) +def test_rejects_an_attachment_it_cannot_send_without_repeating_its_content( + _name, attachments, field +): + emails, adapter = emails_with([]) + + with pytest.raises(EmailError) as raised: + send_code(emails, html=HTML, attachments=attachments) + + error = raised.value + assert (error.name, error.code, error.field) == ( + "validation_error", + "invalid_attachment", + field, + ) + assert "secret-file-content" not in str(error) + assert "115, 101" not in str(error) + assert adapter.requests == [] diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 86395957e..7817b27fd 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -886,13 +886,20 @@ attribute posthog.debug = False attribute posthog.default_client = None attribute posthog.disable_geoip = True attribute posthog.disabled = False +attribute posthog.emails.CodeEmailArgs.attachments: NotRequired[Optional[Sequence[EmailAttachment]]] +attribute posthog.emails.CodeEmailArgs.bcc: NotRequired[Optional[Union[str, Sequence[str]]]] attribute posthog.emails.CodeEmailArgs.capture_engagement: NotRequired[Optional[bool]] +attribute posthog.emails.CodeEmailArgs.cc: NotRequired[Optional[Union[str, Sequence[str]]]] attribute posthog.emails.CodeEmailArgs.from_: NotRequired[Optional[str]] attribute posthog.emails.CodeEmailArgs.html: NotRequired[Optional[str]] attribute posthog.emails.CodeEmailArgs.reply_to: NotRequired[Optional[Union[str, Sequence[str]]]] attribute posthog.emails.CodeEmailArgs.subject: str attribute posthog.emails.CodeEmailArgs.template_key: str attribute posthog.emails.CodeEmailArgs.text: NotRequired[Optional[str]] +attribute posthog.emails.EmailAttachment.content: Union[bytes, bytearray, memoryview, str] +attribute posthog.emails.EmailAttachment.content_id: NotRequired[str] +attribute posthog.emails.EmailAttachment.content_type: NotRequired[str] +attribute posthog.emails.EmailAttachment.filename: str attribute posthog.emails.EmailError.code = code attribute posthog.emails.EmailError.docs_url = docs_url attribute posthog.emails.EmailError.field = field @@ -1342,6 +1349,7 @@ class posthog.consumer.Consumer(queue, api_key, flush_at=100, host=None, on_erro class posthog.contexts.ContextScope(parent=None, fresh: bool = False, capture_exceptions: bool = True, client: Optional[Client] = None) class posthog.emails.AsyncEmails(secret_key: Optional[str], *, host: Optional[str] = None, timeout: float = 15, max_retries: int = 2, disabled: bool = False, http_client: Any = None, migrated_from: Optional[str] = None) class posthog.emails.CodeEmailArgs +class posthog.emails.EmailAttachment class posthog.emails.EmailError(*, name: str, code: str, message: str, status_code: Optional[int] = None, why: Optional[str] = None, fix: Optional[str] = None, docs_url: Optional[str] = None, field: Optional[str] = None, retry_after: Optional[float] = None) class posthog.emails.EmailStatus(id: str, status: str, created_at: datetime, status_at: datetime, reason: Optional[str] = None) class posthog.emails.EmailWarning(code: str, message: str, fix: Optional[str] = None) From f906fb3bb04ab8ccc0cd209bcb7bb5746c92b753 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:18:41 +0000 Subject: [PATCH 05/39] fix: refuse every unknown attachment key, a None key included next(..., None) used None as its no-match value, so a None key hid the unknown keys after it. Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index dc338d21e..bd415a11b 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -470,12 +470,12 @@ def _attachment(index: int, file: Any) -> dict[str, str]: "Read the file and pass its bytes as content instead.", f"{field}__path", ) - unknown = next((name for name in file if name not in _ATTACHMENT_FIELDS), None) - if unknown is not None: + unknown = [name for name in file if name not in _ATTACHMENT_FIELDS] + if unknown: raise _invalid_attachment( "An attachment takes filename, content, content_type and content_id, " - f"not {unknown}.", - f"{field}__{unknown}", + f"not {unknown[0]}.", + f"{field}__{unknown[0]}", ) if not isinstance(file.get("filename"), str): raise _invalid_attachment( From bf687be1e40f3f9c097f325793a505555bff78be Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:20:36 +0000 Subject: [PATCH 06/39] test: cover a None attachment key before an unknown one Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 37dc98576..c50191171 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -954,6 +954,18 @@ def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, conte ], "attachments__0__contentType", ), + ( + "none_key_before_an_unknown_field", + [ + { + None: "ignored", + "filename": "a.txt", + "content": SECRET_CONTENT, + "contentType": "text/plain", + } + ], + "attachments__0__None", + ), ( "content_type_not_text", [ From a0808620494ca60aba0c98718512656384e1b7b9 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:20:01 +0000 Subject: [PATCH 07/39] docs: describe an older server's refusal without its error code Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 4 ++-- posthog/test/test_emails.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index bd415a11b..54f785ef7 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -251,8 +251,8 @@ class CodeEmailArgs(_RecipientArgs): An optional field passed as ``None`` counts as left out. A PostHog version that doesn't take a field yet, such as ``cc``, refuses - the send with ``invalid_request`` and names the field in - ``EmailError.field``. It never drops the field and sends the rest. + the send with an error that names the field in ``EmailError.field``. It + never drops the field and sends the rest. Args: to: One recipient email address. diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index c50191171..ca138f596 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -849,9 +849,9 @@ def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error def test_a_server_that_does_not_take_a_field_refuses_the_send_and_names_it(): refused = { - "type": "invalid_request", - "code": "invalid_request", - "detail": "cc: PostHog doesn't take this field on this route.", + "type": "validation_error", + "code": "invalid_input", + "detail": "PostHog doesn't take this field, so it doesn't send the email without it.", "attr": "cc", } emails, adapter = emails_with([(400, refused, {}), accepted()]) @@ -862,7 +862,7 @@ def test_a_server_that_does_not_take_a_field_refuses_the_send_and_names_it(): error = raised.value assert (error.name, error.code, error.field) == ( "validation_error", - "invalid_request", + "invalid_input", "cc", ) assert len(adapter.requests) == 1 From b93ea5c117741c19427bb215f570b0a33528027a Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:42:38 +0000 Subject: [PATCH 08/39] fix(emails): leave reply_to out when the list is empty Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 7 ++++--- posthog/test/test_emails.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index 8237e2b47..16f342777 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -239,7 +239,7 @@ class CodeEmailArgs(_RecipientArgs): ``html``. HTML with no text in it, such as a single image, goes out without a plain-text part. from_: A verified sender, such as ``"Acme "``. - reply_to: One address or a list of addresses. + reply_to: One address or a list of addresses. An empty list sends no reply-to. capture_engagement: Capture the email's opens and clicks as events of the recipient. Defaults to ``True``. A project whose admin turned engagement capture off captures nothing, whatever the send asks. A @@ -376,8 +376,9 @@ def _sender(args: Mapping[str, Any]) -> dict[str, Any]: sender: dict[str, Any] = {} if "from_" in args: sender["from"] = _nonempty_text(args, "from_") - if "reply_to" in args: - sender["reply_to"] = _addresses("reply_to", args["reply_to"]) + reply_to = _addresses("reply_to", args["reply_to"]) if "reply_to" in args else [] + if reply_to: + sender["reply_to"] = reply_to capture_engagement = args.get("capture_engagement", True) if not isinstance(capture_engagement, bool): raise TypeError("capture_engagement must be True or False.") diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index f84a68ee6..eaf1c0949 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -662,9 +662,9 @@ def send_code(emails: Emails, **overrides: Any): {"html": HTML, "reply_to": ["a@example.com", "b@example.com"]}, ), ( - "empty_address_lists", - {"html": HTML, "reply_to": []}, + "empty_reply_to_left_out", {"html": HTML, "reply_to": []}, + {"html": HTML}, ), ( "none_as_left_out", From f368be744df8849688c995fa9174496328b554a9 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:00:56 +0000 Subject: [PATCH 09/39] fix(emails): refuse an attachment string that is not base64 A text file passed as a str was sent unchecked as if it were Base64. The client now raises invalid_attachment before any request and hints to pass the file's bytes instead. Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 9 +++++++++ posthog/test/test_emails.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/posthog/emails.py b/posthog/emails.py index e7e3a6e38..ade8cb8ac 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -88,6 +88,7 @@ import asyncio import base64 +import binascii import importlib import json import logging @@ -501,6 +502,14 @@ def _base64_content(field: str, content: Any) -> str: if isinstance(content, (bytes, bytearray, memoryview)): return base64.b64encode(bytes(content)).decode("ascii") if isinstance(content, str): + try: + base64.b64decode(content, validate=True) + except (binascii.Error, ValueError): + raise _invalid_attachment( + "An attachment's content string must be Base64. " + "Pass the file's bytes instead, such as text.encode() for text.", + f"{field}__content", + ) from None return content raise _invalid_attachment( "An attachment needs content: bytes or a Base64 string. " diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 22bfdb0ef..a997ef8f3 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -937,6 +937,16 @@ def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, conte "attachments__0__content", ), ("no_content", [{"filename": "a.txt"}], "attachments__0__content"), + ( + "text_not_base64", + [{"filename": "a.txt", "content": "secret-file-content"}], + "attachments__0__content", + ), + ( + "non_ascii_text", + [{"filename": "a.txt", "content": "secret-file-content é"}], + "attachments__0__content", + ), ("no_filename", [{"content": SECRET_CONTENT}], "attachments__0__filename"), ( "path", From 8e58646431f0a4261dc99411cab077fce1942f8e Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:01:01 +0000 Subject: [PATCH 10/39] chore(emails): name the refused-send test after what it guards Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index a997ef8f3..ffce687a7 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -847,7 +847,7 @@ def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error assert adapter.requests == [] -def test_a_server_that_does_not_take_a_field_refuses_the_send_and_names_it(): +def test_a_refused_send_names_the_field_from_attr(): refused = { "type": "validation_error", "code": "invalid_input", From 1b7fd903994f26202d75964a572ddc48f09d9682 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:01:01 +0000 Subject: [PATCH 11/39] chore(emails): drop a comment the name and test already state Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 1 - 1 file changed, 1 deletion(-) diff --git a/posthog/emails.py b/posthog/emails.py index ade8cb8ac..28a235de9 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -442,7 +442,6 @@ def _addresses(field: str, value: Any) -> list[str]: def _invalid_attachment(message: str, field: str) -> EmailError: - # The message never holds the file's content, so the error is safe to log. return EmailError( name="validation_error", code="invalid_attachment", From f5d6f0af755732ad2fb4e0e86be5282d6418d0f0 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:01:06 +0000 Subject: [PATCH 12/39] chore(emails): give cc, bcc and attachments their own changeset The base PR owns emails-code-form.md, so this PR no longer edits it. Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-code-attachments.md | 5 +++++ .sampo/changesets/emails-code-form.md | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .sampo/changesets/emails-code-attachments.md diff --git a/.sampo/changesets/emails-code-attachments.md b/.sampo/changesets/emails-code-attachments.md new file mode 100644 index 000000000..20e4cfbba --- /dev/null +++ b/.sampo/changesets/emails-code-attachments.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. An attachment the client can't send raises `EmailError` with the code `invalid_attachment` before any request. diff --git a/.sampo/changesets/emails-code-form.md b/.sampo/changesets/emails-code-form.md index ea2dadc9d..f02c6d9bd 100644 --- a/.sampo/changesets/emails-code-form.md +++ b/.sampo/changesets/emails-code-form.md @@ -4,6 +4,4 @@ pypi/posthog: minor Send transactional email from content in your code with `posthog.emails.send(template_key=..., subject=..., html=...)`. The client sends the content as is, so render a React email to HTML in your app first. A send that mixes `template_id` and `template_key` fails type checks and raises `TypeError`. -A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `from_`, `reply_to`, `cc`, `bcc`, `attachments` and `max_retries`. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. An optional field of a code send passed as `None` counts as left out. - -An attachment is a `filename` and its `content`, with an optional `content_type` and `content_id`. Its `content` can be `bytes`, `bytearray` or `memoryview`, which the client encodes as Base64 once per send, or a Base64 string, which it sends as is. PostHog can't fetch an attachment from a path or URL. An attachment the client can't send raises `EmailError` with the code `invalid_attachment` before any request, and the error never repeats the file's content. A `template_id` send doesn't take `cc`, `bcc` or `attachments`. +A send with `template_key` captures the email's opens and clicks unless it passes `capture_engagement=False`. A template made in PostHog follows the project's setting, so a `template_id` send doesn't take `capture_engagement`. A code send takes `from_`, `reply_to` and `max_retries`. Set `migrated_from`, such as `"resend"`, on the client or on `Emails` to send it with every email request. An optional field of a code send passed as `None` counts as left out. From ac102e0858af21df51595a97b1bda58b5fb3eb13 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:37:31 +0000 Subject: [PATCH 13/39] fix(emails): refuse an attachment with an empty filename Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 5 +++-- posthog/test/test_emails.py | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index 28a235de9..5e34ba20c 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -479,9 +479,10 @@ def _attachment(index: int, file: Any) -> dict[str, str]: f"not {unknown[0]}.", f"{field}__{unknown[0]}", ) - if not isinstance(file.get("filename"), str): + if not isinstance(file.get("filename"), str) or not file["filename"]: raise _invalid_attachment( - "An attachment needs its filename as a string.", f"{field}__filename" + "An attachment needs a filename, such as 'invoice.pdf'.", + f"{field}__filename", ) attachment = { "filename": file["filename"], diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index ffce687a7..0a580feb2 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -948,6 +948,11 @@ def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, conte "attachments__0__content", ), ("no_filename", [{"content": SECRET_CONTENT}], "attachments__0__filename"), + ( + "empty_filename", + [{"filename": "", "content": SECRET_CONTENT}], + "attachments__0__filename", + ), ( "path", [{"filename": "a.txt", "content": SECRET_CONTENT, "path": "/tmp/a.txt"}], From 85bb3504fd573726f59b946d3bfa30d3184c7340 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:18:57 +0000 Subject: [PATCH 14/39] feat(emails): upload attachments by hash, then send their references A code send hashes each file, asks PostHog which files it lacks, PUTs those straight to PostHog's file storage with the returned headers and without the secret key, then sends the email naming each file by its SHA-256. The client checks 10 files and the 40 MB message estimate before any request. A failed upload sends no email and surfaces the storage's answer. Part of PostHog/michaelr-context#81. Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-code-attachments.md | 2 +- posthog/emails.py | 361 +++++++++++++--- posthog/test/test_emails.py | 409 +++++++++++++++++-- typings/requests/__init__.pyi | 1 + 4 files changed, 681 insertions(+), 92 deletions(-) diff --git a/.sampo/changesets/emails-code-attachments.md b/.sampo/changesets/emails-code-attachments.md index 20e4cfbba..7559b5156 100644 --- a/.sampo/changesets/emails-code-attachments.md +++ b/.sampo/changesets/emails-code-attachments.md @@ -2,4 +2,4 @@ pypi/posthog: minor --- -A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. An attachment the client can't send raises `EmailError` with the code `invalid_attachment` before any request. +A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage, then sends the email with each file's SHA-256. A failed upload raises `EmailError` with the code `upload_failed` and sends no email. diff --git a/posthog/emails.py b/posthog/emails.py index 60089816a..d27c5f42e 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -59,6 +59,14 @@ ``"resend"``, on the client or on :class:`Emails`, so PostHog knows where your sends came from. +Attachments: a code send takes ``attachments``, at most 10 files and 40 MB +per message after Base64 encoding, about 29 MB of files. The client checks +both before any request. It then asks PostHog which files it lacks, uploads +each of those to the URL PostHog answers, without the secret key, and sends the +email naming each file by its SHA-256. A file the project already sent uploads +nothing. A failed upload sends no email. The upload URL's host is not the app +host, so a network that limits outbound requests must allow it. + Errors: a send that is not queued raises :class:`EmailError`. Branch on ``code``. The ones a setup usually hits: @@ -71,6 +79,18 @@ - ``client_disabled``: the client is disabled, so it sends nothing. - ``client_closed``: the client is shut down. +A send with attachments can also raise: + +- ``too_many_attachments`` or ``message_too_large``: the email is over the + limits above. The client raises these before any request. +- ``invalid_attachment``: the client can't send a file, such as one without + content. +- ``upload_failed``: PostHog's file storage refused an upload. ``message`` + and ``why`` carry its answer, and ``field`` names the attachment. +- ``upload_quota_exceeded`` (429): the project uploaded its daily limit of + file bytes. Files it already sent don't count. +- ``field_not_accepted``: the project can't send files yet. + Status: ``emails.get(message_id)`` reads what happened to a sent email, for 30 days after the send, as an :class:`EmailStatus`. A key with only ``transactional_email:read`` can read statuses but not send. ``get`` raises @@ -91,6 +111,7 @@ import asyncio import base64 import binascii +import hashlib import importlib import json import logging @@ -143,6 +164,7 @@ log = logging.getLogger("posthog") _SEND_PATH = "/api/projects/@current/messaging/send/email/" +_UPLOADS_PATH = "/api/projects/@current/messaging/attachments/uploads/" _STATUS_PATH = "/api/projects/@current/messaging/emails/{message_id}/" _TEMPLATE_KEY = re.compile(r"[a-zA-Z0-9_-]{1,400}") _MAX_RETRY_DELAY = 30.0 @@ -150,6 +172,12 @@ _HEADER_TOKEN = re.compile(r"[\x21-\x7e]+") _IN_FLIGHT_CODES = frozenset({"idempotency_key_in_flight"}) _QUOTA_CODES = frozenset({"quota_limited"}) +_DAILY_QUOTA_CODES = frozenset({"upload_quota_exceeded"}) +_MAX_ATTACHMENTS = 10 +_MAX_MESSAGE_BYTES = 40_000_000 +_HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 +_MESSAGE_LIMIT = "40 MB after Base64 encoding, about 29 MB of files" +_STORAGE_ERROR = re.compile(r"<(Code|Message)>([^<]{1,500})") class EmailError(Exception): @@ -206,9 +234,9 @@ class EmailAttachment(TypedDict): Args: filename: The file name the recipient sees, such as ``invite.ics``. - content: The file as ``bytes``, ``bytearray`` or ``memoryview``, which - the client encodes as Base64, or already Base64 encoded as a string. - PostHog can't fetch a file from a path or URL. + content: The file as ``bytes``, ``bytearray`` or ``memoryview``, or + Base64 encoded as a string. PostHog can't fetch a file from a path + or URL. content_type: The MIME type. Without it, the type follows the file name. content_id: Shows the file inline: reference it in ``html`` as ```` with ``content_id`` ``logo``. @@ -272,9 +300,12 @@ class CodeEmailArgs(_RecipientArgs): most 10. bcc: One address or a list of addresses that get a hidden copy, at most 10. - attachments: Files sent with the email, at most 10 and 512 KB in - total. A file the client can't send raises :class:`EmailError` - with ``code`` ``invalid_attachment``. + attachments: Files sent with the email: at most 10 files and 40 MB + per message after Base64 encoding, about 29 MB of files. The + client checks both before any request. It uploads each file that + PostHog doesn't hold yet, then sends the email. A file the client + can't send raises :class:`EmailError` with ``code`` + ``invalid_attachment``. capture_engagement: Capture the email's opens and clicks as events of the recipient. Defaults to ``True``. A project whose admin turned engagement capture off captures nothing, whatever the send asks. A @@ -343,12 +374,30 @@ class EmailStatus: reason: Optional[str] = None +@dataclass(frozen=True) +class _File: + sha256: str + content: bytes + + @property + def size(self) -> int: + return len(self.content) + + +@dataclass(frozen=True) +class _UploadTarget: + url: str + method: str + headers: dict[str, str] + + @dataclass(frozen=True) class _PreparedSend: path: str body: bytes idempotency_key: str max_retries: Optional[int] = None + files: tuple[_File, ...] = () def _prepare(args: Mapping[str, Any]) -> _PreparedSend: @@ -402,15 +451,21 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: if not (content.get("html") or content.get("text")): raise ValueError("html or text must be a nonempty string.") + references, files = _attachments(args.get("attachments", [])) + _check_limits(files, content) + payload: dict[str, Any] = {"to": to, "template_key": template_key, **content} if distinct_id is not None: payload["distinct_id"] = distinct_id payload.update(_sender(args)) + if references: + payload["attachments"] = references return _PreparedSend( path=_SEND_PATH, body=_encode(payload), idempotency_key=idempotency_key, max_retries=max_retries, + files=files, ) @@ -422,9 +477,6 @@ def _sender(args: Mapping[str, Any]) -> dict[str, Any]: addresses = _addresses(field, args[field]) if field in args else [] if addresses: sender[field] = addresses - attachments = _attachments(args["attachments"]) if "attachments" in args else [] - if attachments: - sender["attachments"] = attachments capture_engagement = args.get("capture_engagement", True) if not isinstance(capture_engagement, bool): raise TypeError("capture_engagement must be True or False.") @@ -452,15 +504,18 @@ def _invalid_attachment(message: str, field: str) -> EmailError: ) -def _attachments(attachments: Any) -> list[dict[str, str]]: +def _attachments( + attachments: Any, +) -> tuple[list[dict[str, str]], tuple[_File, ...]]: if isinstance( attachments, (str, bytes, bytearray, memoryview, Mapping) ) or not isinstance(attachments, Sequence): raise _invalid_attachment("attachments must be a list of files.", "attachments") - return [_attachment(index, file) for index, file in enumerate(attachments)] + prepared = [_attachment(index, file) for index, file in enumerate(attachments)] + return [reference for reference, _ in prepared], tuple(file for _, file in prepared) -def _attachment(index: int, file: Any) -> dict[str, str]: +def _attachment(index: int, file: Any) -> tuple[dict[str, str], _File]: field = f"attachments__{index}" if not isinstance(file, Mapping): raise _invalid_attachment( @@ -486,40 +541,93 @@ def _attachment(index: int, file: Any) -> dict[str, str]: "An attachment needs a filename, such as 'invoice.pdf'.", f"{field}__filename", ) - attachment = { - "filename": file["filename"], - "content": _base64_content(field, file.get("content")), - } + content = _file_bytes(field, file.get("content")) + sha256 = hashlib.sha256(content).hexdigest() + reference = {"sha256": sha256, "filename": file["filename"]} for name in ("content_type", "content_id"): if name in file: if not isinstance(file[name], str): raise _invalid_attachment( f"An attachment's {name} must be a string.", f"{field}__{name}" ) - attachment[name] = file[name] - return attachment + reference[name] = file[name] + return reference, _File(sha256=sha256, content=content) -def _base64_content(field: str, content: Any) -> str: +def _file_bytes(field: str, content: Any) -> bytes: if isinstance(content, (bytes, bytearray, memoryview)): - return base64.b64encode(bytes(content)).decode("ascii") - if isinstance(content, str): + file_bytes = bytes(content) + elif isinstance(content, str): try: - base64.b64decode(content, validate=True) + file_bytes = base64.b64decode(content, validate=True) except (binascii.Error, ValueError): raise _invalid_attachment( "An attachment's content string must be Base64. " "Pass the file's bytes instead, such as text.encode() for text.", f"{field}__content", ) from None - return content - raise _invalid_attachment( - "An attachment needs content: bytes or a Base64 string. " - "Convert a list of byte values with bytes(content).", - f"{field}__content", + else: + raise _invalid_attachment( + "An attachment needs content: bytes or a Base64 string. " + "Convert a list of byte values with bytes(content).", + f"{field}__content", + ) + if not file_bytes: + raise _invalid_attachment( + "An attachment's content is empty, and an empty file gives the " + "recipient nothing to open. Leave the file out.", + f"{field}__content", + ) + return file_bytes + + +def _check_limits(files: Sequence[_File], content: Mapping[str, str]) -> None: + if len(files) > _MAX_ATTACHMENTS: + raise _over_the_limit( + "too_many_attachments", + f"An email carries at most {_MAX_ATTACHMENTS} files, so no email was sent.", + fix=f"Send at most {_MAX_ATTACHMENTS} files, and link to any others " + "from the email body.", + ) + bodies = [ + body.encode("utf-8") + for body in (content.get("html"), content.get("text")) + if body + ] + sizes = [file.size for file in files] + [len(body) for body in bodies] + if files and _estimated_message_bytes(sizes) > _MAX_MESSAGE_BYTES: + raise _over_the_limit( + "message_too_large", + f"These files make the email larger than {_MESSAGE_LIMIT}, so no email was sent.", + fix="Send fewer or smaller files, or link to large files from the email body.", + ) + + +def _over_the_limit(code: str, message: str, *, fix: str) -> EmailError: + return EmailError( + name="validation_error", + code=code, + message=message, + why=f"An email carries at most {_MAX_ATTACHMENTS} files and {_MESSAGE_LIMIT}.", + fix=fix, + field="attachments", ) +def _estimated_message_bytes(sizes: Sequence[int]) -> int: + return _HEADERS_AND_MIME_BOUNDARIES_BYTES + sum( + _base64_transfer_bytes(size) for size in sizes + ) + + +def _base64_transfer_bytes(size: int) -> int: + return _ceil_div(_ceil_div(size, 3) * 4 * 78, 76) + + +def _ceil_div(dividend: int, divisor: int) -> int: + return -(-dividend // divisor) + + def _string(args: Mapping[str, Any], name: str) -> str: value = args[name] if not isinstance(value, str): @@ -669,6 +777,8 @@ def _error_name(status: Optional[int], code: str) -> str: else "validation_error" ) if status == 429: + if code in _DAILY_QUOTA_CODES: + return "daily_quota_exceeded" return ( "monthly_quota_exceeded" if code in _QUOTA_CODES else "rate_limit_exceeded" ) @@ -690,10 +800,18 @@ def _retry_after(header: Optional[str]) -> Optional[float]: return max(0.0, seconds) +def _json_object(body: bytes) -> dict[str, Any]: + try: + payload = json.loads(body) + except ValueError: + return {} + return payload if isinstance(payload, dict) else {} + + def _parse_response( - status: int, payload: Any, retry_after_header: Optional[str] + status: int, body: bytes, retry_after_header: Optional[str] ) -> SendEmailResult: - fields = payload if isinstance(payload, dict) else {} + fields = _json_object(body) if 200 <= status < 300: message_id = _text(fields.get("message_id")) if message_id is None: @@ -733,9 +851,9 @@ def _timestamp(value: Any) -> Optional[datetime]: def _parse_status_response( - status: int, payload: Any, retry_after_header: Optional[str] + status: int, body: bytes, retry_after_header: Optional[str] ) -> EmailStatus: - fields = payload if isinstance(payload, dict) else {} + fields = _json_object(body) if not 200 <= status < 300: raise _response_error(fields, status, retry_after_header) message_id = _text(fields.get("id")) @@ -760,6 +878,77 @@ def _parse_status_response( ) +def _upload_slots_parser( + files: Sequence[_File], +) -> _Parse[list[Optional[_UploadTarget]]]: + def parse( + status: int, body: bytes, retry_after_header: Optional[str] + ) -> list[Optional[_UploadTarget]]: + fields = _json_object(body) + if not 200 <= status < 300: + raise _response_error(fields, status, retry_after_header) + slots = fields.get("files") + if not isinstance(slots, list) or len(slots) != len(files): + raise _no_upload_slots(status) + return [_upload_target(slot, file, status) for slot, file in zip(slots, files)] + + return parse + + +def _upload_target(slot: Any, file: _File, status: int) -> Optional[_UploadTarget]: + if not isinstance(slot, dict) or slot.get("sha256") != file.sha256: + raise _no_upload_slots(status) + if slot.get("status") == "present": + return None + url, method, headers = ( + _text(slot.get("url")), + _text(slot.get("method", "PUT")), + slot.get("headers"), + ) + if ( + slot.get("status") != "upload" + or url is None + or method is None + or not isinstance(headers, dict) + or not all(isinstance(value, str) for value in headers.values()) + ): + raise _no_upload_slots(status) + return _UploadTarget(url=url, method=method, headers=headers) + + +def _no_upload_slots(status: int) -> EmailError: + return _sdk_error( + "api_error", "PostHog answered without an upload slot for each file.", status + ) + + +def _upload_parser(index: int) -> _Parse[None]: + def parse(status: int, body: bytes, retry_after_header: Optional[str]) -> None: + if not 200 <= status < 300: + raise _upload_error(index, status, body, retry_after_header) + + return parse + + +def _upload_error( + index: int, status: int, body: bytes, retry_after_header: Optional[str] +) -> EmailError: + answer = dict(_STORAGE_ERROR.findall(body.decode("utf-8", "replace"))) + storage_answer = " ".join(filter(None, (f"HTTP {status}", answer.get("Code")))) + return EmailError( + name=_error_name(status, "upload_failed"), + code="upload_failed", + message=f"PostHog's file storage refused the upload of attachments[{index}] " + f"with {storage_answer}, so no email was sent.", + status_code=status, + why=answer.get("Message"), + fix="Send the email again. If the upload keeps failing, check that the " + "file's bytes don't change while it is sent.", + field=f"attachments__{index}", + retry_after=_retry_after(retry_after_header), + ) + + def _response_error( fields: Mapping[str, Any], status: int, retry_after_header: Optional[str] ) -> EmailError: @@ -784,7 +973,7 @@ def _is_retryable(error: EmailError) -> bool: if error.status_code == 409: return error.code in _IN_FLIGHT_CODES if error.status_code == 429: - return error.code not in _QUOTA_CODES + return error.code not in _QUOTA_CODES | _DAILY_QUOTA_CODES return error.status_code == 503 @@ -836,19 +1025,26 @@ def _headers( "transactional_email:read or transactional_email:write scope." ) _ResultT = TypeVar("_ResultT") -_Parse = Callable[[int, Any, Optional[str]], _ResultT] +_Parse = Callable[[int, bytes, Optional[str]], _ResultT] -class _BearerAuth(requests.auth.AuthBase): +class _ExactAuth(requests.auth.AuthBase): def __init__(self, headers: dict[str, str]) -> None: self._headers = headers def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: # Setting auth explicitly stops requests from substituting .netrc credentials. + request.headers.pop("Authorization", None) request.headers.update(self._headers) return request +def _files_body(files: Sequence[_File]) -> bytes: + return _encode( + {"files": [{"sha256": file.sha256, "size": file.size} for file in files]} + ) + + class _EmailsBase: def __init__( self, @@ -971,8 +1167,9 @@ def send(self, **kwargs: Any) -> SendEmailResult: Raises: EmailError: PostHog did not queue the email, or the client could not tell whether it did. - ``code`` is ``invalid_attachment`` for an attachment the - client can't send, before any request. + With attachments, ``code`` is ``too_many_attachments``, + ``message_too_large`` or ``invalid_attachment`` before any + request, and ``upload_failed`` when an upload failed. TypeError, ValueError: An argument has the wrong type or format. Examples: @@ -1003,6 +1200,8 @@ def send(self, **kwargs: Any) -> SendEmailResult: return result def _deliver(self, prepared: _PreparedSend, secret_key: str) -> SendEmailResult: + if prepared.files: + self._upload(prepared, secret_key) headers = _headers( secret_key, self.migrated_from, @@ -1011,13 +1210,33 @@ def _deliver(self, prepared: _PreparedSend, secret_key: str) -> SendEmailResult: ) return self._call( "POST", - prepared.path, + self.host + prepared.path, prepared.body, headers, _parse_response, prepared.max_retries, ) + def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: + targets = self._call( + "POST", + self.host + _UPLOADS_PATH, + _files_body(prepared.files), + _headers(secret_key, self.migrated_from, has_body=True), + _upload_slots_parser(prepared.files), + prepared.max_retries, + ) + for index, (file, target) in enumerate(zip(prepared.files, targets)): + if target is not None: + self._call( + target.method, + target.url, + file.content, + target.headers, + _upload_parser(index), + prepared.max_retries, + ) + def get(self, message_id: Union[str, UUID]) -> EmailStatus: """Read what happened to an email that :meth:`send` queued. @@ -1046,22 +1265,23 @@ def get(self, message_id: Union[str, UUID]) -> EmailStatus: finish = self._start(_GET_NOT_DONE) try: headers = _headers(self._secret_key(_GET_KEY_NEEDED), self.migrated_from) - return self._call("GET", path, None, headers, _parse_status_response) + return self._call( + "GET", self.host + path, None, headers, _parse_status_response + ) finally: finish() def _call( self, method: str, - path: str, + url: str, body: Optional[bytes], headers: dict[str, str], parse: _Parse[_ResultT], max_retries: Optional[int] = None, ) -> _ResultT: - auth = _BearerAuth(headers) + auth = _ExactAuth(headers) session = self._get_session() - url = self.host + path retries = self.max_retries if max_retries is None else max_retries for attempt in range(retries + 1): try: @@ -1074,13 +1294,9 @@ def _call( allow_redirects=False, ) try: - try: - payload = response.json() - except ValueError: - payload = None return parse( response.status_code, - payload, + response.content, response.headers.get("Retry-After"), ) finally: @@ -1187,6 +1403,8 @@ async def send(self, **kwargs: Any) -> SendEmailResult: async def _deliver( self, prepared: _PreparedSend, secret_key: str ) -> SendEmailResult: + if prepared.files: + await self._upload(prepared, secret_key) headers = _headers( secret_key, self.migrated_from, @@ -1195,13 +1413,33 @@ async def _deliver( ) return await self._call( "POST", - prepared.path, + self.host + prepared.path, prepared.body, headers, _parse_response, prepared.max_retries, ) + async def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: + targets = await self._call( + "POST", + self.host + _UPLOADS_PATH, + _files_body(prepared.files), + _headers(secret_key, self.migrated_from, has_body=True), + _upload_slots_parser(prepared.files), + prepared.max_retries, + ) + for index, (file, target) in enumerate(zip(prepared.files, targets)): + if target is not None: + await self._call( + target.method, + target.url, + file.content, + target.headers, + _upload_parser(index), + prepared.max_retries, + ) + async def get(self, message_id: Union[str, UUID]) -> EmailStatus: """Read what happened to an email that :meth:`send` queued. @@ -1212,42 +1450,37 @@ async def get(self, message_id: Union[str, UUID]) -> EmailStatus: finish = self._start(_GET_NOT_DONE) try: headers = _headers(self._secret_key(_GET_KEY_NEEDED), self.migrated_from) - return await self._call("GET", path, None, headers, _parse_status_response) + return await self._call( + "GET", self.host + path, None, headers, _parse_status_response + ) finally: finish() async def _call( self, method: str, - path: str, + url: str, body: Optional[bytes], headers: dict[str, str], parse: _Parse[_ResultT], max_retries: Optional[int] = None, ) -> _ResultT: httpx, client = self._get_client() - url = self.host + path retries = self.max_retries if max_retries is None else max_retries for attempt in range(retries + 1): try: - # An injected client may carry its own auth and timeout, so set both per request: auth=None keeps - # the secret key's Bearer header, and the request timeout applies whatever the client's default. - response = await client.request( - method, - url, - content=body, - headers=headers, - auth=None, - timeout=self.timeout, + # An injected client may carry its own auth, headers and timeout, so the request sets exactly the + # headers given, sends without the client's auth, and applies its own timeout. + request = client.build_request( + method, url, content=body, timeout=self.timeout ) + request.headers.pop("Authorization", None) + request.headers.update(headers) + response = await client.send(request, auth=None) try: - try: - payload = response.json() - except ValueError: - payload = None return parse( response.status_code, - payload, + response.content, response.headers.get("Retry-After"), ) finally: diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 0a580feb2..c0bc06944 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -25,7 +25,7 @@ f"https://us.posthog.com/api/projects/@current/messaging/emails/{MESSAGE_ID}/" ) -Reply = Union[Exception, tuple[int, Optional[dict[str, Any]], dict[str, str]]] +Reply = Union[Exception, tuple[int, Union[dict[str, Any], bytes, None], dict[str, str]]] def accepted(message_id: str = "msg_1", **fields: Any) -> Reply: @@ -55,7 +55,13 @@ def send( status, body, headers = reply response = requests.Response() response.status_code = status - response._content = b"" if body is None else json.dumps(body).encode() + response._content = ( + body + if isinstance(body, bytes) + else b"" + if body is None + else json.dumps(body).encode() + ) response.headers.update(headers) response.request = request response.url = request.url or "" @@ -728,7 +734,6 @@ def handler(request: httpx.Request) -> httpx.Response: reply_to="help@example.com", cc="manager@example.com", bcc=["audit@example.com"], - attachments=[{"filename": "invite.ics", "content": INVITE}], capture_engagement=False, distinct_id="user_123", idempotency_key="reset:user_123", @@ -751,7 +756,6 @@ def handler(request: httpx.Request) -> httpx.Response: "reply_to": ["help@example.com"], "cc": ["manager@example.com"], "bcc": ["audit@example.com"], - "attachments": [{"filename": "invite.ics", "content": INVITE_BASE64}], "capture_engagement": False, "distinct_id": "user_123", } @@ -868,32 +872,121 @@ def test_a_refused_send_names_the_field_from_attr(): assert len(adapter.requests) == 1 -def test_attachments_are_sent_base64_encoded_once_across_retries(): - emails, adapter = emails_with([rejected(503, "delivery_unavailable"), accepted()]) +UPLOADS_URL = ( + "https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/" +) +BUCKET_URL = "https://files.example.com/team/1/" +INVITE_SHA256 = "215efae54e34ac5657da47b49c8d1bd8ea12649ee1efa838c9a47d7233f18afb" +LOGO = b"\x89PNG\r\n\x1a\n" +LOGO_SHA256 = "4c4b6a3be1314ab86138bef4314dde022e600960d8689a2c8f8631802d20dab6" +SIGNED_HEADERS = { + "content-length": "32", + "x-amz-checksum-sha256": "IV765U40rFZX2ke0nI0b2OoSZJ7h76g4yaR9cjPxivs=", +} + + +def present(sha256: str) -> dict[str, Any]: + return {"sha256": sha256, "status": "present", "expires_at": "2026-12-16T00:00:00Z"} + + +def upload(sha256: str) -> dict[str, Any]: + return { + "sha256": sha256, + "status": "upload", + "url": BUCKET_URL + sha256, + "method": "PUT", + "headers": SIGNED_HEADERS, + "url_expires_at": "2026-11-09T12:15:00Z", + } - send_code( - emails, - html=HTML, - attachments=[ - { - "filename": "invite.ics", - "content": INVITE, - "content_type": "text/calendar; method=REQUEST", - }, - {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, - ], + +def slots(*files: dict[str, Any]) -> Reply: + return (200, {"files": list(files)}, {}) + + +def stored() -> Reply: + return (200, None, {}) + + +def bucket_refused(status: int, code: str, message: str) -> Reply: + return ( + status, + f"{code}{message}".encode(), + {}, ) - first, retry = adapter.requests - assert json.loads(first.body)["attachments"] == [ + +INVITE_AND_LOGO = [ + { + "filename": "invite.ics", + "content": INVITE, + "content_type": "text/calendar; method=REQUEST", + }, + {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, +] + + +def test_uploads_the_files_posthog_lacks_then_sends_their_hashes(): + emails, adapter = emails_with( + [slots(upload(INVITE_SHA256), present(LOGO_SHA256)), stored(), accepted()] + ) + + result = send_code(emails, html=HTML, attachments=INVITE_AND_LOGO) + + assert result.id == "msg_1" + ask, put, sent = adapter.requests + assert (ask.method, ask.url) == ("POST", UPLOADS_URL) + assert ask.headers["Authorization"] == f"Bearer {SECRET_KEY}" + assert json.loads(ask.body) == { + "files": [ + {"sha256": INVITE_SHA256, "size": 32}, + {"sha256": LOGO_SHA256, "size": 8}, + ] + } + assert (put.method, put.url, put.body) == ( + "PUT", + BUCKET_URL + INVITE_SHA256, + INVITE, + ) + assert "Authorization" not in put.headers + assert {name: put.headers[name] for name in SIGNED_HEADERS} == SIGNED_HEADERS + assert (sent.method, sent.url) == ("POST", SEND_URL) + assert json.loads(sent.body)["attachments"] == [ { + "sha256": INVITE_SHA256, "filename": "invite.ics", - "content": INVITE_BASE64, "content_type": "text/calendar; method=REQUEST", }, - {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, + {"sha256": LOGO_SHA256, "filename": "logo.png", "content_id": "logo"}, ] - assert retry.body == first.body + + +def test_sends_without_uploading_when_posthog_has_every_file(): + emails, adapter = emails_with([slots(present(INVITE_SHA256)), accepted()]) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] + ) + + assert [(request.method, request.url) for request in adapter.requests] == [ + ("POST", UPLOADS_URL), + ("POST", SEND_URL), + ] + + +def test_the_upload_never_carries_credentials_the_session_holds(): + adapter = FakeAdapter([slots(upload(INVITE_SHA256)), stored(), accepted()]) + session = requests.Session() + session.mount("https://", adapter) + session.headers["Authorization"] = "Bearer session-default" + emails = Emails(SECRET_KEY, session=session) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] + ) + + put = adapter.requests[1] + assert "Authorization" not in put.headers @parameterized.expand( @@ -904,19 +997,276 @@ def test_attachments_are_sent_base64_encoded_once_across_retries(): "strided_memoryview", memoryview(bytes(b for byte in INVITE for b in (byte, 0)))[::2], ), + ("base64_string", INVITE_BASE64), ] ) -def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, content): - emails, adapter = emails_with([accepted()]) +def test_file_content_from_bytes_or_base64_uploads_the_same_bytes(_name, content): + emails, adapter = emails_with([slots(upload(INVITE_SHA256)), stored(), accepted()]) send_code( emails, html=HTML, attachments=[{"filename": "invite.ics", "content": content}] ) - [request] = adapter.requests - assert json.loads(request.body)["attachments"] == [ - {"filename": "invite.ics", "content": INVITE_BASE64} + ask, put, _sent = adapter.requests + assert json.loads(ask.body) == {"files": [{"sha256": INVITE_SHA256, "size": 32}]} + assert put.body == INVITE + + +@parameterized.expand( + [ + ( + "signed_checksum_mismatch", + bucket_refused( + 400, "BadDigest", "The Content-SHA256 you specified did not match." + ), + 400, + "BadDigest", + ), + ( + "expired_url", + bucket_refused(403, "AccessDenied", "Request has expired"), + 403, + "AccessDenied", + ), ] +) +def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( + _name, refusal, status, storage_code +): + emails, adapter = emails_with([slots(upload(INVITE_SHA256)), refusal]) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + error = raised.value + assert (error.code, error.status_code, error.field) == ( + "upload_failed", + status, + "attachments__0", + ) + assert storage_code in error.message + assert [request.method for request in adapter.requests] == ["POST", "PUT"] + + +def test_an_unavailable_storage_is_retried_before_the_send(): + emails, adapter = emails_with( + [ + slots(upload(INVITE_SHA256)), + bucket_refused(503, "SlowDown", "Please reduce your request rate."), + stored(), + accepted(), + ] + ) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] + ) + + assert [request.method for request in adapter.requests] == [ + "POST", + "PUT", + "PUT", + "POST", + ] + + +def test_a_refused_upload_request_uploads_and_sends_nothing(): + refused = { + "type": "invalid_request", + "code": "field_not_accepted", + "detail": "This project can't send files yet, so PostHog didn't prepare the uploads.", + "attr": "files", + "why": "Sending files isn't available to this project yet.", + "fix": "Send the email without files.", + } + emails, adapter = emails_with([(400, refused, {})]) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + error = raised.value + assert (error.code, error.field, error.why) == ( + "field_not_accepted", + "files", + "Sending files isn't available to this project yet.", + ) + assert [request.url for request in adapter.requests] == [UPLOADS_URL] + + +def test_a_project_over_its_daily_upload_bytes_is_told_without_retrying(): + emails, adapter = emails_with([rejected(429, "upload_quota_exceeded"), slots()]) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert (raised.value.code, raised.value.name) == ( + "upload_quota_exceeded", + "daily_quota_exceeded", + ) + assert len(adapter.requests) == 1 + + +def test_a_retried_send_reuses_its_uploads_and_its_idempotency_key(): + emails, adapter = emails_with( + [ + slots(upload(INVITE_SHA256)), + stored(), + rejected(503, "delivery_unavailable"), + accepted(), + ] + ) + + send_code( + emails, + html=HTML, + idempotency_key="invite:user_123", + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + _ask, _put, first, retry = adapter.requests + assert (retry.url, retry.body) == (SEND_URL, first.body) + assert retry.headers["Idempotency-Key"] == "invite:user_123" + + +@parameterized.expand( + [ + ( + "eleven_files", + [{"filename": f"{i}.txt", "content": b"hi"} for i in range(11)], + "too_many_attachments", + ), + ( + "one_file_too_large_beside_the_body", + [{"filename": "scan.pdf", "content": bytes(29_182_875)}], + "message_too_large", + ), + ( + "files_too_large_together", + [{"filename": f"{i}.pdf", "content": bytes(3_000_000)} for i in range(10)], + "message_too_large", + ), + ] +) +def test_refuses_an_email_over_the_limits_before_any_request(_name, attachments, code): + emails, adapter = emails_with([]) + + with pytest.raises(EmailError) as raised: + send_code(emails, html="Hi!", attachments=attachments) + + error = raised.value + assert (error.name, error.code, error.field) == ( + "validation_error", + code, + "attachments", + ) + assert error.fix + assert adapter.requests == [] + + +LARGEST_SCAN_SHA256 = "da8348171897010f0a7968e94886e059512d508ec8f543ec2af1d353cae83f60" + + +def test_sends_the_largest_file_that_fits_beside_the_body(): + emails, adapter = emails_with([slots(present(LARGEST_SCAN_SHA256)), accepted()]) + + send_code( + emails, + html="Hi!", + attachments=[{"filename": "scan.pdf", "content": bytes(29_182_872)}], + ) + + assert json.loads(adapter.requests[0].body)["files"][0]["size"] == 29_182_872 + + +@pytest.mark.asyncio +async def test_async_send_uploads_without_the_secret_key_then_sends_the_hashes(): + seen: list[httpx.Request] = [] + replies = [ + httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}), + httpx.Response(200), + httpx.Response(202, json={"message_id": "msg_1"}), + ] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return replies.pop(0) + + emails = AsyncEmails( + SECRET_KEY, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), + headers={"Authorization": "Bearer client-default"}, + ), + ) + + result = await emails.send( + to="user@example.com", + template_key="invite", + subject="You're invited", + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert result.id == "msg_1" + ask, put, sent = seen + assert (ask.method, str(ask.url)) == ("POST", UPLOADS_URL) + assert ask.headers["Authorization"] == f"Bearer {SECRET_KEY}" + assert (put.method, str(put.url), put.content) == ( + "PUT", + BUCKET_URL + INVITE_SHA256, + INVITE, + ) + assert "Authorization" not in put.headers + assert ( + put.headers["x-amz-checksum-sha256"] == SIGNED_HEADERS["x-amz-checksum-sha256"] + ) + assert json.loads(sent.content)["attachments"] == [ + {"sha256": INVITE_SHA256, "filename": "invite.ics"} + ] + + +@pytest.mark.asyncio +async def test_async_refused_upload_sends_no_email(): + seen: list[httpx.Request] = [] + replies = [ + httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}), + httpx.Response( + 400, content=b"BadDigestNo." + ), + ] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return replies.pop(0) + + emails = AsyncEmails( + SECRET_KEY, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) + + with pytest.raises(EmailError) as raised: + await emails.send( + to="user@example.com", + template_key="invite", + subject="You're invited", + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert raised.value.code == "upload_failed" + assert [request.method for request in seen] == ["POST", "PUT"] SECRET_CONTENT = b"secret-file-content" @@ -937,6 +1287,11 @@ def test_attachment_content_from_any_bytes_buffer_is_base64_encoded(_name, conte "attachments__0__content", ), ("no_content", [{"filename": "a.txt"}], "attachments__0__content"), + ( + "empty_content", + [{"filename": "a.txt", "content": b""}], + "attachments__0__content", + ), ( "text_not_base64", [{"filename": "a.txt", "content": "secret-file-content"}], diff --git a/typings/requests/__init__.pyi b/typings/requests/__init__.pyi index 1c545b680..b9b223504 100644 --- a/typings/requests/__init__.pyi +++ b/typings/requests/__init__.pyi @@ -10,6 +10,7 @@ class Response: status_code: int ok: bool text: str + content: bytes headers: dict[str, str] def json(self) -> Any: ... def close(self) -> None: ... From 629efcde41ebdf3a3c882cbeec773e8a1801fae2 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:26:08 +0000 Subject: [PATCH 15/39] fix(emails): send the storage upload exactly as signed An injected client's defaults leaked into the presigned PUT: httpx params replaced the URL's signature query, a requests session appended its params, and default headers rode along. The upload is now a standalone request with only the returned URL and headers, sent through the same transport and retries. Part of PostHog/michaelr-context#81. Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 130 ++++++++++++++++++++-------------- posthog/test/test_emails.py | 37 ++++++---- typings/requests/__init__.pyi | 18 +++++ 3 files changed, 118 insertions(+), 67 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index d27c5f42e..1f5153b5e 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -125,6 +125,7 @@ from types import TracebackType from typing import ( Any, + Awaitable, Callable, Mapping, Optional, @@ -1028,13 +1029,12 @@ def _headers( _Parse = Callable[[int, bytes, Optional[str]], _ResultT] -class _ExactAuth(requests.auth.AuthBase): +class _BearerAuth(requests.auth.AuthBase): def __init__(self, headers: dict[str, str]) -> None: self._headers = headers def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: # Setting auth explicitly stops requests from substituting .netrc credentials. - request.headers.pop("Authorization", None) request.headers.update(self._headers) return request @@ -1209,30 +1209,24 @@ def _deliver(self, prepared: _PreparedSend, secret_key: str) -> SendEmailResult: idempotency_key=prepared.idempotency_key, ) return self._call( - "POST", - self.host + prepared.path, - prepared.body, - headers, + self._posthog_request("POST", prepared.path, prepared.body, headers), _parse_response, prepared.max_retries, ) def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: + headers = _headers(secret_key, self.migrated_from, has_body=True) targets = self._call( - "POST", - self.host + _UPLOADS_PATH, - _files_body(prepared.files), - _headers(secret_key, self.migrated_from, has_body=True), + self._posthog_request( + "POST", _UPLOADS_PATH, _files_body(prepared.files), headers + ), _upload_slots_parser(prepared.files), prepared.max_retries, ) for index, (file, target) in enumerate(zip(prepared.files, targets)): if target is not None: self._call( - target.method, - target.url, - file.content, - target.headers, + self._storage_request(target, file.content), _upload_parser(index), prepared.max_retries, ) @@ -1266,33 +1260,46 @@ def get(self, message_id: Union[str, UUID]) -> EmailStatus: try: headers = _headers(self._secret_key(_GET_KEY_NEEDED), self.migrated_from) return self._call( - "GET", self.host + path, None, headers, _parse_status_response + self._posthog_request("GET", path, None, headers), + _parse_status_response, ) finally: finish() + def _posthog_request( + self, method: str, path: str, body: Optional[bytes], headers: dict[str, str] + ) -> Callable[[], requests.Response]: + session, auth = self._get_session(), _BearerAuth(headers) + return lambda: session.request( + method, + self.host + path, + data=body, + auth=auth, + timeout=self.timeout, + allow_redirects=False, + ) + + def _storage_request( + self, target: _UploadTarget, content: bytes + ) -> Callable[[], requests.Response]: + session = self._get_session() + request = requests.Request( + target.method, target.url, data=content, headers=target.headers + ).prepare() + return lambda: session.send( + request, timeout=self.timeout, allow_redirects=False + ) + def _call( self, - method: str, - url: str, - body: Optional[bytes], - headers: dict[str, str], + send: Callable[[], requests.Response], parse: _Parse[_ResultT], max_retries: Optional[int] = None, ) -> _ResultT: - auth = _ExactAuth(headers) - session = self._get_session() retries = self.max_retries if max_retries is None else max_retries for attempt in range(retries + 1): try: - response = session.request( - method, - url, - data=body, - auth=auth, - timeout=self.timeout, - allow_redirects=False, - ) + response = send() try: return parse( response.status_code, @@ -1412,30 +1419,24 @@ async def _deliver( idempotency_key=prepared.idempotency_key, ) return await self._call( - "POST", - self.host + prepared.path, - prepared.body, - headers, + self._posthog_request("POST", prepared.path, prepared.body, headers), _parse_response, prepared.max_retries, ) async def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: + headers = _headers(secret_key, self.migrated_from, has_body=True) targets = await self._call( - "POST", - self.host + _UPLOADS_PATH, - _files_body(prepared.files), - _headers(secret_key, self.migrated_from, has_body=True), + self._posthog_request( + "POST", _UPLOADS_PATH, _files_body(prepared.files), headers + ), _upload_slots_parser(prepared.files), prepared.max_retries, ) for index, (file, target) in enumerate(zip(prepared.files, targets)): if target is not None: await self._call( - target.method, - target.url, - file.content, - target.headers, + self._storage_request(target, file.content), _upload_parser(index), prepared.max_retries, ) @@ -1451,32 +1452,51 @@ async def get(self, message_id: Union[str, UUID]) -> EmailStatus: try: headers = _headers(self._secret_key(_GET_KEY_NEEDED), self.migrated_from) return await self._call( - "GET", self.host + path, None, headers, _parse_status_response + self._posthog_request("GET", path, None, headers), + _parse_status_response, ) finally: finish() + def _posthog_request( + self, method: str, path: str, body: Optional[bytes], headers: dict[str, str] + ) -> Callable[[], Awaitable[Any]]: + _, client = self._get_client() + # An injected client may carry its own auth and timeout, so set both per request: auth=None keeps + # the secret key's Bearer header, and the request timeout applies whatever the client's default. + return lambda: client.request( + method, + self.host + path, + content=body, + headers=headers, + auth=None, + timeout=self.timeout, + ) + + def _storage_request( + self, target: _UploadTarget, content: bytes + ) -> Callable[[], Awaitable[Any]]: + httpx, client = self._get_client() + request = httpx.Request( + target.method, + target.url, + content=content, + headers=target.headers, + extensions={"timeout": httpx.Timeout(self.timeout).as_dict()}, + ) + return lambda: client.send(request, auth=None) + async def _call( self, - method: str, - url: str, - body: Optional[bytes], - headers: dict[str, str], + send: Callable[[], Awaitable[Any]], parse: _Parse[_ResultT], max_retries: Optional[int] = None, ) -> _ResultT: - httpx, client = self._get_client() + httpx, _ = self._get_client() retries = self.max_retries if max_retries is None else max_retries for attempt in range(retries + 1): try: - # An injected client may carry its own auth, headers and timeout, so the request sets exactly the - # headers given, sends without the client's auth, and applies its own timeout. - request = client.build_request( - method, url, content=body, timeout=self.timeout - ) - request.headers.pop("Authorization", None) - request.headers.update(headers) - response = await client.send(request, auth=None) + response = await send() try: return parse( response.status_code, diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index c0bc06944..bb773448a 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -876,6 +876,7 @@ def test_a_refused_send_names_the_field_from_attr(): "https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/" ) BUCKET_URL = "https://files.example.com/team/1/" +SIGNATURE = "?X-Amz-Expires=900&X-Amz-Signature=abc123" INVITE_SHA256 = "215efae54e34ac5657da47b49c8d1bd8ea12649ee1efa838c9a47d7233f18afb" LOGO = b"\x89PNG\r\n\x1a\n" LOGO_SHA256 = "4c4b6a3be1314ab86138bef4314dde022e600960d8689a2c8f8631802d20dab6" @@ -893,7 +894,7 @@ def upload(sha256: str) -> dict[str, Any]: return { "sha256": sha256, "status": "upload", - "url": BUCKET_URL + sha256, + "url": BUCKET_URL + sha256 + SIGNATURE, "method": "PUT", "headers": SIGNED_HEADERS, "url_expires_at": "2026-11-09T12:15:00Z", @@ -945,7 +946,7 @@ def test_uploads_the_files_posthog_lacks_then_sends_their_hashes(): } assert (put.method, put.url, put.body) == ( "PUT", - BUCKET_URL + INVITE_SHA256, + BUCKET_URL + INVITE_SHA256 + SIGNATURE, INVITE, ) assert "Authorization" not in put.headers @@ -974,11 +975,14 @@ def test_sends_without_uploading_when_posthog_has_every_file(): ] -def test_the_upload_never_carries_credentials_the_session_holds(): +def test_the_upload_carries_only_the_signed_url_and_headers_whatever_the_session_holds(): adapter = FakeAdapter([slots(upload(INVITE_SHA256)), stored(), accepted()]) session = requests.Session() session.mount("https://", adapter) - session.headers["Authorization"] = "Bearer session-default" + session.params = {"api_key": "session-default"} + session.headers.update( + {"Authorization": "Bearer session-default", "Content-Type": "text/plain"} + ) emails = Emails(SECRET_KEY, session=session) send_code( @@ -986,7 +990,10 @@ def test_the_upload_never_carries_credentials_the_session_holds(): ) put = adapter.requests[1] - assert "Authorization" not in put.headers + assert put.url == BUCKET_URL + INVITE_SHA256 + SIGNATURE + assert { + name.lower(): value for name, value in put.headers.items() + } == SIGNED_HEADERS @parameterized.expand( @@ -1207,7 +1214,11 @@ def handler(request: httpx.Request) -> httpx.Response: SECRET_KEY, http_client=httpx.AsyncClient( transport=httpx.MockTransport(handler), - headers={"Authorization": "Bearer client-default"}, + params={"api_key": "client-default"}, + headers={ + "Authorization": "Bearer client-default", + "Content-Type": "text/plain", + }, ), ) @@ -1221,17 +1232,19 @@ def handler(request: httpx.Request) -> httpx.Response: assert result.id == "msg_1" ask, put, sent = seen - assert (ask.method, str(ask.url)) == ("POST", UPLOADS_URL) + assert (ask.method, ask.url.path) == ( + "POST", + "/api/projects/@current/messaging/attachments/uploads/", + ) assert ask.headers["Authorization"] == f"Bearer {SECRET_KEY}" assert (put.method, str(put.url), put.content) == ( "PUT", - BUCKET_URL + INVITE_SHA256, + BUCKET_URL + INVITE_SHA256 + SIGNATURE, INVITE, ) - assert "Authorization" not in put.headers - assert ( - put.headers["x-amz-checksum-sha256"] == SIGNED_HEADERS["x-amz-checksum-sha256"] - ) + assert { + name: value for name, value in put.headers.items() if name != "host" + } == SIGNED_HEADERS assert json.loads(sent.content)["attachments"] == [ {"sha256": INVITE_SHA256, "filename": "invite.ics"} ] diff --git a/typings/requests/__init__.pyi b/typings/requests/__init__.pyi index b9b223504..afcd44181 100644 --- a/typings/requests/__init__.pyi +++ b/typings/requests/__init__.pyi @@ -6,6 +6,17 @@ from .auth import AuthBase as _AuthBase class PreparedRequest: headers: dict[str, str] +class Request: + def __init__( + self, + method: str, + url: str, + *, + data: bytes | None = ..., + headers: dict[str, str] | None = ..., + ) -> None: ... + def prepare(self) -> PreparedRequest: ... + class Response: status_code: int ok: bool @@ -20,6 +31,13 @@ class Session: headers: dict[str, str] def mount(self, prefix: str, adapter: adapters.HTTPAdapter) -> None: ... def close(self) -> None: ... + def send( + self, + request: PreparedRequest, + *, + timeout: float | None = ..., + allow_redirects: bool = ..., + ) -> Response: ... def request( self, method: str, From 2a77a17339597ad81e1eb98cd017f51e2e8aaf38 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:30:28 +0000 Subject: [PATCH 16/39] fix(emails): keep the environment's TLS settings and the signed URL private on uploads The standalone storage PUT skipped requests' environment merge, so a custom CA bundle or proxy applied to sends but not to uploads. It now merges the environment's transport settings, without session headers or auth. A storage network error is re-raised without its cause, so the presigned URL never reaches a traceback. Part of PostHog/michaelr-context#81. Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 29 ++++++++++++++--- posthog/test/test_emails.py | 60 +++++++++++++++++++++++++++++++++++ typings/requests/__init__.pyi | 9 ++++++ 3 files changed, 94 insertions(+), 4 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index 1f5153b5e..172f14c35 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -1039,6 +1039,10 @@ def __call__(self, request: requests.PreparedRequest) -> requests.PreparedReques return request +def _storage_unreachable(exc: Exception) -> str: + return f"PostHog's file storage didn't answer ({type(exc).__name__})." + + def _files_body(files: Sequence[_File]) -> bytes: return _encode( {"files": [{"sha256": file.sha256, "size": file.size} for file in files]} @@ -1286,9 +1290,19 @@ def _storage_request( request = requests.Request( target.method, target.url, data=content, headers=target.headers ).prepare() - return lambda: session.send( - request, timeout=self.timeout, allow_redirects=False - ) + transport = session.merge_environment_settings(target.url, {}, None, None, None) + + def send() -> requests.Response: + try: + return session.send( + request, timeout=self.timeout, allow_redirects=False, **transport + ) + except requests.exceptions.RequestException as exc: + raise requests.exceptions.ConnectionError( + _storage_unreachable(exc) + ) from None + + return send def _call( self, @@ -1484,7 +1498,14 @@ def _storage_request( headers=target.headers, extensions={"timeout": httpx.Timeout(self.timeout).as_dict()}, ) - return lambda: client.send(request, auth=None) + + async def send() -> Any: + try: + return await client.send(request, auth=None) + except httpx.RequestError as exc: + raise httpx.ConnectError(_storage_unreachable(exc)) from None + + return send async def _call( self, diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index bb773448a..cb83a62c0 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -3,6 +3,7 @@ import asyncio import json import logging +import traceback from datetime import datetime, timezone from decimal import Decimal from typing import Any, Optional, Union @@ -44,11 +45,13 @@ def __init__(self, replies: list[Reply]) -> None: super().__init__() self.replies = list(replies) self.requests: list[requests.PreparedRequest] = [] + self.verify: list[Any] = [] def send( self, request: requests.PreparedRequest, **kwargs: Any ) -> requests.Response: self.requests.append(request) + self.verify.append(kwargs.get("verify")) reply = self.replies.pop(0) if isinstance(reply, Exception): raise reply @@ -1059,6 +1062,63 @@ def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( assert [request.method for request in adapter.requests] == ["POST", "PUT"] +def test_the_upload_trusts_the_certificates_the_environment_names(monkeypatch): + monkeypatch.setenv("REQUESTS_CA_BUNDLE", "/etc/ssl/corporate-ca.pem") + emails, adapter = emails_with([slots(upload(INVITE_SHA256)), stored(), accepted()]) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] + ) + + assert adapter.verify == ["/etc/ssl/corporate-ca.pem"] * 3 + + +def test_an_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): + unreachable = requests.exceptions.ConnectionError( + f"Max retries exceeded with url: {BUCKET_URL}{INVITE_SHA256}{SIGNATURE}" + ) + emails, adapter = emails_with( + [slots(upload(INVITE_SHA256)), unreachable, unreachable, unreachable] + ) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert raised.value.code == "network_error" + assert "X-Amz-Signature" not in "".join(traceback.format_exception(raised.value)) + assert [request.method for request in adapter.requests] == ["POST"] + ["PUT"] * 3 + + +@pytest.mark.asyncio +async def test_async_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "PUT": + raise httpx.ConnectError(f"Connection refused: {request.url}") + return httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}) + + emails = AsyncEmails( + SECRET_KEY, + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) + + with pytest.raises(EmailError) as raised: + await emails.send( + to="user@example.com", + template_key="invite", + subject="You're invited", + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert raised.value.code == "network_error" + assert "X-Amz-Signature" not in "".join(traceback.format_exception(raised.value)) + + def test_an_unavailable_storage_is_retried_before_the_send(): emails, adapter = emails_with( [ diff --git a/typings/requests/__init__.pyi b/typings/requests/__init__.pyi index afcd44181..91e583b5e 100644 --- a/typings/requests/__init__.pyi +++ b/typings/requests/__init__.pyi @@ -31,12 +31,21 @@ class Session: headers: dict[str, str] def mount(self, prefix: str, adapter: adapters.HTTPAdapter) -> None: ... def close(self) -> None: ... + def merge_environment_settings( + self, + url: str, + proxies: dict[str, str], + stream: bool | None, + verify: bool | str | None, + cert: str | None, + ) -> dict[str, Any]: ... def send( self, request: PreparedRequest, *, timeout: float | None = ..., allow_redirects: bool = ..., + **kwargs: Any, ) -> Response: ... def request( self, From 1fe42335a353a72608e4703deb39dbb103ed48db Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:04:25 +0000 Subject: [PATCH 17/39] fix(emails): upload shared bytes once and retry a storage's passing faults - The client counts attachments before it reads any file. - Bytes that two attachments share go in the upload request once. - The upload parser returns only the uploads still to make, each with its attachment index. - A storage 500, 502 or 504 is retried like 503. - A refused upload no longer takes the names of PostHog's API statuses. - An unreachable storage raises network_error naming the attachment. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-code-attachments.md | 2 +- posthog/emails.py | 145 +++++++++++-------- posthog/test/test_emails.py | 87 ++++++++++- 3 files changed, 167 insertions(+), 67 deletions(-) diff --git a/.sampo/changesets/emails-code-attachments.md b/.sampo/changesets/emails-code-attachments.md index 7559b5156..5b04b446d 100644 --- a/.sampo/changesets/emails-code-attachments.md +++ b/.sampo/changesets/emails-code-attachments.md @@ -2,4 +2,4 @@ pypi/posthog: minor --- -A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage, then sends the email with each file's SHA-256. A failed upload raises `EmailError` with the code `upload_failed` and sends no email. +A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage, then sends the email with each file's SHA-256. A refused upload raises `EmailError` with the code `upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. diff --git a/posthog/emails.py b/posthog/emails.py index 172f14c35..b3da89d33 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -64,8 +64,10 @@ both before any request. It then asks PostHog which files it lacks, uploads each of those to the URL PostHog answers, without the secret key, and sends the email naming each file by its SHA-256. A file the project already sent uploads -nothing. A failed upload sends no email. The upload URL's host is not the app -host, so a network that limits outbound requests must allow it. +nothing, and the same bytes in two attachments upload once. An upload retries +like a send when the storage is unreachable or answers a passing fault. A +failed upload sends no email. The upload URL's host is not the app host, so a +network that limits outbound requests must allow it. Errors: a send that is not queued raises :class:`EmailError`. Branch on ``code``. The ones a setup usually hits: @@ -87,6 +89,8 @@ content. - ``upload_failed``: PostHog's file storage refused an upload. ``message`` and ``why`` carry its answer, and ``field`` names the attachment. +- ``network_error`` with ``field`` naming an attachment: the file storage + didn't answer, such as when a firewall blocks its host. - ``upload_quota_exceeded`` (429): the project uploaded its daily limit of file bytes. Files it already sent don't count. - ``field_not_accepted``: the project can't send files yet. @@ -175,6 +179,8 @@ _QUOTA_CODES = frozenset({"quota_limited"}) _DAILY_QUOTA_CODES = frozenset({"upload_quota_exceeded"}) _MAX_ATTACHMENTS = 10 +# File storage answers these for a passing fault, such as S3's InternalError and SlowDown. +_STORAGE_RETRY_STATUSES = frozenset({500, 502, 503, 504}) _MAX_MESSAGE_BYTES = 40_000_000 _HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 _MESSAGE_LIMIT = "40 MB after Base64 encoding, about 29 MB of files" @@ -379,6 +385,7 @@ class EmailStatus: class _File: sha256: str content: bytes + index: int # The first attachment that carries these bytes. @property def size(self) -> int: @@ -386,7 +393,8 @@ def size(self) -> int: @dataclass(frozen=True) -class _UploadTarget: +class _Upload: + file: _File url: str method: str headers: dict[str, str] @@ -453,7 +461,7 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: raise ValueError("html or text must be a nonempty string.") references, files = _attachments(args.get("attachments", [])) - _check_limits(files, content) + _check_message_size(files, content) payload: dict[str, Any] = {"to": to, "template_key": template_key, **content} if distinct_id is not None: @@ -466,7 +474,7 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: body=_encode(payload), idempotency_key=idempotency_key, max_retries=max_retries, - files=files, + files=_distinct(files), ) @@ -496,6 +504,14 @@ def _addresses(field: str, value: Any) -> list[str]: return list(addresses) +def _distinct(files: Sequence[_File]) -> tuple[_File, ...]: + """Each file once, so bytes that two attachments share upload once.""" + by_hash: dict[str, _File] = {} + for file in files: + by_hash.setdefault(file.sha256, file) + return tuple(by_hash.values()) + + def _invalid_attachment(message: str, field: str) -> EmailError: return EmailError( name="validation_error", @@ -512,6 +528,13 @@ def _attachments( attachments, (str, bytes, bytearray, memoryview, Mapping) ) or not isinstance(attachments, Sequence): raise _invalid_attachment("attachments must be a list of files.", "attachments") + if len(attachments) > _MAX_ATTACHMENTS: + raise _over_the_limit( + "too_many_attachments", + f"An email carries at most {_MAX_ATTACHMENTS} files, so no email was sent.", + fix=f"Send at most {_MAX_ATTACHMENTS} files, and link to any others " + "from the email body.", + ) prepared = [_attachment(index, file) for index, file in enumerate(attachments)] return [reference for reference, _ in prepared], tuple(file for _, file in prepared) @@ -552,7 +575,7 @@ def _attachment(index: int, file: Any) -> tuple[dict[str, str], _File]: f"An attachment's {name} must be a string.", f"{field}__{name}" ) reference[name] = file[name] - return reference, _File(sha256=sha256, content=content) + return reference, _File(sha256=sha256, content=content, index=index) def _file_bytes(field: str, content: Any) -> bytes: @@ -582,14 +605,7 @@ def _file_bytes(field: str, content: Any) -> bytes: return file_bytes -def _check_limits(files: Sequence[_File], content: Mapping[str, str]) -> None: - if len(files) > _MAX_ATTACHMENTS: - raise _over_the_limit( - "too_many_attachments", - f"An email carries at most {_MAX_ATTACHMENTS} files, so no email was sent.", - fix=f"Send at most {_MAX_ATTACHMENTS} files, and link to any others " - "from the email body.", - ) +def _check_message_size(files: Sequence[_File], content: Mapping[str, str]) -> None: bodies = [ body.encode("utf-8") for body in (content.get("html"), content.get("text")) @@ -879,24 +895,27 @@ def _parse_status_response( ) -def _upload_slots_parser( - files: Sequence[_File], -) -> _Parse[list[Optional[_UploadTarget]]]: +def _uploads_parser(files: Sequence[_File]) -> _Parse[list[_Upload]]: + """Parse PostHog's answer into the uploads still to make, leaving out the files it has.""" + def parse( status: int, body: bytes, retry_after_header: Optional[str] - ) -> list[Optional[_UploadTarget]]: + ) -> list[_Upload]: fields = _json_object(body) if not 200 <= status < 300: raise _response_error(fields, status, retry_after_header) slots = fields.get("files") if not isinstance(slots, list) or len(slots) != len(files): raise _no_upload_slots(status) - return [_upload_target(slot, file, status) for slot, file in zip(slots, files)] + uploads = ( + _pending_upload(slot, file, status) for slot, file in zip(slots, files) + ) + return [upload for upload in uploads if upload is not None] return parse -def _upload_target(slot: Any, file: _File, status: int) -> Optional[_UploadTarget]: +def _pending_upload(slot: Any, file: _File, status: int) -> Optional[_Upload]: if not isinstance(slot, dict) or slot.get("sha256") != file.sha256: raise _no_upload_slots(status) if slot.get("status") == "present": @@ -914,7 +933,7 @@ def _upload_target(slot: Any, file: _File, status: int) -> Optional[_UploadTarge or not all(isinstance(value, str) for value in headers.values()) ): raise _no_upload_slots(status) - return _UploadTarget(url=url, method=method, headers=headers) + return _Upload(file=file, url=url, method=method, headers=headers) def _no_upload_slots(status: int) -> EmailError: @@ -937,7 +956,10 @@ def _upload_error( answer = dict(_STORAGE_ERROR.findall(body.decode("utf-8", "replace"))) storage_answer = " ".join(filter(None, (f"HTTP {status}", answer.get("Code")))) return EmailError( - name=_error_name(status, "upload_failed"), + # The names for PostHog's statuses, such as invalid_permission for 403, would mislead here. + name="service_unavailable" + if status in _STORAGE_RETRY_STATUSES + else "application_error", code="upload_failed", message=f"PostHog's file storage refused the upload of attachments[{index}] " f"with {storage_answer}, so no email was sent.", @@ -971,6 +993,8 @@ def _is_retryable(error: EmailError) -> bool: # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. if error.status_code is None: return error.code == "network_error" + if error.code == "upload_failed": + return error.status_code in _STORAGE_RETRY_STATUSES if error.status_code == 409: return error.code in _IN_FLIGHT_CODES if error.status_code == 429: @@ -1039,8 +1063,17 @@ def __call__(self, request: requests.PreparedRequest) -> requests.PreparedReques return request -def _storage_unreachable(exc: Exception) -> str: - return f"PostHog's file storage didn't answer ({type(exc).__name__})." +def _storage_unreachable(index: int, exc: Exception) -> EmailError: + # Built without the request's exception, whose text holds the signed URL. + return EmailError( + name="service_unavailable", + code="network_error", + message=f"PostHog's file storage didn't answer the upload of attachments[{index}] " + f"({type(exc).__name__}), so no email was sent.", + fix="Send the email again. If it keeps failing, check that your network " + "lets the app reach PostHog's file storage.", + field=f"attachments__{index}", + ) def _files_body(files: Sequence[_File]) -> bytes: @@ -1173,7 +1206,9 @@ def send(self, **kwargs: Any) -> SendEmailResult: not tell whether it did. With attachments, ``code`` is ``too_many_attachments``, ``message_too_large`` or ``invalid_attachment`` before any - request, and ``upload_failed`` when an upload failed. + request, ``upload_failed`` when the storage refused an upload, + and ``network_error`` with ``field`` naming the attachment + when the storage didn't answer. TypeError, ValueError: An argument has the wrong type or format. Examples: @@ -1220,20 +1255,19 @@ def _deliver(self, prepared: _PreparedSend, secret_key: str) -> SendEmailResult: def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: headers = _headers(secret_key, self.migrated_from, has_body=True) - targets = self._call( + uploads = self._call( self._posthog_request( "POST", _UPLOADS_PATH, _files_body(prepared.files), headers ), - _upload_slots_parser(prepared.files), + _uploads_parser(prepared.files), prepared.max_retries, ) - for index, (file, target) in enumerate(zip(prepared.files, targets)): - if target is not None: - self._call( - self._storage_request(target, file.content), - _upload_parser(index), - prepared.max_retries, - ) + for upload in uploads: + self._call( + self._storage_request(upload), + _upload_parser(upload.file.index), + prepared.max_retries, + ) def get(self, message_id: Union[str, UUID]) -> EmailStatus: """Read what happened to an email that :meth:`send` queued. @@ -1283,14 +1317,12 @@ def _posthog_request( allow_redirects=False, ) - def _storage_request( - self, target: _UploadTarget, content: bytes - ) -> Callable[[], requests.Response]: + def _storage_request(self, upload: _Upload) -> Callable[[], requests.Response]: session = self._get_session() request = requests.Request( - target.method, target.url, data=content, headers=target.headers + upload.method, upload.url, data=upload.file.content, headers=upload.headers ).prepare() - transport = session.merge_environment_settings(target.url, {}, None, None, None) + transport = session.merge_environment_settings(upload.url, {}, None, None, None) def send() -> requests.Response: try: @@ -1298,9 +1330,7 @@ def send() -> requests.Response: request, timeout=self.timeout, allow_redirects=False, **transport ) except requests.exceptions.RequestException as exc: - raise requests.exceptions.ConnectionError( - _storage_unreachable(exc) - ) from None + raise _storage_unreachable(upload.file.index, exc) from None return send @@ -1440,20 +1470,19 @@ async def _deliver( async def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: headers = _headers(secret_key, self.migrated_from, has_body=True) - targets = await self._call( + uploads = await self._call( self._posthog_request( "POST", _UPLOADS_PATH, _files_body(prepared.files), headers ), - _upload_slots_parser(prepared.files), + _uploads_parser(prepared.files), prepared.max_retries, ) - for index, (file, target) in enumerate(zip(prepared.files, targets)): - if target is not None: - await self._call( - self._storage_request(target, file.content), - _upload_parser(index), - prepared.max_retries, - ) + for upload in uploads: + await self._call( + self._storage_request(upload), + _upload_parser(upload.file.index), + prepared.max_retries, + ) async def get(self, message_id: Union[str, UUID]) -> EmailStatus: """Read what happened to an email that :meth:`send` queued. @@ -1487,15 +1516,13 @@ def _posthog_request( timeout=self.timeout, ) - def _storage_request( - self, target: _UploadTarget, content: bytes - ) -> Callable[[], Awaitable[Any]]: + def _storage_request(self, upload: _Upload) -> Callable[[], Awaitable[Any]]: httpx, client = self._get_client() request = httpx.Request( - target.method, - target.url, - content=content, - headers=target.headers, + upload.method, + upload.url, + content=upload.file.content, + headers=upload.headers, extensions={"timeout": httpx.Timeout(self.timeout).as_dict()}, ) @@ -1503,7 +1530,7 @@ async def send() -> Any: try: return await client.send(request, auth=None) except httpx.RequestError as exc: - raise httpx.ConnectError(_storage_unreachable(exc)) from None + raise _storage_unreachable(upload.file.index, exc) from None return send diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index cb83a62c0..7abdd4586 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1053,7 +1053,8 @@ def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( ) error = raised.value - assert (error.code, error.status_code, error.field) == ( + assert (error.name, error.code, error.status_code, error.field) == ( + "application_error", "upload_failed", status, "attachments__0", @@ -1088,8 +1089,10 @@ def test_an_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): attachments=[{"filename": "invite.ics", "content": INVITE}], ) - assert raised.value.code == "network_error" - assert "X-Amz-Signature" not in "".join(traceback.format_exception(raised.value)) + error = raised.value + assert (error.code, error.field) == ("network_error", "attachments__0") + assert "file storage" in error.message + assert "X-Amz-Signature" not in "".join(traceback.format_exception(error)) assert [request.method for request in adapter.requests] == ["POST"] + ["PUT"] * 3 @@ -1115,15 +1118,25 @@ def handler(request: httpx.Request) -> httpx.Response: attachments=[{"filename": "invite.ics", "content": INVITE}], ) - assert raised.value.code == "network_error" - assert "X-Amz-Signature" not in "".join(traceback.format_exception(raised.value)) + error = raised.value + assert (error.code, error.field) == ("network_error", "attachments__0") + assert "file storage" in error.message + assert "X-Amz-Signature" not in "".join(traceback.format_exception(error)) -def test_an_unavailable_storage_is_retried_before_the_send(): +@parameterized.expand( + [ + ("slow_down", 503, "SlowDown", "Please reduce your request rate."), + ("internal_error", 500, "InternalError", "Please try again."), + ] +) +def test_a_storage_that_asks_for_a_retry_is_retried_before_the_send( + _name, status, code, message +): emails, adapter = emails_with( [ slots(upload(INVITE_SHA256)), - bucket_refused(503, "SlowDown", "Please reduce your request rate."), + bucket_refused(status, code, message), stored(), accepted(), ] @@ -1207,6 +1220,66 @@ def test_a_retried_send_reuses_its_uploads_and_its_idempotency_key(): assert retry.headers["Idempotency-Key"] == "invite:user_123" +def test_the_same_bytes_in_two_attachments_upload_once(): + emails, adapter = emails_with([slots(upload(LOGO_SHA256)), stored(), accepted()]) + + send_code( + emails, + html=HTML, + attachments=[ + {"filename": "logo.png", "content": LOGO, "content_id": "logo"}, + {"filename": "logo-copy.png", "content": LOGO}, + ], + ) + + ask, put, sent = adapter.requests + assert json.loads(ask.body) == {"files": [{"sha256": LOGO_SHA256, "size": 8}]} + assert put.body == LOGO + assert json.loads(sent.body)["attachments"] == [ + {"sha256": LOGO_SHA256, "filename": "logo.png", "content_id": "logo"}, + {"sha256": LOGO_SHA256, "filename": "logo-copy.png"}, + ] + + +def test_a_failed_upload_names_the_first_attachment_with_those_bytes(): + emails, _adapter = emails_with( + [ + slots(present(INVITE_SHA256), upload(LOGO_SHA256)), + bucket_refused(403, "AccessDenied", "Request has expired"), + ] + ) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[ + {"filename": "invite.ics", "content": INVITE}, + {"filename": "logo.png", "content": LOGO}, + {"filename": "logo-copy.png", "content": LOGO}, + ], + ) + + assert raised.value.field == "attachments__1" + + +def test_refuses_eleven_files_before_reading_any_of_them(): + emails, adapter = emails_with([]) + attachments: list[Any] = [ + {"filename": f"{i}.txt", "content": b"hi"} for i in range(10) + ] + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=attachments + [{"filename": "late.txt", "content": None}], + ) + + assert raised.value.code == "too_many_attachments" + assert adapter.requests == [] + + @parameterized.expand( [ ( From 0794b7bc28764e6171b55e067b3f4cef05e7577b Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:10:59 +0000 Subject: [PATCH 18/39] fix(emails): retry a storage's 429 and name what the comments said The previous commit stopped retrying a storage 429, which the general rate-limit branch had retried. Storage 429 is retried again and named rate_limit_exceeded. Names replace the comments that commit added. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 48 +++++++++++++++++++++---------------- posthog/test/test_emails.py | 1 + 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index b3da89d33..f60955669 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -179,8 +179,7 @@ _QUOTA_CODES = frozenset({"quota_limited"}) _DAILY_QUOTA_CODES = frozenset({"upload_quota_exceeded"}) _MAX_ATTACHMENTS = 10 -# File storage answers these for a passing fault, such as S3's InternalError and SlowDown. -_STORAGE_RETRY_STATUSES = frozenset({500, 502, 503, 504}) +_STORAGE_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504}) _MAX_MESSAGE_BYTES = 40_000_000 _HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 _MESSAGE_LIMIT = "40 MB after Base64 encoding, about 29 MB of files" @@ -385,7 +384,7 @@ class EmailStatus: class _File: sha256: str content: bytes - index: int # The first attachment that carries these bytes. + first_attachment_index: int @property def size(self) -> int: @@ -474,7 +473,7 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: body=_encode(payload), idempotency_key=idempotency_key, max_retries=max_retries, - files=_distinct(files), + files=_each_file_once(files), ) @@ -504,8 +503,7 @@ def _addresses(field: str, value: Any) -> list[str]: return list(addresses) -def _distinct(files: Sequence[_File]) -> tuple[_File, ...]: - """Each file once, so bytes that two attachments share upload once.""" +def _each_file_once(files: Sequence[_File]) -> tuple[_File, ...]: by_hash: dict[str, _File] = {} for file in files: by_hash.setdefault(file.sha256, file) @@ -575,7 +573,9 @@ def _attachment(index: int, file: Any) -> tuple[dict[str, str], _File]: f"An attachment's {name} must be a string.", f"{field}__{name}" ) reference[name] = file[name] - return reference, _File(sha256=sha256, content=content, index=index) + return reference, _File( + sha256=sha256, content=content, first_attachment_index=index + ) def _file_bytes(field: str, content: Any) -> bytes: @@ -895,9 +895,7 @@ def _parse_status_response( ) -def _uploads_parser(files: Sequence[_File]) -> _Parse[list[_Upload]]: - """Parse PostHog's answer into the uploads still to make, leaving out the files it has.""" - +def _pending_uploads_parser(files: Sequence[_File]) -> _Parse[list[_Upload]]: def parse( status: int, body: bytes, retry_after_header: Optional[str] ) -> list[_Upload]: @@ -956,10 +954,7 @@ def _upload_error( answer = dict(_STORAGE_ERROR.findall(body.decode("utf-8", "replace"))) storage_answer = " ".join(filter(None, (f"HTTP {status}", answer.get("Code")))) return EmailError( - # The names for PostHog's statuses, such as invalid_permission for 403, would mislead here. - name="service_unavailable" - if status in _STORAGE_RETRY_STATUSES - else "application_error", + name=_storage_error_name(status), code="upload_failed", message=f"PostHog's file storage refused the upload of attachments[{index}] " f"with {storage_answer}, so no email was sent.", @@ -972,6 +967,14 @@ def _upload_error( ) +def _storage_error_name(status: int) -> str: + if status == 429: + return "rate_limit_exceeded" + if status in _STORAGE_RETRY_STATUSES: + return "service_unavailable" + return "application_error" + + def _response_error( fields: Mapping[str, Any], status: int, retry_after_header: Optional[str] ) -> EmailError: @@ -1064,7 +1067,6 @@ def __call__(self, request: requests.PreparedRequest) -> requests.PreparedReques def _storage_unreachable(index: int, exc: Exception) -> EmailError: - # Built without the request's exception, whose text holds the signed URL. return EmailError( name="service_unavailable", code="network_error", @@ -1259,13 +1261,13 @@ def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: self._posthog_request( "POST", _UPLOADS_PATH, _files_body(prepared.files), headers ), - _uploads_parser(prepared.files), + _pending_uploads_parser(prepared.files), prepared.max_retries, ) for upload in uploads: self._call( self._storage_request(upload), - _upload_parser(upload.file.index), + _upload_parser(upload.file.first_attachment_index), prepared.max_retries, ) @@ -1330,7 +1332,9 @@ def send() -> requests.Response: request, timeout=self.timeout, allow_redirects=False, **transport ) except requests.exceptions.RequestException as exc: - raise _storage_unreachable(upload.file.index, exc) from None + raise _storage_unreachable( + upload.file.first_attachment_index, exc + ) from None return send @@ -1474,13 +1478,13 @@ async def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: self._posthog_request( "POST", _UPLOADS_PATH, _files_body(prepared.files), headers ), - _uploads_parser(prepared.files), + _pending_uploads_parser(prepared.files), prepared.max_retries, ) for upload in uploads: await self._call( self._storage_request(upload), - _upload_parser(upload.file.index), + _upload_parser(upload.file.first_attachment_index), prepared.max_retries, ) @@ -1530,7 +1534,9 @@ async def send() -> Any: try: return await client.send(request, auth=None) except httpx.RequestError as exc: - raise _storage_unreachable(upload.file.index, exc) from None + raise _storage_unreachable( + upload.file.first_attachment_index, exc + ) from None return send diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 7abdd4586..57969c629 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1128,6 +1128,7 @@ def handler(request: httpx.Request) -> httpx.Response: [ ("slow_down", 503, "SlowDown", "Please reduce your request rate."), ("internal_error", 500, "InternalError", "Please try again."), + ("too_many_requests", 429, "SlowDown", "Please reduce your request rate."), ] ) def test_a_storage_that_asks_for_a_retry_is_retried_before_the_send( From 391a797bd0c0d7dfe21f1e62522f98adae81baf3 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:15:57 +0000 Subject: [PATCH 19/39] fix(emails): name attachments when PostHog has no upload route A PostHog version without the upload route answered 404, and the send raised a bare not_found with no field. The docs promise that an older PostHog's refusal names the field. It now raises not_found with field attachments, a reason and a fix, and sends nothing. Tests now cover the names a storage fault gets once its retries run out. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 18 +++++++++++ posthog/test/test_emails.py | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/posthog/emails.py b/posthog/emails.py index f60955669..d0c1a0498 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -94,6 +94,8 @@ - ``upload_quota_exceeded`` (429): the project uploaded its daily limit of file bytes. Files it already sent don't count. - ``field_not_accepted``: the project can't send files yet. +- ``not_found`` (404) with ``field`` ``attachments``: this PostHog version + doesn't take files yet. Status: ``emails.get(message_id)`` reads what happened to a sent email, for 30 days after the send, as an :class:`EmailStatus`. A key with only @@ -900,6 +902,8 @@ def parse( status: int, body: bytes, retry_after_header: Optional[str] ) -> list[_Upload]: fields = _json_object(body) + if status == 404: + raise _no_upload_route() if not 200 <= status < 300: raise _response_error(fields, status, retry_after_header) slots = fields.get("files") @@ -934,6 +938,20 @@ def _pending_upload(slot: Any, file: _File, status: int) -> Optional[_Upload]: return _Upload(file=file, url=url, method=method, headers=headers) +def _no_upload_route() -> EmailError: + return EmailError( + name="not_found", + code="not_found", + message="PostHog has no upload route for attachments here, so no email was sent.", + status_code=404, + why="This PostHog version doesn't take attachments yet, or transactional " + "email is not enabled for the project.", + fix="Send the email without attachments, or check that the project can " + "send transactional email.", + field="attachments", + ) + + def _no_upload_slots(status: int) -> EmailError: return _sdk_error( "api_error", "PostHog answered without an upload slot for each file.", status diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 57969c629..1af9096cd 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1182,6 +1182,66 @@ def test_a_refused_upload_request_uploads_and_sends_nothing(): assert [request.url for request in adapter.requests] == [UPLOADS_URL] +@parameterized.expand( + [ + ("throttled", 429, "SlowDown", "rate_limit_exceeded"), + ("unavailable", 503, "ServiceUnavailable", "service_unavailable"), + ] +) +def test_a_storage_out_of_retries_names_its_passing_fault( + _name, status, storage_code, name +): + emails, adapter = emails_with( + [ + slots(upload(INVITE_SHA256)), + bucket_refused(status, storage_code, "Try again."), + ], + max_retries=0, + ) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert (raised.value.name, raised.value.code) == (name, "upload_failed") + assert [request.method for request in adapter.requests] == ["POST", "PUT"] + + +def test_a_posthog_without_the_upload_route_names_attachments_and_sends_nothing(): + emails, adapter = emails_with( + [ + ( + 404, + { + "type": "invalid_request", + "code": "not_found", + "detail": "Endpoint not found.", + }, + {}, + ) + ] + ) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + error = raised.value + assert (error.code, error.status_code, error.field) == ( + "not_found", + 404, + "attachments", + ) + assert "attachments" in error.message + assert [request.url for request in adapter.requests] == [UPLOADS_URL] + + def test_a_project_over_its_daily_upload_bytes_is_told_without_retrying(): emails, adapter = emails_with([rejected(429, "upload_quota_exceeded"), slots()]) From c3005cf6ad8823e2c523197cf4f93e3c43484c73 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:35:33 +0000 Subject: [PATCH 20/39] fix(emails): size attachments before copying them The client copied and hashed every file before it checked the message size. Ten views of one large buffer could exhaust memory before message_too_large. Each file is now sized from its buffer or its Base64 length, the estimate is checked, and only then are files read and hashed. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 91 +++++++++++++++++++++++-------------- posthog/test/test_emails.py | 24 ++++++++++ 2 files changed, 81 insertions(+), 34 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index d0c1a0498..a68366779 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -461,8 +461,7 @@ def _prepare_code_send(args: Mapping[str, Any]) -> _PreparedSend: if not (content.get("html") or content.get("text")): raise ValueError("html or text must be a nonempty string.") - references, files = _attachments(args.get("attachments", [])) - _check_message_size(files, content) + references, files = _attachments(args.get("attachments", []), content) payload: dict[str, Any] = {"to": to, "template_key": template_key, **content} if distinct_id is not None: @@ -522,7 +521,7 @@ def _invalid_attachment(message: str, field: str) -> EmailError: def _attachments( - attachments: Any, + attachments: Any, content: Mapping[str, str] ) -> tuple[list[dict[str, str]], tuple[_File, ...]]: if isinstance( attachments, (str, bytes, bytearray, memoryview, Mapping) @@ -535,11 +534,21 @@ def _attachments( fix=f"Send at most {_MAX_ATTACHMENTS} files, and link to any others " "from the email body.", ) - prepared = [_attachment(index, file) for index, file in enumerate(attachments)] + checked = [ + _checked_attachment(index, file) for index, file in enumerate(attachments) + ] + _check_message_size( + [ + _content_size(f"attachments__{index}", file.get("content")) + for index, file in enumerate(checked) + ], + content, + ) + prepared = [_attachment(index, file) for index, file in enumerate(checked)] return [reference for reference, _ in prepared], tuple(file for _, file in prepared) -def _attachment(index: int, file: Any) -> tuple[dict[str, str], _File]: +def _checked_attachment(index: int, file: Any) -> Mapping[str, Any]: field = f"attachments__{index}" if not isinstance(file, Mapping): raise _invalid_attachment( @@ -565,56 +574,70 @@ def _attachment(index: int, file: Any) -> tuple[dict[str, str], _File]: "An attachment needs a filename, such as 'invoice.pdf'.", f"{field}__filename", ) - content = _file_bytes(field, file.get("content")) - sha256 = hashlib.sha256(content).hexdigest() - reference = {"sha256": sha256, "filename": file["filename"]} for name in ("content_type", "content_id"): - if name in file: - if not isinstance(file[name], str): - raise _invalid_attachment( - f"An attachment's {name} must be a string.", f"{field}__{name}" - ) - reference[name] = file[name] - return reference, _File( - sha256=sha256, content=content, first_attachment_index=index - ) + if name in file and not isinstance(file[name], str): + raise _invalid_attachment( + f"An attachment's {name} must be a string.", f"{field}__{name}" + ) + return file -def _file_bytes(field: str, content: Any) -> bytes: +def _content_size(field: str, content: Any) -> int: if isinstance(content, (bytes, bytearray, memoryview)): - file_bytes = bytes(content) + size = memoryview(content).nbytes elif isinstance(content, str): - try: - file_bytes = base64.b64decode(content, validate=True) - except (binascii.Error, ValueError): - raise _invalid_attachment( - "An attachment's content string must be Base64. " - "Pass the file's bytes instead, such as text.encode() for text.", - f"{field}__content", - ) from None + padding = len(content) - len(content.rstrip("=")) + size = max(0, len(content) * 3 // 4 - padding) else: raise _invalid_attachment( "An attachment needs content: bytes or a Base64 string. " "Convert a list of byte values with bytes(content).", f"{field}__content", ) - if not file_bytes: + if not size: raise _invalid_attachment( "An attachment's content is empty, and an empty file gives the " "recipient nothing to open. Leave the file out.", f"{field}__content", ) - return file_bytes + return size -def _check_message_size(files: Sequence[_File], content: Mapping[str, str]) -> None: - bodies = [ - body.encode("utf-8") +def _attachment(index: int, file: Mapping[str, Any]) -> tuple[dict[str, str], _File]: + content = _file_bytes(f"attachments__{index}", file["content"]) + sha256 = hashlib.sha256(content).hexdigest() + reference = {"sha256": sha256, "filename": file["filename"]} + for name in ("content_type", "content_id"): + if name in file: + reference[name] = file[name] + return reference, _File( + sha256=sha256, content=content, first_attachment_index=index + ) + + +def _file_bytes(field: str, content: Union[bytes, bytearray, memoryview, str]) -> bytes: + if not isinstance(content, str): + return bytes(content) + try: + return base64.b64decode(content, validate=True) + except (binascii.Error, ValueError): + raise _invalid_attachment( + "An attachment's content string must be Base64. " + "Pass the file's bytes instead, such as text.encode() for text.", + f"{field}__content", + ) from None + + +def _check_message_size(file_sizes: Sequence[int], content: Mapping[str, str]) -> None: + body_sizes = [ + len(body.encode("utf-8")) for body in (content.get("html"), content.get("text")) if body ] - sizes = [file.size for file in files] + [len(body) for body in bodies] - if files and _estimated_message_bytes(sizes) > _MAX_MESSAGE_BYTES: + if ( + file_sizes + and _estimated_message_bytes([*file_sizes, *body_sizes]) > _MAX_MESSAGE_BYTES + ): raise _over_the_limit( "message_too_large", f"These files make the email larger than {_MESSAGE_LIMIT}, so no email was sent.", diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 1af9096cd..e30047291 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -4,6 +4,7 @@ import json import logging import traceback +import tracemalloc from datetime import datetime, timezone from decimal import Decimal from typing import Any, Optional, Union @@ -1376,6 +1377,29 @@ def test_refuses_an_email_over_the_limits_before_any_request(_name, attachments, assert adapter.requests == [] +def test_refuses_an_oversized_email_without_copying_its_files(): + emails, adapter = emails_with([]) + scan = memoryview(bytearray(8 * 1024 * 1024)) + + tracemalloc.start() + try: + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[ + {"filename": f"scan-{i}.pdf", "content": scan} for i in range(10) + ], + ) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert raised.value.code == "message_too_large" + assert peak < len(scan) + assert adapter.requests == [] + + LARGEST_SCAN_SHA256 = "da8348171897010f0a7968e94886e059512d508ec8f543ec2af1d353cae83f60" From dd7759e83003f3a0ea014e5a93a1b09322c5e9c7 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:42:27 +0000 Subject: [PATCH 21/39] fix(emails): size Base64 content by its data, not its padding Extra trailing padding decodes, so it must not shrink the estimate. Ticket: PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 3 +-- posthog/test/test_emails.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index a68366779..6717cd142 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -586,8 +586,7 @@ def _content_size(field: str, content: Any) -> int: if isinstance(content, (bytes, bytearray, memoryview)): size = memoryview(content).nbytes elif isinstance(content, str): - padding = len(content) - len(content.rstrip("=")) - size = max(0, len(content) * 3 // 4 - padding) + size = len(content.rstrip("=")) * 3 // 4 else: raise _invalid_attachment( "An attachment needs content: bytes or a Base64 string. " diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index e30047291..678feb211 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1354,6 +1354,16 @@ def test_refuses_eleven_files_before_reading_any_of_them(): [{"filename": "scan.pdf", "content": bytes(29_182_875)}], "message_too_large", ), + ( + "base64_file_too_large_behind_extra_padding", + [ + { + "filename": "scan.pdf", + "content": "AAAA" * (29_182_875 // 3) + "=" * 12, + } + ], + "message_too_large", + ), ( "files_too_large_together", [{"filename": f"{i}.pdf", "content": bytes(3_000_000)} for i in range(10)], From 4b7c311bbf5621930b353f8037084be4385d12f6 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:47:53 +0000 Subject: [PATCH 22/39] test(emails): pin each body's count in the size estimate The server counts html and text each like a file: UTF-8 bytes, Base64 with line breaks. Pin both bodies, their UTF-8 length and per-body rounding. Ticket: PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 678feb211..a080c67e6 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1425,6 +1425,47 @@ def test_sends_the_largest_file_that_fits_beside_the_body(): assert json.loads(adapter.requests[0].body)["files"][0]["size"] == 29_182_872 +SCAN_BESIDE_A_78_BYTE_BODY = bytes(29_182_818) +SCAN_BESIDE_A_78_BYTE_BODY_SHA256 = ( + "23a605f8a0f14f6ad57155dcee6f7e45f783f1a9cd44e6c38f6dfad6d33ef14c" +) + + +@parameterized.expand( + [ + ("html", "html"), + ("text", "text"), + ] +) +def test_counts_a_body_like_a_file_in_the_size_estimate(_name, body_field): + emails, adapter = emails_with( + [slots(present(SCAN_BESIDE_A_78_BYTE_BODY_SHA256)), accepted()] + ) + attachments = [{"filename": "scan.pdf", "content": SCAN_BESIDE_A_78_BYTE_BODY}] + + send_code(emails, attachments=attachments, **{body_field: "a" * 57}) + with pytest.raises(EmailError) as raised: + send_code(emails, attachments=attachments, **{body_field: "é" * 57}) + + assert raised.value.code == "message_too_large" + assert len(adapter.requests) == 2 + + +def test_counts_html_and_text_as_two_bodies(): + emails, adapter = emails_with([]) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html="a", + text="a", + attachments=[{"filename": "scan.pdf", "content": bytes(29_182_872)}], + ) + + assert raised.value.code == "message_too_large" + assert adapter.requests == [] + + @pytest.mark.asyncio async def test_async_send_uploads_without_the_secret_key_then_sends_the_hashes(): seen: list[httpx.Request] = [] From dfc3f8f6af54405936a8659c23448a32c7351b89 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:53:11 +0000 Subject: [PATCH 23/39] fix(emails): report PostHog's own answer to an upload-route 404 A 404 can mean a PostHog without the upload route or sending from code off for the project, so keep the server's reason on attachments instead of naming one cause. Ticket: PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 29 ++++++++++++++++------------- posthog/test/test_emails.py | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index 6717cd142..51415683c 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -95,7 +95,8 @@ file bytes. Files it already sent don't count. - ``field_not_accepted``: the project can't send files yet. - ``not_found`` (404) with ``field`` ``attachments``: this PostHog version - doesn't take files yet. + doesn't take files yet, or sending email from code is off for the project. + ``message`` carries PostHog's answer, and no email was sent. Status: ``emails.get(message_id)`` reads what happened to a sent email, for 30 days after the send, as an :class:`EmailStatus`. A key with only @@ -925,7 +926,7 @@ def parse( ) -> list[_Upload]: fields = _json_object(body) if status == 404: - raise _no_upload_route() + raise _upload_route_not_found(fields, retry_after_header) if not 200 <= status < 300: raise _response_error(fields, status, retry_after_header) slots = fields.get("files") @@ -960,17 +961,19 @@ def _pending_upload(slot: Any, file: _File, status: int) -> Optional[_Upload]: return _Upload(file=file, url=url, method=method, headers=headers) -def _no_upload_route() -> EmailError: - return EmailError( - name="not_found", - code="not_found", - message="PostHog has no upload route for attachments here, so no email was sent.", - status_code=404, - why="This PostHog version doesn't take attachments yet, or transactional " - "email is not enabled for the project.", - fix="Send the email without attachments, or check that the project can " - "send transactional email.", - field="attachments", +def _upload_route_not_found( + fields: Mapping[str, Any], retry_after_header: Optional[str] +) -> EmailError: + unanswered = { + "code": "not_found", + "why": "This PostHog version doesn't take attachments yet, or sending " + "email from code is off for the project.", + "fix": "Send the email without attachments, or check that the project " + "can send email from code.", + } + answered = {key: value for key, value in fields.items() if value is not None} + return _response_error( + {**unanswered, **answered, "attr": "attachments"}, 404, retry_after_header ) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index a080c67e6..bf105acda 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1211,7 +1211,15 @@ def test_a_storage_out_of_retries_names_its_passing_fault( assert [request.method for request in adapter.requests] == ["POST", "PUT"] -def test_a_posthog_without_the_upload_route_names_attachments_and_sends_nothing(): +@parameterized.expand( + [ + ("posthog_without_the_upload_route", "Endpoint not found."), + ("sending_from_code_off_for_the_project", "Not found."), + ] +) +def test_an_upload_route_404_reports_posthogs_answer_on_attachments_and_sends_nothing( + _name, detail +): emails, adapter = emails_with( [ ( @@ -1219,13 +1227,36 @@ def test_a_posthog_without_the_upload_route_names_attachments_and_sends_nothing( { "type": "invalid_request", "code": "not_found", - "detail": "Endpoint not found.", + "detail": detail, + "attr": None, }, {}, ) ] ) + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + error = raised.value + assert (error.name, error.code, error.status_code, error.field, error.message) == ( + "not_found", + "not_found", + 404, + "attachments", + detail, + ) + assert error.why and error.fix + assert [request.url for request in adapter.requests] == [UPLOADS_URL] + + +def test_an_upload_route_404_without_an_answer_still_names_attachments(): + emails, adapter = emails_with([(404, b"

Not Found

", {})]) + with pytest.raises(EmailError) as raised: send_code( emails, @@ -1239,7 +1270,6 @@ def test_a_posthog_without_the_upload_route_names_attachments_and_sends_nothing( 404, "attachments", ) - assert "attachments" in error.message assert [request.url for request in adapter.requests] == [UPLOADS_URL] From 1828040f6695dc5fb9751207f77a603491cb2a01 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:54:35 +0000 Subject: [PATCH 24/39] fix(emails): say which Base64 the SDK takes and where a refused upload goes wrong A wrapped, unpadded or URL-safe string now learns it needs one padded line from base64.b64encode. A storage refusal points at a fresh send or a proxy, not at bytes the caller can't change. Ticket: PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 13 ++++++++----- posthog/test/test_emails.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index 51415683c..dc0ead968 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -244,7 +244,8 @@ class EmailAttachment(TypedDict): Args: filename: The file name the recipient sees, such as ``invite.ics``. content: The file as ``bytes``, ``bytearray`` or ``memoryview``, or - Base64 encoded as a string. PostHog can't fetch a file from a path + Base64 encoded as a string on one line with its padding, as + ``base64.b64encode`` gives. PostHog can't fetch a file from a path or URL. content_type: The MIME type. Without it, the type follows the file name. content_id: Shows the file inline: reference it in ``html`` as @@ -622,8 +623,9 @@ def _file_bytes(field: str, content: Union[bytes, bytearray, memoryview, str]) - return base64.b64decode(content, validate=True) except (binascii.Error, ValueError): raise _invalid_attachment( - "An attachment's content string must be Base64. " - "Pass the file's bytes instead, such as text.encode() for text.", + "An attachment's content string must be standard Base64 on one line " + "with its padding, as base64.b64encode gives. Pass the file's bytes " + "instead, such as text.encode() for text.", f"{field}__content", ) from None @@ -1003,8 +1005,9 @@ def _upload_error( f"with {storage_answer}, so no email was sent.", status_code=status, why=answer.get("Message"), - fix="Send the email again. If the upload keeps failing, check that the " - "file's bytes don't change while it is sent.", + fix="Send the email again, which asks for a fresh upload URL. If the " + "upload keeps failing, check whether a proxy between your app and " + "PostHog's file storage changes or holds up uploads.", field=f"attachments__{index}", retry_after=_retry_after(retry_after_header), ) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index bf105acda..78b8ac783 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import base64 import json import logging import traceback @@ -1061,9 +1062,36 @@ def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( "attachments__0", ) assert storage_code in error.message + assert "proxy" in error.fix assert [request.method for request in adapter.requests] == ["POST", "PUT"] +@parameterized.expand( + [ + ("line_wrapped", base64.encodebytes(bytes(100)).decode()), + ("unpadded", "QUI"), + ("url_safe", base64.urlsafe_b64encode(b"\xfb\xff").decode()), + ] +) +def test_refuses_base64_not_in_one_padded_line_and_says_how_to_encode(_name, content): + emails, adapter = emails_with([]) + + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "a.bin", "content": content}], + ) + + error = raised.value + assert (error.code, error.field) == ( + "invalid_attachment", + "attachments__0__content", + ) + assert "base64.b64encode" in error.message + assert adapter.requests == [] + + def test_the_upload_trusts_the_certificates_the_environment_names(monkeypatch): monkeypatch.setenv("REQUESTS_CA_BUNDLE", "/etc/ssl/corporate-ca.pem") emails, adapter = emails_with([slots(upload(INVITE_SHA256)), stored(), accepted()]) From 5ecd5816325905fce520dc4efe208bcf41975ef1 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:28:38 +0000 Subject: [PATCH 25/39] fix(emails): count the subject in the message estimate like the server The server now adds 75 bytes per 42 UTF-8 bytes of subject, so a long subject beside a near-limit file must be refused before the upload. Ticket: PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 31 ++++++++++++++------ posthog/test/test_emails.py | 57 +++++++++++++++++++++++++++++++------ 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index dc0ead968..f785d9a0d 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -185,6 +185,8 @@ _STORAGE_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504}) _MAX_MESSAGE_BYTES = 40_000_000 _HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 +_SUBJECT_BYTES_PER_ENCODED_WORD = 42 +_ENCODED_WORD_BYTES = 75 _MESSAGE_LIMIT = "40 MB after Base64 encoding, about 29 MB of files" _STORAGE_ERROR = re.compile(r"<(Code|Message)>([^<]{1,500})") @@ -631,14 +633,9 @@ def _file_bytes(field: str, content: Union[bytes, bytearray, memoryview, str]) - def _check_message_size(file_sizes: Sequence[int], content: Mapping[str, str]) -> None: - body_sizes = [ - len(body.encode("utf-8")) - for body in (content.get("html"), content.get("text")) - if body - ] if ( file_sizes - and _estimated_message_bytes([*file_sizes, *body_sizes]) > _MAX_MESSAGE_BYTES + and _estimated_message_bytes(file_sizes, content) > _MAX_MESSAGE_BYTES ): raise _over_the_limit( "message_too_large", @@ -658,9 +655,25 @@ def _over_the_limit(code: str, message: str, *, fix: str) -> EmailError: ) -def _estimated_message_bytes(sizes: Sequence[int]) -> int: - return _HEADERS_AND_MIME_BOUNDARIES_BYTES + sum( - _base64_transfer_bytes(size) for size in sizes +def _estimated_message_bytes( + file_sizes: Sequence[int], content: Mapping[str, str] +) -> int: + body_sizes = [ + len(body.encode("utf-8")) + for body in (content.get("html"), content.get("text")) + if body + ] + return ( + _HEADERS_AND_MIME_BOUNDARIES_BYTES + + sum(_base64_transfer_bytes(size) for size in [*file_sizes, *body_sizes]) + + _encoded_subject_bytes(content.get("subject") or "") + ) + + +def _encoded_subject_bytes(subject: str) -> int: + return ( + _ceil_div(len(subject.encode("utf-8")), _SUBJECT_BYTES_PER_ENCODED_WORD) + * _ENCODED_WORD_BYTES ) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 78b8ac783..a38f596c8 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -17,7 +17,13 @@ from parameterized import parameterized from posthog import AsyncPosthog, Posthog -from posthog.emails import AsyncEmails, EmailError, Emails, EmailStatus +from posthog.emails import ( + AsyncEmails, + EmailError, + Emails, + EmailStatus, + _encoded_subject_bytes, +) from posthog.version import VERSION SECRET_KEY = "phs_test_secret" @@ -1409,7 +1415,7 @@ def test_refuses_eleven_files_before_reading_any_of_them(): ), ( "one_file_too_large_beside_the_body", - [{"filename": "scan.pdf", "content": bytes(29_182_875)}], + [{"filename": "scan.pdf", "content": bytes(29_182_821)}], "message_too_large", ), ( @@ -1417,7 +1423,7 @@ def test_refuses_eleven_files_before_reading_any_of_them(): [ { "filename": "scan.pdf", - "content": "AAAA" * (29_182_875 // 3) + "=" * 12, + "content": "AAAA" * (29_182_821 // 3) + "=" * 12, } ], "message_too_large", @@ -1468,7 +1474,8 @@ def test_refuses_an_oversized_email_without_copying_its_files(): assert adapter.requests == [] -LARGEST_SCAN_SHA256 = "da8348171897010f0a7968e94886e059512d508ec8f543ec2af1d353cae83f60" +LARGEST_SCAN = bytes(29_182_818) +LARGEST_SCAN_SHA256 = "23a605f8a0f14f6ad57155dcee6f7e45f783f1a9cd44e6c38f6dfad6d33ef14c" def test_sends_the_largest_file_that_fits_beside_the_body(): @@ -1477,15 +1484,15 @@ def test_sends_the_largest_file_that_fits_beside_the_body(): send_code( emails, html="Hi!", - attachments=[{"filename": "scan.pdf", "content": bytes(29_182_872)}], + attachments=[{"filename": "scan.pdf", "content": LARGEST_SCAN}], ) - assert json.loads(adapter.requests[0].body)["files"][0]["size"] == 29_182_872 + assert json.loads(adapter.requests[0].body)["files"][0]["size"] == 29_182_818 -SCAN_BESIDE_A_78_BYTE_BODY = bytes(29_182_818) +SCAN_BESIDE_A_78_BYTE_BODY = bytes(29_182_764) SCAN_BESIDE_A_78_BYTE_BODY_SHA256 = ( - "23a605f8a0f14f6ad57155dcee6f7e45f783f1a9cd44e6c38f6dfad6d33ef14c" + "af928bfa2ca810344546c2cdb01a6f5fb9f8545281e4d6c7b75a1bfec9f1f2c3" ) @@ -1517,13 +1524,45 @@ def test_counts_html_and_text_as_two_bodies(): emails, html="a", text="a", - attachments=[{"filename": "scan.pdf", "content": bytes(29_182_872)}], + attachments=[{"filename": "scan.pdf", "content": LARGEST_SCAN}], ) assert raised.value.code == "message_too_large" assert adapter.requests == [] +@parameterized.expand( + [ + ("42_ascii_bytes_count_75", "a" * 42, "a" * 43), + ("21_two_byte_characters_count_75", "é" * 21, "é" * 22), + ] +) +def test_counts_the_subject_in_75_byte_steps_per_42_utf8_bytes( + _name, fitting_subject, refused_subject +): + emails, adapter = emails_with([slots(present(LARGEST_SCAN_SHA256)), accepted()]) + attachments = [{"filename": "scan.pdf", "content": LARGEST_SCAN}] + + send_code(emails, html="Hi!", subject=fitting_subject, attachments=attachments) + with pytest.raises(EmailError) as raised: + send_code(emails, html="Hi!", subject=refused_subject, attachments=attachments) + + assert raised.value.code == "message_too_large" + assert len(adapter.requests) == 2 + + +@parameterized.expand( + [ + ("empty", "", 0), + ("42_ascii_bytes", "a" * 42, 75), + ("43_ascii_bytes", "a" * 43, 150), + ("21_two_byte_characters", "é" * 21, 75), + ] +) +def test_sizes_a_subject_like_the_server(_name, subject, size): + assert _encoded_subject_bytes(subject) == size + + @pytest.mark.asyncio async def test_async_send_uploads_without_the_secret_key_then_sends_the_hashes(): seen: list[httpx.Request] = [] From 7b735ceeba6efec557b4c98acf1e1a8fd4d31632 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:27:52 +0000 Subject: [PATCH 26/39] test(emails): check uploads against their signed checksum like real storage The fake storage now refuses a PUT whose bytes don't match the signed checksum and length, and each upload slot is signed for its own file. A new test asserts every file PostHog lacks lands in storage under its own hash. Refs PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 60 ++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index a38f596c8..001c1fe18 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -2,6 +2,7 @@ import asyncio import base64 +import hashlib import json import logging import traceback @@ -54,6 +55,7 @@ def __init__(self, replies: list[Reply]) -> None: self.replies = list(replies) self.requests: list[requests.PreparedRequest] = [] self.verify: list[Any] = [] + self.stored: dict[str, bytes] = {} def send( self, request: requests.PreparedRequest, **kwargs: Any @@ -63,6 +65,8 @@ def send( reply = self.replies.pop(0) if isinstance(reply, Exception): raise reply + if request.method == "PUT" and reply[0] < 300: + reply = self.store(request, reply) status, body, headers = reply response = requests.Response() response.status_code = status @@ -78,6 +82,19 @@ def send( response.url = request.url or "" return response + def store(self, request: requests.PreparedRequest, reply: Reply) -> Reply: + content = request.body if isinstance(request.body, bytes) else b"" + if dict(signed_headers(content)) != { + name: request.headers.get(name) for name in SIGNED_HEADERS + }: + return bucket_refused( + 400, + "BadDigest", + "The SHA256 you specified did not match the calculated checksum.", + ) + self.stored[(request.url or "").split("?")[0]] = content + return reply + def close(self) -> None: pass @@ -891,6 +908,18 @@ def test_a_refused_send_names_the_field_from_attr(): INVITE_SHA256 = "215efae54e34ac5657da47b49c8d1bd8ea12649ee1efa838c9a47d7233f18afb" LOGO = b"\x89PNG\r\n\x1a\n" LOGO_SHA256 = "4c4b6a3be1314ab86138bef4314dde022e600960d8689a2c8f8631802d20dab6" +FILES = {INVITE_SHA256: INVITE, LOGO_SHA256: LOGO} + + +def signed_headers(content: bytes) -> dict[str, str]: + return { + "content-length": str(len(content)), + "x-amz-checksum-sha256": base64.b64encode( + hashlib.sha256(content).digest() + ).decode(), + } + + SIGNED_HEADERS = { "content-length": "32", "x-amz-checksum-sha256": "IV765U40rFZX2ke0nI0b2OoSZJ7h76g4yaR9cjPxivs=", @@ -907,7 +936,7 @@ def upload(sha256: str) -> dict[str, Any]: "status": "upload", "url": BUCKET_URL + sha256 + SIGNATURE, "method": "PUT", - "headers": SIGNED_HEADERS, + "headers": signed_headers(FILES[sha256]), "url_expires_at": "2026-11-09T12:15:00Z", } @@ -973,6 +1002,35 @@ def test_uploads_the_files_posthog_lacks_then_sends_their_hashes(): ] +def test_stores_each_file_posthog_lacks_under_its_own_hash(): + emails, adapter = emails_with( + [ + slots(upload(INVITE_SHA256), upload(LOGO_SHA256)), + stored(), + stored(), + accepted(), + ] + ) + + result = send_code(emails, html=HTML, attachments=INVITE_AND_LOGO) + + assert result.id == "msg_1" + assert adapter.stored == { + BUCKET_URL + INVITE_SHA256: INVITE, + BUCKET_URL + LOGO_SHA256: LOGO, + } + + +def test_storage_refuses_bytes_that_do_not_match_the_signed_checksum(): + adapter = FakeAdapter([stored()]) + session = requests.Session() + session.mount("https://", adapter) + + refused = session.put(BUCKET_URL + LOGO_SHA256, data=LOGO, headers=SIGNED_HEADERS) + + assert (refused.status_code, adapter.stored) == (400, {}) + + def test_sends_without_uploading_when_posthog_has_every_file(): emails, adapter = emails_with([slots(present(INVITE_SHA256)), accepted()]) From 7ffa68a91623e03fec1b95ae9db8c9ebf13bb97f Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:50:12 +0000 Subject: [PATCH 27/39] fix(emails): size padded Base64 without a copy and never follow a storage redirect Sizing a Base64 string no longer strips its padding into a new string, so refusing an oversized file allocates nothing the size of the file. The async upload passes follow_redirects=False, so an injected client that follows redirects can't turn a storage 302 into a GET that looks like a stored file. The subject's share of the size limit is now pinned through send instead of through a private helper. Refs PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 9 ++-- posthog/test/test_emails.py | 98 ++++++++++++++++++++++++++++++------- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index f785d9a0d..90241d470 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -590,7 +590,10 @@ def _content_size(field: str, content: Any) -> int: if isinstance(content, (bytes, bytearray, memoryview)): size = memoryview(content).nbytes elif isinstance(content, str): - size = len(content.rstrip("=")) * 3 // 4 + data_end = len(content) + while data_end and content[data_end - 1] == "=": + data_end -= 1 + size = data_end * 3 // 4 else: raise _invalid_attachment( "An attachment needs content: bytes or a Base64 string. " @@ -666,7 +669,7 @@ def _estimated_message_bytes( return ( _HEADERS_AND_MIME_BOUNDARIES_BYTES + sum(_base64_transfer_bytes(size) for size in [*file_sizes, *body_sizes]) - + _encoded_subject_bytes(content.get("subject") or "") + + _encoded_subject_bytes(content["subject"]) ) @@ -1591,7 +1594,7 @@ def _storage_request(self, upload: _Upload) -> Callable[[], Awaitable[Any]]: async def send() -> Any: try: - return await client.send(request, auth=None) + return await client.send(request, auth=None, follow_redirects=False) except httpx.RequestError as exc: raise _storage_unreachable( upload.file.first_attachment_index, exc diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 001c1fe18..5fb8d46b9 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -23,7 +23,6 @@ EmailError, Emails, EmailStatus, - _encoded_subject_bytes, ) from posthog.version import VERSION @@ -1532,6 +1531,27 @@ def test_refuses_an_oversized_email_without_copying_its_files(): assert adapter.requests == [] +def test_refuses_oversized_padded_base64_without_copying_it(): + emails, adapter = emails_with([]) + scan = "AAAA" * 10_000_000 + "AA==" + + tracemalloc.start() + try: + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "scan.pdf", "content": scan}], + ) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert raised.value.code == "message_too_large" + assert peak < len(scan) // 4 + assert adapter.requests == [] + + LARGEST_SCAN = bytes(29_182_818) LARGEST_SCAN_SHA256 = "23a605f8a0f14f6ad57155dcee6f7e45f783f1a9cd44e6c38f6dfad6d33ef14c" @@ -1591,34 +1611,76 @@ def test_counts_html_and_text_as_two_bodies(): @parameterized.expand( [ - ("42_ascii_bytes_count_75", "a" * 42, "a" * 43), - ("21_two_byte_characters_count_75", "é" * 21, "é" * 22), + ("42_ascii_bytes_take_75", "a" * 42, 29_182_818), + ("43_ascii_bytes_take_150", "a" * 43, 29_182_764), + ("21_two_byte_characters_take_75", "é" * 21, 29_182_818), + ("22_two_byte_characters_take_150", "é" * 22, 29_182_764), ] ) -def test_counts_the_subject_in_75_byte_steps_per_42_utf8_bytes( - _name, fitting_subject, refused_subject +def test_a_subject_takes_75_bytes_of_the_limit_per_42_utf8_bytes( + _name, subject, largest_file_size ): - emails, adapter = emails_with([slots(present(LARGEST_SCAN_SHA256)), accepted()]) - attachments = [{"filename": "scan.pdf", "content": LARGEST_SCAN}] + largest = bytes(largest_file_size) + emails, adapter = emails_with( + [slots(present(hashlib.sha256(largest).hexdigest())), accepted()] + ) - send_code(emails, html="Hi!", subject=fitting_subject, attachments=attachments) + send_code( + emails, + html="Hi!", + subject=subject, + attachments=[{"filename": "scan.pdf", "content": largest}], + ) with pytest.raises(EmailError) as raised: - send_code(emails, html="Hi!", subject=refused_subject, attachments=attachments) + send_code( + emails, + html="Hi!", + subject=subject, + attachments=[ + {"filename": "scan.pdf", "content": bytes(largest_file_size + 1)} + ], + ) assert raised.value.code == "message_too_large" assert len(adapter.requests) == 2 -@parameterized.expand( - [ - ("empty", "", 0), - ("42_ascii_bytes", "a" * 42, 75), - ("43_ascii_bytes", "a" * 43, 150), - ("21_two_byte_characters", "é" * 21, 75), +@pytest.mark.asyncio +async def test_async_upload_never_follows_a_storage_redirect_even_when_the_client_does(): + seen: list[httpx.Request] = [] + replies = [ + httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}), + httpx.Response(302, headers={"Location": "https://files.example.com/login"}), + httpx.Response(200, text="Sign in"), + httpx.Response(202, json={"message_id": "msg_1"}), ] -) -def test_sizes_a_subject_like_the_server(_name, subject, size): - assert _encoded_subject_bytes(subject) == size + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return replies.pop(0) + + emails = AsyncEmails( + SECRET_KEY, + max_retries=0, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), follow_redirects=True + ), + ) + + with pytest.raises(EmailError) as raised: + await emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + + assert (raised.value.code, raised.value.field) == ( + "upload_failed", + "attachments__0", + ) + assert [request.method for request in seen] == ["POST", "PUT"] @pytest.mark.asyncio From a8dd43b2540baa84d09934ffd8f6cd453de4977e Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:55:48 +0000 Subject: [PATCH 28/39] test(emails): prove the async upload refuses a storage redirect over real sockets The redirect test now runs against a local HTTP server that answers the upload route, redirects the PUT and serves a login page, instead of a scripted transport. It asserts upload_failed and that the login page and the send route were never reached. Refs PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 82 ++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 5fb8d46b9..8dcd0a4ba 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -5,10 +5,12 @@ import hashlib import json import logging +import threading import traceback import tracemalloc from datetime import datetime, timezone from decimal import Decimal +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Any, Optional, Union from unittest import mock @@ -1645,42 +1647,74 @@ def test_a_subject_takes_75_bytes_of_the_limit_per_42_utf8_bytes( assert len(adapter.requests) == 2 -@pytest.mark.asyncio -async def test_async_upload_never_follows_a_storage_redirect_even_when_the_client_does(): - seen: list[httpx.Request] = [] - replies = [ - httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}), - httpx.Response(302, headers={"Location": "https://files.example.com/login"}), - httpx.Response(200, text="Sign in"), - httpx.Response(202, json={"message_id": "msg_1"}), - ] +class PostHogWithARedirectingStorage(BaseHTTPRequestHandler): + hits: list[str] = [] + + def do_POST(self) -> None: + self.hits.append(f"POST {self.path}") + self.rfile.read(int(self.headers["Content-Length"])) + if self.path.endswith("/attachments/uploads/"): + slot = upload(INVITE_SHA256) + slot["url"] = f"http://{self.headers['Host']}/bucket/{INVITE_SHA256}" + self.answer(200, json.dumps({"files": [slot]}).encode()) + else: + self.answer(202, json.dumps({"message_id": "msg_1"}).encode()) + + def do_PUT(self) -> None: + self.hits.append(f"PUT {self.path}") + self.rfile.read(int(self.headers["Content-Length"])) + self.send_response(302) + self.send_header("Location", "/login") + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self) -> None: + self.hits.append(f"GET {self.path}") + self.answer(200, b"Sign in") + + def answer(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: Any) -> None: + pass - def handler(request: httpx.Request) -> httpx.Response: - seen.append(request) - return replies.pop(0) +@pytest.mark.asyncio +async def test_async_upload_never_follows_a_storage_redirect_even_when_the_client_does(): + server = ThreadingHTTPServer(("127.0.0.1", 0), PostHogWithARedirectingStorage) + PostHogWithARedirectingStorage.hits = [] + threading.Thread(target=server.serve_forever, daemon=True).start() emails = AsyncEmails( SECRET_KEY, + host=f"http://127.0.0.1:{server.server_address[1]}", max_retries=0, - http_client=httpx.AsyncClient( - transport=httpx.MockTransport(handler), follow_redirects=True - ), + http_client=httpx.AsyncClient(follow_redirects=True, trust_env=False), ) - with pytest.raises(EmailError) as raised: - await emails.send( - to="user@example.com", - template_key="password-reset", - subject="Reset your password", - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + try: + with pytest.raises(EmailError) as raised: + await emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=HTML, + attachments=[{"filename": "invite.ics", "content": INVITE}], + ) + finally: + server.shutdown() + server.server_close() assert (raised.value.code, raised.value.field) == ( "upload_failed", "attachments__0", ) - assert [request.method for request in seen] == ["POST", "PUT"] + assert PostHogWithARedirectingStorage.hits == [ + "POST /api/projects/@current/messaging/attachments/uploads/", + f"PUT /bucket/{INVITE_SHA256}", + ] @pytest.mark.asyncio From 912f846a25b0db0b2ed4759ddb247a5ab1e38cf0 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:30:28 +0000 Subject: [PATCH 29/39] test(emails): test attachments against fakes that behave like PostHog and S3 The attachment tests now run against one in-memory PostHog with its file storage. The upload and send routes need the secret key and refuse fields outside their schema. Storage refuses a presigned PUT the way S3 does: an added query parameter, an Authorization header, an expired URL or bytes that don't match the signed checksum. A send delivers an email with the stored bytes and replays a repeated idempotency key. Tests assert the delivered email, the bucket and the EmailError, not request shapes. The CA test runs a real TLS handshake against a local storage server with a certificate made in the test. Refs PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 1277 +++++++++++++++++++++++------------ 1 file changed, 861 insertions(+), 416 deletions(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 8dcd0a4ba..d63e3487c 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -3,16 +3,25 @@ import asyncio import base64 import hashlib +import hmac import json import logging +import re +import ssl import threading +import time import traceback import tracemalloc -from datetime import datetime, timezone +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta, timezone from decimal import Decimal from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any, Optional, Union +from ipaddress import ip_address +from pathlib import Path +from typing import Any, Callable, Mapping, Optional, Union from unittest import mock +from urllib.parse import parse_qsl, urlencode, urlsplit +from uuid import uuid4 import httpx import pytest @@ -20,12 +29,7 @@ from parameterized import parameterized from posthog import AsyncPosthog, Posthog -from posthog.emails import ( - AsyncEmails, - EmailError, - Emails, - EmailStatus, -) +from posthog.emails import AsyncEmails, EmailError, Emails, EmailStatus from posthog.version import VERSION SECRET_KEY = "phs_test_secret" @@ -36,7 +40,7 @@ f"https://us.posthog.com/api/projects/@current/messaging/emails/{MESSAGE_ID}/" ) -Reply = Union[Exception, tuple[int, Union[dict[str, Any], bytes, None], dict[str, str]]] +Reply = Union[Exception, tuple[int, Optional[dict[str, Any]], dict[str, str]]] def accepted(message_id: str = "msg_1", **fields: Any) -> Reply: @@ -55,47 +59,23 @@ def __init__(self, replies: list[Reply]) -> None: super().__init__() self.replies = list(replies) self.requests: list[requests.PreparedRequest] = [] - self.verify: list[Any] = [] - self.stored: dict[str, bytes] = {} def send( self, request: requests.PreparedRequest, **kwargs: Any ) -> requests.Response: self.requests.append(request) - self.verify.append(kwargs.get("verify")) reply = self.replies.pop(0) if isinstance(reply, Exception): raise reply - if request.method == "PUT" and reply[0] < 300: - reply = self.store(request, reply) status, body, headers = reply response = requests.Response() response.status_code = status - response._content = ( - body - if isinstance(body, bytes) - else b"" - if body is None - else json.dumps(body).encode() - ) + response._content = b"" if body is None else json.dumps(body).encode() response.headers.update(headers) response.request = request response.url = request.url or "" return response - def store(self, request: requests.PreparedRequest, reply: Reply) -> Reply: - content = request.body if isinstance(request.body, bytes) else b"" - if dict(signed_headers(content)) != { - name: request.headers.get(name) for name in SIGNED_HEADERS - }: - return bucket_refused( - 400, - "BadDigest", - "The SHA256 you specified did not match the calculated checksum.", - ) - self.stored[(request.url or "").split("?")[0]] = content - return reply - def close(self) -> None: pass @@ -901,63 +881,456 @@ def test_a_refused_send_names_the_field_from_attr(): assert len(adapter.requests) == 1 -UPLOADS_URL = ( - "https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/" +POSTHOG_HOST = "https://us.posthog.com" +UPLOADS_PATH = "/api/projects/@current/messaging/attachments/uploads/" +SEND_PATH = "/api/projects/@current/messaging/send/email/" +STORAGE_URL = "https://files.example.com/team/1" +STORAGE_SIGNING_KEY = b"fake-storage-signing-key" +URL_LIFETIME_SECONDS = 900 +SEND_FIELDS = frozenset( + { + "to", + "template_key", + "subject", + "html", + "text", + "from", + "reply_to", + "cc", + "bcc", + "capture_engagement", + "distinct_id", + "attachments", + } ) -BUCKET_URL = "https://files.example.com/team/1/" -SIGNATURE = "?X-Amz-Expires=900&X-Amz-Signature=abc123" -INVITE_SHA256 = "215efae54e34ac5657da47b49c8d1bd8ea12649ee1efa838c9a47d7233f18afb" -LOGO = b"\x89PNG\r\n\x1a\n" -LOGO_SHA256 = "4c4b6a3be1314ab86138bef4314dde022e600960d8689a2c8f8631802d20dab6" -FILES = {INVITE_SHA256: INVITE, LOGO_SHA256: LOGO} +REFERENCE_FIELDS = frozenset({"sha256", "filename", "content_type", "content_id"}) +SHA256_HEX = re.compile(r"[0-9a-f]{64}") -def signed_headers(content: bytes) -> dict[str, str]: - return { - "content-length": str(len(content)), - "x-amz-checksum-sha256": base64.b64encode( - hashlib.sha256(content).digest() - ).decode(), - } +@dataclass(frozen=True) +class Answer: + status: int + body: bytes = b"" + headers: dict[str, str] = field(default_factory=dict) + @classmethod + def of_json(cls, status: int, payload: dict[str, Any]) -> Answer: + return cls( + status, json.dumps(payload).encode(), {"Content-Type": "application/json"} + ) -SIGNED_HEADERS = { - "content-length": "32", - "x-amz-checksum-sha256": "IV765U40rFZX2ke0nI0b2OoSZJ7h76g4yaR9cjPxivs=", -} +def api_error( + status: int, code: str, detail: str, kind: str = "validation_error", **extra: Any +) -> Answer: + return Answer.of_json( + status, {"type": kind, "code": code, "detail": detail, **extra} + ) -def present(sha256: str) -> dict[str, Any]: - return {"sha256": sha256, "status": "present", "expires_at": "2026-12-16T00:00:00Z"} +def storage_error(status: int, code: str, message: str) -> Answer: + return Answer( + status, + f'{code}' + f"{message}".encode(), + {"Content-Type": "application/xml"}, + ) -def upload(sha256: str) -> dict[str, Any]: - return { - "sha256": sha256, - "status": "upload", - "url": BUCKET_URL + sha256 + SIGNATURE, - "method": "PUT", - "headers": signed_headers(FILES[sha256]), - "url_expires_at": "2026-11-09T12:15:00Z", - } +UNAUTHORIZED = api_error( + 401, + "authentication_failed", + "Incorrect authentication credentials.", + kind="authentication_error", +) -def slots(*files: dict[str, Any]) -> Reply: - return (200, {"files": list(files)}, {}) +@dataclass(frozen=True) +class Incoming: + method: str + url: str + headers: Mapping[str, str] + body: bytes -def stored() -> Reply: - return (200, None, {}) + @property + def path(self) -> str: + return urlsplit(self.url).path + @property + def is_authorized(self) -> bool: + return self.headers.get("authorization") == f"Bearer {SECRET_KEY}" -def bucket_refused(status: int, code: str, message: str) -> Reply: - return ( - status, - f"{code}{message}".encode(), - {}, + +class Unreachable(Exception): + pass + + +Handler = Callable[[Incoming], Answer] +Fault = Callable[[Incoming, Handler], Answer] + + +def answers(answer: Answer) -> Fault: + return lambda _incoming, _handle: answer + + +def loses_the_reply(answer: Answer) -> Fault: + def fault(incoming: Incoming, handle: Handler) -> Answer: + handle(incoming) + return answer + + return fault + + +def unreachable(incoming: Incoming, _handle: Handler) -> Answer: + raise Unreachable(f"Max retries exceeded with url: {incoming.url}") + + +def alters_the_bytes(incoming: Incoming, handle: Handler) -> Answer: + return handle(replace(incoming, body=incoming.body[::-1])) + + +@dataclass(frozen=True) +class DeliveredFile: + filename: str + content: bytes + content_type: Optional[str] = None + content_id: Optional[str] = None + + +@dataclass(frozen=True) +class DeliveredEmail: + message_id: str + to: str + cc: tuple[str, ...] + bcc: tuple[str, ...] + subject: str + html: Optional[str] + text: Optional[str] + attachments: tuple[DeliveredFile, ...] + + +def checksum(content: bytes) -> str: + return base64.b64encode(hashlib.sha256(content).digest()).decode() + + +def storage_signature( + method: str, path: str, query: Mapping[str, str], headers: Mapping[str, str] +) -> str: + canonical = "\n".join( + [ + method, + path, + urlencode(sorted(query.items())), + headers.get("content-length", ""), + headers.get("x-amz-checksum-sha256", ""), + ] ) + return hmac.new(STORAGE_SIGNING_KEY, canonical.encode(), hashlib.sha256).hexdigest() +def json_or_none(body: bytes) -> Any: + try: + return json.loads(body) + except ValueError: + return None + + +def files_asked_for(body: bytes) -> Optional[list[tuple[str, int]]]: + payload = json_or_none(body) + if not isinstance(payload, dict) or set(payload) != {"files"}: + return None + files = payload["files"] + if not isinstance(files, list) or not files: + return None + if not all( + isinstance(file, dict) + and set(file) == {"sha256", "size"} + and isinstance(file["sha256"], str) + and SHA256_HEX.fullmatch(file["sha256"]) + and type(file["size"]) is int + and file["size"] > 0 + for file in files + ): + return None + return [(file["sha256"], file["size"]) for file in files] + + +class FakePostHog: + def __init__(self, storage_url: str = STORAGE_URL) -> None: + self.storage_url = storage_url + self.bucket: dict[str, bytes] = {} + self.writes = 0 + self.delivered: list[DeliveredEmail] = [] + self.project_can_send_files = True + self._sent_by_idempotency_key: dict[str, str] = {} + self._faults: dict[str, list[Fault]] = { + "uploads": [], + "send": [], + "storage": [], + } + self._signing_delay = 0.0 + + def inject(self, route: str, *faults: Fault) -> None: + self._faults[route].extend(faults) + + def sign_urls_in_the_past(self, minutes: int) -> None: + self._signing_delay = minutes * 60.0 + + def handle(self, incoming: Incoming) -> Answer: + route, handler = self._route(incoming) + faults = self._faults.get(route) + if faults: + return faults.pop(0)(incoming, handler) + return handler(incoming) + + def _route(self, incoming: Incoming) -> tuple[str, Handler]: + if incoming.url.startswith(self.storage_url + "/"): + return "storage", self._store + if incoming.url.startswith(POSTHOG_HOST + "/"): + if incoming.path == UPLOADS_PATH: + return "uploads", self._answer_uploads + if incoming.path == SEND_PATH: + return "send", self._send + return "unknown", lambda _incoming: api_error( + 404, "not_found", "Not found.", kind="invalid_request" + ) + + def _answer_uploads(self, incoming: Incoming) -> Answer: + if not incoming.is_authorized: + return UNAUTHORIZED + if not self.project_can_send_files: + return api_error( + 400, + "field_not_accepted", + "This project can't send files yet, so PostHog didn't prepare the uploads.", + kind="invalid_request", + attr="files", + why="Sending files isn't available to this project yet.", + fix="Send the email without files.", + ) + files = files_asked_for(incoming.body) + if files is None: + return api_error( + 400, "invalid_input", "Name each file by sha256 and size.", attr="files" + ) + return Answer.of_json( + 200, {"files": [self._slot(sha256, size) for sha256, size in files]} + ) + + def _slot(self, sha256: str, size: int) -> dict[str, Any]: + if sha256 in self.bucket: + return { + "sha256": sha256, + "status": "present", + "expires_at": "2026-12-16T00:00:00Z", + } + url = f"{self.storage_url}/{sha256}" + headers = { + "content-length": str(size), + "x-amz-checksum-sha256": base64.b64encode(bytes.fromhex(sha256)).decode(), + } + query = { + "X-Amz-Date": str(int(time.time() - self._signing_delay)), + "X-Amz-Expires": str(URL_LIFETIME_SECONDS), + "X-Amz-SignedHeaders": "content-length;x-amz-checksum-sha256", + } + signature = storage_signature("PUT", urlsplit(url).path, query, headers) + return { + "sha256": sha256, + "status": "upload", + "url": f"{url}?{urlencode({**query, 'X-Amz-Signature': signature})}", + "method": "PUT", + "headers": headers, + "url_expires_at": "2026-11-09T12:15:00Z", + } + + def _store(self, incoming: Incoming) -> Answer: + if "authorization" in incoming.headers: + return storage_error( + 400, + "InvalidArgument", + "Only one auth mechanism allowed; only the X-Amz-Algorithm query " + "parameter, Signature query string parameter or the Authorization " + "header should be specified", + ) + query = dict(parse_qsl(urlsplit(incoming.url).query, keep_blank_values=True)) + signature = query.pop("X-Amz-Signature", "") + expected = storage_signature( + incoming.method, incoming.path, query, incoming.headers + ) + if not hmac.compare_digest(signature, expected): + return storage_error( + 403, + "SignatureDoesNotMatch", + "The request signature we calculated does not match the signature you provided.", + ) + if time.time() > int(query["X-Amz-Date"]) + int(query["X-Amz-Expires"]): + return storage_error(403, "AccessDenied", "Request has expired") + if incoming.headers["content-length"] != str( + len(incoming.body) + ) or incoming.headers["x-amz-checksum-sha256"] != checksum(incoming.body): + return storage_error( + 400, + "BadDigest", + "The SHA256 you specified did not match the calculated checksum.", + ) + self.bucket[incoming.path.rsplit("/", 1)[1]] = incoming.body + self.writes += 1 + return Answer(200) + + def _send(self, incoming: Incoming) -> Answer: + if not incoming.is_authorized: + return UNAUTHORIZED + idempotency_key = incoming.headers.get("idempotency-key", "") + if idempotency_key in self._sent_by_idempotency_key: + return Answer.of_json( + 200, + { + "message_id": self._sent_by_idempotency_key[idempotency_key], + "replayed": True, + }, + ) + payload = json_or_none(incoming.body) + if not isinstance(payload, dict): + return api_error(400, "invalid_input", "Send a JSON object.") + unknown = sorted(set(payload) - SEND_FIELDS) + if unknown: + return api_error( + 400, + "invalid_input", + f"PostHog doesn't take {unknown[0]}.", + attr=unknown[0], + ) + references = payload.get("attachments", []) + for index, reference in enumerate(references): + refused = self._refused_reference(index, reference) + if refused is not None: + return refused + message_id = str(uuid4()) + self.delivered.append(self._email(message_id, payload, references)) + if idempotency_key: + self._sent_by_idempotency_key[idempotency_key] = message_id + return Answer.of_json(202, {"message_id": message_id}) + + def _refused_reference(self, index: int, reference: Any) -> Optional[Answer]: + if ( + not isinstance(reference, dict) + or not {"sha256", "filename"} <= set(reference) <= REFERENCE_FIELDS + ): + return api_error( + 400, + "invalid_input", + "An attachment names sha256 and filename.", + attr=f"attachments__{index}", + ) + if reference["sha256"] not in self.bucket: + return api_error( + 400, + "attachment_not_uploaded", + "PostHog doesn't hold this file. Upload it first.", + attr=f"attachments__{index}", + ) + return None + + def _email( + self, message_id: str, payload: dict[str, Any], references: list[Any] + ) -> DeliveredEmail: + return DeliveredEmail( + message_id=message_id, + to=payload["to"], + cc=tuple(payload.get("cc", ())), + bcc=tuple(payload.get("bcc", ())), + subject=payload["subject"], + html=payload.get("html"), + text=payload.get("text"), + attachments=tuple( + DeliveredFile( + filename=reference["filename"], + content=self.bucket[reference["sha256"]], + content_type=reference.get("content_type"), + content_id=reference.get("content_id"), + ) + for reference in references + ), + ) + + +def lowercase(headers: Any) -> dict[str, str]: + return {name.lower(): value for name, value in headers.items()} + + +class FakePostHogAdapter(requests.adapters.BaseAdapter): + def __init__(self, posthog: FakePostHog) -> None: + super().__init__() + self.posthog = posthog + + def send( + self, request: requests.PreparedRequest, **kwargs: Any + ) -> requests.Response: + incoming = Incoming( + request.method or "", + request.url or "", + lowercase(request.headers), + request.body if isinstance(request.body, bytes) else b"", + ) + try: + answer = self.posthog.handle(incoming) + except Unreachable as exc: + raise requests.exceptions.ConnectionError(str(exc)) + response = requests.Response() + response.status_code = answer.status + response._content = answer.body + response.headers.update(answer.headers) + response.request = request + response.url = incoming.url + return response + + def close(self) -> None: + pass + + +def httpx_transport(posthog: FakePostHog) -> httpx.MockTransport: + def handle(request: httpx.Request) -> httpx.Response: + incoming = Incoming( + request.method, str(request.url), lowercase(request.headers), request.read() + ) + try: + answer = posthog.handle(incoming) + except Unreachable as exc: + raise httpx.ConnectError(str(exc), request=request) + return httpx.Response( + answer.status, content=answer.body, headers=answer.headers + ) + + return httpx.MockTransport(handle) + + +def session_on(posthog: FakePostHog, prefix: str = "https://") -> requests.Session: + session = requests.Session() + session.mount(prefix, FakePostHogAdapter(posthog)) + return session + + +def emails_on(posthog: FakePostHog, **options: Any) -> Emails: + return Emails(SECRET_KEY, session=session_on(posthog), **options) + + +def async_emails_on( + posthog: FakePostHog, max_retries: int = 2, **client_options: Any +) -> AsyncEmails: + return AsyncEmails( + SECRET_KEY, + max_retries=max_retries, + http_client=httpx.AsyncClient( + transport=httpx_transport(posthog), **client_options + ), + ) + + +INVITE_SHA256 = "215efae54e34ac5657da47b49c8d1bd8ea12649ee1efa838c9a47d7233f18afb" +LOGO = b"\x89PNG\r\n\x1a\n" +LOGO_SHA256 = "4c4b6a3be1314ab86138bef4314dde022e600960d8689a2c8f8631802d20dab6" +INVITE_FILE = {"filename": "invite.ics", "content": INVITE} INVITE_AND_LOGO = [ { "filename": "invite.ics", @@ -966,104 +1339,143 @@ def bucket_refused(status: int, code: str, message: str) -> Reply: }, {"filename": "logo.png", "content": "iVBORw0KGgo=", "content_id": "logo"}, ] +DELIVERED_INVITE_AND_LOGO = ( + DeliveredFile("invite.ics", INVITE, content_type="text/calendar; method=REQUEST"), + DeliveredFile("logo.png", LOGO, content_id="logo"), +) -def test_uploads_the_files_posthog_lacks_then_sends_their_hashes(): - emails, adapter = emails_with( - [slots(upload(INVITE_SHA256), present(LOGO_SHA256)), stored(), accepted()] - ) +def test_delivers_each_file_with_its_name_type_and_content_id(): + posthog = FakePostHog() - result = send_code(emails, html=HTML, attachments=INVITE_AND_LOGO) + result = send_code(emails_on(posthog), html=HTML, attachments=INVITE_AND_LOGO) - assert result.id == "msg_1" - ask, put, sent = adapter.requests - assert (ask.method, ask.url) == ("POST", UPLOADS_URL) - assert ask.headers["Authorization"] == f"Bearer {SECRET_KEY}" - assert json.loads(ask.body) == { - "files": [ - {"sha256": INVITE_SHA256, "size": 32}, - {"sha256": LOGO_SHA256, "size": 8}, - ] - } - assert (put.method, put.url, put.body) == ( - "PUT", - BUCKET_URL + INVITE_SHA256 + SIGNATURE, - INVITE, - ) - assert "Authorization" not in put.headers - assert {name: put.headers[name] for name in SIGNED_HEADERS} == SIGNED_HEADERS - assert (sent.method, sent.url) == ("POST", SEND_URL) - assert json.loads(sent.body)["attachments"] == [ - { - "sha256": INVITE_SHA256, - "filename": "invite.ics", - "content_type": "text/calendar; method=REQUEST", - }, - {"sha256": LOGO_SHA256, "filename": "logo.png", "content_id": "logo"}, - ] + [email] = posthog.delivered + assert result.id == email.message_id + assert email.attachments == DELIVERED_INVITE_AND_LOGO -def test_stores_each_file_posthog_lacks_under_its_own_hash(): - emails, adapter = emails_with( - [ - slots(upload(INVITE_SHA256), upload(LOGO_SHA256)), - stored(), - stored(), - accepted(), - ] - ) +def test_delivers_cc_and_bcc_beside_the_files(): + posthog = FakePostHog() - result = send_code(emails, html=HTML, attachments=INVITE_AND_LOGO) + send_code( + emails_on(posthog), + html=HTML, + cc="manager@example.com", + bcc=["audit@example.com", "archive@example.com"], + attachments=[INVITE_FILE], + ) - assert result.id == "msg_1" - assert adapter.stored == { - BUCKET_URL + INVITE_SHA256: INVITE, - BUCKET_URL + LOGO_SHA256: LOGO, - } + [email] = posthog.delivered + assert (email.cc, email.bcc) == ( + ("manager@example.com",), + ("audit@example.com", "archive@example.com"), + ) + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) -def test_storage_refuses_bytes_that_do_not_match_the_signed_checksum(): - adapter = FakeAdapter([stored()]) - session = requests.Session() - session.mount("https://", adapter) +def test_stores_only_the_files_posthog_lacks_under_their_own_hash(): + posthog = FakePostHog() + posthog.bucket[LOGO_SHA256] = LOGO - refused = session.put(BUCKET_URL + LOGO_SHA256, data=LOGO, headers=SIGNED_HEADERS) + send_code(emails_on(posthog), html=HTML, attachments=INVITE_AND_LOGO) - assert (refused.status_code, adapter.stored) == (400, {}) + assert posthog.bucket == {INVITE_SHA256: INVITE, LOGO_SHA256: LOGO} + assert posthog.writes == 1 + [email] = posthog.delivered + assert email.attachments == DELIVERED_INVITE_AND_LOGO def test_sends_without_uploading_when_posthog_has_every_file(): - emails, adapter = emails_with([slots(present(INVITE_SHA256)), accepted()]) + posthog = FakePostHog() + posthog.bucket[INVITE_SHA256] = INVITE - send_code( - emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) - assert [(request.method, request.url) for request in adapter.requests] == [ - ("POST", UPLOADS_URL), - ("POST", SEND_URL), - ] + assert posthog.writes == 0 + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) def test_the_upload_carries_only_the_signed_url_and_headers_whatever_the_session_holds(): - adapter = FakeAdapter([slots(upload(INVITE_SHA256)), stored(), accepted()]) - session = requests.Session() - session.mount("https://", adapter) + posthog = FakePostHog() + session = session_on(posthog) session.params = {"api_key": "session-default"} session.headers.update( {"Authorization": "Bearer session-default", "Content-Type": "text/plain"} ) - emails = Emails(SECRET_KEY, session=session) - send_code( - emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] + send_code(Emails(SECRET_KEY, session=session), html=HTML, attachments=[INVITE_FILE]) + + assert posthog.bucket == {INVITE_SHA256: INVITE} + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) + + +def upload_slot(posthog: FakePostHog, content: bytes) -> dict[str, Any]: + asked = posthog.handle( + Incoming( + "POST", + POSTHOG_HOST + UPLOADS_PATH, + {"authorization": f"Bearer {SECRET_KEY}"}, + json.dumps( + { + "files": [ + { + "sha256": hashlib.sha256(content).hexdigest(), + "size": len(content), + } + ] + } + ).encode(), + ) ) + return json.loads(asked.body)["files"][0] - put = adapter.requests[1] - assert put.url == BUCKET_URL + INVITE_SHA256 + SIGNATURE - assert { - name.lower(): value for name, value in put.headers.items() - } == SIGNED_HEADERS + +@parameterized.expand( + [ + ( + "bytes_that_differ_from_the_signed_checksum", + lambda slot: (slot["url"], slot["headers"], INVITE[::-1]), + 400, + "BadDigest", + ), + ( + "an_added_query_parameter", + lambda slot: (slot["url"] + "&api_key=x", slot["headers"], INVITE), + 403, + "SignatureDoesNotMatch", + ), + ( + "no_signed_headers", + lambda slot: (slot["url"], {}, INVITE), + 403, + "SignatureDoesNotMatch", + ), + ( + "an_authorization_header_beside_the_signature", + lambda slot: ( + slot["url"], + {**slot["headers"], "Authorization": f"Bearer {SECRET_KEY}"}, + INVITE, + ), + 400, + "InvalidArgument", + ), + ] +) +def test_the_fake_storage_refuses_what_s3_refuses(_name, tamper, status, code): + posthog = FakePostHog() + url, headers, body = tamper(upload_slot(posthog, INVITE)) + + refused = session_on(posthog).put(url, data=body, headers=headers) + + assert (refused.status_code, f"{code}" in refused.text) == ( + status, + True, + ) + assert posthog.bucket == {} @parameterized.expand( @@ -1078,57 +1490,65 @@ def test_the_upload_carries_only_the_signed_url_and_headers_whatever_the_session ] ) def test_file_content_from_bytes_or_base64_uploads_the_same_bytes(_name, content): - emails, adapter = emails_with([slots(upload(INVITE_SHA256)), stored(), accepted()]) + posthog = FakePostHog() send_code( - emails, html=HTML, attachments=[{"filename": "invite.ics", "content": content}] + emails_on(posthog), + html=HTML, + attachments=[{"filename": "invite.ics", "content": content}], ) - ask, put, _sent = adapter.requests - assert json.loads(ask.body) == {"files": [{"sha256": INVITE_SHA256, "size": 32}]} - assert put.body == INVITE + assert posthog.bucket == {INVITE_SHA256: INVITE} + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) + + +def bytes_altered_on_the_way(posthog: FakePostHog) -> None: + posthog.inject("storage", alters_the_bytes) + + +def url_signed_16_minutes_ago(posthog: FakePostHog) -> None: + posthog.sign_urls_in_the_past(16) @parameterized.expand( [ ( - "signed_checksum_mismatch", - bucket_refused( - 400, "BadDigest", "The Content-SHA256 you specified did not match." - ), + "bytes_altered_on_the_way", + bytes_altered_on_the_way, 400, "BadDigest", + "The SHA256 you specified did not match the calculated checksum.", ), ( "expired_url", - bucket_refused(403, "AccessDenied", "Request has expired"), + url_signed_16_minutes_ago, 403, "AccessDenied", + "Request has expired", ), ] ) def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( - _name, refusal, status, storage_code + _name, arrange, status, storage_code, storage_message ): - emails, adapter = emails_with([slots(upload(INVITE_SHA256)), refusal]) + posthog = FakePostHog() + arrange(posthog) with pytest.raises(EmailError) as raised: - send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) error = raised.value - assert (error.name, error.code, error.status_code, error.field) == ( + assert (error.name, error.code, error.status_code, error.field, error.why) == ( "application_error", "upload_failed", status, "attachments__0", + storage_message, ) assert storage_code in error.message assert "proxy" in error.fix - assert [request.method for request in adapter.requests] == ["POST", "PUT"] + assert (posthog.delivered, posthog.bucket) == ([], {}) @parameterized.expand( @@ -1139,11 +1559,11 @@ def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( ] ) def test_refuses_base64_not_in_one_padded_line_and_says_how_to_encode(_name, content): - emails, adapter = emails_with([]) + posthog = FakePostHog() with pytest.raises(EmailError) as raised: send_code( - emails, + emails_on(posthog), html=HTML, attachments=[{"filename": "a.bin", "content": content}], ) @@ -1154,118 +1574,207 @@ def test_refuses_base64_not_in_one_padded_line_and_says_how_to_encode(_name, con "attachments__0__content", ) assert "base64.b64encode" in error.message - assert adapter.requests == [] + assert (posthog.delivered, posthog.bucket) == ([], {}) + + +def self_signed_certificate(directory: Path) -> tuple[Path, Path]: + pytest.importorskip("cryptography") + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + now = datetime.now(timezone.utc) + certificate = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=5)) + .not_valid_after(now + timedelta(days=1)) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ip_address("127.0.0.1"))]), + critical=False, + ) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False + ) + .add_extension( + x509.SubjectKeyIdentifier.from_public_key(key.public_key()), critical=False + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key(key.public_key()), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + certificate_path, key_path = directory / "storage.pem", directory / "storage.key" + certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + return certificate_path, key_path + + +def storage_handler(posthog: FakePostHog) -> type[BaseHTTPRequestHandler]: + class Storage(BaseHTTPRequestHandler): + def do_PUT(self) -> None: + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + origin = f"https://127.0.0.1:{self.server.server_address[1]}" + answer = posthog.handle( + Incoming("PUT", origin + self.path, lowercase(self.headers), body) + ) + self.send_response(answer.status) + for name, value in answer.headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(answer.body))) + self.end_headers() + self.wfile.write(answer.body) + + def log_message(self, *args: Any) -> None: + pass + + return Storage + + +@pytest.fixture +def posthog_with_tls_storage(tmp_path): + certificate, key = self_signed_certificate(tmp_path) + posthog = FakePostHog() + server = ThreadingHTTPServer(("127.0.0.1", 0), storage_handler(posthog)) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certificate, key) + server.socket = context.wrap_socket(server.socket, server_side=True) + posthog.storage_url = f"https://127.0.0.1:{server.server_address[1]}/bucket" + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield posthog, certificate + finally: + server.shutdown() + server.server_close() -def test_the_upload_trusts_the_certificates_the_environment_names(monkeypatch): - monkeypatch.setenv("REQUESTS_CA_BUNDLE", "/etc/ssl/corporate-ca.pem") - emails, adapter = emails_with([slots(upload(INVITE_SHA256)), stored(), accepted()]) +def test_the_upload_trusts_the_certificates_the_environment_names( + posthog_with_tls_storage, monkeypatch +): + posthog, certificate = posthog_with_tls_storage + monkeypatch.setenv("REQUESTS_CA_BUNDLE", str(certificate)) + emails = Emails(SECRET_KEY, session=session_on(posthog, POSTHOG_HOST)) - send_code( - emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] - ) + send_code(emails, html=HTML, attachments=[INVITE_FILE]) - assert adapter.verify == ["/etc/ssl/corporate-ca.pem"] * 3 + assert posthog.bucket == {INVITE_SHA256: INVITE} + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) -def test_an_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): - unreachable = requests.exceptions.ConnectionError( - f"Max retries exceeded with url: {BUCKET_URL}{INVITE_SHA256}{SIGNATURE}" - ) - emails, adapter = emails_with( - [slots(upload(INVITE_SHA256)), unreachable, unreachable, unreachable] +def test_the_upload_refuses_a_storage_whose_certificate_nothing_trusts( + posthog_with_tls_storage, monkeypatch +): + posthog, _certificate = posthog_with_tls_storage + monkeypatch.delenv("REQUESTS_CA_BUNDLE", raising=False) + monkeypatch.delenv("CURL_CA_BUNDLE", raising=False) + emails = Emails(SECRET_KEY, session=session_on(posthog, POSTHOG_HOST)) + + with pytest.raises(EmailError) as raised: + send_code(emails, html=HTML, attachments=[INVITE_FILE]) + + assert (raised.value.code, raised.value.field) == ( + "network_error", + "attachments__0", ) + assert (posthog.delivered, posthog.bucket) == ([], {}) + + +def test_an_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): + posthog = FakePostHog() + posthog.inject("storage", unreachable, unreachable, unreachable) with pytest.raises(EmailError) as raised: - send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) error = raised.value assert (error.code, error.field) == ("network_error", "attachments__0") assert "file storage" in error.message assert "X-Amz-Signature" not in "".join(traceback.format_exception(error)) - assert [request.method for request in adapter.requests] == ["POST"] + ["PUT"] * 3 + assert (posthog.delivered, posthog.bucket) == ([], {}) @pytest.mark.asyncio async def test_async_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): - def handler(request: httpx.Request) -> httpx.Response: - if request.method == "PUT": - raise httpx.ConnectError(f"Connection refused: {request.url}") - return httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}) - - emails = AsyncEmails( - SECRET_KEY, - max_retries=0, - http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), - ) + posthog = FakePostHog() + posthog.inject("storage", unreachable) with pytest.raises(EmailError) as raised: - await emails.send( - to="user@example.com", - template_key="invite", - subject="You're invited", + await send_code( + async_emails_on(posthog, max_retries=0), html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], + attachments=[INVITE_FILE], ) error = raised.value assert (error.code, error.field) == ("network_error", "attachments__0") assert "file storage" in error.message assert "X-Amz-Signature" not in "".join(traceback.format_exception(error)) + assert (posthog.delivered, posthog.bucket) == ([], {}) @parameterized.expand( [ - ("slow_down", 503, "SlowDown", "Please reduce your request rate."), - ("internal_error", 500, "InternalError", "Please try again."), - ("too_many_requests", 429, "SlowDown", "Please reduce your request rate."), + ( + "slow_down", + answers(storage_error(503, "SlowDown", "Please reduce your request rate.")), + ), + ( + "internal_error", + answers(storage_error(500, "InternalError", "Please try again.")), + ), + ( + "too_many_requests", + answers(storage_error(429, "SlowDown", "Please reduce your request rate.")), + ), + ("connection_dropped", unreachable), ] ) -def test_a_storage_that_asks_for_a_retry_is_retried_before_the_send( - _name, status, code, message -): - emails, adapter = emails_with( - [ - slots(upload(INVITE_SHA256)), - bucket_refused(status, code, message), - stored(), - accepted(), - ] - ) +def test_a_storage_that_asks_for_a_retry_is_retried_before_the_send(_name, fault): + posthog = FakePostHog() + posthog.inject("storage", fault) - send_code( - emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) - assert [request.method for request in adapter.requests] == [ - "POST", - "PUT", - "PUT", - "POST", - ] + assert posthog.bucket == {INVITE_SHA256: INVITE} + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) -def test_a_refused_upload_request_uploads_and_sends_nothing(): - refused = { - "type": "invalid_request", - "code": "field_not_accepted", - "detail": "This project can't send files yet, so PostHog didn't prepare the uploads.", - "attr": "files", - "why": "Sending files isn't available to this project yet.", - "fix": "Send the email without files.", - } - emails, adapter = emails_with([(400, refused, {})]) +def test_a_project_that_cannot_send_files_uploads_and_sends_nothing(): + posthog = FakePostHog() + posthog.project_can_send_files = False with pytest.raises(EmailError) as raised: - send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) error = raised.value assert (error.code, error.field, error.why) == ( @@ -1273,7 +1782,7 @@ def test_a_refused_upload_request_uploads_and_sends_nothing(): "files", "Sending files isn't available to this project yet.", ) - assert [request.url for request in adapter.requests] == [UPLOADS_URL] + assert (posthog.delivered, posthog.bucket) == ([], {}) @parameterized.expand( @@ -1285,23 +1794,18 @@ def test_a_refused_upload_request_uploads_and_sends_nothing(): def test_a_storage_out_of_retries_names_its_passing_fault( _name, status, storage_code, name ): - emails, adapter = emails_with( - [ - slots(upload(INVITE_SHA256)), - bucket_refused(status, storage_code, "Try again."), - ], - max_retries=0, + posthog = FakePostHog() + posthog.inject( + "storage", answers(storage_error(status, storage_code, "Try again.")) ) with pytest.raises(EmailError) as raised: send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], + emails_on(posthog, max_retries=0), html=HTML, attachments=[INVITE_FILE] ) assert (raised.value.name, raised.value.code) == (name, "upload_failed") - assert [request.method for request in adapter.requests] == ["POST", "PUT"] + assert (posthog.delivered, posthog.bucket) == ([], {}) @parameterized.expand( @@ -1313,27 +1817,14 @@ def test_a_storage_out_of_retries_names_its_passing_fault( def test_an_upload_route_404_reports_posthogs_answer_on_attachments_and_sends_nothing( _name, detail ): - emails, adapter = emails_with( - [ - ( - 404, - { - "type": "invalid_request", - "code": "not_found", - "detail": detail, - "attr": None, - }, - {}, - ) - ] + posthog = FakePostHog() + posthog.inject( + "uploads", + answers(api_error(404, "not_found", detail, kind="invalid_request", attr=None)), ) with pytest.raises(EmailError) as raised: - send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) error = raised.value assert (error.name, error.code, error.status_code, error.field, error.message) == ( @@ -1344,18 +1835,15 @@ def test_an_upload_route_404_reports_posthogs_answer_on_attachments_and_sends_no detail, ) assert error.why and error.fix - assert [request.url for request in adapter.requests] == [UPLOADS_URL] + assert (posthog.delivered, posthog.bucket) == ([], {}) def test_an_upload_route_404_without_an_answer_still_names_attachments(): - emails, adapter = emails_with([(404, b"

Not Found

", {})]) + posthog = FakePostHog() + posthog.inject("uploads", answers(Answer(404, b"

Not Found

"))) with pytest.raises(EmailError) as raised: - send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) error = raised.value assert (error.code, error.status_code, error.field) == ( @@ -1363,53 +1851,59 @@ def test_an_upload_route_404_without_an_answer_still_names_attachments(): 404, "attachments", ) - assert [request.url for request in adapter.requests] == [UPLOADS_URL] + assert (posthog.delivered, posthog.bucket) == ([], {}) def test_a_project_over_its_daily_upload_bytes_is_told_without_retrying(): - emails, adapter = emails_with([rejected(429, "upload_quota_exceeded"), slots()]) + posthog = FakePostHog() + posthog.inject( + "uploads", + answers( + api_error( + 429, + "upload_quota_exceeded", + "The project uploaded its daily limit of file bytes.", + kind="throttled_error", + ) + ), + ) with pytest.raises(EmailError) as raised: - send_code( - emails, - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) assert (raised.value.code, raised.value.name) == ( "upload_quota_exceeded", "daily_quota_exceeded", ) - assert len(adapter.requests) == 1 + assert (posthog.delivered, posthog.bucket) == ([], {}) def test_a_retried_send_reuses_its_uploads_and_its_idempotency_key(): - emails, adapter = emails_with( - [ - slots(upload(INVITE_SHA256)), - stored(), - rejected(503, "delivery_unavailable"), - accepted(), - ] + posthog = FakePostHog() + posthog.inject( + "send", + loses_the_reply( + api_error(503, "delivery_unavailable", "Try again.", kind="server_error") + ), ) - send_code( - emails, + result = send_code( + emails_on(posthog), html=HTML, idempotency_key="invite:user_123", - attachments=[{"filename": "invite.ics", "content": INVITE}], + attachments=[INVITE_FILE], ) - _ask, _put, first, retry = adapter.requests - assert (retry.url, retry.body) == (SEND_URL, first.body) - assert retry.headers["Idempotency-Key"] == "invite:user_123" + [email] = posthog.delivered + assert (result.id, result.replayed) == (email.message_id, True) + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) def test_the_same_bytes_in_two_attachments_upload_once(): - emails, adapter = emails_with([slots(upload(LOGO_SHA256)), stored(), accepted()]) + posthog = FakePostHog() send_code( - emails, + emails_on(posthog), html=HTML, attachments=[ {"filename": "logo.png", "content": LOGO, "content_id": "logo"}, @@ -1417,52 +1911,52 @@ def test_the_same_bytes_in_two_attachments_upload_once(): ], ) - ask, put, sent = adapter.requests - assert json.loads(ask.body) == {"files": [{"sha256": LOGO_SHA256, "size": 8}]} - assert put.body == LOGO - assert json.loads(sent.body)["attachments"] == [ - {"sha256": LOGO_SHA256, "filename": "logo.png", "content_id": "logo"}, - {"sha256": LOGO_SHA256, "filename": "logo-copy.png"}, - ] + assert (posthog.bucket, posthog.writes) == ({LOGO_SHA256: LOGO}, 1) + [email] = posthog.delivered + assert email.attachments == ( + DeliveredFile("logo.png", LOGO, content_id="logo"), + DeliveredFile("logo-copy.png", LOGO), + ) def test_a_failed_upload_names_the_first_attachment_with_those_bytes(): - emails, _adapter = emails_with( - [ - slots(present(INVITE_SHA256), upload(LOGO_SHA256)), - bucket_refused(403, "AccessDenied", "Request has expired"), - ] - ) + posthog = FakePostHog() + posthog.bucket[INVITE_SHA256] = INVITE + posthog.sign_urls_in_the_past(16) with pytest.raises(EmailError) as raised: send_code( - emails, + emails_on(posthog), html=HTML, attachments=[ - {"filename": "invite.ics", "content": INVITE}, + INVITE_FILE, {"filename": "logo.png", "content": LOGO}, {"filename": "logo-copy.png", "content": LOGO}, ], ) - assert raised.value.field == "attachments__1" + assert (raised.value.code, raised.value.field) == ( + "upload_failed", + "attachments__1", + ) + assert posthog.delivered == [] def test_refuses_eleven_files_before_reading_any_of_them(): - emails, adapter = emails_with([]) + posthog = FakePostHog() attachments: list[Any] = [ {"filename": f"{i}.txt", "content": b"hi"} for i in range(10) ] with pytest.raises(EmailError) as raised: send_code( - emails, + emails_on(posthog), html=HTML, attachments=attachments + [{"filename": "late.txt", "content": None}], ) assert raised.value.code == "too_many_attachments" - assert adapter.requests == [] + assert (posthog.delivered, posthog.bucket) == ([], {}) @parameterized.expand( @@ -1495,10 +1989,10 @@ def test_refuses_eleven_files_before_reading_any_of_them(): ] ) def test_refuses_an_email_over_the_limits_before_any_request(_name, attachments, code): - emails, adapter = emails_with([]) + posthog = FakePostHog() with pytest.raises(EmailError) as raised: - send_code(emails, html="Hi!", attachments=attachments) + send_code(emails_on(posthog), html="Hi!", attachments=attachments) error = raised.value assert (error.name, error.code, error.field) == ( @@ -1507,7 +2001,7 @@ def test_refuses_an_email_over_the_limits_before_any_request(_name, attachments, "attachments", ) assert error.fix - assert adapter.requests == [] + assert (posthog.delivered, posthog.bucket) == ([], {}) def test_refuses_an_oversized_email_without_copying_its_files(): @@ -1555,25 +2049,22 @@ def test_refuses_oversized_padded_base64_without_copying_it(): LARGEST_SCAN = bytes(29_182_818) -LARGEST_SCAN_SHA256 = "23a605f8a0f14f6ad57155dcee6f7e45f783f1a9cd44e6c38f6dfad6d33ef14c" def test_sends_the_largest_file_that_fits_beside_the_body(): - emails, adapter = emails_with([slots(present(LARGEST_SCAN_SHA256)), accepted()]) + posthog = FakePostHog() send_code( - emails, + emails_on(posthog), html="Hi!", attachments=[{"filename": "scan.pdf", "content": LARGEST_SCAN}], ) - assert json.loads(adapter.requests[0].body)["files"][0]["size"] == 29_182_818 + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("scan.pdf", LARGEST_SCAN),) SCAN_BESIDE_A_78_BYTE_BODY = bytes(29_182_764) -SCAN_BESIDE_A_78_BYTE_BODY_SHA256 = ( - "af928bfa2ca810344546c2cdb01a6f5fb9f8545281e4d6c7b75a1bfec9f1f2c3" -) @parameterized.expand( @@ -1583,9 +2074,8 @@ def test_sends_the_largest_file_that_fits_beside_the_body(): ] ) def test_counts_a_body_like_a_file_in_the_size_estimate(_name, body_field): - emails, adapter = emails_with( - [slots(present(SCAN_BESIDE_A_78_BYTE_BODY_SHA256)), accepted()] - ) + posthog = FakePostHog() + emails = emails_on(posthog) attachments = [{"filename": "scan.pdf", "content": SCAN_BESIDE_A_78_BYTE_BODY}] send_code(emails, attachments=attachments, **{body_field: "a" * 57}) @@ -1593,22 +2083,22 @@ def test_counts_a_body_like_a_file_in_the_size_estimate(_name, body_field): send_code(emails, attachments=attachments, **{body_field: "é" * 57}) assert raised.value.code == "message_too_large" - assert len(adapter.requests) == 2 + assert len(posthog.delivered) == 1 def test_counts_html_and_text_as_two_bodies(): - emails, adapter = emails_with([]) + posthog = FakePostHog() with pytest.raises(EmailError) as raised: send_code( - emails, + emails_on(posthog), html="a", text="a", attachments=[{"filename": "scan.pdf", "content": LARGEST_SCAN}], ) assert raised.value.code == "message_too_large" - assert adapter.requests == [] + assert (posthog.delivered, posthog.bucket) == ([], {}) @parameterized.expand( @@ -1622,16 +2112,14 @@ def test_counts_html_and_text_as_two_bodies(): def test_a_subject_takes_75_bytes_of_the_limit_per_42_utf8_bytes( _name, subject, largest_file_size ): - largest = bytes(largest_file_size) - emails, adapter = emails_with( - [slots(present(hashlib.sha256(largest).hexdigest())), accepted()] - ) + posthog = FakePostHog() + emails = emails_on(posthog) send_code( emails, html="Hi!", subject=subject, - attachments=[{"filename": "scan.pdf", "content": largest}], + attachments=[{"filename": "scan.pdf", "content": bytes(largest_file_size)}], ) with pytest.raises(EmailError) as raised: send_code( @@ -1644,7 +2132,7 @@ def test_a_subject_takes_75_bytes_of_the_limit_per_42_utf8_bytes( ) assert raised.value.code == "message_too_large" - assert len(adapter.requests) == 2 + assert len(posthog.delivered) == 1 class PostHogWithARedirectingStorage(BaseHTTPRequestHandler): @@ -1654,8 +2142,14 @@ def do_POST(self) -> None: self.hits.append(f"POST {self.path}") self.rfile.read(int(self.headers["Content-Length"])) if self.path.endswith("/attachments/uploads/"): - slot = upload(INVITE_SHA256) - slot["url"] = f"http://{self.headers['Host']}/bucket/{INVITE_SHA256}" + slot = { + "sha256": INVITE_SHA256, + "status": "upload", + "url": f"http://{self.headers['Host']}/bucket/{INVITE_SHA256}", + "method": "PUT", + "headers": {"content-length": str(len(INVITE))}, + "url_expires_at": "2026-11-09T12:15:00Z", + } self.answer(200, json.dumps({"files": [slot]}).encode()) else: self.answer(202, json.dumps({"message_id": "msg_1"}).encode()) @@ -1718,88 +2212,39 @@ async def test_async_upload_never_follows_a_storage_redirect_even_when_the_clien @pytest.mark.asyncio -async def test_async_send_uploads_without_the_secret_key_then_sends_the_hashes(): - seen: list[httpx.Request] = [] - replies = [ - httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}), - httpx.Response(200), - httpx.Response(202, json={"message_id": "msg_1"}), - ] - - def handler(request: httpx.Request) -> httpx.Response: - seen.append(request) - return replies.pop(0) - - emails = AsyncEmails( - SECRET_KEY, - http_client=httpx.AsyncClient( - transport=httpx.MockTransport(handler), - params={"api_key": "client-default"}, - headers={ - "Authorization": "Bearer client-default", - "Content-Type": "text/plain", - }, - ), +async def test_async_upload_carries_only_the_signed_url_and_headers_whatever_the_client_holds(): + posthog = FakePostHog() + emails = async_emails_on( + posthog, + params={"api_key": "client-default"}, + headers={ + "Authorization": "Bearer client-default", + "Content-Type": "text/plain", + }, ) - result = await emails.send( - to="user@example.com", - template_key="invite", - subject="You're invited", - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + result = await send_code(emails, html=HTML, attachments=INVITE_AND_LOGO) - assert result.id == "msg_1" - ask, put, sent = seen - assert (ask.method, ask.url.path) == ( - "POST", - "/api/projects/@current/messaging/attachments/uploads/", - ) - assert ask.headers["Authorization"] == f"Bearer {SECRET_KEY}" - assert (put.method, str(put.url), put.content) == ( - "PUT", - BUCKET_URL + INVITE_SHA256 + SIGNATURE, - INVITE, - ) - assert { - name: value for name, value in put.headers.items() if name != "host" - } == SIGNED_HEADERS - assert json.loads(sent.content)["attachments"] == [ - {"sha256": INVITE_SHA256, "filename": "invite.ics"} - ] + assert posthog.bucket == {INVITE_SHA256: INVITE, LOGO_SHA256: LOGO} + [email] = posthog.delivered + assert result.id == email.message_id + assert email.attachments == DELIVERED_INVITE_AND_LOGO @pytest.mark.asyncio async def test_async_refused_upload_sends_no_email(): - seen: list[httpx.Request] = [] - replies = [ - httpx.Response(200, json={"files": [upload(INVITE_SHA256)]}), - httpx.Response( - 400, content=b"BadDigestNo." - ), - ] - - def handler(request: httpx.Request) -> httpx.Response: - seen.append(request) - return replies.pop(0) - - emails = AsyncEmails( - SECRET_KEY, - http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), - ) + posthog = FakePostHog() + posthog.sign_urls_in_the_past(16) with pytest.raises(EmailError) as raised: - await emails.send( - to="user@example.com", - template_key="invite", - subject="You're invited", - html=HTML, - attachments=[{"filename": "invite.ics", "content": INVITE}], - ) + await send_code(async_emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) - assert raised.value.code == "upload_failed" - assert [request.method for request in seen] == ["POST", "PUT"] + assert (raised.value.code, raised.value.field, raised.value.status_code) == ( + "upload_failed", + "attachments__0", + 403, + ) + assert (posthog.delivered, posthog.bucket) == ([], {}) SECRET_CONTENT = b"secret-file-content" @@ -1882,10 +2327,10 @@ def handler(request: httpx.Request) -> httpx.Response: def test_rejects_an_attachment_it_cannot_send_without_repeating_its_content( _name, attachments, field ): - emails, adapter = emails_with([]) + posthog = FakePostHog() with pytest.raises(EmailError) as raised: - send_code(emails, html=HTML, attachments=attachments) + send_code(emails_on(posthog), html=HTML, attachments=attachments) error = raised.value assert (error.name, error.code, error.field) == ( @@ -1895,4 +2340,4 @@ def test_rejects_an_attachment_it_cannot_send_without_repeating_its_content( ) assert "secret-file-content" not in str(error) assert "115, 101" not in str(error) - assert adapter.requests == [] + assert (posthog.delivered, posthog.bucket) == ([], {}) From 5e8e8992369c214edb1d4e953e0d8ebe387033a1 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:40:53 +0000 Subject: [PATCH 30/39] fix(emails): count the text PostHog derives and upload only over https Without text, PostHog derives the plain text from the HTML and counts it in the message size, so an HTML-only email near the limit uploaded its files and was then refused. The client now counts the HTML twice, since the derived text is never longer. A plain HTTP upload URL from misconfigured storage would carry the file and its signed URL unencrypted. The client now stops before any upload with api_error naming the attachment, unless the client's host is plain HTTP too, as in local development. The fake PostHog now checks each route's method and records the content type storage keeps. Tests assert stored files, not storage keys, and the tests that only exercised the fake are gone. Refs PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-code-attachments.md | 2 +- posthog/emails.py | 62 +++-- posthog/test/test_emails.py | 249 ++++++++++--------- 3 files changed, 176 insertions(+), 137 deletions(-) diff --git a/.sampo/changesets/emails-code-attachments.md b/.sampo/changesets/emails-code-attachments.md index 5b04b446d..021138277 100644 --- a/.sampo/changesets/emails-code-attachments.md +++ b/.sampo/changesets/emails-code-attachments.md @@ -2,4 +2,4 @@ pypi/posthog: minor --- -A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage, then sends the email with each file's SHA-256. A refused upload raises `EmailError` with the code `upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. +A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files, with the HTML counted twice when there is no `text`, since PostHog derives the text from it. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage over https, then sends the email with each file's SHA-256. A refused upload raises `EmailError` with the code `upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. diff --git a/posthog/emails.py b/posthog/emails.py index 90241d470..e830cfe23 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -59,15 +59,18 @@ ``"resend"``, on the client or on :class:`Emails`, so PostHog knows where your sends came from. -Attachments: a code send takes ``attachments``, at most 10 files and 40 MB -per message after Base64 encoding, about 29 MB of files. The client checks +Attachments: a code send takes ``attachments``, at most 10 files and 40 MB per +message after Base64 encoding, about 29 MB of files. Without ``text``, the +HTML counts twice, since PostHog derives the text from it. The client checks both before any request. It then asks PostHog which files it lacks, uploads -each of those to the URL PostHog answers, without the secret key, and sends the -email naming each file by its SHA-256. A file the project already sent uploads -nothing, and the same bytes in two attachments upload once. An upload retries -like a send when the storage is unreachable or answers a passing fault. A -failed upload sends no email. The upload URL's host is not the app host, so a -network that limits outbound requests must allow it. +each of those to the https URL PostHog answers, without the secret key, and +sends the email naming each file by its SHA-256. A file the project already +sent uploads nothing, and the same bytes in two attachments upload once. An +upload retries like a send when the storage is unreachable or answers a +passing fault. A failed upload sends no email. A plain HTTP upload URL is +refused before any upload unless the client's host is plain HTTP too, as in +local development. The upload URL's host is not the app host, so a network +that limits outbound requests must allow it. Errors: a send that is not queued raises :class:`EmailError`. Branch on ``code``. The ones a setup usually hits: @@ -91,6 +94,8 @@ and ``why`` carry its answer, and ``field`` names the attachment. - ``network_error`` with ``field`` naming an attachment: the file storage didn't answer, such as when a firewall blocks its host. +- ``api_error`` with ``field`` naming an attachment: PostHog named an upload + URL that isn't https. Fix the file storage's URL. Nothing was uploaded. - ``upload_quota_exceeded`` (429): the project uploaded its daily limit of file bytes. Files it already sent don't count. - ``field_not_accepted``: the project can't send files yet. @@ -661,11 +666,8 @@ def _over_the_limit(code: str, message: str, *, fix: str) -> EmailError: def _estimated_message_bytes( file_sizes: Sequence[int], content: Mapping[str, str] ) -> int: - body_sizes = [ - len(body.encode("utf-8")) - for body in (content.get("html"), content.get("text")) - if body - ] + html, text = content.get("html") or "", content.get("text") or "" + body_sizes = [len(body.encode("utf-8")) for body in (html, text or html) if body] return ( _HEADERS_AND_MIME_BOUNDARIES_BYTES + sum(_base64_transfer_bytes(size) for size in [*file_sizes, *body_sizes]) @@ -938,7 +940,9 @@ def _parse_status_response( ) -def _pending_uploads_parser(files: Sequence[_File]) -> _Parse[list[_Upload]]: +def _pending_uploads_parser( + files: Sequence[_File], plain_http_allowed: bool +) -> _Parse[list[_Upload]]: def parse( status: int, body: bytes, retry_after_header: Optional[str] ) -> list[_Upload]: @@ -950,10 +954,14 @@ def parse( slots = fields.get("files") if not isinstance(slots, list) or len(slots) != len(files): raise _no_upload_slots(status) - uploads = ( + pending = ( _pending_upload(slot, file, status) for slot, file in zip(slots, files) ) - return [upload for upload in uploads if upload is not None] + uploads = [upload for upload in pending if upload is not None] + for upload in uploads: + if not plain_http_allowed and not upload.url.lower().startswith("https://"): + raise _upload_url_not_https(upload.file.first_attachment_index, status) + return uploads return parse @@ -995,6 +1003,20 @@ def _upload_route_not_found( ) +def _upload_url_not_https(index: int, status: int) -> EmailError: + return EmailError( + name=_error_name(status, "api_error"), + code="api_error", + message=f"PostHog named an upload URL that isn't https, so attachments[{index}] " + "would travel unencrypted. No email was sent.", + status_code=status, + why="PostHog's file storage is set up with a plain HTTP URL.", + fix="Fix the file storage's URL so it uses https. For local development " + "over plain HTTP, give the client a host that starts with http://.", + field=f"attachments__{index}", + ) + + def _no_upload_slots(status: int) -> EmailError: return _sdk_error( "api_error", "PostHog answered without an upload slot for each file.", status @@ -1323,7 +1345,9 @@ def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: self._posthog_request( "POST", _UPLOADS_PATH, _files_body(prepared.files), headers ), - _pending_uploads_parser(prepared.files), + _pending_uploads_parser( + prepared.files, self.host.lower().startswith("http://") + ), prepared.max_retries, ) for upload in uploads: @@ -1540,7 +1564,9 @@ async def _upload(self, prepared: _PreparedSend, secret_key: str) -> None: self._posthog_request( "POST", _UPLOADS_PATH, _files_body(prepared.files), headers ), - _pending_uploads_parser(prepared.files), + _pending_uploads_parser( + prepared.files, self.host.lower().startswith("http://") + ), prepared.max_retries, ) for upload in uploads: diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index d63e3487c..aa9281c7f 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1056,9 +1056,13 @@ def files_asked_for(body: bytes) -> Optional[list[tuple[str, int]]]: class FakePostHog: - def __init__(self, storage_url: str = STORAGE_URL) -> None: + def __init__( + self, storage_url: str = STORAGE_URL, posthog_host: str = POSTHOG_HOST + ) -> None: self.storage_url = storage_url - self.bucket: dict[str, bytes] = {} + self.posthog_host = posthog_host + self._bucket: dict[str, bytes] = {} + self._content_types: dict[str, Optional[str]] = {} self.writes = 0 self.delivered: list[DeliveredEmail] = [] self.project_can_send_files = True @@ -1070,6 +1074,17 @@ def __init__(self, storage_url: str = STORAGE_URL) -> None: } self._signing_delay = 0.0 + @property + def stored_files(self) -> list[bytes]: + return sorted(self._bucket.values()) + + @property + def stored_content_types(self) -> list[Optional[str]]: + return list(self._content_types.values()) + + def already_has(self, content: bytes) -> None: + self._bucket[hashlib.sha256(content).hexdigest()] = content + def inject(self, route: str, *faults: Fault) -> None: self._faults[route].extend(faults) @@ -1085,12 +1100,26 @@ def handle(self, incoming: Incoming) -> Answer: def _route(self, incoming: Incoming) -> tuple[str, Handler]: if incoming.url.startswith(self.storage_url + "/"): + if incoming.method != "PUT": + return "storage", lambda _incoming: storage_error( + 405, + "MethodNotAllowed", + "The specified method is not allowed against this resource.", + ) return "storage", self._store - if incoming.url.startswith(POSTHOG_HOST + "/"): - if incoming.path == UPLOADS_PATH: - return "uploads", self._answer_uploads - if incoming.path == SEND_PATH: - return "send", self._send + if incoming.url.startswith(self.posthog_host + "/") and incoming.path in ( + UPLOADS_PATH, + SEND_PATH, + ): + route = "uploads" if incoming.path == UPLOADS_PATH else "send" + if incoming.method != "POST": + return route, lambda _incoming: api_error( + 405, + "method_not_allowed", + f'Method "{incoming.method}" not allowed.', + kind="invalid_request", + ) + return route, (self._answer_uploads if route == "uploads" else self._send) return "unknown", lambda _incoming: api_error( 404, "not_found", "Not found.", kind="invalid_request" ) @@ -1118,7 +1147,7 @@ def _answer_uploads(self, incoming: Incoming) -> Answer: ) def _slot(self, sha256: str, size: int) -> dict[str, Any]: - if sha256 in self.bucket: + if sha256 in self._bucket: return { "sha256": sha256, "status": "present", @@ -1174,7 +1203,9 @@ def _store(self, incoming: Incoming) -> Answer: "BadDigest", "The SHA256 you specified did not match the calculated checksum.", ) - self.bucket[incoming.path.rsplit("/", 1)[1]] = incoming.body + key = incoming.path.rsplit("/", 1)[1] + self._bucket[key] = incoming.body + self._content_types[key] = incoming.headers.get("content-type") self.writes += 1 return Answer(200) @@ -1223,7 +1254,7 @@ def _refused_reference(self, index: int, reference: Any) -> Optional[Answer]: "An attachment names sha256 and filename.", attr=f"attachments__{index}", ) - if reference["sha256"] not in self.bucket: + if reference["sha256"] not in self._bucket: return api_error( 400, "attachment_not_uploaded", @@ -1246,7 +1277,7 @@ def _email( attachments=tuple( DeliveredFile( filename=reference["filename"], - content=self.bucket[reference["sha256"]], + content=self._bucket[reference["sha256"]], content_type=reference.get("content_type"), content_id=reference.get("content_id"), ) @@ -1376,11 +1407,11 @@ def test_delivers_cc_and_bcc_beside_the_files(): def test_stores_only_the_files_posthog_lacks_under_their_own_hash(): posthog = FakePostHog() - posthog.bucket[LOGO_SHA256] = LOGO + posthog.already_has(LOGO) send_code(emails_on(posthog), html=HTML, attachments=INVITE_AND_LOGO) - assert posthog.bucket == {INVITE_SHA256: INVITE, LOGO_SHA256: LOGO} + assert posthog.stored_files == sorted([INVITE, LOGO]) assert posthog.writes == 1 [email] = posthog.delivered assert email.attachments == DELIVERED_INVITE_AND_LOGO @@ -1388,7 +1419,7 @@ def test_stores_only_the_files_posthog_lacks_under_their_own_hash(): def test_sends_without_uploading_when_posthog_has_every_file(): posthog = FakePostHog() - posthog.bucket[INVITE_SHA256] = INVITE + posthog.already_has(INVITE) send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) @@ -1407,77 +1438,11 @@ def test_the_upload_carries_only_the_signed_url_and_headers_whatever_the_session send_code(Emails(SECRET_KEY, session=session), html=HTML, attachments=[INVITE_FILE]) - assert posthog.bucket == {INVITE_SHA256: INVITE} + assert (posthog.stored_files, posthog.stored_content_types) == ([INVITE], [None]) [email] = posthog.delivered assert email.attachments == (DeliveredFile("invite.ics", INVITE),) -def upload_slot(posthog: FakePostHog, content: bytes) -> dict[str, Any]: - asked = posthog.handle( - Incoming( - "POST", - POSTHOG_HOST + UPLOADS_PATH, - {"authorization": f"Bearer {SECRET_KEY}"}, - json.dumps( - { - "files": [ - { - "sha256": hashlib.sha256(content).hexdigest(), - "size": len(content), - } - ] - } - ).encode(), - ) - ) - return json.loads(asked.body)["files"][0] - - -@parameterized.expand( - [ - ( - "bytes_that_differ_from_the_signed_checksum", - lambda slot: (slot["url"], slot["headers"], INVITE[::-1]), - 400, - "BadDigest", - ), - ( - "an_added_query_parameter", - lambda slot: (slot["url"] + "&api_key=x", slot["headers"], INVITE), - 403, - "SignatureDoesNotMatch", - ), - ( - "no_signed_headers", - lambda slot: (slot["url"], {}, INVITE), - 403, - "SignatureDoesNotMatch", - ), - ( - "an_authorization_header_beside_the_signature", - lambda slot: ( - slot["url"], - {**slot["headers"], "Authorization": f"Bearer {SECRET_KEY}"}, - INVITE, - ), - 400, - "InvalidArgument", - ), - ] -) -def test_the_fake_storage_refuses_what_s3_refuses(_name, tamper, status, code): - posthog = FakePostHog() - url, headers, body = tamper(upload_slot(posthog, INVITE)) - - refused = session_on(posthog).put(url, data=body, headers=headers) - - assert (refused.status_code, f"{code}" in refused.text) == ( - status, - True, - ) - assert posthog.bucket == {} - - @parameterized.expand( [ ("bytearray", bytearray(INVITE)), @@ -1498,7 +1463,7 @@ def test_file_content_from_bytes_or_base64_uploads_the_same_bytes(_name, content attachments=[{"filename": "invite.ics", "content": content}], ) - assert posthog.bucket == {INVITE_SHA256: INVITE} + assert posthog.stored_files == [INVITE] [email] = posthog.delivered assert email.attachments == (DeliveredFile("invite.ics", INVITE),) @@ -1548,7 +1513,7 @@ def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( ) assert storage_code in error.message assert "proxy" in error.fix - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) @parameterized.expand( @@ -1574,7 +1539,7 @@ def test_refuses_base64_not_in_one_padded_line_and_says_how_to_encode(_name, con "attachments__0__content", ) assert "base64.b64encode" in error.message - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) def self_signed_certificate(directory: Path) -> tuple[Path, Path]: @@ -1685,7 +1650,7 @@ def test_the_upload_trusts_the_certificates_the_environment_names( send_code(emails, html=HTML, attachments=[INVITE_FILE]) - assert posthog.bucket == {INVITE_SHA256: INVITE} + assert posthog.stored_files == [INVITE] [email] = posthog.delivered assert email.attachments == (DeliveredFile("invite.ics", INVITE),) @@ -1705,7 +1670,7 @@ def test_the_upload_refuses_a_storage_whose_certificate_nothing_trusts( "network_error", "attachments__0", ) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) def test_an_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): @@ -1719,7 +1684,7 @@ def test_an_unreachable_storage_never_shows_the_signed_url_in_the_traceback(): assert (error.code, error.field) == ("network_error", "attachments__0") assert "file storage" in error.message assert "X-Amz-Signature" not in "".join(traceback.format_exception(error)) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) @pytest.mark.asyncio @@ -1738,7 +1703,7 @@ async def test_async_unreachable_storage_never_shows_the_signed_url_in_the_trace assert (error.code, error.field) == ("network_error", "attachments__0") assert "file storage" in error.message assert "X-Amz-Signature" not in "".join(traceback.format_exception(error)) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) @parameterized.expand( @@ -1764,7 +1729,7 @@ def test_a_storage_that_asks_for_a_retry_is_retried_before_the_send(_name, fault send_code(emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) - assert posthog.bucket == {INVITE_SHA256: INVITE} + assert posthog.stored_files == [INVITE] [email] = posthog.delivered assert email.attachments == (DeliveredFile("invite.ics", INVITE),) @@ -1782,7 +1747,7 @@ def test_a_project_that_cannot_send_files_uploads_and_sends_nothing(): "files", "Sending files isn't available to this project yet.", ) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) @parameterized.expand( @@ -1805,7 +1770,7 @@ def test_a_storage_out_of_retries_names_its_passing_fault( ) assert (raised.value.name, raised.value.code) == (name, "upload_failed") - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) @parameterized.expand( @@ -1835,7 +1800,7 @@ def test_an_upload_route_404_reports_posthogs_answer_on_attachments_and_sends_no detail, ) assert error.why and error.fix - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) def test_an_upload_route_404_without_an_answer_still_names_attachments(): @@ -1851,7 +1816,7 @@ def test_an_upload_route_404_without_an_answer_still_names_attachments(): 404, "attachments", ) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) def test_a_project_over_its_daily_upload_bytes_is_told_without_retrying(): @@ -1875,7 +1840,7 @@ def test_a_project_over_its_daily_upload_bytes_is_told_without_retrying(): "upload_quota_exceeded", "daily_quota_exceeded", ) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) def test_a_retried_send_reuses_its_uploads_and_its_idempotency_key(): @@ -1911,7 +1876,7 @@ def test_the_same_bytes_in_two_attachments_upload_once(): ], ) - assert (posthog.bucket, posthog.writes) == ({LOGO_SHA256: LOGO}, 1) + assert (posthog.stored_files, posthog.writes) == ([LOGO], 1) [email] = posthog.delivered assert email.attachments == ( DeliveredFile("logo.png", LOGO, content_id="logo"), @@ -1921,7 +1886,7 @@ def test_the_same_bytes_in_two_attachments_upload_once(): def test_a_failed_upload_names_the_first_attachment_with_those_bytes(): posthog = FakePostHog() - posthog.bucket[INVITE_SHA256] = INVITE + posthog.already_has(INVITE) posthog.sign_urls_in_the_past(16) with pytest.raises(EmailError) as raised: @@ -1956,7 +1921,7 @@ def test_refuses_eleven_files_before_reading_any_of_them(): ) assert raised.value.code == "too_many_attachments" - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) @parameterized.expand( @@ -2001,7 +1966,7 @@ def test_refuses_an_email_over_the_limits_before_any_request(_name, attachments, "attachments", ) assert error.fix - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) def test_refuses_an_oversized_email_without_copying_its_files(): @@ -2048,7 +2013,7 @@ def test_refuses_oversized_padded_base64_without_copying_it(): assert adapter.requests == [] -LARGEST_SCAN = bytes(29_182_818) +LARGEST_SCAN = bytes(29_182_815) def test_sends_the_largest_file_that_fits_beside_the_body(): @@ -2064,49 +2029,94 @@ def test_sends_the_largest_file_that_fits_beside_the_body(): assert email.attachments == (DeliveredFile("scan.pdf", LARGEST_SCAN),) -SCAN_BESIDE_A_78_BYTE_BODY = bytes(29_182_764) +SCAN_BESIDE_A_78_BYTE_BODY = bytes(29_182_761) @parameterized.expand( [ - ("html", "html"), - ("text", "text"), + ("html", "html", "text"), + ("text", "text", "html"), ] ) -def test_counts_a_body_like_a_file_in_the_size_estimate(_name, body_field): +def test_counts_a_body_like_a_file_in_the_size_estimate( + _name, body_field, other_body_field +): posthog = FakePostHog() emails = emails_on(posthog) attachments = [{"filename": "scan.pdf", "content": SCAN_BESIDE_A_78_BYTE_BODY}] - send_code(emails, attachments=attachments, **{body_field: "a" * 57}) + send_code( + emails, attachments=attachments, **{body_field: "a" * 57, other_body_field: "b"} + ) with pytest.raises(EmailError) as raised: - send_code(emails, attachments=attachments, **{body_field: "é" * 57}) + send_code( + emails, + attachments=attachments, + **{body_field: "é" * 57, other_body_field: "b"}, + ) assert raised.value.code == "message_too_large" assert len(posthog.delivered) == 1 -def test_counts_html_and_text_as_two_bodies(): +def test_counts_the_html_twice_without_text_since_posthog_derives_the_text(): posthog = FakePostHog() + emails = emails_on(posthog) + attachments = [{"filename": "scan.pdf", "content": SCAN_BESIDE_A_78_BYTE_BODY}] + + with pytest.raises(EmailError) as raised: + send_code(emails, html="a" * 57, attachments=attachments) + send_code(emails, html="a" * 57, text="b", attachments=attachments) + + assert raised.value.code == "message_too_large" + assert len(posthog.delivered) == 1 + + +def test_uploads_only_to_an_https_storage_url(): + posthog = FakePostHog(storage_url="http://files.example.com/team/1") with pytest.raises(EmailError) as raised: send_code( emails_on(posthog), - html="a", - text="a", - attachments=[{"filename": "scan.pdf", "content": LARGEST_SCAN}], + html=HTML, + attachments=[ + {"filename": "notes.txt", "content": b"notes"}, + {"filename": "invite.ics", "content": INVITE}, + ], ) - assert raised.value.code == "message_too_large" - assert (posthog.delivered, posthog.bucket) == ([], {}) + error = raised.value + assert (error.code, error.field) == ("api_error", "attachments__0") + assert "https" in error.message and "file storage" in (error.fix or "") + assert "files.example.com" not in str(error) + str(vars(error)) + assert (posthog.delivered, posthog.stored_files, posthog.writes) == ([], [], 0) + + +def test_a_plain_http_host_for_local_development_uploads_over_plain_http(): + posthog = FakePostHog( + storage_url="http://localhost:8333/team/1", + posthog_host="http://localhost:8000", + ) + emails = Emails( + SECRET_KEY, + host="http://localhost:8000", + session=session_on(posthog, "http://"), + ) + + send_code( + emails, html=HTML, attachments=[{"filename": "invite.ics", "content": INVITE}] + ) + + [email] = posthog.delivered + assert email.attachments == (DeliveredFile("invite.ics", INVITE),) @parameterized.expand( [ - ("42_ascii_bytes_take_75", "a" * 42, 29_182_818), - ("43_ascii_bytes_take_150", "a" * 43, 29_182_764), - ("21_two_byte_characters_take_75", "é" * 21, 29_182_818), - ("22_two_byte_characters_take_150", "é" * 22, 29_182_764), + ("42_ascii_bytes_take_75", "a" * 42, 29_182_815), + ("43_ascii_bytes_take_150", "a" * 43, 29_182_758), + ("21_two_byte_characters_take_75", "é" * 21, 29_182_815), + ("22_two_byte_characters_take_150", "é" * 22, 29_182_758), ] ) def test_a_subject_takes_75_bytes_of_the_limit_per_42_utf8_bytes( @@ -2225,7 +2235,10 @@ async def test_async_upload_carries_only_the_signed_url_and_headers_whatever_the result = await send_code(emails, html=HTML, attachments=INVITE_AND_LOGO) - assert posthog.bucket == {INVITE_SHA256: INVITE, LOGO_SHA256: LOGO} + assert (posthog.stored_files, posthog.stored_content_types) == ( + sorted([INVITE, LOGO]), + [None, None], + ) [email] = posthog.delivered assert result.id == email.message_id assert email.attachments == DELIVERED_INVITE_AND_LOGO @@ -2244,7 +2257,7 @@ async def test_async_refused_upload_sends_no_email(): "attachments__0", 403, ) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) SECRET_CONTENT = b"secret-file-content" @@ -2340,4 +2353,4 @@ def test_rejects_an_attachment_it_cannot_send_without_repeating_its_content( ) assert "secret-file-content" not in str(error) assert "115, 101" not in str(error) - assert (posthog.delivered, posthog.bucket) == ([], {}) + assert (posthog.delivered, posthog.stored_files) == ([], []) From 29b1d4ade086abd85dbc1637096797fc6f9eb345 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:28:05 +0000 Subject: [PATCH 31/39] fix(emails): name the caller's attachment in upload refusals and read storage answers whole An upload-route refusal named the wire field (files, files__1__size), which the caller never passed. It now names attachments, or the first attachment holding that file. A session with stream=True left a storage answer unread until after the upload, so a cut-off answer lost its attachment; the storage request now always reads the answer at once. The cc and bcc tests now assert the recipients of the delivered email, and the template-content tests are back to main's shape. Refs PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 19 ++++- posthog/test/test_emails.py | 145 ++++++++++++++++++++++++++++++++---- 2 files changed, 146 insertions(+), 18 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index e830cfe23..520fdb003 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -98,7 +98,8 @@ URL that isn't https. Fix the file storage's URL. Nothing was uploaded. - ``upload_quota_exceeded`` (429): the project uploaded its daily limit of file bytes. Files it already sent don't count. -- ``field_not_accepted``: the project can't send files yet. +- ``field_not_accepted`` with ``field`` ``attachments``: the project can't send + files yet. - ``not_found`` (404) with ``field`` ``attachments``: this PostHog version doesn't take files yet, or sending email from code is off for the project. ``message`` carries PostHog's answer, and no email was sent. @@ -950,7 +951,8 @@ def parse( if status == 404: raise _upload_route_not_found(fields, retry_after_header) if not 200 <= status < 300: - raise _response_error(fields, status, retry_after_header) + attr = _attachments_field(fields.get("attr"), files) + raise _response_error({**fields, "attr": attr}, status, retry_after_header) slots = fields.get("files") if not isinstance(slots, list) or len(slots) != len(files): raise _no_upload_slots(status) @@ -966,6 +968,15 @@ def parse( return parse +def _attachments_field(attr: Any, files: Sequence[_File]) -> Any: + if not isinstance(attr, str) or not attr.startswith("files"): + return attr + index = attr.split("__")[1] if attr.startswith("files__") else "" + if index.isdigit() and int(index) < len(files): + return f"attachments__{files[int(index)].first_attachment_index}" + return "attachments" + + def _pending_upload(slot: Any, file: _File, status: int) -> Optional[_Upload]: if not isinstance(slot, dict) or slot.get("sha256") != file.sha256: raise _no_upload_slots(status) @@ -1410,7 +1421,9 @@ def _storage_request(self, upload: _Upload) -> Callable[[], requests.Response]: request = requests.Request( upload.method, upload.url, data=upload.file.content, headers=upload.headers ).prepare() - transport = session.merge_environment_settings(upload.url, {}, None, None, None) + transport = session.merge_environment_settings( + upload.url, {}, False, None, None + ) def send() -> requests.Response: try: diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index aa9281c7f..0265d6c5c 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -661,8 +661,6 @@ def send_code(emails: Emails, **overrides: Any): "text": "Reset it here.", "from_": "Acme ", "reply_to": "help@example.com", - "cc": "manager@example.com", - "bcc": ["audit@example.com", "archive@example.com"], "capture_engagement": False, "distinct_id": "user_123", }, @@ -671,8 +669,6 @@ def send_code(emails: Emails, **overrides: Any): "text": "Reset it here.", "from": "Acme ", "reply_to": ["help@example.com"], - "cc": ["manager@example.com"], - "bcc": ["audit@example.com", "archive@example.com"], "capture_engagement": False, "distinct_id": "user_123", }, @@ -683,8 +679,8 @@ def send_code(emails: Emails, **overrides: Any): {"html": HTML, "reply_to": ["a@example.com", "b@example.com"]}, ), ( - "empty_lists_left_out", - {"html": HTML, "reply_to": [], "cc": [], "bcc": [], "attachments": []}, + "empty_reply_to_left_out", + {"html": HTML, "reply_to": []}, {"html": HTML}, ), ( @@ -694,9 +690,6 @@ def send_code(emails: Emails, **overrides: Any): "text": None, "from_": None, "reply_to": None, - "cc": None, - "bcc": None, - "attachments": None, "capture_engagement": None, "distinct_id": None, }, @@ -741,8 +734,6 @@ def handler(request: httpx.Request) -> httpx.Response: text="Reset it here.", from_="Acme ", reply_to="help@example.com", - cc="manager@example.com", - bcc=["audit@example.com"], capture_engagement=False, distinct_id="user_123", idempotency_key="reset:user_123", @@ -763,8 +754,6 @@ def handler(request: httpx.Request) -> httpx.Response: "text": "Reset it here.", "from": "Acme ", "reply_to": ["help@example.com"], - "cc": ["manager@example.com"], - "bcc": ["audit@example.com"], "capture_engagement": False, "distinct_id": "user_123", } @@ -1360,7 +1349,6 @@ def async_emails_on( INVITE_SHA256 = "215efae54e34ac5657da47b49c8d1bd8ea12649ee1efa838c9a47d7233f18afb" LOGO = b"\x89PNG\r\n\x1a\n" -LOGO_SHA256 = "4c4b6a3be1314ab86138bef4314dde022e600960d8689a2c8f8631802d20dab6" INVITE_FILE = {"filename": "invite.ics", "content": INVITE} INVITE_AND_LOGO = [ { @@ -1405,6 +1393,59 @@ def test_delivers_cc_and_bcc_beside_the_files(): assert email.attachments == (DeliveredFile("invite.ics", INVITE),) +@parameterized.expand( + [ + ( + "one_address_each", + {"cc": "manager@example.com", "bcc": "audit@example.com"}, + ("manager@example.com",), + ("audit@example.com",), + ), + ( + "lists", + { + "cc": ["manager@example.com", "finance@example.com"], + "bcc": ["audit@example.com"], + }, + ("manager@example.com", "finance@example.com"), + ("audit@example.com",), + ), + ("empty_lists", {"cc": [], "bcc": [], "attachments": []}, (), ()), + ("none", {"cc": None, "bcc": None, "attachments": None}, (), ()), + ] +) +def test_delivers_cc_and_bcc_as_given(_name, overrides, cc, bcc): + posthog = FakePostHog() + + send_code(emails_on(posthog), html=HTML, **overrides) + + [email] = posthog.delivered + assert (email.to, email.cc, email.bcc, email.attachments) == ( + "user@example.com", + cc, + bcc, + (), + ) + + +@pytest.mark.asyncio +async def test_async_delivers_cc_and_bcc(): + posthog = FakePostHog() + + await send_code( + async_emails_on(posthog), + html=HTML, + cc="manager@example.com", + bcc=["audit@example.com", "archive@example.com"], + ) + + [email] = posthog.delivered + assert (email.cc, email.bcc) == ( + ("manager@example.com",), + ("audit@example.com", "archive@example.com"), + ) + + def test_stores_only_the_files_posthog_lacks_under_their_own_hash(): posthog = FakePostHog() posthog.already_has(LOGO) @@ -1744,12 +1785,44 @@ def test_a_project_that_cannot_send_files_uploads_and_sends_nothing(): error = raised.value assert (error.code, error.field, error.why) == ( "field_not_accepted", - "files", + "attachments", "Sending files isn't available to this project yet.", ) assert (posthog.delivered, posthog.stored_files) == ([], []) +def test_an_upload_refusal_naming_a_file_names_the_first_attachment_with_those_bytes(): + posthog = FakePostHog() + posthog.inject( + "uploads", + answers( + api_error( + 400, + "message_too_large", + "This file is too large to send.", + attr="files__1__size", + ) + ), + ) + + with pytest.raises(EmailError) as raised: + send_code( + emails_on(posthog), + html=HTML, + attachments=[ + INVITE_FILE, + {"filename": "invite-copy.ics", "content": INVITE}, + {"filename": "logo.png", "content": LOGO}, + ], + ) + + assert (raised.value.code, raised.value.field) == ( + "message_too_large", + "attachments__2", + ) + assert (posthog.delivered, posthog.stored_files) == ([], []) + + @parameterized.expand( [ ("throttled", 429, "SlowDown", "rate_limit_exceeded"), @@ -2186,6 +2259,48 @@ def log_message(self, *args: Any) -> None: pass +class PostHogWithAStorageThatHangsUp(PostHogWithARedirectingStorage): + def do_PUT(self) -> None: + self.hits.append(f"PUT {self.path}") + self.rfile.read(int(self.headers["Content-Length"])) + self.send_response(500) + self.send_header("Content-Length", "100") + self.end_headers() + self.wfile.write(b"") + self.close_connection = True + + +def test_a_storage_answer_cut_short_names_the_attachment_even_on_a_streaming_session(): + server = ThreadingHTTPServer(("127.0.0.1", 0), PostHogWithAStorageThatHangsUp) + PostHogWithAStorageThatHangsUp.hits = [] + threading.Thread(target=server.serve_forever, daemon=True).start() + session = requests.Session() + session.stream = True + session.trust_env = False + emails = Emails( + SECRET_KEY, + host=f"http://127.0.0.1:{server.server_address[1]}", + max_retries=0, + session=session, + ) + + try: + with pytest.raises(EmailError) as raised: + send_code(emails, html=HTML, attachments=[INVITE_FILE]) + finally: + server.shutdown() + server.server_close() + + assert (raised.value.code, raised.value.field) == ( + "network_error", + "attachments__0", + ) + assert PostHogWithAStorageThatHangsUp.hits == [ + "POST /api/projects/@current/messaging/attachments/uploads/", + f"PUT /bucket/{INVITE_SHA256}", + ] + + @pytest.mark.asyncio async def test_async_upload_never_follows_a_storage_redirect_even_when_the_client_does(): server = ThreadingHTTPServer(("127.0.0.1", 0), PostHogWithARedirectingStorage) From ae0b1ff96bdb7c7108bf9994d87217a9b1b64ec0 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:37:59 +0000 Subject: [PATCH 32/39] test(emails): serve the test's TLS storage at TLS 1.2 or later CodeQL flags the local HTTPS storage server's SSLContext for allowing TLS 1.0 and 1.1. Pin the minimum version. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 1 + 1 file changed, 1 insertion(+) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 0265d6c5c..04f60254c 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1671,6 +1671,7 @@ def posthog_with_tls_storage(tmp_path): posthog = FakePostHog() server = ThreadingHTTPServer(("127.0.0.1", 0), storage_handler(posthog)) context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 context.load_cert_chain(certificate, key) server.socket = context.wrap_socket(server.socket, server_side=True) posthog.storage_url = f"https://127.0.0.1:{server.server_address[1]}/bucket" From 0098de8a22b6377ea8055f00a50f1064cb48cc29 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:10:06 +0000 Subject: [PATCH 33/39] test(emails): assert outcomes, not request traces, in the real-socket upload tests The redirect and cut-short storage tests compared the server's request list, which held the storage object path. They now assert the caller's error, that no email was sent, and that the redirect was never followed. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 04f60254c..04e7fe30f 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -2220,10 +2220,10 @@ def test_a_subject_takes_75_bytes_of_the_limit_per_42_utf8_bytes( class PostHogWithARedirectingStorage(BaseHTTPRequestHandler): - hits: list[str] = [] + emails_sent = 0 + redirect_followed = False def do_POST(self) -> None: - self.hits.append(f"POST {self.path}") self.rfile.read(int(self.headers["Content-Length"])) if self.path.endswith("/attachments/uploads/"): slot = { @@ -2236,10 +2236,10 @@ def do_POST(self) -> None: } self.answer(200, json.dumps({"files": [slot]}).encode()) else: + type(self).emails_sent += 1 self.answer(202, json.dumps({"message_id": "msg_1"}).encode()) def do_PUT(self) -> None: - self.hits.append(f"PUT {self.path}") self.rfile.read(int(self.headers["Content-Length"])) self.send_response(302) self.send_header("Location", "/login") @@ -2247,7 +2247,7 @@ def do_PUT(self) -> None: self.end_headers() def do_GET(self) -> None: - self.hits.append(f"GET {self.path}") + type(self).redirect_followed = True self.answer(200, b"Sign in") def answer(self, status: int, body: bytes) -> None: @@ -2262,7 +2262,6 @@ def log_message(self, *args: Any) -> None: class PostHogWithAStorageThatHangsUp(PostHogWithARedirectingStorage): def do_PUT(self) -> None: - self.hits.append(f"PUT {self.path}") self.rfile.read(int(self.headers["Content-Length"])) self.send_response(500) self.send_header("Content-Length", "100") @@ -2273,7 +2272,7 @@ def do_PUT(self) -> None: def test_a_storage_answer_cut_short_names_the_attachment_even_on_a_streaming_session(): server = ThreadingHTTPServer(("127.0.0.1", 0), PostHogWithAStorageThatHangsUp) - PostHogWithAStorageThatHangsUp.hits = [] + PostHogWithAStorageThatHangsUp.emails_sent = 0 threading.Thread(target=server.serve_forever, daemon=True).start() session = requests.Session() session.stream = True @@ -2296,16 +2295,14 @@ def test_a_storage_answer_cut_short_names_the_attachment_even_on_a_streaming_ses "network_error", "attachments__0", ) - assert PostHogWithAStorageThatHangsUp.hits == [ - "POST /api/projects/@current/messaging/attachments/uploads/", - f"PUT /bucket/{INVITE_SHA256}", - ] + assert PostHogWithAStorageThatHangsUp.emails_sent == 0 @pytest.mark.asyncio async def test_async_upload_never_follows_a_storage_redirect_even_when_the_client_does(): server = ThreadingHTTPServer(("127.0.0.1", 0), PostHogWithARedirectingStorage) - PostHogWithARedirectingStorage.hits = [] + PostHogWithARedirectingStorage.emails_sent = 0 + PostHogWithARedirectingStorage.redirect_followed = False threading.Thread(target=server.serve_forever, daemon=True).start() emails = AsyncEmails( SECRET_KEY, @@ -2331,10 +2328,8 @@ async def test_async_upload_never_follows_a_storage_redirect_even_when_the_clien "upload_failed", "attachments__0", ) - assert PostHogWithARedirectingStorage.hits == [ - "POST /api/projects/@current/messaging/attachments/uploads/", - f"PUT /bucket/{INVITE_SHA256}", - ] + assert PostHogWithARedirectingStorage.emails_sent == 0 + assert not PostHogWithARedirectingStorage.redirect_followed @pytest.mark.asyncio From 3db6c986ce60d3a59f3ad1feded930c6eb4c7e20 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:27:16 +0000 Subject: [PATCH 34/39] test(emails): run the field refusal and size tests against the PostHog fake The cc refusal test now sends to a PostHog that doesn't take cc and asserts the caller's error and that nothing was delivered, instead of counting requests on a canned transport. The two oversized-email tests assert that nothing was stored or delivered. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 04e7fe30f..1f2e9c021 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -850,24 +850,14 @@ def test_rejects_a_code_send_that_mixes_or_misses_fields(_name, overrides, error def test_a_refused_send_names_the_field_from_attr(): - refused = { - "type": "validation_error", - "code": "invalid_input", - "detail": "PostHog doesn't take this field, so it doesn't send the email without it.", - "attr": "cc", - } - emails, adapter = emails_with([(400, refused, {}), accepted()]) + posthog = FakePostHog() + posthog.send_fields = SEND_FIELDS - {"cc"} with pytest.raises(EmailError) as raised: - send_code(emails, html=HTML, cc="manager@example.com") + send_code(emails_on(posthog), html=HTML, cc="manager@example.com") - error = raised.value - assert (error.name, error.code, error.field) == ( - "validation_error", - "invalid_input", - "cc", - ) - assert len(adapter.requests) == 1 + assert (raised.value.code, raised.value.field) == ("invalid_input", "cc") + assert posthog.delivered == [] POSTHOG_HOST = "https://us.posthog.com" @@ -1055,6 +1045,7 @@ def __init__( self.writes = 0 self.delivered: list[DeliveredEmail] = [] self.project_can_send_files = True + self.send_fields = SEND_FIELDS self._sent_by_idempotency_key: dict[str, str] = {} self._faults: dict[str, list[Fault]] = { "uploads": [], @@ -1213,7 +1204,7 @@ def _send(self, incoming: Incoming) -> Answer: payload = json_or_none(incoming.body) if not isinstance(payload, dict): return api_error(400, "invalid_input", "Send a JSON object.") - unknown = sorted(set(payload) - SEND_FIELDS) + unknown = sorted(set(payload) - self.send_fields) if unknown: return api_error( 400, @@ -2044,7 +2035,8 @@ def test_refuses_an_email_over_the_limits_before_any_request(_name, attachments, def test_refuses_an_oversized_email_without_copying_its_files(): - emails, adapter = emails_with([]) + posthog = FakePostHog() + emails = emails_on(posthog) scan = memoryview(bytearray(8 * 1024 * 1024)) tracemalloc.start() @@ -2063,11 +2055,13 @@ def test_refuses_an_oversized_email_without_copying_its_files(): assert raised.value.code == "message_too_large" assert peak < len(scan) - assert adapter.requests == [] + assert posthog.stored_files == [] + assert posthog.delivered == [] def test_refuses_oversized_padded_base64_without_copying_it(): - emails, adapter = emails_with([]) + posthog = FakePostHog() + emails = emails_on(posthog) scan = "AAAA" * 10_000_000 + "AA==" tracemalloc.start() @@ -2084,7 +2078,8 @@ def test_refuses_oversized_padded_base64_without_copying_it(): assert raised.value.code == "message_too_large" assert peak < len(scan) // 4 - assert adapter.requests == [] + assert posthog.stored_files == [] + assert posthog.delivered == [] LARGEST_SCAN = bytes(29_182_815) From 9c8aa031fcf8d07cf99b80be843df7c5277d69d2 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:44:35 +0000 Subject: [PATCH 35/39] test(emails): prove an upload never follows storage to another server A client that follows a 307 or 308 on the PUT sends the file again to the new URL, even over plain HTTP, and reports success. Both upload paths already refuse redirects. New sync and async tests have the storage answer 307 to a plain-HTTP server and assert upload_failed on the attachment without either URL, that the other server received nothing, and that no email was sent. Without the fix, requests and an httpx client with follow_redirects deliver the file there. The docs and changeset say the client never follows a redirect. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-code-attachments.md | 2 +- posthog/emails.py | 5 +- posthog/test/test_emails.py | 77 +++++++++++++++++++- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/.sampo/changesets/emails-code-attachments.md b/.sampo/changesets/emails-code-attachments.md index 021138277..73c58be50 100644 --- a/.sampo/changesets/emails-code-attachments.md +++ b/.sampo/changesets/emails-code-attachments.md @@ -2,4 +2,4 @@ pypi/posthog: minor --- -A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files, with the HTML counted twice when there is no `text`, since PostHog derives the text from it. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage over https, then sends the email with each file's SHA-256. A refused upload raises `EmailError` with the code `upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. +A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files, with the HTML counted twice when there is no `text`, since PostHog derives the text from it. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage over https, never following a redirect, then sends the email with each file's SHA-256. A refused or redirected upload raises `EmailError` with the code `upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. diff --git a/posthog/emails.py b/posthog/emails.py index 520fdb003..77ce7a50b 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -90,8 +90,9 @@ limits above. The client raises these before any request. - ``invalid_attachment``: the client can't send a file, such as one without content. -- ``upload_failed``: PostHog's file storage refused an upload. ``message`` - and ``why`` carry its answer, and ``field`` names the attachment. +- ``upload_failed``: PostHog's file storage refused an upload or answered + with a redirect, which the client never follows. ``message`` and ``why`` + carry its answer, and ``field`` names the attachment. - ``network_error`` with ``field`` naming an attachment: the file storage didn't answer, such as when a firewall blocks its host. - ``api_error`` with ``field`` naming an attachment: PostHog named an upload diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 1f2e9c021..cc18ba85a 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -18,7 +18,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from ipaddress import ip_address from pathlib import Path -from typing import Any, Callable, Mapping, Optional, Union +from typing import Any, Callable, Mapping, Optional, Protocol, Union from unittest import mock from urllib.parse import parse_qsl, urlencode, urlsplit from uuid import uuid4 @@ -1270,8 +1270,21 @@ def lowercase(headers: Any) -> dict[str, str]: return {name.lower(): value for name, value in headers.items()} +class Server(Protocol): + def handle(self, incoming: Incoming) -> Answer: ... + + +class AnotherServer: + def __init__(self) -> None: + self.received: list[bytes] = [] + + def handle(self, incoming: Incoming) -> Answer: + self.received.append(incoming.body) + return Answer(200) + + class FakePostHogAdapter(requests.adapters.BaseAdapter): - def __init__(self, posthog: FakePostHog) -> None: + def __init__(self, posthog: Server) -> None: super().__init__() self.posthog = posthog @@ -1300,7 +1313,7 @@ def close(self) -> None: pass -def httpx_transport(posthog: FakePostHog) -> httpx.MockTransport: +def httpx_transport(posthog: Server) -> httpx.MockTransport: def handle(request: httpx.Request) -> httpx.Response: incoming = Incoming( request.method, str(request.url), lowercase(request.headers), request.read() @@ -2327,6 +2340,64 @@ async def test_async_upload_never_follows_a_storage_redirect_even_when_the_clien assert not PostHogWithARedirectingStorage.redirect_followed +ELSEWHERE = "http://elsewhere.example" +REDIRECTED_ELSEWHERE = Answer(307, headers={"Location": f"{ELSEWHERE}/bucket/copy"}) + + +def error_text(error: EmailError) -> str: + return " ".join(str(part) for part in (error, error.message, error.why, error.fix)) + + +def test_an_upload_never_follows_storage_to_another_server(): + posthog, elsewhere = FakePostHog(), AnotherServer() + posthog.inject("storage", answers(REDIRECTED_ELSEWHERE)) + session = session_on(posthog) + session.mount(ELSEWHERE, FakePostHogAdapter(elsewhere)) + + with pytest.raises(EmailError) as raised: + send_code( + Emails(SECRET_KEY, session=session), html=HTML, attachments=[INVITE_FILE] + ) + + assert (raised.value.code, raised.value.field) == ( + "upload_failed", + "attachments__0", + ) + assert "elsewhere.example" not in error_text(raised.value) + assert "files.example.com" not in error_text(raised.value) + assert elsewhere.received == [] + assert posthog.delivered == [] + + +@pytest.mark.asyncio +async def test_async_upload_never_follows_storage_to_another_server(): + posthog, elsewhere = FakePostHog(), AnotherServer() + posthog.inject("storage", answers(REDIRECTED_ELSEWHERE)) + emails = async_emails_on( + posthog, + follow_redirects=True, + mounts={ELSEWHERE: httpx_transport(elsewhere)}, + ) + + with pytest.raises(EmailError) as raised: + await emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=HTML, + attachments=[INVITE_FILE], + ) + + assert (raised.value.code, raised.value.field) == ( + "upload_failed", + "attachments__0", + ) + assert "elsewhere.example" not in error_text(raised.value) + assert "files.example.com" not in error_text(raised.value) + assert elsewhere.received == [] + assert posthog.delivered == [] + + @pytest.mark.asyncio async def test_async_upload_carries_only_the_signed_url_and_headers_whatever_the_client_holds(): posthog = FakePostHog() From f3f5fa3960c554c5709f4f3deb8e30a10229e7ca Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:20:21 +0000 Subject: [PATCH 36/39] test(emails): name the lost-reply retry test for what it proves The test shows a send retried after a lost reply replays with its idempotency key. It can't observe whether uploads are asked for again, so the name no longer promises that. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index cc18ba85a..2cbe28589 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1921,7 +1921,7 @@ def test_a_project_over_its_daily_upload_bytes_is_told_without_retrying(): assert (posthog.delivered, posthog.stored_files) == ([], []) -def test_a_retried_send_reuses_its_uploads_and_its_idempotency_key(): +def test_a_send_retried_after_a_lost_reply_replays_with_its_idempotency_key(): posthog = FakePostHog() posthog.inject( "send", From b32cc2dd5a60d69f4655846e945740472710fbab Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:36:46 +0000 Subject: [PATCH 37/39] fix(emails): refuse extra Base64 padding before copying the string The size check walked back over any number of trailing "=", so a short Base64 string followed by tens of millions of "=" passed as a few bytes, and b64decode then copied and scanned all of it, blocking the async client's event loop. Standard Base64 has at most two "=", so a third now raises invalid_attachment with the Base64 advice during that walk, before any copy. The extra-padding size case is refused this way now. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- posthog/emails.py | 18 ++++++++++++------ posthog/test/test_emails.py | 37 +++++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/posthog/emails.py b/posthog/emails.py index 77ce7a50b..e598a733d 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -600,6 +600,8 @@ def _content_size(field: str, content: Any) -> int: data_end = len(content) while data_end and content[data_end - 1] == "=": data_end -= 1 + if len(content) - data_end > 2: + raise _not_standard_base64(field) size = data_end * 3 // 4 else: raise _invalid_attachment( @@ -634,12 +636,16 @@ def _file_bytes(field: str, content: Union[bytes, bytearray, memoryview, str]) - try: return base64.b64decode(content, validate=True) except (binascii.Error, ValueError): - raise _invalid_attachment( - "An attachment's content string must be standard Base64 on one line " - "with its padding, as base64.b64encode gives. Pass the file's bytes " - "instead, such as text.encode() for text.", - f"{field}__content", - ) from None + raise _not_standard_base64(field) from None + + +def _not_standard_base64(field: str) -> EmailError: + return _invalid_attachment( + "An attachment's content string must be standard Base64 on one line " + "with its padding, as base64.b64encode gives. Pass the file's bytes " + "instead, such as text.encode() for text.", + f"{field}__content", + ) def _check_message_size(file_sizes: Sequence[int], content: Mapping[str, str]) -> None: diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 2cbe28589..f0dd06163 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -2014,16 +2014,6 @@ def test_refuses_eleven_files_before_reading_any_of_them(): [{"filename": "scan.pdf", "content": bytes(29_182_821)}], "message_too_large", ), - ( - "base64_file_too_large_behind_extra_padding", - [ - { - "filename": "scan.pdf", - "content": "AAAA" * (29_182_821 // 3) + "=" * 12, - } - ], - "message_too_large", - ), ( "files_too_large_together", [{"filename": f"{i}.pdf", "content": bytes(3_000_000)} for i in range(10)], @@ -2095,6 +2085,33 @@ def test_refuses_oversized_padded_base64_without_copying_it(): assert posthog.delivered == [] +def test_refuses_extra_base64_padding_without_copying_it(): + posthog = FakePostHog() + emails = emails_on(posthog) + content = "YWJj" + "=" * 40_000_000 + + tracemalloc.start() + try: + with pytest.raises(EmailError) as raised: + send_code( + emails, + html=HTML, + attachments=[{"filename": "a.txt", "content": content}], + ) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert (raised.value.code, raised.value.field) == ( + "invalid_attachment", + "attachments__0__content", + ) + assert "base64.b64encode" in raised.value.message + assert peak < len(content) // 4 + assert posthog.stored_files == [] + assert posthog.delivered == [] + + LARGEST_SCAN = bytes(29_182_815) From 7de6993edcfe162b78bfc8456e1ff3ef457cc4b6 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:22:17 +0000 Subject: [PATCH 38/39] feat(emails): name a failed attachment upload attachment_upload_failed Go, JS, Ruby and PHP raise attachment_upload_failed for a refused or redirected upload, and the docs list one code for every SDK. The code was upload_failed only in this unreleased change, so it takes the shared name in the error, the tests, the docstrings and the changeset. PostHog/michaelr-context#81 Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/emails-code-attachments.md | 2 +- posthog/emails.py | 16 ++++++++-------- posthog/test/test_emails.py | 14 +++++++------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.sampo/changesets/emails-code-attachments.md b/.sampo/changesets/emails-code-attachments.md index 73c58be50..48bc79ed0 100644 --- a/.sampo/changesets/emails-code-attachments.md +++ b/.sampo/changesets/emails-code-attachments.md @@ -2,4 +2,4 @@ pypi/posthog: minor --- -A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files, with the HTML counted twice when there is no `text`, since PostHog derives the text from it. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage over https, never following a redirect, then sends the email with each file's SHA-256. A refused or redirected upload raises `EmailError` with the code `upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. +A code send with `posthog.emails.send` takes `cc`, `bcc` and `attachments`: at most 10 files and 40 MB per message after Base64 encoding, about 29 MB of files, with the HTML counted twice when there is no `text`, since PostHog derives the text from it. An attachment is a `filename` and its `content`, as bytes or a Base64 string, with an optional `content_type` and `content_id`. The client checks the limits before any request, uploads each file PostHog doesn't hold yet straight to PostHog's file storage over https, never following a redirect, then sends the email with each file's SHA-256. A refused or redirected upload raises `EmailError` with the code `attachment_upload_failed`, and an unreachable storage `network_error` with `field` naming the attachment. Either way no email is sent. diff --git a/posthog/emails.py b/posthog/emails.py index e598a733d..c345aca32 100644 --- a/posthog/emails.py +++ b/posthog/emails.py @@ -90,9 +90,9 @@ limits above. The client raises these before any request. - ``invalid_attachment``: the client can't send a file, such as one without content. -- ``upload_failed``: PostHog's file storage refused an upload or answered - with a redirect, which the client never follows. ``message`` and ``why`` - carry its answer, and ``field`` names the attachment. +- ``attachment_upload_failed``: PostHog's file storage refused an upload or + answered with a redirect, which the client never follows. ``message`` and + ``why`` carry its answer, and ``field`` names the attachment. - ``network_error`` with ``field`` naming an attachment: the file storage didn't answer, such as when a firewall blocks its host. - ``api_error`` with ``field`` naming an attachment: PostHog named an upload @@ -1056,7 +1056,7 @@ def _upload_error( storage_answer = " ".join(filter(None, (f"HTTP {status}", answer.get("Code")))) return EmailError( name=_storage_error_name(status), - code="upload_failed", + code="attachment_upload_failed", message=f"PostHog's file storage refused the upload of attachments[{index}] " f"with {storage_answer}, so no email was sent.", status_code=status, @@ -1098,7 +1098,7 @@ def _is_retryable(error: EmailError) -> bool: # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. if error.status_code is None: return error.code == "network_error" - if error.code == "upload_failed": + if error.code == "attachment_upload_failed": return error.status_code in _STORAGE_RETRY_STATUSES if error.status_code == 409: return error.code in _IN_FLIGHT_CODES @@ -1310,9 +1310,9 @@ def send(self, **kwargs: Any) -> SendEmailResult: not tell whether it did. With attachments, ``code`` is ``too_many_attachments``, ``message_too_large`` or ``invalid_attachment`` before any - request, ``upload_failed`` when the storage refused an upload, - and ``network_error`` with ``field`` naming the attachment - when the storage didn't answer. + request, ``attachment_upload_failed`` when the storage + refused an upload, and ``network_error`` with ``field`` + naming the attachment when the storage didn't answer. TypeError, ValueError: An argument has the wrong type or format. Examples: diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index f0dd06163..8ca06683d 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -1551,7 +1551,7 @@ def test_a_refused_upload_sends_no_email_and_surfaces_the_storage_answer( error = raised.value assert (error.name, error.code, error.status_code, error.field, error.why) == ( "application_error", - "upload_failed", + "attachment_upload_failed", status, "attachments__0", storage_message, @@ -1847,7 +1847,7 @@ def test_a_storage_out_of_retries_names_its_passing_fault( emails_on(posthog, max_retries=0), html=HTML, attachments=[INVITE_FILE] ) - assert (raised.value.name, raised.value.code) == (name, "upload_failed") + assert (raised.value.name, raised.value.code) == (name, "attachment_upload_failed") assert (posthog.delivered, posthog.stored_files) == ([], []) @@ -1979,7 +1979,7 @@ def test_a_failed_upload_names_the_first_attachment_with_those_bytes(): ) assert (raised.value.code, raised.value.field) == ( - "upload_failed", + "attachment_upload_failed", "attachments__1", ) assert posthog.delivered == [] @@ -2350,7 +2350,7 @@ async def test_async_upload_never_follows_a_storage_redirect_even_when_the_clien server.server_close() assert (raised.value.code, raised.value.field) == ( - "upload_failed", + "attachment_upload_failed", "attachments__0", ) assert PostHogWithARedirectingStorage.emails_sent == 0 @@ -2377,7 +2377,7 @@ def test_an_upload_never_follows_storage_to_another_server(): ) assert (raised.value.code, raised.value.field) == ( - "upload_failed", + "attachment_upload_failed", "attachments__0", ) assert "elsewhere.example" not in error_text(raised.value) @@ -2406,7 +2406,7 @@ async def test_async_upload_never_follows_storage_to_another_server(): ) assert (raised.value.code, raised.value.field) == ( - "upload_failed", + "attachment_upload_failed", "attachments__0", ) assert "elsewhere.example" not in error_text(raised.value) @@ -2447,7 +2447,7 @@ async def test_async_refused_upload_sends_no_email(): await send_code(async_emails_on(posthog), html=HTML, attachments=[INVITE_FILE]) assert (raised.value.code, raised.value.field, raised.value.status_code) == ( - "upload_failed", + "attachment_upload_failed", "attachments__0", 403, ) From 174a60bb2c42199f3f4bfcac74779c43968fd793 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:09:06 +0000 Subject: [PATCH 39/39] test(emails): follow a storage 307 to another server over real sockets The two cross-server redirect tests now run against a local PostHog whose storage answers the upload with a 307 to a second local server. They assert the second server receives nothing and no email is sent, instead of injecting the redirect through mocked transports. Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_emails.py | 130 ++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 49 deletions(-) diff --git a/posthog/test/test_emails.py b/posthog/test/test_emails.py index 09b143658..82b8a8dce 100644 --- a/posthog/test/test_emails.py +++ b/posthog/test/test_emails.py @@ -12,13 +12,14 @@ import time import traceback import tracemalloc +from contextlib import contextmanager from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone from decimal import Decimal from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer from ipaddress import ip_address from pathlib import Path -from typing import Any, Callable, Iterator, Mapping, Optional, Protocol, Union +from typing import Any, Callable, Iterator, Mapping, Optional, Union from unittest import mock from urllib.parse import parse_qsl, urlencode, urlsplit from uuid import uuid4 @@ -1371,21 +1372,8 @@ def lowercase(headers: Any) -> dict[str, str]: return {name.lower(): value for name, value in headers.items()} -class Server(Protocol): - def handle(self, incoming: Incoming) -> Answer: ... - - -class AnotherServer: - def __init__(self) -> None: - self.received: list[bytes] = [] - - def handle(self, incoming: Incoming) -> Answer: - self.received.append(incoming.body) - return Answer(200) - - class FakePostHogAdapter(requests.adapters.BaseAdapter): - def __init__(self, posthog: Server) -> None: + def __init__(self, posthog: FakePostHog) -> None: super().__init__() self.posthog = posthog @@ -1414,7 +1402,7 @@ def close(self) -> None: pass -def httpx_transport(posthog: Server) -> httpx.MockTransport: +def httpx_transport(posthog: FakePostHog) -> httpx.MockTransport: def handle(request: httpx.Request) -> httpx.Response: incoming = Incoming( request.method, str(request.url), lowercase(request.headers), request.read() @@ -2458,8 +2446,52 @@ async def test_async_upload_never_follows_a_storage_redirect_even_when_the_clien assert not PostHogWithARedirectingStorage.redirect_followed -ELSEWHERE = "http://elsewhere.example" -REDIRECTED_ELSEWHERE = Answer(307, headers={"Location": f"{ELSEWHERE}/bucket/copy"}) +class ServerElsewhere(BaseHTTPRequestHandler): + requests_received = 0 + + def do_PUT(self) -> None: + type(self).requests_received += 1 + self.rfile.read(int(self.headers.get("Content-Length", 0))) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + do_GET = do_POST = do_PUT + + def log_message(self, *args: Any) -> None: + pass + + +class PostHogWithStorageRedirectingElsewhere(PostHogWithARedirectingStorage): + elsewhere = "" + + def do_PUT(self) -> None: + self.rfile.read(int(self.headers["Content-Length"])) + self.send_response(307) + self.send_header("Location", f"{self.elsewhere}/bucket/copy") + self.send_header("Content-Length", "0") + self.end_headers() + + +@contextmanager +def serving(handler: type[BaseHTTPRequestHandler]) -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +@contextmanager +def posthog_redirecting_storage_elsewhere() -> Iterator[tuple[str, str]]: + ServerElsewhere.requests_received = 0 + PostHogWithStorageRedirectingElsewhere.emails_sent = 0 + with serving(ServerElsewhere) as elsewhere: + PostHogWithStorageRedirectingElsewhere.elsewhere = elsewhere + with serving(PostHogWithStorageRedirectingElsewhere) as posthog: + yield posthog, elsewhere def error_text(error: EmailError) -> str: @@ -2467,53 +2499,53 @@ def error_text(error: EmailError) -> str: def test_an_upload_never_follows_storage_to_another_server(): - posthog, elsewhere = FakePostHog(), AnotherServer() - posthog.inject("storage", answers(REDIRECTED_ELSEWHERE)) - session = session_on(posthog) - session.mount(ELSEWHERE, FakePostHogAdapter(elsewhere)) + session = requests.Session() + session.trust_env = False - with pytest.raises(EmailError) as raised: - send_code( - Emails(SECRET_KEY, session=session), html=HTML, attachments=[INVITE_FILE] - ) + with posthog_redirecting_storage_elsewhere() as (posthog, elsewhere): + with pytest.raises(EmailError) as raised: + send_code( + Emails(SECRET_KEY, host=posthog, max_retries=0, session=session), + html=HTML, + attachments=[INVITE_FILE], + ) assert (raised.value.code, raised.value.field) == ( "attachment_upload_failed", "attachments__0", ) - assert "elsewhere.example" not in error_text(raised.value) - assert "files.example.com" not in error_text(raised.value) - assert elsewhere.received == [] - assert posthog.delivered == [] + assert elsewhere not in error_text(raised.value) + assert "/bucket/" not in error_text(raised.value) + assert ServerElsewhere.requests_received == 0 + assert PostHogWithStorageRedirectingElsewhere.emails_sent == 0 @pytest.mark.asyncio async def test_async_upload_never_follows_storage_to_another_server(): - posthog, elsewhere = FakePostHog(), AnotherServer() - posthog.inject("storage", answers(REDIRECTED_ELSEWHERE)) - emails = async_emails_on( - posthog, - follow_redirects=True, - mounts={ELSEWHERE: httpx_transport(elsewhere)}, - ) - - with pytest.raises(EmailError) as raised: - await emails.send( - to="user@example.com", - template_key="password-reset", - subject="Reset your password", - html=HTML, - attachments=[INVITE_FILE], + with posthog_redirecting_storage_elsewhere() as (posthog, elsewhere): + emails = AsyncEmails( + SECRET_KEY, + host=posthog, + max_retries=0, + http_client=httpx.AsyncClient(follow_redirects=True, trust_env=False), ) + with pytest.raises(EmailError) as raised: + await emails.send( + to="user@example.com", + template_key="password-reset", + subject="Reset your password", + html=HTML, + attachments=[INVITE_FILE], + ) assert (raised.value.code, raised.value.field) == ( "attachment_upload_failed", "attachments__0", ) - assert "elsewhere.example" not in error_text(raised.value) - assert "files.example.com" not in error_text(raised.value) - assert elsewhere.received == [] - assert posthog.delivered == [] + assert elsewhere not in error_text(raised.value) + assert "/bucket/" not in error_text(raised.value) + assert ServerElsewhere.requests_received == 0 + assert PostHogWithStorageRedirectingElsewhere.emails_sent == 0 @pytest.mark.asyncio