From 156067d37bfb02962a82692e075be2b618f5bc97 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:47:51 +0000 Subject: [PATCH 01/31] feat(emails): send transactional email and read its status Adds client.emails.send for a template made in PostHog (template_id) or content your code rendered (template_key, subject, html, text), and client.emails.get for the status of a sent email. Ports posthog-node and posthog-python's emails to Ruby: the same wire body, app host rules, idempotency key, retries and error names. Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-send.md | 5 + lib/posthog/client.rb | 28 ++- lib/posthog/defaults.rb | 5 + lib/posthog/email_error.rb | 57 +++++ lib/posthog/emails.rb | 478 +++++++++++++++++++++++++++++++++++ lib/posthog/transport.rb | 13 - lib/posthog/utils.rb | 18 ++ public_api_snapshot.txt | 57 +++++ spec/posthog/emails_spec.rb | 482 ++++++++++++++++++++++++++++++++++++ 9 files changed, 1129 insertions(+), 14 deletions(-) create mode 100644 .changeset/emails-send.md create mode 100644 lib/posthog/email_error.rb create mode 100644 lib/posthog/emails.rb create mode 100644 spec/posthog/emails_spec.rb diff --git a/.changeset/emails-send.md b/.changeset/emails-send.md new file mode 100644 index 00000000..0119c487 --- /dev/null +++ b/.changeset/emails-send.md @@ -0,0 +1,5 @@ +--- +"posthog-ruby": minor +--- + +Send transactional email with `client.emails.send`, from an email template made in PostHog (`template_id:` with `template_data:`) or from content your code rendered (`template_key:`, `subject:` and `html:` or `text:`). Read what happened to a sent email with `client.emails.get(message_id)`. Sends need `secret_key`, a project secret API key with the `hog_flow:write` scope, and go to the PostHog Cloud app host of the client's `host`; set `app_host` for any other host. A send retries network errors, rate limits and an unavailable service with the same idempotency key, and raises `PostHog::EmailError` when PostHog doesn't queue the email. To send a React email, render it to HTML in your app first. New client options: `app_host`, `email_request_timeout_seconds`, `email_request_max_retries` and `migrated_from`. diff --git a/lib/posthog/client.rb b/lib/posthog/client.rb index 1b4912f9..5292f44d 100644 --- a/lib/posthog/client.rb +++ b/lib/posthog/client.rb @@ -15,6 +15,7 @@ require 'posthog/feature_flag_evaluations' require 'posthog/send_feature_flags_options' require 'posthog/exception_capture' +require 'posthog/emails' require 'posthog/internal/context' module PostHog @@ -127,6 +128,14 @@ def _decrement_instance_count(api_key) # @option opts [Boolean] :compress_request Set to +false+ to disable gzip compression for batch uploads. # @option opts [Object] :flag_definition_cache_provider An object implementing the {FlagDefinitionCacheProvider} # interface for distributed flag definition caching. + # @option opts [String, nil] :app_host Your PostHog app host, such as `https://posthog.example.com`, for {#emails}. + # Needed only when +:host+ is not PostHog Cloud, such as a reverse proxy, which must not receive the secret key. + # @option opts [Numeric] :email_request_timeout_seconds How long each {#emails} request may take, in seconds. + # Defaults to 15. + # @option opts [Integer] :email_request_max_retries How many times {#emails} retries a request after a network + # error, a rate limit or an unavailable service. Defaults to 2. Set to 0 when a job runner already retries. + # @option opts [String, nil] :migrated_from The email provider you moved from, such as `resend`. {#emails} sends + # it with every request. # @option opts [Boolean] :is_server +true+ to stamp captured events with `$is_server => true` so PostHog # attributes them as server-side. Defaults to +true+. Set to +false+ when running posthog-ruby as a # client/CLI so the device OS is attributed normally and `$is_server` is omitted. @@ -227,12 +236,28 @@ def initialize(opts = {}) ) end + @emails = Emails.new( + secret_key: secret_key, + host: opts[:host], + app_host: opts[:app_host], + timeout: opts.fetch(:email_request_timeout_seconds, Defaults::Emails::REQUEST_TIMEOUT_SECONDS), + max_retries: opts.fetch(:email_request_max_retries, Defaults::Emails::REQUEST_MAX_RETRIES), + migrated_from: opts[:migrated_from], + user_agent: @headers['User-Agent'], + disabled: @disabled + ) @before_send = opts[:before_send] @is_server = opts.fetch(:is_server, true) != false @release_id = normalize_string_option(ENV.fetch(RELEASE_ID_ENV_VAR, nil), blank_as_nil: true) @deprecation_emitted_for = Concurrent::Set.new end + # Sends transactional email from email templates made in PostHog or from content your code rendered, and reads + # the status of each email sent. Needs +:secret_key+, a project secret API key with the `hog_flow:write` scope. + # + # @return [PostHog::Emails] + attr_reader :emails + # When sync_mode is enabled, blocks until in-flight requests are complete # and returns true. Timeout parameter has no effect. # @@ -977,7 +1002,8 @@ def shutdown(timeout: nil) @distinct_id_has_sent_flag_calls_mutex.synchronize do @distinct_id_has_sent_flag_calls.clear end - flushed + emails_settled = @emails.close(timeout: deadline && [deadline - monotonic_time, 0].max) + flushed &&= emails_settled ensure @shutdown_mutex.synchronize do @shutdown_result = flushed diff --git a/lib/posthog/defaults.rb b/lib/posthog/defaults.rb index d152ec46..af57f356 100644 --- a/lib/posthog/defaults.rb +++ b/lib/posthog/defaults.rb @@ -26,6 +26,11 @@ module FeatureFlags FLAG_REQUEST_MAX_RETRIES = 1 end + module Emails + REQUEST_TIMEOUT_SECONDS = 15 + REQUEST_MAX_RETRIES = 2 + end + module Queue MAX_SIZE = 10_000 end diff --git a/lib/posthog/email_error.rb b/lib/posthog/email_error.rb new file mode 100644 index 00000000..0133b7b1 --- /dev/null +++ b/lib/posthog/email_error.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module PostHog + # PostHog did not queue the email, or the client could not tell whether it did. + # + # Branch on {#code}, PostHog's stable error code, or on {#name}, a coarse category of the failure. + # The message never contains the secret key or the request body. + class EmailError < StandardError + # @return [String] A coarse category of the failure, such as `validation_error`, `rate_limit_exceeded` + # or `service_unavailable`. + attr_reader :name + # @return [String] PostHog's error code, such as `quota_limited`, or an SDK code such as `network_error`. + attr_reader :code + # @return [Integer, nil] The HTTP status, when PostHog answered. + attr_reader :status_code + # @return [String, nil] Why PostHog refused the email. + attr_reader :why + # @return [String, nil] How to fix the request. + attr_reader :fix + # @return [String, nil] A link to the docs for this error. + attr_reader :docs_url + # @return [String, nil] The request field the error is about, when PostHog named one. + attr_reader :field + # @return [Float, nil] How long PostHog asked the client to wait before it sends again, in seconds. + attr_reader :retry_after + + # @api private + def initialize(code:, message:, status_code: nil, why: nil, fix: nil, docs_url: nil, field: nil, retry_after: nil) + super(message) + @name = self.class.name_for(status_code, code) + @code = code + @status_code = status_code + @why = why + @fix = fix + @docs_url = docs_url + @field = field + @retry_after = retry_after + end + + # @api private + def self.name_for(status_code, code) + return 'missing_api_key' if code == 'missing_secret_key' || status_code == 401 + return code == 'network_error' ? 'service_unavailable' : 'application_error' if status_code.nil? + + case status_code + when 400 then 'validation_error' + when 403 then 'invalid_permission' + when 404 then 'not_found' + when 409 then code == 'template_outdated' ? 'validation_error' : 'concurrent_idempotent_requests' + when 422 then code == 'idempotency_key_reused' ? 'invalid_idempotent_request' : 'validation_error' + when 429 then code == 'quota_limited' ? 'monthly_quota_exceeded' : 'rate_limit_exceeded' + when 503 then 'service_unavailable' + else 'application_error' + end + end + end +end diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb new file mode 100644 index 00000000..f9faf438 --- /dev/null +++ b/lib/posthog/emails.rb @@ -0,0 +1,478 @@ +# frozen_string_literal: true + +require 'json' +require 'net/http' +require 'openssl' +require 'securerandom' +require 'time' + +require 'posthog/email_error' +require 'posthog/logging' +require 'posthog/utils' + +module PostHog + # Sends transactional email, from an email template made in PostHog or from content your code rendered, + # and reads the status of each email sent. Reach it through {Client#emails}. + # + # A send needs the client's `secret_key`, a project secret API key (`phs_...`) with the `hog_flow:write` scope. + # Sends go to the PostHog app host: a client on PostHog Cloud uses its region's app host, and any other `host` + # needs `app_host`, because a reverse proxy for ingestion must not receive the secret key. + # + # Each send retries network errors, rate limits and an unavailable service, waiting about 1 and then 2 seconds, + # or the server's `Retry-After` when that is longer. Every attempt sends the same idempotency key, so a retry + # never sends twice. + # + # @example A template made in PostHog + # posthog.emails.send( + # to: 'user@example.com', + # template_id: '0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b', + # template_data: { name: 'Ada' }, + # distinct_id: 'user_123', + # idempotency_key: 'welcome:user_123' + # ) + # + # @example Content your code rendered, such as React Email output + # posthog.emails.send( + # to: 'user@example.com', + # template_key: 'password-reset', + # subject: 'Reset your password', + # html: rendered_html, + # distinct_id: 'user_123' + # ) + class Emails + include PostHog::Logging + include PostHog::Utils + + # The queued email. A result means PostHog queued the email, not that it was delivered. + # + # @!attribute [r] id + # @return [String] The queued email's message ID. + # @!attribute [r] replayed + # @return [Boolean] Whether PostHog had already accepted this idempotency key and returned the first result. + # @!attribute [r] warnings + # @return [Array] + SendResult = Struct.new(:id, :replayed, :warnings, keyword_init: true) + + # A warning PostHog returned with a queued email. + Warning = Struct.new(:code, :message, :fix, keyword_init: true) + + # What happened to an email that {Emails#send} queued. + # + # @!attribute [r] status + # @return [String] `queued`, `sent`, `failed`, `delivered`, `bounced` or `complained`. Later SDK versions + # can see values this list does not have yet. + # @!attribute [r] reason + # @return [String, nil] Why a failed email was not sent, such as `recipient_suppressed`, or the bounce type + # of a bounced email: `hard`, `soft` or `unknown`. + Status = Struct.new(:id, :status, :created_at, :status_at, :reason, keyword_init: true) + + SEND_PATH = '/api/projects/@current/messaging/send/email/' + STATUS_PATH = '/api/projects/@current/messaging/emails/%s/' + US_APP_HOST = 'https://us.posthog.com' + EU_APP_HOST = 'https://eu.posthog.com' + CLOUD_APP_HOSTS = { + 'https://us.i.posthog.com' => US_APP_HOST, + 'https://us.posthog.com' => US_APP_HOST, + 'https://app.posthog.com' => US_APP_HOST, + 'https://eu.i.posthog.com' => EU_APP_HOST, + 'https://eu.posthog.com' => EU_APP_HOST + }.freeze + MAX_RETRY_DELAY_SECONDS = 30.0 + UUID_PATTERN = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/ + IDEMPOTENCY_KEY_PATTERN = /\A[\x21-\x7e]{1,255}\z/ + TEMPLATE_KEY_PATTERN = /\A[a-zA-Z0-9_-]{1,400}\z/ + HEADER_TOKEN_PATTERN = /\A[\x21-\x7e]+\z/ + SEND_KEY_NEEDED = 'Sending email needs secret_key, a project secret API key with the hog_flow:write scope.' + READ_KEY_NEEDED = 'Reading an email status needs secret_key, a project secret API key with the hog_flow:read ' \ + 'or hog_flow:write scope.' + TEMPLATE_FORM_WHY = 'A template made in PostHog sets its own sender, subject and content, and follows the ' \ + "project's engagement setting." + CODE_FORM_WHY = 'A send with template_key takes its content from your code.' + TEMPLATE_FIELDS = %i[template_data].freeze + CODE_FIELDS = %i[subject html text from reply_to capture_engagement].freeze + NETWORK_ERRORS = [ + Timeout::Error, SocketError, SystemCallError, IOError, EOFError, OpenSSL::SSL::SSLError, + Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError + ].freeze + private_constant :SEND_PATH, :STATUS_PATH, :US_APP_HOST, :EU_APP_HOST, :CLOUD_APP_HOSTS, + :MAX_RETRY_DELAY_SECONDS, :UUID_PATTERN, :IDEMPOTENCY_KEY_PATTERN, :TEMPLATE_KEY_PATTERN, + :HEADER_TOKEN_PATTERN, :SEND_KEY_NEEDED, :READ_KEY_NEEDED, :TEMPLATE_FORM_WHY, + :CODE_FORM_WHY, :TEMPLATE_FIELDS, :CODE_FIELDS, :NETWORK_ERRORS + + # @api private + def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_from:, user_agent:, disabled:) + check_options(timeout, max_retries, migrated_from) + @secret_key = secret_key.is_a?(String) ? secret_key.strip : nil + @app_host = resolve_app_host(host, app_host) + @timeout = timeout + @max_retries = max_retries + @migrated_from = migrated_from + @user_agent = user_agent + @disabled = disabled + @closed = false + @in_flight = 0 + @mutex = Mutex.new + @idle = ConditionVariable.new + end + + # Queues one email, from a template made in PostHog or from content your code rendered. + # + # Pass exactly one form. The template form names a template made in PostHog with `template_id`. The code form + # names the email with `template_key` and passes `subject` and `html`, `text` or both. To send a React email, + # render it to HTML in your app first. + # + # @param to [String] One recipient email address. + # @param template_id [String] Template form: the UUID of a template made transactional in PostHog. + # @param template_data [Hash] Template form: values for the template's `{{ variables.* }}` placeholders. + # @param template_key [String] Code form: names the email, such as `password-reset`, so its metrics, logs and + # status group under it in PostHog. Letters, digits, hyphens and underscores, at most 400. + # @param subject [String] Code form: the subject line. + # @param html [String] Code form: the HTML body. + # @param text [String] Code form: the plain-text body. Without it, PostHog derives it from `html`. + # @param from [String] Code form: a verified sender, such as `Acme `. Without it, the + # project's default sender. + # @param reply_to [String, Array] Code form: one address or a list of addresses. + # @param capture_engagement [Boolean] Code form: whether PostHog captures the email's opens and clicks as events + # of the recipient. Defaults to +true+. A project whose admin turned engagement capture off captures nothing. + # @param distinct_id [String] The recipient's distinct ID in your product. The email's events go to this + # person. Without it, they go to a person whose distinct ID is the email address. + # @param idempotency_key [String] 1 to 255 visible ASCII characters. A send with a key PostHog has already + # accepted queues nothing and returns the first send's result. Without it, the client makes a UUID that covers + # its own retries. Pass a key derived from what the email is for, such as `welcome:user_123`, when a job runner + # retries the send. + # @param max_retries [Integer] Overrides the client's `email_request_max_retries` for this send. +0+ turns + # retries off. + # @return [SendResult] The queued email. + # @raise [EmailError] PostHog did not queue the email, or the client could not tell whether it did. + # @raise [ArgumentError] An argument has the wrong type or format, or the call mixes the two forms. + def send( + to:, template_id: nil, template_data: nil, template_key: nil, subject: nil, html: nil, text: nil, from: nil, + reply_to: nil, capture_engagement: nil, distinct_id: nil, idempotency_key: nil, max_retries: nil + ) + fields = { + template_data: template_data, subject: subject, html: html, text: text, from: from, reply_to: reply_to, + capture_engagement: capture_engagement + }.compact + check_recipient(to, distinct_id, idempotency_key, max_retries) + payload = content(template_id, template_key, fields).merge(to: to) + payload[:distinct_id] = distinct_id unless distinct_id.nil? + body = JSON.generate(payload) + headers = { 'Content-Type' => 'application/json', 'Idempotency-Key' => idempotency_key || SecureRandom.uuid } + + result = call('no email was sent', SEND_KEY_NEEDED) do |secret_key| + uri = URI(@app_host + SEND_PATH) + with_retries(max_retries || @max_retries) do + parse_send_reply(request(Net::HTTP::Post.new(uri), uri, secret_key, headers, body)) + end + end + result.warnings.each { |warning| logger.warn("PostHog email warning #{warning.code}: #{warning.message}") } + result + end + + # Reads what happened to an email that {#send} queued, for 30 days after the send. + # + # @param message_id [String] The {SendResult#id} that {#send} returned. + # @return [Status] The email's latest status. + # @raise [EmailError] PostHog did not answer with a status. {EmailError#code} is `not_found` for an unknown ID, + # an email older than 30 days, or for a few seconds after the send. + # @raise [ArgumentError] +message_id+ is not a message ID. + def get(message_id) + raise ArgumentError, 'message_id must be the message ID that send returned.' unless uuid?(message_id) + + call('no email status was read', READ_KEY_NEEDED) do |secret_key| + uri = URI(@app_host + format(STATUS_PATH, message_id: message_id.downcase)) + with_retries(@max_retries) { parse_status_reply(request(Net::HTTP::Get.new(uri), uri, secret_key, {})) } + end + end + + # Refuses new sends and waits for the sends and status reads in flight, retries included, to settle. + # + # @api private + # @param timeout [Numeric, nil] Maximum seconds to wait, or +nil+ to wait until they settle. + # @return [Boolean] +true+ if nothing is in flight any more. + def close(timeout: nil) + deadline = timeout && (monotonic_time + timeout) + @mutex.synchronize do + @closed = true + until @in_flight.zero? + remaining = deadline && (deadline - monotonic_time) + break if remaining && remaining <= 0 + + @idle.wait(@mutex, remaining) + end + @in_flight.zero? + end + end + + private + + def check_options(timeout, max_retries, migrated_from) + raise ArgumentError, 'email_request_timeout_seconds must be a positive number.' unless positive?(timeout) + raise ArgumentError, 'email_request_max_retries must be a nonnegative integer.' unless retry_count?(max_retries) + return if migrated_from.nil? || (migrated_from.is_a?(String) && HEADER_TOKEN_PATTERN.match?(migrated_from)) + + raise ArgumentError, "migrated_from must be visible ASCII characters, such as 'resend'." + end + + def positive?(value) + value.is_a?(Numeric) && value.real? && value.positive? && value.finite? + end + + def retry_count?(value) + value.is_a?(Integer) && value >= 0 + end + + def uuid?(value) + value.is_a?(String) && UUID_PATTERN.match?(value) + end + + def resolve_app_host(host, app_host) + explicit = app_host.is_a?(String) ? app_host.strip.chomp('/') : '' + return CLOUD_APP_HOSTS.fetch(explicit, explicit) unless explicit.empty? + + CLOUD_APP_HOSTS[host.to_s.strip.chomp('/')] + end + + def check_recipient(to, distinct_id, idempotency_key, max_retries) + raise ArgumentError, 'to must be one email address as a nonempty string.' unless nonempty_string?(to) + unless distinct_id.nil? || nonempty_string?(distinct_id) + raise ArgumentError, 'distinct_id must be a nonempty string.' + end + unless idempotency_key.nil? || (idempotency_key.is_a?(String) && IDEMPOTENCY_KEY_PATTERN.match?(idempotency_key)) + raise ArgumentError, 'idempotency_key must be 1 to 255 visible ASCII characters without spaces.' + end + return if max_retries.nil? || retry_count?(max_retries) + + raise ArgumentError, 'max_retries must be a nonnegative integer.' + end + + def nonempty_string?(value) + value.is_a?(String) && !value.strip.empty? + end + + def content(template_id, template_key, fields) + if template_id.nil? == template_key.nil? + raise ArgumentError, 'Send either template_id, or template_key with subject and html or text.' + end + + template_key.nil? ? template_content(template_id, fields) : code_content(template_key, fields) + end + + def template_content(template_id, fields) + refuse_fields(fields, allowed: TEMPLATE_FIELDS, form: :template_id, other_form: :template_key, + why: TEMPLATE_FORM_WHY) + raise ArgumentError, "template_id must be the template's UUID." unless uuid?(template_id) + + template_data = fields.fetch(:template_data, {}) + raise ArgumentError, 'template_data must be a Hash.' unless template_data.is_a?(Hash) + + { template_id: template_id.downcase, message_data: isoify_dates(template_data) } + end + + def code_content(template_key, fields) + refuse_fields(fields, allowed: CODE_FIELDS, form: :template_key, other_form: :template_id, + why: CODE_FORM_WHY) + unless template_key.is_a?(String) && TEMPLATE_KEY_PATTERN.match?(template_key) + raise ArgumentError, + 'template_key can only have letters, digits, hyphens (-) and underscores (_), at most 400.' + end + raise ArgumentError, 'A send with template_key needs a subject.' unless fields[:subject].is_a?(String) + unless fields.key?(:html) || fields.key?(:text) + raise ArgumentError, + 'A send with template_key needs html, text or both.' + end + + %i[html text from].each do |name| + raise ArgumentError, "#{name} must be a string." if fields.key?(name) && !fields[name].is_a?(String) + end + unless [nil, true, false].include?(fields[:capture_engagement]) + raise ArgumentError, 'capture_engagement must be true or false.' + end + + code = fields.slice(:subject, :html, :text, :from).merge(template_key: template_key) + code[:reply_to] = addresses(fields[:reply_to]) if fields.key?(:reply_to) + code[:capture_engagement] = fields.fetch(:capture_engagement, true) + code + end + + def refuse_fields(fields, allowed:, form:, other_form:, why:) + field = fields.keys.find { |name| !allowed.include?(name) } + return if field.nil? + + message = "A send with #{form} doesn't take #{field}, so no email was sent." + raise ArgumentError, "#{message} #{why} Remove #{field}, or send with #{other_form}." + end + + def addresses(reply_to) + list = reply_to.is_a?(String) ? [reply_to] : reply_to + return list if list.is_a?(Array) && list.all? { |address| nonempty_string?(address) } + + raise ArgumentError, 'reply_to must be one address or a list of addresses.' + end + + def call(nothing_done, key_needed) + admit(nothing_done) + begin + unless @app_host + # A custom host is often an ingestion proxy, which must not receive the secret key. + raise EmailError.new( + code: 'missing_app_host', + message: "The client's host is not PostHog Cloud, so the client can't tell where to send email. " \ + 'Set app_host to your PostHog app host on the client.' + ) + end + raise EmailError.new(code: 'missing_secret_key', message: key_needed) if @secret_key.nil? || @secret_key.empty? + + yield @secret_key + ensure + finish + end + end + + def admit(nothing_done) + if @disabled + raise EmailError.new(code: 'client_disabled', + message: "The PostHog client is disabled, so #{nothing_done}.") + end + + @mutex.synchronize do + if @closed + raise EmailError.new(code: 'client_closed', + message: "The PostHog client is shut down, so #{nothing_done}.") + end + + @in_flight += 1 + end + end + + def finish + @mutex.synchronize do + @in_flight -= 1 + @idle.broadcast if @in_flight.zero? + end + end + + def with_retries(max_retries) + attempt = 0 + begin + yield + rescue EmailError => e + delay = retry_delay(e, attempt, max_retries) + raise if delay.nil? + + sleep(delay) + attempt += 1 + retry + end + end + + def retry_delay(error, attempt, max_retries) + return nil if attempt >= max_retries || !retryable?(error) + # Retrying before a longer Retry-After would break the server's rate limit. + return nil if error.retry_after && error.retry_after > MAX_RETRY_DELAY_SECONDS + + backoff = [MAX_RETRY_DELAY_SECONDS, (2**attempt) * rand(0.8..1.2)].min + [backoff, error.retry_after || 0].max + end + + def retryable?(error) + # A send carries an idempotency key, so a retry can't queue a second email. A status read changes nothing. + case error.status_code + when nil then error.code == 'network_error' + when 409 then error.code == 'idempotency_key_in_flight' + when 429 then error.code != 'quota_limited' + else error.status_code == 503 + end + end + + def request(request, uri, secret_key, headers, body = nil) + request['Authorization'] = "Bearer #{secret_key}" + request['Accept'] = 'application/json' + request['User-Agent'] = @user_agent if @user_agent + request['X-PostHog-Migrated-From'] = @migrated_from if @migrated_from + headers.each { |name, value| request[name] = value } + request.body = body if body + + response = Net::HTTP.start( + uri.hostname, uri.port, + use_ssl: uri.scheme == 'https', open_timeout: @timeout, read_timeout: @timeout, write_timeout: @timeout, + max_retries: 0 + ) { |http| http.request(request) } + [response.code.to_i, parse_json(response.body), parse_retry_after(response['Retry-After'])] + rescue *NETWORK_ERRORS + raise EmailError.new(code: 'network_error', message: 'The request to PostHog failed.') + end + + def parse_json(body) + JSON.parse(body.to_s) + rescue JSON::ParserError + nil + end + + def parse_send_reply((status, payload, retry_after)) + fields = payload.is_a?(Hash) ? payload : {} + raise reply_error(fields, status, retry_after) unless status.between?(200, 299) + + id = text(fields['message_id']) + unless id + raise EmailError.new(code: 'api_error', message: 'PostHog answered without a message ID.', + status_code: status) + end + + SendResult.new(id: id, replayed: fields['replayed'] == true, warnings: warnings(fields['warnings'])) + end + + def warnings(entries) + return [] unless entries.is_a?(Array) + + entries.grep(Hash).map do |entry| + Warning.new( + code: text(entry['code']) || 'warning', + message: text(entry['detail']) || text(entry['message']) || '', + fix: text(entry['fix']) + ) + end + end + + def parse_status_reply((status, payload, retry_after)) + fields = payload.is_a?(Hash) ? payload : {} + raise reply_error(fields, status, retry_after) unless status.between?(200, 299) + + record = { + id: text(fields['id']), + status: text(fields['status']), + created_at: timestamp(fields['created_at']), + status_at: timestamp(fields['status_at']) + } + if record.value?(nil) + raise EmailError.new(code: 'api_error', message: 'PostHog answered without an email status.', + status_code: status) + end + + Status.new(**record, reason: text(fields['reason'])) + end + + def timestamp(value) + text(value) && Time.iso8601(value) + rescue ArgumentError + nil + end + + def reply_error(fields, status, retry_after) + EmailError.new( + code: text(fields['code']) || 'api_error', + message: text(fields['detail']) || "PostHog answered HTTP #{status}.", + status_code: status, + why: text(fields['why']), + fix: text(fields['fix']), + docs_url: text(fields['docs_url']), + field: text(fields['attr']), + retry_after: retry_after + ) + end + + def text(value) + value if value.is_a?(String) && !value.empty? + end + end +end diff --git a/lib/posthog/transport.rb b/lib/posthog/transport.rb index 0198a758..c2db0fc9 100644 --- a/lib/posthog/transport.rb +++ b/lib/posthog/transport.rb @@ -151,19 +151,6 @@ def retry_delay_seconds @backoff_policy.next_interval.to_f / 1000 end - def parse_retry_after(value) - return nil if value.nil? || value.empty? - - seconds = Float(value, exception: false) - return seconds if seconds && seconds >= 0 - - parsed_time = Time.httpdate(value) - delay = parsed_time - Time.now - delay.positive? ? delay : nil - rescue ArgumentError - nil - end - # Sends a request for the batch, returns [status_code, body] def send_request(api_key, batch) @last_retry_after = nil diff --git a/lib/posthog/utils.rb b/lib/posthog/utils.rb index 141ee77a..2ae66f1a 100644 --- a/lib/posthog/utils.rb +++ b/lib/posthog/utils.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'time' + require 'posthog/flag_evaluation_errors' module PostHog @@ -135,6 +137,22 @@ def get_by_symbol_or_string_key(hash, key) end end + # public: Seconds to wait from a Retry-After header, given as seconds or as an HTTP date, or nil when the + # header is absent, malformed or in the past + # + def parse_retry_after(value) + return nil if value.nil? || value.empty? + + seconds = Float(value, exception: false) + return seconds if seconds&.finite? && seconds >= 0 + + parsed_time = Time.httpdate(value) + delay = parsed_time - Time.now + delay.positive? ? delay : nil + rescue ArgumentError + nil + end + # public: Current monotonic clock time in seconds, for measuring durations # def monotonic_time diff --git a/public_api_snapshot.txt b/public_api_snapshot.txt index 97e7eb53..16679caa 100644 --- a/public_api_snapshot.txt +++ b/public_api_snapshot.txt @@ -10,6 +10,7 @@ instance_method PostHog::Client#capture(attrs) instance_method PostHog::Client#capture_exception(exception, distinct_id = ..., additional_properties = ..., flags: ..., mechanism: ...) instance_method PostHog::Client#clear() instance_method PostHog::Client#dequeue_last_message() +instance_method PostHog::Client#emails() instance_method PostHog::Client#enabled?() instance_method PostHog::Client#evaluate_flags(distinct_id, groups: ..., person_properties: ..., group_properties: ..., only_evaluate_locally: ..., disable_geoip: ..., flag_keys: ...) instance_method PostHog::Client#feature_flags_loaded?() @@ -39,6 +40,9 @@ constant PostHog::Defaults::BackoffPolicy::MAX_TIMEOUT_MS: Integer constant PostHog::Defaults::BackoffPolicy::MIN_TIMEOUT_MS: Integer constant PostHog::Defaults::BackoffPolicy::MULTIPLIER: Float constant PostHog::Defaults::BackoffPolicy::RANDOMIZATION_FACTOR: Float +module PostHog::Defaults::Emails +constant PostHog::Defaults::Emails::REQUEST_MAX_RETRIES: Integer +constant PostHog::Defaults::Emails::REQUEST_TIMEOUT_SECONDS: Integer module PostHog::Defaults::FeatureFlags constant PostHog::Defaults::FeatureFlags::FLAG_REQUEST_MAX_RETRIES: Integer constant PostHog::Defaults::FeatureFlags::FLAG_REQUEST_TIMEOUT_SECONDS: Integer @@ -59,6 +63,59 @@ constant PostHog::Defaults::Request::PATH: String constant PostHog::Defaults::Request::PORT: Integer constant PostHog::Defaults::Request::RETRIES: Integer constant PostHog::Defaults::Request::SSL: Boolean +class PostHog::EmailError < StandardError +instance_method PostHog::EmailError#code() +instance_method PostHog::EmailError#docs_url() +instance_method PostHog::EmailError#field() +instance_method PostHog::EmailError#fix() +instance_method PostHog::EmailError#name() +instance_method PostHog::EmailError#retry_after() +instance_method PostHog::EmailError#status_code() +instance_method PostHog::EmailError#why() +class PostHog::Emails +instance_method PostHog::Emails#get(message_id) +instance_method PostHog::Emails#send(to:, template_id: ..., template_data: ..., template_key: ..., subject: ..., html: ..., text: ..., from: ..., reply_to: ..., capture_engagement: ..., distinct_id: ..., idempotency_key: ..., max_retries: ...) +class_method PostHog::Emails.logger() +class PostHog::Emails::SendResult < Struct +instance_method PostHog::Emails::SendResult#id() +instance_method PostHog::Emails::SendResult#id=(_) +instance_method PostHog::Emails::SendResult#replayed() +instance_method PostHog::Emails::SendResult#replayed=(_) +instance_method PostHog::Emails::SendResult#warnings() +instance_method PostHog::Emails::SendResult#warnings=(_) +class_method PostHog::Emails::SendResult.[](*arg0) +class_method PostHog::Emails::SendResult.inspect() +class_method PostHog::Emails::SendResult.keyword_init?() +class_method PostHog::Emails::SendResult.members() +class_method PostHog::Emails::SendResult.new(*arg0) +class PostHog::Emails::Status < Struct +instance_method PostHog::Emails::Status#created_at() +instance_method PostHog::Emails::Status#created_at=(_) +instance_method PostHog::Emails::Status#id() +instance_method PostHog::Emails::Status#id=(_) +instance_method PostHog::Emails::Status#reason() +instance_method PostHog::Emails::Status#reason=(_) +instance_method PostHog::Emails::Status#status() +instance_method PostHog::Emails::Status#status=(_) +instance_method PostHog::Emails::Status#status_at() +instance_method PostHog::Emails::Status#status_at=(_) +class_method PostHog::Emails::Status.[](*arg0) +class_method PostHog::Emails::Status.inspect() +class_method PostHog::Emails::Status.keyword_init?() +class_method PostHog::Emails::Status.members() +class_method PostHog::Emails::Status.new(*arg0) +class PostHog::Emails::Warning < Struct +instance_method PostHog::Emails::Warning#code() +instance_method PostHog::Emails::Warning#code=(_) +instance_method PostHog::Emails::Warning#fix() +instance_method PostHog::Emails::Warning#fix=(_) +instance_method PostHog::Emails::Warning#message() +instance_method PostHog::Emails::Warning#message=(_) +class_method PostHog::Emails::Warning.[](*arg0) +class_method PostHog::Emails::Warning.inspect() +class_method PostHog::Emails::Warning.keyword_init?() +class_method PostHog::Emails::Warning.members() +class_method PostHog::Emails::Warning.new(*arg0) class PostHog::FeatureFlagEvaluations instance_method PostHog::FeatureFlagEvaluations#distinct_id() instance_method PostHog::FeatureFlagEvaluations#enabled?(key, default_value: ...) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb new file mode 100644 index 00000000..96f16327 --- /dev/null +++ b/spec/posthog/emails_spec.rb @@ -0,0 +1,482 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module PostHog + describe Emails do + let(:send_url) { 'https://us.posthog.com/api/projects/@current/messaging/send/email/' } + let(:template_id) { '0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b' } + let(:message_id) { '01a18356-c56c-0001-e494-7666bc915188' } + let(:status_url) { "https://us.posthog.com/api/projects/@current/messaging/emails/#{message_id}/" } + let(:uuid_v4) { /\A\h{8}-\h{4}-4\h{3}-[89ab]\h{3}-\h{12}\z/ } + let(:options) { { api_key: API_KEY, secret_key: ' phs_test ', test_mode: true } } + let(:client) { Client.new(options) } + let(:emails) { client.emails.tap { |emails| allow(emails).to receive(:sleep) { |seconds| delays << seconds } } } + let(:delays) { [] } + + before do + stub_request(:get, %r{/flags/definitions}).to_return(status: 200, body: '{"flags":[]}') + end + + after { client.shutdown(timeout: 1) } + + def accept(message_id = 'msg_1', **extra) + { status: 202, body: JSON.generate({ message_id: message_id }.merge(extra)) } + end + + def refuse(status, code, detail: nil, headers: {}, **fields) + { status: status, body: JSON.generate({ code: code, detail: detail }.merge(fields).compact), headers: headers } + end + + def sent_bodies + bodies = [] + expect(a_request(:post, send_url).with do |request| + bodies << JSON.parse(request.body) + end).to have_been_made.at_least_once + bodies + end + + def send_template(**overrides) + emails.send(to: 'user@example.com', template_id: template_id, **overrides) + end + + def send_code(**overrides) + emails.send(to: 'user@example.com', template_key: 'password-reset', subject: 'Reset your password', + html: '

Reset

', **overrides) + end + + describe 'a send of a template made in PostHog' do + it 'posts the template and its data to the app host with the secret key' do + stub_request(:post, send_url).to_return(accept) + + result = send_template(template_id: template_id.upcase, template_data: { name: 'Ada' }, distinct_id: 'user_123') + + expect(result.id).to eq('msg_1') + expect(result.replayed).to be(false) + expect( + a_request(:post, send_url).with( + body: { template_id: template_id, message_data: { name: 'Ada' }, to: 'user@example.com', + distinct_id: 'user_123' }, + headers: { + 'Authorization' => 'Bearer phs_test', + 'Content-Type' => 'application/json', + 'Accept' => 'application/json', + 'User-Agent' => "posthog-ruby/#{PostHog::VERSION}" + } + ) + ).to have_been_made.once + end + + it 'sends an empty message_data and no distinct_id when the caller passes neither' do + stub_request(:post, send_url).to_return(accept) + + send_template + + expect(sent_bodies).to eq([{ 'template_id' => template_id, 'message_data' => {}, 'to' => 'user@example.com' }]) + end + + it 'sends dates in template_data as ISO 8601' do + stub_request(:post, send_url).to_return(accept) + + send_template(template_data: { renews_at: Time.utc(2026, 11, 2, 9, 30) }) + + expect(sent_bodies.first['message_data']).to eq('renews_at' => '2026-11-02T09:30:00.000Z') + end + + it 'refuses code content, because the template sets its own' do + expect { send_template(subject: 'Hi') } + .to raise_error(ArgumentError, /template_id doesn't take subject.*send with template_key/) + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + end + + describe 'a send of content from your code' do + it 'posts the content under its key and captures engagement by default' do + stub_request(:post, send_url).to_return(accept) + + send_code(text: 'Reset', from: 'Acme ', reply_to: 'help@example.com', + distinct_id: 'user_123') + + expect(sent_bodies).to eq([{ + 'template_key' => 'password-reset', + 'subject' => 'Reset your password', + 'html' => '

Reset

', + 'text' => 'Reset', + 'from' => 'Acme ', + 'reply_to' => ['help@example.com'], + 'capture_engagement' => true, + 'to' => 'user@example.com', + 'distinct_id' => 'user_123' + }]) + end + + it 'turns engagement capture off when the caller asks' do + stub_request(:post, send_url).to_return(accept) + + send_code(capture_engagement: false, reply_to: %w[a@example.com b@example.com]) + + expect(sent_bodies.first).to include('capture_engagement' => false, + 'reply_to' => %w[a@example.com b@example.com]) + end + + it 'sends text alone' do + stub_request(:post, send_url).to_return(accept) + + emails.send(to: 'user@example.com', template_key: 'digest', subject: 'Digest', text: 'Hello') + + expect(sent_bodies.first.keys).to contain_exactly('template_key', 'subject', 'text', 'capture_engagement', 'to') + end + + it 'refuses template_data, because local templates are not supported' do + expect { send_code(template_data: { name: 'Ada' }) } + .to raise_error(ArgumentError, /template_key doesn't take template_data/) + end + + it 'needs a subject and html or text' do + expect { emails.send(to: 'user@example.com', template_key: 'k', html: '

') } + .to raise_error(ArgumentError, /needs a subject/) + expect { emails.send(to: 'user@example.com', template_key: 'k', subject: 'Hi') } + .to raise_error(ArgumentError, /needs html, text or both/) + end + + it 'refuses a template_key with characters PostHog does not take' do + expect { send_code(template_key: 'password reset') }.to raise_error(ArgumentError, /template_key can only/) + end + end + + describe 'argument checks' do + it 'needs exactly one form' do + expect { emails.send(to: 'user@example.com') }.to raise_error(ArgumentError, /either template_id, or/) + expect { send_code(template_id: template_id) }.to raise_error(ArgumentError, /either template_id, or/) + end + + it 'refuses bad arguments before any request' do + expect { send_template(template_id: 'welcome') }.to raise_error(ArgumentError, /template's UUID/) + expect { send_template(to: ' ') }.to raise_error(ArgumentError, /to must be/) + expect { send_template(distinct_id: '') }.to raise_error(ArgumentError, /distinct_id/) + expect { send_template(idempotency_key: 'has space') }.to raise_error(ArgumentError, /idempotency_key/) + expect { send_template(max_retries: -1) }.to raise_error(ArgumentError, /max_retries/) + expect { send_template(template_data: []) }.to raise_error(ArgumentError, /template_data must be a Hash/) + expect { send_code(capture_engagement: 'no') }.to raise_error(ArgumentError, /capture_engagement/) + expect { send_code(reply_to: [1]) }.to raise_error(ArgumentError, /reply_to/) + expect { send_template(cc: 'x@example.com') }.to raise_error(ArgumentError, /unknown keyword: :cc/) + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + end + + describe 'client setup' do + it 'maps an EU ingestion host to the EU app host' do + options[:host] = 'https://eu.i.posthog.com/' + stub_request(:post, 'https://eu.posthog.com/api/projects/@current/messaging/send/email/').to_return(accept) + + expect(send_template.id).to eq('msg_1') + end + + it 'refuses a custom host without app_host, so a proxy never receives the secret key' do + options[:host] = 'https://proxy.example.com' + + expect { send_template }.to raise_error(EmailError) { |error| + expect(error.code).to eq('missing_app_host') + expect(error.name).to eq('application_error') + } + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + + it 'sends to app_host when the client sets it' do + options.merge!(host: 'https://proxy.example.com', app_host: 'https://posthog.example.com/') + stub_request(:post, 'https://posthog.example.com/api/projects/@current/messaging/send/email/').to_return(accept) + + expect(send_template.id).to eq('msg_1') + end + + it 'refuses a send without a secret key' do + options.delete(:secret_key) + + expect { send_template }.to raise_error(EmailError) { |error| + expect(error.code).to eq('missing_secret_key') + expect(error.name).to eq('missing_api_key') + } + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + + it 'sends nothing from a disabled client' do + options[:api_key] = nil + + expect { send_template }.to raise_error(EmailError) { |error| expect(error.code).to eq('client_disabled') } + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + + it 'names the provider the app moved from on every request' do + options[:migrated_from] = 'resend' + stub_request(:post, send_url).to_return(accept) + + send_template + + expect(a_request(:post, send_url).with(headers: { 'X-PostHog-Migrated-From' => 'resend' })).to have_been_made + end + + it 'refuses email options it cannot use' do + expect { Client.new(options.merge(email_request_max_retries: -1)) }.to raise_error(ArgumentError, /max_retries/) + expect { Client.new(options.merge(email_request_timeout_seconds: 0)) }.to raise_error(ArgumentError, /timeout/) + expect do + Client.new(options.merge(migrated_from: 'my provider')) + end.to raise_error(ArgumentError, /migrated_from/) + end + end + + describe 'retries' do + it 'retries an unavailable service with the same generated idempotency key' do + stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable'), accept) + + expect(send_template.id).to eq('msg_1') + + keys = [] + expect(a_request(:post, send_url).with { |request| keys << request.headers['Idempotency-Key'] }) + .to have_been_made.twice + expect(keys.uniq.size).to eq(1) + expect(keys.first).to match(uuid_v4) + expect(delays.first).to be_between(0.8, 1.2) + end + + it 'keeps the caller idempotency key' do + stub_request(:post, send_url).to_return(accept) + + send_template(idempotency_key: 'welcome:user_123') + + expect(a_request(:post, send_url).with(headers: { 'Idempotency-Key' => 'welcome:user_123' })).to have_been_made + end + + it 'makes 3 attempts by default and reports the last error' do + stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable', detail: 'Try later.')) + + expect { send_template }.to raise_error(EmailError, 'Try later.') { |error| + expect(error.name).to eq('service_unavailable') + expect(error.status_code).to eq(503) + } + expect(a_request(:post, send_url)).to have_been_made.times(3) + expect(delays.size).to eq(2) + end + + it 'makes 1 attempt when the send turns retries off' do + stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable')) + + expect { send_template(max_retries: 0) }.to raise_error(EmailError) + expect(a_request(:post, send_url)).to have_been_made.once + end + + it 'uses the client retry limit' do + options[:email_request_max_retries] = 0 + stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable')) + + expect { send_template }.to raise_error(EmailError) + expect(a_request(:post, send_url)).to have_been_made.once + end + + [ + [503, 'delivery_unavailable', 2], + [429, 'throttled', 2], + [409, 'idempotency_key_in_flight', 2], + [429, 'quota_limited', 1], + [422, 'recipient_suppressed', 1] + ].each do |status, code, attempts| + it "makes #{attempts} attempts for a #{status} #{code}" do + stub_request(:post, send_url).to_return(refuse(status, code), accept) + + begin + send_template + rescue EmailError + nil + end + expect(a_request(:post, send_url)).to have_been_made.times(attempts) + end + end + + it 'retries a network error, and reports it as service_unavailable' do + stub_request(:post, send_url).to_raise(Errno::ECONNREFUSED) + + expect { send_template }.to raise_error(EmailError) { |error| + expect(error.code).to eq('network_error') + expect(error.name).to eq('service_unavailable') + expect(error.cause).to be_a(Errno::ECONNREFUSED) + } + expect(a_request(:post, send_url)).to have_been_made.times(3) + end + + it 'retries a timeout' do + stub_request(:post, send_url).to_timeout.then.to_return(accept) + + expect(send_template.id).to eq('msg_1') + end + + it 'waits for a longer Retry-After' do + stub_request(:post, send_url).to_return(refuse(429, 'throttled', headers: { 'Retry-After' => '5' }), accept) + + send_template + + expect(delays).to eq([5.0]) + end + + it 'fails at once on a Retry-After over 30 seconds' do + stub_request(:post, send_url).to_return(refuse(429, 'throttled', headers: { 'Retry-After' => '60' })) + + expect { send_template }.to raise_error(EmailError) { |error| + expect(error.code).to eq('throttled') + expect(error.retry_after).to eq(60.0) + } + expect(a_request(:post, send_url)).to have_been_made.once + end + + it 'does not retry an acceptance without a message ID' do + stub_request(:post, send_url).to_return(status: 202, body: '{}') + + expect { send_template }.to raise_error(EmailError) { |error| + expect(error.code).to eq('api_error') + expect(error.status_code).to eq(202) + } + expect(a_request(:post, send_url)).to have_been_made.once + end + end + + describe 'results and errors' do + it 'returns a replay and logs each warning' do + stub_request(:post, send_url).to_return( + accept('msg_1', replayed: true, warnings: [{ code: 'unverified_link', detail: 'Check links.', fix: 'Fix.' }]) + ) + expect(emails.logger).to receive(:warn).with('PostHog email warning unverified_link: Check links.') + + result = send_template + + expect(result.replayed).to be(true) + expect(result.warnings).to eq([Emails::Warning.new(code: 'unverified_link', message: 'Check links.', + fix: 'Fix.')]) + end + + it "keeps PostHog's error fields" do + stub_request(:post, send_url).to_return( + refuse(400, 'sender_not_verified', detail: 'Verify it.', attr: 'from', why: 'Unverified.', + fix: 'Verify the domain.', docs_url: 'https://posthog.com/docs/x') + ) + + expect { send_code }.to raise_error(EmailError, 'Verify it.') { |error| + expect([error.code, error.name, error.status_code]).to eq(['sender_not_verified', 'validation_error', 400]) + expect([error.field, error.why, error.fix, error.docs_url]) + .to eq(['from', 'Unverified.', 'Verify the domain.', 'https://posthog.com/docs/x']) + } + end + + it 'reports an answer without a code as api_error with the name of its status' do + stub_request(:post, send_url).to_return(status: 403, body: 'Forbidden') + + expect { send_template }.to raise_error(EmailError, 'PostHog answered HTTP 403.') { |error| + expect([error.code, error.name]).to eq(%w[api_error invalid_permission]) + } + end + + [ + [401, 'authentication_failed', 'missing_api_key'], + [404, 'not_found', 'not_found'], + [409, 'template_outdated', 'validation_error'], + [409, 'idempotency_key_in_flight', 'concurrent_idempotent_requests'], + [422, 'idempotency_key_reused', 'invalid_idempotent_request'], + [422, 'recipient_suppressed', 'validation_error'], + [429, 'quota_limited', 'monthly_quota_exceeded'], + [429, 'throttled', 'rate_limit_exceeded'], + [500, 'server_error', 'application_error'] + ].each do |status, code, name| + it "names a #{status} #{code} #{name}" do + stub_request(:post, send_url).to_return(refuse(status, code)) + + expect { send_template(max_retries: 0) }.to raise_error(EmailError) { |error| + expect([error.code, error.name, error.status_code]).to eq([code, name, status]) + } + end + end + end + + describe '#get' do + let(:record) do + { id: message_id, status: 'bounced', created_at: '2026-10-28T16:27:44.364000Z', + status_at: '2026-10-28T16:30:02.100000Z', reason: 'hard' } + end + + it "reads the email's status" do + stub_request(:get, status_url).to_return(status: 200, body: JSON.generate(record)) + + status = emails.get(message_id.upcase) + + expect(status.to_h).to eq( + id: message_id, status: 'bounced', reason: 'hard', + created_at: Time.utc(2026, 10, 28, 16, 27, 44.364r), status_at: Time.utc(2026, 10, 28, 16, 30, 2.1r) + ) + expect(a_request(:get, + status_url).with(headers: { 'Authorization' => 'Bearer phs_test' })).to have_been_made.once + end + + it 'passes a status it does not know through' do + stub_request(:get, status_url).to_return(status: 200, + body: JSON.generate(record.merge( + status: 'archived', reason: nil + ))) + + expect(emails.get(message_id)).to have_attributes(status: 'archived', reason: nil) + end + + it 'does not retry an unknown message ID' do + stub_request(:get, status_url).to_return(refuse(404, 'not_found')) + + expect { emails.get(message_id) }.to raise_error(EmailError) { |error| expect(error.code).to eq('not_found') } + expect(a_request(:get, status_url)).to have_been_made.once + end + + it 'reports a record without its times as api_error' do + stub_request(:get, status_url).to_return(status: 200, body: JSON.generate(record.merge(status_at: 'soon'))) + + expect { emails.get(message_id) }.to raise_error(EmailError) { |error| expect(error.code).to eq('api_error') } + end + + it 'refuses an ID that is not a message ID' do + expect { emails.get('msg_1') }.to raise_error(ArgumentError, /message_id/) + end + end + + describe 'shutdown' do + it 'refuses sends after the client shuts down' do + client.shutdown + + expect { send_template }.to raise_error(EmailError) { |error| expect(error.code).to eq('client_closed') } + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + + context 'with a send in its retry backoff' do + let(:backoff_started) { Queue.new } + + before do + allow(emails).to receive(:sleep) do |_seconds| + backoff_started << true + Kernel.sleep(0.3) + end + stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable'), accept) + end + + it 'waits for the retry to settle' do + sender = Thread.new { send_template } + backoff_started.pop + + expect(client.shutdown(timeout: 5)).to be(true) + expect(a_request(:post, send_url)).to have_been_made.twice + expect(sender.value.id).to eq('msg_1') + end + + it 'keeps to its timeout' do + sender = Thread.new { send_template } + backoff_started.pop + + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + expect(client.shutdown(timeout: 0.05)).to be(false) + expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 0.25 + sender.join + end + end + end + end +end From 16a2c6258003208238cf44b0fe4b5b60ef1ad780 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:54:07 +0000 Subject: [PATCH 02/31] fix(emails): forget the parent's requests in a forked child A child forked while another thread sent email inherited a positive in-flight count that no thread could settle, so shutdown waited for its whole timeout, or forever without one. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 11 +++++++++++ spec/posthog/emails_spec.rb | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index f9faf438..2388c48b 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -111,6 +111,7 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f @disabled = disabled @closed = false @in_flight = 0 + @pid = Process.pid @mutex = Mutex.new @idle = ConditionVariable.new end @@ -193,6 +194,7 @@ def get(message_id) def close(timeout: nil) deadline = timeout && (monotonic_time + timeout) @mutex.synchronize do + forget_parent_requests @closed = true until @in_flight.zero? remaining = deadline && (deadline - monotonic_time) @@ -336,6 +338,7 @@ def admit(nothing_done) end @mutex.synchronize do + forget_parent_requests if @closed raise EmailError.new(code: 'client_closed', message: "The PostHog client is shut down, so #{nothing_done}.") @@ -345,6 +348,14 @@ def admit(nothing_done) end end + # A forked child inherits the parent's count, but not the threads that would settle those requests. + def forget_parent_requests + return if @pid == Process.pid + + @pid = Process.pid + @in_flight = 0 + end + def finish @mutex.synchronize do @in_flight -= 1 diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 96f16327..f0d2ec34 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -447,6 +447,25 @@ def send_code(**overrides) expect(a_request(:any, %r{/messaging/})).not_to have_been_made end + it 'does not wait in a forked child for a send that runs in the parent', skip: !Process.respond_to?(:fork) do + request_started = Queue.new + release = Queue.new + stub_request(:post, send_url).to_return do + request_started << true + release.pop + accept + end + sender = Thread.new { send_template } + request_started.pop + + child = fork { exit!(client.shutdown(timeout: 5) ? 0 : 1) } + _, status = Process.wait2(child) + release << true + + expect(status.exitstatus).to eq(0) + expect(sender.value.id).to eq('msg_1') + end + context 'with a send in its retry backoff' do let(:backoff_started) { Queue.new } From e33feda0c14c96beaf38bb91ff15f115d6e02a2f Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:04:23 +0000 Subject: [PATCH 03/31] fix(emails): retry a send whose response body was cut off Net::HTTP treats a connection closed partway through a body as the end of the body, so a truncated 202 raised a non-retryable api_error. Turn EOF suppression off so the cut-off read raises a network error and the send retries with the same idempotency key. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 4 +++- spec/posthog/emails_spec.rb | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index 2388c48b..cf8953e4 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -407,7 +407,9 @@ def request(request, uri, secret_key, headers, body = nil) response = Net::HTTP.start( uri.hostname, uri.port, use_ssl: uri.scheme == 'https', open_timeout: @timeout, read_timeout: @timeout, write_timeout: @timeout, - max_retries: 0 + # Net::HTTP returns a body cut off by a closed connection as if it were whole. Raise instead, so the + # send retries with the same idempotency key. + max_retries: 0, ignore_eof: false ) { |http| http.request(request) } [response.code.to_i, parse_json(response.body), parse_retry_after(response['Retry-After'])] rescue *NETWORK_ERRORS diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index f0d2ec34..232b1323 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -308,6 +308,29 @@ def send_code(**overrides) expect(send_template.id).to eq('msg_1') end + it 'retries an acceptance whose body was cut off' do + server = TCPServer.new('127.0.0.1', 0) + replies = ['{"message_id":"msg', JSON.generate(message_id: 'msg_1')] + responder = Thread.new do + Thread.current.report_on_exception = false + replies.each do |body| + connection = server.accept + connection.readpartial(65_536) + connection.write("HTTP/1.1 202 Accepted\r\nContent-Type: application/json\r\n" \ + "Content-Length: 22\r\nConnection: close\r\n\r\n#{body}") + connection.close + end + end + options.merge!(host: 'http://127.0.0.1:9', app_host: "http://127.0.0.1:#{server.addr[1]}") + WebMock.disable_net_connect!(allow_localhost: true) + + expect(send_template.id).to eq('msg_1') + expect(responder.join(5)).to be_truthy + ensure + WebMock.disable_net_connect! + server&.close + end + it 'waits for a longer Retry-After' do stub_request(:post, send_url).to_return(refuse(429, 'throttled', headers: { 'Retry-After' => '5' }), accept) From d945d7514138ecf0ee38d2391b9d25336bb09094 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:59:00 +0000 Subject: [PATCH 04/31] feat(emails): send cc, bcc and attachments with emails.send The code form takes cc and bcc (one address or a list) and attachments: Hashes with filename, content, and an optional content_type and content_id. A binary String, such as File.binread returns, or a list of byte values is encoded as Base64 once, before the retry loop; any other String is taken as Base64 already and must look like it. Empty lists are left out, so a PostHog version without the fields still sends. A file the client can't send raises EmailError invalid_attachment with the field, before any request. Mirrors posthog-node (posthog-js#5514) and posthog-python#1172; the server holds the limits. Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-attachments.md | 5 ++ lib/posthog/email_error.rb | 5 +- lib/posthog/emails.rb | 99 ++++++++++++++++++++++++++--- public_api_snapshot.txt | 2 +- spec/posthog/emails_spec.rb | 104 ++++++++++++++++++++++++++++++- 5 files changed, 204 insertions(+), 11 deletions(-) create mode 100644 .changeset/emails-attachments.md diff --git a/.changeset/emails-attachments.md b/.changeset/emails-attachments.md new file mode 100644 index 00000000..e3de20b4 --- /dev/null +++ b/.changeset/emails-attachments.md @@ -0,0 +1,5 @@ +--- +"posthog-ruby": minor +--- + +`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values, and the client encodes them as Base64; any other String is taken as the file already encoded as Base64. A file the client can't send, such as one given by `path`, raises `PostHog::EmailError` with code `invalid_attachment` before any request. diff --git a/lib/posthog/email_error.rb b/lib/posthog/email_error.rb index 0133b7b1..30cf8471 100644 --- a/lib/posthog/email_error.rb +++ b/lib/posthog/email_error.rb @@ -37,10 +37,13 @@ def initialize(code:, message:, status_code: nil, why: nil, fix: nil, docs_url: @retry_after = retry_after end + SDK_ERROR_NAMES = { 'network_error' => 'service_unavailable', 'invalid_attachment' => 'validation_error' }.freeze + private_constant :SDK_ERROR_NAMES + # @api private def self.name_for(status_code, code) return 'missing_api_key' if code == 'missing_secret_key' || status_code == 401 - return code == 'network_error' ? 'service_unavailable' : 'application_error' if status_code.nil? + return SDK_ERROR_NAMES.fetch(code, 'application_error') if status_code.nil? case status_code when 400 then 'validation_error' diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index cf8953e4..81375724 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -89,7 +89,9 @@ class Emails "project's engagement setting." CODE_FORM_WHY = 'A send with template_key takes its content from your code.' TEMPLATE_FIELDS = %i[template_data].freeze - CODE_FIELDS = %i[subject html text from reply_to capture_engagement].freeze + CODE_FIELDS = %i[subject html text from reply_to cc bcc attachments capture_engagement].freeze + ATTACHMENT_FIELDS = %w[filename content content_type content_id].freeze + BASE64_PATTERN = %r{\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?\z} NETWORK_ERRORS = [ Timeout::Error, SocketError, SystemCallError, IOError, EOFError, OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError @@ -97,7 +99,8 @@ class Emails private_constant :SEND_PATH, :STATUS_PATH, :US_APP_HOST, :EU_APP_HOST, :CLOUD_APP_HOSTS, :MAX_RETRY_DELAY_SECONDS, :UUID_PATTERN, :IDEMPOTENCY_KEY_PATTERN, :TEMPLATE_KEY_PATTERN, :HEADER_TOKEN_PATTERN, :SEND_KEY_NEEDED, :READ_KEY_NEEDED, :TEMPLATE_FORM_WHY, - :CODE_FORM_WHY, :TEMPLATE_FIELDS, :CODE_FIELDS, :NETWORK_ERRORS + :CODE_FORM_WHY, :TEMPLATE_FIELDS, :CODE_FIELDS, :ATTACHMENT_FIELDS, :BASE64_PATTERN, + :NETWORK_ERRORS # @api private def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_from:, user_agent:, disabled:) @@ -133,6 +136,18 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # @param from [String] Code form: a verified sender, such as `Acme `. Without it, the # project's default sender. # @param reply_to [String, Array] Code form: one address or a list of addresses. + # @param cc [String, Array] Code form: one address or a list of addresses that get a visible copy, at + # most 10. An empty list sends no copy. + # @param bcc [String, Array] Code form: one address or a list of addresses that get a hidden copy, at + # most 10. An empty list sends no copy. + # @param attachments [Array] Code form: files sent with the email, at most 10 and 512 KB in total. Each + # file is a Hash with: + # - +filename+ [String] the name the recipient sees, such as `invoice.pdf`; + # - +content+ [String, Array] the file's bytes, as a binary String such as `File.binread(path)` + # returns or as a list of byte values, which the client encodes as Base64. Any other String is taken as the + # file already Base64 encoded. PostHog can't fetch a file from a path or URL; + # - +content_type+ [String] the MIME type, optional. Without it, the type follows the file name; + # - +content_id+ [String] optional: shows the file inline where `html` has `` with this ID. # @param capture_engagement [Boolean] Code form: whether PostHog captures the email's opens and clicks as events # of the recipient. Defaults to +true+. A project whose admin turned engagement capture off captures nothing. # @param distinct_id [String] The recipient's distinct ID in your product. The email's events go to this @@ -145,14 +160,16 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # retries off. # @return [SendResult] The queued email. # @raise [EmailError] PostHog did not queue the email, or the client could not tell whether it did. + # {EmailError#code} is `invalid_attachment`, before any request, for a file the client can't send. # @raise [ArgumentError] An argument has the wrong type or format, or the call mixes the two forms. def send( to:, template_id: nil, template_data: nil, template_key: nil, subject: nil, html: nil, text: nil, from: nil, - reply_to: nil, capture_engagement: nil, distinct_id: nil, idempotency_key: nil, max_retries: nil + reply_to: nil, cc: nil, bcc: nil, attachments: nil, capture_engagement: nil, distinct_id: nil, + idempotency_key: nil, max_retries: nil ) fields = { template_data: template_data, subject: subject, html: html, text: text, from: from, reply_to: reply_to, - capture_engagement: capture_engagement + cc: cc, bcc: bcc, attachments: attachments, capture_engagement: capture_engagement }.compact check_recipient(to, distinct_id, idempotency_key, max_retries) payload = content(template_id, template_key, fields).merge(to: to) @@ -292,7 +309,14 @@ def code_content(template_key, fields) end code = fields.slice(:subject, :html, :text, :from).merge(template_key: template_key) - code[:reply_to] = addresses(fields[:reply_to]) if fields.key?(:reply_to) + code[:reply_to] = addresses(:reply_to, fields[:reply_to]) if fields.key?(:reply_to) + # An empty list goes out as no list, because a PostHog version without the field refuses it even empty. + %i[cc bcc].each do |name| + list = addresses(name, fields[name]) if fields.key?(name) + code[name] = list unless list.nil? || list.empty? + end + files = attachment_list(fields[:attachments]) if fields.key?(:attachments) + code[:attachments] = files unless files.nil? || files.empty? code[:capture_engagement] = fields.fetch(:capture_engagement, true) code end @@ -305,11 +329,70 @@ def refuse_fields(fields, allowed:, form:, other_form:, why:) raise ArgumentError, "#{message} #{why} Remove #{field}, or send with #{other_form}." end - def addresses(reply_to) - list = reply_to.is_a?(String) ? [reply_to] : reply_to + def addresses(name, value) + list = value.is_a?(String) ? [value] : value return list if list.is_a?(Array) && list.all? { |address| nonempty_string?(address) } - raise ArgumentError, 'reply_to must be one address or a list of addresses.' + raise ArgumentError, "#{name} must be one address or a list of addresses." + end + + def attachment_list(files) + raise invalid_attachment('attachments must be a list of files.', 'attachments') unless files.is_a?(Array) + + files.each_with_index.map { |file, index| attachment(file, "attachments__#{index}") } + end + + def attachment(file, field) + unless file.is_a?(Hash) + raise invalid_attachment("An attachment is a Hash with filename and content. Pass the file's bytes as its " \ + 'content.', field) + end + + file = file.transform_keys(&:to_s).compact + if file.key?('path') + raise invalid_attachment("PostHog can't fetch an attachment from a path or URL. Read the file with " \ + 'File.binread and pass it as content instead.', "#{field}__path") + end + unknown = file.keys.find { |key| !ATTACHMENT_FIELDS.include?(key) } + if unknown + raise invalid_attachment("An attachment takes filename, content, content_type and content_id, not #{unknown}.", + "#{field}__#{unknown}") + end + + %w[filename content_type content_id].each do |name| + next if name != 'filename' && !file.key?(name) + next if file[name].is_a?(String) + + raise invalid_attachment("An attachment's #{name} must be a String.", "#{field}__#{name}") + end + file.merge('content' => base64_content(file['content'], "#{field}__content")) + end + + # A binary String or a list of byte values is the file itself. Any other String is the file already in Base64: + # Ruby has one String type for bytes and text, and File.binread returns a binary one. + def base64_content(content, field) + bytes = file_bytes(content) + return [bytes].pack('m0') if bytes + return content if content.is_a?(String) && BASE64_PATTERN.match?(content) + + if content.is_a?(String) + raise invalid_attachment("An attachment's content is text that isn't Base64. Pass the file's bytes, read " \ + 'with File.binread, or the file encoded as Base64.', field) + end + raise invalid_attachment('An attachment needs content: the bytes File.binread returns, a list of byte values, ' \ + 'or the file encoded as Base64.', field) + end + + def file_bytes(content) + return content if content.is_a?(String) && content.encoding == Encoding::BINARY + return unless content.is_a?(Array) && content.all? { |byte| byte.is_a?(Integer) && byte.between?(0, 255) } + + content.pack('C*') + end + + # The message never holds the file's content, so the error is safe to log. + def invalid_attachment(message, field) + EmailError.new(code: 'invalid_attachment', message: message, field: field) end def call(nothing_done, key_needed) diff --git a/public_api_snapshot.txt b/public_api_snapshot.txt index 16679caa..4417a854 100644 --- a/public_api_snapshot.txt +++ b/public_api_snapshot.txt @@ -74,7 +74,7 @@ instance_method PostHog::EmailError#status_code() instance_method PostHog::EmailError#why() class PostHog::Emails instance_method PostHog::Emails#get(message_id) -instance_method PostHog::Emails#send(to:, template_id: ..., template_data: ..., template_key: ..., subject: ..., html: ..., text: ..., from: ..., reply_to: ..., capture_engagement: ..., distinct_id: ..., idempotency_key: ..., max_retries: ...) +instance_method PostHog::Emails#send(to:, template_id: ..., template_data: ..., template_key: ..., subject: ..., html: ..., text: ..., from: ..., reply_to: ..., cc: ..., bcc: ..., attachments: ..., capture_engagement: ..., distinct_id: ..., idempotency_key: ..., max_retries: ...) class_method PostHog::Emails.logger() class PostHog::Emails::SendResult < Struct instance_method PostHog::Emails::SendResult#id() diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 232b1323..6616e364 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -3,6 +3,12 @@ require 'spec_helper' module PostHog + module Base64Check + def self.decode(text) + text.unpack1('m0') + end + end + describe Emails do let(:send_url) { 'https://us.posthog.com/api/projects/@current/messaging/send/email/' } let(:template_id) { '0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b' } @@ -144,6 +150,102 @@ def send_code(**overrides) end end + describe 'copies and attachments' do + let(:pdf) { "%PDF-1.7\n\xFF\x00\xE2".b } + + def refused_attachment(**overrides) + send_code(**overrides) + rescue EmailError => e + e + end + + it 'sends cc and bcc as lists' do + stub_request(:post, send_url).to_return(accept) + + send_code(cc: 'finance@example.com', bcc: %w[audit@example.com archive@example.com]) + + expect(sent_bodies.first).to include('cc' => ['finance@example.com'], + 'bcc' => %w[audit@example.com archive@example.com]) + end + + it 'sends the bytes of each file as Base64, with its type and content ID' do + stub_request(:post, send_url).to_return(accept) + + send_code(attachments: [ + { filename: 'invoice.pdf', content: pdf, content_type: 'application/pdf' }, + { 'filename' => 'logo.png', 'content' => [137, 80, 78, 71], 'content_id' => 'logo' }, + { filename: 'notes.txt', content: 'aGVsbG8=' } + ]) + + expect(sent_bodies.first['attachments']).to eq([ + { 'filename' => 'invoice.pdf', + 'content' => 'JVBERi0xLjcK/wDi', + 'content_type' => 'application/pdf' }, + { 'filename' => 'logo.png', 'content' => 'iVBORw==', + 'content_id' => 'logo' }, + { 'filename' => 'notes.txt', 'content' => 'aGVsbG8=' } + ]) + expect(Base64Check.decode(sent_bodies.first['attachments'][0]['content'])).to eq(pdf) + end + + it 'leaves out empty lists, so a PostHog version without these fields still sends the email' do + stub_request(:post, send_url).to_return(accept) + + send_code(cc: [], bcc: [], attachments: []) + + expect(sent_bodies.first.keys).not_to include('cc', 'bcc', 'attachments') + end + + it 'sends the same encoded files on every attempt' do + stub_request(:post, send_url).to_return(refuse(503, 'service_unavailable'), accept) + + send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect(a_request(:post, send_url)).to have_been_made.twice + expect(sent_bodies.map { |body| body['attachments'] }) + .to eq([[{ 'filename' => 'invoice.pdf', 'content' => 'JVBERi0xLjcK/wDi' }]]) + end + + it 'refuses a file it cannot send before any request, naming the field and never the content' do + cases = { + { attachments: { filename: 'a.pdf', content: pdf } } => ['attachments', /list of files/], + { attachments: ['a.pdf'] } => ['attachments__0', /Hash with filename and content/], + { attachments: [{ filename: 'a.pdf', path: '/tmp/a.pdf' }] } => ['attachments__0__path', /File.binread/], + { attachments: [{ filename: 'a.pdf', content: pdf, headers: {} }] } => + ['attachments__0__headers', /not headers/], + { attachments: [{ filename: 'a.pdf' }] } => ['attachments__0__content', /needs content/], + { attachments: [{ filename: 'a.pdf', content: [256] }] } => ['attachments__0__content', /needs content/], + { attachments: [{ filename: 'a.pdf', content: 'secret text, not Base64' }] } => + ['attachments__0__content', /isn't Base64/], + { attachments: [{ filename: :a, content: pdf }] } => ['attachments__0__filename', /filename/], + { attachments: [{ filename: 'a.pdf', content: pdf, content_type: 1 }] } => + ['attachments__0__content_type', /content_type/] + } + + cases.each do |arguments, (field, message)| + error = refused_attachment(**arguments) + expect(error).to be_a(EmailError), "#{arguments} raised #{error.inspect}" + expect([error.code, error.name, error.field]).to eq(['invalid_attachment', 'validation_error', field]) + expect(error.message).to match(message) + expect(error.message).not_to include('secret text') + end + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + + it 'refuses an address list that is not one', :aggregate_failures do + expect { send_code(cc: [nil]) }.to raise_error(ArgumentError, /cc must be one address or a list/) + expect { send_code(bcc: 3) }.to raise_error(ArgumentError, /bcc must be one address or a list/) + end + + it 'refuses them on a template send, because the template sets its own recipients and files' do + %i[cc bcc attachments].each do |field| + expect { send_template(field => []) } + .to raise_error(ArgumentError, /template_id doesn't take #{field}.*send with template_key/) + end + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + end + describe 'argument checks' do it 'needs exactly one form' do expect { emails.send(to: 'user@example.com') }.to raise_error(ArgumentError, /either template_id, or/) @@ -159,7 +261,7 @@ def send_code(**overrides) expect { send_template(template_data: []) }.to raise_error(ArgumentError, /template_data must be a Hash/) expect { send_code(capture_engagement: 'no') }.to raise_error(ArgumentError, /capture_engagement/) expect { send_code(reply_to: [1]) }.to raise_error(ArgumentError, /reply_to/) - expect { send_template(cc: 'x@example.com') }.to raise_error(ArgumentError, /unknown keyword: :cc/) + expect { send_template(headers: {}) }.to raise_error(ArgumentError, /unknown keyword: :headers/) expect(a_request(:any, %r{/messaging/})).not_to have_been_made end end From 90c1afbd3f5ef537b7d4748a31581cd768bb1ae1 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:07:18 +0000 Subject: [PATCH 05/31] fix(emails): answer the round 3 review - Explicit nil email options fall back to the defaults, and the client checks them before the worker, instance count and flags poller start. - app_host must be an http or https URL with a host. - A refusal whose body broke off keeps its status and Retry-After; a corrupt compressed body is a network error and retries. - skip_ssl_verification also applies to email requests. - Specs: real-socket server reads whole requests and runs without WebMock, the backoff spec waits on a queue instead of a fixed sleep, and the warning spec reads the logger output. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/client.rb | 23 ++++--- lib/posthog/emails.rb | 48 ++++++++++--- spec/posthog/emails_spec.rb | 134 ++++++++++++++++++++++++++++++------ 3 files changed, 164 insertions(+), 41 deletions(-) diff --git a/lib/posthog/client.rb b/lib/posthog/client.rb index 5292f44d..33aea387 100644 --- a/lib/posthog/client.rb +++ b/lib/posthog/client.rb @@ -163,6 +163,19 @@ def initialize(opts = {}) @lib = opts[:_lib] || 'posthog-ruby' @lib_version = (opts[:_lib_version] || PostHog::VERSION).to_s @headers = Defaults::Request::HEADERS.merge('User-Agent' => "#{@lib}/#{@lib_version}") + # Built before the worker, the instance count and the flags poller start, so a bad email option raises + # before anything runs or sends the secret key. + @emails = Emails.new( + secret_key: secret_key, + host: opts[:host], + app_host: opts[:app_host], + timeout: opts[:email_request_timeout_seconds] || Defaults::Emails::REQUEST_TIMEOUT_SECONDS, + max_retries: opts[:email_request_max_retries] || Defaults::Emails::REQUEST_MAX_RETRIES, + migrated_from: opts[:migrated_from], + user_agent: @headers['User-Agent'], + disabled: @disabled, + skip_ssl_verification: opts[:skip_ssl_verification] == true + ) @worker_mutex = Mutex.new @shutdown_mutex = Mutex.new @shutdown_condition = ConditionVariable.new @@ -236,16 +249,6 @@ def initialize(opts = {}) ) end - @emails = Emails.new( - secret_key: secret_key, - host: opts[:host], - app_host: opts[:app_host], - timeout: opts.fetch(:email_request_timeout_seconds, Defaults::Emails::REQUEST_TIMEOUT_SECONDS), - max_retries: opts.fetch(:email_request_max_retries, Defaults::Emails::REQUEST_MAX_RETRIES), - migrated_from: opts[:migrated_from], - user_agent: @headers['User-Agent'], - disabled: @disabled - ) @before_send = opts[:before_send] @is_server = opts.fetch(:is_server, true) != false @release_id = normalize_string_option(ENV.fetch(RELEASE_ID_ENV_VAR, nil), blank_as_nil: true) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index cf8953e4..be011cf1 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -5,6 +5,8 @@ require 'openssl' require 'securerandom' require 'time' +require 'uri' +require 'zlib' require 'posthog/email_error' require 'posthog/logging' @@ -92,7 +94,7 @@ class Emails CODE_FIELDS = %i[subject html text from reply_to capture_engagement].freeze NETWORK_ERRORS = [ Timeout::Error, SocketError, SystemCallError, IOError, EOFError, OpenSSL::SSL::SSLError, - Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError + Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error ].freeze private_constant :SEND_PATH, :STATUS_PATH, :US_APP_HOST, :EU_APP_HOST, :CLOUD_APP_HOSTS, :MAX_RETRY_DELAY_SECONDS, :UUID_PATTERN, :IDEMPOTENCY_KEY_PATTERN, :TEMPLATE_KEY_PATTERN, @@ -100,7 +102,8 @@ class Emails :CODE_FORM_WHY, :TEMPLATE_FIELDS, :CODE_FIELDS, :NETWORK_ERRORS # @api private - def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_from:, user_agent:, disabled:) + def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_from:, user_agent:, disabled:, + skip_ssl_verification: false) check_options(timeout, max_retries, migrated_from) @secret_key = secret_key.is_a?(String) ? secret_key.strip : nil @app_host = resolve_app_host(host, app_host) @@ -109,6 +112,7 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f @migrated_from = migrated_from @user_agent = user_agent @disabled = disabled + @skip_ssl_verification = skip_ssl_verification @closed = false @in_flight = 0 @pid = Process.pid @@ -230,9 +234,20 @@ def uuid?(value) def resolve_app_host(host, app_host) explicit = app_host.is_a?(String) ? app_host.strip.chomp('/') : '' - return CLOUD_APP_HOSTS.fetch(explicit, explicit) unless explicit.empty? + return CLOUD_APP_HOSTS[host.to_s.strip.chomp('/')] if explicit.empty? + # A host without a scheme would send the secret key to the wrong place, such as localhost. + unless web_url?(explicit) + raise ArgumentError, 'app_host must be an http or https URL, such as https://us.posthog.com.' + end + + CLOUD_APP_HOSTS.fetch(explicit, explicit) + end - CLOUD_APP_HOSTS[host.to_s.strip.chomp('/')] + def web_url?(value) + uri = URI(value) + %w[http https].include?(uri.scheme) && !uri.host.to_s.empty? + rescue URI::InvalidURIError + false end def check_recipient(to, distinct_id, idempotency_key, max_retries) @@ -404,18 +419,29 @@ def request(request, uri, secret_key, headers, body = nil) headers.each { |name, value| request[name] = value } request.body = body if body - response = Net::HTTP.start( - uri.hostname, uri.port, - use_ssl: uri.scheme == 'https', open_timeout: @timeout, read_timeout: @timeout, write_timeout: @timeout, - # Net::HTTP returns a body cut off by a closed connection as if it were whole. Raise instead, so the - # send retries with the same idempotency key. - max_retries: 0, ignore_eof: false - ) { |http| http.request(request) } + head = nil + response = Net::HTTP.start(uri.hostname, uri.port, connection_options(uri)) do |http| + http.request(request) { |started| head = started } + end [response.code.to_i, parse_json(response.body), parse_retry_after(response['Retry-After'])] rescue *NETWORK_ERRORS + # A refusal whose body broke off still carries its status and Retry-After. An acceptance whose body broke off + # has no message ID, so it retries with the same idempotency key. + return [head.code.to_i, nil, parse_retry_after(head['Retry-After'])] if head && !head.is_a?(Net::HTTPSuccess) + raise EmailError.new(code: 'network_error', message: 'The request to PostHog failed.') end + def connection_options(uri) + options = { + use_ssl: uri.scheme == 'https', open_timeout: @timeout, read_timeout: @timeout, write_timeout: @timeout, + # Net::HTTP returns a body cut off by a closed connection as if it were whole. Raise instead. + max_retries: 0, ignore_eof: false + } + options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE if @skip_ssl_verification + options + end + def parse_json(body) JSON.parse(body.to_s) rescue JSON::ParserError diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 232b1323..19763ae7 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -36,6 +36,60 @@ def sent_bodies bodies end + # Serves each raw HTTP response once, in order, on a local socket, and points the client's app host at it. WebMock + # is off meanwhile: its Net::HTTP wrapper reads the whole body before it hands over the response. + def serve(*responses, tls: nil) + server = TCPServer.new('127.0.0.1', 0) + listener = tls ? OpenSSL::SSL::SSLServer.new(server, tls) : server + responder = Thread.new do + Thread.current.report_on_exception = false + responses.each do |response| + connection = listener.accept + read_request(connection) + connection.write(response) + connection.close + end + end + options.merge!(host: 'http://127.0.0.1:9', app_host: "#{tls ? 'https' : 'http'}://127.0.0.1:#{server.addr[1]}") + WebMock.disable! + yield + expect(responder.join(5)).to be_truthy + ensure + WebMock.enable! + server&.close + end + + # Reads one whole request, so closing the connection never resets a client still writing its body. + def read_request(connection) + request = +'' + request << connection.readpartial(65_536) until request.include?("\r\n\r\n") + head, body = request.split("\r\n\r\n", 2) + length = head[/^content-length: *(\d+)/i, 1].to_i + body << connection.readpartial(65_536) while body.bytesize < length + end + + def http_response(status, body, headers = {}) + head = headers.merge('Content-Type' => 'application/json', 'Connection' => 'close') + head['Content-Length'] ||= body.bytesize.to_s + "HTTP/1.1 #{status}\r\n#{head.map { |name, value| "#{name}: #{value}\r\n" }.join}\r\n#{body}" + end + + def self_signed_tls + key = OpenSSL::PKey::RSA.new(2048) + cert = OpenSSL::X509::Certificate.new + cert.version = 2 + cert.serial = 1 + cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/CN=127.0.0.1') + cert.public_key = key.public_key + cert.not_before = Time.now - 60 + cert.not_after = Time.now + 3600 + cert.sign(key, OpenSSL::Digest.new('SHA256')) + OpenSSL::SSL::SSLContext.new.tap do |context| + context.cert = cert + context.key = key + end + end + def send_template(**overrides) emails.send(to: 'user@example.com', template_id: template_id, **overrides) end @@ -221,6 +275,31 @@ def send_code(**overrides) expect do Client.new(options.merge(migrated_from: 'my provider')) end.to raise_error(ArgumentError, /migrated_from/) + %w[posthog.example.com ftp://posthog.example.com https://].each do |app_host| + expect { Client.new(options.merge(app_host: app_host)) }.to raise_error(ArgumentError, /app_host/) + end + end + + it 'refuses a bad email option before it polls flags with the secret key' do + expect do + Client.new(options.merge(test_mode: false, email_request_timeout_seconds: 0)) + end.to raise_error(ArgumentError, /timeout/) + expect(a_request(:get, %r{/flags/definitions})).not_to have_been_made + end + + it 'uses the defaults for email options passed as nil' do + stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable'), refuse(503, 'x'), accept) + options.merge!(email_request_timeout_seconds: nil, email_request_max_retries: nil) + + expect(send_template.id).to eq('msg_1') + end + + it 'skips certificate checks when the client sets skip_ssl_verification' do + options[:skip_ssl_verification] = true + + serve(http_response('202 Accepted', JSON.generate(message_id: 'msg_1')), tls: self_signed_tls) do + expect(send_template.id).to eq('msg_1') + end end end @@ -309,26 +388,32 @@ def send_code(**overrides) end it 'retries an acceptance whose body was cut off' do - server = TCPServer.new('127.0.0.1', 0) - replies = ['{"message_id":"msg', JSON.generate(message_id: 'msg_1')] - responder = Thread.new do - Thread.current.report_on_exception = false - replies.each do |body| - connection = server.accept - connection.readpartial(65_536) - connection.write("HTTP/1.1 202 Accepted\r\nContent-Type: application/json\r\n" \ - "Content-Length: 22\r\nConnection: close\r\n\r\n#{body}") - connection.close - end + cut_off = http_response('202 Accepted', '{"message_id":"msg', 'Content-Length' => '22') + + serve(cut_off, http_response('202 Accepted', JSON.generate(message_id: 'msg_1'))) do + expect(send_template.id).to eq('msg_1') end - options.merge!(host: 'http://127.0.0.1:9', app_host: "http://127.0.0.1:#{server.addr[1]}") - WebMock.disable_net_connect!(allow_localhost: true) + end - expect(send_template.id).to eq('msg_1') - expect(responder.join(5)).to be_truthy - ensure - WebMock.disable_net_connect! - server&.close + it 'retries an acceptance whose compressed body is corrupt' do + # A gzip header, then a deflate block of a type that does not exist. + corrupt = http_response('202 Accepted', "\x1F\x8B\x08\x00\x00\x00\x00\x00\x00\x03\xFF\xFF".b, + 'Content-Encoding' => 'gzip') + + serve(corrupt, http_response('202 Accepted', JSON.generate(message_id: 'msg_1'))) do + expect(send_template.id).to eq('msg_1') + end + end + + it "keeps a refusal's status and Retry-After when its body was cut off" do + cut_off = http_response('429 Too Many Requests', '{"code":"thro', + 'Content-Length' => '40', 'Retry-After' => '60') + + serve(cut_off) do + expect { send_template }.to raise_error(EmailError) { |error| + expect([error.status_code, error.name, error.retry_after]).to eq([429, 'rate_limit_exceeded', 60.0]) + } + end end it 'waits for a longer Retry-After' do @@ -365,10 +450,13 @@ def send_code(**overrides) stub_request(:post, send_url).to_return( accept('msg_1', replayed: true, warnings: [{ code: 'unverified_link', detail: 'Check links.', fix: 'Fix.' }]) ) - expect(emails.logger).to receive(:warn).with('PostHog email warning unverified_link: Check links.') + log = StringIO.new + PostHog::Logging.logger = Logger.new(log, level: Logger::WARN) result = send_template + expect(log.string).to include('PostHog email warning unverified_link: Check links.') + expect(result.replayed).to be(true) expect(result.warnings).to eq([Emails::Warning.new(code: 'unverified_link', message: 'Check links.', fix: 'Fix.')]) @@ -491,11 +579,12 @@ def send_code(**overrides) context 'with a send in its retry backoff' do let(:backoff_started) { Queue.new } + let(:release_retry) { Queue.new } before do allow(emails).to receive(:sleep) do |_seconds| backoff_started << true - Kernel.sleep(0.3) + release_retry.pop end stub_request(:post, send_url).to_return(refuse(503, 'delivery_unavailable'), accept) end @@ -503,6 +592,10 @@ def send_code(**overrides) it 'waits for the retry to settle' do sender = Thread.new { send_template } backoff_started.pop + Thread.new do + Kernel.sleep(0.05) + release_retry << true + end expect(client.shutdown(timeout: 5)).to be(true) expect(a_request(:post, send_url)).to have_been_made.twice @@ -516,6 +609,7 @@ def send_code(**overrides) started = Process.clock_gettime(Process::CLOCK_MONOTONIC) expect(client.shutdown(timeout: 0.05)).to be(false) expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 0.25 + release_retry << true sender.join end end From e94874ceeb6ae34fa8c5da1b02cf1f6ddb7eb60d Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:16:27 +0000 Subject: [PATCH 06/31] fix(emails): retry a refusal whose body broke off A reply whose body broke off is a network error again, so a cut-off 409 idempotency_key_in_flight retries as before. It now keeps the head's Retry-After: the retry waits for it, and one over 30 seconds raises at once. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 8 +++----- spec/posthog/emails_spec.rb | 22 ++++++++++++++++++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index be011cf1..c1a9cebc 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -425,11 +425,9 @@ def request(request, uri, secret_key, headers, body = nil) end [response.code.to_i, parse_json(response.body), parse_retry_after(response['Retry-After'])] rescue *NETWORK_ERRORS - # A refusal whose body broke off still carries its status and Retry-After. An acceptance whose body broke off - # has no message ID, so it retries with the same idempotency key. - return [head.code.to_i, nil, parse_retry_after(head['Retry-After'])] if head && !head.is_a?(Net::HTTPSuccess) - - raise EmailError.new(code: 'network_error', message: 'The request to PostHog failed.') + # A reply whose body broke off retries with the same idempotency key, after its Retry-After when it sent one. + raise EmailError.new(code: 'network_error', message: 'The request to PostHog failed.', + retry_after: head && parse_retry_after(head['Retry-After'])) end def connection_options(uri) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 19763ae7..274a6256 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -405,17 +405,35 @@ def send_code(**overrides) end end - it "keeps a refusal's status and Retry-After when its body was cut off" do + it 'retries a refusal whose body was cut off, as a network error' do + cut_off = http_response('409 Conflict', '{"code":"idempotency_key_in', 'Content-Length' => '60') + + serve(cut_off, http_response('202 Accepted', JSON.generate(message_id: 'msg_1'))) do + expect(send_template.id).to eq('msg_1') + end + end + + it 'keeps the Retry-After of a refusal whose body was cut off' do cut_off = http_response('429 Too Many Requests', '{"code":"thro', 'Content-Length' => '40', 'Retry-After' => '60') serve(cut_off) do expect { send_template }.to raise_error(EmailError) { |error| - expect([error.status_code, error.name, error.retry_after]).to eq([429, 'rate_limit_exceeded', 60.0]) + expect([error.code, error.retry_after]).to eq(['network_error', 60.0]) } end end + it 'waits out the Retry-After of a cut-off refusal before it retries' do + cut_off = http_response('503 Service Unavailable', '{"code":"deliv', + 'Content-Length' => '40', 'Retry-After' => '3') + serve(cut_off, http_response('202 Accepted', JSON.generate(message_id: 'msg_1'))) do + emails + expect(send_template.id).to eq('msg_1') + end + expect(delays).to eq([3.0]) + end + it 'waits for a longer Retry-After' do stub_request(:post, send_url).to_return(refuse(429, 'throttled', headers: { 'Retry-After' => '5' }), accept) From c843d535cd5f8c127429ac1c212458dfd4257f1f Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:23:44 +0000 Subject: [PATCH 07/31] test(emails): prove shutdown waits for a send in its retry backoff The spec released the retry on a 50 ms timer, so it passed even when shutdown returned at once. It now holds the retry until it has seen shutdown still running, then releases it and expects true. Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 274a6256..9d2fea35 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -610,12 +610,12 @@ def send_code(**overrides) it 'waits for the retry to settle' do sender = Thread.new { send_template } backoff_started.pop - Thread.new do - Kernel.sleep(0.05) - release_retry << true - end + shutdown = Thread.new { client.shutdown(timeout: 5) } + + expect(shutdown.join(0.2)).to be_nil + release_retry << true - expect(client.shutdown(timeout: 5)).to be(true) + expect(shutdown.value).to be(true) expect(a_request(:post, send_url)).to have_been_made.twice expect(sender.value.id).to eq('msg_1') end From a3a1aab0703b822a56b2d64e92572acdef02b0fa Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:40:03 +0000 Subject: [PATCH 08/31] fix(emails): refuse non-ASCII content as invalid_attachment, take wrapped Base64 - A String that isn't ASCII, such as a PDF read with File.read or UTF-16 text, raised a bare ArgumentError from the Base64 match. It now raises invalid_attachment with the field and the File.binread hint. - Base64 with line breaks, as Base64.encode64 writes it, is sent without them instead of being refused. - The retry spec checks both attempts carry the encoded file, and the Base64Check helper is inlined. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 18 +++++++++++------- spec/posthog/emails_spec.rb | 28 ++++++++++++++++++---------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index 2a46c93a..186c7f85 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -388,14 +388,18 @@ def attachment(file, field) def base64_content(content, field) bytes = file_bytes(content) return [bytes].pack('m0') if bytes - return content if content.is_a?(String) && BASE64_PATTERN.match?(content) - - if content.is_a?(String) - raise invalid_attachment("An attachment's content is text that isn't Base64. Pass the file's bytes, read " \ - 'with File.binread, or the file encoded as Base64.', field) + unless content.is_a?(String) + raise invalid_attachment('An attachment needs content: the bytes File.binread returns, a list of byte ' \ + 'values, or the file encoded as Base64.', field) end - raise invalid_attachment('An attachment needs content: the bytes File.binread returns, a list of byte values, ' \ - 'or the file encoded as Base64.', field) + + # Base64.encode64 breaks lines every 60 characters, and PostHog takes Base64 without them. A String that isn't + # ASCII, such as a PDF read with File.read, can't be Base64. + encoded = content.ascii_only? && content.delete("\r\n") + return encoded if encoded && BASE64_PATTERN.match?(encoded) + + raise invalid_attachment("An attachment's content is text that isn't Base64. Pass the file's bytes, read " \ + 'with File.binread, or the file encoded as Base64.', field) end def file_bytes(content) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 5abb19b0..37ed20ad 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -3,12 +3,6 @@ require 'spec_helper' module PostHog - module Base64Check - def self.decode(text) - text.unpack1('m0') - end - end - describe Emails do let(:send_url) { 'https://us.posthog.com/api/projects/@current/messaging/send/email/' } let(:template_id) { '0190f6d2-7c1e-7b3a-9f5e-1c2d3e4f5a6b' } @@ -239,7 +233,7 @@ def refused_attachment(**overrides) 'content_id' => 'logo' }, { 'filename' => 'notes.txt', 'content' => 'aGVsbG8=' } ]) - expect(Base64Check.decode(sent_bodies.first['attachments'][0]['content'])).to eq(pdf) + expect(sent_bodies.first['attachments'][0]['content'].unpack1('m0')).to eq(pdf) end it 'leaves out empty lists, so a PostHog version without these fields still sends the email' do @@ -255,9 +249,19 @@ def refused_attachment(**overrides) send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) - expect(a_request(:post, send_url)).to have_been_made.twice - expect(sent_bodies.map { |body| body['attachments'] }) - .to eq([[{ 'filename' => 'invoice.pdf', 'content' => 'JVBERi0xLjcK/wDi' }]]) + encoded_once = [{ 'filename' => 'invoice.pdf', 'content' => 'JVBERi0xLjcK/wDi' }] + expect(a_request(:post, send_url).with { |request| JSON.parse(request.body)['attachments'] == encoded_once }) + .to have_been_made.twice + end + + it 'takes Base64 with line breaks, as Base64.encode64 writes it' do + stub_request(:post, send_url).to_return(accept) + file = (0..99).to_a.pack('C*') + + send_code(attachments: [{ filename: 'data.bin', content: [file].pack('m') }]) + + expect([file].pack('m')).to include("\n") + expect(sent_bodies.first['attachments'][0]['content']).to eq([file].pack('m0')) end it 'refuses a file it cannot send before any request, naming the field and never the content' do @@ -271,6 +275,10 @@ def refused_attachment(**overrides) { attachments: [{ filename: 'a.pdf', content: [256] }] } => ['attachments__0__content', /needs content/], { attachments: [{ filename: 'a.pdf', content: 'secret text, not Base64' }] } => ['attachments__0__content', /isn't Base64/], + { attachments: [{ filename: 'a.pdf', content: "%PDF secret text \xE2\xE3".dup.force_encoding('UTF-8') }] } => + ['attachments__0__content', /File.binread/], + { attachments: [{ filename: 'a.txt', content: 'aGVsbG8='.encode('UTF-16LE') }] } => + ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: :a, content: pdf }] } => ['attachments__0__filename', /filename/], { attachments: [{ filename: 'a.pdf', content: pdf, content_type: 1 }] } => ['attachments__0__content_type', /content_type/] From e29b8a735c96434a3d3b733d9ef36cf552183dc8 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:44:58 +0000 Subject: [PATCH 09/31] fix(emails): take line-broken Base64 only in Base64.encode64's layout Removing every line break let a text file read with File.read, such as "ABCD1234\nEFGH5678\n", pass as Base64, so the recipient got a garbled file. Line breaks are now removed only when the content is exactly what Base64.encode64 writes; other text is refused as invalid_attachment. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 11 +++++++---- spec/posthog/emails_spec.rb | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index 186c7f85..c7d29461 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -393,10 +393,13 @@ def base64_content(content, field) 'values, or the file encoded as Base64.', field) end - # Base64.encode64 breaks lines every 60 characters, and PostHog takes Base64 without them. A String that isn't - # ASCII, such as a PDF read with File.read, can't be Base64. - encoded = content.ascii_only? && content.delete("\r\n") - return encoded if encoded && BASE64_PATTERN.match?(encoded) + # A String that isn't ASCII, such as a PDF read with File.read, can't be Base64. + if content.ascii_only? + return content if BASE64_PATTERN.match?(content) + # PostHog takes Base64 without line breaks. Only Base64.encode64's own layout loses them, so a text file + # that happens to hold Base64 characters on several lines is still refused. + return content.delete("\n") if content.include?("\n") && [content.unpack1('m')].pack('m') == content + end raise invalid_attachment("An attachment's content is text that isn't Base64. Pass the file's bytes, read " \ 'with File.binread, or the file encoded as Base64.', field) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 37ed20ad..e455a603 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -276,7 +276,9 @@ def refused_attachment(**overrides) { attachments: [{ filename: 'a.pdf', content: 'secret text, not Base64' }] } => ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: 'a.pdf', content: "%PDF secret text \xE2\xE3".dup.force_encoding('UTF-8') }] } => - ['attachments__0__content', /File.binread/], + ['attachments__0__content', /isn't Base64/], + { attachments: [{ filename: 'codes.txt', content: "ABCD1234\nEFGH5678\n" }] } => + ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: 'a.txt', content: 'aGVsbG8='.encode('UTF-16LE') }] } => ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: :a, content: pdf }] } => ['attachments__0__filename', /filename/], From df3ff9200273011a7c1079b4d097252d5b2b985d Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:16:35 +0000 Subject: [PATCH 10/31] refactor(emails): name the attachment rules instead of commenting them Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index c7d29461..59e2e13c 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -325,13 +325,8 @@ def code_content(template_key, fields) code = fields.slice(:subject, :html, :text, :from).merge(template_key: template_key) code[:reply_to] = addresses(:reply_to, fields[:reply_to]) if fields.key?(:reply_to) - # An empty list goes out as no list, because a PostHog version without the field refuses it even empty. - %i[cc bcc].each do |name| - list = addresses(name, fields[name]) if fields.key?(name) - code[name] = list unless list.nil? || list.empty? - end - files = attachment_list(fields[:attachments]) if fields.key?(:attachments) - code[:attachments] = files unless files.nil? || files.empty? + %i[cc bcc].each { |name| put_unless_empty(code, name, addresses(name, fields[name])) if fields.key?(name) } + put_unless_empty(code, :attachments, attachment_list(fields[:attachments])) if fields.key?(:attachments) code[:capture_engagement] = fields.fetch(:capture_engagement, true) code end @@ -344,6 +339,10 @@ def refuse_fields(fields, allowed:, form:, other_form:, why:) raise ArgumentError, "#{message} #{why} Remove #{field}, or send with #{other_form}." end + def put_unless_empty(code, name, list) + code[name] = list unless list.empty? + end + def addresses(name, value) list = value.is_a?(String) ? [value] : value return list if list.is_a?(Array) && list.all? { |address| nonempty_string?(address) } @@ -383,8 +382,6 @@ def attachment(file, field) file.merge('content' => base64_content(file['content'], "#{field}__content")) end - # A binary String or a list of byte values is the file itself. Any other String is the file already in Base64: - # Ruby has one String type for bytes and text, and File.binread returns a binary one. def base64_content(content, field) bytes = file_bytes(content) return [bytes].pack('m0') if bytes @@ -392,19 +389,21 @@ def base64_content(content, field) raise invalid_attachment('An attachment needs content: the bytes File.binread returns, a list of byte ' \ 'values, or the file encoded as Base64.', field) end - - # A String that isn't ASCII, such as a PDF read with File.read, can't be Base64. - if content.ascii_only? - return content if BASE64_PATTERN.match?(content) - # PostHog takes Base64 without line breaks. Only Base64.encode64's own layout loses them, so a text file - # that happens to hold Base64 characters on several lines is still refused. - return content.delete("\n") if content.include?("\n") && [content.unpack1('m')].pack('m') == content - end + return content if single_line_base64?(content) + return content.delete("\n") if encode64_layout?(content) raise invalid_attachment("An attachment's content is text that isn't Base64. Pass the file's bytes, read " \ 'with File.binread, or the file encoded as Base64.', field) end + def single_line_base64?(content) + content.ascii_only? && BASE64_PATTERN.match?(content) + end + + def encode64_layout?(content) + content.ascii_only? && content.include?("\n") && [content.unpack1('m')].pack('m') == content + end + def file_bytes(content) return content if content.is_a?(String) && content.encoding == Encoding::BINARY return unless content.is_a?(Array) && content.all? { |byte| byte.is_a?(Integer) && byte.between?(0, 255) } @@ -412,7 +411,6 @@ def file_bytes(content) content.pack('C*') end - # The message never holds the file's content, so the error is safe to log. def invalid_attachment(message, field) EmailError.new(code: 'invalid_attachment', message: message, field: field) end From 294a89a0928a361a008a18f6dbb96f8cdc08c62b Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:10:46 +0000 Subject: [PATCH 11/31] feat(emails): upload attachments, then send them by hash An email now carries up to 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, checks both limits before any request, asks PostHog where to upload the files it doesn't have yet, PUTs them to file storage without the secret key, and sends the files' SHA-256 hashes. A refused upload sends no email and names what storage answered. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-attachments.md | 2 +- lib/posthog/email_attachments.rb | 148 ++++++++++++++++++++++++ lib/posthog/email_error.rb | 7 +- lib/posthog/emails.rb | 168 +++++++++++++-------------- spec/posthog/emails_spec.rb | 188 ++++++++++++++++++++++++++++--- 5 files changed, 412 insertions(+), 101 deletions(-) create mode 100644 lib/posthog/email_attachments.rb diff --git a/.changeset/emails-attachments.md b/.changeset/emails-attachments.md index e3de20b4..b29514e5 100644 --- a/.changeset/emails-attachments.md +++ b/.changeset/emails-attachments.md @@ -2,4 +2,4 @@ "posthog-ruby": minor --- -`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values, and the client encodes them as Base64; any other String is taken as the file already encoded as Base64. A file the client can't send, such as one given by `path`, raises `PostHog::EmailError` with code `invalid_attachment` before any request. +`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and no email is sent. diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb new file mode 100644 index 00000000..6d327db3 --- /dev/null +++ b/lib/posthog/email_attachments.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require 'digest' + +require 'posthog/email_error' + +module PostHog + # The files of one send: each file's bytes and SHA-256, checked against the limits of one email before any request. + # + # @api private + class EmailAttachments + MAX_FILES = 10 + MAX_MESSAGE_BYTES = 40_000_000 + HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 + LIMIT = '10 files and 40 MB per email after Base64 encoding, about 29 MB of files' + FIELDS = %w[filename content content_type content_id].freeze + BASE64_PATTERN = %r{\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?\z} + + # One file to attach: its bytes, their SHA-256, and what the send says about it. + Attachment = Struct.new(:bytes, :sha256, :reference, :field, keyword_init: true) + + # @param files [Object] What the caller passed as `attachments`. + # @param bodies [Array] The email's HTML and text bodies, which count towards its size. + # @raise [EmailError] A file the client can't send, or more than one email can carry. + def self.from(files, bodies) + raise invalid('attachments must be a list of files.', 'attachments') unless files.is_a?(Array) + + new(files.each_with_index.map { |file, index| attachment(file, "attachments__#{index}") }, bodies) + end + + def self.attachment(file, field) + unless file.is_a?(Hash) + raise invalid("An attachment is a Hash with filename and content. Pass the file's bytes as its content.", field) + end + + file = file.transform_keys(&:to_s).compact + check_keys(file, field) + bytes = file_bytes(file['content'], "#{field}__content") + Attachment.new(bytes: bytes, sha256: Digest::SHA256.hexdigest(bytes), field: field, + reference: file.slice('filename', 'content_type', 'content_id')) + end + + def self.check_keys(file, field) + if file.key?('path') + raise invalid("PostHog can't fetch an attachment from a path or URL. Read the file with File.binread and " \ + 'pass it as content instead.', "#{field}__path") + end + unknown = file.keys.find { |key| !FIELDS.include?(key) } + if unknown + raise invalid("An attachment takes filename, content, content_type and content_id, not #{unknown}.", + "#{field}__#{unknown}") + end + + %w[filename content_type content_id].each do |name| + next if name != 'filename' && !file.key?(name) + next if file[name].is_a?(String) + + raise invalid("An attachment's #{name} must be a String.", "#{field}__#{name}") + end + end + + def self.file_bytes(content, field) + bytes = raw_bytes(content) || base64_bytes(content, field) + raise invalid("An attachment can't be empty. Pass the file's bytes as its content.", field) if bytes.empty? + + bytes + end + + def self.raw_bytes(content) + return content if content.is_a?(String) && content.encoding == Encoding::BINARY + return unless content.is_a?(Array) && content.all? { |byte| byte.is_a?(Integer) && byte.between?(0, 255) } + + content.pack('C*') + end + + def self.base64_bytes(content, field) + unless content.is_a?(String) + raise invalid('An attachment needs content: the bytes File.binread returns, a list of byte values, or the ' \ + 'file encoded as Base64.', field) + end + return content.unpack1('m0') if content.ascii_only? && BASE64_PATTERN.match?(content) + return content.unpack1('m') if encode64_layout?(content) + + raise invalid("An attachment's content is text that isn't Base64. Pass the file's bytes, read with " \ + 'File.binread, or the file encoded as Base64.', field) + end + + def self.encode64_layout?(content) + content.ascii_only? && content.include?("\n") && [content.unpack1('m')].pack('m') == content + end + + def self.invalid(message, field) + EmailError.new(code: 'invalid_attachment', message: message, field: field) + end + + private_class_method :new, :attachment, :check_keys, :file_bytes, :raw_bytes, :base64_bytes, + :encode64_layout?, :invalid + + def initialize(attachments, bodies) + @attachments = attachments + check_fits(bodies) + end + + def empty? + @attachments.empty? + end + + # @return [Array] The files as the send names them. + def references + @attachments.map { |attachment| { 'sha256' => attachment.sha256 }.merge(attachment.reference) } + end + + # @return [Array] Each distinct file once, in the order the caller gave them. + def distinct + @attachments.uniq(&:sha256) + end + + private + + def check_fits(bodies) + if @attachments.size > MAX_FILES + raise refusal('too_many_attachments', "An email carries at most #{MAX_FILES} files, so no email was sent.", + why: "An email carries at most #{MAX_FILES} files.", + fix: "Attach at most #{MAX_FILES} files, and link to any others from the email body.") + end + return if message_bytes(bodies) <= MAX_MESSAGE_BYTES + + raise refusal('message_too_large', 'The files are too large for one email, so no email was sent.', + why: "PostHog's email provider takes emails up to 40 MB, and Base64 encoding makes each file " \ + 'about a third larger.', + fix: "Keep the email under #{LIMIT}: attach fewer or smaller files, or link to large files " \ + 'from the email body.') + end + + def message_bytes(bodies) + files = @attachments.sum { |attachment| encoded_bytes(attachment.bytes.bytesize) } + files + bodies.sum(&:bytesize) + HEADERS_AND_MIME_BOUNDARIES_BYTES + end + + def encoded_bytes(size) + (((size + 2) / 3 * 4 * 78) + 75) / 76 + end + + def refusal(code, message, why:, fix:) + EmailError.new(code: code, message: message, field: 'attachments', why: why, fix: fix) + end + end +end diff --git a/lib/posthog/email_error.rb b/lib/posthog/email_error.rb index 30cf8471..82a5f988 100644 --- a/lib/posthog/email_error.rb +++ b/lib/posthog/email_error.rb @@ -37,7 +37,12 @@ def initialize(code:, message:, status_code: nil, why: nil, fix: nil, docs_url: @retry_after = retry_after end - SDK_ERROR_NAMES = { 'network_error' => 'service_unavailable', 'invalid_attachment' => 'validation_error' }.freeze + SDK_ERROR_NAMES = { + 'network_error' => 'service_unavailable', + 'invalid_attachment' => 'validation_error', + 'too_many_attachments' => 'validation_error', + 'message_too_large' => 'validation_error' + }.freeze private_constant :SDK_ERROR_NAMES # @api private diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index cc003a00..a8ab7a34 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -8,6 +8,7 @@ require 'uri' require 'zlib' +require 'posthog/email_attachments' require 'posthog/email_error' require 'posthog/logging' require 'posthog/utils' @@ -71,6 +72,7 @@ class Emails Status = Struct.new(:id, :status, :created_at, :status_at, :reason, keyword_init: true) SEND_PATH = '/api/projects/@current/messaging/send/email/' + UPLOADS_PATH = '/api/projects/@current/messaging/attachments/uploads/' STATUS_PATH = '/api/projects/@current/messaging/emails/%s/' US_APP_HOST = 'https://us.posthog.com' EU_APP_HOST = 'https://eu.posthog.com' @@ -86,6 +88,7 @@ class Emails IDEMPOTENCY_KEY_PATTERN = /\A[\x21-\x7e]{1,255}\z/ TEMPLATE_KEY_PATTERN = /\A[a-zA-Z0-9_-]{1,400}\z/ HEADER_TOKEN_PATTERN = /\A[\x21-\x7e]+\z/ + JSON_CONTENT = { 'Content-Type' => 'application/json' }.freeze SEND_KEY_NEEDED = 'Sending email needs secret_key, a project secret API key with the ' \ 'transactional_email:write scope.' READ_KEY_NEEDED = 'Reading an email status needs secret_key, a project secret API key with the ' \ @@ -95,17 +98,14 @@ class Emails CODE_FORM_WHY = 'A send with template_key takes its content from your code.' TEMPLATE_FIELDS = %i[template_data].freeze CODE_FIELDS = %i[subject html text from reply_to cc bcc attachments capture_engagement].freeze - ATTACHMENT_FIELDS = %w[filename content content_type content_id].freeze - BASE64_PATTERN = %r{\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?\z} NETWORK_ERRORS = [ Timeout::Error, SocketError, SystemCallError, IOError, EOFError, OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, Zlib::Error ].freeze - private_constant :SEND_PATH, :STATUS_PATH, :US_APP_HOST, :EU_APP_HOST, :CLOUD_APP_HOSTS, + private_constant :SEND_PATH, :UPLOADS_PATH, :STATUS_PATH, :US_APP_HOST, :EU_APP_HOST, :CLOUD_APP_HOSTS, :MAX_RETRY_DELAY_SECONDS, :UUID_PATTERN, :IDEMPOTENCY_KEY_PATTERN, :TEMPLATE_KEY_PATTERN, - :HEADER_TOKEN_PATTERN, :SEND_KEY_NEEDED, :READ_KEY_NEEDED, :TEMPLATE_FORM_WHY, - :CODE_FORM_WHY, :TEMPLATE_FIELDS, :CODE_FIELDS, :ATTACHMENT_FIELDS, :BASE64_PATTERN, - :NETWORK_ERRORS + :HEADER_TOKEN_PATTERN, :JSON_CONTENT, :SEND_KEY_NEEDED, :READ_KEY_NEEDED, :TEMPLATE_FORM_WHY, + :CODE_FORM_WHY, :TEMPLATE_FIELDS, :CODE_FIELDS, :NETWORK_ERRORS # @api private def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_from:, user_agent:, disabled:, @@ -147,12 +147,14 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # most 10. An empty list sends no copy. # @param bcc [String, Array] Code form: one address or a list of addresses that get a hidden copy, at # most 10. An empty list sends no copy. - # @param attachments [Array] Code form: files sent with the email, at most 10 and 512 KB in total. Each - # file is a Hash with: + # @param attachments [Array] Code form: files sent with the email, at most 10 files and 40 MB per email + # after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't + # have yet straight to PostHog's file storage, then sends the email naming the files by their SHA-256. A file + # sent before by the same project uploads nothing again. Each file is a Hash with: # - +filename+ [String] the name the recipient sees, such as `invoice.pdf`; # - +content+ [String, Array] the file's bytes, as a binary String such as `File.binread(path)` - # returns or as a list of byte values, which the client encodes as Base64. Any other String is taken as the - # file already Base64 encoded. PostHog can't fetch a file from a path or URL; + # returns or as a list of byte values. Any other String is taken as the file encoded as Base64. PostHog + # can't fetch a file from a path or URL; # - +content_type+ [String] the MIME type, optional. Without it, the type follows the file name; # - +content_id+ [String] optional: shows the file inline where `html` has `` with this ID. # @param capture_engagement [Boolean] Code form: whether PostHog captures the email's opens and clicks as events @@ -167,7 +169,9 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # retries off. # @return [SendResult] The queued email. # @raise [EmailError] PostHog did not queue the email, or the client could not tell whether it did. - # {EmailError#code} is `invalid_attachment`, before any request, for a file the client can't send. + # Before any request, {EmailError#code} is `invalid_attachment` for a file the client can't send, + # `too_many_attachments` for more than 10 files, and `message_too_large` for files too large for one email. + # It is `attachment_upload_failed` when file storage refused an upload, and then no email was sent. # @raise [ArgumentError] An argument has the wrong type or format, or the call mixes the two forms. def send( to:, template_id: nil, template_data: nil, template_key: nil, subject: nil, html: nil, text: nil, from: nil, @@ -179,12 +183,14 @@ def send( cc: cc, bcc: bcc, attachments: attachments, capture_engagement: capture_engagement }.compact check_recipient(to, distinct_id, idempotency_key, max_retries) - payload = content(template_id, template_key, fields).merge(to: to) + payload, files = content(template_id, template_key, fields) + payload = payload.merge(to: to) payload[:distinct_id] = distinct_id unless distinct_id.nil? body = JSON.generate(payload) headers = { 'Content-Type' => 'application/json', 'Idempotency-Key' => idempotency_key || SecureRandom.uuid } result = call('no email was sent', SEND_KEY_NEEDED) do |secret_key| + upload(files, secret_key, max_retries || @max_retries) unless files.nil? || files.empty? uri = URI(@app_host + SEND_PATH) with_retries(max_retries || @max_retries) do parse_send_reply(request(Net::HTTP::Post.new(uri), uri, secret_key, headers, body)) @@ -303,7 +309,7 @@ def template_content(template_id, fields) template_data = fields.fetch(:template_data, {}) raise ArgumentError, 'template_data must be a Hash.' unless template_data.is_a?(Hash) - { template_id: template_id.downcase, message_data: isoify_dates(template_data) } + [{ template_id: template_id.downcase, message_data: isoify_dates(template_data) }, nil] end def code_content(template_key, fields) @@ -329,9 +335,12 @@ def code_content(template_key, fields) code = fields.slice(:subject, :html, :text, :from).merge(template_key: template_key) code[:reply_to] = addresses(:reply_to, fields[:reply_to]) if fields.key?(:reply_to) %i[cc bcc].each { |name| put_unless_empty(code, name, addresses(name, fields[name])) if fields.key?(name) } - put_unless_empty(code, :attachments, attachment_list(fields[:attachments])) if fields.key?(:attachments) + if fields.key?(:attachments) + files = EmailAttachments.from(fields[:attachments], fields.values_at(:html, :text).compact) + put_unless_empty(code, :attachments, files.references) + end code[:capture_engagement] = fields.fetch(:capture_engagement, true) - code + [code, files] end def refuse_fields(fields, allowed:, form:, other_form:, why:) @@ -353,71 +362,6 @@ def addresses(name, value) raise ArgumentError, "#{name} must be one address or a list of addresses." end - def attachment_list(files) - raise invalid_attachment('attachments must be a list of files.', 'attachments') unless files.is_a?(Array) - - files.each_with_index.map { |file, index| attachment(file, "attachments__#{index}") } - end - - def attachment(file, field) - unless file.is_a?(Hash) - raise invalid_attachment("An attachment is a Hash with filename and content. Pass the file's bytes as its " \ - 'content.', field) - end - - file = file.transform_keys(&:to_s).compact - if file.key?('path') - raise invalid_attachment("PostHog can't fetch an attachment from a path or URL. Read the file with " \ - 'File.binread and pass it as content instead.', "#{field}__path") - end - unknown = file.keys.find { |key| !ATTACHMENT_FIELDS.include?(key) } - if unknown - raise invalid_attachment("An attachment takes filename, content, content_type and content_id, not #{unknown}.", - "#{field}__#{unknown}") - end - - %w[filename content_type content_id].each do |name| - next if name != 'filename' && !file.key?(name) - next if file[name].is_a?(String) - - raise invalid_attachment("An attachment's #{name} must be a String.", "#{field}__#{name}") - end - file.merge('content' => base64_content(file['content'], "#{field}__content")) - end - - def base64_content(content, field) - bytes = file_bytes(content) - return [bytes].pack('m0') if bytes - unless content.is_a?(String) - raise invalid_attachment('An attachment needs content: the bytes File.binread returns, a list of byte ' \ - 'values, or the file encoded as Base64.', field) - end - return content if single_line_base64?(content) - return content.delete("\n") if encode64_layout?(content) - - raise invalid_attachment("An attachment's content is text that isn't Base64. Pass the file's bytes, read " \ - 'with File.binread, or the file encoded as Base64.', field) - end - - def single_line_base64?(content) - content.ascii_only? && BASE64_PATTERN.match?(content) - end - - def encode64_layout?(content) - content.ascii_only? && content.include?("\n") && [content.unpack1('m')].pack('m') == content - end - - def file_bytes(content) - return content if content.is_a?(String) && content.encoding == Encoding::BINARY - return unless content.is_a?(Array) && content.all? { |byte| byte.is_a?(Integer) && byte.between?(0, 255) } - - content.pack('C*') - end - - def invalid_attachment(message, field) - EmailError.new(code: 'invalid_attachment', message: message, field: field) - end - def call(nothing_done, key_needed) admit(nothing_done) begin @@ -497,16 +441,76 @@ def retryable?(error) case error.status_code when nil then error.code == 'network_error' when 409 then error.code == 'idempotency_key_in_flight' - when 429 then error.code != 'quota_limited' + when 429 then !%w[quota_limited upload_quota_exceeded].include?(error.code) else error.status_code == 503 end end + def upload(files, secret_key, max_retries) + distinct = files.distinct + uri = URI(@app_host + UPLOADS_PATH) + body = JSON.generate(files: distinct.map { |file| { sha256: file.sha256, size: file.bytes.bytesize } }) + slots = with_retries(max_retries) do + parse_uploads_reply(request(Net::HTTP::Post.new(uri), uri, secret_key, JSON_CONTENT, body), distinct) + end + distinct.zip(slots).each { |file, slot| put_file(file, slot, max_retries) if slot['status'] == 'upload' } + end + + def parse_uploads_reply((status, payload, retry_after), files) + fields = payload.is_a?(Hash) ? payload : {} + raise reply_error(fields, status, retry_after) unless status.between?(200, 299) + + slots = fields['files'] + return slots if slots.is_a?(Array) && slots.size == files.size && files.zip(slots).all? { |pair| slot?(*pair) } + + raise EmailError.new(code: 'api_error', message: "PostHog answered without the files' uploads.", + status_code: status) + end + + def slot?(file, slot) + return false unless slot.is_a?(Hash) && slot['sha256'] == file.sha256 + return slot['status'] == 'present' unless slot['status'] == 'upload' + + slot['method'] == 'PUT' && web_url?(slot['url'].to_s) && slot['headers'].is_a?(Hash) && + slot['headers'].all? { |name, value| name.is_a?(String) && value.is_a?(String) } + end + + def put_file(file, slot, max_retries) + uri = URI(slot['url']) + with_retries(max_retries) do + request = Net::HTTP::Put.new(uri) + request['Content-Type'] = 'application/octet-stream' + status, answer = exchange(request, uri, slot['headers'], file.bytes) + raise upload_failed(file, status, answer) unless status.between?(200, 299) + end + rescue EmailError => e + raise unless e.code == 'network_error' + + raise EmailError.new(code: 'network_error', field: file.field, + message: "The upload of #{file.reference['filename']} to file storage failed, so no " \ + 'email was sent.') + end + + def upload_failed(file, status, answer) + answer = answer.to_s.dup.force_encoding(Encoding::UTF_8).scrub + storage_said = [answer[%r{([^<]*)}, 1], answer[%r{([^<]*)}, 1]].compact + EmailError.new( + code: 'attachment_upload_failed', status_code: status, field: file.field, + message: "File storage refused the upload of #{file.reference['filename']} with HTTP #{status}" \ + "#{" (#{storage_said.join(': ')})" unless storage_said.empty?}, so no email was sent." + ) + end + def request(request, uri, secret_key, headers, body = nil) request['Authorization'] = "Bearer #{secret_key}" request['Accept'] = 'application/json' request['User-Agent'] = @user_agent if @user_agent request['X-PostHog-Migrated-From'] = @migrated_from if @migrated_from + status, answer, retry_after = exchange(request, uri, headers, body) + [status, parse_json(answer), retry_after] + end + + def exchange(request, uri, headers, body) headers.each { |name, value| request[name] = value } request.body = body if body @@ -514,7 +518,7 @@ def request(request, uri, secret_key, headers, body = nil) response = Net::HTTP.start(uri.hostname, uri.port, connection_options(uri)) do |http| http.request(request) { |started| head = started } end - [response.code.to_i, parse_json(response.body), parse_retry_after(response['Retry-After'])] + [response.code.to_i, response.body, parse_retry_after(response['Retry-After'])] rescue *NETWORK_ERRORS # A reply whose body broke off retries with the same idempotency key, after its Retry-After when it sent one. raise EmailError.new(code: 'network_error', message: 'The request to PostHog failed.', diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 4609c8c2..3a3d2e14 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -199,7 +199,30 @@ def send_code(**overrides) end describe 'copies and attachments' do + let(:uploads_url) { 'https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/' } + let(:upload_url) { 'https://files.example.com/team/99882c55?X-Amz-Signature=abc' } let(:pdf) { "%PDF-1.7\n\xFF\x00\xE2".b } + let(:pdf_sha256) { '99882c5503918dac954710e17022f6243565344d43c0f30ac70d19c560f857c4' } + let(:png) { "\x89PNG".b } + let(:png_sha256) { '0f4636c78f65d3639ece5a064b5ae753e3408614a14fb18ab4d7540d2c248543' } + let(:five_pdfs_sha256) { 'dfb00ae2891aa7dac580fd90519992e7a8d6914de7a683c0ecc071f65868101e' } + let(:largest_that_fits_sha256) { '28f197eae8bd57e7c2a94d2062d2cef0a04b34e7fd3e0c743d88cda2217f069c' } + let(:upload_headers) do + { 'Content-Length' => '12', 'x-amz-checksum-sha256' => 'mYgsVQORjayVRxDhcCL2JDVlNE1DwPMKxw0ZxWD4V8Q=' } + end + + def upload_slot(sha256, url: upload_url) + { sha256: sha256, status: 'upload', url: url, method: 'PUT', headers: upload_headers, + url_expires_at: '2026-11-09T12:15:00Z' } + end + + def present_slot(sha256) + { sha256: sha256, status: 'present', expires_at: '2026-12-16T12:00:00Z' } + end + + def answer_uploads(*slots) + { status: 200, body: JSON.generate(files: slots) } + end def refused_attachment(**overrides) send_code(**overrides) @@ -216,24 +239,29 @@ def refused_attachment(**overrides) 'bcc' => %w[audit@example.com archive@example.com]) end - it 'sends the bytes of each file as Base64, with its type and content ID' do + it 'uploads each file PostHog asks for to its URL, without the secret key, then sends the files by hash' do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256), present_slot(png_sha256))) + stub_request(:put, upload_url).to_return(status: 200) stub_request(:post, send_url).to_return(accept) send_code(attachments: [ { filename: 'invoice.pdf', content: pdf, content_type: 'application/pdf' }, - { 'filename' => 'logo.png', 'content' => [137, 80, 78, 71], 'content_id' => 'logo' }, - { filename: 'notes.txt', content: 'aGVsbG8=' } + { 'filename' => 'logo.png', 'content' => png, 'content_id' => 'logo' } ]) + expect(a_request(:post, uploads_url).with( + headers: { 'Authorization' => 'Bearer phs_test' }, + body: { files: [{ sha256: pdf_sha256, size: 12 }, { sha256: png_sha256, size: 4 }] }.to_json + )).to have_been_made.once + expect(a_request(:put, upload_url).with(body: pdf, headers: upload_headers) do |request| + !request.headers.key?('Authorization') + end).to have_been_made.once expect(sent_bodies.first['attachments']).to eq([ - { 'filename' => 'invoice.pdf', - 'content' => 'JVBERi0xLjcK/wDi', + { 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf', 'content_type' => 'application/pdf' }, - { 'filename' => 'logo.png', 'content' => 'iVBORw==', - 'content_id' => 'logo' }, - { 'filename' => 'notes.txt', 'content' => 'aGVsbG8=' } + { 'sha256' => png_sha256, 'filename' => 'logo.png', + 'content_id' => 'logo' } ]) - expect(sent_bodies.first['attachments'][0]['content'].unpack1('m0')).to eq(pdf) end it 'leaves out empty lists, so a PostHog version without these fields still sends the email' do @@ -244,24 +272,149 @@ def refused_attachment(**overrides) expect(sent_bodies.first.keys).not_to include('cc', 'bcc', 'attachments') end - it 'sends the same encoded files on every attempt' do + it 'uploads once and sends the same files on every attempt of the send' do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_return(status: 200) stub_request(:post, send_url).to_return(refuse(503, 'service_unavailable'), accept) send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) - encoded_once = [{ 'filename' => 'invoice.pdf', 'content' => 'JVBERi0xLjcK/wDi' }] - expect(a_request(:post, send_url).with { |request| JSON.parse(request.body)['attachments'] == encoded_once }) - .to have_been_made.twice + expect(a_request(:put, upload_url)).to have_been_made.once + expect(a_request(:post, send_url).with do |request| + JSON.parse(request.body)['attachments'] == [{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }] + end).to have_been_made.twice end - it 'takes Base64 with line breaks, as Base64.encode64 writes it' do + it 'takes the file as bytes, a list of byte values, or Base64 with or without line breaks' do + file = pdf * 5 + stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(five_pdfs_sha256))) stub_request(:post, send_url).to_return(accept) - file = (0..99).to_a.pack('C*') - send_code(attachments: [{ filename: 'data.bin', content: [file].pack('m') }]) + [file, file.bytes, [file].pack('m0'), [file].pack('m')].each do |content| + send_code(attachments: [{ filename: 'invoice.pdf', content: content }]) + end expect([file].pack('m')).to include("\n") - expect(sent_bodies.first['attachments'][0]['content']).to eq([file].pack('m0')) + expect(a_request(:post, uploads_url).with(body: { files: [{ sha256: five_pdfs_sha256, size: 60 }] }.to_json)) + .to have_been_made.times(4) + end + + it 'uploads a file attached twice once, and sends both references' do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_return(status: 200) + stub_request(:post, send_url).to_return(accept) + + send_code(attachments: [{ filename: 'a.pdf', content: pdf }, { filename: 'b.pdf', content: pdf }]) + + expect(a_request(:post, uploads_url).with(body: { files: [{ sha256: pdf_sha256, size: 12 }] }.to_json)) + .to have_been_made.once + expect(a_request(:put, upload_url)).to have_been_made.once + expect(sent_bodies.first['attachments'].map { |file| file['filename'] }).to eq(%w[a.pdf b.pdf]) + end + + it 'sends nothing when storage refuses an upload, and says what storage answered' do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_return( + status: 400, + body: 'BadDigest' \ + 'The checksum did not match.' + ) + + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.status_code, error.field]) + .to eq(['attachment_upload_failed', 400, 'attachments__0']) + expect(error.message).to eq('File storage refused the upload of invoice.pdf with HTTP 400 ' \ + '(BadDigest: The checksum did not match.), so no email was sent.') + expect(a_request(:post, send_url)).not_to have_been_made + end + + it 'names the file storage and the file when an upload never reaches it' do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_timeout + + error = refused_attachment(max_retries: 0, attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.name, error.field]).to eq(%w[network_error service_unavailable attachments__0]) + expect(error.message).to eq('The upload of invoice.pdf to file storage failed, so no email was sent.') + expect(a_request(:post, send_url)).not_to have_been_made + end + + it "retries an upload storage couldn't take, then sends" do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_return({ status: 503, body: 'SlowDown' }, + { status: 200 }) + stub_request(:post, send_url).to_return(accept) + + expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') + expect(a_request(:put, upload_url)).to have_been_made.twice + expect(delays.size).to eq(1) + end + + it "sends nothing when PostHog refuses the uploads, and passes on PostHog's reason" do + stub_request(:post, uploads_url).to_return( + refuse(400, 'field_not_accepted', detail: "This project can't send files yet.", attr: 'files', + why: 'Not available yet.', fix: 'Send without files.') + ) + + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.name, error.field, error.why, error.fix]) + .to eq(['field_not_accepted', 'validation_error', 'files', 'Not available yet.', 'Send without files.']) + expect(a_request(:put, /files.example.com/)).not_to have_been_made + expect(a_request(:post, send_url)).not_to have_been_made + end + + it "doesn't retry a project over its daily upload bytes" do + stub_request(:post, uploads_url).to_return(refuse(429, 'upload_quota_exceeded')) + + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect(error.code).to eq('upload_quota_exceeded') + expect(a_request(:post, uploads_url)).to have_been_made.once + end + + it "sends nothing when PostHog's answer doesn't say where each file goes" do + answers = [ + answer_uploads, + answer_uploads(present_slot(png_sha256)), + answer_uploads(upload_slot(pdf_sha256).merge(method: 'POST')), + answer_uploads(upload_slot(pdf_sha256, url: 'file:///etc/passwd')), + answer_uploads(upload_slot(pdf_sha256).merge(headers: { 'Content-Length' => 12 })) + ] + stub_request(:post, uploads_url).to_return(*answers) + + errors = answers.map { refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) } + + expect(errors.map(&:code)).to all(eq('api_error')) + expect(a_request(:put, /./)).not_to have_been_made + expect(a_request(:post, send_url)).not_to have_been_made + end + + it 'refuses more files or bytes than one email carries before any request' do + eleven = Array.new(11) { |index| { filename: "#{index}.txt", content: "file #{index}".b } } + largest_that_fits = ("\0" * 29_182_875).b + + too_many = refused_attachment(attachments: eleven) + too_large = refused_attachment(attachments: [{ filename: 'scan.pdf', content: "#{largest_that_fits}\0".b }]) + too_large_with_body = refused_attachment(html: 'abcd', + attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]) + + expect([too_many.code, too_many.name, too_many.field]) + .to eq(%w[too_many_attachments validation_error attachments]) + expect([too_large, too_large_with_body].map { |error| [error.code, error.name, error.field] }) + .to all(eq(%w[message_too_large validation_error attachments])) + expect(too_large.fix).to include('40 MB per email after Base64 encoding, about 29 MB of files') + expect(a_request(:any, %r{/messaging/})).not_to have_been_made + end + + it 'uploads the largest file one email carries' do + largest_that_fits = ("\0" * 29_182_875).b + stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) + stub_request(:post, send_url).to_return(accept) + + expect(send_code(html: '', attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]).id) + .to eq('msg_1') end it 'refuses a file it cannot send before any request, naming the field and never the content' do @@ -273,6 +426,7 @@ def refused_attachment(**overrides) ['attachments__0__headers', /not headers/], { attachments: [{ filename: 'a.pdf' }] } => ['attachments__0__content', /needs content/], { attachments: [{ filename: 'a.pdf', content: [256] }] } => ['attachments__0__content', /needs content/], + { attachments: [{ filename: 'a.pdf', content: ''.b }] } => ['attachments__0__content', /can't be empty/], { attachments: [{ filename: 'a.pdf', content: 'secret text, not Base64' }] } => ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: 'a.pdf', content: "%PDF secret text \xE2\xE3".dup.force_encoding('UTF-8') }] } => From d94b54119eed191364dd3adf2ae1c636954a1336 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:22:28 +0000 Subject: [PATCH 12/31] refactor(emails): build the files beside the payload, decode Base64 strictly Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 6 ++---- lib/posthog/emails.rb | 15 ++++++--------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index 6d327db3..9d602afd 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -14,7 +14,6 @@ class EmailAttachments HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 LIMIT = '10 files and 40 MB per email after Base64 encoding, about 29 MB of files' FIELDS = %w[filename content content_type content_id].freeze - BASE64_PATTERN = %r{\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?\z} # One file to attach: its bytes, their SHA-256, and what the send says about it. Attachment = Struct.new(:bytes, :sha256, :reference, :field, keyword_init: true) @@ -78,9 +77,8 @@ def self.base64_bytes(content, field) raise invalid('An attachment needs content: the bytes File.binread returns, a list of byte values, or the ' \ 'file encoded as Base64.', field) end - return content.unpack1('m0') if content.ascii_only? && BASE64_PATTERN.match?(content) - return content.unpack1('m') if encode64_layout?(content) - + encode64_layout?(content) ? content.unpack1('m') : content.unpack1('m0') + rescue ArgumentError raise invalid("An attachment's content is text that isn't Base64. Pass the file's bytes, read with " \ 'File.binread, or the file encoded as Base64.', field) end diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index a8ab7a34..ba33fe28 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -183,14 +183,15 @@ def send( cc: cc, bcc: bcc, attachments: attachments, capture_engagement: capture_engagement }.compact check_recipient(to, distinct_id, idempotency_key, max_retries) - payload, files = content(template_id, template_key, fields) - payload = payload.merge(to: to) + payload = content(template_id, template_key, fields).merge(to: to) + files = EmailAttachments.from(fields.fetch(:attachments, []), fields.values_at(:html, :text).compact) + payload[:attachments] = files.references unless files.empty? payload[:distinct_id] = distinct_id unless distinct_id.nil? body = JSON.generate(payload) headers = { 'Content-Type' => 'application/json', 'Idempotency-Key' => idempotency_key || SecureRandom.uuid } result = call('no email was sent', SEND_KEY_NEEDED) do |secret_key| - upload(files, secret_key, max_retries || @max_retries) unless files.nil? || files.empty? + upload(files, secret_key, max_retries || @max_retries) unless files.empty? uri = URI(@app_host + SEND_PATH) with_retries(max_retries || @max_retries) do parse_send_reply(request(Net::HTTP::Post.new(uri), uri, secret_key, headers, body)) @@ -309,7 +310,7 @@ def template_content(template_id, fields) template_data = fields.fetch(:template_data, {}) raise ArgumentError, 'template_data must be a Hash.' unless template_data.is_a?(Hash) - [{ template_id: template_id.downcase, message_data: isoify_dates(template_data) }, nil] + { template_id: template_id.downcase, message_data: isoify_dates(template_data) } end def code_content(template_key, fields) @@ -335,12 +336,8 @@ def code_content(template_key, fields) code = fields.slice(:subject, :html, :text, :from).merge(template_key: template_key) code[:reply_to] = addresses(:reply_to, fields[:reply_to]) if fields.key?(:reply_to) %i[cc bcc].each { |name| put_unless_empty(code, name, addresses(name, fields[name])) if fields.key?(name) } - if fields.key?(:attachments) - files = EmailAttachments.from(fields[:attachments], fields.values_at(:html, :text).compact) - put_unless_empty(code, :attachments, files.references) - end code[:capture_engagement] = fields.fetch(:capture_engagement, true) - [code, files] + code end def refuse_fields(fields, allowed:, form:, other_form:, why:) From c4d93e8e9c8e78a8dcfd97cd0d0a42ebd59b50e1 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:36:11 +0000 Subject: [PATCH 13/31] fix(emails): wait for file storage's Retry-After before retrying an upload Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 8 ++++---- spec/posthog/emails_spec.rb | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index ba33fe28..e656eaff 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -477,8 +477,8 @@ def put_file(file, slot, max_retries) with_retries(max_retries) do request = Net::HTTP::Put.new(uri) request['Content-Type'] = 'application/octet-stream' - status, answer = exchange(request, uri, slot['headers'], file.bytes) - raise upload_failed(file, status, answer) unless status.between?(200, 299) + status, answer, retry_after = exchange(request, uri, slot['headers'], file.bytes) + raise upload_failed(file, status, answer, retry_after) unless status.between?(200, 299) end rescue EmailError => e raise unless e.code == 'network_error' @@ -488,11 +488,11 @@ def put_file(file, slot, max_retries) 'email was sent.') end - def upload_failed(file, status, answer) + def upload_failed(file, status, answer, retry_after) answer = answer.to_s.dup.force_encoding(Encoding::UTF_8).scrub storage_said = [answer[%r{([^<]*)}, 1], answer[%r{([^<]*)}, 1]].compact EmailError.new( - code: 'attachment_upload_failed', status_code: status, field: file.field, + code: 'attachment_upload_failed', status_code: status, field: file.field, retry_after: retry_after, message: "File storage refused the upload of #{file.reference['filename']} with HTTP #{status}" \ "#{" (#{storage_said.join(': ')})" unless storage_said.empty?}, so no email was sent." ) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 3a3d2e14..3b4f849c 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -351,6 +351,22 @@ def refused_attachment(**overrides) expect(delays.size).to eq(1) end + it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_return({ status: 503, headers: { 'Retry-After' => '3' } }, + { status: 200 }, + { status: 503, headers: { 'Retry-After' => '60' } }) + stub_request(:post, send_url).to_return(accept) + + send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect(delays.size).to eq(1) + expect(delays.first).to be >= 3 + expect([error.code, error.retry_after]).to eq(['attachment_upload_failed', 60.0]) + expect(a_request(:put, upload_url)).to have_been_made.times(3) + end + it "sends nothing when PostHog refuses the uploads, and passes on PostHog's reason" do stub_request(:post, uploads_url).to_return( refuse(400, 'field_not_accepted', detail: "This project can't send files yet.", attr: 'files', @@ -361,7 +377,7 @@ def refused_attachment(**overrides) expect([error.code, error.name, error.field, error.why, error.fix]) .to eq(['field_not_accepted', 'validation_error', 'files', 'Not available yet.', 'Send without files.']) - expect(a_request(:put, /files.example.com/)).not_to have_been_made + expect(a_request(:put, upload_url)).not_to have_been_made expect(a_request(:post, send_url)).not_to have_been_made end From 3cea49b5c5f31f199e6afc50bcbfc8a39ad18a48 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:40:44 +0000 Subject: [PATCH 14/31] fix(emails): count the files before reading them, hash only files that fit Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 45 ++++++++++++++++++-------------- spec/posthog/emails_spec.rb | 8 ++++++ 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index 9d602afd..c7a95c5c 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -16,13 +16,18 @@ class EmailAttachments FIELDS = %w[filename content content_type content_id].freeze # One file to attach: its bytes, their SHA-256, and what the send says about it. - Attachment = Struct.new(:bytes, :sha256, :reference, :field, keyword_init: true) + Attachment = Struct.new(:bytes, :reference, :field, keyword_init: true) do + def sha256 + @sha256 ||= Digest::SHA256.hexdigest(bytes) + end + end # @param files [Object] What the caller passed as `attachments`. # @param bodies [Array] The email's HTML and text bodies, which count towards its size. # @raise [EmailError] A file the client can't send, or more than one email can carry. def self.from(files, bodies) raise invalid('attachments must be a list of files.', 'attachments') unless files.is_a?(Array) + raise too_many_files if files.size > MAX_FILES new(files.each_with_index.map { |file, index| attachment(file, "attachments__#{index}") }, bodies) end @@ -35,8 +40,7 @@ def self.attachment(file, field) file = file.transform_keys(&:to_s).compact check_keys(file, field) bytes = file_bytes(file['content'], "#{field}__content") - Attachment.new(bytes: bytes, sha256: Digest::SHA256.hexdigest(bytes), field: field, - reference: file.slice('filename', 'content_type', 'content_id')) + Attachment.new(bytes: bytes, field: field, reference: file.slice('filename', 'content_type', 'content_id')) end def self.check_keys(file, field) @@ -91,12 +95,19 @@ def self.invalid(message, field) EmailError.new(code: 'invalid_attachment', message: message, field: field) end + def self.too_many_files + EmailError.new(code: 'too_many_attachments', field: 'attachments', + message: "An email carries at most #{MAX_FILES} files, so no email was sent.", + why: "An email carries at most #{MAX_FILES} files.", + fix: "Attach at most #{MAX_FILES} files, and link to any others from the email body.") + end + private_class_method :new, :attachment, :check_keys, :file_bytes, :raw_bytes, :base64_bytes, - :encode64_layout?, :invalid + :encode64_layout?, :invalid, :too_many_files def initialize(attachments, bodies) @attachments = attachments - check_fits(bodies) + check_size(bodies) end def empty? @@ -115,19 +126,17 @@ def distinct private - def check_fits(bodies) - if @attachments.size > MAX_FILES - raise refusal('too_many_attachments', "An email carries at most #{MAX_FILES} files, so no email was sent.", - why: "An email carries at most #{MAX_FILES} files.", - fix: "Attach at most #{MAX_FILES} files, and link to any others from the email body.") - end + def check_size(bodies) return if message_bytes(bodies) <= MAX_MESSAGE_BYTES - raise refusal('message_too_large', 'The files are too large for one email, so no email was sent.', - why: "PostHog's email provider takes emails up to 40 MB, and Base64 encoding makes each file " \ - 'about a third larger.', - fix: "Keep the email under #{LIMIT}: attach fewer or smaller files, or link to large files " \ - 'from the email body.') + raise EmailError.new( + code: 'message_too_large', field: 'attachments', + message: 'The files are too large for one email, so no email was sent.', + why: "PostHog's email provider takes emails up to 40 MB, and Base64 encoding makes each file about a " \ + 'third larger.', + fix: "Keep the email under #{LIMIT}: attach fewer or smaller files, or link to large files from the email " \ + 'body.' + ) end def message_bytes(bodies) @@ -138,9 +147,5 @@ def message_bytes(bodies) def encoded_bytes(size) (((size + 2) / 3 * 4 * 78) + 75) / 76 end - - def refusal(code, message, why:, fix:) - EmailError.new(code: code, message: message, field: 'attachments', why: why, fix: fix) - end end end diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 3b4f849c..acc20ea6 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -424,6 +424,14 @@ def refused_attachment(**overrides) expect(a_request(:any, %r{/messaging/})).not_to have_been_made end + it 'counts the files before it reads any of them' do + files = Array.new(10) { |index| { filename: "#{index}.txt", content: "file #{index}".b } } + + error = refused_attachment(attachments: files + [{ filename: 'scan.pdf', path: '/tmp/scan.pdf' }]) + + expect(error.code).to eq('too_many_attachments') + end + it 'uploads the largest file one email carries' do largest_that_fits = ("\0" * 29_182_875).b stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) From 03b4d0b7942d619fafdc3d38022e81994019cf95 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:49:54 +0000 Subject: [PATCH 15/31] fix(emails): keep storage's Retry-After when its upload answer breaks off A PUT whose answer broke off raised a network error naming the file but dropped the Retry-After storage had sent, so callers couldn't wait it out. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/emails.rb | 2 +- spec/posthog/emails_spec.rb | 26 ++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index e656eaff..c518e288 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -483,7 +483,7 @@ def put_file(file, slot, max_retries) rescue EmailError => e raise unless e.code == 'network_error' - raise EmailError.new(code: 'network_error', field: file.field, + raise EmailError.new(code: 'network_error', field: file.field, retry_after: e.retry_after, message: "The upload of #{file.reference['filename']} to file storage failed, so no " \ 'email was sent.') end diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index acc20ea6..beed40c9 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -36,21 +36,23 @@ def sent_bodies bodies end - # Serves each raw HTTP response once, in order, on a local socket, and points the client's app host at it. WebMock - # is off meanwhile: its Net::HTTP wrapper reads the whole body before it hands over the response. + # Serves each raw HTTP response once, in order, on a local socket, and points the client's app host at it. A + # response can be a lambda that takes the socket's URL. WebMock is off meanwhile: its Net::HTTP wrapper reads the + # whole body before it hands over the response. def serve(*responses, tls: nil) server = TCPServer.new('127.0.0.1', 0) listener = tls ? OpenSSL::SSL::SSLServer.new(server, tls) : server + base_url = "#{tls ? 'https' : 'http'}://127.0.0.1:#{server.addr[1]}" responder = Thread.new do Thread.current.report_on_exception = false responses.each do |response| connection = listener.accept read_request(connection) - connection.write(response) + connection.write(response.respond_to?(:call) ? response.call(base_url) : response) connection.close end end - options.merge!(host: 'http://127.0.0.1:9', app_host: "#{tls ? 'https' : 'http'}://127.0.0.1:#{server.addr[1]}") + options.merge!(host: 'http://127.0.0.1:9', app_host: base_url) WebMock.disable! yield expect(responder.join(5)).to be_truthy @@ -367,6 +369,22 @@ def refused_attachment(**overrides) expect(a_request(:put, upload_url)).to have_been_made.times(3) end + it "keeps storage's Retry-After when its answer to an upload was cut off" do + uploads = lambda do |base_url| + slot = upload_slot(pdf_sha256, url: "#{base_url}/team/#{pdf_sha256}") + http_response('200 OK', JSON.generate(files: [slot])) + end + cut_off = http_response('503 Service Unavailable', 'Slow', + 'Content-Length' => '60', 'Retry-After' => '60') + + serve(uploads, cut_off) do + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.field, error.retry_after]).to eq(['network_error', 'attachments__0', 60.0]) + expect(error.message).to eq('The upload of invoice.pdf to file storage failed, so no email was sent.') + end + end + it "sends nothing when PostHog refuses the uploads, and passes on PostHog's reason" do stub_request(:post, uploads_url).to_return( refuse(400, 'field_not_accepted', detail: "This project can't send files yet.", attr: 'files', From e56f70c07277b76dee956c8fd1ff0de9a3b78763 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:55:45 +0000 Subject: [PATCH 16/31] fix(emails): count each body's UTF-8 bytes towards the email's size The send carries the bodies as JSON in UTF-8, so a UTF-16 body counted twice its sent size and refused emails the server takes. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 10 +++++++++- spec/posthog/emails_spec.rb | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index c7a95c5c..4d17d093 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -141,7 +141,15 @@ def check_size(bodies) def message_bytes(bodies) files = @attachments.sum { |attachment| encoded_bytes(attachment.bytes.bytesize) } - files + bodies.sum(&:bytesize) + HEADERS_AND_MIME_BOUNDARIES_BYTES + files + bodies.sum { |body| utf8_bytesize(body) } + HEADERS_AND_MIME_BOUNDARIES_BYTES + end + + # The send carries each body as JSON, which transcodes it to UTF-8, so a UTF-16 body counts its UTF-8 bytes. + def utf8_bytesize(body) + body.encoding == Encoding::BINARY ? body.bytesize : body.encode(Encoding::UTF_8).bytesize + rescue EncodingError + # Text that can't become UTF-8 fails when the send builds its JSON; count its bytes as they are until then. + body.bytesize end def encoded_bytes(size) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index beed40c9..9288bca2 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -459,6 +459,20 @@ def refused_attachment(**overrides) .to eq('msg_1') end + it 'counts a body by the UTF-8 bytes it is sent as, whatever its encoding' do + largest_that_fits = ("\0" * 29_182_875).b + stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) + stub_request(:post, send_url).to_return(accept) + attachments = [{ filename: 'scan.pdf', content: largest_that_fits }] + + sent = send_code(html: 'abc'.encode(Encoding::UTF_16LE), attachments: attachments) + too_large = refused_attachment(html: 'abcd'.encode(Encoding::UTF_16LE), attachments: attachments) + + expect(sent.id).to eq('msg_1') + expect(sent_bodies.first['html']).to eq('abc') + expect(too_large.code).to eq('message_too_large') + end + it 'refuses a file it cannot send before any request, naming the field and never the content' do cases = { { attachments: { filename: 'a.pdf', content: pdf } } => ['attachments', /list of files/], From 592d442bb06b52980e15c0a76e7628011f4bc639 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:00:05 +0000 Subject: [PATCH 17/31] fix(emails): count each body like a file towards the email's size PostHog's check counts each body's UTF-8 bytes at their Base64 size with line breaks, as it does a file's. Counting raw bytes let a send pass here and be refused after its uploads. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 3 ++- spec/posthog/emails_spec.rb | 27 ++++++++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index 4d17d093..2790b823 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -141,7 +141,8 @@ def check_size(bodies) def message_bytes(bodies) files = @attachments.sum { |attachment| encoded_bytes(attachment.bytes.bytesize) } - files + bodies.sum { |body| utf8_bytesize(body) } + HEADERS_AND_MIME_BOUNDARIES_BYTES + # PostHog counts each body like a file, so a send that passes here passes its check after the uploads. + files + bodies.sum { |body| encoded_bytes(utf8_bytesize(body)) } + HEADERS_AND_MIME_BOUNDARIES_BYTES end # The send carries each body as JSON, which transcodes it to UTF-8, so a UTF-16 body counts its UTF-8 bytes. diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 9288bca2..21714b5d 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -459,18 +459,27 @@ def refused_attachment(**overrides) .to eq('msg_1') end - it 'counts a body by the UTF-8 bytes it is sent as, whatever its encoding' do - largest_that_fits = ("\0" * 29_182_875).b - stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) + it "counts each body like a file: its UTF-8 bytes' Base64 size with line breaks" do + stub_request(:post, uploads_url).to_return do |request| + answer_uploads(*JSON.parse(request.body)['files'].map { |file| present_slot(file['sha256']) }) + end stub_request(:post, send_url).to_return(accept) - attachments = [{ filename: 'scan.pdf', content: largest_that_fits }] + # Each file leaves room for 57 characters of body, but not for 58. + room_for_57_ascii = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] # 81 bytes left + room_for_57_two_byte = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] # 159 bytes left - sent = send_code(html: 'abc'.encode(Encoding::UTF_16LE), attachments: attachments) - too_large = refused_attachment(html: 'abcd'.encode(Encoding::UTF_16LE), attachments: attachments) + sent = [ + send_code(html: 'a' * 57, attachments: room_for_57_ascii), # counts 78 + send_code(html: '', text: ('é' * 57).encode(Encoding::UTF_16LE), attachments: room_for_57_two_byte) # 156 + ] + refused = [ + refused_attachment(html: 'a' * 58, attachments: room_for_57_ascii), # counts 83 + refused_attachment(html: '', text: 'é' * 58, attachments: room_for_57_two_byte) # counts 161 + ] - expect(sent.id).to eq('msg_1') - expect(sent_bodies.first['html']).to eq('abc') - expect(too_large.code).to eq('message_too_large') + expect(refused.map(&:code)).to eq(%w[message_too_large message_too_large]) + expect(sent.map(&:id)).to eq(%w[msg_1 msg_1]) + expect(sent_bodies.map { |body| body['text'] }).to eq([nil, 'é' * 57]) end it 'refuses a file it cannot send before any request, naming the field and never the content' do From 7db66ddc4a343ec6a1393c6d0a30d55d93d55edd Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:26:27 +0000 Subject: [PATCH 18/31] fix(emails): answer QA panel findings and the upload-route 404 contract - A 404 from the upload route raises not_found on attachments with PostHog's own reason, and sends nothing (spec #71, Compatibility). - Base64 content wrapped at 60, 64 or 76 characters, with LF or CRLF, decodes; text in any other line layout is still refused. - A refused upload is named by storage's failure (service_unavailable for 5xx, else application_error), not by PostHog's categories. - Inline cc/bcc assignment; drop inline comments for names. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-attachments.md | 2 +- lib/posthog/email_attachments.rb | 18 ++++++----- lib/posthog/email_error.rb | 3 +- lib/posthog/emails.rb | 17 +++++----- spec/posthog/emails_spec.rb | 55 +++++++++++++++++++++++++------- 5 files changed, 66 insertions(+), 29 deletions(-) diff --git a/.changeset/emails-attachments.md b/.changeset/emails-attachments.md index b29514e5..04fd4f91 100644 --- a/.changeset/emails-attachments.md +++ b/.changeset/emails-attachments.md @@ -2,4 +2,4 @@ "posthog-ruby": minor --- -`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and no email is sent. +`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index 2790b823..7ba6d261 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -14,6 +14,7 @@ class EmailAttachments HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 LIMIT = '10 files and 40 MB per email after Base64 encoding, about 29 MB of files' FIELDS = %w[filename content content_type content_id].freeze + BASE64_LINE_WIDTHS = [60, 64, 76].freeze # One file to attach: its bytes, their SHA-256, and what the send says about it. Attachment = Struct.new(:bytes, :reference, :field, keyword_init: true) do @@ -81,14 +82,18 @@ def self.base64_bytes(content, field) raise invalid('An attachment needs content: the bytes File.binread returns, a list of byte values, or the ' \ 'file encoded as Base64.', field) end - encode64_layout?(content) ? content.unpack1('m') : content.unpack1('m0') - rescue ArgumentError + bytes = content.delete("\r\n").unpack1('m0') + raise ArgumentError if content.include?("\n") && !wrapped_base64?(content, bytes) + + bytes + rescue ArgumentError, EncodingError raise invalid("An attachment's content is text that isn't Base64. Pass the file's bytes, read with " \ 'File.binread, or the file encoded as Base64.', field) end - def self.encode64_layout?(content) - content.ascii_only? && content.include?("\n") && [content.unpack1('m')].pack('m') == content + def self.wrapped_base64?(content, bytes) + encoded = [bytes].pack('m0') + BASE64_LINE_WIDTHS.any? { |width| encoded.scan(/.{1,#{width}}/).join("\n") == content.delete("\r").chomp } end def self.invalid(message, field) @@ -103,7 +108,7 @@ def self.too_many_files end private_class_method :new, :attachment, :check_keys, :file_bytes, :raw_bytes, :base64_bytes, - :encode64_layout?, :invalid, :too_many_files + :wrapped_base64?, :invalid, :too_many_files def initialize(attachments, bodies) @attachments = attachments @@ -141,15 +146,12 @@ def check_size(bodies) def message_bytes(bodies) files = @attachments.sum { |attachment| encoded_bytes(attachment.bytes.bytesize) } - # PostHog counts each body like a file, so a send that passes here passes its check after the uploads. files + bodies.sum { |body| encoded_bytes(utf8_bytesize(body)) } + HEADERS_AND_MIME_BOUNDARIES_BYTES end - # The send carries each body as JSON, which transcodes it to UTF-8, so a UTF-16 body counts its UTF-8 bytes. def utf8_bytesize(body) body.encoding == Encoding::BINARY ? body.bytesize : body.encode(Encoding::UTF_8).bytesize rescue EncodingError - # Text that can't become UTF-8 fails when the send builds its JSON; count its bytes as they are until then. body.bytesize end diff --git a/lib/posthog/email_error.rb b/lib/posthog/email_error.rb index 82a5f988..e55f51e5 100644 --- a/lib/posthog/email_error.rb +++ b/lib/posthog/email_error.rb @@ -11,7 +11,7 @@ class EmailError < StandardError attr_reader :name # @return [String] PostHog's error code, such as `quota_limited`, or an SDK code such as `network_error`. attr_reader :code - # @return [Integer, nil] The HTTP status, when PostHog answered. + # @return [Integer, nil] The HTTP status, when PostHog answered, or file storage's for `attachment_upload_failed`. attr_reader :status_code # @return [String, nil] Why PostHog refused the email. attr_reader :why @@ -48,6 +48,7 @@ def initialize(code:, message:, status_code: nil, why: nil, fix: nil, docs_url: # @api private def self.name_for(status_code, code) return 'missing_api_key' if code == 'missing_secret_key' || status_code == 401 + return status_code.to_i >= 500 ? 'service_unavailable' : 'application_error' if code == 'attachment_upload_failed' return SDK_ERROR_NAMES.fetch(code, 'application_error') if status_code.nil? case status_code diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index c518e288..e2e1e823 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -153,8 +153,8 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # sent before by the same project uploads nothing again. Each file is a Hash with: # - +filename+ [String] the name the recipient sees, such as `invoice.pdf`; # - +content+ [String, Array] the file's bytes, as a binary String such as `File.binread(path)` - # returns or as a list of byte values. Any other String is taken as the file encoded as Base64. PostHog - # can't fetch a file from a path or URL; + # returns or as a list of byte values. Any other String is taken as the file encoded as Base64, on one + # line or wrapped at 60, 64 or 76 characters. PostHog can't fetch a file from a path or URL; # - +content_type+ [String] the MIME type, optional. Without it, the type follows the file name; # - +content_id+ [String] optional: shows the file inline where `html` has `` with this ID. # @param capture_engagement [Boolean] Code form: whether PostHog captures the email's opens and clicks as events @@ -171,7 +171,8 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # @raise [EmailError] PostHog did not queue the email, or the client could not tell whether it did. # Before any request, {EmailError#code} is `invalid_attachment` for a file the client can't send, # `too_many_attachments` for more than 10 files, and `message_too_large` for files too large for one email. - # It is `attachment_upload_failed` when file storage refused an upload, and then no email was sent. + # It is `attachment_upload_failed` when file storage refused an upload, and `not_found` with + # {EmailError#field} `attachments` when PostHog has no upload route for this project. No email was sent. # @raise [ArgumentError] An argument has the wrong type or format, or the call mixes the two forms. def send( to:, template_id: nil, template_data: nil, template_key: nil, subject: nil, html: nil, text: nil, from: nil, @@ -335,7 +336,10 @@ def code_content(template_key, fields) code = fields.slice(:subject, :html, :text, :from).merge(template_key: template_key) code[:reply_to] = addresses(:reply_to, fields[:reply_to]) if fields.key?(:reply_to) - %i[cc bcc].each { |name| put_unless_empty(code, name, addresses(name, fields[name])) if fields.key?(name) } + %i[cc bcc].each do |name| + list = fields.key?(name) ? addresses(name, fields[name]) : [] + code[name] = list unless list.empty? + end code[:capture_engagement] = fields.fetch(:capture_engagement, true) code end @@ -348,10 +352,6 @@ def refuse_fields(fields, allowed:, form:, other_form:, why:) raise ArgumentError, "#{message} #{why} Remove #{field}, or send with #{other_form}." end - def put_unless_empty(code, name, list) - code[name] = list unless list.empty? - end - def addresses(name, value) list = value.is_a?(String) ? [value] : value return list if list.is_a?(Array) && list.all? { |address| nonempty_string?(address) } @@ -455,6 +455,7 @@ def upload(files, secret_key, max_retries) def parse_uploads_reply((status, payload, retry_after), files) fields = payload.is_a?(Hash) ? payload : {} + fields = { 'attr' => 'attachments' }.merge(fields.compact) if status == 404 raise reply_error(fields, status, retry_after) unless status.between?(200, 299) slots = fields['files'] diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 21714b5d..fd50930c 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -287,18 +287,21 @@ def refused_attachment(**overrides) end).to have_been_made.twice end - it 'takes the file as bytes, a list of byte values, or Base64 with or without line breaks' do + it 'takes the file as bytes, a list of byte values, or Base64 in any line layout' do file = pdf * 5 + mime_lines = [file].pack('m0').scan(/.{1,76}/) + pem_lines = [file].pack('m0').scan(/.{1,64}/) stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(five_pdfs_sha256))) stub_request(:post, send_url).to_return(accept) - [file, file.bytes, [file].pack('m0'), [file].pack('m')].each do |content| + [file, file.bytes, [file].pack('m0'), [file].pack('m'), "#{mime_lines.join("\n")}\n", + "#{mime_lines.join("\r\n")}\r\n", pem_lines.join("\n")].each do |content| send_code(attachments: [{ filename: 'invoice.pdf', content: content }]) end - expect([file].pack('m')).to include("\n") + expect([mime_lines.size, pem_lines.size]).to eq([2, 2]) expect(a_request(:post, uploads_url).with(body: { files: [{ sha256: five_pdfs_sha256, size: 60 }] }.to_json)) - .to have_been_made.times(4) + .to have_been_made.times(7) end it 'uploads a file attached twice once, and sends both references' do @@ -331,6 +334,33 @@ def refused_attachment(**overrides) expect(a_request(:post, send_url)).not_to have_been_made end + it "names a refused upload by storage's failure, never by PostHog's categories" do + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) + stub_request(:put, upload_url).to_return({ status: 403, body: 'AccessDenied' }, + { status: 503, body: 'SlowDown' }) + attachments = [{ filename: 'invoice.pdf', content: pdf }] + + expired = refused_attachment(max_retries: 0, attachments: attachments) + unavailable = refused_attachment(max_retries: 0, attachments: attachments) + + expect([expired.name, expired.status_code]).to eq(['application_error', 403]) + expect([unavailable.name, unavailable.status_code]).to eq(['service_unavailable', 503]) + end + + it "sends nothing on a PostHog without the upload route, and passes on PostHog's answer" do + stub_request(:post, uploads_url).to_return( + refuse(404, 'not_found', detail: 'Endpoint not found.', type: 'invalid_request', attr: nil) + ) + + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.name, error.status_code, error.field]) + .to eq(['not_found', 'not_found', 404, 'attachments']) + expect(error.message).to eq('Endpoint not found.') + expect(a_request(:put, /./)).not_to have_been_made + expect(a_request(:post, send_url)).not_to have_been_made + end + it 'names the file storage and the file when an upload never reaches it' do stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) stub_request(:put, upload_url).to_timeout @@ -464,17 +494,20 @@ def refused_attachment(**overrides) answer_uploads(*JSON.parse(request.body)['files'].map { |file| present_slot(file['sha256']) }) end stub_request(:post, send_url).to_return(accept) - # Each file leaves room for 57 characters of body, but not for 58. - room_for_57_ascii = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] # 81 bytes left - room_for_57_two_byte = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] # 159 bytes left + leaving_81_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] + leaving_159_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] + ascii_that_fits = 'a' * 57 + ascii_one_too_many = 'a' * 58 + two_byte_that_fits = 'é' * 57 + two_byte_one_too_many = 'é' * 58 sent = [ - send_code(html: 'a' * 57, attachments: room_for_57_ascii), # counts 78 - send_code(html: '', text: ('é' * 57).encode(Encoding::UTF_16LE), attachments: room_for_57_two_byte) # 156 + send_code(html: ascii_that_fits, attachments: leaving_81_bytes), + send_code(html: '', text: two_byte_that_fits.encode(Encoding::UTF_16LE), attachments: leaving_159_bytes) ] refused = [ - refused_attachment(html: 'a' * 58, attachments: room_for_57_ascii), # counts 83 - refused_attachment(html: '', text: 'é' * 58, attachments: room_for_57_two_byte) # counts 161 + refused_attachment(html: ascii_one_too_many, attachments: leaving_81_bytes), + refused_attachment(html: '', text: two_byte_one_too_many, attachments: leaving_159_bytes) ] expect(refused.map(&:code)).to eq(%w[message_too_large message_too_large]) From 484811d6278cc6f5e03e632a85f7eadc394b19bd Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:31:51 +0000 Subject: [PATCH 19/31] perf(emails): check Base64 line widths without splitting the file The layout check re-encoded the file and split it into lines for each width, which allocated millions of strings for the largest file. It now walks the line breaks once. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 26 +++++++++++++++++++------- spec/posthog/emails_spec.rb | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index 7ba6d261..d4b49465 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -82,18 +82,30 @@ def self.base64_bytes(content, field) raise invalid('An attachment needs content: the bytes File.binread returns, a list of byte values, or the ' \ 'file encoded as Base64.', field) end - bytes = content.delete("\r\n").unpack1('m0') - raise ArgumentError if content.include?("\n") && !wrapped_base64?(content, bytes) + raise ArgumentError if content.include?("\n") && !wrapped_base64?(content) - bytes + content.delete("\r\n").unpack1('m0') rescue ArgumentError, EncodingError raise invalid("An attachment's content is text that isn't Base64. Pass the file's bytes, read with " \ 'File.binread, or the file encoded as Base64.', field) end - def self.wrapped_base64?(content, bytes) - encoded = [bytes].pack('m0') - BASE64_LINE_WIDTHS.any? { |width| encoded.scan(/.{1,#{width}}/).join("\n") == content.delete("\r").chomp } + def self.wrapped_base64?(content) + text = content.delete("\r").chomp + width = text.index("\n") + return true if width.nil? + + text.ascii_only? && BASE64_LINE_WIDTHS.include?(width) && lines_of_width?(text, width) + end + + def self.lines_of_width?(text, width) + line_break = width + while (next_break = text.index("\n", line_break + 1)) + return false unless next_break == line_break + width + 1 + + line_break = next_break + end + (text.length - line_break - 1).between?(1, width) end def self.invalid(message, field) @@ -108,7 +120,7 @@ def self.too_many_files end private_class_method :new, :attachment, :check_keys, :file_bytes, :raw_bytes, :base64_bytes, - :wrapped_base64?, :invalid, :too_many_files + :wrapped_base64?, :lines_of_width?, :invalid, :too_many_files def initialize(attachments, bodies) @attachments = attachments diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index fd50930c..afd0ff35 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -489,6 +489,20 @@ def refused_attachment(**overrides) .to eq('msg_1') end + it 'reads the largest file wrapped as Base64 without splitting it into lines' do + largest_that_fits = ("\0" * 29_182_875).b + wrapped = "#{[largest_that_fits].pack('m0').scan(/.{1,76}/).join("\n")}\n" + stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) + stub_request(:post, send_url).to_return(accept) + emails + GC.start + + allocated_before = GC.stat(:total_allocated_objects) + send_code(html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) + + expect(GC.stat(:total_allocated_objects) - allocated_before).to be < 100_000 + end + it "counts each body like a file: its UTF-8 bytes' Base64 size with line breaks" do stub_request(:post, uploads_url).to_return do |request| answer_uploads(*JSON.parse(request.body)['files'].map { |file| present_slot(file['sha256']) }) From 8b7d237470977625283116adfc15a471f565e6fe Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:35:05 +0000 Subject: [PATCH 20/31] fix(emails): send files to storage only over https when PostHog is on https An upload URL without https from a PostHog reached over https now stops the send before any file goes out, with a reason and a fix. Local setups that reach PostHog over http still upload over http. Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-attachments.md | 2 +- lib/posthog/emails.rb | 26 ++++++++++++++++++++++---- spec/posthog/emails_spec.rb | 25 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/.changeset/emails-attachments.md b/.changeset/emails-attachments.md index 04fd4f91..59e1bb5d 100644 --- a/.changeset/emails-attachments.md +++ b/.changeset/emails-attachments.md @@ -2,4 +2,4 @@ "posthog-ruby": minor --- -`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. +`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. With `app_host` on https, files only go to storage over https: an upload URL without https raises `api_error` on `attachments` before any file goes out. diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index e2e1e823..b7b3a252 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -172,7 +172,9 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # Before any request, {EmailError#code} is `invalid_attachment` for a file the client can't send, # `too_many_attachments` for more than 10 files, and `message_too_large` for files too large for one email. # It is `attachment_upload_failed` when file storage refused an upload, and `not_found` with - # {EmailError#field} `attachments` when PostHog has no upload route for this project. No email was sent. + # {EmailError#field} `attachments` when PostHog has no upload route for this project. With `app_host` on + # https, an upload URL without https raises `api_error` on `attachments` before any file goes out. No email + # was sent. # @raise [ArgumentError] An argument has the wrong type or format, or the call mixes the two forms. def send( to:, template_id: nil, template_data: nil, template_key: nil, subject: nil, html: nil, text: nil, from: nil, @@ -459,10 +461,26 @@ def parse_uploads_reply((status, payload, retry_after), files) raise reply_error(fields, status, retry_after) unless status.between?(200, 299) slots = fields['files'] - return slots if slots.is_a?(Array) && slots.size == files.size && files.zip(slots).all? { |pair| slot?(*pair) } + unless slots.is_a?(Array) && slots.size == files.size && files.zip(slots).all? { |pair| slot?(*pair) } + raise EmailError.new(code: 'api_error', message: "PostHog answered without the files' uploads.", + status_code: status) + end + raise upload_without_tls(status) if slots.any? { |slot| without_tls?(slot) } + + slots + end + + def without_tls?(slot) + slot['status'] == 'upload' && URI(@app_host).scheme == 'https' && URI(slot['url']).scheme != 'https' + end - raise EmailError.new(code: 'api_error', message: "PostHog answered without the files' uploads.", - status_code: status) + def upload_without_tls(status) + EmailError.new(code: 'api_error', field: 'attachments', status_code: status, + message: 'PostHog gave an upload URL without https, so no file was uploaded and no email was ' \ + 'sent.', + why: 'Your app_host uses https, and the files would have gone to storage without TLS.', + fix: "Serve your PostHog's file storage over https, or reach PostHog over http in local " \ + 'development.') end def slot?(file, slot) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index afd0ff35..00da9a42 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -455,6 +455,31 @@ def refused_attachment(**overrides) expect(a_request(:post, send_url)).not_to have_been_made end + it 'sends no file without TLS when PostHog is reached over https' do + plain_url = 'http://files.example.com/team/99882c55?X-Amz-Signature=abc' + stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256, url: plain_url))) + + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.field]).to eq(%w[api_error attachments]) + expect([error.message, error.fix]).to all(include('https')) + expect(a_request(:put, /./)).not_to have_been_made + expect(a_request(:post, send_url)).not_to have_been_made + end + + it 'uploads over http when PostHog itself is reached over http' do + options.merge!(host: 'http://127.0.0.1:9', app_host: 'http://posthog.test') + plain_url = 'http://storage.test:8333/team/99882c55?X-Amz-Signature=abc' + stub_request(:post, 'http://posthog.test/api/projects/@current/messaging/attachments/uploads/') + .to_return(answer_uploads(upload_slot(pdf_sha256, url: plain_url))) + stub_request(:put, plain_url).to_return(status: 200) + stub_request(:post, 'http://posthog.test/api/projects/@current/messaging/send/email/').to_return(accept) + + send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect(a_request(:put, plain_url).with(body: pdf)).to have_been_made.once + end + it 'refuses more files or bytes than one email carries before any request' do eleven = Array.new(11) { |index| { filename: "#{index}.txt", content: "file #{index}".b } } largest_that_fits = ("\0" * 29_182_875).b From 7115f4bcd12544c8d2bf45031ce7c5340f84112a Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 04:42:27 +0000 Subject: [PATCH 21/31] fix(emails): retry storage's 500, 502 and 504, and name its 401 as storage's A PUT to the same signed URL with the same bytes is safe to repeat, so a transient storage error no longer stops the send while retries remain. A storage 401 is no longer reported as a missing PostHog key. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_error.rb | 2 +- lib/posthog/emails.rb | 1 + spec/posthog/emails_spec.rb | 12 ++++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/posthog/email_error.rb b/lib/posthog/email_error.rb index e55f51e5..c362072b 100644 --- a/lib/posthog/email_error.rb +++ b/lib/posthog/email_error.rb @@ -47,8 +47,8 @@ def initialize(code:, message:, status_code: nil, why: nil, fix: nil, docs_url: # @api private def self.name_for(status_code, code) - return 'missing_api_key' if code == 'missing_secret_key' || status_code == 401 return status_code.to_i >= 500 ? 'service_unavailable' : 'application_error' if code == 'attachment_upload_failed' + return 'missing_api_key' if code == 'missing_secret_key' || status_code == 401 return SDK_ERROR_NAMES.fetch(code, 'application_error') if status_code.nil? case status_code diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index b7b3a252..8fa6de69 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -441,6 +441,7 @@ def retryable?(error) when nil then error.code == 'network_error' when 409 then error.code == 'idempotency_key_in_flight' when 429 then !%w[quota_limited upload_quota_exceeded].include?(error.code) + when 500, 502, 504 then error.code == 'attachment_upload_failed' else error.status_code == 503 end end diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 00da9a42..8c509ea4 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -337,13 +337,16 @@ def refused_attachment(**overrides) it "names a refused upload by storage's failure, never by PostHog's categories" do stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) stub_request(:put, upload_url).to_return({ status: 403, body: 'AccessDenied' }, + { status: 401 }, { status: 503, body: 'SlowDown' }) attachments = [{ filename: 'invoice.pdf', content: pdf }] expired = refused_attachment(max_retries: 0, attachments: attachments) + unauthorized = refused_attachment(max_retries: 0, attachments: attachments) unavailable = refused_attachment(max_retries: 0, attachments: attachments) expect([expired.name, expired.status_code]).to eq(['application_error', 403]) + expect([unauthorized.name, unauthorized.status_code]).to eq(['application_error', 401]) expect([unavailable.name, unavailable.status_code]).to eq(['service_unavailable', 503]) end @@ -375,12 +378,13 @@ def refused_attachment(**overrides) it "retries an upload storage couldn't take, then sends" do stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) stub_request(:put, upload_url).to_return({ status: 503, body: 'SlowDown' }, - { status: 200 }) + { status: 500, body: 'InternalError' }, + { status: 502 }, { status: 504 }, { status: 200 }) stub_request(:post, send_url).to_return(accept) - expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') - expect(a_request(:put, upload_url)).to have_been_made.twice - expect(delays.size).to eq(1) + expect(send_code(max_retries: 4, attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') + expect(a_request(:put, upload_url)).to have_been_made.times(5) + expect(delays.size).to eq(4) end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do From 4bc4f3f9212e085a9f02adebda522eef38016fca Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:25:59 +0000 Subject: [PATCH 22/31] fix(emails): count the subject towards the email's size as PostHog does PostHog's estimate now adds 75 bytes for every 42 bytes of the subject's UTF-8, rounded up, and nothing for an empty subject. The local check matches it, so a long subject beside a near-limit file is refused before any upload. Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-attachments.md | 2 +- lib/posthog/email_attachments.rb | 27 ++++++++++++++------ lib/posthog/emails.rb | 3 ++- spec/posthog/emails_spec.rb | 43 ++++++++++++++++++++++++++------ 4 files changed, 57 insertions(+), 18 deletions(-) diff --git a/.changeset/emails-attachments.md b/.changeset/emails-attachments.md index 59e1bb5d..d2004638 100644 --- a/.changeset/emails-attachments.md +++ b/.changeset/emails-attachments.md @@ -2,4 +2,4 @@ "posthog-ruby": minor --- -`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. With `app_host` on https, files only go to storage over https: an upload URL without https raises `api_error` on `attachments` before any file goes out. +`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files; the subject and bodies count towards the 40 MB as PostHog counts them. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. With `app_host` on https, files only go to storage over https: an upload URL without https raises `api_error` on `attachments` before any file goes out. diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index d4b49465..0372c01e 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -12,6 +12,8 @@ class EmailAttachments MAX_FILES = 10 MAX_MESSAGE_BYTES = 40_000_000 HEADERS_AND_MIME_BOUNDARIES_BYTES = 64 * 1024 + SUBJECT_BYTES_PER_ENCODED_WORD = 42 + ENCODED_WORD_BYTES = 75 LIMIT = '10 files and 40 MB per email after Base64 encoding, about 29 MB of files' FIELDS = %w[filename content content_type content_id].freeze BASE64_LINE_WIDTHS = [60, 64, 76].freeze @@ -25,12 +27,13 @@ def sha256 # @param files [Object] What the caller passed as `attachments`. # @param bodies [Array] The email's HTML and text bodies, which count towards its size. + # @param subject [String, nil] The email's subject, which counts towards its size. # @raise [EmailError] A file the client can't send, or more than one email can carry. - def self.from(files, bodies) + def self.from(files, bodies, subject) raise invalid('attachments must be a list of files.', 'attachments') unless files.is_a?(Array) raise too_many_files if files.size > MAX_FILES - new(files.each_with_index.map { |file, index| attachment(file, "attachments__#{index}") }, bodies) + new(files.each_with_index.map { |file, index| attachment(file, "attachments__#{index}") }, bodies, subject) end def self.attachment(file, field) @@ -122,9 +125,9 @@ def self.too_many_files private_class_method :new, :attachment, :check_keys, :file_bytes, :raw_bytes, :base64_bytes, :wrapped_base64?, :lines_of_width?, :invalid, :too_many_files - def initialize(attachments, bodies) + def initialize(attachments, bodies, subject) @attachments = attachments - check_size(bodies) + check_size(bodies, subject) end def empty? @@ -143,8 +146,8 @@ def distinct private - def check_size(bodies) - return if message_bytes(bodies) <= MAX_MESSAGE_BYTES + def check_size(bodies, subject) + return if message_bytes(bodies, subject) <= MAX_MESSAGE_BYTES raise EmailError.new( code: 'message_too_large', field: 'attachments', @@ -156,9 +159,17 @@ def check_size(bodies) ) end - def message_bytes(bodies) + def message_bytes(bodies, subject) files = @attachments.sum { |attachment| encoded_bytes(attachment.bytes.bytesize) } - files + bodies.sum { |body| encoded_bytes(utf8_bytesize(body)) } + HEADERS_AND_MIME_BOUNDARIES_BYTES + files + bodies.sum { |body| encoded_bytes(utf8_bytesize(body)) } + subject_bytes(subject) + + HEADERS_AND_MIME_BOUNDARIES_BYTES + end + + def subject_bytes(subject) + return 0 unless subject.is_a?(String) + + (utf8_bytesize(subject) + SUBJECT_BYTES_PER_ENCODED_WORD - 1) / SUBJECT_BYTES_PER_ENCODED_WORD * + ENCODED_WORD_BYTES end def utf8_bytesize(body) diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index 8fa6de69..5c885ab3 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -187,7 +187,8 @@ def send( }.compact check_recipient(to, distinct_id, idempotency_key, max_retries) payload = content(template_id, template_key, fields).merge(to: to) - files = EmailAttachments.from(fields.fetch(:attachments, []), fields.values_at(:html, :text).compact) + files = EmailAttachments.from(fields.fetch(:attachments, []), fields.values_at(:html, :text).compact, + fields[:subject]) payload[:attachments] = files.references unless files.empty? payload[:distinct_id] = distinct_id unless distinct_id.nil? body = JSON.generate(payload) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 8c509ea4..212aa096 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -489,8 +489,9 @@ def refused_attachment(**overrides) largest_that_fits = ("\0" * 29_182_875).b too_many = refused_attachment(attachments: eleven) - too_large = refused_attachment(attachments: [{ filename: 'scan.pdf', content: "#{largest_that_fits}\0".b }]) - too_large_with_body = refused_attachment(html: 'abcd', + too_large = refused_attachment(subject: '', + attachments: [{ filename: 'scan.pdf', content: "#{largest_that_fits}\0".b }]) + too_large_with_body = refused_attachment(subject: '', html: 'abcd', attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]) expect([too_many.code, too_many.name, too_many.field]) @@ -514,7 +515,7 @@ def refused_attachment(**overrides) stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) stub_request(:post, send_url).to_return(accept) - expect(send_code(html: '', attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]).id) + expect(send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]).id) .to eq('msg_1') end @@ -527,7 +528,7 @@ def refused_attachment(**overrides) GC.start allocated_before = GC.stat(:total_allocated_objects) - send_code(html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) + send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) expect(GC.stat(:total_allocated_objects) - allocated_before).to be < 100_000 end @@ -545,12 +546,13 @@ def refused_attachment(**overrides) two_byte_one_too_many = 'é' * 58 sent = [ - send_code(html: ascii_that_fits, attachments: leaving_81_bytes), - send_code(html: '', text: two_byte_that_fits.encode(Encoding::UTF_16LE), attachments: leaving_159_bytes) + send_code(subject: '', html: ascii_that_fits, attachments: leaving_81_bytes), + send_code(subject: '', html: '', text: two_byte_that_fits.encode(Encoding::UTF_16LE), + attachments: leaving_159_bytes) ] refused = [ - refused_attachment(html: ascii_one_too_many, attachments: leaving_81_bytes), - refused_attachment(html: '', text: two_byte_one_too_many, attachments: leaving_159_bytes) + refused_attachment(subject: '', html: ascii_one_too_many, attachments: leaving_81_bytes), + refused_attachment(subject: '', html: '', text: two_byte_one_too_many, attachments: leaving_159_bytes) ] expect(refused.map(&:code)).to eq(%w[message_too_large message_too_large]) @@ -558,6 +560,31 @@ def refused_attachment(**overrides) expect(sent_bodies.map { |body| body['text'] }).to eq([nil, 'é' * 57]) end + it 'counts the subject in 75-byte steps per 42 of its UTF-8 bytes, and an empty subject not at all' do + stub_request(:post, uploads_url).to_return do |request| + answer_uploads(*JSON.parse(request.body)['files'].map { |file| present_slot(file['sha256']) }) + end + stub_request(:post, send_url).to_return(accept) + leaving_77_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_821).b }] + leaving_3_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_875).b }] + one_ascii_word = 'a' * 42 + two_ascii_words = 'a' * 43 + one_two_byte_word = 'é' * 21 + two_two_byte_words = 'é' * 22 + + sent = [one_ascii_word, one_two_byte_word].map do |subject| + send_code(subject: subject, html: '', attachments: leaving_77_bytes) + end + sent << send_code(subject: '', html: '', attachments: leaving_3_bytes) + refused = [two_ascii_words, two_two_byte_words].map do |subject| + refused_attachment(subject: subject, html: '', attachments: leaving_77_bytes) + end + refused << refused_attachment(subject: 'a', html: '', attachments: leaving_3_bytes) + + expect(sent.map(&:id)).to eq(%w[msg_1 msg_1 msg_1]) + expect(refused.map(&:code)).to all(eq('message_too_large')) + end + it 'refuses a file it cannot send before any request, naming the field and never the content' do cases = { { attachments: { filename: 'a.pdf', content: pdf } } => ['attachments', /list of files/], From 547e46652b7aed03b69c31196f60d41a02072e85 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:10:24 +0000 Subject: [PATCH 23/31] fix(emails): answer the QA panel on the subject round - Base64 with lone carriage returns is refused like any other unknown line layout, instead of having them stripped. - Wrapped Base64 reads the same whatever the app's input record separator. - A 404 from the upload route without PostHog's JSON, such as a proxy's page, is still not_found on attachments. - Tests assert what a caller sees: the largest wrapped file is sent with its decoded size, and storage retries are counted at the storage boundary. Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 7 ++++-- lib/posthog/emails.rb | 2 +- spec/posthog/emails_spec.rb | 40 +++++++++++++++++++++++++------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index 0372c01e..d51e326c 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -85,7 +85,7 @@ def self.base64_bytes(content, field) raise invalid('An attachment needs content: the bytes File.binread returns, a list of byte values, or the ' \ 'file encoded as Base64.', field) end - raise ArgumentError if content.include?("\n") && !wrapped_base64?(content) + raise ArgumentError if content.match?(/[\r\n]/) && !wrapped_base64?(content) content.delete("\r\n").unpack1('m0') rescue ArgumentError, EncodingError @@ -94,7 +94,10 @@ def self.base64_bytes(content, field) end def self.wrapped_base64?(content) - text = content.delete("\r").chomp + text = content.gsub("\r\n", "\n") + return false if text.include?("\r") + + text = text.delete_suffix("\n") width = text.index("\n") return true if width.nil? diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index 5c885ab3..8eb90183 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -459,7 +459,7 @@ def upload(files, secret_key, max_retries) def parse_uploads_reply((status, payload, retry_after), files) fields = payload.is_a?(Hash) ? payload : {} - fields = { 'attr' => 'attachments' }.merge(fields.compact) if status == 404 + fields = { 'code' => 'not_found', 'attr' => 'attachments' }.merge(fields.compact) if status == 404 raise reply_error(fields, status, retry_after) unless status.between?(200, 299) slots = fields['files'] diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 212aa096..1fe363f9 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'English' require 'spec_helper' module PostHog @@ -304,6 +305,21 @@ def refused_attachment(**overrides) .to have_been_made.times(7) end + it "reads wrapped Base64 whatever the app's input record separator" do + file = pdf * 5 + wrapped = "#{[file].pack('m0').scan(/.{1,76}/).join("\n")}\n" + stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(five_pdfs_sha256))) + stub_request(:post, send_url).to_return(accept) + separator = $INPUT_RECORD_SEPARATOR + $INPUT_RECORD_SEPARATOR = nil + + result = send_code(attachments: [{ filename: 'invoice.pdf', content: wrapped }]) + + expect(result.id).to eq('msg_1') + ensure + $INPUT_RECORD_SEPARATOR = separator + end + it 'uploads a file attached twice once, and sends both references' do stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) stub_request(:put, upload_url).to_return(status: 200) @@ -352,14 +368,17 @@ def refused_attachment(**overrides) it "sends nothing on a PostHog without the upload route, and passes on PostHog's answer" do stub_request(:post, uploads_url).to_return( - refuse(404, 'not_found', detail: 'Endpoint not found.', type: 'invalid_request', attr: nil) + refuse(404, 'not_found', detail: 'Endpoint not found.', type: 'invalid_request', attr: nil), + { status: 404, body: 'Not Found', headers: { 'Content-Type' => 'text/html' } } ) - error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + error, proxied = Array.new(2) { refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) } expect([error.code, error.name, error.status_code, error.field]) .to eq(['not_found', 'not_found', 404, 'attachments']) expect(error.message).to eq('Endpoint not found.') + expect([proxied.code, proxied.name, proxied.status_code, proxied.field]) + .to eq(['not_found', 'not_found', 404, 'attachments']) expect(a_request(:put, /./)).not_to have_been_made expect(a_request(:post, send_url)).not_to have_been_made end @@ -384,7 +403,6 @@ def refused_attachment(**overrides) expect(send_code(max_retries: 4, attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(a_request(:put, upload_url)).to have_been_made.times(5) - expect(delays.size).to eq(4) end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do @@ -519,18 +537,20 @@ def refused_attachment(**overrides) .to eq('msg_1') end - it 'reads the largest file wrapped as Base64 without splitting it into lines' do + it 'reads the largest file one email carries wrapped as Base64, and sends it' do largest_that_fits = ("\0" * 29_182_875).b wrapped = "#{[largest_that_fits].pack('m0').scan(/.{1,76}/).join("\n")}\n" stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) stub_request(:post, send_url).to_return(accept) - emails - GC.start - allocated_before = GC.stat(:total_allocated_objects) - send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) + result = send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) - expect(GC.stat(:total_allocated_objects) - allocated_before).to be < 100_000 + expect(result.id).to eq('msg_1') + expect(a_request(:post, uploads_url).with( + body: { files: [{ sha256: largest_that_fits_sha256, size: 29_182_875 }] }.to_json + )).to have_been_made.once + expect(sent_bodies.first['attachments']).to eq([{ 'sha256' => largest_that_fits_sha256, + 'filename' => 'scan.pdf' }]) end it "counts each body like a file: its UTF-8 bytes' Base64 size with line breaks" do @@ -601,6 +621,8 @@ def refused_attachment(**overrides) ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: 'codes.txt', content: "ABCD1234\nEFGH5678\n" }] } => ['attachments__0__content', /isn't Base64/], + { attachments: [{ filename: 'codes.txt', content: "ABCD1234\rEFGH5678\r" }] } => + ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: 'a.txt', content: 'aGVsbG8='.encode('UTF-16LE') }] } => ['attachments__0__content', /isn't Base64/], { attachments: [{ filename: :a, content: pdf }] } => ['attachments__0__filename', /filename/], From 5f86c3776ff780a167ce2dc356c8faf387f24ac4 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:17:32 +0000 Subject: [PATCH 24/31] test(emails): run attachment sends against a fake PostHog and file storage The success paths ran against canned answers that accepted any upload and any send. They now run against an in-memory stand-in that behaves like the real routes and bucket: PostHog asks only for the files storage lacks, storage keeps a PUT only when its bytes match the signed SHA-256 and it carries no Authorization, and a send is accepted only when every file it names is stored. The tests assert the bytes in the bucket. Stubs remain only to inject faults. Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 174 ++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 75 deletions(-) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 1fe363f9..80cd2751 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -233,8 +233,50 @@ def refused_attachment(**overrides) e end + let(:bucket) { {} } + + def fake_posthog(app: 'https://us.posthog.com', storage: 'https://files.example.com/team') + routes = "#{app}/api/projects/@current/messaging" + stub_request(:post, "#{routes}/attachments/uploads/").to_return { |request| plan_uploads(request, storage) } + stub_request(:put, %r{/team/\h{64}\?}).to_return { |request| store(request) } + stub_request(:post, "#{routes}/send/email/").to_return { |request| accept_send(request) } + end + + def plan_uploads(request, storage) + return refuse(401, 'authentication_failed') unless request.headers['Authorization'] == 'Bearer phs_test' + + slots = JSON.parse(request.body)['files'].map do |file| + sha256 = file['sha256'] + next present_slot(sha256) if bucket.key?(sha256) + + upload_slot(sha256, url: "#{storage}/#{sha256}?X-Amz-Signature=fake") + .merge(headers: { 'x-amz-checksum-sha256' => [[sha256].pack('H*')].pack('m0') }) + end + answer_uploads(*slots) + end + + def store(request) + denied = { status: 403, body: 'AccessDenied' } + return denied if request.headers.key?('Authorization') + + bytes = request.body.b + sha256 = request.uri.path[/\h{64}\z/] + signed = request.headers['X-Amz-Checksum-Sha256'] == [Digest::SHA256.digest(bytes)].pack('m0') + return { status: 400, body: 'BadDigest' } unless signed + + bucket[sha256] = bytes + { status: 200 } + end + + def accept_send(request) + return refuse(401, 'authentication_failed') unless request.headers['Authorization'] == 'Bearer phs_test' + + files = JSON.parse(request.body).fetch('attachments', []) + files.all? { |file| bucket.key?(file['sha256']) } ? accept : refuse(400, 'attachment_not_uploaded') + end + it 'sends cc and bcc as lists' do - stub_request(:post, send_url).to_return(accept) + fake_posthog send_code(cc: 'finance@example.com', bcc: %w[audit@example.com archive@example.com]) @@ -242,23 +284,18 @@ def refused_attachment(**overrides) 'bcc' => %w[audit@example.com archive@example.com]) end - it 'uploads each file PostHog asks for to its URL, without the secret key, then sends the files by hash' do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256), present_slot(png_sha256))) - stub_request(:put, upload_url).to_return(status: 200) - stub_request(:post, send_url).to_return(accept) + it 'uploads each file storage lacks, without the secret key, then sends the files by hash' do + fake_posthog + bucket[png_sha256] = png - send_code(attachments: [ - { filename: 'invoice.pdf', content: pdf, content_type: 'application/pdf' }, - { 'filename' => 'logo.png', 'content' => png, 'content_id' => 'logo' } - ]) - - expect(a_request(:post, uploads_url).with( - headers: { 'Authorization' => 'Bearer phs_test' }, - body: { files: [{ sha256: pdf_sha256, size: 12 }, { sha256: png_sha256, size: 4 }] }.to_json - )).to have_been_made.once - expect(a_request(:put, upload_url).with(body: pdf, headers: upload_headers) do |request| - !request.headers.key?('Authorization') - end).to have_been_made.once + result = send_code(attachments: [ + { filename: 'invoice.pdf', content: pdf, content_type: 'application/pdf' }, + { 'filename' => 'logo.png', 'content' => png, 'content_id' => 'logo' } + ]) + + expect(result.id).to eq('msg_1') + expect(bucket).to eq(png_sha256 => png, pdf_sha256 => pdf) + expect(a_request(:put, /./)).to have_been_made.once expect(sent_bodies.first['attachments']).to eq([ { 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf', 'content_type' => 'application/pdf' }, @@ -268,7 +305,7 @@ def refused_attachment(**overrides) end it 'leaves out empty lists, so a PostHog version without these fields still sends the email' do - stub_request(:post, send_url).to_return(accept) + fake_posthog send_code(cc: [], bcc: [], attachments: []) @@ -276,13 +313,14 @@ def refused_attachment(**overrides) end it 'uploads once and sends the same files on every attempt of the send' do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_return(status: 200) - stub_request(:post, send_url).to_return(refuse(503, 'service_unavailable'), accept) + fake_posthog + stub_request(:post, send_url).to_return(refuse(503, 'service_unavailable')) + .then.to_return { |request| accept_send(request) } - send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') - expect(a_request(:put, upload_url)).to have_been_made.once + expect(bucket).to eq(pdf_sha256 => pdf) + expect(a_request(:put, /./)).to have_been_made.once expect(a_request(:post, send_url).with do |request| JSON.parse(request.body)['attachments'] == [{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }] end).to have_been_made.twice @@ -292,44 +330,40 @@ def refused_attachment(**overrides) file = pdf * 5 mime_lines = [file].pack('m0').scan(/.{1,76}/) pem_lines = [file].pack('m0').scan(/.{1,64}/) - stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(five_pdfs_sha256))) - stub_request(:post, send_url).to_return(accept) + fake_posthog - [file, file.bytes, [file].pack('m0'), [file].pack('m'), "#{mime_lines.join("\n")}\n", - "#{mime_lines.join("\r\n")}\r\n", pem_lines.join("\n")].each do |content| + results = [file, file.bytes, [file].pack('m0'), [file].pack('m'), "#{mime_lines.join("\n")}\n", + "#{mime_lines.join("\r\n")}\r\n", pem_lines.join("\n")].map do |content| send_code(attachments: [{ filename: 'invoice.pdf', content: content }]) end expect([mime_lines.size, pem_lines.size]).to eq([2, 2]) - expect(a_request(:post, uploads_url).with(body: { files: [{ sha256: five_pdfs_sha256, size: 60 }] }.to_json)) - .to have_been_made.times(7) + expect(results.map(&:id)).to all(eq('msg_1')) + expect(bucket).to eq(five_pdfs_sha256 => file) end it "reads wrapped Base64 whatever the app's input record separator" do file = pdf * 5 wrapped = "#{[file].pack('m0').scan(/.{1,76}/).join("\n")}\n" - stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(five_pdfs_sha256))) - stub_request(:post, send_url).to_return(accept) + fake_posthog separator = $INPUT_RECORD_SEPARATOR $INPUT_RECORD_SEPARATOR = nil result = send_code(attachments: [{ filename: 'invoice.pdf', content: wrapped }]) expect(result.id).to eq('msg_1') + expect(bucket).to eq(five_pdfs_sha256 => file) ensure $INPUT_RECORD_SEPARATOR = separator end it 'uploads a file attached twice once, and sends both references' do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_return(status: 200) - stub_request(:post, send_url).to_return(accept) + fake_posthog send_code(attachments: [{ filename: 'a.pdf', content: pdf }, { filename: 'b.pdf', content: pdf }]) - expect(a_request(:post, uploads_url).with(body: { files: [{ sha256: pdf_sha256, size: 12 }] }.to_json)) - .to have_been_made.once - expect(a_request(:put, upload_url)).to have_been_made.once + expect(bucket).to eq(pdf_sha256 => pdf) + expect(a_request(:put, /./)).to have_been_made.once expect(sent_bodies.first['attachments'].map { |file| file['filename'] }).to eq(%w[a.pdf b.pdf]) end @@ -395,30 +429,33 @@ def refused_attachment(**overrides) end it "retries an upload storage couldn't take, then sends" do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_return({ status: 503, body: 'SlowDown' }, - { status: 500, body: 'InternalError' }, - { status: 502 }, { status: 504 }, { status: 200 }) - stub_request(:post, send_url).to_return(accept) + fake_posthog + stub_request(:put, %r{/team/\h{64}\?}) + .to_return({ status: 503, body: 'SlowDown' }, + { status: 500, body: 'InternalError' }, + { status: 502 }, { status: 504 }) + .then.to_return { |request| store(request) } expect(send_code(max_retries: 4, attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') - expect(a_request(:put, upload_url)).to have_been_made.times(5) + expect(bucket).to eq(pdf_sha256 => pdf) + expect(a_request(:put, /./)).to have_been_made.times(5) end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_return({ status: 503, headers: { 'Retry-After' => '3' } }, - { status: 200 }, - { status: 503, headers: { 'Retry-After' => '60' } }) - stub_request(:post, send_url).to_return(accept) + fake_posthog + stub_request(:put, %r{/team/\h{64}\?}) + .to_return(status: 503, headers: { 'Retry-After' => '3' }) + .then.to_return { |request| store(request) } + .then.to_return(status: 503, headers: { 'Retry-After' => '60' }) - send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) - error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + sent = send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + error = refused_attachment(attachments: [{ filename: 'logo.png', content: png }]) - expect(delays.size).to eq(1) + expect(sent.id).to eq('msg_1') expect(delays.first).to be >= 3 expect([error.code, error.retry_after]).to eq(['attachment_upload_failed', 60.0]) - expect(a_request(:put, upload_url)).to have_been_made.times(3) + expect(bucket).to eq(pdf_sha256 => pdf) + expect(a_request(:put, /./)).to have_been_made.times(3) end it "keeps storage's Retry-After when its answer to an upload was cut off" do @@ -491,15 +528,11 @@ def refused_attachment(**overrides) it 'uploads over http when PostHog itself is reached over http' do options.merge!(host: 'http://127.0.0.1:9', app_host: 'http://posthog.test') - plain_url = 'http://storage.test:8333/team/99882c55?X-Amz-Signature=abc' - stub_request(:post, 'http://posthog.test/api/projects/@current/messaging/attachments/uploads/') - .to_return(answer_uploads(upload_slot(pdf_sha256, url: plain_url))) - stub_request(:put, plain_url).to_return(status: 200) - stub_request(:post, 'http://posthog.test/api/projects/@current/messaging/send/email/').to_return(accept) - - send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + fake_posthog(app: 'http://posthog.test', storage: 'http://storage.test:8333/team') - expect(a_request(:put, plain_url).with(body: pdf)).to have_been_made.once + expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') + expect(bucket).to eq(pdf_sha256 => pdf) + expect(a_request(:put, %r{\Ahttp://storage\.test:8333/})).to have_been_made.once end it 'refuses more files or bytes than one email carries before any request' do @@ -530,34 +563,28 @@ def refused_attachment(**overrides) it 'uploads the largest file one email carries' do largest_that_fits = ("\0" * 29_182_875).b - stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) - stub_request(:post, send_url).to_return(accept) + fake_posthog expect(send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]).id) .to eq('msg_1') + expect(bucket).to eq(largest_that_fits_sha256 => largest_that_fits) end it 'reads the largest file one email carries wrapped as Base64, and sends it' do largest_that_fits = ("\0" * 29_182_875).b wrapped = "#{[largest_that_fits].pack('m0').scan(/.{1,76}/).join("\n")}\n" - stub_request(:post, uploads_url).to_return(answer_uploads(present_slot(largest_that_fits_sha256))) - stub_request(:post, send_url).to_return(accept) + fake_posthog result = send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) expect(result.id).to eq('msg_1') - expect(a_request(:post, uploads_url).with( - body: { files: [{ sha256: largest_that_fits_sha256, size: 29_182_875 }] }.to_json - )).to have_been_made.once + expect(bucket).to eq(largest_that_fits_sha256 => largest_that_fits) expect(sent_bodies.first['attachments']).to eq([{ 'sha256' => largest_that_fits_sha256, 'filename' => 'scan.pdf' }]) end it "counts each body like a file: its UTF-8 bytes' Base64 size with line breaks" do - stub_request(:post, uploads_url).to_return do |request| - answer_uploads(*JSON.parse(request.body)['files'].map { |file| present_slot(file['sha256']) }) - end - stub_request(:post, send_url).to_return(accept) + fake_posthog leaving_81_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] leaving_159_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] ascii_that_fits = 'a' * 57 @@ -581,10 +608,7 @@ def refused_attachment(**overrides) end it 'counts the subject in 75-byte steps per 42 of its UTF-8 bytes, and an empty subject not at all' do - stub_request(:post, uploads_url).to_return do |request| - answer_uploads(*JSON.parse(request.body)['files'].map { |file| present_slot(file['sha256']) }) - end - stub_request(:post, send_url).to_return(accept) + fake_posthog leaving_77_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_821).b }] leaving_3_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_875).b }] one_ascii_word = 'a' * 42 From fe145f00f082bc8819c80fc044ae1e749161d8a4 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:27:33 +0000 Subject: [PATCH 25/31] test(emails): check uploads against a fake that enforces the signed size and digest The fake PostHog now refuses a malformed manifest and an unsigned send, and the fake storage takes a PUT only when its bytes match the signed size, checksum and digest. Tests read the stored files and the accepted emails instead of counting requests. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 260 ++++++++++++++++++------------------ 1 file changed, 133 insertions(+), 127 deletions(-) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 80cd2751..0fe187d1 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -202,38 +202,19 @@ def send_code(**overrides) end describe 'copies and attachments' do - let(:uploads_url) { 'https://us.posthog.com/api/projects/@current/messaging/attachments/uploads/' } - let(:upload_url) { 'https://files.example.com/team/99882c55?X-Amz-Signature=abc' } let(:pdf) { "%PDF-1.7\n\xFF\x00\xE2".b } let(:pdf_sha256) { '99882c5503918dac954710e17022f6243565344d43c0f30ac70d19c560f857c4' } let(:png) { "\x89PNG".b } let(:png_sha256) { '0f4636c78f65d3639ece5a064b5ae753e3408614a14fb18ab4d7540d2c248543' } let(:five_pdfs_sha256) { 'dfb00ae2891aa7dac580fd90519992e7a8d6914de7a683c0ecc071f65868101e' } let(:largest_that_fits_sha256) { '28f197eae8bd57e7c2a94d2062d2cef0a04b34e7fd3e0c743d88cda2217f069c' } - let(:upload_headers) do - { 'Content-Length' => '12', 'x-amz-checksum-sha256' => 'mYgsVQORjayVRxDhcCL2JDVlNE1DwPMKxw0ZxWD4V8Q=' } - end - - def upload_slot(sha256, url: upload_url) - { sha256: sha256, status: 'upload', url: url, method: 'PUT', headers: upload_headers, - url_expires_at: '2026-11-09T12:15:00Z' } - end - - def present_slot(sha256) - { sha256: sha256, status: 'present', expires_at: '2026-12-16T12:00:00Z' } - end - - def answer_uploads(*slots) - { status: 200, body: JSON.generate(files: slots) } - end - - def refused_attachment(**overrides) - send_code(**overrides) - rescue EmailError => e - e - end - let(:bucket) { {} } + let(:storage_requests) { [] } + let(:posthog_requests) { [] } + let(:outbox) { [] } + let(:faults) { { uploads: [], storage: [], send: [] } } + + before { fake_posthog } def fake_posthog(app: 'https://us.posthog.com', storage: 'https://files.example.com/team') routes = "#{app}/api/projects/@current/messaging" @@ -243,49 +224,96 @@ def fake_posthog(app: 'https://us.posthog.com', storage: 'https://files.example. end def plan_uploads(request, storage) - return refuse(401, 'authentication_failed') unless request.headers['Authorization'] == 'Bearer phs_test' + files = record(:uploads, request)['files'] + return faults[:uploads].shift if faults[:uploads].any? + return refuse(401, 'authentication_failed') unless signed_in?(request) + return refuse(400, 'invalid_request', attr: 'files') unless files.all? { |file| manifest_entry?(file) } + + answer_uploads(*files.map { |file| plan(file, storage) }) + end - slots = JSON.parse(request.body)['files'].map do |file| - sha256 = file['sha256'] - next present_slot(sha256) if bucket.key?(sha256) + def plan(file, storage) + bucket.key?(file['sha256']) ? present_slot(file['sha256']) : slot(file, storage) + end - upload_slot(sha256, url: "#{storage}/#{sha256}?X-Amz-Signature=fake") - .merge(headers: { 'x-amz-checksum-sha256' => [[sha256].pack('H*')].pack('m0') }) - end - answer_uploads(*slots) + def manifest_entry?(file) + file['sha256'].to_s.match?(/\A\h{64}\z/) && file['size'].is_a?(Integer) && file['size'].positive? + end + + def slot(file, storage) + sha256 = file['sha256'] + upload_slot(sha256, url: "#{storage}/#{sha256}?size=#{file['size']}&X-Amz-Signature=fake", + headers: { 'x-amz-checksum-sha256' => [[sha256].pack('H*')].pack('m0') }) end def store(request) - denied = { status: 403, body: 'AccessDenied' } - return denied if request.headers.key?('Authorization') + sha256 = request.uri.path[/\h{64}\z/] + storage_requests << sha256 + return faults[:storage].shift if faults[:storage].any? + return storage_error(403, 'AccessDenied') if request.headers.key?('Authorization') bytes = request.body.b - sha256 = request.uri.path[/\h{64}\z/] - signed = request.headers['X-Amz-Checksum-Sha256'] == [Digest::SHA256.digest(bytes)].pack('m0') - return { status: 400, body: 'BadDigest' } unless signed + signed = request.headers['X-Amz-Checksum-Sha256'] == [[sha256].pack('H*')].pack('m0') && + bytes.bytesize == request.uri.query_values['size'].to_i + return storage_error(403, 'SignatureDoesNotMatch') unless signed + return storage_error(400, 'BadDigest') if Digest::SHA256.hexdigest(bytes) != sha256 bucket[sha256] = bytes { status: 200 } end + def storage_error(status, code) + { status: status, body: "#{code}" } + end + def accept_send(request) - return refuse(401, 'authentication_failed') unless request.headers['Authorization'] == 'Bearer phs_test' + email = record(:send, request) + return faults[:send].shift if faults[:send].any? + return refuse(401, 'authentication_failed') unless signed_in?(request) + unless email.fetch('attachments', []).all? { |file| bucket.key?(file['sha256']) } + return refuse(400, 'attachment_not_uploaded', attr: 'attachments') + end - files = JSON.parse(request.body).fetch('attachments', []) - files.all? { |file| bucket.key?(file['sha256']) } ? accept : refuse(400, 'attachment_not_uploaded') + outbox << email + accept end - it 'sends cc and bcc as lists' do - fake_posthog + def record(route, request) + JSON.parse(request.body).tap { |body| posthog_requests << [route, body] } + end + def signed_in?(request) + request.headers['Authorization'] == 'Bearer phs_test' + end + + def upload_slot(sha256, url: 'https://files.example.com/team/99882c55?X-Amz-Signature=abc', + headers: { 'x-amz-checksum-sha256' => 'mYgsVQORjayVRxDhcCL2JDVlNE1DwPMKxw0ZxWD4V8Q=' }) + { sha256: sha256, status: 'upload', url: url, method: 'PUT', headers: headers, + url_expires_at: '2026-11-09T12:15:00Z' } + end + + def present_slot(sha256) + { sha256: sha256, status: 'present', expires_at: '2026-12-16T12:00:00Z' } + end + + def answer_uploads(*slots) + { status: 200, body: JSON.generate(files: slots) } + end + + def refused_attachment(**overrides) + send_code(**overrides) + rescue EmailError => e + e + end + + it 'sends cc and bcc as lists' do send_code(cc: 'finance@example.com', bcc: %w[audit@example.com archive@example.com]) - expect(sent_bodies.first).to include('cc' => ['finance@example.com'], - 'bcc' => %w[audit@example.com archive@example.com]) + expect(outbox.first).to include('cc' => ['finance@example.com'], + 'bcc' => %w[audit@example.com archive@example.com]) end it 'uploads each file storage lacks, without the secret key, then sends the files by hash' do - fake_posthog bucket[png_sha256] = png result = send_code(attachments: [ @@ -295,42 +323,37 @@ def accept_send(request) expect(result.id).to eq('msg_1') expect(bucket).to eq(png_sha256 => png, pdf_sha256 => pdf) - expect(a_request(:put, /./)).to have_been_made.once - expect(sent_bodies.first['attachments']).to eq([ - { 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf', - 'content_type' => 'application/pdf' }, - { 'sha256' => png_sha256, 'filename' => 'logo.png', - 'content_id' => 'logo' } - ]) + expect(storage_requests).to eq([pdf_sha256]) + expect(outbox.first['attachments']).to eq([ + { 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf', + 'content_type' => 'application/pdf' }, + { 'sha256' => png_sha256, 'filename' => 'logo.png', + 'content_id' => 'logo' } + ]) end it 'leaves out empty lists, so a PostHog version without these fields still sends the email' do - fake_posthog - send_code(cc: [], bcc: [], attachments: []) - expect(sent_bodies.first.keys).not_to include('cc', 'bcc', 'attachments') + expect(outbox.first.keys).not_to include('cc', 'bcc', 'attachments') + expect(posthog_requests.map(&:first)).to eq([:send]) end it 'uploads once and sends the same files on every attempt of the send' do - fake_posthog - stub_request(:post, send_url).to_return(refuse(503, 'service_unavailable')) - .then.to_return { |request| accept_send(request) } + faults[:send] << refuse(503, 'service_unavailable') expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(a_request(:put, /./)).to have_been_made.once - expect(a_request(:post, send_url).with do |request| - JSON.parse(request.body)['attachments'] == [{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }] - end).to have_been_made.twice + expect(storage_requests).to eq([pdf_sha256]) + attempts = posthog_requests.filter_map { |route, email| email['attachments'] if route == :send } + expect(attempts).to eq([[{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }]] * 2) end it 'takes the file as bytes, a list of byte values, or Base64 in any line layout' do file = pdf * 5 mime_lines = [file].pack('m0').scan(/.{1,76}/) pem_lines = [file].pack('m0').scan(/.{1,64}/) - fake_posthog results = [file, file.bytes, [file].pack('m0'), [file].pack('m'), "#{mime_lines.join("\n")}\n", "#{mime_lines.join("\r\n")}\r\n", pem_lines.join("\n")].map do |content| @@ -345,7 +368,6 @@ def accept_send(request) it "reads wrapped Base64 whatever the app's input record separator" do file = pdf * 5 wrapped = "#{[file].pack('m0').scan(/.{1,76}/).join("\n")}\n" - fake_posthog separator = $INPUT_RECORD_SEPARATOR $INPUT_RECORD_SEPARATOR = nil @@ -358,22 +380,20 @@ def accept_send(request) end it 'uploads a file attached twice once, and sends both references' do - fake_posthog - send_code(attachments: [{ filename: 'a.pdf', content: pdf }, { filename: 'b.pdf', content: pdf }]) expect(bucket).to eq(pdf_sha256 => pdf) - expect(a_request(:put, /./)).to have_been_made.once - expect(sent_bodies.first['attachments'].map { |file| file['filename'] }).to eq(%w[a.pdf b.pdf]) + expect(storage_requests).to eq([pdf_sha256]) + expect(posthog_requests.first).to eq([:uploads, { 'files' => [{ 'sha256' => pdf_sha256, 'size' => 12 }] }]) + expect(outbox.first['attachments'].map { |file| file['filename'] }).to eq(%w[a.pdf b.pdf]) end it 'sends nothing when storage refuses an upload, and says what storage answered' do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_return( + faults[:storage] << { status: 400, body: 'BadDigest' \ 'The checksum did not match.' - ) + } error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) @@ -381,14 +401,13 @@ def accept_send(request) .to eq(['attachment_upload_failed', 400, 'attachments__0']) expect(error.message).to eq('File storage refused the upload of invoice.pdf with HTTP 400 ' \ '(BadDigest: The checksum did not match.), so no email was sent.') - expect(a_request(:post, send_url)).not_to have_been_made + expect(bucket).to be_empty + expect(outbox).to be_empty end it "names a refused upload by storage's failure, never by PostHog's categories" do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_return({ status: 403, body: 'AccessDenied' }, - { status: 401 }, - { status: 503, body: 'SlowDown' }) + faults[:storage].push({ status: 403, body: 'AccessDenied' }, { status: 401 }, + { status: 503, body: 'SlowDown' }) attachments = [{ filename: 'invoice.pdf', content: pdf }] expired = refused_attachment(max_retries: 0, attachments: attachments) @@ -398,10 +417,11 @@ def accept_send(request) expect([expired.name, expired.status_code]).to eq(['application_error', 403]) expect([unauthorized.name, unauthorized.status_code]).to eq(['application_error', 401]) expect([unavailable.name, unavailable.status_code]).to eq(['service_unavailable', 503]) + expect(outbox).to be_empty end it "sends nothing on a PostHog without the upload route, and passes on PostHog's answer" do - stub_request(:post, uploads_url).to_return( + faults[:uploads].push( refuse(404, 'not_found', detail: 'Endpoint not found.', type: 'invalid_request', attr: nil), { status: 404, body: 'Not Found', headers: { 'Content-Type' => 'text/html' } } ) @@ -413,49 +433,42 @@ def accept_send(request) expect(error.message).to eq('Endpoint not found.') expect([proxied.code, proxied.name, proxied.status_code, proxied.field]) .to eq(['not_found', 'not_found', 404, 'attachments']) - expect(a_request(:put, /./)).not_to have_been_made - expect(a_request(:post, send_url)).not_to have_been_made + expect(storage_requests).to be_empty + expect(outbox).to be_empty end it 'names the file storage and the file when an upload never reaches it' do - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256))) - stub_request(:put, upload_url).to_timeout + stub_request(:put, %r{/team/\h{64}\?}).to_timeout error = refused_attachment(max_retries: 0, attachments: [{ filename: 'invoice.pdf', content: pdf }]) expect([error.code, error.name, error.field]).to eq(%w[network_error service_unavailable attachments__0]) expect(error.message).to eq('The upload of invoice.pdf to file storage failed, so no email was sent.') - expect(a_request(:post, send_url)).not_to have_been_made + expect(outbox).to be_empty end it "retries an upload storage couldn't take, then sends" do - fake_posthog - stub_request(:put, %r{/team/\h{64}\?}) - .to_return({ status: 503, body: 'SlowDown' }, - { status: 500, body: 'InternalError' }, - { status: 502 }, { status: 504 }) - .then.to_return { |request| store(request) } + faults[:storage].push({ status: 503, body: 'SlowDown' }, + { status: 500, body: 'InternalError' }, + { status: 502 }, { status: 504 }) expect(send_code(max_retries: 4, attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(a_request(:put, /./)).to have_been_made.times(5) + expect(storage_requests).to eq([pdf_sha256] * 5) end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do - fake_posthog - stub_request(:put, %r{/team/\h{64}\?}) - .to_return(status: 503, headers: { 'Retry-After' => '3' }) - .then.to_return { |request| store(request) } - .then.to_return(status: 503, headers: { 'Retry-After' => '60' }) - + faults[:storage] << { status: 503, headers: { 'Retry-After' => '3' } } sent = send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + faults[:storage] << { status: 503, headers: { 'Retry-After' => '60' } } + error = refused_attachment(attachments: [{ filename: 'logo.png', content: png }]) expect(sent.id).to eq('msg_1') expect(delays.first).to be >= 3 expect([error.code, error.retry_after]).to eq(['attachment_upload_failed', 60.0]) expect(bucket).to eq(pdf_sha256 => pdf) - expect(a_request(:put, /./)).to have_been_made.times(3) + expect(storage_requests).to eq([pdf_sha256, pdf_sha256, png_sha256]) end it "keeps storage's Retry-After when its answer to an upload was cut off" do @@ -475,55 +488,53 @@ def accept_send(request) end it "sends nothing when PostHog refuses the uploads, and passes on PostHog's reason" do - stub_request(:post, uploads_url).to_return( - refuse(400, 'field_not_accepted', detail: "This project can't send files yet.", attr: 'files', - why: 'Not available yet.', fix: 'Send without files.') - ) + faults[:uploads] << refuse(400, 'field_not_accepted', detail: "This project can't send files yet.", + attr: 'files', why: 'Not available yet.', + fix: 'Send without files.') error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) expect([error.code, error.name, error.field, error.why, error.fix]) .to eq(['field_not_accepted', 'validation_error', 'files', 'Not available yet.', 'Send without files.']) - expect(a_request(:put, upload_url)).not_to have_been_made - expect(a_request(:post, send_url)).not_to have_been_made + expect(storage_requests).to be_empty + expect(outbox).to be_empty end it "doesn't retry a project over its daily upload bytes" do - stub_request(:post, uploads_url).to_return(refuse(429, 'upload_quota_exceeded')) + faults[:uploads] << refuse(429, 'upload_quota_exceeded') error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) expect(error.code).to eq('upload_quota_exceeded') - expect(a_request(:post, uploads_url)).to have_been_made.once + expect(posthog_requests.map(&:first)).to eq([:uploads]) end it "sends nothing when PostHog's answer doesn't say where each file goes" do - answers = [ + faults[:uploads].push( answer_uploads, answer_uploads(present_slot(png_sha256)), answer_uploads(upload_slot(pdf_sha256).merge(method: 'POST')), answer_uploads(upload_slot(pdf_sha256, url: 'file:///etc/passwd')), - answer_uploads(upload_slot(pdf_sha256).merge(headers: { 'Content-Length' => 12 })) - ] - stub_request(:post, uploads_url).to_return(*answers) + answer_uploads(upload_slot(pdf_sha256, headers: { 'Content-Length' => 12 })) + ) - errors = answers.map { refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) } + errors = Array.new(5) { refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) } expect(errors.map(&:code)).to all(eq('api_error')) - expect(a_request(:put, /./)).not_to have_been_made - expect(a_request(:post, send_url)).not_to have_been_made + expect(storage_requests).to be_empty + expect(outbox).to be_empty end it 'sends no file without TLS when PostHog is reached over https' do - plain_url = 'http://files.example.com/team/99882c55?X-Amz-Signature=abc' - stub_request(:post, uploads_url).to_return(answer_uploads(upload_slot(pdf_sha256, url: plain_url))) + plain_url = "http://files.example.com/team/#{pdf_sha256}?size=12&X-Amz-Signature=fake" + faults[:uploads] << answer_uploads(upload_slot(pdf_sha256, url: plain_url)) error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) expect([error.code, error.field]).to eq(%w[api_error attachments]) expect([error.message, error.fix]).to all(include('https')) - expect(a_request(:put, /./)).not_to have_been_made - expect(a_request(:post, send_url)).not_to have_been_made + expect(storage_requests).to be_empty + expect(outbox).to be_empty end it 'uploads over http when PostHog itself is reached over http' do @@ -532,7 +543,7 @@ def accept_send(request) expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(a_request(:put, %r{\Ahttp://storage\.test:8333/})).to have_been_made.once + expect(storage_requests).to eq([pdf_sha256]) end it 'refuses more files or bytes than one email carries before any request' do @@ -550,7 +561,7 @@ def accept_send(request) expect([too_large, too_large_with_body].map { |error| [error.code, error.name, error.field] }) .to all(eq(%w[message_too_large validation_error attachments])) expect(too_large.fix).to include('40 MB per email after Base64 encoding, about 29 MB of files') - expect(a_request(:any, %r{/messaging/})).not_to have_been_made + expect(posthog_requests).to be_empty end it 'counts the files before it reads any of them' do @@ -563,7 +574,6 @@ def accept_send(request) it 'uploads the largest file one email carries' do largest_that_fits = ("\0" * 29_182_875).b - fake_posthog expect(send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: largest_that_fits }]).id) .to eq('msg_1') @@ -573,18 +583,15 @@ def accept_send(request) it 'reads the largest file one email carries wrapped as Base64, and sends it' do largest_that_fits = ("\0" * 29_182_875).b wrapped = "#{[largest_that_fits].pack('m0').scan(/.{1,76}/).join("\n")}\n" - fake_posthog result = send_code(subject: '', html: '', attachments: [{ filename: 'scan.pdf', content: wrapped }]) expect(result.id).to eq('msg_1') expect(bucket).to eq(largest_that_fits_sha256 => largest_that_fits) - expect(sent_bodies.first['attachments']).to eq([{ 'sha256' => largest_that_fits_sha256, - 'filename' => 'scan.pdf' }]) + expect(outbox.first['attachments']).to eq([{ 'sha256' => largest_that_fits_sha256, 'filename' => 'scan.pdf' }]) end it "counts each body like a file: its UTF-8 bytes' Base64 size with line breaks" do - fake_posthog leaving_81_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] leaving_159_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] ascii_that_fits = 'a' * 57 @@ -604,11 +611,10 @@ def accept_send(request) expect(refused.map(&:code)).to eq(%w[message_too_large message_too_large]) expect(sent.map(&:id)).to eq(%w[msg_1 msg_1]) - expect(sent_bodies.map { |body| body['text'] }).to eq([nil, 'é' * 57]) + expect(outbox.map { |email| email['text'] }).to eq([nil, 'é' * 57]) end it 'counts the subject in 75-byte steps per 42 of its UTF-8 bytes, and an empty subject not at all' do - fake_posthog leaving_77_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_821).b }] leaving_3_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_875).b }] one_ascii_word = 'a' * 42 @@ -661,7 +667,7 @@ def accept_send(request) expect(error.message).to match(message) expect(error.message).not_to include('secret text') end - expect(a_request(:any, %r{/messaging/})).not_to have_been_made + expect(posthog_requests).to be_empty end it 'refuses an address list that is not one', :aggregate_failures do @@ -674,7 +680,7 @@ def accept_send(request) expect { send_template(field => []) } .to raise_error(ArgumentError, /template_id doesn't take #{field}.*send with template_key/) end - expect(a_request(:any, %r{/messaging/})).not_to have_been_made + expect(posthog_requests).to be_empty end end From ab93bd53268129086bc191c0ac2fc52f755393ec Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:37:55 +0000 Subject: [PATCH 26/31] test(emails): assert stored files, upload quota and accepted email, not requests The fake PostHog counts the bytes of each upload slot it issues against the team's daily quota, as the upload route does, so a file uploaded twice shows in the quota. The tests no longer read any request log. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 50 ++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 0fe187d1..17c1b4fc 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -209,8 +209,7 @@ def send_code(**overrides) let(:five_pdfs_sha256) { 'dfb00ae2891aa7dac580fd90519992e7a8d6914de7a683c0ecc071f65868101e' } let(:largest_that_fits_sha256) { '28f197eae8bd57e7c2a94d2062d2cef0a04b34e7fd3e0c743d88cda2217f069c' } let(:bucket) { {} } - let(:storage_requests) { [] } - let(:posthog_requests) { [] } + let(:quota) { { used: 0 } } let(:outbox) { [] } let(:faults) { { uploads: [], storage: [], send: [] } } @@ -224,7 +223,7 @@ def fake_posthog(app: 'https://us.posthog.com', storage: 'https://files.example. end def plan_uploads(request, storage) - files = record(:uploads, request)['files'] + files = JSON.parse(request.body)['files'] return faults[:uploads].shift if faults[:uploads].any? return refuse(401, 'authentication_failed') unless signed_in?(request) return refuse(400, 'invalid_request', attr: 'files') unless files.all? { |file| manifest_entry?(file) } @@ -233,7 +232,10 @@ def plan_uploads(request, storage) end def plan(file, storage) - bucket.key?(file['sha256']) ? present_slot(file['sha256']) : slot(file, storage) + return present_slot(file['sha256']) if bucket.key?(file['sha256']) + + quota[:used] += file['size'] + slot(file, storage) end def manifest_entry?(file) @@ -248,7 +250,6 @@ def slot(file, storage) def store(request) sha256 = request.uri.path[/\h{64}\z/] - storage_requests << sha256 return faults[:storage].shift if faults[:storage].any? return storage_error(403, 'AccessDenied') if request.headers.key?('Authorization') @@ -267,7 +268,7 @@ def storage_error(status, code) end def accept_send(request) - email = record(:send, request) + email = JSON.parse(request.body) return faults[:send].shift if faults[:send].any? return refuse(401, 'authentication_failed') unless signed_in?(request) unless email.fetch('attachments', []).all? { |file| bucket.key?(file['sha256']) } @@ -278,10 +279,6 @@ def accept_send(request) accept end - def record(route, request) - JSON.parse(request.body).tap { |body| posthog_requests << [route, body] } - end - def signed_in?(request) request.headers['Authorization'] == 'Bearer phs_test' end @@ -323,7 +320,7 @@ def refused_attachment(**overrides) expect(result.id).to eq('msg_1') expect(bucket).to eq(png_sha256 => png, pdf_sha256 => pdf) - expect(storage_requests).to eq([pdf_sha256]) + expect(quota[:used]).to eq(pdf.bytesize) expect(outbox.first['attachments']).to eq([ { 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf', 'content_type' => 'application/pdf' }, @@ -333,10 +330,11 @@ def refused_attachment(**overrides) end it 'leaves out empty lists, so a PostHog version without these fields still sends the email' do + faults[:uploads] << refuse(404, 'not_found') + send_code(cc: [], bcc: [], attachments: []) expect(outbox.first.keys).not_to include('cc', 'bcc', 'attachments') - expect(posthog_requests.map(&:first)).to eq([:send]) end it 'uploads once and sends the same files on every attempt of the send' do @@ -345,9 +343,8 @@ def refused_attachment(**overrides) expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(storage_requests).to eq([pdf_sha256]) - attempts = posthog_requests.filter_map { |route, email| email['attachments'] if route == :send } - expect(attempts).to eq([[{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }]] * 2) + expect(quota[:used]).to eq(pdf.bytesize) + expect(outbox.first['attachments']).to eq([{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }]) end it 'takes the file as bytes, a list of byte values, or Base64 in any line layout' do @@ -383,8 +380,7 @@ def refused_attachment(**overrides) send_code(attachments: [{ filename: 'a.pdf', content: pdf }, { filename: 'b.pdf', content: pdf }]) expect(bucket).to eq(pdf_sha256 => pdf) - expect(storage_requests).to eq([pdf_sha256]) - expect(posthog_requests.first).to eq([:uploads, { 'files' => [{ 'sha256' => pdf_sha256, 'size' => 12 }] }]) + expect(quota[:used]).to eq(pdf.bytesize) expect(outbox.first['attachments'].map { |file| file['filename'] }).to eq(%w[a.pdf b.pdf]) end @@ -433,7 +429,7 @@ def refused_attachment(**overrides) expect(error.message).to eq('Endpoint not found.') expect([proxied.code, proxied.name, proxied.status_code, proxied.field]) .to eq(['not_found', 'not_found', 404, 'attachments']) - expect(storage_requests).to be_empty + expect(bucket).to be_empty expect(outbox).to be_empty end @@ -454,7 +450,7 @@ def refused_attachment(**overrides) expect(send_code(max_retries: 4, attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(storage_requests).to eq([pdf_sha256] * 5) + expect(faults[:storage]).to be_empty end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do @@ -468,7 +464,6 @@ def refused_attachment(**overrides) expect(delays.first).to be >= 3 expect([error.code, error.retry_after]).to eq(['attachment_upload_failed', 60.0]) expect(bucket).to eq(pdf_sha256 => pdf) - expect(storage_requests).to eq([pdf_sha256, pdf_sha256, png_sha256]) end it "keeps storage's Retry-After when its answer to an upload was cut off" do @@ -496,7 +491,7 @@ def refused_attachment(**overrides) expect([error.code, error.name, error.field, error.why, error.fix]) .to eq(['field_not_accepted', 'validation_error', 'files', 'Not available yet.', 'Send without files.']) - expect(storage_requests).to be_empty + expect(bucket).to be_empty expect(outbox).to be_empty end @@ -506,7 +501,7 @@ def refused_attachment(**overrides) error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) expect(error.code).to eq('upload_quota_exceeded') - expect(posthog_requests.map(&:first)).to eq([:uploads]) + expect(outbox).to be_empty end it "sends nothing when PostHog's answer doesn't say where each file goes" do @@ -521,7 +516,7 @@ def refused_attachment(**overrides) errors = Array.new(5) { refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) } expect(errors.map(&:code)).to all(eq('api_error')) - expect(storage_requests).to be_empty + expect(bucket).to be_empty expect(outbox).to be_empty end @@ -533,7 +528,7 @@ def refused_attachment(**overrides) expect([error.code, error.field]).to eq(%w[api_error attachments]) expect([error.message, error.fix]).to all(include('https')) - expect(storage_requests).to be_empty + expect(bucket).to be_empty expect(outbox).to be_empty end @@ -543,7 +538,6 @@ def refused_attachment(**overrides) expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(storage_requests).to eq([pdf_sha256]) end it 'refuses more files or bytes than one email carries before any request' do @@ -561,7 +555,7 @@ def refused_attachment(**overrides) expect([too_large, too_large_with_body].map { |error| [error.code, error.name, error.field] }) .to all(eq(%w[message_too_large validation_error attachments])) expect(too_large.fix).to include('40 MB per email after Base64 encoding, about 29 MB of files') - expect(posthog_requests).to be_empty + expect([quota[:used], bucket, outbox]).to eq([0, {}, []]) end it 'counts the files before it reads any of them' do @@ -667,7 +661,7 @@ def refused_attachment(**overrides) expect(error.message).to match(message) expect(error.message).not_to include('secret text') end - expect(posthog_requests).to be_empty + expect([quota[:used], bucket, outbox]).to eq([0, {}, []]) end it 'refuses an address list that is not one', :aggregate_failures do @@ -680,7 +674,7 @@ def refused_attachment(**overrides) expect { send_template(field => []) } .to raise_error(ArgumentError, /template_id doesn't take #{field}.*send with template_key/) end - expect(posthog_requests).to be_empty + expect([quota[:used], bucket, outbox]).to eq([0, {}, []]) end end From ef6f0ad30e943d6a3f5877fffe1de878b3a4a6c6 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:45:48 +0000 Subject: [PATCH 27/31] test(emails): meter the bytes storage writes, so a repeated upload shows The fake storage adds each written file's bytes to a meter, as object storage reports bytes uploaded, so a file sent to storage twice fails the tests. The retry test now rests on the send succeeding and the stored file. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 17c1b4fc..9993fbad 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -209,7 +209,7 @@ def send_code(**overrides) let(:five_pdfs_sha256) { 'dfb00ae2891aa7dac580fd90519992e7a8d6914de7a683c0ecc071f65868101e' } let(:largest_that_fits_sha256) { '28f197eae8bd57e7c2a94d2062d2cef0a04b34e7fd3e0c743d88cda2217f069c' } let(:bucket) { {} } - let(:quota) { { used: 0 } } + let(:usage) { { quota_bytes: 0, stored_bytes: 0 } } let(:outbox) { [] } let(:faults) { { uploads: [], storage: [], send: [] } } @@ -234,7 +234,7 @@ def plan_uploads(request, storage) def plan(file, storage) return present_slot(file['sha256']) if bucket.key?(file['sha256']) - quota[:used] += file['size'] + usage[:quota_bytes] += file['size'] slot(file, storage) end @@ -259,6 +259,7 @@ def store(request) return storage_error(403, 'SignatureDoesNotMatch') unless signed return storage_error(400, 'BadDigest') if Digest::SHA256.hexdigest(bytes) != sha256 + usage[:stored_bytes] += bytes.bytesize bucket[sha256] = bytes { status: 200 } end @@ -320,7 +321,7 @@ def refused_attachment(**overrides) expect(result.id).to eq('msg_1') expect(bucket).to eq(png_sha256 => png, pdf_sha256 => pdf) - expect(quota[:used]).to eq(pdf.bytesize) + expect(usage).to eq(quota_bytes: pdf.bytesize, stored_bytes: pdf.bytesize) expect(outbox.first['attachments']).to eq([ { 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf', 'content_type' => 'application/pdf' }, @@ -343,7 +344,7 @@ def refused_attachment(**overrides) expect(send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(quota[:used]).to eq(pdf.bytesize) + expect(usage).to eq(quota_bytes: pdf.bytesize, stored_bytes: pdf.bytesize) expect(outbox.first['attachments']).to eq([{ 'sha256' => pdf_sha256, 'filename' => 'invoice.pdf' }]) end @@ -380,7 +381,7 @@ def refused_attachment(**overrides) send_code(attachments: [{ filename: 'a.pdf', content: pdf }, { filename: 'b.pdf', content: pdf }]) expect(bucket).to eq(pdf_sha256 => pdf) - expect(quota[:used]).to eq(pdf.bytesize) + expect(usage).to eq(quota_bytes: pdf.bytesize, stored_bytes: pdf.bytesize) expect(outbox.first['attachments'].map { |file| file['filename'] }).to eq(%w[a.pdf b.pdf]) end @@ -450,7 +451,6 @@ def refused_attachment(**overrides) expect(send_code(max_retries: 4, attachments: [{ filename: 'invoice.pdf', content: pdf }]).id).to eq('msg_1') expect(bucket).to eq(pdf_sha256 => pdf) - expect(faults[:storage]).to be_empty end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do @@ -555,7 +555,7 @@ def refused_attachment(**overrides) expect([too_large, too_large_with_body].map { |error| [error.code, error.name, error.field] }) .to all(eq(%w[message_too_large validation_error attachments])) expect(too_large.fix).to include('40 MB per email after Base64 encoding, about 29 MB of files') - expect([quota[:used], bucket, outbox]).to eq([0, {}, []]) + expect([usage.values.sum, bucket, outbox]).to eq([0, {}, []]) end it 'counts the files before it reads any of them' do @@ -661,7 +661,7 @@ def refused_attachment(**overrides) expect(error.message).to match(message) expect(error.message).not_to include('secret text') end - expect([quota[:used], bucket, outbox]).to eq([0, {}, []]) + expect([usage.values.sum, bucket, outbox]).to eq([0, {}, []]) end it 'refuses an address list that is not one', :aggregate_failures do @@ -674,7 +674,7 @@ def refused_attachment(**overrides) expect { send_template(field => []) } .to raise_error(ArgumentError, /template_id doesn't take #{field}.*send with template_key/) end - expect([quota[:used], bucket, outbox]).to eq([0, {}, []]) + expect([usage.values.sum, bucket, outbox]).to eq([0, {}, []]) end end From 8e2b20a88f97fd244ed1299e1d9665b142c6f173 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:25:48 +0000 Subject: [PATCH 28/31] test(emails): prove storage's Retry-After by storage staying busy, not by sleep's argument The attachment tests run on a fake clock that the client's waits advance. Fake storage answers 503 SlowDown with its Retry-After until that time has passed, so a client that retries early fails the send. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 9993fbad..3d8852e4 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -212,6 +212,8 @@ def send_code(**overrides) let(:usage) { { quota_bytes: 0, stored_bytes: 0 } } let(:outbox) { [] } let(:faults) { { uploads: [], storage: [], send: [] } } + let(:clock) { { now: 0.0, storage_busy_until: 0.0 } } + let(:emails) { client.emails.tap { |emails| allow(emails).to receive(:sleep) { |seconds| pass(seconds) } } } before { fake_posthog } @@ -251,6 +253,7 @@ def slot(file, storage) def store(request) sha256 = request.uri.path[/\h{64}\z/] return faults[:storage].shift if faults[:storage].any? + return storage_busy if clock[:now] < clock[:storage_busy_until] return storage_error(403, 'AccessDenied') if request.headers.key?('Authorization') bytes = request.body.b @@ -264,6 +267,15 @@ def store(request) { status: 200 } end + def pass(seconds) + clock[:now] += seconds + end + + def storage_busy + retry_after = (clock[:storage_busy_until] - clock[:now]).ceil + storage_error(503, 'SlowDown').merge(headers: { 'Retry-After' => retry_after.to_s }) + end + def storage_error(status, code) { status: status, body: "#{code}" } end @@ -454,14 +466,13 @@ def refused_attachment(**overrides) end it "waits for storage's Retry-After, and stops when it asks for longer than 30 seconds" do - faults[:storage] << { status: 503, headers: { 'Retry-After' => '3' } } - sent = send_code(attachments: [{ filename: 'invoice.pdf', content: pdf }]) - faults[:storage] << { status: 503, headers: { 'Retry-After' => '60' } } + clock[:storage_busy_until] = 3.0 + sent = send_code(max_retries: 1, attachments: [{ filename: 'invoice.pdf', content: pdf }]) + clock[:storage_busy_until] = clock[:now] + 60 - error = refused_attachment(attachments: [{ filename: 'logo.png', content: png }]) + error = refused_attachment(max_retries: 1, attachments: [{ filename: 'logo.png', content: png }]) expect(sent.id).to eq('msg_1') - expect(delays.first).to be >= 3 expect([error.code, error.retry_after]).to eq(['attachment_upload_failed', 60.0]) expect(bucket).to eq(pdf_sha256 => pdf) end From 011bdd90c72baff9660a4e47627b3b34da0e0399 Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 07:58:33 +0000 Subject: [PATCH 29/31] fix(emails): count derived text, and name the file an http upload URL would carry When text is empty, PostHog derives it from the HTML and counts it, so the local estimate counts the HTML twice; the derived text is never longer. An upload URL without https now refuses on the file's field and names the file, never the URL, unless app_host is an http URL for local development. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- .changeset/emails-attachments.md | 2 +- lib/posthog/emails.rb | 32 ++++++++++++++---------- spec/posthog/emails_spec.rb | 43 +++++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/.changeset/emails-attachments.md b/.changeset/emails-attachments.md index d2004638..8a7ba290 100644 --- a/.changeset/emails-attachments.md +++ b/.changeset/emails-attachments.md @@ -2,4 +2,4 @@ "posthog-ruby": minor --- -`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files; the subject and bodies count towards the 40 MB as PostHog counts them. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. With `app_host` on https, files only go to storage over https: an upload URL without https raises `api_error` on `attachments` before any file goes out. +`client.emails.send` with `template_key:` takes `cc:`, `bcc:` and `attachments:`. `cc` and `bcc` take one address or a list. Each attachment is a Hash with `filename`, `content`, and an optional `content_type` and `content_id` (`content_id` shows the file inline). Pass the file's bytes as `content`, such as `File.binread('invoice.pdf')` or a list of byte values; any other String is taken as the file encoded as Base64, on one line or wrapped at 60, 64 or 76 characters. An email carries at most 10 files and 40 MB after Base64 encoding, about 29 MB of files; the subject and bodies count towards the 40 MB as PostHog counts them, the HTML twice when there is no text, because PostHog derives the text from it. The client hashes each file, uploads the files PostHog doesn't have yet straight to PostHog's file storage, and then sends the email with the files' SHA-256 hashes; a file the project sent before uploads nothing again. A file the client can't send raises `PostHog::EmailError` before any request, with code `invalid_attachment`, `too_many_attachments` or `message_too_large`. An upload that file storage refuses raises `attachment_upload_failed`, and a PostHog without the upload route for the project answers `not_found` on `attachments`; either way no email is sent. Files go to storage only over https, unless `app_host` is an http URL for local development: an upload URL without https raises `api_error` on the file's field, such as `attachments__0`, before any file goes out. diff --git a/lib/posthog/emails.rb b/lib/posthog/emails.rb index 8eb90183..4200de20 100644 --- a/lib/posthog/emails.rb +++ b/lib/posthog/emails.rb @@ -172,9 +172,9 @@ def initialize(secret_key:, host:, app_host:, timeout:, max_retries:, migrated_f # Before any request, {EmailError#code} is `invalid_attachment` for a file the client can't send, # `too_many_attachments` for more than 10 files, and `message_too_large` for files too large for one email. # It is `attachment_upload_failed` when file storage refused an upload, and `not_found` with - # {EmailError#field} `attachments` when PostHog has no upload route for this project. With `app_host` on - # https, an upload URL without https raises `api_error` on `attachments` before any file goes out. No email - # was sent. + # {EmailError#field} `attachments` when PostHog has no upload route for this project. Unless `app_host` is + # an http URL, an upload URL without https raises `api_error` on the file's field, such as + # `attachments__0`, before any file goes out. No email was sent. # @raise [ArgumentError] An argument has the wrong type or format, or the call mixes the two forms. def send( to:, template_id: nil, template_data: nil, template_key: nil, subject: nil, html: nil, text: nil, from: nil, @@ -187,8 +187,7 @@ def send( }.compact check_recipient(to, distinct_id, idempotency_key, max_retries) payload = content(template_id, template_key, fields).merge(to: to) - files = EmailAttachments.from(fields.fetch(:attachments, []), fields.values_at(:html, :text).compact, - fields[:subject]) + files = EmailAttachments.from(fields.fetch(:attachments, []), bodies_with_derived_text(fields), fields[:subject]) payload[:attachments] = files.references unless files.empty? payload[:distinct_id] = distinct_id unless distinct_id.nil? body = JSON.generate(payload) @@ -263,6 +262,11 @@ def uuid?(value) value.is_a?(String) && UUID_PATTERN.match?(value) end + def bodies_with_derived_text(fields) + html, text = fields.values_at(:html, :text) + [html, text.to_s.empty? ? html : text].compact + end + def resolve_app_host(host, app_host) explicit = app_host.is_a?(String) ? app_host.strip.chomp('/') : '' return CLOUD_APP_HOSTS[host.to_s.strip.chomp('/')] if explicit.empty? @@ -467,7 +471,8 @@ def parse_uploads_reply((status, payload, retry_after), files) raise EmailError.new(code: 'api_error', message: "PostHog answered without the files' uploads.", status_code: status) end - raise upload_without_tls(status) if slots.any? { |slot| without_tls?(slot) } + file_without_tls, = files.zip(slots).find { |_, slot| without_tls?(slot) } + raise upload_without_tls(file_without_tls, status) if file_without_tls slots end @@ -476,13 +481,14 @@ def without_tls?(slot) slot['status'] == 'upload' && URI(@app_host).scheme == 'https' && URI(slot['url']).scheme != 'https' end - def upload_without_tls(status) - EmailError.new(code: 'api_error', field: 'attachments', status_code: status, - message: 'PostHog gave an upload URL without https, so no file was uploaded and no email was ' \ - 'sent.', - why: 'Your app_host uses https, and the files would have gone to storage without TLS.', - fix: "Serve your PostHog's file storage over https, or reach PostHog over http in local " \ - 'development.') + def upload_without_tls(file, status) + filename = file.reference['filename'] + EmailError.new(code: 'api_error', field: file.field, status_code: status, + message: "PostHog gave an upload URL for #{filename} that isn't https, so no file was uploaded " \ + 'and no email was sent.', + why: "Your app_host uses https, and #{filename} would have gone to file storage unencrypted.", + fix: "Fix the file storage's URL so it uses https. For local development over http, set " \ + 'app_host to an http URL.') end def slot?(file, slot) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 3d8852e4..e097b059 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -531,19 +531,22 @@ def refused_attachment(**overrides) expect(outbox).to be_empty end - it 'sends no file without TLS when PostHog is reached over https' do - plain_url = "http://files.example.com/team/#{pdf_sha256}?size=12&X-Amz-Signature=fake" - faults[:uploads] << answer_uploads(upload_slot(pdf_sha256, url: plain_url)) + it "sends no file to storage without https, naming the file and never the storage's URL" do + fake_posthog(storage: 'http://files.example.com/team') + bucket[png_sha256] = png - error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + error = refused_attachment(attachments: [{ filename: 'logo.png', content: png }, + { filename: 'invoice.pdf', content: pdf }]) - expect([error.code, error.field]).to eq(%w[api_error attachments]) - expect([error.message, error.fix]).to all(include('https')) - expect(bucket).to be_empty + expect([error.code, error.field]).to eq(%w[api_error attachments__1]) + expect(error.message).to include('invoice.pdf') + expect(error.fix).to include("Fix the file storage's URL") + expect([error.message, error.why, error.fix].join).not_to include('files.example.com') + expect(bucket).to eq(png_sha256 => png) expect(outbox).to be_empty end - it 'uploads over http when PostHog itself is reached over http' do + it 'uploads over http when app_host is an http URL, for local development' do options.merge!(host: 'http://127.0.0.1:9', app_host: 'http://posthog.test') fake_posthog(app: 'http://posthog.test', storage: 'http://storage.test:8333/team') @@ -597,7 +600,6 @@ def refused_attachment(**overrides) end it "counts each body like a file: its UTF-8 bytes' Base64 size with line breaks" do - leaving_81_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] leaving_159_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] ascii_that_fits = 'a' * 57 ascii_one_too_many = 'a' * 58 @@ -605,18 +607,35 @@ def refused_attachment(**overrides) two_byte_one_too_many = 'é' * 58 sent = [ - send_code(subject: '', html: ascii_that_fits, attachments: leaving_81_bytes), + send_code(subject: '', html: ascii_that_fits, text: ascii_that_fits, attachments: leaving_159_bytes), send_code(subject: '', html: '', text: two_byte_that_fits.encode(Encoding::UTF_16LE), attachments: leaving_159_bytes) ] refused = [ - refused_attachment(subject: '', html: ascii_one_too_many, attachments: leaving_81_bytes), + refused_attachment(subject: '', html: ascii_one_too_many, text: ascii_that_fits, + attachments: leaving_159_bytes), refused_attachment(subject: '', html: '', text: two_byte_one_too_many, attachments: leaving_159_bytes) ] expect(refused.map(&:code)).to eq(%w[message_too_large message_too_large]) expect(sent.map(&:id)).to eq(%w[msg_1 msg_1]) - expect(outbox.map { |email| email['text'] }).to eq([nil, 'é' * 57]) + expect(outbox.map { |email| email['text'] }).to eq(['a' * 57, 'é' * 57]) + end + + it 'counts the HTML twice when there is no text, because PostHog derives the text from it' do + leaving_81_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_818).b }] + leaving_159_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_761).b }] + html = 'a' * 57 + + sent = send_code(subject: '', html: html, attachments: leaving_159_bytes) + refused = [nil, ''].map do |text| + refused_attachment(subject: '', html: html, text: text, attachments: leaving_81_bytes) + end + + expect(sent.id).to eq('msg_1') + expect(refused.map(&:code)).to all(eq('message_too_large')) + expect(usage[:quota_bytes]).to eq(29_182_761) + expect(outbox.size).to eq(1) end it 'counts the subject in 75-byte steps per 42 of its UTF-8 bytes, and an empty subject not at all' do From 8e2bc6389f0e10393eb0faf574ae657e8d51625d Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:21:49 +0000 Subject: [PATCH 30/31] fix(emails): leave the size of an email without files to PostHog The local 40 MB estimate guards uploads, so it now runs only when the email has files, as in the Go SDK. Large HTML without files goes to PostHog as before. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- lib/posthog/email_attachments.rb | 2 +- spec/posthog/emails_spec.rb | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/posthog/email_attachments.rb b/lib/posthog/email_attachments.rb index d51e326c..75b1a8b9 100644 --- a/lib/posthog/email_attachments.rb +++ b/lib/posthog/email_attachments.rb @@ -150,7 +150,7 @@ def distinct private def check_size(bodies, subject) - return if message_bytes(bodies, subject) <= MAX_MESSAGE_BYTES + return if empty? || message_bytes(bodies, subject) <= MAX_MESSAGE_BYTES raise EmailError.new( code: 'message_too_large', field: 'attachments', diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index e097b059..0c533845 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -213,7 +213,9 @@ def send_code(**overrides) let(:outbox) { [] } let(:faults) { { uploads: [], storage: [], send: [] } } let(:clock) { { now: 0.0, storage_busy_until: 0.0 } } - let(:emails) { client.emails.tap { |emails| allow(emails).to receive(:sleep) { |seconds| pass(seconds) } } } + let(:emails) do + client.emails.tap { |emails| allow(emails).to receive(:sleep) { |seconds| clock[:now] += seconds } } + end before { fake_posthog } @@ -267,10 +269,6 @@ def store(request) { status: 200 } end - def pass(seconds) - clock[:now] += seconds - end - def storage_busy retry_after = (clock[:storage_busy_until] - clock[:now]).ceil storage_error(503, 'SlowDown').merge(headers: { 'Retry-After' => retry_after.to_s }) @@ -638,6 +636,15 @@ def refused_attachment(**overrides) expect(outbox.size).to eq(1) end + it 'leaves the size of an email without files to PostHog' do + html = "

#{'a' * 20_000_000}

" + + results = [nil, []].map { |attachments| send_code(subject: '', html: html, attachments: attachments) } + + expect(results.map(&:id)).to eq(%w[msg_1 msg_1]) + expect(outbox.map { |email| email['html'].bytesize }).to eq([html.bytesize] * 2) + end + it 'counts the subject in 75-byte steps per 42 of its UTF-8 bytes, and an empty subject not at all' do leaving_77_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_821).b }] leaving_3_bytes = [{ filename: 'scan.pdf', content: ("\0" * 29_182_875).b }] From 5659e3eb45ceb71f6d1f79ac87fd63d83e5952ed Mon Sep 17 00:00:00 2001 From: "Reichenbach, Michael" <755327+Silthus@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:47:00 +0000 Subject: [PATCH 31/31] test(emails): show an upload never follows storage's redirect Fake storage answers the PUT with a 307 to a second, plain-http storage. The second storage receives nothing, the attachment fails with attachment_upload_failed naming the file but not where storage pointed, and no email is sent. Net::HTTP doesn't follow redirects, so only the test is new. Refs PostHog/michaelr-context#83 Co-Authored-By: Claude Opus 5.5 --- spec/posthog/emails_spec.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/spec/posthog/emails_spec.rb b/spec/posthog/emails_spec.rb index 0c533845..f25e96d0 100644 --- a/spec/posthog/emails_spec.rb +++ b/spec/posthog/emails_spec.rb @@ -412,6 +412,28 @@ def refused_attachment(**overrides) expect(outbox).to be_empty end + it 'never follows storage redirecting an upload elsewhere, and names the file but not the URL' do + elsewhere = {} + stub_request(:put, %r{\Ahttp://elsewhere\.test/}).to_return do |request| + elsewhere[request.uri.to_s] = request.body + { status: 200 } + end + faults[:storage] << { + status: 307, headers: { 'Location' => "http://elsewhere.test/stolen/#{pdf_sha256}" }, + body: 'TemporaryRedirectPlease re-send this request to the specified ' \ + 'temporary endpoint.elsewhere.test' + } + + error = refused_attachment(attachments: [{ filename: 'invoice.pdf', content: pdf }]) + + expect([error.code, error.field, error.status_code]).to eq(['attachment_upload_failed', 'attachments__0', 307]) + expect(error.message).to include('invoice.pdf') + expect([error.message, error.why, error.fix].join).not_to include('elsewhere.test') + expect(elsewhere).to be_empty + expect(bucket).to be_empty + expect(outbox).to be_empty + end + it "names a refused upload by storage's failure, never by PostHog's categories" do faults[:storage].push({ status: 403, body: 'AccessDenied' }, { status: 401 }, { status: 503, body: 'SlowDown' })