Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ BASETEN_API_KEY=
BASETEN_API_BASE_URL=https://inference.baseten.co/v1
BASETEN_MODEL=meta-models/muse-spark-open-1026

# OpenCode Go key, for drawing-set reviews. Muse Spark there trains on drawings; see docs/inference.md.
# OpenCode Go key (oc_sk_...), the variable Pi and OpenCode read, for drawing-set reviews.
# Muse Spark there trains on drawings; see docs/inference.md.
OPENCODE_API_KEY=

# Setup and model reference: docs/inference.md and docs/models.md.
Expand Down
15 changes: 11 additions & 4 deletions backend/application.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import { ExecutionError } from "#lib/review/execution.ts";
import { CloudReviewError } from "#lib/review/cloud.ts";
import { CloudReviews, CloudSandbox, unavailableCloudSandbox } from "./cloud/service.ts";
import { modalSandboxLayer } from "./cloud/modal.ts";
import { isCloudProvider } from "./cloud/contracts.ts";
import { executionPersistence } from "./review/execution-store.ts";
import { cloudPackageLimits } from "./cloud/transfer.ts";
import { CloudReviewSynchronization } from "./cloud/sync.ts";
Expand Down Expand Up @@ -201,7 +202,9 @@ export class Application extends Context.Service<
const executionOptions = { owner, queries: operations, port, runPromise: entry.runPromise };

// Count checks stay on Meta or Baseten; see the guard in ReviewExecution.
const countProvider = providers.find((provider) => provider.provider !== "opencode-go");
const countProvider = providers.find(
(provider) => provider.provider === "meta" || provider.provider === "baseten",
);

const connected = {
...executionOptions,
Expand Down Expand Up @@ -393,10 +396,14 @@ export class Application extends Context.Service<
restorePending: pendingRestore,
};

// A custom endpoint's key stays on this computer; it never reaches the cloud.
const cloudProviders = providers.filter(isCloudProvider);
const [cloudProvider] = cloudProviders;

const withProvider =
inference === undefined
cloudProvider === undefined
? cloudOptions
: { ...cloudOptions, provider: inference, providers };
: { ...cloudOptions, provider: cloudProvider, providers: cloudProviders };

const cloud = yield* CloudReviews.pipe(
Effect.provide(
Expand All @@ -410,7 +417,7 @@ export class Application extends Context.Service<
cloudSandbox ??
(config.modal === undefined
? unavailableCloudSandbox
: modalSandboxLayer(config.modal, inference?.apiKey)),
: modalSandboxLayer(config.modal, cloudProvider?.apiKey)),
),
),
),
Expand Down
13 changes: 11 additions & 2 deletions backend/cloud/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,17 @@ export const CloudReviewRecord = Schema.Struct({
/** Persisted record type stays private to cloud application and adapters. */
export type CloudReviewRecord = typeof CloudReviewRecord.Type;

/** Provider injection uses the canonical application route and a separately supplied secret. */
export const CloudProvider = InferenceConfiguration;
/**
* Provider injection uses the canonical application route and a separately supplied secret. A
* custom endpoint is not a cloud route: its key stays on the computer that saved it.
*/
export const CloudProvider = Schema.Struct({
...InferenceConfiguration.fields,
provider: Schema.Literals(["meta", "baseten", "opencode-go"]),
});

/** Whether a configured provider may run cloud reviews. */
export const isCloudProvider = Schema.is(CloudProvider);

/** Cloud outbound policy admits exactly these provider routes. */
export const cloudProviderEndpoints = {
Expand Down
9 changes: 4 additions & 5 deletions backend/cloud/sandbox-port.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
import type { Effect, Stream } from "effect";
import type { InferenceConfiguration } from "#lib/backend-rpc.ts";
import type { CloudReviewError, CloudTransferProgress } from "#lib/review/cloud.ts";
import type { PackageFile } from "#lib/review/packages.ts";
import type { CloudCachedFile, CloudPendingRetain } from "./cache.ts";
import type { CloudReviewRecord } from "./contracts.ts";
import type { CloudProvider, CloudReviewRecord } from "./contracts.ts";
import type { CloudRecoveredWorkerProgress } from "./progress.ts";

/** Sandbox mechanics stay behind the cloud application's replay and ownership policy. */
export interface CloudSandboxPort {
/** Resolve the published image and every outbound provider secret without starting compute. */
readonly preflight: (
providers: ReadonlyArray<typeof InferenceConfiguration.Type>,
providers: ReadonlyArray<typeof CloudProvider.Type>,
) => Effect.Effect<void, CloudReviewError>;
/** Change signals are hints; callers reconcile the durable job after subscribing and on reconnect. */
readonly changes: (record: CloudReviewRecord) => Stream.Stream<void, CloudReviewError>;
Expand All @@ -23,8 +22,8 @@ export interface CloudSandboxPort {
) => Effect.Effect<string, CloudReviewError>;
readonly dispatch: (
record: CloudReviewRecord,
provider: typeof InferenceConfiguration.Type,
alternatives: ReadonlyArray<typeof InferenceConfiguration.Type>,
provider: typeof CloudProvider.Type,
alternatives: ReadonlyArray<typeof CloudProvider.Type>,
) => Effect.Effect<void, CloudReviewError>;
readonly progress: (
record: CloudReviewRecord,
Expand Down
17 changes: 13 additions & 4 deletions backend/cloud/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { randomUUID } from "node:crypto";
import type { InferenceConfiguration, ModalConfiguration } from "#lib/backend-rpc.ts";
import type { ModalConfiguration } from "#lib/backend-rpc.ts";
import {
CloudReviewError,
CloudReviewSetup,
Expand All @@ -21,9 +21,10 @@ import {
type CloudReviewRequest,
type CloudWorkspaceEstimate,
} from "#lib/review/cloud.ts";
import { reviewModels } from "#lib/review/models.ts";
import { reviewModels, runsInCloud } from "#lib/review/models.ts";
import {
cloudEngineIdentity,
type CloudProvider,
cloudProviderEndpoints,
cloudWorkerMinimumTimeoutMs,
type CloudReviewRecord,
Expand Down Expand Up @@ -63,8 +64,9 @@ interface WorkspaceVersions {
export interface CloudReviewConfiguration {
readonly directory: string;
readonly modal?: typeof ModalConfiguration.Type;
readonly provider?: typeof InferenceConfiguration.Type;
readonly providers?: ReadonlyArray<typeof InferenceConfiguration.Type>;
/** Never a custom endpoint; the parent keeps its key on this computer. */
readonly provider?: typeof CloudProvider.Type;
readonly providers?: ReadonlyArray<typeof CloudProvider.Type>;
readonly inspect: (
request: typeof CloudReviewRequest.Type,
) => Effect.Effect<WorkspaceVersions, CloudReviewError>;
Expand Down Expand Up @@ -546,6 +548,13 @@ export class CloudReviews extends Context.Service<

const model = reviewModels.find((item) => item.id === request.model);

if (model !== undefined && !runsInCloud(model.provider))
return yield* new CloudReviewError({
reason: "configuration",
message:
"A custom endpoint's key stays on this computer, so it runs reviews here only. Run this review here, or choose a Meta, Baseten or OpenCode Go model to push it to the cloud. Your workspace has not been uploaded.",
});

const provider =
request.model === undefined
? config.provider
Expand Down
11 changes: 8 additions & 3 deletions backend/inference/pacing.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { ReviewProvider } from "#lib/review/models.ts";
import { Option, Predicate, Redacted, Schema } from "effect";
import { createHash } from "node:crypto";
import type { ReviewProvider } from "#lib/review/models.ts";

/** Per-minute request and token allowances a provider enforces for one credential. */
export interface RateLimits {
Expand All @@ -15,8 +15,13 @@ export interface RateLimits {
export const reportedRateLimits = {
meta: { requestsPerMinute: 3000, tokensPerMinute: 4_000_000 },
baseten: { requestsPerMinute: 100, tokensPerMinute: 2_000_000 },
// Go publishes dollar allowances, not per-minute limits; this is a cautious starting point.
"opencode-go": { requestsPerMinute: 300, tokensPerMinute: 4_000_000 },
// OpenCode Go publishes no request rate; it asks for typical coding agent traffic and monitors for
// abuse. This ceiling is MOKE's own choice, well above three reviewers' measured 8 requests a
// minute, so a burst of quick answers still looks like one person's agents.
"opencode-go": { requestsPerMinute: 60, tokensPerMinute: 5_000_000 },
// A custom endpoint's limits are unknown until it sends these headers. A 429 starts the shared
// cooldown, so the generous start is corrected on first refusal.
"openai-compatible": { requestsPerMinute: 600, tokensPerMinute: 10_000_000 },
} as const satisfies Record<typeof ReviewProvider.Type, RateLimits>;

/** Monotonic time and timers; tests supply a manual clock. */
Expand Down
Loading
Loading