From 067c86d45f8bafd44668811b88dad5200efb2ec6 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:44:13 -0600 Subject: [PATCH 1/4] feat(bps): align with SWIP-74 rev 8 (BPS-lite) - wire: Join{cohort, addr}, Ack{status, challenge}, Broadcast{soc, kind, challenge, index}; protocol id pubsub/1.0.0 - per-stream random 32-byte challenge salts the session feed topic; the first valid frame (DATA or empty AUTH) claims the publisher role - pending admin streams outside the fan-out bound with a claim deadline - per-cohort cursor replaces dedup; ordered validation with violations reset and blocklisted - resource bounds, inactivity reclaim, per-cohort counters and metrics - subscribers re-verify deliveries against spec and cursor - API websocket bridge and OpenAPI updated to the new frames Co-Authored-By: Claude Opus 5.5 --- openapi/Swarm.yaml | 47 ++- pkg/api/api_test.go | 3 +- pkg/api/bps.go | 184 ++++----- pkg/api/bps_test.go | 320 +++++++--------- pkg/api/export_test.go | 2 +- pkg/bps/TODO | 62 +-- pkg/bps/binding.go | 81 ---- pkg/bps/bps.go | 849 +++++++++++++++++++++++++---------------- pkg/bps/bps_test.go | 479 ++++++++++++++++++++--- pkg/bps/export_test.go | 13 + pkg/bps/id.go | 111 ++++++ pkg/bps/metrics.go | 88 +++++ pkg/bps/pb/bps.pb.go | 722 ++++++++++++++++++++++++++--------- pkg/bps/pb/bps.proto | 55 ++- pkg/node/node.go | 6 +- 15 files changed, 1976 insertions(+), 1046 deletions(-) delete mode 100644 pkg/bps/binding.go create mode 100644 pkg/bps/export_test.go create mode 100644 pkg/bps/id.go create mode 100644 pkg/bps/metrics.go diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index b4963bbada3..9a146f5d48a 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -979,9 +979,12 @@ paths: get: summary: Subscribe to a BPS cohort description: | - WebSocket. The node joins the cohort `keccak256(keccak256("bps-claim" | topic) | owner)` - at `broker` and sends every broadcast as a binary frame with the full SOC bytes - (`id | signature | span | payload`). Broadcasts not signed by `owner` are dropped. + WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` + at `broker` as `identity` and sends every update as a binary frame + `challenge (32) | index (8, big-endian) | soc`, where `soc` is the full single-owner + chunk (`id | signature | span | payload`) signed by `owner` with + `id = keccak256(keccak256(topic | challenge) | index)`. Updates that do not verify, + or whose index is below the subscriber's cursor, are dropped. The endpoint is read-only: any client frame closes the socket with 4002. Close codes: 4002 invalid message, 4003 broker stream closed. tags: @@ -1011,6 +1014,12 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" required: true description: Ethereum address identifying the joiner. + - in: query + name: cursor + schema: + type: integer + required: false + description: Lowest update index to deliver. Defaults to 0. responses: "200": description: Establishes a WebSocket subscription for incoming broadcasts @@ -1025,19 +1034,21 @@ paths: "/bps/publish/{owner}/{topic}": get: - summary: Claim and publish to a BPS cohort + summary: Publish to a BPS cohort as its admin description: | - WebSocket. Text frames carry JSON control messages; binary frames carry SOCs. - - 1. Server sends `{"type":"challenge","challenge":"","broker":"","id":""}`. - 2. Client signs a SOC with `id` wrapping the payload `challenge | broker` and sends - `{"type":"claim","signature":""}` within 30 seconds. - 3. Server verifies the claim locally, forwards it to the broker and sends - `{"type":"claim_sent"}`. This does not mean the broker accepted it; a rejection - closes the socket with 4003. - 4. Client sends each broadcast as a binary frame containing a full SOC at the cohort address. - - Close codes: 4001 invalid claim, 4002 invalid message (bad SOC, wrong frame type, + WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` + at `broker` declaring `owner` as its address. + + 1. Server sends `{"type":"challenge","challenge":"","broker":"","dataTopic":"","authTopic":""}`. + `dataTopic` is `keccak256(topic | challenge)`, the session feed for updates; + `authTopic` is `keccak256("bps-service:v1" | 0x02 | topic | challenge)`. + 2. Client sends binary frames `kind (1) | index (8, big-endian) | soc`, where `kind` is + 1 (DATA) or 2 (AUTH) and `soc` is a single-owner chunk signed by `owner` with + `id = keccak256(sessionTopic | index)`. An AUTH chunk is empty and only claims the + stream. The first frame must arrive within 30 seconds. Indexes must not be reused + across sessions; frames below the broker's cursor are dropped silently. + + Close codes: 4001 invalid soc, 4002 invalid message (wrong frame type, unknown kind, claim timeout), 4003 broker stream closed. tags: - BPS @@ -1060,12 +1071,6 @@ paths: $ref: "SwarmCommon.yaml#/components/schemas/SwarmAddress" required: true description: Overlay address of the broker node. Must not be this node. - - in: query - name: identity - schema: - $ref: "SwarmCommon.yaml#/components/schemas/EthereumAddress" - required: true - description: Ethereum address identifying the joiner. responses: "200": description: Establishes a WebSocket subscription for publishing broadcasts diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index feec2c6e9f4..c342a370e2b 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -26,6 +26,7 @@ import ( mockac "github.com/ethersphere/bee/v2/pkg/accesscontrol/mock" accountingmock "github.com/ethersphere/bee/v2/pkg/accounting/mock" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/feeds" "github.com/ethersphere/bee/v2/pkg/file/pipeline" @@ -95,7 +96,7 @@ type testServerOptions struct { Resolver resolver.Interface Pss pss.Interface Gsoc gsoc.Listener - Bps api.BPSService + Bps bps.Interface WsPath string WsPingPeriod time.Duration Logger log.Logger diff --git a/pkg/api/bps.go b/pkg/api/bps.go index f802cb2bed3..32fe0d5283a 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -6,6 +6,7 @@ package api import ( "context" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -15,69 +16,43 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/v2/pkg/bps" - "github.com/ethersphere/bee/v2/pkg/cac" - "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/jsonhttp" "github.com/ethersphere/bee/v2/pkg/log" - "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" "github.com/gorilla/mux" "github.com/gorilla/websocket" ) const ( - bpsCloseInvalidClaim = 4001 // claim failed local verification or was malformed + bpsCloseInvalidSOC = 4001 // a chunk that does not validate under the session challenge bpsCloseInvalidMessage = 4002 // frame violates the endpoint protocol bpsCloseBrokerGone = 4003 // the p2p stream to the broker ended - bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code - bpsMaxFrameSize = swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + swarm.ChunkSize + bpsMaxCloseReason = 123 // websocket control frame payload limit minus the code + bpsFrameHeader = 1 + 8 // publish frame header: kind | index (big-endian) + bpsMaxSOCSize = swarm.HashSize + swarm.SocSignatureSize + swarm.SpanSize + swarm.ChunkSize + bpsMaxFrameSize = bpsFrameHeader + bpsMaxSOCSize ) -// bpsTopicPrefix domain-separates the cohort SOC id from the owner's other SOCs. -var bpsTopicPrefix = []byte("bps-claim") - -// bpsClaimTimeout bounds how long a publisher may take to send its claim. +// bpsClaimTimeout bounds how long a publisher may take to send its first frame. var bpsClaimTimeout = 30 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") type bpsRequest struct { owner common.Address - id []byte topic []byte - addr swarm.Address broker swarm.Address identity []byte + cursor uint64 } type bpsChallengeMessage struct { Type string `json:"type"` Challenge string `json:"challenge"` Broker string `json:"broker"` - ID string `json:"id"` -} - -type bpsClaimMessage struct { - Type string `json:"type"` - Signature string `json:"signature"` -} - -type bpsClaimSentMessage struct { - Type string `json:"type"` -} - -// bpsTopicAddress derives the cohort SOC id and address for owner and topic. -func bpsTopicAddress(owner common.Address, topic []byte) ([]byte, swarm.Address, error) { - id, err := crypto.LegacyKeccak256(append(append([]byte{}, bpsTopicPrefix...), topic...)) - if err != nil { - return nil, swarm.ZeroAddress, err - } - addr, err := soc.CreateAddress(id, owner.Bytes()) - if err != nil { - return nil, swarm.ZeroAddress, err - } - return id, addr, nil + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` } func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) { @@ -86,6 +61,10 @@ func (s *Service) bpsSubscribeWsHandler(w http.ResponseWriter, r *http.Request) if !ok { return } + if len(req.identity) == 0 { + jsonhttp.BadRequest(w, "missing identity") + return + } conn, ok := s.bpsUpgrade(w, r, logger) if !ok { return @@ -127,7 +106,8 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger queries := struct { Broker swarm.Address `map:"broker"` - Identity []byte `map:"identity" validate:"required,len=20"` + Identity []byte `map:"identity" validate:"omitempty,len=20"` + Cursor uint64 `map:"cursor"` }{} if response := s.mapStructure(r.URL.Query(), &queries); response != nil { response("invalid query params", logger, w) @@ -146,19 +126,12 @@ func (s *Service) bpsParseRequest(w http.ResponseWriter, r *http.Request, logger return bpsRequest{}, false } - id, addr, err := bpsTopicAddress(paths.Owner, paths.Topic) - if err != nil { - logger.Debug("derive topic address failed", "error", err) - jsonhttp.InternalServerError(w, "derive topic address failed") - return bpsRequest{}, false - } return bpsRequest{ owner: paths.Owner, - id: id, topic: paths.Topic, - addr: addr, broker: queries.Broker, identity: queries.Identity, + cursor: queries.Cursor, }, true } @@ -258,11 +231,11 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo }() sess, err := s.bps.Join(ctx, bps.JoinRequest{ - Broker: req.broker, - Binding: bps.BindingFeed, - Topic: req.topic, - Principal: req.owner.Bytes(), - Identity: req.identity, + Broker: req.broker, + Topic: req.topic, + Admin: req.owner.Bytes(), + Addr: req.identity, + Cursor: req.cursor, }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -275,11 +248,16 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo for { select { - case data, ok := <-sess.Messages(): + case msg, ok := <-sess.Messages(): if !ok { s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(sess.Err())) return } + // challenge | index | soc: what the client needs to re-verify the chunk + data := make([]byte, 0, bps.ChallengeSize+8+len(msg.SOC)) + data = append(data, msg.Challenge...) + data = binary.BigEndian.AppendUint64(data, msg.Index) + data = append(data, msg.SOC...) if err := s.bpsWrite(conn, websocket.BinaryMessage, data); err != nil { logger.Debug("write broadcast failed", "error", err) return @@ -303,43 +281,25 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo } } -// bpsVerifyClaim checks a claim frame the same way the broker will and -// returns the claim SOC bytes, or a close code and reason on failure. -func bpsVerifyClaim(f bpsFrame, req bpsRequest, challenge []byte) ([]byte, int, string) { - if f.typ != websocket.TextMessage { - return nil, bpsCloseInvalidMessage, "expected claim" - } - var m bpsClaimMessage - if err := json.Unmarshal(f.data, &m); err != nil { - return nil, bpsCloseInvalidClaim, "malformed claim" - } - if m.Type != "claim" { - return nil, bpsCloseInvalidMessage, "expected claim" - } - sig, err := hex.DecodeString(m.Signature) - if err != nil || len(sig) != swarm.SocSignatureSize { - return nil, bpsCloseInvalidClaim, "invalid signature" +// bpsParseFrame checks a publish frame kind | index | soc the same way the +// broker will, or returns a close code and reason on failure. +func bpsParseFrame(f bpsFrame, req bpsRequest, challenge []byte) (bps.Kind, uint64, []byte, int, string) { + if f.typ != websocket.BinaryMessage { + return 0, 0, nil, bpsCloseInvalidMessage, "expected binary frame" } - payload := make([]byte, 0, len(challenge)+swarm.HashSize) - payload = append(append(payload, challenge...), req.broker.Bytes()...) - ch, err := cac.New(payload) - if err != nil { - return nil, bpsCloseInvalidClaim, "invalid claim payload" + if len(f.data) < bpsFrameHeader { + return 0, 0, nil, bpsCloseInvalidMessage, "short frame" } - sc, err := soc.NewSigned(req.id, ch, req.owner.Bytes(), sig) - if err != nil { - return nil, bpsCloseInvalidClaim, "invalid claim" + kind := bps.Kind(f.data[0]) + if kind != bps.KindData && kind != bps.KindAuth { + return 0, 0, nil, bpsCloseInvalidMessage, "unknown kind" } - chunk, err := sc.Chunk() - if err != nil { - return nil, bpsCloseInvalidClaim, "invalid claim" + index := binary.BigEndian.Uint64(f.data[1:bpsFrameHeader]) + chunk := f.data[bpsFrameHeader:] + if err := bps.Verify(kind, challenge, index, chunk, req.topic, req.owner.Bytes()); err != nil { + return 0, 0, nil, bpsCloseInvalidSOC, "invalid soc" } - // NewSigned does not verify the signature; Valid recovers the signer - // and checks that it maps to the cohort address. - if !chunk.Address().Equal(req.addr) || !soc.Valid(chunk) { - return nil, bpsCloseInvalidClaim, "claim verification failed" - } - return chunk.Data(), 0, "" + return kind, index, chunk, 0, "" } func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log.Logger) { @@ -362,12 +322,12 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. _ = conn.Close() }() + // the publisher declares the admin's address and claims with its first frame sess, err := s.bps.Join(ctx, bps.JoinRequest{ - Broker: req.broker, - Binding: bps.BindingFeed, - Topic: req.topic, - Principal: req.owner.Bytes(), - Identity: req.identity, + Broker: req.broker, + Topic: req.topic, + Admin: req.owner.Bytes(), + Addr: req.owner.Bytes(), }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -377,16 +337,27 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. defer sess.Close() challenge := sess.Challenge() - if len(challenge) != swarm.HashSize { + if len(challenge) != bps.ChallengeSize { logger.Debug("invalid challenge length", "broker", req.broker, "length", len(challenge)) s.bpsClose(conn, bpsCloseBrokerGone, "invalid challenge") return } + dataTopic, err := bps.SessionTopic(bps.KindData, req.topic, challenge) + if err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, "session topic") + return + } + authTopic, err := bps.SessionTopic(bps.KindAuth, req.topic, challenge) + if err != nil { + s.bpsClose(conn, bpsCloseBrokerGone, "session topic") + return + } if err := s.bpsWriteJSON(conn, bpsChallengeMessage{ Type: "challenge", Challenge: hex.EncodeToString(challenge), Broker: req.broker.String(), - ID: hex.EncodeToString(req.id), + DataTopic: hex.EncodeToString(dataTopic), + AuthTopic: hex.EncodeToString(authTopic), }); err != nil { logger.Debug("write challenge failed", "error", err) return @@ -401,36 +372,19 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. for { select { case f := <-frames: - if !claimed { - claim, code, reason := bpsVerifyClaim(f, req, challenge) - if code != 0 { - s.bpsClose(conn, code, reason) - return - } - if err := sess.Claim(ctx, claim); err != nil { - s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) - return - } - claimed = true - claimTimer.Stop() - if err := s.bpsWriteJSON(conn, bpsClaimSentMessage{Type: "claim_sent"}); err != nil { - logger.Debug("write claim_sent failed", "error", err) - return - } - continue - } - if f.typ != websocket.BinaryMessage { - s.bpsClose(conn, bpsCloseInvalidMessage, "expected binary soc") - return - } - if !soc.Valid(swarm.NewChunk(req.addr, f.data)) { - s.bpsClose(conn, bpsCloseInvalidMessage, "invalid soc") + kind, index, chunk, code, reason := bpsParseFrame(f, req, challenge) + if code != 0 { + s.bpsClose(conn, code, reason) return } - if err := sess.Publish(ctx, f.data); err != nil { + if err := sess.Publish(ctx, kind, index, chunk); err != nil { s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) return } + if !claimed { + claimed = true + claimTimer.Stop() + } case <-claimTimer.C: if !claimed { s.bpsClose(conn, bpsCloseInvalidMessage, "claim timeout") diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index a10d8403f26..ed651235116 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -7,6 +7,7 @@ package api_test import ( "bytes" "context" + "encoding/binary" "encoding/hex" "errors" "fmt" @@ -17,6 +18,7 @@ import ( "time" "github.com/ethersphere/bee/v2/pkg/api" + "github.com/ethersphere/bee/v2/pkg/bps" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/jsonhttp" @@ -31,14 +33,13 @@ type fakeBPS struct { sess *fakeSession joinErr error - mu sync.Mutex - broker swarm.Address - addr swarm.Address + mu sync.Mutex + req bps.JoinRequest } -func (f *fakeBPS) Join(_ context.Context, broker, addr swarm.Address) (api.BPSSession, error) { +func (f *fakeBPS) Join(_ context.Context, req bps.JoinRequest) (bps.Session, error) { f.mu.Lock() - f.broker, f.addr = broker, addr + f.req = req f.mu.Unlock() if f.joinErr != nil { return nil, f.joinErr @@ -46,15 +47,19 @@ func (f *fakeBPS) Join(_ context.Context, broker, addr swarm.Address) (api.BPSSe return f.sess, nil } +type published struct { + kind bps.Kind + index uint64 + soc []byte +} + type fakeSession struct { challenge []byte - msgs chan []byte + msgs chan bps.Message done chan struct{} - claims chan []byte - published chan []byte + published chan published closed chan struct{} - claimErr error publishErr error doneOnce sync.Once @@ -67,32 +72,24 @@ func newFakeSession() *fakeSession { copy(challenge, "bps-test-challenge") return &fakeSession{ challenge: challenge, - msgs: make(chan []byte), + msgs: make(chan bps.Message), done: make(chan struct{}), - claims: make(chan []byte, 1), - published: make(chan []byte, 8), + published: make(chan published, 8), closed: make(chan struct{}), } } -func (f *fakeSession) Challenge() []byte { return f.challenge } -func (f *fakeSession) Messages() <-chan []byte { return f.msgs } -func (f *fakeSession) Done() <-chan struct{} { return f.done } -func (f *fakeSession) Err() error { return f.err } - -func (f *fakeSession) Claim(_ context.Context, b []byte) error { - if f.claimErr != nil { - return f.claimErr - } - f.claims <- b - return nil -} +func (f *fakeSession) Challenge() []byte { return f.challenge } +func (f *fakeSession) Messages() <-chan bps.Message { return f.msgs } +func (f *fakeSession) Cursor() uint64 { return 0 } +func (f *fakeSession) Done() <-chan struct{} { return f.done } +func (f *fakeSession) Err() error { return f.err } -func (f *fakeSession) Publish(_ context.Context, b []byte) error { +func (f *fakeSession) Publish(_ context.Context, kind bps.Kind, index uint64, b []byte) error { if f.publishErr != nil { return f.publishErr } - f.published <- b + f.published <- published{kind: kind, index: index, soc: b} return nil } @@ -116,9 +113,8 @@ type bpsFixture struct { sess *fakeSession signer crypto.Signer owner []byte + identity []byte topic []byte - id []byte - addr swarm.Address broker swarm.Address self swarm.Address } @@ -136,25 +132,18 @@ func newBPSFixture(t *testing.T) *bpsFixture { } topic := make([]byte, 32) copy(topic, "bps-test-topic") - id, err := crypto.LegacyKeccak256(append([]byte("bps-claim"), topic...)) - if err != nil { - t.Fatal(err) - } - addr, err := soc.CreateAddress(id, owner.Bytes()) - if err != nil { - t.Fatal(err) - } + identity := make([]byte, 20) + copy(identity, "bps-test-subscriber") sess := newFakeSession() f := &bpsFixture{ - bps: &fakeBPS{sess: sess}, - sess: sess, - signer: signer, - owner: owner.Bytes(), - topic: topic, - id: id, - addr: addr, - broker: swarm.RandAddress(t), - self: swarm.RandAddress(t), + bps: &fakeBPS{sess: sess}, + sess: sess, + signer: signer, + owner: owner.Bytes(), + identity: identity, + topic: topic, + broker: swarm.RandAddress(t), + self: swarm.RandAddress(t), } f.client, _, f.listener, _ = newTestServer(t, testServerOptions{ Bps: f.bps, @@ -170,7 +159,8 @@ func (f *bpsFixture) path(endpoint string) string { func (f *bpsFixture) dial(t *testing.T, endpoint string) *websocket.Conn { t.Helper() - u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(endpoint), RawQuery: "broker=" + f.broker.String()} + q := "broker=" + f.broker.String() + "&identity=" + hex.EncodeToString(f.identity) + u := url.URL{Scheme: "ws", Host: f.listener, Path: f.path(endpoint), RawQuery: q} conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) if err != nil { t.Fatalf("dial %s: %v", u.String(), err) @@ -182,26 +172,30 @@ func (f *bpsFixture) dial(t *testing.T, endpoint string) *websocket.Conn { return conn } -// signedSOC returns full SOC bytes at f.addr wrapping payload. -func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, payload []byte) []byte { +// signedSOC returns the SOC bytes of the session feed update of kind at index, +// signed by signer under challenge. +func (f *bpsFixture) signedSOC(t *testing.T, signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { t.Helper() + id, err := bps.ID(kind, f.topic, challenge, index) + if err != nil { + t.Fatal(err) + } ch, err := cac.New(payload) if err != nil { t.Fatal(err) } - c, err := soc.New(f.id, ch).Sign(signer) + c, err := soc.New(id, ch).Sign(signer) if err != nil { t.Fatal(err) } return c.Data() } -// claimSignature signs the claim payload challenge|broker with signer. -func (f *bpsFixture) claimSignature(t *testing.T, signer crypto.Signer, challenge []byte) string { - t.Helper() - payload := append(append([]byte{}, challenge...), f.broker.Bytes()...) - data := f.signedSOC(t, signer, payload) - return hex.EncodeToString(data[swarm.HashSize : swarm.HashSize+swarm.SocSignatureSize]) +// frame returns a publish frame kind | index | soc. +func frame(kind bps.Kind, index uint64, chunk []byte) []byte { + b := []byte{byte(kind)} + b = binary.BigEndian.AppendUint64(b, index) + return append(b, chunk...) } func expectClose(t *testing.T, conn *websocket.Conn, code int) { @@ -263,43 +257,37 @@ func TestBPSPreUpgrade(t *testing.T) { func TestBPSSubscribe(t *testing.T) { t.Parallel() - t.Run("forwards valid soc", func(t *testing.T) { + t.Run("forwards delivery", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) conn := f.dial(t, "subscribe") - msg := f.signedSOC(t, f.signer, []byte("hello cohort")) - f.sess.msgs <- msg + chunk := f.signedSOC(t, f.signer, bps.KindData, f.sess.challenge, 7, []byte("hello cohort")) + f.sess.msgs <- bps.Message{SOC: chunk, Challenge: f.sess.challenge, Index: 7} typ, got, err := conn.ReadMessage() if err != nil { t.Fatal(err) } - if typ != websocket.BinaryMessage || !bytes.Equal(got, msg) { - t.Fatalf("got type %d %x, want binary %x", typ, got, msg) + want := append(append([]byte{}, f.sess.challenge...), 0, 0, 0, 0, 0, 0, 0, 7) + want = append(want, chunk...) + if typ != websocket.BinaryMessage || !bytes.Equal(got, want) { + t.Fatalf("got type %d %x, want binary %x", typ, got, want) } f.bps.mu.Lock() - gotAddr, gotBroker := f.bps.addr, f.bps.broker + req := f.bps.req f.bps.mu.Unlock() - if !gotAddr.Equal(f.addr) || !gotBroker.Equal(f.broker) { - t.Fatalf("join got addr %s broker %s, want %s %s", gotAddr, gotBroker, f.addr, f.broker) + if !req.Broker.Equal(f.broker) || !bytes.Equal(req.Admin, f.owner) || !bytes.Equal(req.Addr, f.identity) || !bytes.Equal(req.Topic, f.topic) { + t.Fatalf("unexpected join request %+v", req) } }) - t.Run("drops invalid soc", func(t *testing.T) { + t.Run("missing identity", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := f.dial(t, "subscribe") - other, _ := crypto.GenerateSecp256k1Key() - f.sess.msgs <- f.signedSOC(t, crypto.NewDefaultSigner(other), []byte("forged")) - f.sess.msgs <- []byte("garbage") - valid := f.signedSOC(t, f.signer, []byte("real")) - f.sess.msgs <- valid - _, got, err := conn.ReadMessage() - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(got, valid) { - t.Fatalf("got %x, want only the valid soc %x", got, valid) - } + jsonhttptest.Request(t, f.client, http.MethodGet, f.path("subscribe")+"?broker="+f.broker.String(), http.StatusBadRequest, + jsonhttptest.WithExpectedJSONResponse(jsonhttp.StatusResponse{ + Code: http.StatusBadRequest, Message: "missing identity", + }), + ) }) t.Run("client frame closes 4002", func(t *testing.T) { @@ -357,12 +345,16 @@ func readChallenge(t *testing.T, f *bpsFixture, conn *websocket.Conn) []byte { Type string `json:"type"` Challenge string `json:"challenge"` Broker string `json:"broker"` - ID string `json:"id"` + DataTopic string `json:"dataTopic"` + AuthTopic string `json:"authTopic"` } if err := conn.ReadJSON(&m); err != nil { t.Fatal(err) } - if m.Type != "challenge" || m.Broker != f.broker.String() || m.ID != hex.EncodeToString(f.id) { + dataTopic, _ := bps.SessionTopic(bps.KindData, f.topic, f.sess.challenge) + authTopic, _ := bps.SessionTopic(bps.KindAuth, f.topic, f.sess.challenge) + if m.Type != "challenge" || m.Broker != f.broker.String() || + m.DataTopic != hex.EncodeToString(dataTopic) || m.AuthTopic != hex.EncodeToString(authTopic) { t.Fatalf("unexpected challenge message %+v", m) } challenge, err := hex.DecodeString(m.Challenge) @@ -372,69 +364,45 @@ func readChallenge(t *testing.T, f *bpsFixture, conn *websocket.Conn) []byte { return challenge } -func sendClaim(t *testing.T, conn *websocket.Conn, sig string) { - t.Helper() - if err := conn.WriteJSON(map[string]string{"type": "claim", "signature": sig}); err != nil { - t.Fatal(err) - } -} - -func expectClaimSent(t *testing.T, conn *websocket.Conn) { +func expectPublished(t *testing.T, f *bpsFixture, kind bps.Kind, index uint64, chunk []byte) { t.Helper() - var m struct { - Type string `json:"type"` - } - if err := conn.ReadJSON(&m); err != nil { - t.Fatal(err) - } - if m.Type != "claim_sent" { - t.Fatalf("got %q, want claim_sent", m.Type) + select { + case got := <-f.sess.published: + if got.kind != kind || got.index != index || !bytes.Equal(got.soc, chunk) { + t.Fatalf("published %d/%d %x, want %d/%d %x", got.kind, got.index, got.soc, kind, index, chunk) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for publish") } } -// claimed dials publish and completes the claim handshake. -func claimed(t *testing.T, f *bpsFixture) *websocket.Conn { - t.Helper() - conn := f.dial(t, "publish") - challenge := readChallenge(t, f, conn) - sendClaim(t, conn, f.claimSignature(t, f.signer, challenge)) - expectClaimSent(t, conn) - return conn -} - func TestBPSPublish(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := claimed(t, f) + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) - claim := <-f.sess.claims - if !soc.Valid(swarm.NewChunk(f.addr, claim)) { - t.Fatal("claim passed to session is not a valid soc at the topic address") + f.bps.mu.Lock() + req := f.bps.req + f.bps.mu.Unlock() + if !bytes.Equal(req.Addr, f.owner) { + t.Fatalf("publisher joined as %x, want the admin %x", req.Addr, f.owner) } - sc, err := soc.FromChunk(swarm.NewChunk(f.addr, claim)) - if err != nil { + + auth := f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, auth)); err != nil { t.Fatal(err) } - wantPayload := append(append([]byte{}, f.sess.challenge...), f.broker.Bytes()...) - if got := sc.WrappedChunk().Data()[swarm.SpanSize:]; !bytes.Equal(got, wantPayload) { - t.Fatalf("claim payload %x, want %x", got, wantPayload) - } + expectPublished(t, f, bps.KindAuth, 0, auth) - msg := f.signedSOC(t, f.signer, []byte("broadcast")) - if err := conn.WriteMessage(websocket.BinaryMessage, msg); err != nil { + data := f.signedSOC(t, f.signer, bps.KindData, challenge, 3, []byte("broadcast")) + if err := conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 3, data)); err != nil { t.Fatal(err) } - select { - case got := <-f.sess.published: - if !bytes.Equal(got, msg) { - t.Fatalf("published %x, want %x", got, msg) - } - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for publish") - } + expectPublished(t, f, bps.KindData, 3, data) } -func TestBPSPublishClaimErrors(t *testing.T) { +func TestBPSPublishFrameErrors(t *testing.T) { t.Parallel() other, _ := crypto.GenerateSecp256k1Key() @@ -448,49 +416,56 @@ func TestBPSPublishClaimErrors(t *testing.T) { { name: "wrong signer", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { - sendClaim(t, conn, f.claimSignature(t, otherSigner, challenge)) + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, otherSigner, bps.KindData, challenge, 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { name: "wrong challenge", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { - sendClaim(t, conn, f.claimSignature(t, f.signer, make([]byte, 32))) + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, make([]byte, 32), 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "malformed json", - send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteMessage(websocket.TextMessage, []byte("{not json")) + name: "index not the signed one", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 1, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "short signature", - send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - sendClaim(t, conn, "abcd") + name: "auth relabelled as data", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "0x-prefixed signature", + name: "auth with payload", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { - sendClaim(t, conn, "0x"+f.claimSignature(t, f.signer, challenge)) + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, []byte("x")))) }, - code: api.BPSCloseInvalidClaim, + code: api.BPSCloseInvalidSOC, }, { - name: "wrong message type", + name: "unknown kind", + send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.Kind(9), 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) + }, + code: api.BPSCloseInvalidMessage, + }, + { + name: "short frame", send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteJSON(map[string]string{"type": "hello"}) + _ = conn.WriteMessage(websocket.BinaryMessage, []byte{1, 2}) }, code: api.BPSCloseInvalidMessage, }, { - name: "binary before claim", - send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, f.signer, []byte("early"))) + name: "text frame", + send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { + _ = conn.WriteJSON(map[string]string{"type": "claim"}) }, code: api.BPSCloseInvalidMessage, }, @@ -503,61 +478,33 @@ func TestBPSPublishClaimErrors(t *testing.T) { tc.send(t, f, conn, challenge) expectClose(t, conn, tc.code) waitSessionClosed(t, f) - if len(f.sess.claims) != 0 { - t.Fatal("rejected claim reached the session") + if len(f.sess.published) != 0 { + t.Fatal("rejected frame reached the session") } }) } } -func TestBPSPublishAfterClaimErrors(t *testing.T) { +func TestBPSPublishSessionErrors(t *testing.T) { t.Parallel() - t.Run("invalid soc closes 4002", func(t *testing.T) { - t.Parallel() - f := newBPSFixture(t) - conn := claimed(t, f) - other, _ := crypto.GenerateSecp256k1Key() - _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, crypto.NewDefaultSigner(other), []byte("forged"))) - expectClose(t, conn, api.BPSCloseInvalidMessage) - if len(f.sess.published) != 0 { - t.Fatal("invalid soc was published") - } - }) - - t.Run("text frame closes 4002", func(t *testing.T) { - t.Parallel() - f := newBPSFixture(t) - conn := claimed(t, f) - _ = conn.WriteMessage(websocket.TextMessage, []byte(`{"type":"claim"}`)) - expectClose(t, conn, api.BPSCloseInvalidMessage) - }) - t.Run("broker gone closes 4003", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := claimed(t, f) + conn := f.dial(t, "publish") + readChallenge(t, f, conn) f.sess.end(errors.New("stream reset")) expectClose(t, conn, api.BPSCloseBrokerGone) waitSessionClosed(t, f) }) - t.Run("claim write error closes 4003", func(t *testing.T) { - t.Parallel() - f := newBPSFixture(t) - f.sess.claimErr = errors.New("write failed") - conn := f.dial(t, "publish") - challenge := readChallenge(t, f, conn) - sendClaim(t, conn, f.claimSignature(t, f.signer, challenge)) - expectClose(t, conn, api.BPSCloseBrokerGone) - }) - t.Run("publish error closes 4003", func(t *testing.T) { t.Parallel() f := newBPSFixture(t) f.sess.publishErr = errors.New("write failed") - conn := claimed(t, f) - _ = conn.WriteMessage(websocket.BinaryMessage, f.signedSOC(t, f.signer, []byte("x"))) + conn := f.dial(t, "publish") + challenge := readChallenge(t, f, conn) + _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) expectClose(t, conn, api.BPSCloseBrokerGone) }) } @@ -570,17 +517,14 @@ func TestBPSPublishClientGoneBeforeClaim(t *testing.T) { // Send a close frame rather than Close, which the dial cleanup would // report as an error on the already closed connection. _ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) - select { - case <-f.sess.closed: - case <-time.After(5 * time.Second): - t.Fatal("session not closed after client went away") - } + waitSessionClosed(t, f) } func TestBPSPublishOversizedFrame(t *testing.T) { t.Parallel() f := newBPSFixture(t) - conn := claimed(t, f) + conn := f.dial(t, "publish") + readChallenge(t, f, conn) _ = conn.WriteMessage(websocket.BinaryMessage, make([]byte, 5000)) expectClose(t, conn, websocket.CloseMessageTooBig) waitSessionClosed(t, f) diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index e63f502828c..499fc6d8aa3 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -144,7 +144,7 @@ func NewParseError(entry, value string, cause error) error { } const ( - BPSCloseInvalidClaim = bpsCloseInvalidClaim + BPSCloseInvalidSOC = bpsCloseInvalidSOC BPSCloseInvalidMessage = bpsCloseInvalidMessage BPSCloseBrokerGone = bpsCloseBrokerGone ) diff --git a/pkg/bps/TODO b/pkg/bps/TODO index c4e0b659d75..f9011f1c0ce 100644 --- a/pkg/bps/TODO +++ b/pkg/bps/TODO @@ -1,52 +1,10 @@ -● Here's what's still open, from the earlier reviews plus a few smaller things I noticed while working in the code. - - Protocol and security - - 1. Publisher tracking (#7): add cohort.publisher and allow only one publisher per cohort. On takeover, close the old publisher's stream. Doing - the check, the swap and the eviction under one lock also stops two claims from racing each other. - 2. Owner checks (#8): - - Store the owner recovered from the claim in the cohort. - - The broker should check that each broadcast is a valid SOC signed by that owner. - - Subscribers should check too, which means deciding how they learn the owner. - 3. Members keyed by stream, not overlay (#9): a second join from the same peer currently overwrites the first, and either stream's left then - removes the other. Fixing this also closes the case where the publisher stalls sending on ch1 after the fan-out goroutine has exited. - 4. Domain-separated topic ID: use id = keccak("bps-claim" | t) so the claim chunk can't overwrite the owner's own SOC or GSOC in Swarm. This is a - client-side convention; a helper function would enforce it. - 5. Limits on cohort creation: reject Join.Topic values that aren't 32 bytes, and cap cohorts per peer and in total. - 6. Replay and ordering of broadcasts: lastSeen is commented out, so nothing stops a broker from re-sending old messages. - - Client (Join) - - 7. The first message on tx is always sent as the claim. Give the claim its own path or a state flag. - 8. Wrong log messages: several write errors are logged as "read join ack". - - Code quality - - 9. newMember panics if rand.Read fails; it should return an error. - 10. The ad-hoc errors.New messages should be bps:-prefixed sentinel errors. - 11. soc.FromChunk's error is ignored. It can't fail once Valid passes, but it deserves a comment. - 12. The fan-out goroutine takes the service-wide s.mtx for every message. A lock per cohort will be needed once there are many cohorts. - 13. Stale comments: - - The package doc still describes a "hello-world wire protocol used to greet other peers". - - The handler comment still says the "next challenge changes randomly", but nonces are now per member. - - Integration - - 14. Wiring: nothing in pkg/node/node.go calls bps.New yet. It needs swarmAddress and o.FullNodeMode, and the protocol has to be registered with - p2p. - - Tests - - 15. Claim cases not covered yet: - - a claim signed for a different broker - - a stale nonce, e.g. from an earlier stream - - a bad signature, or a signer other than the owner - - publisher takeover, once #1 is done - - a subscriber leaving and being removed - - an unverified broadcast being rejected, once #2 is done - - Housekeeping - - 16. Unrelated diff: regenerated .pb.go files in 12 unrelated packages are still modified in the working tree. They're safe to revert. - - I'd do 3, then 1 and 2 together (tracking the publisher and storing the owner are really one change), then 7, then 14. +Aligned with SWIP-74 rev 8 (ethersphere/SWIPs#111, SWIPs/swip-74.md). Still open: + +1. Client backoff: a peer whose join was refused or whose stream was reset MUST back off + (randomised exponential, 1s-30s, full jitter) before rejoining. Join is one-shot; the + API websocket closes instead of retrying, so a reconnecting client owns the backoff. +2. Spec open points (?) follow the draft as written: the kind byte in the service prefix, + streams per peer per cohort = 2, claim deadline = 30s. +3. Per-cohort counters are exposed via Service.Counters; there is no API endpoint for them + yet (prometheus carries the aggregate per outcome). +4. Conformance 12 (interop with a full SWIP-60 broker) is untestable until SWIP-60 exists. diff --git a/pkg/bps/binding.go b/pkg/bps/binding.go deleted file mode 100644 index f2482bda946..00000000000 --- a/pkg/bps/binding.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2026 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package bps - -import ( - "bytes" - "errors" - "fmt" - - "github.com/ethersphere/bee/v2/pkg/bps/pb" - "github.com/ethersphere/bee/v2/pkg/crypto" - "github.com/ethersphere/bee/v2/pkg/soc" - "github.com/ethersphere/bee/v2/pkg/swarm" -) - -// claimIDPrefix domain-separates the cohort's claim SOC id from the -// principal's other single owner chunks. -var claimIDPrefix = []byte("bps-claim") - -// errUnsupportedBinding is returned for a Join naming a topic binding this -// node does not implement. -var errUnsupportedBinding = errors.New("bps: unsupported topic binding") - -// errClaimNotQualified is returned when a claim SOC does not satisfy its -// cohort's topic binding. -var errClaimNotQualified = errors.New("bps: claim does not qualify") - -// binding decides how a cohort's publisher claim is verified. The topic -// binding named in Join picks the implementation; feed is the only one for now. -type binding interface { - // claimAddress returns the SOC address a valid claim must have for the - // cohort's topic and governing principal. - claimAddress(topic, principal []byte) (swarm.Address, error) - // verifyClaim checks the decoded claim's owner and signed payload. It - // returns nil when the claim promotes the stream to publisher. - verifyClaim(s *soc.SOC, principal, challenge, broker, topic []byte) error -} - -// bindingFor returns the binding rules for b, or errUnsupportedBinding. -func bindingFor(b pb.TopicBinding) (binding, error) { - switch b { - case pb.TopicBinding_BINDING_FEED: - return feedBinding{}, nil - } - return nil, fmt.Errorf("%s: %w", b, errUnsupportedBinding) -} - -// feedBinding verifies the publisher claim for a feed topic. The claim is a -// SOC with id keccak256("bps-claim" | topic), owned by the governing principal, -// whose wrapped payload signs the broker overlay and feed topic together with -// the per-member challenge. -type feedBinding struct{} - -func (feedBinding) claimAddress(topic, principal []byte) (swarm.Address, error) { - id, err := feedClaimID(topic) - if err != nil { - return swarm.ZeroAddress, err - } - return soc.CreateAddress(id, principal) -} - -func (feedBinding) verifyClaim(s *soc.SOC, principal, challenge, broker, topic []byte) error { - if !bytes.Equal(s.OwnerAddress(), principal) { - return fmt.Errorf("claim owner is not the principal: %w", errClaimNotQualified) - } - innerProof := make([]byte, 0, len(challenge)+len(broker)+len(topic)) - innerProof = append(innerProof, challenge...) - innerProof = append(innerProof, broker...) - innerProof = append(innerProof, topic...) - if !bytes.Equal(s.WrappedChunk().Data()[swarm.SpanSize:], innerProof) { - return fmt.Errorf("claim payload mismatch: %w", errClaimNotQualified) - } - return nil -} - -// feedClaimID derives the claim SOC id for a feed topic. -func feedClaimID(topic []byte) ([]byte, error) { - return crypto.LegacyKeccak256(append(append([]byte{}, claimIDPrefix...), topic...)) -} diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index 53756ade28b..f05db6767ed 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -2,8 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package bps exposes a simple hello-world wire protocol -// used to greet other peers. +// Package bps implements BPS-lite (SWIP-74): a single-publisher live stream +// over a feed, through one broker, one hop. +// +// A cohort is identified by its spec {topic, FEED_TOPIC, admin}. Every peer +// joins with the spec and its own address and is answered with a random +// per-stream challenge. The admin publishes ordinary single-owner chunks that +// are updates of the session feed on keccak256(topic | challenge); its first +// valid frame claims the stream. The broker keeps one cursor per cohort and +// delivers every accepted update to every subscriber stream. package bps import ( @@ -13,13 +20,14 @@ import ( "errors" "fmt" "sync" + "sync/atomic" + "time" "github.com/ethersphere/bee/v2/pkg/bps/pb" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" "github.com/ethersphere/bee/v2/pkg/p2p" "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" - "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" ) @@ -27,43 +35,105 @@ import ( const loggerName = "bps" const ( - protocolName = "bps" + protocolName = "pubsub" protocolVersion = "1.0.0" - streamName = "bps" + streamName = "pubsub" ) -// broadcastBufferSize is the per-member buffer for broadcast messages. -// Delivery is lossy by design (a slow member must not stall the cohort), -// so the buffer absorbs short bursts while the member's stream is busy. -const broadcastBufferSize = 1 - var ( - errNotBroker = errors.New("not a broker") - errJoinRejected = errors.New("bps: join rejected") - errSessionClosed = errors.New("bps: session closed") + errNotBroker = errors.New("bps: not a broker") + errJoinRefused = errors.New("bps: join refused") + errInvalidJoin = errors.New("bps: invalid join") + errSessionClosed = errors.New("bps: session closed") + errProtocolError = errors.New("bps: protocol error") + errViolation = errors.New("bps: protocol violation") + errWrongStream = errors.New("bps: frame from subscriber stream") + errCohortReclaimed = errors.New("bps: cohort reclaimed") ) -// Service is the bps protocol service. +// Options are the broker's resource bounds. Zero values take the defaults. +type Options struct { + MaxSubscribers int // subscriber streams per cohort + MaxCohorts int // live cohorts per broker + MaxCohortsPerPeer int // cohorts per peer connection + MaxStreamsPerPeerCohort int // streams per peer connection per cohort + InactivityDeadline time.Duration // reclaims a cohort with no accepted frame + ClaimDeadline time.Duration // disconnects a pending stream that has not claimed + QueueSize int // outbound frames per subscriber stream + ViolationBlocklist time.Duration // blocklist duration for a protocol violation + ClaimTimeoutBlocklist time.Duration // blocklist duration for a claim timeout +} + +// DefaultOptions are the SWIP-74 recommended bounds. +var DefaultOptions = Options{ + MaxSubscribers: 1024, + MaxCohorts: 4096, + MaxCohortsPerPeer: 16, + MaxStreamsPerPeerCohort: 2, + InactivityDeadline: 10 * time.Minute, + ClaimDeadline: 30 * time.Second, + QueueSize: 64, + ViolationBlocklist: 10 * time.Minute, + ClaimTimeoutBlocklist: time.Minute, +} + +func (o Options) withDefaults() Options { + d := DefaultOptions + if o.MaxSubscribers > 0 { + d.MaxSubscribers = o.MaxSubscribers + } + if o.MaxCohorts > 0 { + d.MaxCohorts = o.MaxCohorts + } + if o.MaxCohortsPerPeer > 0 { + d.MaxCohortsPerPeer = o.MaxCohortsPerPeer + } + if o.MaxStreamsPerPeerCohort > 0 { + d.MaxStreamsPerPeerCohort = o.MaxStreamsPerPeerCohort + } + if o.InactivityDeadline > 0 { + d.InactivityDeadline = o.InactivityDeadline + } + if o.ClaimDeadline > 0 { + d.ClaimDeadline = o.ClaimDeadline + } + if o.QueueSize > 0 { + d.QueueSize = o.QueueSize + } + if o.ViolationBlocklist > 0 { + d.ViolationBlocklist = o.ViolationBlocklist + } + if o.ClaimTimeoutBlocklist > 0 { + d.ClaimTimeoutBlocklist = o.ClaimTimeoutBlocklist + } + return d +} + +// Service is the bps protocol service. On a full node it is also the broker. type Service struct { fullNode bool - mtx sync.Mutex - selfOverlay swarm.Address // this node's overlay. used to verify challenges streamer p2p.Streamer - cohorts map[string]*cohort - // cohort registry - keep track of members, publisher, channels, challenge, streams? - // this is only relevant for the actual broker. subscribers don't care of manage this state at all. - // registry *CohortRegistry - - logger log.Logger + blocklister p2p.Blocklister + opts Options + metrics metrics + logger log.Logger + + mtx sync.Mutex + cohorts map[string]*cohort // by canonical spec + peers map[string]map[string]int // peer overlay -> cohort key -> streams + closed bool } -// New returns a new bps Service. -func New(streamer p2p.Streamer, overlay swarm.Address, fullNode bool, logger log.Logger) *Service { +// New returns a new bps Service. The blocklister may be nil. +func New(streamer p2p.Streamer, blocklister p2p.Blocklister, fullNode bool, logger log.Logger, o Options) *Service { return &Service{ fullNode: fullNode, - selfOverlay: overlay, streamer: streamer, + blocklister: blocklister, + opts: o.withDefaults(), + metrics: newMetrics(), cohorts: make(map[string]*cohort), + peers: make(map[string]map[string]int), logger: logger.WithName(loggerName).Register(), } } @@ -82,26 +152,48 @@ func (s *Service) Protocol() p2p.ProtocolSpec { } } -// Interface is the bps surface consumed by the API. It is satisfied by -// *Service and exists so consumers can depend on behaviour, not the concrete -// implementation. +// Close reclaims every cohort and resets their streams. +func (s *Service) Close() error { + s.mtx.Lock() + s.closed = true + cohorts := s.cohorts + s.cohorts = make(map[string]*cohort) + s.mtx.Unlock() + for _, co := range cohorts { + for _, m := range co.shutdown() { + m.reset() + } + } + return nil +} + +// Interface is the bps surface consumed by the API. type Interface interface { - // Join joins a topic's cohort at a broker and returns the live session. + // Join joins a cohort at a broker and returns the live session. Join(ctx context.Context, req JoinRequest) (Session, error) } +// Message is a verified DATA delivery: the chunk and the frame fields its id +// was derived from. +type Message struct { + SOC []byte + Challenge []byte + Index uint64 +} + // Session is a single joined p2p stream to a broker. type Session interface { - // Challenge is the nonce the broker issued for this stream. + // Challenge is the challenge the broker issued for this stream. Every + // chunk published on the session must be signed under it (see ID). Challenge() []byte - // Messages yields raw SOC bytes broadcast by the broker. The channel is - // closed when the stream ends. - Messages() <-chan []byte - // Claim writes the publisher claim SOC to the broker. The broker does not - // reply. - Claim(ctx context.Context, soc []byte) error - // Publish writes a broadcast SOC to the broker. - Publish(ctx context.Context, soc []byte) error + // Messages yields the DATA deliveries that verify against the cohort spec + // and the session cursor. The channel is closed when the stream ends. + Messages() <-chan Message + // Cursor is the lowest DATA index the session will deliver next. + Cursor() uint64 + // Publish writes a chunk of the given kind and index, signed under the + // session challenge, to the broker. The broker does not reply. + Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error // Done is closed when the p2p stream ends. Done() <-chan struct{} // Err reports why the stream ended. Valid after Done is closed. @@ -110,128 +202,94 @@ type Session interface { Close() error } -// TopicBinding is the topic binding carried by Join, wrapped so consumers do -// not depend on the generated protobuf package. -type TopicBinding int - -const ( - BindingUndefined TopicBinding = iota - BindingFeed -) - -// proto maps the wrapped binding to its wire value. -func (b TopicBinding) proto() pb.TopicBinding { - switch b { - case BindingFeed: - return pb.TopicBinding_BINDING_FEED - default: - return pb.TopicBinding_BINDING_UNDEFINED - } +// JoinRequest describes a cohort join. +type JoinRequest struct { + Broker swarm.Address // the broker + Topic []byte // 32 bytes: the feed topic + Admin []byte // 20 bytes: the publisher + Addr []byte // 20 bytes: the joining stream's identity + Cursor uint64 // the lowest DATA index to deliver, the subscriber's cursor for (topic, admin) } -// JoinRequest describes a cohort join. Identity is the joiner's address, -// injected from the API rather than taken from the peer address, so a node can -// join on behalf of a client. -type JoinRequest struct { - Broker swarm.Address // where to join the topic - Binding TopicBinding // which kind of topic binding - Topic []byte // the topic - Principal []byte // the "owner/admin" - Identity []byte // the joining node's identity +func (r JoinRequest) spec() *pb.CohortSpec { + return &pb.CohortSpec{Topic: r.Topic, Binding: pb.TopicBinding_FEED_TOPIC, Admin: r.Admin} } -// Join onto a topic at the broker. This is called on the subscriber. +// Join joins a cohort at the broker. A peer whose join was refused or whose +// stream was reset must back off before rejoining. func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { + spec := req.spec() + if err := validateJoin(&pb.Join{Cohort: spec, Addr: req.Addr}); err != nil { + return nil, err + } stream, err := s.streamer.NewStream(ctx, req.Broker, nil, protocolName, protocolVersion, streamName) if err != nil { return nil, fmt.Errorf("new stream: %w", err) } - ctx, cancel := context.WithCancel(ctx) - w, r := protobuf.NewWriterAndReader(stream) - joinMsg := pb.Join{ - Topic: req.Topic, - Binding: req.Binding.proto(), - Principal: req.Principal, - Identity: req.Identity, + if err := w.WriteMsgWithContext(ctx, &pb.Join{Cohort: spec, Addr: req.Addr}); err != nil { + _ = stream.Reset() + return nil, fmt.Errorf("write join: %w", err) } - if err := w.WriteMsgWithContext(ctx, &joinMsg); err != nil { - cancel() - stream.Reset() - return nil, fmt.Errorf("write join msg: %w", err) + var ack pb.Ack + if err := r.ReadMsgWithContext(ctx, &ack); err != nil { + _ = stream.Reset() + return nil, fmt.Errorf("read ack: %w", err) } - joinAck := pb.JoinAck{} - if err := r.ReadMsgWithContext(ctx, &joinAck); err != nil { - cancel() - stream.Reset() - return nil, fmt.Errorf("read join ack: %w", err) + if ack.Status != pb.Status_OK { + _ = stream.Reset() + return nil, fmt.Errorf("join %s: %w", ack.Status, errJoinRefused) } - if joinAck.Status != pb.Status_STATUS_OK { - cancel() - stream.Reset() - return nil, fmt.Errorf("join %s: %w", joinAck.Status, errJoinRejected) + if len(ack.Challenge) != ChallengeSize { + _ = stream.Reset() + return nil, fmt.Errorf("challenge length %d: %w", len(ack.Challenge), errProtocolError) } + ctx, cancel := context.WithCancel(context.Background()) sess := &session{ ctx: ctx, cancel: cancel, - challenge: joinAck.Challenge, - rx: make(chan []byte, broadcastBufferSize), - tx: make(chan []byte), + w: w, + challenge: ack.Challenge, + rx: make(chan Message), done: make(chan struct{}), } + sess.cursor.Store(req.Cursor) - // reading should always yield broadcast msgs go func() { - defer stream.FullClose() - var readErr error - defer func() { sess.finish(readErr) }() + defer func() { + sess.finish(readErr) + if readErr != nil && ctx.Err() == nil { + _ = stream.Reset() + } else { + _ = stream.FullClose() + } + }() + go func() { + <-ctx.Done() + _ = stream.Reset() + }() for { - msg := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &msg); err != nil { - if ctx.Err() == nil { - s.logger.Error(err, "read broadcast") - } + var f pb.Broadcast + if err := r.ReadMsgWithContext(ctx, &f); err != nil { readErr = err return } - - // we might want to do some input validation to see that the broker isn't tricking us - - select { - case sess.rx <- msg.Soc: - default: + // re-verify end to end: drop what is below the cursor or does + // not validate against the spec; never compare the challenge + if f.Kind != pb.Kind_DATA || f.Index < sess.cursor.Load() { + continue } - } - }() - - // the write loop will only be used in the case of a publisher. - // for readers this is essentially a noop. - go func() { - defer stream.FullClose() - - select { - case cl := <-sess.tx: - claim := pb.Broadcast{Soc: cl} - if err := w.WriteMsgWithContext(ctx, &claim); err != nil { - s.logger.Error(err, "write claim") - return + if err := verify(&f, req.Topic, req.Admin); err != nil { + s.logger.Debug("dropping delivery", "error", err) + continue } - case <-ctx.Done(): - return - } - - for { + sess.cursor.Store(f.Index + 1) select { - case bcast := <-sess.tx: - msg := pb.Broadcast{Soc: bcast} - if err := w.WriteMsgWithContext(ctx, &msg); err != nil { - s.logger.Error(err, "write broadcast") - return - } + case sess.rx <- Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index}: case <-ctx.Done(): return } @@ -245,43 +303,39 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { type session struct { ctx context.Context cancel context.CancelFunc + wmtx sync.Mutex + w protobuf.Writer challenge []byte - rx chan []byte - tx chan []byte + cursor atomic.Uint64 + rx chan Message done chan struct{} err error once sync.Once } -func (s *session) Challenge() []byte { return s.challenge } -func (s *session) Messages() <-chan []byte { return s.rx } -func (s *session) Done() <-chan struct{} { return s.done } -func (s *session) Err() error { return s.err } +func (s *session) Challenge() []byte { return s.challenge } +func (s *session) Messages() <-chan Message { return s.rx } +func (s *session) Cursor() uint64 { return s.cursor.Load() } +func (s *session) Done() <-chan struct{} { return s.done } +func (s *session) Err() error { return s.err } func (s *session) Close() error { s.cancel() return nil } -func (s *session) Claim(ctx context.Context, soc []byte) error { +func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error { select { - case s.tx <- soc: - return nil case <-s.done: return errSessionClosed - case <-ctx.Done(): - return ctx.Err() + default: } -} - -func (s *session) Publish(ctx context.Context, soc []byte) error { - select { - case s.tx <- soc: - return nil - case <-s.done: - return errSessionClosed - case <-ctx.Done(): - return ctx.Err() + s.wmtx.Lock() + defer s.wmtx.Unlock() + f := pb.Broadcast{Soc: soc, Kind: kind.proto(), Challenge: s.challenge, Index: index} + if err := s.w.WriteMsgWithContext(ctx, &f); err != nil { + return fmt.Errorf("write broadcast: %w", err) } + return nil } // finish records the terminal error and signals stream completion exactly once. @@ -293,62 +347,71 @@ func (s *session) finish(err error) { }) } +// validateJoin rejects a Join whose spec has a value outside SWIP-74 or whose +// addr is not 20 bytes. +func validateJoin(join *pb.Join) error { + c := join.Cohort + switch { + case c == nil: + return fmt.Errorf("no cohort: %w", errInvalidJoin) + case len(c.Topic) != swarm.HashSize: + return fmt.Errorf("topic length %d: %w", len(c.Topic), errInvalidJoin) + case c.Binding != pb.TopicBinding_FEED_TOPIC: + return fmt.Errorf("binding %s: %w", c.Binding, errInvalidJoin) + case len(c.Admin) != crypto.AddressSize: + return fmt.Errorf("admin length %d: %w", len(c.Admin), errInvalidJoin) + case len(join.Addr) != crypto.AddressSize: + return fmt.Errorf("addr length %d: %w", len(join.Addr), errInvalidJoin) + } + return nil +} + // handler is the protocol handler on the broker. func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) error { if !s.fullNode { - stream.Reset() + _ = stream.Reset() return errNotBroker } - ctx, cancel := context.WithCancel(ctx) - defer cancel() - defer stream.FullClose() w, r := protobuf.NewWriterAndReader(stream) var join pb.Join if err := r.ReadMsgWithContext(ctx, &join); err != nil { - return fmt.Errorf("read sys message: %w", err) + _ = stream.Reset() + return fmt.Errorf("read join: %w", err) } - if err := validateJoin(&join); err != nil { - if werr := w.WriteMsgWithContext(ctx, &pb.JoinAck{Status: pb.Status_STATUS_REJECTED}); werr != nil { - return fmt.Errorf("write join ack: %w", werr) - } - stream.Reset() - return err + return s.refuse(ctx, w, stream, pb.Status_REJECTED, err) } + // re-marshal so that the cohort key ignores fields BPS-lite does not define + join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Admin: join.Cohort.Admin} - // add to the cohort and get a channel to receive the broadcasts on - ch, challenge, err := s.joinCohort(&join) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + co, m, err := s.join(p.Address, &join, func() { + cancel() + _ = stream.Reset() + }) if err != nil { - if werr := w.WriteMsgWithContext(ctx, &pb.JoinAck{Status: pb.Status_STATUS_REJECTED}); werr != nil { - return fmt.Errorf("write join ack: %w", werr) - } - stream.Reset() - return err + return s.refuse(ctx, w, stream, pb.Status_FULL, err) } - // register in the cohort and return the secret - // peer is trying to join the cohort. accept and return the challenge - ack := pb.JoinAck{Challenge: challenge, Status: pb.Status_STATUS_OK} - if err := w.WriteMsgWithContext(ctx, &ack); err != nil { - s.left(join.Identity, join.Topic) - return fmt.Errorf("write claim: %w", err) + defer s.leave(p.Address, co, m) + + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: m.challenge}); err != nil { + m.reset() + return fmt.Errorf("write ack: %w", err) } - defer s.left(join.Identity, join.Topic) + // the writer side: deliveries to a subscriber stream var wg sync.WaitGroup - // the writer side - reads messages off the publisher channel - // and pushes them to the subscriber stream wg.Add(1) go func() { defer wg.Done() - defer cancel() for { - // await messages, then write to the stream once they come in select { - case msg := <-ch: - m := pb.Broadcast{Soc: msg} - if err := w.WriteMsgWithContext(ctx, &m); err != nil { - s.logger.Error(err, "write broadcast") + case f := <-m.queue: + if err := w.WriteMsgWithContext(ctx, f); err != nil { + m.reset() return } case <-ctx.Done(): @@ -356,199 +419,323 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } } }() + defer wg.Wait() + + // the reader side: publications and claims + for { + var f pb.Broadcast + if err := r.ReadMsgWithContext(ctx, &f); err != nil { + if ctx.Err() != nil { + return nil + } + m.reset() + return nil + } + if err := s.handleFrame(co, m, &f); err != nil { + if errors.Is(err, errViolation) && s.blocklister != nil { + if berr := s.blocklister.Blocklist(p.Address, s.opts.ViolationBlocklist, err.Error()); berr != nil { + s.logger.Debug("blocklist failed", "peer_address", p.Address, "error", berr) + } + } + m.reset() + return err + } + } +} - // peer is trying to claim the cohort. if the challenge doesn't add up - kick them off - // check if claim signature pk matches the feed owner address on the registry - // if it does - this is an atomic swap - the current stream becomes the publisher stream - // and the next challenge changes randomly, so that if needed, the publisher can reclaim - // later and rejoin + reclaim. +// refuse answers a Join with a non-OK Ack, which ends the stream. +func (s *Service) refuse(ctx context.Context, w protobuf.Writer, stream p2p.Stream, status pb.Status, cause error) error { + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: status}); err != nil { + _ = stream.Reset() + return fmt.Errorf("write ack: %w", err) + } + _ = stream.FullClose() + s.logger.Debug("join refused", "status", status, "error", cause) + return nil +} - // writer side - we first try to read a claim. we can wait indefinitely here. - // once a claim is accepted, we prompte the stream to be a publisher stream - // then continuously try to read from it broadcast messages and push them over the - // publisher channel - wg.Add(1) - go func() { - defer wg.Done() - defer cancel() - claim := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &claim); err != nil { - s.logger.Error(err, "read claim") - return - } +// handleFrame validates a Broadcast arriving at the broker per SWIP-74 and +// accepts, drops or punishes it. A returned error resets the stream. +func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { + co.mtx.Lock() + role, cursor := m.role, co.cursor + co.mtx.Unlock() + + // 1. only a publisher or pending stream may send + if role == roleSubscriber { + s.count(&co.counters.wrongStream, "wrong_stream") + return fmt.Errorf("%w: %w", errViolation, errWrongStream) + } + // 2. a kind BPS-lite does not define is dropped + if f.Kind != pb.Kind_DATA && f.Kind != pb.Kind_AUTH { + s.count(&co.counters.unknownKind, "unknown_kind") + return nil + } + // 3. the challenge must be the stream's + if !bytes.Equal(f.Challenge, m.challenge) { + s.count(&co.counters.wrongChallenge, "wrong_challenge") + return nil + } + // 4. on a publisher stream a DATA frame below the cursor is a retransmit; + // on a pending stream it is checked after the signature, so that it claims + if role == rolePublisher && f.Kind == pb.Kind_DATA && f.Index < cursor { + s.count(&co.counters.retransmit, "retransmit") + return nil + } + // 5. the chunk must be the admin's at the id derived from the frame + if err := verify(f, co.spec.Topic, co.spec.Admin); err != nil { + s.count(&co.counters.invalidSOC, "invalid_soc") + return fmt.Errorf("%w: %w", errViolation, err) + } - // do the checks on the claim, if it is wrong then we reset the stream - ch1, err := s.claim(&join, claim.Soc) - if err != nil { - stream.Reset() - return - } - defer close(ch1) - for { - // await messages, then write to the cohort channel - m := pb.Broadcast{} - if err := r.ReadMsgWithContext(ctx, &m); err != nil { - s.logger.Error(err, "read broadcast") - return + var overflow []*member + co.mtx.Lock() + if m.role == rolePending { + m.role = rolePublisher + m.claimTimer.Stop() + } + switch { + case f.Kind == pb.Kind_AUTH: + co.lastActivity = time.Now() + case f.Index < co.cursor: + s.count(&co.counters.retransmit, "retransmit") + default: + co.cursor = f.Index + 1 + co.lastActivity = time.Now() + s.metrics.Delivered.Inc() + for sub := range co.members { + if sub.role != roleSubscriber { + continue } select { - case ch1 <- m.Soc: - case <-ctx.Done(): - return + case sub.queue <- f: + default: + s.count(&co.counters.queueReset, "queue_reset") + overflow = append(overflow, sub) } } - }() - wg.Wait() + } + co.mtx.Unlock() + for _, sub := range overflow { + sub.reset() + } return nil } -var ( - errInvalidProof = errors.New("bps: invalid proof") - errInvalidTopic = errors.New("bps: invalid topic") - errInvalidPrincipal = errors.New("bps: invalid principal") - errInvalidIdentity = errors.New("bps: invalid identity") - errCohortMismatch = errors.New("bps: cohort binding mismatch") -) +// join attaches a stream to the cohort named by the spec, creating the +// cohort if no live one has it. An error means a capacity bound was hit. +func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort, *member, error) { + key, err := join.Cohort.Marshal() + if err != nil { + return nil, nil, err + } + k := string(key) -// validateJoin rejects a Join naming malformed or unsupported fields. -func validateJoin(join *pb.Join) error { - if len(join.Topic) != swarm.HashSize { - return errInvalidTopic + s.mtx.Lock() + defer s.mtx.Unlock() + if s.closed { + return nil, nil, errSessionClosed } - if len(join.Principal) != crypto.AddressSize { - return errInvalidPrincipal + + pc := s.peers[peer.ByteString()] + if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerPeer { + return nil, nil, errors.New("cohorts per peer") } - if len(join.Identity) != crypto.AddressSize { - return errInvalidIdentity + if pc[k] >= s.opts.MaxStreamsPerPeerCohort { + return nil, nil, errors.New("streams per peer per cohort") } - if _, err := bindingFor(join.Binding); err != nil { - return err + + role := roleSubscriber + if bytes.Equal(join.Addr, join.Cohort.Admin) { + role = rolePending } - return nil -} -// join/open operation in one - the peer either joins an existing -// cohort or creates one by joining a previously unregistered topic. -// returns the cohort challenge and the channel that data will be sent over -func (s *Service) joinCohort(join *pb.Join) (chan []byte, []byte, error) { - s.mtx.Lock() - defer s.mtx.Unlock() - t := string(join.Topic) - co, ok := s.cohorts[t] + co, ok := s.cohorts[k] + if !ok { + if len(s.cohorts) >= s.opts.MaxCohorts { + return nil, nil, errors.New("cohorts per broker") + } + co = &cohort{ + key: k, + spec: join.Cohort, + members: make(map[*member]struct{}), + lastActivity: time.Now(), + } + co.inactivity = time.AfterFunc(s.opts.InactivityDeadline, func() { s.reclaim(co) }) + s.cohorts[k] = co + s.metrics.Cohorts.Set(float64(len(s.cohorts))) + } - m := newMember(join.Identity) - if ok { - if co.binding != join.Binding || !bytes.Equal(co.principal, join.Principal) { - return nil, nil, errCohortMismatch + co.mtx.Lock() + defer co.mtx.Unlock() + if role == roleSubscriber && co.subscribers >= s.opts.MaxSubscribers { + if len(co.members) == 0 { + s.removeCohort(co) } - co.members[string(join.Identity)] = m - return m.ch, m.challenge, nil + return nil, nil, errors.New("subscribers per cohort") + } + + challenge := make([]byte, ChallengeSize) + if _, err := rand.Read(challenge); err != nil { + if len(co.members) == 0 { + s.removeCohort(co) + } + return nil, nil, fmt.Errorf("challenge: %w", err) + } + m := &member{ + role: role, + challenge: challenge, + queue: make(chan *pb.Broadcast, s.opts.QueueSize), } - members := make(map[string]*member) - members[string(join.Identity)] = m - s.cohorts[t] = &cohort{ - topic: join.Topic, - binding: join.Binding, - principal: join.Principal, - members: members, + var once sync.Once + m.reset = func() { once.Do(reset) } + if role == rolePending { + m.claimTimer = time.AfterFunc(s.opts.ClaimDeadline, func() { s.claimTimeout(peer, co, m) }) + } else { + co.subscribers++ } + co.members[m] = struct{}{} - return m.ch, m.challenge, nil + if pc == nil { + pc = make(map[string]int) + s.peers[peer.ByteString()] = pc + } + pc[k]++ + return co, m, nil } -func (s *Service) left(identity, topic []byte) { +// leave detaches a stream. A cohort with no attached streams is reclaimed at once. +func (s *Service) leave(peer swarm.Address, co *cohort, m *member) { s.mtx.Lock() defer s.mtx.Unlock() - t := string(topic) - co, ok := s.cohorts[t] - if ok { - delete(co.members, string(identity)) - if len(co.members) == 0 { - delete(s.cohorts, t) + + if pc := s.peers[peer.ByteString()]; pc != nil { + if pc[co.key]--; pc[co.key] <= 0 { + delete(pc, co.key) + } + if len(pc) == 0 { + delete(s.peers, peer.ByteString()) } } -} -func (s *Service) claim(join *pb.Join, socBlob []byte) (chan []byte, error) { - s.mtx.Lock() - defer s.mtx.Unlock() - t := string(join.Topic) - co, ok := s.cohorts[t] - if !ok { - return nil, errors.New("tried to claim non-existent topic") + co.mtx.Lock() + defer co.mtx.Unlock() + if _, ok := co.members[m]; !ok { + return } - member, ok := co.members[string(join.Identity)] - if !ok { - return nil, errors.New("no such member") + delete(co.members, m) + if m.claimTimer != nil { + m.claimTimer.Stop() } - b, err := bindingFor(co.binding) - if err != nil { - return nil, err + if m.role == roleSubscriber { + co.subscribers-- } - addr, err := b.claimAddress(co.topic, co.principal) - if err != nil { - return nil, fmt.Errorf("claim address: %w", err) + if len(co.members) == 0 && s.cohorts[co.key] == co { + s.removeCohort(co) } - chunk := swarm.NewChunk(addr, socBlob) - if !soc.Valid(chunk) { - return nil, errInvalidProof +} + +// removeCohort forgets the cohort. Must be called with s.mtx and co.mtx held. +func (s *Service) removeCohort(co *cohort) { + delete(s.cohorts, co.key) + co.inactivity.Stop() + s.metrics.Cohorts.Set(float64(len(s.cohorts))) +} + +// reclaim forgets a cohort on which no frame was accepted for the inactivity +// deadline, and resets every stream in it. +func (s *Service) reclaim(co *cohort) { + s.mtx.Lock() + if s.cohorts[co.key] != co { + s.mtx.Unlock() + return } - proofSoc, err := soc.FromChunk(chunk) - if err != nil { - return nil, fmt.Errorf("claim soc: %w", err) + co.mtx.Lock() + if idle := time.Since(co.lastActivity); idle < s.opts.InactivityDeadline { + co.inactivity.Reset(s.opts.InactivityDeadline - idle) + co.mtx.Unlock() + s.mtx.Unlock() + return } - if err := b.verifyClaim(proofSoc, co.principal, member.challenge, s.selfOverlay.Bytes(), co.topic); err != nil { - return nil, err + s.removeCohort(co) + co.mtx.Unlock() + s.mtx.Unlock() + + s.logger.Debug("cohort reclaimed", "error", errCohortReclaimed) + for _, m := range co.shutdown() { + m.reset() } +} - publisher := member - txCh := make(chan []byte) - go func() { - for v := range txCh { - s.mtx.Lock() - co, ok := s.cohorts[t] - if !ok { - s.mtx.Unlock() - return - } - for _, m := range co.members { - if m == publisher { - continue - } - select { - case m.ch <- v: - default: - } - } - s.mtx.Unlock() +// claimTimeout disconnects a pending stream that has not claimed in time. +func (s *Service) claimTimeout(peer swarm.Address, co *cohort, m *member) { + co.mtx.Lock() + pending := m.role == rolePending + co.mtx.Unlock() + if !pending { + return + } + s.count(&co.counters.claimTimeout, "claim_timeout") + if s.blocklister != nil { + if err := s.blocklister.Blocklist(peer, s.opts.ClaimTimeoutBlocklist, "bps claim timeout"); err != nil { + s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) } - }() - return txCh, nil + } + m.reset() } -type cohort struct { - topic []byte // topic is the feed topic - binding pb.TopicBinding - principal []byte // governing identity allowed to publish - // lastSeen uint64 // last feed update index, used to prevent replay and circumvent dedup logic (for now) - members map[string]*member +// Counters returns the counters of the live cohort {topic, FEED_TOPIC, admin}. +func (s *Service) Counters(topic, admin []byte) (Counters, bool) { + key, err := (&pb.CohortSpec{Topic: topic, Binding: pb.TopicBinding_FEED_TOPIC, Admin: admin}).Marshal() + if err != nil { + return Counters{}, false + } + s.mtx.Lock() + co, ok := s.cohorts[string(key)] + s.mtx.Unlock() + if !ok { + return Counters{}, false + } + return co.counters.snapshot(), true } -func newMember(identity []byte) *member { - challenge := make([]byte, 32) - if _, err := rand.Read(challenge); err != nil { - panic(err) - } +type role int - return &member{ - identity: identity, - ch: make(chan []byte, broadcastBufferSize), - challenge: challenge, +const ( + roleSubscriber role = iota + 1 // receives, never publishes + rolePending // declared the admin's addr, has not claimed yet + rolePublisher // claimed: may publish, receives nothing +) + +type cohort struct { + key string + spec *pb.CohortSpec + inactivity *time.Timer + counters counters + + mtx sync.Mutex + cursor uint64 // the lowest DATA index accepted next + lastActivity time.Time + members map[*member]struct{} + subscribers int +} + +// shutdown returns the cohort's streams, for the caller to reset. +func (co *cohort) shutdown() []*member { + co.mtx.Lock() + defer co.mtx.Unlock() + ms := make([]*member, 0, len(co.members)) + for m := range co.members { + ms = append(ms, m) } + return ms } +// member is one stream attached to a cohort. role is guarded by cohort.mtx. type member struct { - identity []byte - ch chan []byte - challenge []byte + role role + challenge []byte + queue chan *pb.Broadcast + reset func() + claimTimer *time.Timer } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index fb9b67c6fe6..e01e31d1998 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -7,98 +7,481 @@ package bps_test import ( "bytes" "context" + "errors" + "sync" "testing" "time" "github.com/ethersphere/bee/v2/pkg/bps" + "github.com/ethersphere/bee/v2/pkg/bps/pb" "github.com/ethersphere/bee/v2/pkg/cac" "github.com/ethersphere/bee/v2/pkg/crypto" "github.com/ethersphere/bee/v2/pkg/log" + "github.com/ethersphere/bee/v2/pkg/p2p" + "github.com/ethersphere/bee/v2/pkg/p2p/protobuf" "github.com/ethersphere/bee/v2/pkg/p2p/streamtest" "github.com/ethersphere/bee/v2/pkg/soc" "github.com/ethersphere/bee/v2/pkg/swarm" ) -func TestClaimAndBroadcast(t *testing.T) { - t.Parallel() +const waitFor = 5 * time.Second - logger := log.Noop +type blocklister struct { + mu sync.Mutex + peers map[string]string +} - brokerAddr := swarm.RandAddress(t) - broker := bps.New(nil, brokerAddr, true, logger) +func (b *blocklister) NetworkStatus() p2p.NetworkStatus { return p2p.NetworkStatusAvailable } - // each client gets its own recorder so that the broker sees distinct peer overlays - newClient := func() *bps.Service { - recorder := streamtest.New( - streamtest.WithProtocols(broker.Protocol()), - streamtest.WithBaseAddr(swarm.RandAddress(t)), - ) - return bps.New(recorder, swarm.RandAddress(t), false, logger) - } - publisher := newClient() - subscriber := newClient() +func (b *blocklister) Blocklist(overlay swarm.Address, _ time.Duration, reason string) error { + b.mu.Lock() + defer b.mu.Unlock() + b.peers[overlay.ByteString()] = reason + return nil +} + +func (b *blocklister) blocklisted(overlay swarm.Address) bool { + b.mu.Lock() + defer b.mu.Unlock() + _, ok := b.peers[overlay.ByteString()] + return ok +} + +type env struct { + t *testing.T + broker *bps.Service + brokerAddr swarm.Address + bl *blocklister + signer crypto.Signer + admin []byte + topic []byte +} - // the public topic is the soc address of the claim: keccak(id | owner) +func newEnv(t *testing.T, o bps.Options) *env { + t.Helper() + bl := &blocklister{peers: make(map[string]string)} + broker := bps.New(nil, bl, true, log.Noop, o) + t.Cleanup(func() { _ = broker.Close() }) key, err := crypto.GenerateSecp256k1Key() if err != nil { t.Fatal(err) } signer := crypto.NewDefaultSigner(key) - owner, err := crypto.NewEthereumAddress(key.PublicKey) - if err != nil { - t.Fatal(err) - } - id := make([]byte, swarm.HashSize) - copy(id, "bps-test-topic") - topic, err := soc.CreateAddress(id, owner) + admin, err := signer.EthereumAddress() if err != nil { t.Fatal(err) } + topic := make([]byte, swarm.HashSize) + copy(topic, "bps-test-topic") + return &env{t: t, broker: broker, brokerAddr: swarm.RandAddress(t), bl: bl, signer: signer, admin: admin.Bytes(), topic: topic} +} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +// client returns a node whose streams reach the broker from overlay. +func (e *env) client(overlay swarm.Address) (*bps.Service, *streamtest.Recorder) { + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(overlay)) + return bps.New(rec, nil, false, log.Noop, bps.Options{}), rec +} - _, subRx, _, _, err := subscriber.Join(ctx, brokerAddr, topic.Bytes()) +func (e *env) join(c *bps.Service, addr []byte) bps.Session { + e.t.Helper() + s, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Topic: e.topic, Admin: e.admin, Addr: addr}) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } + e.t.Cleanup(func() { _ = s.Close() }) + return s +} - challenge, pubRx, pubTx, claim, err := publisher.Join(ctx, brokerAddr, topic.Bytes()) +func (e *env) subscribe() bps.Session { + e.t.Helper() + c, _ := e.client(swarm.RandAddress(e.t)) + addr := make([]byte, 20) + copy(addr, swarm.RandAddress(e.t).Bytes()) + return e.join(c, addr) +} + +func (e *env) publisher() (bps.Session, swarm.Address) { + e.t.Helper() + overlay := swarm.RandAddress(e.t) + c, _ := e.client(overlay) + return e.join(c, e.admin), overlay +} + +func (e *env) chunk(signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { + e.t.Helper() + id, err := bps.ID(kind, e.topic, challenge, index) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } - - // claim payload: challenge | broker overlay - payload := append(append([]byte{}, challenge...), brokerAddr.Bytes()...) ch, err := cac.New(payload) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } - proof, err := soc.New(id, ch).Sign(signer) + s, err := soc.New(id, ch).Sign(signer) if err != nil { - t.Fatal(err) + e.t.Fatal(err) } - claim(proof.Data()) + return s.Data() +} - msg := []byte("hello cohort") - select { - case pubTx <- msg: - case <-time.After(time.Second): - t.Fatal("timed out sending broadcast") +func (e *env) publish(s bps.Session, kind bps.Kind, index uint64, payload []byte) []byte { + e.t.Helper() + c := e.chunk(e.signer, kind, s.Challenge(), index, payload) + if err := s.Publish(context.Background(), kind, index, c); err != nil { + e.t.Fatal(err) } + return c +} +func (e *env) counters() bps.Counters { + e.t.Helper() + c, ok := e.broker.Counters(e.topic, e.admin) + if !ok { + e.t.Fatal("no live cohort") + } + return c +} + +func expectMessage(t *testing.T, s bps.Session, index uint64, chunk []byte) { + t.Helper() select { - case got := <-subRx: - if !bytes.Equal(got, msg) { - t.Fatalf("got message %q, want %q", got, msg) + case m := <-s.Messages(): + if m.Index != index || !bytes.Equal(m.SOC, chunk) { + t.Fatalf("got index %d %x, want %d %x", m.Index, m.SOC, index, chunk) } - case <-time.After(time.Second): - t.Fatal("timed out waiting for subscriber to receive broadcast") + case <-time.After(waitFor): + t.Fatalf("timed out waiting for index %d", index) } +} +func expectNoMessage(t *testing.T, s bps.Session) { + t.Helper() select { - case got := <-pubRx: - t.Fatalf("publisher received its own broadcast %q", got) + case m, ok := <-s.Messages(): + if ok { + t.Fatalf("unexpected message at index %d", m.Index) + } case <-time.After(100 * time.Millisecond): } } + +func expectDone(t *testing.T, s bps.Session) { + t.Helper() + select { + case <-s.Done(): + case <-time.After(waitFor): + t.Fatal("stream not ended") + } +} + +func eventually(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(waitFor) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("condition not met") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestBroadcast(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.publisher() + + if len(pub.Challenge()) != bps.ChallengeSize || bytes.Equal(pub.Challenge(), sub.Challenge()) { + t.Fatal("want a distinct 32-byte challenge per stream") + } + + // AUTH claims and is never delivered + e.publish(pub, bps.KindAuth, 0, nil) + c0 := e.publish(pub, bps.KindData, 0, []byte("hello cohort")) + expectMessage(t, sub, 0, c0) + expectNoMessage(t, pub) +} + +func TestCursor(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, _ := e.publisher() + + c5 := e.publish(pub, bps.KindData, 5, []byte("five")) // gaps are allowed + expectMessage(t, sub, 5, c5) + e.publish(pub, bps.KindData, 3, []byte("three")) // retransmit + e.publish(pub, bps.KindData, 5, []byte("five")) // retransmit + c6 := e.publish(pub, bps.KindData, 6, []byte("six")) + expectMessage(t, sub, 6, c6) + if got := e.counters().Retransmit; got != 2 { + t.Fatalf("retransmit %d, want 2", got) + } +} + +func TestPublisherReconnects(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + + pub1, _ := e.publisher() + for i := uint64(0); i < 3; i++ { + expectMessage(t, sub, i, e.publish(pub1, bps.KindData, i, []byte{byte(i)})) + } + _ = pub1.Close() + expectDone(t, pub1) + + // the cohort and its cursor persist; a valid frame below the cursor + // still claims the new stream and is counted as a retransmit + pub2, _ := e.publisher() + if bytes.Equal(pub1.Challenge(), pub2.Challenge()) { + t.Fatal("challenge reused") + } + e.publish(pub2, bps.KindData, 1, []byte{1}) + c3 := e.publish(pub2, bps.KindData, 3, []byte{3}) + expectMessage(t, sub, 3, c3) + if got := e.counters().Retransmit; got != 1 { + t.Fatalf("retransmit %d, want 1", got) + } +} + +func TestTwoPublisherStreams(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub1, _ := e.publisher() + pub2, _ := e.publisher() + + c0 := e.publish(pub1, bps.KindData, 0, []byte("a")) + expectMessage(t, sub, 0, c0) + e.publish(pub2, bps.KindData, 0, []byte("a")) // signed for its own stream, delivered once + c1 := e.publish(pub2, bps.KindData, 1, []byte("b")) + expectMessage(t, sub, 1, c1) + expectNoMessage(t, pub1) +} + +func TestViolations(t *testing.T) { + t.Parallel() + + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + otherSigner := crypto.NewDefaultSigner(other) + + t.Run("publication from a subscriber stream", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + overlay := swarm.RandAddress(t) + c, _ := e.client(overlay) + sub := e.join(c, swarm.RandAddress(t).Bytes()[:20]) + keep := e.subscribe() + e.publish(sub, bps.KindData, 0, []byte("x")) + expectDone(t, sub) + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().WrongStream; got != 1 { + t.Fatalf("wrong_stream %d, want 1", got) + } + expectNoMessage(t, keep) + }) + + for _, tc := range []struct { + name string + chunk func(e *env, challenge []byte) (bps.Kind, uint64, []byte) + }{ + {"wrong signer", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(otherSigner, bps.KindData, ch, 0, []byte("x")) + }}, + {"signed for another challenge", func(e *env, _ []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindData, make([]byte, 32), 0, []byte("x")) + }}, + {"index not the signed one", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 1, e.chunk(e.signer, bps.KindData, ch, 0, []byte("x")) + }}, + {"auth relabelled as data", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindData, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, nil) + }}, + {"auth with payload", func(e *env, ch []byte) (bps.Kind, uint64, []byte) { + return bps.KindAuth, 0, e.chunk(e.signer, bps.KindAuth, ch, 0, []byte("x")) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + pub, overlay := e.publisher() + kind, index, chunk := tc.chunk(e, pub.Challenge()) + if err := pub.Publish(context.Background(), kind, index, chunk); err != nil { + t.Fatal(err) + } + expectDone(t, pub) + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().InvalidSOC; got != 1 { + t.Fatalf("invalid_soc %d, want 1", got) + } + expectNoMessage(t, sub) + }) + } +} + +// raw opens a stream to the broker and joins with join, returning the Ack. +func (e *env) raw(join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { + e.t.Helper() + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(swarm.RandAddress(e.t))) + stream, err := rec.NewStream(context.Background(), e.brokerAddr, nil, bps.ProtocolName, bps.ProtocolVersion, bps.StreamName) + if err != nil { + e.t.Fatal(err) + } + e.t.Cleanup(func() { _ = stream.Reset() }) + w, r := protobuf.NewWriterAndReader(stream) + if err := w.WriteMsg(join); err != nil { + e.t.Fatal(err) + } + var ack pb.Ack + if err := r.ReadMsg(&ack); err != nil { + e.t.Fatal(err) + } + return w, r, &ack +} + +func (e *env) spec() *pb.CohortSpec { + return &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC, Admin: e.admin} +} + +func TestDroppedFrames(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Addr: e.admin}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + write := func(f *pb.Broadcast) { + t.Helper() + if err := w.WriteMsg(f); err != nil { + t.Fatal(err) + } + } + + // another kind and another challenge are dropped, not punished + write(&pb.Broadcast{Kind: pb.Kind(7), Challenge: ack.Challenge, Soc: []byte("roster")}) + other := make([]byte, 32) + write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: other, Soc: e.chunk(e.signer, bps.KindData, other, 0, []byte("old"))}) + c := e.chunk(e.signer, bps.KindData, ack.Challenge, 0, []byte("ok")) + write(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ack.Challenge, Index: 0, Soc: c}) + expectMessage(t, sub, 0, c) + + got := e.counters() + if got.UnknownKind != 1 || got.WrongChallenge != 1 || got.InvalidSOC != 0 { + t.Fatalf("counters %+v", got) + } +} + +func TestJoinRejected(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + for _, tc := range []struct { + name string + join *pb.Join + }{ + {"no cohort", &pb.Join{Addr: e.admin}}, + {"unspecified binding", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Admin: e.admin}, Addr: e.admin}}, + {"short topic", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic[:31], Binding: pb.TopicBinding_FEED_TOPIC, Admin: e.admin}, Addr: e.admin}}, + {"no admin", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC}, Addr: e.admin}}, + {"short addr", &pb.Join{Cohort: e.spec(), Addr: e.admin[:19]}}, + } { + _, _, ack := e.raw(tc.join) + if ack.Status != pb.Status_REJECTED || len(ack.Challenge) != 0 { + t.Fatalf("%s: got %s", tc.name, ack.Status) + } + } +} + +func TestBounds(t *testing.T) { + t.Parallel() + + t.Run("subscribers per cohort, admin admitted outside", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxSubscribers: 1}) + sub := e.subscribe() + if _, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Addr: make([]byte, 20)}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + pub, _ := e.publisher() + expectMessage(t, sub, 0, e.publish(pub, bps.KindData, 0, []byte("x"))) + }) + + t.Run("cohorts per broker", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxCohorts: 1}) + e.subscribe() + spec := e.spec() + spec.Topic = make([]byte, 32) + if _, _, ack := e.raw(&pb.Join{Cohort: spec, Addr: e.admin}); ack.Status != pb.Status_FULL { + t.Fatalf("got %s, want FULL", ack.Status) + } + }) + + t.Run("streams and cohorts per peer", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{MaxStreamsPerPeerCohort: 1, MaxCohortsPerPeer: 1}) + c, _ := e.client(swarm.RandAddress(t)) + e.join(c, e.admin) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Topic: e.topic, Admin: e.admin, Addr: e.admin}); err == nil { + t.Fatal("want a second stream on the cohort refused") + } + other := make([]byte, 32) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Topic: other, Admin: e.admin, Addr: e.admin}); err == nil { + t.Fatal("want a second cohort refused") + } + }) + + t.Run("claim deadline", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{ClaimDeadline: 50 * time.Millisecond}) + e.subscribe() + pub, overlay := e.publisher() + expectDone(t, pub) + eventually(t, func() bool { return e.bl.blocklisted(overlay) }) + if got := e.counters().ClaimTimeout; got != 1 { + t.Fatalf("claim_timeout %d, want 1", got) + } + }) + + t.Run("inactivity deadline", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{InactivityDeadline: 100 * time.Millisecond}) + sub := e.subscribe() + pub, _ := e.publisher() + e.publish(pub, bps.KindAuth, 0, nil) + expectDone(t, sub) + expectDone(t, pub) + if _, ok := e.broker.Counters(e.topic, e.admin); ok { + t.Fatal("cohort not reclaimed") + } + }) +} + +func TestVerify(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + challenge := make([]byte, 32) + copy(challenge, "challenge") + + c := e.chunk(e.signer, bps.KindData, challenge, 9, []byte("x")) + if err := bps.Verify(bps.KindData, challenge, 9, c, e.topic, e.admin); err != nil { + t.Fatal(err) + } + other := make([]byte, 20) + for _, err := range []error{ + bps.Verify(bps.KindData, challenge, 9, c, e.topic, other), + bps.Verify(bps.KindData, challenge[:31], 9, c, e.topic, e.admin), + bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.admin), + bps.Verify(bps.KindData, challenge, 9, c, make([]byte, 32), e.admin), + bps.Verify(bps.KindData, challenge, 9, c[:10], e.topic, e.admin), + } { + if !errors.Is(err, bps.ErrInvalidSOC) { + t.Fatalf("got %v, want invalid soc", err) + } + } +} diff --git a/pkg/bps/export_test.go b/pkg/bps/export_test.go new file mode 100644 index 00000000000..24c353ff0b8 --- /dev/null +++ b/pkg/bps/export_test.go @@ -0,0 +1,13 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +const ( + ProtocolName = protocolName + ProtocolVersion = protocolVersion + StreamName = streamName +) + +var ErrInvalidSOC = errInvalidSOC diff --git a/pkg/bps/id.go b/pkg/bps/id.go new file mode 100644 index 00000000000..b5a2cbdaf6b --- /dev/null +++ b/pkg/bps/id.go @@ -0,0 +1,111 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + + "github.com/ethersphere/bee/v2/pkg/bps/pb" + "github.com/ethersphere/bee/v2/pkg/crypto" + "github.com/ethersphere/bee/v2/pkg/soc" + "github.com/ethersphere/bee/v2/pkg/swarm" +) + +// ChallengeSize is the length of the per-stream challenge issued in Ack. +const ChallengeSize = 32 + +// servicePrefix domain-separates the session feeds of every kind but DATA. +var servicePrefix = []byte("bps-service:v1") + +// errInvalidSOC is returned for a chunk that does not have the id derived +// from its frame, or does not validate as the admin's single-owner chunk. +var errInvalidSOC = errors.New("bps: invalid soc") + +// Kind is what a chunk on the wire is. +type Kind int + +const ( + KindUnspecified Kind = iota + KindData // an update of the stream's session feed + KindAuth // an empty chunk that claims the stream +) + +func (k Kind) proto() pb.Kind { + switch k { + case KindData: + return pb.Kind_DATA + case KindAuth: + return pb.Kind_AUTH + default: + return pb.Kind_KIND_UNSPECIFIED + } +} + +// SessionTopic returns the topic of the session feed of the given kind: +// keccak256(prefix | topic | challenge), with the prefix empty for DATA and +// "bps-service:v1" | kind for any other kind. +func SessionTopic(kind Kind, topic, challenge []byte) ([]byte, error) { + return sessionTopic(kind.proto(), topic, challenge) +} + +// ID returns the single-owner chunk id of the session feed update at index: +// keccak256(SessionTopic(kind, topic, challenge) | index), index big-endian. +func ID(kind Kind, topic, challenge []byte, index uint64) ([]byte, error) { + return id(kind.proto(), topic, challenge, index) +} + +func sessionTopic(kind pb.Kind, topic, challenge []byte) ([]byte, error) { + var prefix []byte + if kind != pb.Kind_DATA { + prefix = append(append([]byte{}, servicePrefix...), byte(kind)) + } + return crypto.LegacyKeccak256(append(append(prefix, topic...), challenge...)) +} + +func id(kind pb.Kind, topic, challenge []byte, index uint64) ([]byte, error) { + topicS, err := sessionTopic(kind, topic, challenge) + if err != nil { + return nil, err + } + return crypto.LegacyKeccak256(binary.BigEndian.AppendUint64(topicS, index)) +} + +// Verify checks that the frame fields kind, challenge and index give the id of +// the chunk, and that the chunk validates as admin's single-owner chunk at +// keccak256(id | admin). An AUTH chunk must moreover be empty. +func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, admin []byte) error { + return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, admin) +} + +func verify(f *pb.Broadcast, topic, admin []byte) error { + if len(f.Challenge) != ChallengeSize { + return fmt.Errorf("challenge length %d: %w", len(f.Challenge), errInvalidSOC) + } + want, err := id(f.Kind, topic, f.Challenge, f.Index) + if err != nil { + return err + } + if len(f.Soc) < swarm.HashSize || !bytes.Equal(f.Soc[:swarm.HashSize], want) { + return fmt.Errorf("id not derived from frame: %w", errInvalidSOC) + } + addr, err := soc.CreateAddress(want, admin) + if err != nil { + return err + } + if !soc.Valid(swarm.NewChunk(addr, f.Soc)) { + return fmt.Errorf("not the admin's soc: %w", errInvalidSOC) + } + if f.Kind == pb.Kind_AUTH { + // span 0, no payload + if len(f.Soc) != swarm.HashSize+swarm.SocSignatureSize+swarm.SpanSize || + binary.LittleEndian.Uint64(f.Soc[swarm.HashSize+swarm.SocSignatureSize:]) != 0 { + return fmt.Errorf("auth chunk not empty: %w", errInvalidSOC) + } + } + return nil +} diff --git a/pkg/bps/metrics.go b/pkg/bps/metrics.go new file mode 100644 index 00000000000..a6497338433 --- /dev/null +++ b/pkg/bps/metrics.go @@ -0,0 +1,88 @@ +// Copyright 2026 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bps + +import ( + "sync/atomic" + + m "github.com/ethersphere/bee/v2/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +// Counters are the per-cohort counters of the silent outcomes SWIP-74 makes a +// broker expose. +type Counters struct { + WrongStream uint64 // a frame from a subscriber stream + UnknownKind uint64 // a frame of a kind BPS-lite does not define + WrongChallenge uint64 // a frame whose challenge is not its stream's + InvalidSOC uint64 // a chunk that does not validate at the derived id + ClaimTimeout uint64 // a pending stream disconnected at the claim deadline + Retransmit uint64 // a DATA frame below the cursor + QueueReset uint64 // a subscriber stream reset for a full queue +} + +type counters struct { + wrongStream atomic.Uint64 + unknownKind atomic.Uint64 + wrongChallenge atomic.Uint64 + invalidSOC atomic.Uint64 + claimTimeout atomic.Uint64 + retransmit atomic.Uint64 + queueReset atomic.Uint64 +} + +func (c *counters) snapshot() Counters { + return Counters{ + WrongStream: c.wrongStream.Load(), + UnknownKind: c.unknownKind.Load(), + WrongChallenge: c.wrongChallenge.Load(), + InvalidSOC: c.invalidSOC.Load(), + ClaimTimeout: c.claimTimeout.Load(), + Retransmit: c.retransmit.Load(), + QueueReset: c.queueReset.Load(), + } +} + +type metrics struct { + Outcomes *prometheus.CounterVec + Delivered prometheus.Counter + Cohorts prometheus.Gauge +} + +func newMetrics() metrics { + subsystem := "bps" + + return metrics{ + Outcomes: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "dropped_frames", + Help: "Frames dropped or streams reset by the broker, by outcome.", + }, []string{"outcome"}), + Delivered: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "accepted_updates", + Help: "DATA updates accepted by the broker.", + }), + Cohorts: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "cohorts", + Help: "Live cohorts on the broker.", + }), + } +} + +// Metrics returns the prometheus collectors of the service. +func (s *Service) Metrics() []prometheus.Collector { + return m.PrometheusCollectorsFromFields(s.metrics) +} + +// count increments the per-cohort counter c and its prometheus outcome. +func (s *Service) count(c *atomic.Uint64, outcome string) { + c.Add(1) + s.metrics.Outcomes.WithLabelValues(outcome).Inc() +} diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index c0e2696e9a3..416d48cd973 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -22,21 +22,22 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package +// What the topic binds to. One binding. type TopicBinding int32 const ( - TopicBinding_BINDING_UNDEFINED TopicBinding = 0 - TopicBinding_BINDING_FEED TopicBinding = 1 + TopicBinding_TOPIC_BINDING_UNSPECIFIED TopicBinding = 0 + TopicBinding_FEED_TOPIC TopicBinding = 4 ) var TopicBinding_name = map[int32]string{ - 0: "BINDING_UNDEFINED", - 1: "BINDING_FEED", + 0: "TOPIC_BINDING_UNSPECIFIED", + 4: "FEED_TOPIC", } var TopicBinding_value = map[string]int32{ - "BINDING_UNDEFINED": 0, - "BINDING_FEED": 1, + "TOPIC_BINDING_UNSPECIFIED": 0, + "FEED_TOPIC": 4, } func (x TopicBinding) String() string { @@ -50,24 +51,24 @@ func (TopicBinding) EnumDescriptor() ([]byte, []int) { type Status int32 const ( - Status_STATUS_UNDEFINED Status = 0 - Status_STATUS_OK Status = 1 - Status_STATUS_FULL Status = 2 - Status_STATUS_REJECTED Status = 3 + Status_STATUS_UNSPECIFIED Status = 0 + Status_OK Status = 1 + Status_FULL Status = 2 + Status_REJECTED Status = 3 ) var Status_name = map[int32]string{ - 0: "STATUS_UNDEFINED", - 1: "STATUS_OK", - 2: "STATUS_FULL", - 3: "STATUS_REJECTED", + 0: "STATUS_UNSPECIFIED", + 1: "OK", + 2: "FULL", + 3: "REJECTED", } var Status_value = map[string]int32{ - "STATUS_UNDEFINED": 0, - "STATUS_OK": 1, - "STATUS_FULL": 2, - "STATUS_REJECTED": 3, + "STATUS_UNSPECIFIED": 0, + "OK": 1, + "FULL": 2, + "REJECTED": 3, } func (x Status) String() string { @@ -78,23 +79,54 @@ func (Status) EnumDescriptor() ([]byte, []int) { return fileDescriptor_786299925f8760d5, []int{1} } -type Broadcast struct { - // either service or broadcast - Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` +// What a chunk on this wire is. +type Kind int32 + +const ( + Kind_KIND_UNSPECIFIED Kind = 0 + Kind_DATA Kind = 1 + Kind_AUTH Kind = 2 +) + +var Kind_name = map[int32]string{ + 0: "KIND_UNSPECIFIED", + 1: "DATA", + 2: "AUTH", } -func (m *Broadcast) Reset() { *m = Broadcast{} } -func (m *Broadcast) String() string { return proto.CompactTextString(m) } -func (*Broadcast) ProtoMessage() {} -func (*Broadcast) Descriptor() ([]byte, []int) { +var Kind_value = map[string]int32{ + "KIND_UNSPECIFIED": 0, + "DATA": 1, + "AUTH": 2, +} + +func (x Kind) String() string { + return proto.EnumName(Kind_name, int32(x)) +} + +func (Kind) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} + +// The cohort's identity, keyed by its canonical serialisation. +type CohortSpec struct { + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` + Admin []byte `protobuf:"bytes,3,opt,name=admin,proto3" json:"admin,omitempty"` +} + +func (m *CohortSpec) Reset() { *m = CohortSpec{} } +func (m *CohortSpec) String() string { return proto.CompactTextString(m) } +func (*CohortSpec) ProtoMessage() {} +func (*CohortSpec) Descriptor() ([]byte, []int) { return fileDescriptor_786299925f8760d5, []int{0} } -func (m *Broadcast) XXX_Unmarshal(b []byte) error { +func (m *CohortSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *CohortSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_Broadcast.Marshal(b, m, deterministic) + return xxx_messageInfo_CohortSpec.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -104,31 +136,43 @@ func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *Broadcast) XXX_Merge(src proto.Message) { - xxx_messageInfo_Broadcast.Merge(m, src) +func (m *CohortSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_CohortSpec.Merge(m, src) } -func (m *Broadcast) XXX_Size() int { +func (m *CohortSpec) XXX_Size() int { return m.Size() } -func (m *Broadcast) XXX_DiscardUnknown() { - xxx_messageInfo_Broadcast.DiscardUnknown(m) +func (m *CohortSpec) XXX_DiscardUnknown() { + xxx_messageInfo_CohortSpec.DiscardUnknown(m) } -var xxx_messageInfo_Broadcast proto.InternalMessageInfo +var xxx_messageInfo_CohortSpec proto.InternalMessageInfo -func (m *Broadcast) GetSoc() []byte { +func (m *CohortSpec) GetTopic() []byte { if m != nil { - return m.Soc + return m.Topic + } + return nil +} + +func (m *CohortSpec) GetBinding() TopicBinding { + if m != nil { + return m.Binding + } + return TopicBinding_TOPIC_BINDING_UNSPECIFIED +} + +func (m *CohortSpec) GetAdmin() []byte { + if m != nil { + return m.Admin } return nil } +// Peer -> broker: the first frame on a fresh stream. type Join struct { - Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` - Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` - Principal []byte `protobuf:"bytes,3,opt,name=principal,proto3" json:"principal,omitempty"` - // ============================ - Identity []byte `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` + Cohort *CohortSpec `protobuf:"bytes,1,opt,name=cohort,proto3" json:"cohort,omitempty"` + Addr []byte `protobuf:"bytes,2,opt,name=addr,proto3" json:"addr,omitempty"` } func (m *Join) Reset() { *m = Join{} } @@ -164,51 +208,94 @@ func (m *Join) XXX_DiscardUnknown() { var xxx_messageInfo_Join proto.InternalMessageInfo -func (m *Join) GetTopic() []byte { +func (m *Join) GetCohort() *CohortSpec { if m != nil { - return m.Topic + return m.Cohort } return nil } -func (m *Join) GetBinding() TopicBinding { +func (m *Join) GetAddr() []byte { if m != nil { - return m.Binding + return m.Addr } - return TopicBinding_BINDING_UNDEFINED + return nil +} + +// Broker -> peer, answering Join. A non-OK Ack ends the stream. +type Ack struct { + Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` + Challenge []byte `protobuf:"bytes,2,opt,name=challenge,proto3" json:"challenge,omitempty"` } -func (m *Join) GetPrincipal() []byte { +func (m *Ack) Reset() { *m = Ack{} } +func (m *Ack) String() string { return proto.CompactTextString(m) } +func (*Ack) ProtoMessage() {} +func (*Ack) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{2} +} +func (m *Ack) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Ack) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Ack.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Ack) XXX_Merge(src proto.Message) { + xxx_messageInfo_Ack.Merge(m, src) +} +func (m *Ack) XXX_Size() int { + return m.Size() +} +func (m *Ack) XXX_DiscardUnknown() { + xxx_messageInfo_Ack.DiscardUnknown(m) +} + +var xxx_messageInfo_Ack proto.InternalMessageInfo + +func (m *Ack) GetStatus() Status { if m != nil { - return m.Principal + return m.Status } - return nil + return Status_STATUS_UNSPECIFIED } -func (m *Join) GetIdentity() []byte { +func (m *Ack) GetChallenge() []byte { if m != nil { - return m.Identity + return m.Challenge } return nil } -type JoinAck struct { - Challenge []byte `protobuf:"bytes,1,opt,name=challenge,proto3" json:"challenge,omitempty"` - Status Status `protobuf:"varint,2,opt,name=status,proto3,enum=bps.Status" json:"status,omitempty"` +// Every frame after the handshake. soc is an ordinary single-owner chunk +// whose id is derived from kind, challenge and index (see bps.ID). +type Broadcast struct { + Soc []byte `protobuf:"bytes,1,opt,name=soc,proto3" json:"soc,omitempty"` + Kind Kind `protobuf:"varint,2,opt,name=kind,proto3,enum=bps.Kind" json:"kind,omitempty"` + Challenge []byte `protobuf:"bytes,3,opt,name=challenge,proto3" json:"challenge,omitempty"` + Index uint64 `protobuf:"varint,4,opt,name=index,proto3" json:"index,omitempty"` } -func (m *JoinAck) Reset() { *m = JoinAck{} } -func (m *JoinAck) String() string { return proto.CompactTextString(m) } -func (*JoinAck) ProtoMessage() {} -func (*JoinAck) Descriptor() ([]byte, []int) { - return fileDescriptor_786299925f8760d5, []int{2} +func (m *Broadcast) Reset() { *m = Broadcast{} } +func (m *Broadcast) String() string { return proto.CompactTextString(m) } +func (*Broadcast) ProtoMessage() {} +func (*Broadcast) Descriptor() ([]byte, []int) { + return fileDescriptor_786299925f8760d5, []int{3} } -func (m *JoinAck) XXX_Unmarshal(b []byte) error { +func (m *Broadcast) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *JoinAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *Broadcast) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_JoinAck.Marshal(b, m, deterministic) + return xxx_messageInfo_Broadcast.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -218,68 +305,89 @@ func (m *JoinAck) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *JoinAck) XXX_Merge(src proto.Message) { - xxx_messageInfo_JoinAck.Merge(m, src) +func (m *Broadcast) XXX_Merge(src proto.Message) { + xxx_messageInfo_Broadcast.Merge(m, src) } -func (m *JoinAck) XXX_Size() int { +func (m *Broadcast) XXX_Size() int { return m.Size() } -func (m *JoinAck) XXX_DiscardUnknown() { - xxx_messageInfo_JoinAck.DiscardUnknown(m) +func (m *Broadcast) XXX_DiscardUnknown() { + xxx_messageInfo_Broadcast.DiscardUnknown(m) } -var xxx_messageInfo_JoinAck proto.InternalMessageInfo +var xxx_messageInfo_Broadcast proto.InternalMessageInfo + +func (m *Broadcast) GetSoc() []byte { + if m != nil { + return m.Soc + } + return nil +} + +func (m *Broadcast) GetKind() Kind { + if m != nil { + return m.Kind + } + return Kind_KIND_UNSPECIFIED +} -func (m *JoinAck) GetChallenge() []byte { +func (m *Broadcast) GetChallenge() []byte { if m != nil { return m.Challenge } return nil } -func (m *JoinAck) GetStatus() Status { +func (m *Broadcast) GetIndex() uint64 { if m != nil { - return m.Status + return m.Index } - return Status_STATUS_UNDEFINED + return 0 } func init() { proto.RegisterEnum("bps.TopicBinding", TopicBinding_name, TopicBinding_value) proto.RegisterEnum("bps.Status", Status_name, Status_value) - proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") + proto.RegisterEnum("bps.Kind", Kind_name, Kind_value) + proto.RegisterType((*CohortSpec)(nil), "bps.CohortSpec") proto.RegisterType((*Join)(nil), "bps.Join") - proto.RegisterType((*JoinAck)(nil), "bps.JoinAck") + proto.RegisterType((*Ack)(nil), "bps.Ack") + proto.RegisterType((*Broadcast)(nil), "bps.Broadcast") } func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 328 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0x31, 0x4f, 0xc2, 0x40, - 0x18, 0x86, 0x7b, 0x14, 0xc1, 0x7e, 0xa0, 0x1c, 0x9f, 0x98, 0x34, 0x06, 0x1b, 0x82, 0x0b, 0xc1, - 0x84, 0x41, 0x07, 0x67, 0x6a, 0x0f, 0x03, 0x92, 0x9a, 0xb4, 0x65, 0x71, 0x31, 0x6d, 0x21, 0x78, - 0x91, 0xb4, 0x17, 0x7a, 0x0e, 0xce, 0xfe, 0x01, 0x7f, 0x96, 0x23, 0xa3, 0xa3, 0x81, 0x3f, 0x62, - 0x5a, 0x8a, 0xe8, 0x76, 0xef, 0xf3, 0xde, 0xbd, 0x79, 0x92, 0x03, 0x2d, 0x10, 0x49, 0x4f, 0x2c, - 0x63, 0x19, 0xa3, 0x1a, 0x88, 0xa4, 0x7d, 0x0e, 0x9a, 0xb9, 0x8c, 0xfd, 0x69, 0xe8, 0x27, 0x12, - 0x29, 0xa8, 0x49, 0x1c, 0xea, 0xa4, 0x45, 0x3a, 0x55, 0x27, 0x3d, 0xb6, 0xdf, 0x09, 0x14, 0x47, - 0x31, 0x8f, 0xb0, 0x01, 0x07, 0x32, 0x16, 0x7c, 0x57, 0x6e, 0x03, 0x5e, 0x42, 0x39, 0xe0, 0xd1, - 0x94, 0x47, 0x73, 0xbd, 0xd0, 0x22, 0x9d, 0xe3, 0xab, 0x7a, 0x2f, 0xdd, 0xf7, 0xd2, 0xd2, 0xdc, - 0x16, 0xce, 0xee, 0x06, 0x36, 0x41, 0x13, 0x4b, 0x1e, 0x85, 0x5c, 0xf8, 0x0b, 0x5d, 0xcd, 0x66, - 0xf6, 0x00, 0xcf, 0xe0, 0x90, 0x4f, 0x67, 0x91, 0xe4, 0xf2, 0x4d, 0x2f, 0x66, 0xe5, 0x6f, 0x6e, - 0x8f, 0xa1, 0x9c, 0x4a, 0xf4, 0xc3, 0x97, 0x74, 0x24, 0x7c, 0xf6, 0x17, 0x8b, 0x59, 0x34, 0x9f, - 0xe5, 0x2e, 0x7b, 0x80, 0x17, 0x50, 0x4a, 0xa4, 0x2f, 0x5f, 0x93, 0x5c, 0xa7, 0x92, 0xe9, 0xb8, - 0x19, 0x72, 0xf2, 0xaa, 0x7b, 0x03, 0xd5, 0xbf, 0x82, 0x78, 0x0a, 0x75, 0x73, 0x68, 0x5b, 0x43, - 0xfb, 0xee, 0x69, 0x62, 0x5b, 0x6c, 0x30, 0xb4, 0x99, 0x45, 0x15, 0xa4, 0x50, 0xdd, 0xe1, 0x01, - 0x63, 0x16, 0x25, 0x5d, 0x17, 0x4a, 0xdb, 0x29, 0x6c, 0x00, 0x75, 0xbd, 0xbe, 0x37, 0x71, 0xff, - 0xbd, 0x38, 0x02, 0x2d, 0xa7, 0x0f, 0xf7, 0x94, 0x60, 0x0d, 0x2a, 0x79, 0x1c, 0x4c, 0xc6, 0x63, - 0x5a, 0xc0, 0x13, 0xa8, 0xe5, 0xc0, 0x61, 0x23, 0x76, 0xeb, 0x31, 0x8b, 0xaa, 0x66, 0xf3, 0x73, - 0x6d, 0x90, 0xd5, 0xda, 0x20, 0xdf, 0x6b, 0x83, 0x7c, 0x6c, 0x0c, 0x65, 0xb5, 0x31, 0x94, 0xaf, - 0x8d, 0xa1, 0x3c, 0x16, 0x44, 0x10, 0x94, 0xb2, 0xaf, 0xba, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, - 0x32, 0x9a, 0x54, 0x7b, 0xb7, 0x01, 0x00, 0x00, + // 416 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xc1, 0x6a, 0xdb, 0x40, + 0x10, 0x86, 0x2d, 0x69, 0xab, 0xda, 0x13, 0xe3, 0x6e, 0x87, 0x50, 0x5c, 0x48, 0x84, 0x71, 0x0f, + 0x35, 0x2e, 0x84, 0xe2, 0x9e, 0x0b, 0x95, 0x25, 0xb9, 0x51, 0x1c, 0x9c, 0xb0, 0x92, 0x2f, 0xbd, + 0x18, 0x49, 0x2b, 0x92, 0x25, 0xe9, 0xae, 0xb0, 0x54, 0xe8, 0x63, 0xf4, 0xb1, 0x7a, 0xcc, 0xb1, + 0xc7, 0x62, 0xbf, 0x48, 0xd1, 0x4a, 0x21, 0x8d, 0x6f, 0x33, 0xf3, 0xcf, 0x7c, 0xb3, 0xff, 0xb0, + 0xd0, 0x4b, 0x8b, 0xf2, 0xac, 0xd8, 0xaa, 0x4a, 0xa1, 0x95, 0x16, 0xe5, 0x38, 0x07, 0xf0, 0xd4, + 0xad, 0xda, 0x56, 0x51, 0x91, 0x67, 0x78, 0x0c, 0x2f, 0x2a, 0x55, 0x88, 0x6c, 0x68, 0x8c, 0x8c, + 0x49, 0x9f, 0x35, 0x09, 0x7e, 0x80, 0x97, 0xa9, 0x90, 0x5c, 0xc8, 0x9b, 0xa1, 0x39, 0x32, 0x26, + 0x83, 0xd9, 0xeb, 0xb3, 0x9a, 0x12, 0xd7, 0xe2, 0xbc, 0x11, 0xd8, 0x63, 0x47, 0x8d, 0x48, 0xf8, + 0x77, 0x21, 0x87, 0x56, 0x83, 0xd0, 0xc9, 0xd8, 0x03, 0x72, 0xa1, 0x84, 0xc4, 0xf7, 0x60, 0x67, + 0x7a, 0x9d, 0xde, 0x70, 0x34, 0x7b, 0xa5, 0x49, 0x4f, 0x2f, 0x60, 0xad, 0x8c, 0x08, 0x24, 0xe1, + 0x7c, 0xab, 0x17, 0xf6, 0x99, 0x8e, 0xc7, 0xe7, 0x60, 0xb9, 0xd9, 0x1d, 0xbe, 0x03, 0xbb, 0xac, + 0x92, 0xea, 0x47, 0xa9, 0x19, 0x83, 0xd9, 0x91, 0x66, 0x44, 0xba, 0xc4, 0x5a, 0x09, 0x4f, 0xa0, + 0x97, 0xdd, 0x26, 0xf7, 0xf7, 0xb9, 0xbc, 0xc9, 0x5b, 0xc8, 0x53, 0x61, 0x5c, 0x40, 0x6f, 0xbe, + 0x55, 0x09, 0xcf, 0x92, 0xb2, 0x42, 0x0a, 0x56, 0xa9, 0x1e, 0x2d, 0xd7, 0x21, 0x9e, 0x02, 0xb9, + 0x13, 0x92, 0xb7, 0x6e, 0x7b, 0x9a, 0xbf, 0x14, 0x92, 0x33, 0x5d, 0x7e, 0xce, 0xb6, 0x0e, 0xd8, + 0xf5, 0x01, 0x84, 0xe4, 0xf9, 0xcf, 0x21, 0x19, 0x19, 0x13, 0xc2, 0x9a, 0x64, 0xfa, 0x19, 0xfa, + 0xff, 0xdf, 0x0b, 0x4f, 0xe1, 0x6d, 0x7c, 0x75, 0x1d, 0x7a, 0x9b, 0x79, 0xb8, 0xf2, 0xc3, 0xd5, + 0xd7, 0xcd, 0x7a, 0x15, 0x5d, 0x07, 0x5e, 0xb8, 0x08, 0x03, 0x9f, 0x76, 0x70, 0x00, 0xb0, 0x08, + 0x02, 0x7f, 0xa3, 0x7b, 0x28, 0x99, 0x7e, 0x01, 0xbb, 0x31, 0x88, 0x6f, 0x00, 0xa3, 0xd8, 0x8d, + 0xd7, 0xd1, 0xc1, 0x84, 0x0d, 0xe6, 0xd5, 0x92, 0x1a, 0xd8, 0x05, 0xb2, 0x58, 0x5f, 0x5e, 0x52, + 0x13, 0xfb, 0xd0, 0x65, 0xc1, 0x45, 0xe0, 0xc5, 0x81, 0x4f, 0xad, 0xe9, 0x47, 0x20, 0xb5, 0x05, + 0x3c, 0x06, 0xba, 0x0c, 0x57, 0xfe, 0xc1, 0x74, 0x17, 0x88, 0xef, 0xc6, 0x6e, 0x33, 0xef, 0xae, + 0xe3, 0x73, 0x6a, 0xce, 0x4f, 0x7e, 0xef, 0x1c, 0xe3, 0x61, 0xe7, 0x18, 0x7f, 0x77, 0x8e, 0xf1, + 0x6b, 0xef, 0x74, 0x1e, 0xf6, 0x4e, 0xe7, 0xcf, 0xde, 0xe9, 0x7c, 0x33, 0x8b, 0x34, 0xb5, 0xf5, + 0x27, 0xfa, 0xf4, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xc2, 0xe3, 0x2e, 0x00, 0x51, 0x02, 0x00, 0x00, } -func (m *Broadcast) Marshal() (dAtA []byte, err error) { +func (m *CohortSpec) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -289,20 +397,32 @@ func (m *Broadcast) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Broadcast) MarshalTo(dAtA []byte) (int, error) { +func (m *CohortSpec) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *CohortSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Soc) > 0 { - i -= len(m.Soc) - copy(dAtA[i:], m.Soc) - i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + if len(m.Admin) > 0 { + i -= len(m.Admin) + copy(dAtA[i:], m.Admin) + i = encodeVarintBps(dAtA, i, uint64(len(m.Admin))) + i-- + dAtA[i] = 0x1a + } + if m.Binding != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + i-- + dAtA[i] = 0x10 + } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) i-- dAtA[i] = 0xa } @@ -329,36 +449,64 @@ func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Identity) > 0 { - i -= len(m.Identity) - copy(dAtA[i:], m.Identity) - i = encodeVarintBps(dAtA, i, uint64(len(m.Identity))) + if len(m.Addr) > 0 { + i -= len(m.Addr) + copy(dAtA[i:], m.Addr) + i = encodeVarintBps(dAtA, i, uint64(len(m.Addr))) i-- - dAtA[i] = 0x22 + dAtA[i] = 0x12 } - if len(m.Principal) > 0 { - i -= len(m.Principal) - copy(dAtA[i:], m.Principal) - i = encodeVarintBps(dAtA, i, uint64(len(m.Principal))) + if m.Cohort != nil { + { + size, err := m.Cohort.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintBps(dAtA, i, uint64(size)) + } i-- - dAtA[i] = 0x1a + dAtA[i] = 0xa } - if m.Binding != 0 { - i = encodeVarintBps(dAtA, i, uint64(m.Binding)) + return len(dAtA) - i, nil +} + +func (m *Ack) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Ack) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Ack) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) i-- - dAtA[i] = 0x10 + dAtA[i] = 0x12 } - if len(m.Topic) > 0 { - i -= len(m.Topic) - copy(dAtA[i:], m.Topic) - i = encodeVarintBps(dAtA, i, uint64(len(m.Topic))) + if m.Status != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Status)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func (m *JoinAck) Marshal() (dAtA []byte, err error) { +func (m *Broadcast) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -368,26 +516,38 @@ func (m *JoinAck) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *JoinAck) MarshalTo(dAtA []byte) (int, error) { +func (m *Broadcast) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *JoinAck) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Broadcast) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if m.Status != 0 { - i = encodeVarintBps(dAtA, i, uint64(m.Status)) + if m.Index != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Index)) i-- - dAtA[i] = 0x10 + dAtA[i] = 0x20 } if len(m.Challenge) > 0 { i -= len(m.Challenge) copy(dAtA[i:], m.Challenge) i = encodeVarintBps(dAtA, i, uint64(len(m.Challenge))) i-- + dAtA[i] = 0x1a + } + if m.Kind != 0 { + i = encodeVarintBps(dAtA, i, uint64(m.Kind)) + i-- + dAtA[i] = 0x10 + } + if len(m.Soc) > 0 { + i -= len(m.Soc) + copy(dAtA[i:], m.Soc) + i = encodeVarintBps(dAtA, i, uint64(len(m.Soc))) + i-- dAtA[i] = 0xa } return len(dAtA) - i, nil @@ -404,13 +564,20 @@ func encodeVarintBps(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *Broadcast) Size() (n int) { +func (m *CohortSpec) Size() (n int) { if m == nil { return 0 } var l int _ = l - l = len(m.Soc) + l = len(m.Topic) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Binding != 0 { + n += 1 + sovBps(uint64(m.Binding)) + } + l = len(m.Admin) if l > 0 { n += 1 + l + sovBps(uint64(l)) } @@ -423,36 +590,52 @@ func (m *Join) Size() (n int) { } var l int _ = l - l = len(m.Topic) - if l > 0 { + if m.Cohort != nil { + l = m.Cohort.Size() n += 1 + l + sovBps(uint64(l)) } - if m.Binding != 0 { - n += 1 + sovBps(uint64(m.Binding)) - } - l = len(m.Principal) + l = len(m.Addr) if l > 0 { n += 1 + l + sovBps(uint64(l)) } - l = len(m.Identity) + return n +} + +func (m *Ack) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != 0 { + n += 1 + sovBps(uint64(m.Status)) + } + l = len(m.Challenge) if l > 0 { n += 1 + l + sovBps(uint64(l)) } return n } -func (m *JoinAck) Size() (n int) { +func (m *Broadcast) Size() (n int) { if m == nil { return 0 } var l int _ = l + l = len(m.Soc) + if l > 0 { + n += 1 + l + sovBps(uint64(l)) + } + if m.Kind != 0 { + n += 1 + sovBps(uint64(m.Kind)) + } l = len(m.Challenge) if l > 0 { n += 1 + l + sovBps(uint64(l)) } - if m.Status != 0 { - n += 1 + sovBps(uint64(m.Status)) + if m.Index != 0 { + n += 1 + sovBps(uint64(m.Index)) } return n } @@ -463,7 +646,7 @@ func sovBps(x uint64) (n int) { func sozBps(x uint64) (n int) { return sovBps(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *Broadcast) Unmarshal(dAtA []byte) error { +func (m *CohortSpec) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -486,15 +669,15 @@ func (m *Broadcast) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Broadcast: wiretype end group for non-group") + return fmt.Errorf("proto: CohortSpec: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Broadcast: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: CohortSpec: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -521,9 +704,62 @@ func (m *Broadcast) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) - if m.Soc == nil { - m.Soc = []byte{} + m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) + if m.Topic == nil { + m.Topic = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) + } + m.Binding = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Binding |= TopicBinding(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Admin", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Admin = append(m.Admin[:0], dAtA[iNdEx:postIndex]...) + if m.Admin == nil { + m.Admin = []byte{} } iNdEx = postIndex default: @@ -581,9 +817,9 @@ func (m *Join) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Cohort", wireType) } - var byteLen int + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowBps @@ -593,48 +829,31 @@ func (m *Join) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + if msglen < 0 { return ErrInvalidLengthBps } - postIndex := iNdEx + byteLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthBps } if postIndex > l { return io.ErrUnexpectedEOF } - m.Topic = append(m.Topic[:0], dAtA[iNdEx:postIndex]...) - if m.Topic == nil { - m.Topic = []byte{} + if m.Cohort == nil { + m.Cohort = &CohortSpec{} + } + if err := m.Cohort.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err } iNdEx = postIndex case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Binding", wireType) - } - m.Binding = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowBps - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Binding |= TopicBinding(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Principal", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Addr", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -661,14 +880,86 @@ func (m *Join) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Principal = append(m.Principal[:0], dAtA[iNdEx:postIndex]...) - if m.Principal == nil { - m.Principal = []byte{} + m.Addr = append(m.Addr[:0], dAtA[iNdEx:postIndex]...) + if m.Addr == nil { + m.Addr = []byte{} } iNdEx = postIndex - case 4: + default: + iNdEx = preIndex + skippy, err := skipBps(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthBps + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Ack) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ack: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ack: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= Status(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -695,9 +986,9 @@ func (m *Join) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) - if m.Identity == nil { - m.Identity = []byte{} + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} } iNdEx = postIndex default: @@ -724,7 +1015,7 @@ func (m *Join) Unmarshal(dAtA []byte) error { } return nil } -func (m *JoinAck) Unmarshal(dAtA []byte) error { +func (m *Broadcast) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -747,13 +1038,66 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: JoinAck: wiretype end group for non-group") + return fmt.Errorf("proto: Broadcast: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: JoinAck: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Broadcast: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Soc", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthBps + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthBps + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Soc = append(m.Soc[:0], dAtA[iNdEx:postIndex]...) + if m.Soc == nil { + m.Soc = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Kind", wireType) + } + m.Kind = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBps + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Kind |= Kind(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) } @@ -787,11 +1131,11 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { m.Challenge = []byte{} } iNdEx = postIndex - case 2: + case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } - m.Status = 0 + m.Index = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowBps @@ -801,7 +1145,7 @@ func (m *JoinAck) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= Status(b&0x7F) << shift + m.Index |= uint64(b&0x7F) << shift if b < 0x80 { break } diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index d112c17d114..f2a9250efce 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -2,39 +2,58 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Wire format of SWIP-74 (BPS-lite). Field and enum numbers are shared with +// SWIP-60, which extends these messages by adding fields and values only. syntax = "proto3"; package bps; option go_package = "pb"; -message Broadcast { - // either service or broadcast - bytes soc = 1; +// What the topic binds to. One binding. +enum TopicBinding { + TOPIC_BINDING_UNSPECIFIED = 0; // invalid on the wire + FEED_TOPIC = 4; // the stream is a feed on the topic } -enum TopicBinding { - BINDING_UNDEFINED = 0; - BINDING_FEED = 1; +// The cohort's identity, keyed by its canonical serialisation. +message CohortSpec { + bytes topic = 1; // 32 bytes + TopicBinding binding = 2; // FEED_TOPIC + bytes admin = 3; // 20 bytes, required } +// Peer -> broker: the first frame on a fresh stream. message Join { - bytes topic = 1; - TopicBinding binding = 2; // which kind of topic binding - bytes principal = 3; // the "owner" - // ============================ - bytes identity = 4; // who am i + CohortSpec cohort = 1; + bytes addr = 2; // 20 bytes, required: the stream's identity } enum Status { - STATUS_UNDEFINED = 0; - STATUS_OK = 1; - STATUS_FULL = 2; - STATUS_REJECTED = 3; + STATUS_UNSPECIFIED = 0; // invalid on the wire + OK = 1; + FULL = 2; // a capacity bound + REJECTED = 3; // a spec value outside SWIP-74, or a malformed Join } -message JoinAck { - bytes challenge = 1; // the challenge to sign in order to claim the cohort's publishing role stream - Status status = 2; +// Broker -> peer, answering Join. A non-OK Ack ends the stream. +message Ack { + Status status = 1; + bytes challenge = 2; // iff OK: 32 random bytes for this stream } +// What a chunk on this wire is. +enum Kind { + KIND_UNSPECIFIED = 0; // invalid on the wire + DATA = 1; // an update of the stream's session feed + AUTH = 2; // an empty chunk: claims the stream, never delivered +} + +// Every frame after the handshake. soc is an ordinary single-owner chunk +// whose id is derived from kind, challenge and index (see bps.ID). +message Broadcast { + bytes soc = 1; + Kind kind = 2; + bytes challenge = 3; // 32 bytes + uint64 index = 4; +} diff --git a/pkg/node/node.go b/pkg/node/node.go index e481d524ac7..a5f947a5bb0 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -114,6 +114,7 @@ type Bee struct { pullSyncCloser io.Closer pssCloser io.Closer gsocCloser io.Closer + bpsCloser io.Closer transactionMonitorCloser io.Closer transactionCloser io.Closer listenerCloser io.Closer @@ -1104,7 +1105,8 @@ func NewBee( b.pssCloser = pssService b.gsocCloser = gsocService - bpsService := bps.New(p2ps, swarmAddress, o.FullNodeMode, logger) + bpsService := bps.New(p2ps, p2ps, o.FullNodeMode, logger, bps.DefaultOptions) + b.bpsCloser = bpsService if o.FullNodeMode { if err = p2ps.AddProtocol(bpsService.Protocol()); err != nil { return nil, fmt.Errorf("bps protocol: %w", err) @@ -1401,6 +1403,7 @@ func NewBee( if o.APIAddr != "" { // register metrics from components apiService.MustRegisterMetrics(p2ps.Metrics()...) + apiService.MustRegisterMetrics(bpsService.Metrics()...) apiService.MustRegisterMetrics(pingPong.Metrics()...) apiService.MustRegisterMetrics(acc.Metrics()...) apiService.MustRegisterMetrics(localStore.Metrics()...) @@ -1487,6 +1490,7 @@ func (b *Bee) shutdownClosers() []namedCloser { return []namedCloser{ {b.pssCloser, "pss"}, {b.gsocCloser, "gsoc"}, + {b.bpsCloser, "bps"}, {b.pusherCloser, "pusher"}, {b.pullerCloser, "puller"}, {b.accountingCloser, "accounting"}, From 7e590ce305a102b131d61e5baea4ef44daa92eb4 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:02:03 -0600 Subject: [PATCH 2/4] chore: pr comments --- openapi/Swarm.yaml | 8 +- pkg/api/bps.go | 40 +++++----- pkg/api/bps_test.go | 24 ++++-- pkg/api/export_test.go | 10 +-- pkg/api/status_test.go | 1 - pkg/bps/TODO | 10 --- pkg/bps/bps.go | 174 ++++++++++++++++++++++------------------- pkg/bps/bps_test.go | 62 +++++++-------- pkg/bps/id.go | 18 ++--- pkg/bps/metrics.go | 6 +- pkg/bps/pb/bps.pb.go | 109 +++++++++++++------------- pkg/bps/pb/bps.proto | 6 +- 12 files changed, 238 insertions(+), 230 deletions(-) delete mode 100644 pkg/bps/TODO diff --git a/openapi/Swarm.yaml b/openapi/Swarm.yaml index 9a146f5d48a..2bc42c59573 100644 --- a/openapi/Swarm.yaml +++ b/openapi/Swarm.yaml @@ -1034,7 +1034,7 @@ paths: "/bps/publish/{owner}/{topic}": get: - summary: Publish to a BPS cohort as its admin + summary: Publish to a BPS cohort as its principal description: | WebSocket. The node joins the BPS-lite (SWIP-74) cohort `{topic, FEED_TOPIC, owner}` at `broker` declaring `owner` as its address. @@ -1044,12 +1044,12 @@ paths: `authTopic` is `keccak256("bps-service:v1" | 0x02 | topic | challenge)`. 2. Client sends binary frames `kind (1) | index (8, big-endian) | soc`, where `kind` is 1 (DATA) or 2 (AUTH) and `soc` is a single-owner chunk signed by `owner` with - `id = keccak256(sessionTopic | index)`. An AUTH chunk is empty and only claims the - stream. The first frame must arrive within 30 seconds. Indexes must not be reused + `id = keccak256(sessionTopic | index)`. An AUTH chunk is empty and only authenticates + the stream. The first frame must arrive within 30 seconds. Indexes must not be reused across sessions; frames below the broker's cursor are dropped silently. Close codes: 4001 invalid soc, 4002 invalid message (wrong frame type, unknown kind, - claim timeout), 4003 broker stream closed. + auth timeout), 4003 broker stream closed. tags: - BPS parameters: diff --git a/pkg/api/bps.go b/pkg/api/bps.go index 32fe0d5283a..a846ec3bcde 100644 --- a/pkg/api/bps.go +++ b/pkg/api/bps.go @@ -34,8 +34,8 @@ const ( bpsMaxFrameSize = bpsFrameHeader + bpsMaxSOCSize ) -// bpsClaimTimeout bounds how long a publisher may take to send its first frame. -var bpsClaimTimeout = 30 * time.Second +// bpsAuthTimeout bounds how long a publisher may take to send its first frame. +var bpsAuthTimeout = 30 * time.Second var errBPSBrokerGone = errors.New("broker stream closed") @@ -231,11 +231,10 @@ func (s *Service) bpsSubscribeWs(conn *websocket.Conn, req bpsRequest, logger lo }() sess, err := s.bps.Join(ctx, bps.JoinRequest{ - Broker: req.broker, - Topic: req.topic, - Admin: req.owner.Bytes(), - Addr: req.identity, - Cursor: req.cursor, + Broker: req.broker, + Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, + Identity: req.identity, + Cursor: req.cursor, }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -322,12 +321,11 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. _ = conn.Close() }() - // the publisher declares the admin's address and claims with its first frame + // the publisher declares the principal as its identity and authenticates with its first frame sess, err := s.bps.Join(ctx, bps.JoinRequest{ - Broker: req.broker, - Topic: req.topic, - Admin: req.owner.Bytes(), - Addr: req.owner.Bytes(), + Broker: req.broker, + Spec: bps.CohortSpec{Topic: req.topic, Principal: req.owner.Bytes()}, + Identity: req.owner.Bytes(), }) if err != nil { logger.Debug("join failed", "broker", req.broker, "error", err) @@ -363,11 +361,11 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. return } - claimTimer := time.NewTimer(bpsClaimTimeout) - defer claimTimer.Stop() + authTimer := time.NewTimer(bpsAuthTimeout) + defer authTimer.Stop() frames, gone := bpsReadFrames(conn, quit) - claimed := false + authenticated := false for { select { @@ -381,13 +379,13 @@ func (s *Service) bpsPublishWs(conn *websocket.Conn, req bpsRequest, logger log. s.bpsClose(conn, bpsCloseBrokerGone, bpsErrReason(err)) return } - if !claimed { - claimed = true - claimTimer.Stop() + if !authenticated { + authenticated = true + authTimer.Stop() } - case <-claimTimer.C: - if !claimed { - s.bpsClose(conn, bpsCloseInvalidMessage, "claim timeout") + case <-authTimer.C: + if !authenticated { + s.bpsClose(conn, bpsCloseInvalidMessage, "auth timeout") return } case <-sess.Done(): diff --git a/pkg/api/bps_test.go b/pkg/api/bps_test.go index ed651235116..97dd8a8d6e2 100644 --- a/pkg/api/bps_test.go +++ b/pkg/api/bps_test.go @@ -275,7 +275,7 @@ func TestBPSSubscribe(t *testing.T) { f.bps.mu.Lock() req := f.bps.req f.bps.mu.Unlock() - if !req.Broker.Equal(f.broker) || !bytes.Equal(req.Admin, f.owner) || !bytes.Equal(req.Addr, f.identity) || !bytes.Equal(req.Topic, f.topic) { + if !req.Broker.Equal(f.broker) || !bytes.Equal(req.Spec.Principal, f.owner) || !bytes.Equal(req.Identity, f.identity) || !bytes.Equal(req.Spec.Topic, f.topic) { t.Fatalf("unexpected join request %+v", req) } }) @@ -385,8 +385,8 @@ func TestBPSPublish(t *testing.T) { f.bps.mu.Lock() req := f.bps.req f.bps.mu.Unlock() - if !bytes.Equal(req.Addr, f.owner) { - t.Fatalf("publisher joined as %x, want the admin %x", req.Addr, f.owner) + if !bytes.Equal(req.Identity, f.owner) { + t.Fatalf("publisher joined as %x, want the principal %x", req.Identity, f.owner) } auth := f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil) @@ -416,6 +416,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "wrong signer", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, otherSigner, bps.KindData, challenge, 0, []byte("x")))) }, code: api.BPSCloseInvalidSOC, @@ -423,6 +424,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "wrong challenge", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, _ []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindData, make([]byte, 32), 0, []byte("x")))) }, code: api.BPSCloseInvalidSOC, @@ -430,6 +432,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "index not the signed one", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 1, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) }, code: api.BPSCloseInvalidSOC, @@ -437,6 +440,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "auth relabelled as data", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindData, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, nil))) }, code: api.BPSCloseInvalidSOC, @@ -444,6 +448,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "auth with payload", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.KindAuth, 0, f.signedSOC(t, f.signer, bps.KindAuth, challenge, 0, []byte("x")))) }, code: api.BPSCloseInvalidSOC, @@ -451,6 +456,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "unknown kind", send: func(t *testing.T, f *bpsFixture, conn *websocket.Conn, challenge []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, frame(bps.Kind(9), 0, f.signedSOC(t, f.signer, bps.KindData, challenge, 0, []byte("x")))) }, code: api.BPSCloseInvalidMessage, @@ -458,6 +464,7 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "short frame", send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { + t.Helper() _ = conn.WriteMessage(websocket.BinaryMessage, []byte{1, 2}) }, code: api.BPSCloseInvalidMessage, @@ -465,7 +472,8 @@ func TestBPSPublishFrameErrors(t *testing.T) { { name: "text frame", send: func(t *testing.T, _ *bpsFixture, conn *websocket.Conn, _ []byte) { - _ = conn.WriteJSON(map[string]string{"type": "claim"}) + t.Helper() + _ = conn.WriteJSON(map[string]string{"type": "auth"}) }, code: api.BPSCloseInvalidMessage, }, @@ -509,7 +517,7 @@ func TestBPSPublishSessionErrors(t *testing.T) { }) } -func TestBPSPublishClientGoneBeforeClaim(t *testing.T) { +func TestBPSPublishClientGoneBeforeAuth(t *testing.T) { t.Parallel() f := newBPSFixture(t) conn := f.dial(t, "publish") @@ -530,9 +538,9 @@ func TestBPSPublishOversizedFrame(t *testing.T) { waitSessionClosed(t, f) } -// Not parallel: mutates the package-level claim timeout. -func TestBPSPublishClaimTimeout(t *testing.T) { - restore := api.SetBPSClaimTimeout(100 * time.Millisecond) +// Not parallel: mutates the package-level auth timeout. +func TestBPSPublishAuthTimeout(t *testing.T) { + restore := api.SetBPSAuthTimeout(100 * time.Millisecond) defer restore() f := newBPSFixture(t) conn := f.dial(t, "publish") diff --git a/pkg/api/export_test.go b/pkg/api/export_test.go index 499fc6d8aa3..226a5cc5bc3 100644 --- a/pkg/api/export_test.go +++ b/pkg/api/export_test.go @@ -149,10 +149,10 @@ const ( BPSCloseBrokerGone = bpsCloseBrokerGone ) -// SetBPSClaimTimeout overrides the claim timeout and returns a restore func. +// SetBPSAuthTimeout overrides the auth timeout and returns a restore func. // Callers must not run in parallel with other bps tests. -func SetBPSClaimTimeout(d time.Duration) func() { - prev := bpsClaimTimeout - bpsClaimTimeout = d - return func() { bpsClaimTimeout = prev } +func SetBPSAuthTimeout(d time.Duration) func() { + prev := bpsAuthTimeout + bpsAuthTimeout = d + return func() { bpsAuthTimeout = prev } } diff --git a/pkg/api/status_test.go b/pkg/api/status_test.go index 9da5797caf8..dffc1ab4f4c 100644 --- a/pkg/api/status_test.go +++ b/pkg/api/status_test.go @@ -74,7 +74,6 @@ func TestGetStatus(t *testing.T) { jsonhttptest.WithExpectedJSONResponse(ssr), ) }) - } // TestGetStatusPeersIncludesBootnodes is a regression test for diff --git a/pkg/bps/TODO b/pkg/bps/TODO deleted file mode 100644 index f9011f1c0ce..00000000000 --- a/pkg/bps/TODO +++ /dev/null @@ -1,10 +0,0 @@ -Aligned with SWIP-74 rev 8 (ethersphere/SWIPs#111, SWIPs/swip-74.md). Still open: - -1. Client backoff: a peer whose join was refused or whose stream was reset MUST back off - (randomised exponential, 1s-30s, full jitter) before rejoining. Join is one-shot; the - API websocket closes instead of retrying, so a reconnecting client owns the backoff. -2. Spec open points (?) follow the draft as written: the kind byte in the service prefix, - streams per peer per cohort = 2, claim deadline = 30s. -3. Per-cohort counters are exposed via Service.Counters; there is no API endpoint for them - yet (prometheus carries the aggregate per outcome). -4. Conformance 12 (interop with a full SWIP-60 broker) is untestable until SWIP-60 exists. diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index f05db6767ed..f7ff4616700 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -5,11 +5,11 @@ // Package bps implements BPS-lite (SWIP-74): a single-publisher live stream // over a feed, through one broker, one hop. // -// A cohort is identified by its spec {topic, FEED_TOPIC, admin}. Every peer -// joins with the spec and its own address and is answered with a random -// per-stream challenge. The admin publishes ordinary single-owner chunks that +// A cohort is identified by its spec {topic, FEED_TOPIC, principal}. Every +// peer joins with the spec and its own identity and is answered with a random +// per-stream challenge. The principal publishes ordinary single-owner chunks that // are updates of the session feed on keccak256(topic | challenge); its first -// valid frame claims the stream. The broker keeps one cursor per cohort and +// valid frame authenticates the stream. The broker keeps one cursor per cohort and // delivers every accepted update to every subscriber stream. package bps @@ -35,9 +35,9 @@ import ( const loggerName = "bps" const ( - protocolName = "pubsub" + protocolName = "bps" protocolVersion = "1.0.0" - streamName = "pubsub" + streamName = "bps" ) var ( @@ -49,6 +49,7 @@ var ( errViolation = errors.New("bps: protocol violation") errWrongStream = errors.New("bps: frame from subscriber stream") errCohortReclaimed = errors.New("bps: cohort reclaimed") + errFull = errors.New("bps: capacity bound") ) // Options are the broker's resource bounds. Zero values take the defaults. @@ -57,24 +58,24 @@ type Options struct { MaxCohorts int // live cohorts per broker MaxCohortsPerPeer int // cohorts per peer connection MaxStreamsPerPeerCohort int // streams per peer connection per cohort - InactivityDeadline time.Duration // reclaims a cohort with no accepted frame - ClaimDeadline time.Duration // disconnects a pending stream that has not claimed + InactivityTimeout time.Duration // reclaims a cohort with no accepted frame + AuthTimeout time.Duration // disconnects a pending stream that has not authenticated QueueSize int // outbound frames per subscriber stream ViolationBlocklist time.Duration // blocklist duration for a protocol violation - ClaimTimeoutBlocklist time.Duration // blocklist duration for a claim timeout + AuthTimeoutBlocklist time.Duration // blocklist duration for an auth timeout } // DefaultOptions are the SWIP-74 recommended bounds. var DefaultOptions = Options{ MaxSubscribers: 1024, - MaxCohorts: 4096, + MaxCohorts: 512, MaxCohortsPerPeer: 16, MaxStreamsPerPeerCohort: 2, - InactivityDeadline: 10 * time.Minute, - ClaimDeadline: 30 * time.Second, + InactivityTimeout: 10 * time.Minute, + AuthTimeout: 30 * time.Second, QueueSize: 64, ViolationBlocklist: 10 * time.Minute, - ClaimTimeoutBlocklist: time.Minute, + AuthTimeoutBlocklist: time.Minute, } func (o Options) withDefaults() Options { @@ -91,11 +92,11 @@ func (o Options) withDefaults() Options { if o.MaxStreamsPerPeerCohort > 0 { d.MaxStreamsPerPeerCohort = o.MaxStreamsPerPeerCohort } - if o.InactivityDeadline > 0 { - d.InactivityDeadline = o.InactivityDeadline + if o.InactivityTimeout > 0 { + d.InactivityTimeout = o.InactivityTimeout } - if o.ClaimDeadline > 0 { - d.ClaimDeadline = o.ClaimDeadline + if o.AuthTimeout > 0 { + d.AuthTimeout = o.AuthTimeout } if o.QueueSize > 0 { d.QueueSize = o.QueueSize @@ -103,8 +104,8 @@ func (o Options) withDefaults() Options { if o.ViolationBlocklist > 0 { d.ViolationBlocklist = o.ViolationBlocklist } - if o.ClaimTimeoutBlocklist > 0 { - d.ClaimTimeoutBlocklist = o.ClaimTimeoutBlocklist + if o.AuthTimeoutBlocklist > 0 { + d.AuthTimeoutBlocklist = o.AuthTimeoutBlocklist } return d } @@ -202,24 +203,30 @@ type Session interface { Close() error } -// JoinRequest describes a cohort join. -type JoinRequest struct { - Broker swarm.Address // the broker - Topic []byte // 32 bytes: the feed topic - Admin []byte // 20 bytes: the publisher - Addr []byte // 20 bytes: the joining stream's identity - Cursor uint64 // the lowest DATA index to deliver, the subscriber's cursor for (topic, admin) +// CohortSpec names a cohort: the feed of the principal on the topic. The +// topic binding is always FEED_TOPIC. +type CohortSpec struct { + Topic []byte // 32 bytes: the feed topic + Principal []byte // 20 bytes: the publisher +} + +func (c CohortSpec) proto() *pb.CohortSpec { + return &pb.CohortSpec{Topic: c.Topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: c.Principal} } -func (r JoinRequest) spec() *pb.CohortSpec { - return &pb.CohortSpec{Topic: r.Topic, Binding: pb.TopicBinding_FEED_TOPIC, Admin: r.Admin} +// JoinRequest describes a cohort join. +type JoinRequest struct { + Broker swarm.Address // the broker + Spec CohortSpec // the cohort to join + Identity []byte // 20 bytes: the joining stream's identity + Cursor uint64 // the lowest DATA index to deliver, the subscriber's cursor for the cohort } // Join joins a cohort at the broker. A peer whose join was refused or whose // stream was reset must back off before rejoining. func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { - spec := req.spec() - if err := validateJoin(&pb.Join{Cohort: spec, Addr: req.Addr}); err != nil { + join := &pb.Join{Cohort: req.Spec.proto(), Identity: req.Identity} + if err := validateJoin(join); err != nil { return nil, err } stream, err := s.streamer.NewStream(ctx, req.Broker, nil, protocolName, protocolVersion, streamName) @@ -228,7 +235,7 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { } w, r := protobuf.NewWriterAndReader(stream) - if err := w.WriteMsgWithContext(ctx, &pb.Join{Cohort: spec, Addr: req.Addr}); err != nil { + if err := w.WriteMsgWithContext(ctx, join); err != nil { _ = stream.Reset() return nil, fmt.Errorf("write join: %w", err) } @@ -283,7 +290,7 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { if f.Kind != pb.Kind_DATA || f.Index < sess.cursor.Load() { continue } - if err := verify(&f, req.Topic, req.Admin); err != nil { + if err := verify(&f, req.Spec.Topic, req.Spec.Principal); err != nil { s.logger.Debug("dropping delivery", "error", err) continue } @@ -348,20 +355,20 @@ func (s *session) finish(err error) { } // validateJoin rejects a Join whose spec has a value outside SWIP-74 or whose -// addr is not 20 bytes. +// identity is not 20 bytes. func validateJoin(join *pb.Join) error { - c := join.Cohort + cohortSpec := join.Cohort switch { - case c == nil: + case cohortSpec == nil: return fmt.Errorf("no cohort: %w", errInvalidJoin) - case len(c.Topic) != swarm.HashSize: - return fmt.Errorf("topic length %d: %w", len(c.Topic), errInvalidJoin) - case c.Binding != pb.TopicBinding_FEED_TOPIC: - return fmt.Errorf("binding %s: %w", c.Binding, errInvalidJoin) - case len(c.Admin) != crypto.AddressSize: - return fmt.Errorf("admin length %d: %w", len(c.Admin), errInvalidJoin) - case len(join.Addr) != crypto.AddressSize: - return fmt.Errorf("addr length %d: %w", len(join.Addr), errInvalidJoin) + case len(cohortSpec.Topic) != swarm.HashSize: + return fmt.Errorf("topic length %d: %w", len(cohortSpec.Topic), errInvalidJoin) + case cohortSpec.Binding != pb.TopicBinding_FEED_TOPIC: + return fmt.Errorf("binding %s: %w", cohortSpec.Binding, errInvalidJoin) + case len(cohortSpec.Principal) != crypto.AddressSize: + return fmt.Errorf("principal length %d: %w", len(cohortSpec.Principal), errInvalidJoin) + case len(join.Identity) != crypto.AddressSize: + return fmt.Errorf("identity length %d: %w", len(join.Identity), errInvalidJoin) } return nil } @@ -383,7 +390,7 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er return s.refuse(ctx, w, stream, pb.Status_REJECTED, err) } // re-marshal so that the cohort key ignores fields BPS-lite does not define - join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Admin: join.Cohort.Admin} + join.Cohort = &pb.CohortSpec{Topic: join.Cohort.Topic, Binding: join.Cohort.Binding, Principal: join.Cohort.Principal} ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -393,8 +400,13 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er _ = stream.Reset() }) if err != nil { - return s.refuse(ctx, w, stream, pb.Status_FULL, err) + status := pb.Status_REJECTED + if errors.Is(err, errFull) { + status = pb.Status_FULL + } + return s.refuse(ctx, w, stream, status, err) } + // detach on every exit, normal or not defer s.leave(p.Address, co, m) if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: m.challenge}); err != nil { @@ -403,10 +415,9 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } // the writer side: deliveries to a subscriber stream - var wg sync.WaitGroup - wg.Add(1) + writerDone := make(chan struct{}) go func() { - defer wg.Done() + defer close(writerDone) for { select { case f := <-m.queue: @@ -419,9 +430,9 @@ func (s *Service) handler(ctx context.Context, p p2p.Peer, stream p2p.Stream) er } } }() - defer wg.Wait() + defer func() { <-writerDone }() - // the reader side: publications and claims + // the reader side: publications and authentications for { var f pb.Broadcast if err := r.ReadMsgWithContext(ctx, &f); err != nil { @@ -477,13 +488,13 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { return nil } // 4. on a publisher stream a DATA frame below the cursor is a retransmit; - // on a pending stream it is checked after the signature, so that it claims + // on a pending stream it is checked after the signature, so that it authenticates if role == rolePublisher && f.Kind == pb.Kind_DATA && f.Index < cursor { s.count(&co.counters.retransmit, "retransmit") return nil } - // 5. the chunk must be the admin's at the id derived from the frame - if err := verify(f, co.spec.Topic, co.spec.Admin); err != nil { + // 5. the chunk must be the principal's at the id derived from the frame + if err := verify(f, co.spec.Topic, co.spec.Principal); err != nil { s.count(&co.counters.invalidSOC, "invalid_soc") return fmt.Errorf("%w: %w", errViolation, err) } @@ -492,7 +503,7 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { co.mtx.Lock() if m.role == rolePending { m.role = rolePublisher - m.claimTimer.Stop() + m.authTimer.Stop() } switch { case f.Kind == pb.Kind_AUTH: @@ -523,11 +534,12 @@ func (s *Service) handleFrame(co *cohort, m *member, f *pb.Broadcast) error { } // join attaches a stream to the cohort named by the spec, creating the -// cohort if no live one has it. An error means a capacity bound was hit. +// cohort if no live one has it. An error wrapping errFull means a capacity +// bound was hit. func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort, *member, error) { key, err := join.Cohort.Marshal() if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("marshal cohort spec: %w", err) } k := string(key) @@ -539,21 +551,21 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort pc := s.peers[peer.ByteString()] if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerPeer { - return nil, nil, errors.New("cohorts per peer") + return nil, nil, fmt.Errorf("cohorts per peer: %w", errFull) } if pc[k] >= s.opts.MaxStreamsPerPeerCohort { - return nil, nil, errors.New("streams per peer per cohort") + return nil, nil, fmt.Errorf("streams per peer per cohort: %w", errFull) } role := roleSubscriber - if bytes.Equal(join.Addr, join.Cohort.Admin) { + if bytes.Equal(join.Identity, join.Cohort.Principal) { role = rolePending } co, ok := s.cohorts[k] if !ok { if len(s.cohorts) >= s.opts.MaxCohorts { - return nil, nil, errors.New("cohorts per broker") + return nil, nil, fmt.Errorf("cohorts per broker: %w", errFull) } co = &cohort{ key: k, @@ -561,7 +573,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort members: make(map[*member]struct{}), lastActivity: time.Now(), } - co.inactivity = time.AfterFunc(s.opts.InactivityDeadline, func() { s.reclaim(co) }) + co.inactivity = time.AfterFunc(s.opts.InactivityTimeout, func() { s.reclaim(co) }) s.cohorts[k] = co s.metrics.Cohorts.Set(float64(len(s.cohorts))) } @@ -572,7 +584,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort if len(co.members) == 0 { s.removeCohort(co) } - return nil, nil, errors.New("subscribers per cohort") + return nil, nil, fmt.Errorf("subscribers per cohort: %w", errFull) } challenge := make([]byte, ChallengeSize) @@ -590,7 +602,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort var once sync.Once m.reset = func() { once.Do(reset) } if role == rolePending { - m.claimTimer = time.AfterFunc(s.opts.ClaimDeadline, func() { s.claimTimeout(peer, co, m) }) + m.authTimer = time.AfterFunc(s.opts.AuthTimeout, func() { s.authTimeout(peer, co, m) }) } else { co.subscribers++ } @@ -624,8 +636,8 @@ func (s *Service) leave(peer swarm.Address, co *cohort, m *member) { return } delete(co.members, m) - if m.claimTimer != nil { - m.claimTimer.Stop() + if m.authTimer != nil { + m.authTimer.Stop() } if m.role == roleSubscriber { co.subscribers-- @@ -651,8 +663,8 @@ func (s *Service) reclaim(co *cohort) { return } co.mtx.Lock() - if idle := time.Since(co.lastActivity); idle < s.opts.InactivityDeadline { - co.inactivity.Reset(s.opts.InactivityDeadline - idle) + if idle := time.Since(co.lastActivity); idle < s.opts.InactivityTimeout { + co.inactivity.Reset(s.opts.InactivityTimeout - idle) co.mtx.Unlock() s.mtx.Unlock() return @@ -667,26 +679,26 @@ func (s *Service) reclaim(co *cohort) { } } -// claimTimeout disconnects a pending stream that has not claimed in time. -func (s *Service) claimTimeout(peer swarm.Address, co *cohort, m *member) { +// authTimeout disconnects a pending stream that has not authenticated in time. +func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { co.mtx.Lock() pending := m.role == rolePending co.mtx.Unlock() if !pending { return } - s.count(&co.counters.claimTimeout, "claim_timeout") + s.count(&co.counters.authTimeout, "auth_timeout") if s.blocklister != nil { - if err := s.blocklister.Blocklist(peer, s.opts.ClaimTimeoutBlocklist, "bps claim timeout"); err != nil { + if err := s.blocklister.Blocklist(peer, s.opts.AuthTimeoutBlocklist, "bps auth timeout"); err != nil { s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) } } m.reset() } -// Counters returns the counters of the live cohort {topic, FEED_TOPIC, admin}. -func (s *Service) Counters(topic, admin []byte) (Counters, bool) { - key, err := (&pb.CohortSpec{Topic: topic, Binding: pb.TopicBinding_FEED_TOPIC, Admin: admin}).Marshal() +// Counters returns the counters of the live cohort named by the spec. +func (s *Service) Counters(spec CohortSpec) (Counters, bool) { + key, err := spec.proto().Marshal() if err != nil { return Counters{}, false } @@ -703,8 +715,8 @@ type role int const ( roleSubscriber role = iota + 1 // receives, never publishes - rolePending // declared the admin's addr, has not claimed yet - rolePublisher // claimed: may publish, receives nothing + rolePending // declared the principal as identity, not authenticated yet + rolePublisher // authenticated: may publish, receives nothing ) type cohort struct { @@ -733,9 +745,9 @@ func (co *cohort) shutdown() []*member { // member is one stream attached to a cohort. role is guarded by cohort.mtx. type member struct { - role role - challenge []byte - queue chan *pb.Broadcast - reset func() - claimTimer *time.Timer + role role + challenge []byte + queue chan *pb.Broadcast + reset func() + authTimer *time.Timer } diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index e01e31d1998..c88ec7a2470 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -53,7 +53,7 @@ type env struct { brokerAddr swarm.Address bl *blocklister signer crypto.Signer - admin []byte + principal []byte topic []byte } @@ -73,7 +73,7 @@ func newEnv(t *testing.T, o bps.Options) *env { } topic := make([]byte, swarm.HashSize) copy(topic, "bps-test-topic") - return &env{t: t, broker: broker, brokerAddr: swarm.RandAddress(t), bl: bl, signer: signer, admin: admin.Bytes(), topic: topic} + return &env{t: t, broker: broker, brokerAddr: swarm.RandAddress(t), bl: bl, signer: signer, principal: admin.Bytes(), topic: topic} } // client returns a node whose streams reach the broker from overlay. @@ -84,7 +84,7 @@ func (e *env) client(overlay swarm.Address) (*bps.Service, *streamtest.Recorder) func (e *env) join(c *bps.Service, addr []byte) bps.Session { e.t.Helper() - s, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Topic: e.topic, Admin: e.admin, Addr: addr}) + s, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: addr}) if err != nil { e.t.Fatal(err) } @@ -104,7 +104,7 @@ func (e *env) publisher() (bps.Session, swarm.Address) { e.t.Helper() overlay := swarm.RandAddress(e.t) c, _ := e.client(overlay) - return e.join(c, e.admin), overlay + return e.join(c, e.principal), overlay } func (e *env) chunk(signer crypto.Signer, kind bps.Kind, challenge []byte, index uint64, payload []byte) []byte { @@ -135,7 +135,7 @@ func (e *env) publish(s bps.Session, kind bps.Kind, index uint64, payload []byte func (e *env) counters() bps.Counters { e.t.Helper() - c, ok := e.broker.Counters(e.topic, e.admin) + c, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}) if !ok { e.t.Fatal("no live cohort") } @@ -195,7 +195,7 @@ func TestBroadcast(t *testing.T) { t.Fatal("want a distinct 32-byte challenge per stream") } - // AUTH claims and is never delivered + // AUTH authenticates and is never delivered e.publish(pub, bps.KindAuth, 0, nil) c0 := e.publish(pub, bps.KindData, 0, []byte("hello cohort")) expectMessage(t, sub, 0, c0) @@ -232,7 +232,7 @@ func TestPublisherReconnects(t *testing.T) { expectDone(t, pub1) // the cohort and its cursor persist; a valid frame below the cursor - // still claims the new stream and is counted as a retransmit + // still authenticates the new stream and is counted as a retransmit pub2, _ := e.publisher() if bytes.Equal(pub1.Challenge(), pub2.Challenge()) { t.Fatal("challenge reused") @@ -345,14 +345,14 @@ func (e *env) raw(join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { } func (e *env) spec() *pb.CohortSpec { - return &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC, Admin: e.admin} + return &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC, Principal: e.principal} } func TestDroppedFrames(t *testing.T) { t.Parallel() e := newEnv(t, bps.Options{}) sub := e.subscribe() - w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Addr: e.admin}) + w, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Identity: e.principal}) if ack.Status != pb.Status_OK { t.Fatalf("status %s", ack.Status) } @@ -384,11 +384,11 @@ func TestJoinRejected(t *testing.T) { name string join *pb.Join }{ - {"no cohort", &pb.Join{Addr: e.admin}}, - {"unspecified binding", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Admin: e.admin}, Addr: e.admin}}, - {"short topic", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic[:31], Binding: pb.TopicBinding_FEED_TOPIC, Admin: e.admin}, Addr: e.admin}}, - {"no admin", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC}, Addr: e.admin}}, - {"short addr", &pb.Join{Cohort: e.spec(), Addr: e.admin[:19]}}, + {"no cohort", &pb.Join{Identity: e.principal}}, + {"unspecified binding", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: e.principal}}, + {"short topic", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic[:31], Binding: pb.TopicBinding_FEED_TOPIC, Principal: e.principal}, Identity: e.principal}}, + {"no admin", &pb.Join{Cohort: &pb.CohortSpec{Topic: e.topic, Binding: pb.TopicBinding_FEED_TOPIC}, Identity: e.principal}}, + {"short addr", &pb.Join{Cohort: e.spec(), Identity: e.principal[:19]}}, } { _, _, ack := e.raw(tc.join) if ack.Status != pb.Status_REJECTED || len(ack.Challenge) != 0 { @@ -400,11 +400,11 @@ func TestJoinRejected(t *testing.T) { func TestBounds(t *testing.T) { t.Parallel() - t.Run("subscribers per cohort, admin admitted outside", func(t *testing.T) { + t.Run("subscribers per cohort, principal admitted outside", func(t *testing.T) { t.Parallel() e := newEnv(t, bps.Options{MaxSubscribers: 1}) sub := e.subscribe() - if _, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Addr: make([]byte, 20)}); ack.Status != pb.Status_FULL { + if _, _, ack := e.raw(&pb.Join{Cohort: e.spec(), Identity: make([]byte, 20)}); ack.Status != pb.Status_FULL { t.Fatalf("got %s, want FULL", ack.Status) } pub, _ := e.publisher() @@ -417,7 +417,7 @@ func TestBounds(t *testing.T) { e.subscribe() spec := e.spec() spec.Topic = make([]byte, 32) - if _, _, ack := e.raw(&pb.Join{Cohort: spec, Addr: e.admin}); ack.Status != pb.Status_FULL { + if _, _, ack := e.raw(&pb.Join{Cohort: spec, Identity: e.principal}); ack.Status != pb.Status_FULL { t.Fatalf("got %s, want FULL", ack.Status) } }) @@ -426,37 +426,37 @@ func TestBounds(t *testing.T) { t.Parallel() e := newEnv(t, bps.Options{MaxStreamsPerPeerCohort: 1, MaxCohortsPerPeer: 1}) c, _ := e.client(swarm.RandAddress(t)) - e.join(c, e.admin) - if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Topic: e.topic, Admin: e.admin, Addr: e.admin}); err == nil { + e.join(c, e.principal) + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: e.topic, Principal: e.principal}, Identity: e.principal}); err == nil { t.Fatal("want a second stream on the cohort refused") } other := make([]byte, 32) - if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Topic: other, Admin: e.admin, Addr: e.admin}); err == nil { + if _, err := c.Join(context.Background(), bps.JoinRequest{Broker: e.brokerAddr, Spec: bps.CohortSpec{Topic: other, Principal: e.principal}, Identity: e.principal}); err == nil { t.Fatal("want a second cohort refused") } }) - t.Run("claim deadline", func(t *testing.T) { + t.Run("auth timeout", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{ClaimDeadline: 50 * time.Millisecond}) + e := newEnv(t, bps.Options{AuthTimeout: 50 * time.Millisecond}) e.subscribe() pub, overlay := e.publisher() expectDone(t, pub) eventually(t, func() bool { return e.bl.blocklisted(overlay) }) - if got := e.counters().ClaimTimeout; got != 1 { - t.Fatalf("claim_timeout %d, want 1", got) + if got := e.counters().AuthTimeout; got != 1 { + t.Fatalf("auth_timeout %d, want 1", got) } }) t.Run("inactivity deadline", func(t *testing.T) { t.Parallel() - e := newEnv(t, bps.Options{InactivityDeadline: 100 * time.Millisecond}) + e := newEnv(t, bps.Options{InactivityTimeout: 100 * time.Millisecond}) sub := e.subscribe() pub, _ := e.publisher() e.publish(pub, bps.KindAuth, 0, nil) expectDone(t, sub) expectDone(t, pub) - if _, ok := e.broker.Counters(e.topic, e.admin); ok { + if _, ok := e.broker.Counters(bps.CohortSpec{Topic: e.topic, Principal: e.principal}); ok { t.Fatal("cohort not reclaimed") } }) @@ -469,16 +469,16 @@ func TestVerify(t *testing.T) { copy(challenge, "challenge") c := e.chunk(e.signer, bps.KindData, challenge, 9, []byte("x")) - if err := bps.Verify(bps.KindData, challenge, 9, c, e.topic, e.admin); err != nil { + if err := bps.Verify(bps.KindData, challenge, 9, c, e.topic, e.principal); err != nil { t.Fatal(err) } other := make([]byte, 20) for _, err := range []error{ bps.Verify(bps.KindData, challenge, 9, c, e.topic, other), - bps.Verify(bps.KindData, challenge[:31], 9, c, e.topic, e.admin), - bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.admin), - bps.Verify(bps.KindData, challenge, 9, c, make([]byte, 32), e.admin), - bps.Verify(bps.KindData, challenge, 9, c[:10], e.topic, e.admin), + bps.Verify(bps.KindData, challenge[:31], 9, c, e.topic, e.principal), + bps.Verify(bps.KindAuth, challenge, 9, c, e.topic, e.principal), + bps.Verify(bps.KindData, challenge, 9, c, make([]byte, 32), e.principal), + bps.Verify(bps.KindData, challenge, 9, c[:10], e.topic, e.principal), } { if !errors.Is(err, bps.ErrInvalidSOC) { t.Fatalf("got %v, want invalid soc", err) diff --git a/pkg/bps/id.go b/pkg/bps/id.go index b5a2cbdaf6b..001a373779e 100644 --- a/pkg/bps/id.go +++ b/pkg/bps/id.go @@ -23,7 +23,7 @@ const ChallengeSize = 32 var servicePrefix = []byte("bps-service:v1") // errInvalidSOC is returned for a chunk that does not have the id derived -// from its frame, or does not validate as the admin's single-owner chunk. +// from its frame, or does not validate as the principal's single-owner chunk. var errInvalidSOC = errors.New("bps: invalid soc") // Kind is what a chunk on the wire is. @@ -32,7 +32,7 @@ type Kind int const ( KindUnspecified Kind = iota KindData // an update of the stream's session feed - KindAuth // an empty chunk that claims the stream + KindAuth // an empty chunk that authenticates the stream ) func (k Kind) proto() pb.Kind { @@ -76,13 +76,13 @@ func id(kind pb.Kind, topic, challenge []byte, index uint64) ([]byte, error) { } // Verify checks that the frame fields kind, challenge and index give the id of -// the chunk, and that the chunk validates as admin's single-owner chunk at -// keccak256(id | admin). An AUTH chunk must moreover be empty. -func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, admin []byte) error { - return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, admin) +// the chunk, and that the chunk validates as the principal's single-owner chunk +// at keccak256(id | principal). An AUTH chunk must moreover be empty. +func Verify(kind Kind, challenge []byte, index uint64, chunk, topic, principal []byte) error { + return verify(&pb.Broadcast{Soc: chunk, Kind: kind.proto(), Challenge: challenge, Index: index}, topic, principal) } -func verify(f *pb.Broadcast, topic, admin []byte) error { +func verify(f *pb.Broadcast, topic, principal []byte) error { if len(f.Challenge) != ChallengeSize { return fmt.Errorf("challenge length %d: %w", len(f.Challenge), errInvalidSOC) } @@ -93,12 +93,12 @@ func verify(f *pb.Broadcast, topic, admin []byte) error { if len(f.Soc) < swarm.HashSize || !bytes.Equal(f.Soc[:swarm.HashSize], want) { return fmt.Errorf("id not derived from frame: %w", errInvalidSOC) } - addr, err := soc.CreateAddress(want, admin) + addr, err := soc.CreateAddress(want, principal) if err != nil { return err } if !soc.Valid(swarm.NewChunk(addr, f.Soc)) { - return fmt.Errorf("not the admin's soc: %w", errInvalidSOC) + return fmt.Errorf("not the principal's soc: %w", errInvalidSOC) } if f.Kind == pb.Kind_AUTH { // span 0, no payload diff --git a/pkg/bps/metrics.go b/pkg/bps/metrics.go index a6497338433..0473653e6dd 100644 --- a/pkg/bps/metrics.go +++ b/pkg/bps/metrics.go @@ -18,7 +18,7 @@ type Counters struct { UnknownKind uint64 // a frame of a kind BPS-lite does not define WrongChallenge uint64 // a frame whose challenge is not its stream's InvalidSOC uint64 // a chunk that does not validate at the derived id - ClaimTimeout uint64 // a pending stream disconnected at the claim deadline + AuthTimeout uint64 // a pending stream disconnected at the auth timeout Retransmit uint64 // a DATA frame below the cursor QueueReset uint64 // a subscriber stream reset for a full queue } @@ -28,7 +28,7 @@ type counters struct { unknownKind atomic.Uint64 wrongChallenge atomic.Uint64 invalidSOC atomic.Uint64 - claimTimeout atomic.Uint64 + authTimeout atomic.Uint64 retransmit atomic.Uint64 queueReset atomic.Uint64 } @@ -39,7 +39,7 @@ func (c *counters) snapshot() Counters { UnknownKind: c.unknownKind.Load(), WrongChallenge: c.wrongChallenge.Load(), InvalidSOC: c.invalidSOC.Load(), - ClaimTimeout: c.claimTimeout.Load(), + AuthTimeout: c.authTimeout.Load(), Retransmit: c.retransmit.Load(), QueueReset: c.queueReset.Load(), } diff --git a/pkg/bps/pb/bps.pb.go b/pkg/bps/pb/bps.pb.go index 416d48cd973..9218a354c1c 100644 --- a/pkg/bps/pb/bps.pb.go +++ b/pkg/bps/pb/bps.pb.go @@ -110,9 +110,9 @@ func (Kind) EnumDescriptor() ([]byte, []int) { // The cohort's identity, keyed by its canonical serialisation. type CohortSpec struct { - Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` - Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` - Admin []byte `protobuf:"bytes,3,opt,name=admin,proto3" json:"admin,omitempty"` + Topic []byte `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Binding TopicBinding `protobuf:"varint,2,opt,name=binding,proto3,enum=bps.TopicBinding" json:"binding,omitempty"` + Principal []byte `protobuf:"bytes,3,opt,name=principal,proto3" json:"principal,omitempty"` } func (m *CohortSpec) Reset() { *m = CohortSpec{} } @@ -162,17 +162,17 @@ func (m *CohortSpec) GetBinding() TopicBinding { return TopicBinding_TOPIC_BINDING_UNSPECIFIED } -func (m *CohortSpec) GetAdmin() []byte { +func (m *CohortSpec) GetPrincipal() []byte { if m != nil { - return m.Admin + return m.Principal } return nil } // Peer -> broker: the first frame on a fresh stream. type Join struct { - Cohort *CohortSpec `protobuf:"bytes,1,opt,name=cohort,proto3" json:"cohort,omitempty"` - Addr []byte `protobuf:"bytes,2,opt,name=addr,proto3" json:"addr,omitempty"` + Cohort *CohortSpec `protobuf:"bytes,1,opt,name=cohort,proto3" json:"cohort,omitempty"` + Identity []byte `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"` } func (m *Join) Reset() { *m = Join{} } @@ -215,9 +215,9 @@ func (m *Join) GetCohort() *CohortSpec { return nil } -func (m *Join) GetAddr() []byte { +func (m *Join) GetIdentity() []byte { if m != nil { - return m.Addr + return m.Identity } return nil } @@ -358,33 +358,34 @@ func init() { func init() { proto.RegisterFile("bps.proto", fileDescriptor_786299925f8760d5) } var fileDescriptor_786299925f8760d5 = []byte{ - // 416 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xc1, 0x6a, 0xdb, 0x40, - 0x10, 0x86, 0x2d, 0x69, 0xab, 0xda, 0x13, 0xe3, 0x6e, 0x87, 0x50, 0x5c, 0x48, 0x84, 0x71, 0x0f, - 0x35, 0x2e, 0x84, 0xe2, 0x9e, 0x0b, 0x95, 0x25, 0xb9, 0x51, 0x1c, 0x9c, 0xb0, 0x92, 0x2f, 0xbd, - 0x18, 0x49, 0x2b, 0x92, 0x25, 0xe9, 0xae, 0xb0, 0x54, 0xe8, 0x63, 0xf4, 0xb1, 0x7a, 0xcc, 0xb1, - 0xc7, 0x62, 0xbf, 0x48, 0xd1, 0x4a, 0x21, 0x8d, 0x6f, 0x33, 0xf3, 0xcf, 0x7c, 0xb3, 0xff, 0xb0, - 0xd0, 0x4b, 0x8b, 0xf2, 0xac, 0xd8, 0xaa, 0x4a, 0xa1, 0x95, 0x16, 0xe5, 0x38, 0x07, 0xf0, 0xd4, - 0xad, 0xda, 0x56, 0x51, 0x91, 0x67, 0x78, 0x0c, 0x2f, 0x2a, 0x55, 0x88, 0x6c, 0x68, 0x8c, 0x8c, - 0x49, 0x9f, 0x35, 0x09, 0x7e, 0x80, 0x97, 0xa9, 0x90, 0x5c, 0xc8, 0x9b, 0xa1, 0x39, 0x32, 0x26, - 0x83, 0xd9, 0xeb, 0xb3, 0x9a, 0x12, 0xd7, 0xe2, 0xbc, 0x11, 0xd8, 0x63, 0x47, 0x8d, 0x48, 0xf8, - 0x77, 0x21, 0x87, 0x56, 0x83, 0xd0, 0xc9, 0xd8, 0x03, 0x72, 0xa1, 0x84, 0xc4, 0xf7, 0x60, 0x67, - 0x7a, 0x9d, 0xde, 0x70, 0x34, 0x7b, 0xa5, 0x49, 0x4f, 0x2f, 0x60, 0xad, 0x8c, 0x08, 0x24, 0xe1, - 0x7c, 0xab, 0x17, 0xf6, 0x99, 0x8e, 0xc7, 0xe7, 0x60, 0xb9, 0xd9, 0x1d, 0xbe, 0x03, 0xbb, 0xac, - 0x92, 0xea, 0x47, 0xa9, 0x19, 0x83, 0xd9, 0x91, 0x66, 0x44, 0xba, 0xc4, 0x5a, 0x09, 0x4f, 0xa0, - 0x97, 0xdd, 0x26, 0xf7, 0xf7, 0xb9, 0xbc, 0xc9, 0x5b, 0xc8, 0x53, 0x61, 0x5c, 0x40, 0x6f, 0xbe, - 0x55, 0x09, 0xcf, 0x92, 0xb2, 0x42, 0x0a, 0x56, 0xa9, 0x1e, 0x2d, 0xd7, 0x21, 0x9e, 0x02, 0xb9, - 0x13, 0x92, 0xb7, 0x6e, 0x7b, 0x9a, 0xbf, 0x14, 0x92, 0x33, 0x5d, 0x7e, 0xce, 0xb6, 0x0e, 0xd8, - 0xf5, 0x01, 0x84, 0xe4, 0xf9, 0xcf, 0x21, 0x19, 0x19, 0x13, 0xc2, 0x9a, 0x64, 0xfa, 0x19, 0xfa, - 0xff, 0xdf, 0x0b, 0x4f, 0xe1, 0x6d, 0x7c, 0x75, 0x1d, 0x7a, 0x9b, 0x79, 0xb8, 0xf2, 0xc3, 0xd5, - 0xd7, 0xcd, 0x7a, 0x15, 0x5d, 0x07, 0x5e, 0xb8, 0x08, 0x03, 0x9f, 0x76, 0x70, 0x00, 0xb0, 0x08, - 0x02, 0x7f, 0xa3, 0x7b, 0x28, 0x99, 0x7e, 0x01, 0xbb, 0x31, 0x88, 0x6f, 0x00, 0xa3, 0xd8, 0x8d, - 0xd7, 0xd1, 0xc1, 0x84, 0x0d, 0xe6, 0xd5, 0x92, 0x1a, 0xd8, 0x05, 0xb2, 0x58, 0x5f, 0x5e, 0x52, - 0x13, 0xfb, 0xd0, 0x65, 0xc1, 0x45, 0xe0, 0xc5, 0x81, 0x4f, 0xad, 0xe9, 0x47, 0x20, 0xb5, 0x05, - 0x3c, 0x06, 0xba, 0x0c, 0x57, 0xfe, 0xc1, 0x74, 0x17, 0x88, 0xef, 0xc6, 0x6e, 0x33, 0xef, 0xae, - 0xe3, 0x73, 0x6a, 0xce, 0x4f, 0x7e, 0xef, 0x1c, 0xe3, 0x61, 0xe7, 0x18, 0x7f, 0x77, 0x8e, 0xf1, - 0x6b, 0xef, 0x74, 0x1e, 0xf6, 0x4e, 0xe7, 0xcf, 0xde, 0xe9, 0x7c, 0x33, 0x8b, 0x34, 0xb5, 0xf5, - 0x27, 0xfa, 0xf4, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xc2, 0xe3, 0x2e, 0x00, 0x51, 0x02, 0x00, 0x00, + // 424 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x92, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0xc7, 0xe3, 0x0f, 0x4c, 0x32, 0x8d, 0xc2, 0x32, 0xaa, 0x50, 0x40, 0xad, 0x15, 0x85, 0x03, + 0x51, 0x90, 0x2a, 0x14, 0xce, 0x48, 0x38, 0xb1, 0x43, 0xdd, 0x54, 0x69, 0xb5, 0x76, 0x2e, 0x5c, + 0x22, 0x7f, 0xa9, 0x5d, 0x35, 0xec, 0xae, 0xe2, 0x45, 0x82, 0xb7, 0xe0, 0xb1, 0x38, 0xf6, 0xc8, + 0x11, 0x25, 0x2f, 0x82, 0xbc, 0x4e, 0x08, 0xe4, 0xb6, 0xf3, 0xff, 0xcf, 0xfc, 0xe6, 0x43, 0x0b, + 0xad, 0x54, 0x96, 0x17, 0x72, 0x2d, 0x94, 0x40, 0x2b, 0x95, 0x65, 0xff, 0x0b, 0xc0, 0x44, 0xdc, + 0x8b, 0xb5, 0x8a, 0x64, 0x91, 0xe1, 0x29, 0x3c, 0x51, 0x42, 0xb2, 0xac, 0x6b, 0xf4, 0x8c, 0x41, + 0x9b, 0xd6, 0x01, 0xbe, 0x85, 0xa7, 0x29, 0xe3, 0x39, 0xe3, 0x77, 0x5d, 0xb3, 0x67, 0x0c, 0x3a, + 0xa3, 0xe7, 0x17, 0x15, 0x25, 0xae, 0xcc, 0x71, 0x6d, 0xd0, 0x7d, 0x06, 0x9e, 0x41, 0x4b, 0xae, + 0x19, 0xcf, 0x98, 0x4c, 0x56, 0x5d, 0x4b, 0x63, 0x0e, 0x42, 0x7f, 0x06, 0xf6, 0x95, 0x60, 0x1c, + 0xdf, 0x80, 0x93, 0xe9, 0xb6, 0xba, 0xd3, 0xc9, 0xe8, 0x99, 0x26, 0x1e, 0x26, 0xa1, 0x3b, 0x1b, + 0x5f, 0x41, 0x93, 0xe5, 0x05, 0x57, 0x4c, 0x7d, 0xd7, 0xcd, 0xdb, 0xf4, 0x6f, 0xdc, 0xbf, 0x04, + 0xcb, 0xcb, 0x1e, 0xf0, 0x35, 0x38, 0xa5, 0x4a, 0xd4, 0xd7, 0x52, 0xb3, 0x3a, 0xa3, 0x13, 0xcd, + 0x8a, 0xb4, 0x44, 0x77, 0x56, 0x35, 0x56, 0x76, 0x9f, 0xac, 0x56, 0x05, 0xbf, 0x2b, 0x76, 0xa0, + 0x83, 0xd0, 0x97, 0xd0, 0x1a, 0xaf, 0x45, 0x92, 0x67, 0x49, 0xa9, 0x90, 0x80, 0x55, 0x8a, 0xfd, + 0x09, 0xaa, 0x27, 0x9e, 0x83, 0xfd, 0xc0, 0x78, 0xbe, 0xdb, 0xbe, 0xa5, 0xf9, 0x33, 0xc6, 0x73, + 0xaa, 0xe5, 0xff, 0xd9, 0xd6, 0x11, 0xbb, 0xba, 0x29, 0xe3, 0x79, 0xf1, 0xad, 0x6b, 0xf7, 0x8c, + 0x81, 0x4d, 0xeb, 0x60, 0xf8, 0x01, 0xda, 0xff, 0xde, 0x0f, 0xcf, 0xe1, 0x65, 0x7c, 0x73, 0x1b, + 0x4e, 0x96, 0xe3, 0x70, 0xee, 0x87, 0xf3, 0x4f, 0xcb, 0xc5, 0x3c, 0xba, 0x0d, 0x26, 0xe1, 0x34, + 0x0c, 0x7c, 0xd2, 0xc0, 0x0e, 0xc0, 0x34, 0x08, 0xfc, 0xa5, 0xce, 0x21, 0xf6, 0xf0, 0x23, 0x38, + 0xf5, 0x82, 0xf8, 0x02, 0x30, 0x8a, 0xbd, 0x78, 0x11, 0x1d, 0x55, 0x38, 0x60, 0xde, 0xcc, 0x88, + 0x81, 0x4d, 0xb0, 0xa7, 0x8b, 0xeb, 0x6b, 0x62, 0x62, 0x1b, 0x9a, 0x34, 0xb8, 0x0a, 0x26, 0x71, + 0xe0, 0x13, 0x6b, 0xf8, 0x0e, 0xec, 0x6a, 0x05, 0x3c, 0x05, 0x32, 0x0b, 0xe7, 0xfe, 0x51, 0x75, + 0x13, 0x6c, 0xdf, 0x8b, 0xbd, 0xba, 0xde, 0x5b, 0xc4, 0x97, 0xc4, 0x1c, 0x9f, 0xfd, 0xdc, 0xb8, + 0xc6, 0xe3, 0xc6, 0x35, 0x7e, 0x6f, 0x5c, 0xe3, 0xc7, 0xd6, 0x6d, 0x3c, 0x6e, 0xdd, 0xc6, 0xaf, + 0xad, 0xdb, 0xf8, 0x6c, 0xca, 0x34, 0x75, 0xf4, 0xa7, 0x7a, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, + 0x30, 0x35, 0xa6, 0xe5, 0x61, 0x02, 0x00, 0x00, } func (m *CohortSpec) Marshal() (dAtA []byte, err error) { @@ -407,10 +408,10 @@ func (m *CohortSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Admin) > 0 { - i -= len(m.Admin) - copy(dAtA[i:], m.Admin) - i = encodeVarintBps(dAtA, i, uint64(len(m.Admin))) + if len(m.Principal) > 0 { + i -= len(m.Principal) + copy(dAtA[i:], m.Principal) + i = encodeVarintBps(dAtA, i, uint64(len(m.Principal))) i-- dAtA[i] = 0x1a } @@ -449,10 +450,10 @@ func (m *Join) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Addr) > 0 { - i -= len(m.Addr) - copy(dAtA[i:], m.Addr) - i = encodeVarintBps(dAtA, i, uint64(len(m.Addr))) + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = encodeVarintBps(dAtA, i, uint64(len(m.Identity))) i-- dAtA[i] = 0x12 } @@ -577,7 +578,7 @@ func (m *CohortSpec) Size() (n int) { if m.Binding != 0 { n += 1 + sovBps(uint64(m.Binding)) } - l = len(m.Admin) + l = len(m.Principal) if l > 0 { n += 1 + l + sovBps(uint64(l)) } @@ -594,7 +595,7 @@ func (m *Join) Size() (n int) { l = m.Cohort.Size() n += 1 + l + sovBps(uint64(l)) } - l = len(m.Addr) + l = len(m.Identity) if l > 0 { n += 1 + l + sovBps(uint64(l)) } @@ -730,7 +731,7 @@ func (m *CohortSpec) Unmarshal(dAtA []byte) error { } case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Admin", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Principal", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -757,9 +758,9 @@ func (m *CohortSpec) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Admin = append(m.Admin[:0], dAtA[iNdEx:postIndex]...) - if m.Admin == nil { - m.Admin = []byte{} + m.Principal = append(m.Principal[:0], dAtA[iNdEx:postIndex]...) + if m.Principal == nil { + m.Principal = []byte{} } iNdEx = postIndex default: @@ -853,7 +854,7 @@ func (m *Join) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Addr", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -880,9 +881,9 @@ func (m *Join) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Addr = append(m.Addr[:0], dAtA[iNdEx:postIndex]...) - if m.Addr == nil { - m.Addr = []byte{} + m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) + if m.Identity == nil { + m.Identity = []byte{} } iNdEx = postIndex default: diff --git a/pkg/bps/pb/bps.proto b/pkg/bps/pb/bps.proto index f2a9250efce..a61dd2e2eab 100644 --- a/pkg/bps/pb/bps.proto +++ b/pkg/bps/pb/bps.proto @@ -20,13 +20,13 @@ enum TopicBinding { message CohortSpec { bytes topic = 1; // 32 bytes TopicBinding binding = 2; // FEED_TOPIC - bytes admin = 3; // 20 bytes, required + bytes principal = 3; // 20 bytes, required: the publisher } // Peer -> broker: the first frame on a fresh stream. message Join { CohortSpec cohort = 1; - bytes addr = 2; // 20 bytes, required: the stream's identity + bytes identity = 2; // 20 bytes, required: the stream's identity } enum Status { @@ -46,7 +46,7 @@ message Ack { enum Kind { KIND_UNSPECIFIED = 0; // invalid on the wire DATA = 1; // an update of the stream's session feed - AUTH = 2; // an empty chunk: claims the stream, never delivered + AUTH = 2; // an empty chunk: authenticates the stream, never delivered } // Every frame after the handshake. soc is an ordinary single-owner chunk From 6dca0e5cb2726329ed0717915784f5dc842d9ca5 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:16:47 -0600 Subject: [PATCH 3/4] chore: pr review --- pkg/bps/bps.go | 95 ++++++++++++++++++++++++------------------ pkg/bps/bps_test.go | 75 ++++++++++++++++++++++++++++++--- pkg/bps/export_test.go | 6 ++- 3 files changed, 129 insertions(+), 47 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index f7ff4616700..f65242739f4 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -50,6 +50,8 @@ var ( errWrongStream = errors.New("bps: frame from subscriber stream") errCohortReclaimed = errors.New("bps: cohort reclaimed") errFull = errors.New("bps: capacity bound") + errNotPublisher = errors.New("bps: session is not a publisher") + errInvalidDelivery = errors.New("bps: invalid delivery") ) // Options are the broker's resource bounds. Zero values take the defaults. @@ -193,7 +195,8 @@ type Session interface { // Cursor is the lowest DATA index the session will deliver next. Cursor() uint64 // Publish writes a chunk of the given kind and index, signed under the - // session challenge, to the broker. The broker does not reply. + // session challenge, to the broker. The broker does not reply. Only a + // session whose identity is the cohort principal may publish. Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error // Done is closed when the p2p stream ends. Done() <-chan struct{} @@ -259,51 +262,57 @@ func (s *Service) Join(ctx context.Context, req JoinRequest) (Session, error) { cancel: cancel, w: w, challenge: ack.Challenge, + publisher: bytes.Equal(req.Identity, req.Spec.Principal), rx: make(chan Message), done: make(chan struct{}), } sess.cursor.Store(req.Cursor) - go func() { - var readErr error - defer func() { - sess.finish(readErr) - if readErr != nil && ctx.Err() == nil { - _ = stream.Reset() - } else { - _ = stream.FullClose() - } - }() - go func() { - <-ctx.Done() - _ = stream.Reset() - }() + go sess.readLoop(stream, r, req.Spec) - for { - var f pb.Broadcast - if err := r.ReadMsgWithContext(ctx, &f); err != nil { - readErr = err - return - } - // re-verify end to end: drop what is below the cursor or does - // not validate against the spec; never compare the challenge - if f.Kind != pb.Kind_DATA || f.Index < sess.cursor.Load() { - continue - } - if err := verify(&f, req.Spec.Topic, req.Spec.Principal); err != nil { - s.logger.Debug("dropping delivery", "error", err) - continue - } - sess.cursor.Store(f.Index + 1) - select { - case sess.rx <- Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index}: - case <-ctx.Done(): - return - } + return sess, nil +} + +// readLoop delivers the broker's frames until the stream ends. +func (s *session) readLoop(stream p2p.Stream, r protobuf.Reader, spec CohortSpec) { + var readErr error + defer func() { + s.finish(readErr) + if readErr != nil && s.ctx.Err() == nil { + _ = stream.Reset() + } else { + _ = stream.FullClose() } }() + go func() { + <-s.ctx.Done() + _ = stream.Reset() + }() - return sess, nil + for { + var f pb.Broadcast + if err := r.ReadMsgWithContext(s.ctx, &f); err != nil { + readErr = err + return + } + // a retransmit below the cursor is skipped. The challenge is not + // compared: it is the publisher stream's, which the subscriber never sees. + if f.Kind != pb.Kind_DATA || f.Index < s.cursor.Load() { + continue + } + // re-verify end to end: a broker that forwards a chunk not valid + // for the spec is misbehaving, so the stream is ended + if err := verify(&f, spec.Topic, spec.Principal); err != nil { + readErr = fmt.Errorf("%w: %w", errInvalidDelivery, err) + return + } + s.cursor.Store(f.Index + 1) + select { + case s.rx <- Message{SOC: f.Soc, Challenge: f.Challenge, Index: f.Index}: + case <-s.ctx.Done(): + return + } + } } // session is the concrete Session implementation. @@ -313,6 +322,7 @@ type session struct { wmtx sync.Mutex w protobuf.Writer challenge []byte + publisher bool // the identity is the cohort principal cursor atomic.Uint64 rx chan Message done chan struct{} @@ -331,6 +341,9 @@ func (s *session) Close() error { } func (s *session) Publish(ctx context.Context, kind Kind, index uint64, soc []byte) error { + if !s.publisher { + return errNotPublisher + } select { case <-s.done: return errSessionClosed @@ -551,10 +564,10 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort pc := s.peers[peer.ByteString()] if pc[k] == 0 && len(pc) >= s.opts.MaxCohortsPerPeer { - return nil, nil, fmt.Errorf("cohorts per peer: %w", errFull) + return nil, nil, fmt.Errorf("too many cohorts per peer: %w", errFull) } if pc[k] >= s.opts.MaxStreamsPerPeerCohort { - return nil, nil, fmt.Errorf("streams per peer per cohort: %w", errFull) + return nil, nil, fmt.Errorf("too many streams per peer per cohort: %w", errFull) } role := roleSubscriber @@ -565,7 +578,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort co, ok := s.cohorts[k] if !ok { if len(s.cohorts) >= s.opts.MaxCohorts { - return nil, nil, fmt.Errorf("cohorts per broker: %w", errFull) + return nil, nil, fmt.Errorf("too many cohorts per broker: %w", errFull) } co = &cohort{ key: k, @@ -584,7 +597,7 @@ func (s *Service) join(peer swarm.Address, join *pb.Join, reset func()) (*cohort if len(co.members) == 0 { s.removeCohort(co) } - return nil, nil, fmt.Errorf("subscribers per cohort: %w", errFull) + return nil, nil, fmt.Errorf("too many subscribers per cohort: %w", errFull) } challenge := make([]byte, ChallengeSize) diff --git a/pkg/bps/bps_test.go b/pkg/bps/bps_test.go index c88ec7a2470..4fb14fc7686 100644 --- a/pkg/bps/bps_test.go +++ b/pkg/bps/bps_test.go @@ -269,15 +269,32 @@ func TestViolations(t *testing.T) { } otherSigner := crypto.NewDefaultSigner(other) + t.Run("publish on a subscriber session", func(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + sub := e.subscribe() + c := e.chunk(e.signer, bps.KindData, sub.Challenge(), 0, []byte("x")) + if err := sub.Publish(context.Background(), bps.KindData, 0, c); !errors.Is(err, bps.ErrNotPublisher) { + t.Fatalf("got %v, want %v", err, bps.ErrNotPublisher) + } + }) + t.Run("publication from a subscriber stream", func(t *testing.T) { t.Parallel() e := newEnv(t, bps.Options{}) overlay := swarm.RandAddress(t) - c, _ := e.client(overlay) - sub := e.join(c, swarm.RandAddress(t).Bytes()[:20]) keep := e.subscribe() - e.publish(sub, bps.KindData, 0, []byte("x")) - expectDone(t, sub) + w, r, ack := e.rawFrom(overlay, &pb.Join{Cohort: e.spec(), Identity: swarm.RandAddress(t).Bytes()[:20]}) + if ack.Status != pb.Status_OK { + t.Fatalf("status %s", ack.Status) + } + c := e.chunk(e.signer, bps.KindData, ack.Challenge, 0, []byte("x")) + if err := w.WriteMsg(&pb.Broadcast{Kind: pb.Kind_DATA, Challenge: ack.Challenge, Soc: c}); err != nil { + t.Fatal(err) + } + if err := r.ReadMsg(&pb.Broadcast{}); err == nil { + t.Fatal("stream not ended") + } eventually(t, func() bool { return e.bl.blocklisted(overlay) }) if got := e.counters().WrongStream; got != 1 { t.Fatalf("wrong_stream %d, want 1", got) @@ -327,7 +344,13 @@ func TestViolations(t *testing.T) { // raw opens a stream to the broker and joins with join, returning the Ack. func (e *env) raw(join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { e.t.Helper() - rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(swarm.RandAddress(e.t))) + return e.rawFrom(swarm.RandAddress(e.t), join) +} + +// rawFrom is raw from the given overlay. +func (e *env) rawFrom(overlay swarm.Address, join *pb.Join) (protobuf.Writer, protobuf.Reader, *pb.Ack) { + e.t.Helper() + rec := streamtest.New(streamtest.WithProtocols(e.broker.Protocol()), streamtest.WithBaseAddr(overlay)) stream, err := rec.NewStream(context.Background(), e.brokerAddr, nil, bps.ProtocolName, bps.ProtocolVersion, bps.StreamName) if err != nil { e.t.Fatal(err) @@ -377,6 +400,48 @@ func TestDroppedFrames(t *testing.T) { } } +// TestInvalidDelivery checks that a subscriber ends the session when the +// broker forwards a chunk that does not verify against the cohort spec. +func TestInvalidDelivery(t *testing.T) { + t.Parallel() + e := newEnv(t, bps.Options{}) + other, err := crypto.GenerateSecp256k1Key() + if err != nil { + t.Fatal(err) + } + challenge := make([]byte, bps.ChallengeSize) + bad := e.chunk(crypto.NewDefaultSigner(other), bps.KindData, challenge, 0, []byte("x")) + broker := p2p.ProtocolSpec{ + Name: bps.ProtocolName, + Version: bps.ProtocolVersion, + StreamSpecs: []p2p.StreamSpec{{ + Name: bps.StreamName, + Handler: func(ctx context.Context, _ p2p.Peer, stream p2p.Stream) error { + w, r := protobuf.NewWriterAndReader(stream) + if err := r.ReadMsgWithContext(ctx, &pb.Join{}); err != nil { + return err + } + if err := w.WriteMsgWithContext(ctx, &pb.Ack{Status: pb.Status_OK, Challenge: challenge}); err != nil { + return err + } + if err := w.WriteMsgWithContext(ctx, &pb.Broadcast{Kind: pb.Kind_DATA, Challenge: challenge, Soc: bad}); err != nil { + return err + } + // hold the stream until the subscriber ends it + _ = r.ReadMsgWithContext(ctx, &pb.Broadcast{}) + return nil + }, + }}, + } + rec := streamtest.New(streamtest.WithProtocols(broker), streamtest.WithBaseAddr(swarm.RandAddress(t))) + c := bps.New(rec, nil, false, log.Noop, bps.Options{}) + sub := e.join(c, swarm.RandAddress(t).Bytes()[:20]) + expectDone(t, sub) + if err := sub.Err(); !errors.Is(err, bps.ErrInvalidDelivery) { + t.Fatalf("got %v, want %v", err, bps.ErrInvalidDelivery) + } +} + func TestJoinRejected(t *testing.T) { t.Parallel() e := newEnv(t, bps.Options{}) diff --git a/pkg/bps/export_test.go b/pkg/bps/export_test.go index 24c353ff0b8..58f68d2fa7e 100644 --- a/pkg/bps/export_test.go +++ b/pkg/bps/export_test.go @@ -10,4 +10,8 @@ const ( StreamName = streamName ) -var ErrInvalidSOC = errInvalidSOC +var ( + ErrInvalidSOC = errInvalidSOC + ErrNotPublisher = errNotPublisher + ErrInvalidDelivery = errInvalidDelivery +) From 2c2a2b735d01dd6c46e4b3f7f37aede0293cdc93 Mon Sep 17 00:00:00 2001 From: acud <12988138+acud@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:42:10 -0600 Subject: [PATCH 4/4] chore: pr changes --- pkg/bps/bps.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/bps/bps.go b/pkg/bps/bps.go index f65242739f4..bbf45feb72b 100644 --- a/pkg/bps/bps.go +++ b/pkg/bps/bps.go @@ -64,7 +64,7 @@ type Options struct { AuthTimeout time.Duration // disconnects a pending stream that has not authenticated QueueSize int // outbound frames per subscriber stream ViolationBlocklist time.Duration // blocklist duration for a protocol violation - AuthTimeoutBlocklist time.Duration // blocklist duration for an auth timeout + BlocklistDuration time.Duration // blocklist duration for an auth timeout } // DefaultOptions are the SWIP-74 recommended bounds. @@ -77,7 +77,7 @@ var DefaultOptions = Options{ AuthTimeout: 30 * time.Second, QueueSize: 64, ViolationBlocklist: 10 * time.Minute, - AuthTimeoutBlocklist: time.Minute, + BlocklistDuration: time.Minute, } func (o Options) withDefaults() Options { @@ -106,8 +106,8 @@ func (o Options) withDefaults() Options { if o.ViolationBlocklist > 0 { d.ViolationBlocklist = o.ViolationBlocklist } - if o.AuthTimeoutBlocklist > 0 { - d.AuthTimeoutBlocklist = o.AuthTimeoutBlocklist + if o.BlocklistDuration > 0 { + d.BlocklistDuration = o.BlocklistDuration } return d } @@ -702,7 +702,7 @@ func (s *Service) authTimeout(peer swarm.Address, co *cohort, m *member) { } s.count(&co.counters.authTimeout, "auth_timeout") if s.blocklister != nil { - if err := s.blocklister.Blocklist(peer, s.opts.AuthTimeoutBlocklist, "bps auth timeout"); err != nil { + if err := s.blocklister.Blocklist(peer, s.opts.BlocklistDuration, "bps auth timeout"); err != nil { s.logger.Debug("blocklist failed", "peer_address", peer, "error", err) } }