diff --git a/config.go b/config.go index 332ff2d4..99b59b50 100644 --- a/config.go +++ b/config.go @@ -13,7 +13,7 @@ type NodeAddress = conn.NodeAddress // WithNodes returns a NodeSource containing the provided mapping from // application-specific IDs to types implementing NodeAddress. // Node IDs must be greater than 0. -func WithNodes[T NodeAddress](nodes map[uint32]T) NodeSource { +func WithNodes[T NodeAddress](nodes map[ID]T) NodeSource { return conn.WithNodes(nodes) } @@ -25,6 +25,15 @@ func WithNodeList(addrsList []string) NodeSource { return conn.WithNodeList(addrsList) } +// ID identifies a node. The application chooses the IDs of configured nodes +// with [WithNodes], or [WithNodeList] assigns them in list order. Servers +// configured with [WithPeers] must agree on the ID of each peer, because a +// server announces its own ID when it connects. ID 0 is reserved: +// it marks a handler-only server and a client that announces no ID. A server +// assigns IDs from 2^20 upward to back-channel clients, skipping configured +// IDs. Under [WithStreamDedup], the peer with the lower ID dials the other. +type ID = conn.ID + // Node encapsulates the state of a node on which a remote procedure call can be // performed. Nodes are created as part of a [Config] built with [NewConfig]. type Node = conn.Node diff --git a/doc/dev-guide.md b/doc/dev-guide.md index 7761569b..ce3e33f1 100644 --- a/doc/dev-guide.md +++ b/doc/dev-guide.md @@ -186,7 +186,7 @@ classDiagram } class RequestHandler { <> - HandleRequest(ctx, msg, release, send) + HandleRequest(ctx, senderID, msg, release, send) } class BidiStream { <> @@ -198,7 +198,7 @@ classDiagram AcceptPeer(ctx, stream) InboundChannel } class Transport { - id uint32 + id ID shared bool Enqueue(Request) StoreChannel(Channel) @@ -258,12 +258,12 @@ classDiagram direction LR class Config { <> - NodeIDs() []uint32 + NodeIDs() []ID Size() int Context(parent) ConfigContext } class Node { - id uint32 + id ID addr string Context(parent) NodeContext IsShared() bool @@ -445,7 +445,7 @@ sequenceDiagram The root `gorums.Server` ties the layers together on the server side. It registers a `stream.Server` with its gRPC server, uses its `conn.InboundManager` as the server's `stream.PeerAcceptor`, and serves as the `stream.RequestHandler` for inbound streams, for its local node, and for back-channel requests on its outbound peer configuration. -`Server.HandleRequest` creates a `ServerContext` per request and runs the registered `Handler`, wrapped by any `ServerInterceptor`s. +`Server.HandleRequest` creates a `ServerContext` per request, records the sender ID that the channel passes in, and runs the registered `Handler`, wrapped by any `ServerInterceptor`s. A `gorums.Message` wraps a `stream.Message` together with its decoded proto payload. ```mermaid @@ -454,7 +454,7 @@ classDiagram class Server { handlers map[string]Handler RegisterHandler(method, Handler) - HandleRequest(ctx, msg, release, send) + HandleRequest(ctx, senderID, msg, release, send) ConnectedPeers() Config ConnectedClients() Config } @@ -519,7 +519,7 @@ sequenceDiagram SS->>IC: Serve(): receive loop P->>IC: request Message IC->>D: push handler job - D->>H: HandleRequest(ctx, msg, release, send) + D->>H: HandleRequest(ctx, senderID, msg, release, send) H->>H: Handler(ServerContext, Message) H->>IC: send(reply): push on sendQueue IC->>P: Send(reply) on send loop diff --git a/doc/user-guide.md b/doc/user-guide.md index 4358dae7..4511ae8d 100644 --- a/doc/user-guide.md +++ b/doc/user-guide.md @@ -650,31 +650,31 @@ func majorityWrite(responses *gorums.Responses[*WriteResponse]) (*WriteResponse, return nil, gorums.ErrIncomplete } - // Process the map[uint32]*WriteResponse + // Process the map[gorums.ID]*WriteResponse return aggregateWrites(replies), nil } // allSuccessful collects all successful responses; errored nodes are skipped. -func allSuccessful(responses *gorums.Responses[*Response]) map[uint32]*Response { +func allSuccessful(responses *gorums.Responses[*Response]) map[gorums.ID]*Response { return responses.Results().IgnoreErrors().CollectAll() } // allResponses collects all responses; errored nodes are included with a zero value. // Only use this when you know all nodes will succeed or you handle zero values. -func allResponses(responses *gorums.Responses[*Response]) map[uint32]*Response { +func allResponses(responses *gorums.Responses[*Response]) map[gorums.ID]*Response { return responses.Results().CollectAll() } ``` #### Per-Node Error Inspection -`CollectN` and `CollectAll` return `map[uint32]Resp` and do not preserve errors. +`CollectN` and `CollectAll` return `map[gorums.ID]Resp` and do not preserve errors. When you need the actual error from a specific node, range over the results sequence directly: ```go func inspectPerNodeErrors(responses *gorums.Responses[*Response]) { var errs []error - replies := make(map[uint32]*Response) + replies := make(map[gorums.ID]*Response) for result := range responses.Results() { if result.Err != nil { @@ -1543,6 +1543,117 @@ It returns no error, because closing a connection has no failure that a caller c A `Server` configured with `WithPeers` owns its peer configuration, and `Stop` and `GracefulStop` close it. +## Node IDs + +Gorums identifies each node with a `gorums.ID`. +The type is an alias for `uint32`, so a value from a protobuf `uint32` field needs no conversion. + +### Rules for Node IDs + +* ID 0 is reserved. + It marks a handler-only server and a client that announces no ID. +* With `WithNodes`, the application chooses the ID of each node. + With `WithNodeList`, Gorums assigns IDs in list order, starting after the highest ID in the configuration's connection pool, or from 1 for a new pool. +* Servers configured with `WithPeers` must agree on the ID of each peer, because a server announces its own ID when it connects. +* A server assigns IDs from 2^20 upward to back-channel clients, and it skips configured IDs. +* Under `WithStreamDedup`, the peer with the lower ID dials the other. + +### Choosing Node IDs + +Many protocols number their replicas from 1 to n, and use the number for leader rotation, slice indexes, or signature bitfields. +Use these numbers directly as Gorums IDs: + +```go +type replica struct { + addr string +} + +func (r replica) Addr() string { return r.addr } + +nodes := map[gorums.ID]replica{ + 1: {addr: "10.0.0.1:9000"}, + 2: {addr: "10.0.0.2:9000"}, + 3: {addr: "10.0.0.3:9000"}, +} +srv := gorums.NewServer( + gorums.WithPeers(myID, gorums.WithNodes(nodes), dialOpts), +) +``` + +Give each node an explicit ID in the cluster configuration, and let each node read its own ID from that configuration. +An explicit ID stays the same when the address list changes. +If you derive IDs from an address list instead, sort the list first, as the storage example does, so that all nodes compute the same IDs. + +### Mapping Application IDs to Node IDs + +An application may identify nodes by strings, such as host names or UUIDs. +Then give each node a Gorums ID in the shared cluster configuration, and keep a map in each direction: + +```go +type member struct { + Name string // application ID + ID gorums.ID // Gorums ID + Address string +} + +func (m member) Addr() string { return m.Address } + +byID := make(map[gorums.ID]member) +idOf := make(map[string]gorums.ID) +for _, m := range members { + byID[m.ID] = m + idOf[m.Name] = m.ID +} +cfg, closeFn, err := gorums.NewConfig(gorums.WithNodes(byID), dialOpts) +``` + +`cfg.Node(idOf[name])` returns the node for an application ID, or nil if the configuration has no such node. +`byID[id].Name` returns the application ID for a Gorums ID. + +Nodes that join without a shared configuration can derive their IDs from their names with a hash, so that all nodes compute the same ID without coordination: + +```go +func idOf(name string) gorums.ID { + h := fnv.New32a() + h.Write([]byte(name)) + return max(h.Sum32(), 1) // ID 0 is reserved +} +``` + +Two names can hash to the same 32-bit ID; the probability is about one in a million for 100 nodes. +Check for duplicate IDs when you build the node map. +Do not use hashed IDs in a protocol that needs IDs from 1 to n. + +### Finding the Sender of a Request + +A handler gets the ID of the node that sent a request from `ServerContext.SenderID`. +With `Config.Node`, it can then find the sender's node: + +```go +func (srv *storageSrv) WriteQC(ctx gorums.ServerContext, req *WriteRequest) (*WriteResponse, error) { + if sender := ctx.PeerConfig().Node(ctx.SenderID()); sender != nil { + log.Printf("write from peer %d at %s", sender.ID(), sender.Address()) + } + // ... +} +``` + +`SenderID` identifies the sender as this server knows it: + +* a peer configured with `WithPeers` has its configured ID; +* a back-channel client has the ID that this server assigned to it, as in `ConnectedClients()`; the ID changes when the client reconnects; +* a request on a connection that this server dialed has the dialed node's ID; +* a request that this server sends to itself has this server's own ID. + +`SenderID` returns 0 for a client that announces no ID, or an ID that this server does not know. + +The sender is the last hop. +When a message travels through several nodes, for example down a tree, `SenderID` identifies the node that forwarded it, not the node where the message started. +Put the origin in the message if the handler needs it. + +The sender asserts its ID when it connects, and the server does not authenticate it. +A Byzantine fault-tolerant protocol must still sign its messages and check the signer. + ## Latency-Based Node Selection Gorums tracks the round-trip latency to each node as an exponentially weighted diff --git a/examples/storage/server.go b/examples/storage/server.go index 17183d8e..d3010f33 100644 --- a/examples/storage/server.go +++ b/examples/storage/server.go @@ -74,7 +74,7 @@ func runLocalCluster(srvOpts gorums.ServerOption) ([]string, func(), error) { addrs := make([]string, len(servers)) for i, srv := range servers { addrs[i] = srv.Addr() - registerAndServe(srv, uint32(i+1)) + registerAndServe(srv, gorums.ID(i+1)) } // Wait for all servers to see each other before opening the client REPL. @@ -99,19 +99,19 @@ func runLocalCluster(srvOpts gorums.ServerOption) ([]string, func(), error) { // assignment regardless of the order of addresses in the input list. // The server's own address must be included in the peer list. // It returns an error if the server's address is not found in the peer list. -func peerConfig(address string, peers []string) (uint32, gorums.NodeSource, error) { +func peerConfig(address string, peers []string) (gorums.ID, gorums.NodeSource, error) { sorted := slices.Clone(peers) slices.Sort(sorted) idx := slices.Index(sorted, address) if idx < 0 { return 0, nil, fmt.Errorf("server address %q not found in -addrs list", address) } - return uint32(idx + 1), gorums.WithNodeList(sorted), nil + return gorums.ID(idx + 1), gorums.WithNodeList(sorted), nil } // registerAndServe registers the storage service on srv and starts serving in // a background goroutine. The server log output is labelled with the node ID. -func registerAndServe(srv *gorums.Server, id uint32) { +func registerAndServe(srv *gorums.Server, id gorums.ID) { storage := newStorageServer(os.Stderr, fmt.Sprintf("node %d", id)) pb.RegisterStorageServer(srv, storage) go func() { diff --git a/gorumstest/errors.go b/gorumstest/errors.go index c8b95f87..88bfb23f 100644 --- a/gorumstest/errors.go +++ b/gorumstest/errors.go @@ -11,7 +11,7 @@ import ( // QuorumCallError returns a [gorums.QuorumCallError] with cause // [gorums.ErrIncomplete] and one node error for each entry in nodeErrors. // The node errors are in ascending node ID order. -func QuorumCallError(nodeErrors map[uint32]error) gorums.QuorumCallError { +func QuorumCallError(nodeErrors map[gorums.ID]error) gorums.QuorumCallError { errs := make([]conn.NodeError, 0, len(nodeErrors)) for _, nodeID := range slices.Sorted(maps.Keys(nodeErrors)) { errs = append(errs, conn.NewNodeError(nodeID, nodeErrors[nodeID])) diff --git a/gorumstest/gorumstest.go b/gorumstest/gorumstest.go index a9c050b1..8e6cfcc6 100644 --- a/gorumstest/gorumstest.go +++ b/gorumstest/gorumstest.go @@ -186,12 +186,10 @@ func Node(t testing.TB, srvFn func(i int) ServerIface, opts ...Option) *gorums.N } // PeerNode returns the node with the given id in cfg, or fails the test. -func PeerNode(t testing.TB, cfg gorums.Config, id uint32) *gorums.Node { +func PeerNode(t testing.TB, cfg gorums.Config, id gorums.ID) *gorums.Node { t.Helper() - for _, node := range cfg { - if node.ID() == id { - return node - } + if node := cfg.Node(id); node != nil { + return node } t.Fatalf("node %d not in config %v", id, cfg.NodeIDs()) return nil diff --git a/internal/conn/config.go b/internal/conn/config.go index 97d1c49d..31dad02e 100644 --- a/internal/conn/config.go +++ b/internal/conn/config.go @@ -115,8 +115,8 @@ func (c Config) Extend(nodes NodeSource) (Config, error) { } // NodeIDs returns a slice of this configuration's Node IDs. -func (c Config) NodeIDs() []uint32 { - ids := make([]uint32, len(c)) +func (c Config) NodeIDs() []ID { + ids := make([]ID, len(c)) for i, node := range c { ids[i] = node.ID() } @@ -146,14 +146,22 @@ func (c Config) Equal(b Config) bool { return true } +// Node returns the node in c with the given ID, or nil if c has no such node. +func (c Config) Node(id ID) *Node { + if i := slices.IndexFunc(c, func(n *Node) bool { return n.id == id }); i >= 0 { + return c[i] + } + return nil +} + // Contains reports whether c contains a node with the given ID. -func (c Config) Contains(id uint32) bool { - return slices.ContainsFunc(c, func(n *Node) bool { return n.id == id }) +func (c Config) Contains(id ID) bool { + return c.Node(id) != nil } // Add returns a new Config containing nodes from c and nodes with the specified IDs. // Duplicate IDs and IDs not found in the manager are ignored. -func (c Config) Add(ids ...uint32) Config { +func (c Config) Add(ids ...ID) Config { if len(c) == 0 { return nil } @@ -186,7 +194,7 @@ func (c Config) Union(other Config) Config { } // Remove returns a new Config excluding nodes with the specified IDs. -func (c Config) Remove(ids ...uint32) Config { +func (c Config) Remove(ids ...ID) Config { if len(c) == 0 { return nil } @@ -325,7 +333,7 @@ func (c Config) WithoutErrors(err QuorumCallError, errorTypes ...error) Config { return false } // Build a set of node IDs to exclude. - excludeSet := newSet[uint32]() + excludeSet := newSet[ID]() for _, ne := range err.errors { if exclude(ne.cause) { excludeSet.add(ne.nodeID) diff --git a/internal/conn/config_test.go b/internal/conn/config_test.go index e84026a2..26271554 100644 --- a/internal/conn/config_test.go +++ b/internal/conn/config_test.go @@ -106,6 +106,32 @@ func TestConfigWatch(t *testing.T) { }) } +func TestConfigNode(t *testing.T) { + n1 := newTestNode(1, stream.NewChannelWithState(nil)) + n3 := newTestNode(3, stream.NewChannelWithState(nil)) + tests := []struct { + name string + cfg Config + id ID + want *Node + }{ + {name: "First", cfg: Config{n1, n3}, id: 1, want: n1}, + {name: "Last", cfg: Config{n1, n3}, id: 3, want: n3}, + {name: "NotSortedByID", cfg: Config{n3, n1}, id: 1, want: n1}, + {name: "Absent", cfg: Config{n1, n3}, id: 2, want: nil}, + {name: "ReservedZero", cfg: Config{n1, n3}, id: 0, want: nil}, + {name: "Empty", cfg: Config{}, id: 1, want: nil}, + {name: "Nil", cfg: nil, id: 1, want: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.cfg.Node(tt.id); got != tt.want { + t.Errorf("Node(%d) = node %d, want node %d", tt.id, got.ID(), tt.want.ID()) + } + }) + } +} + func TestConfigSort(t *testing.T) { const unmeasured = -1 * time.Second // makeNode returns a node with the given latency and last error. diff --git a/internal/conn/dial_options.go b/internal/conn/dial_options.go index 47778bf6..cc021c05 100644 --- a/internal/conn/dial_options.go +++ b/internal/conn/dial_options.go @@ -23,7 +23,7 @@ type DialOptions struct { SendBufferSize uint Metadata metadata.MD Handler stream.RequestHandler - LocalNodeID uint32 // if non-zero, skip setting handler on this node ID + LocalNodeID ID // if non-zero, skip setting handler on this node ID StreamDedup bool // reuse a lower-ID peer's dialed stream instead of dialing back InboundManager *InboundManager // set when the configuration carries a server (peer or back-channel client); enables eager reconnect and, with StreamDedup, borrowing Err error // records misuse of a dial option; surfaced by NewConfig diff --git a/internal/conn/errors.go b/internal/conn/errors.go index 23339f05..8a6ef48b 100644 --- a/internal/conn/errors.go +++ b/internal/conn/errors.go @@ -70,12 +70,12 @@ func (e QuorumCallError) Error() string { // NodeError reports on a failed RPC call from a specific node. type NodeError struct { cause error - nodeID uint32 + nodeID ID } // NewNodeError builds a [NodeError] for the given node ID and cause. It is used // by the call engine to record per-node failures in a [QuorumCallError]. -func NewNodeError(nodeID uint32, cause error) NodeError { +func NewNodeError(nodeID ID, cause error) NodeError { return NodeError{cause: cause, nodeID: nodeID} } diff --git a/internal/conn/inbound_manager.go b/internal/conn/inbound_manager.go index dba1789c..3b7df035 100644 --- a/internal/conn/inbound_manager.go +++ b/internal/conn/inbound_manager.go @@ -26,8 +26,8 @@ const gorumsNodeIDKey = "gorums-node-id" var errSelfNodeIDStream = status.Error(codes.InvalidArgument, "gorums: inbound stream claims the server's own node ID") // nodeID extracts the NodeID from the gorums-node-id metadata key in ctx. -// It returns 0 if the key is absent, empty, or not a valid uint32 greater than zero. -func nodeID(ctx context.Context) uint32 { +// It returns 0 if the key is absent, empty, or not a valid ID greater than zero. +func nodeID(ctx context.Context) ID { md, ok := metadata.FromIncomingContext(ctx) if !ok { return 0 @@ -40,7 +40,7 @@ func nodeID(ctx context.Context) uint32 { if err != nil || id == 0 { return 0 } - return uint32(id) + return ID(id) } // hasPeerMetadata reports whether ctx contains the gorums-node-id metadata key, @@ -56,7 +56,7 @@ func hasPeerMetadata(ctx context.Context) bool { } // MetadataWithNodeID returns a metadata.MD containing the gorums-node-id key with the given id value. -func MetadataWithNodeID(id uint32) metadata.MD { +func MetadataWithNodeID(id ID) metadata.MD { return metadata.Pairs(gorumsNodeIDKey, strconv.FormatUint(uint64(id), 10)) } @@ -74,9 +74,9 @@ func MetadataWithNodeID(id uint32) metadata.MD { // InboundManager is safe for concurrent use. type InboundManager struct { mu sync.RWMutex - myID uint32 // this server's own NodeID; always present in inboundPeers - knownNodes map[uint32]*Node // pre-created configured peers, including self when configured - clientNodes map[uint32]*Node // dynamically assigned peer-capable clients + myID ID // this server's own NodeID; always present in inboundPeers + knownNodes map[ID]*Node // pre-created configured peers, including self when configured + clientNodes map[ID]*Node // dynamically assigned peer-capable clients peerConfig Config // the server's peer Config; set once by SetPeerConfig after NewConfig builds it connectedPeers Config // auto-updated connectivity-filtered subset of peerConfig, sorted by ID inboundPeers Config // auto-updated slice of known peers with an inbound stream, sorted by ID @@ -109,11 +109,11 @@ const ClientIDStart = 1 << 20 // their send queue and request dispatch capacities; a dispatchSize of 0 // selects the default. It returns an error if peerNodes is invalid, for // example because of an invalid address or a duplicate node. -func NewInboundManager(myID uint32, peerNodes NodeSource, sendBufferSize, dispatchSize uint, onConfigChange func(Config), handler stream.RequestHandler) (*InboundManager, error) { +func NewInboundManager(myID ID, peerNodes NodeSource, sendBufferSize, dispatchSize uint, onConfigChange func(Config), handler stream.RequestHandler) (*InboundManager, error) { im := &InboundManager{ myID: myID, - knownNodes: make(map[uint32]*Node), - clientNodes: make(map[uint32]*Node), + knownNodes: make(map[ID]*Node), + clientNodes: make(map[ID]*Node), sendBufferSize: sendBufferSize, dispatchSize: dispatchSize, handler: handler, @@ -188,7 +188,7 @@ func (im *InboundManager) ConnectedClients() Config { } // NodeID returns this server's own nodeID. -func (im *InboundManager) NodeID() uint32 { +func (im *InboundManager) NodeID() ID { if im == nil { return 0 } @@ -208,7 +208,7 @@ func (im *InboundManager) nextMsgID() uint64 { // construction before any peers connect, so no locking is needed. // If id equals myID, a local (in-process) node is created instead of an // inbound node, enabling direct handler invocation without a network round-trip. -func (im *InboundManager) newNode(id uint32, addr string) (*Node, error) { +func (im *InboundManager) newNode(id ID, addr string) (*Node, error) { var node *Node if id == im.myID && im.handler != nil { node = newLocalNode(id, addr, im.nextMsgID, im.handler, nil) @@ -226,7 +226,7 @@ func (im *InboundManager) newNode(id uint32, addr string) (*Node, error) { // peer's channel slot before the peer first connects. The returned node's // channel reference is shared, so channel attachments and replacements remain // visible to holders. -func (im *InboundManager) knownPeer(id uint32) *Node { +func (im *InboundManager) knownPeer(id ID) *Node { if im == nil { return nil } @@ -237,7 +237,7 @@ func (im *InboundManager) knownPeer(id uint32) *Node { // isKnown returns true if the given NodeID is a known peer. // Returns false for id == 0 (external clients) or unknown IDs. -func (im *InboundManager) isKnown(id uint32) bool { +func (im *InboundManager) isKnown(id ID) bool { if id == 0 { return false } @@ -293,7 +293,7 @@ func (im *InboundManager) AcceptPeer(streamCtx context.Context, inboundStream st // as active while keeping the prior one live until its own stream ends, so the // node never goes dark mid-handover; see [Node.attachStream]. The returned // cleanup function detaches and closes this registration's channel. -func (im *InboundManager) registerPeer(streamCtx context.Context, inboundStream stream.BidiStream, id uint32) (*stream.InboundChannel, func(), error) { +func (im *InboundManager) registerPeer(streamCtx context.Context, inboundStream stream.BidiStream, id ID) (*stream.InboundChannel, func(), error) { im.mu.Lock() defer im.mu.Unlock() node := im.knownNodes[id] @@ -343,10 +343,10 @@ func (im *InboundManager) acceptClient(streamCtx context.Context, inboundStream // nextAvailableClientID returns the next unoccupied dynamic client ID. // The caller must hold im.mu. -func (im *InboundManager) nextAvailableClientID() (uint32, error) { +func (im *InboundManager) nextAvailableClientID() (ID, error) { const maxNodeID = uint64(1<<32 - 1) for im.nextClientID <= maxNodeID { - id := uint32(im.nextClientID) + id := ID(im.nextClientID) im.nextClientID++ if _, exists := im.knownNodes[id]; exists { continue diff --git a/internal/conn/inbound_manager_test.go b/internal/conn/inbound_manager_test.go index 6210133a..3426693a 100644 --- a/internal/conn/inbound_manager_test.go +++ b/internal/conn/inbound_manager_test.go @@ -19,7 +19,7 @@ import ( // requestHandlerFunc adapts a function to [stream.RequestHandler]. type requestHandlerFunc func(context.Context, *stream.Message, func(), func(*stream.Message)) -func (f requestHandlerFunc) HandleRequest(ctx context.Context, msg *stream.Message, release func(), send func(*stream.Message)) { +func (f requestHandlerFunc) HandleRequest(ctx context.Context, _ ID, msg *stream.Message, release func(), send func(*stream.Message)) { f(ctx, msg, release, send) } diff --git a/internal/conn/node.go b/internal/conn/node.go index bdc1f66d..3819b05f 100644 --- a/internal/conn/node.go +++ b/internal/conn/node.go @@ -42,11 +42,17 @@ func sharedNodeTransport(peer *Node) *stream.Transport { return stream.NewSharedTransport(transport) } +// ID identifies a node. ID 0 is reserved for a node that has no ID: a +// handler-only server, or a client that announces no ID. Dynamic client IDs +// start at [ClientIDStart] and skip configured IDs. Under stream +// deduplication, the peer with the lower ID dials. +type ID = stream.ID + // Node encapsulates the state of a node on which a remote procedure call // can be performed. type Node struct { // Only assigned at creation. - id uint32 + id ID addr string mgr *outboundManager // owning manager for this node @@ -63,7 +69,7 @@ type Node struct { } // newNode creates a Node with stable identity fields and its transport. -func newNode(id uint32, addr string, mgr *outboundManager, transport *stream.Transport) *Node { +func newNode(id ID, addr string, mgr *outboundManager, transport *stream.Transport) *Node { return &Node{id: id, addr: addr, mgr: mgr, transport: transport} } @@ -95,7 +101,7 @@ func (n *Node) Context(parent context.Context) *NodeContext { // nodeOptions contains configuration options for creating a new Node. type nodeOptions struct { - ID uint32 + ID ID SendBufferSize uint MsgIDGen func() uint64 Metadata metadata.MD @@ -140,7 +146,7 @@ func newOutboundNode(addr string, opts nodeOptions) (*Node, error) { // newInboundNode creates a Node for a known peer or client without an active // channel; the channel is attached when the peer's stream arrives. -func newInboundNode(id uint32, addr string, msgIDGen func() uint64) *Node { +func newInboundNode(id ID, addr string, msgIDGen func() uint64) *Node { return newNode(id, addr, nil, stream.NewTransport(id, msgIDGen)) } @@ -148,7 +154,7 @@ func newInboundNode(id uint32, addr string, msgIDGen func() uint64) *Node { // network. It is used for the self-node when a server calls its own peers, // which include itself. The provided handler serves requests directly without // a gRPC round-trip. -func newLocalNode(id uint32, addr string, msgIDGen func() uint64, handler stream.RequestHandler, mgr *outboundManager) *Node { +func newLocalNode(id ID, addr string, msgIDGen func() uint64, handler stream.RequestHandler, mgr *outboundManager) *Node { transport := stream.NewTransport(id, msgIDGen) n := newNode(id, addr, mgr, transport) transport.StoreChannel(stream.NewLocalChannel(id, handler)) @@ -270,7 +276,7 @@ func (n *Node) close() error { } // ID returns the ID of n. -func (n *Node) ID() uint32 { +func (n *Node) ID() ID { if n != nil { return n.id } diff --git a/internal/conn/node_source.go b/internal/conn/node_source.go index 828d08db..f4482c57 100644 --- a/internal/conn/node_source.go +++ b/internal/conn/node_source.go @@ -18,7 +18,7 @@ type NodeSource interface { // Implemented by [outboundManager] and [InboundManager]. type nodeRegistry interface { Nodes() []*Node - newNode(id uint32, addr string) (*Node, error) + newNode(id ID, addr string) (*Node, error) } // NodeAddress must be implemented by types that can be used as node addresses. @@ -29,11 +29,11 @@ type NodeAddress interface { // WithNodes returns a NodeSource containing the provided mapping from // application-specific IDs to types implementing NodeAddress. // Node IDs must be greater than 0. -func WithNodes[T NodeAddress](nodes map[uint32]T) NodeSource { +func WithNodes[T NodeAddress](nodes map[ID]T) NodeSource { return nodeMap[T](nodes) } -type nodeMap[T NodeAddress] map[uint32]T +type nodeMap[T NodeAddress] map[ID]T func (nm nodeMap[T]) newConfig(registry nodeRegistry) (Config, error) { if len(nm) == 0 { @@ -67,7 +67,7 @@ func (nl nodeList) newConfig(registry nodeRegistry) (Config, error) { builder := newNodeBuilder(registry, len(nl)) nextID := builder.nextID() for i, addr := range nl { - id := nextID + uint32(i) + id := nextID + ID(i) if err := builder.add(id, addr); err != nil { return nil, err } @@ -79,17 +79,17 @@ func (nl nodeList) newConfig(registry nodeRegistry) (Config, error) { // It encapsulates the common logic shared between WithNodes and WithNodeList. type nodeBuilder struct { registry nodeRegistry - addrToID map[string]uint32 // normalized address -> node ID - idToNode map[uint32]*Node // existing node ID -> node - maxID uint32 // maximum existing node ID + addrToID map[string]ID // normalized address -> node ID + idToNode map[ID]*Node // existing node ID -> node + maxID ID // maximum existing node ID nodes Config } // newNodeBuilder creates a new nodeBuilder initialized with existing nodes from the registry. func newNodeBuilder(registry nodeRegistry, capacity int) *nodeBuilder { - addrToID := make(map[string]uint32, capacity) - idToNode := make(map[uint32]*Node, capacity) - maxID := uint32(0) + addrToID := make(map[string]ID, capacity) + idToNode := make(map[ID]*Node, capacity) + maxID := ID(0) // Populate with existing nodes from the registry (already normalized) for _, existingNode := range registry.Nodes() { id := existingNode.ID() @@ -107,7 +107,7 @@ func newNodeBuilder(registry nodeRegistry, capacity int) *nodeBuilder { } // add creates or reuses a node with the given ID and address. -func (b *nodeBuilder) add(id uint32, addr string) error { +func (b *nodeBuilder) add(id ID, addr string) error { if id == 0 { return fmt.Errorf("gorums: node 0 is reserved") } @@ -146,7 +146,7 @@ func (b *nodeBuilder) configuration() Config { } // nextID returns the next available node ID (max existing ID + 1). -func (b *nodeBuilder) nextID() uint32 { +func (b *nodeBuilder) nextID() ID { return b.maxID + 1 } diff --git a/internal/conn/outbound_manager.go b/internal/conn/outbound_manager.go index 91b4e74e..85bea67f 100644 --- a/internal/conn/outbound_manager.go +++ b/internal/conn/outbound_manager.go @@ -16,7 +16,7 @@ import ( type outboundManager struct { mu sync.Mutex nodes []*Node - lookup map[uint32]*Node + lookup map[ID]*Node closed bool // guarded by mu; set once by Close closeOnce sync.Once logger *log.Logger @@ -28,7 +28,7 @@ type outboundManager struct { // nodes added to the manager. func newOutboundManager(opts ...DialOption) *outboundManager { m := &outboundManager{ - lookup: make(map[uint32]*Node), + lookup: make(map[ID]*Node), opts: NewDialOptions(), } for _, opt := range opts { @@ -75,7 +75,7 @@ func (m *outboundManager) Close() { } // Node returns the node with the given identifier if present. -func (m *outboundManager) Node(id uint32) (node *Node, found bool) { +func (m *outboundManager) Node(id ID) (node *Node, found bool) { m.mu.Lock() defer m.mu.Unlock() node, found = m.lookup[id] @@ -115,7 +115,7 @@ func (m *outboundManager) isClosed() bool { return m.closed } -func (m *outboundManager) newNode(id uint32, addr string) (*Node, error) { +func (m *outboundManager) newNode(id ID, addr string) (*Node, error) { if _, found := m.Node(id); found { return nil, fmt.Errorf("gorums: node %d already exists", id) } diff --git a/internal/conn/testhelpers.go b/internal/conn/testhelpers.go index af3fff28..c6f717e8 100644 --- a/internal/conn/testhelpers.go +++ b/internal/conn/testhelpers.go @@ -10,6 +10,6 @@ import "github.com/relab/gorums/internal/stream" // NewNodeForTest builds a node with the given transport and no owning manager, // for tests in other packages that assemble a [Config] by hand. This function // should only be used in tests. -func NewNodeForTest(id uint32, transport *stream.Transport) *Node { +func NewNodeForTest(id ID, transport *stream.Transport) *Node { return newNode(id, "", nil, transport) } diff --git a/internal/impl/aliases.go b/internal/impl/aliases.go index a5ff55e4..ff254504 100644 --- a/internal/impl/aliases.go +++ b/internal/impl/aliases.go @@ -5,6 +5,7 @@ import "github.com/relab/gorums/internal/conn" // Aliases for the connectivity types the call engine operates on, so engine // code reads the same as the public API in package gorums. type ( + ID = conn.ID Node = conn.Node Config = conn.Config ConfigContext = conn.ConfigContext diff --git a/internal/impl/call_context.go b/internal/impl/call_context.go index 3aeaa9cd..267c6a71 100644 --- a/internal/impl/call_context.go +++ b/internal/impl/call_context.go @@ -3,7 +3,6 @@ package impl import ( "context" "math/rand/v2" - "slices" "sync" "sync/atomic" @@ -125,16 +124,10 @@ func (c *CallContext[Req, Resp]) Nodes() []*Node { return c.config.Nodes() } -// Node returns the node with the given ID. -func (c *CallContext[Req, Resp]) Node(id uint32) *Node { - nodes := c.config.Nodes() - index := slices.IndexFunc(nodes, func(n *Node) bool { - return n.ID() == id - }) - if index != -1 { - return nodes[index] - } - return nil +// Node returns the node with the given ID, or nil if the call's configuration +// has no such node. +func (c *CallContext[Req, Resp]) Node(id ID) *Node { + return c.config.Node(id) } // Size returns the number of nodes in this configuration. @@ -144,7 +137,7 @@ func (c *CallContext[Req, Resp]) Size() int { // reportNodeError sends an error response for the given node to responseChan. // It is a no-op for fire-and-forget calls where responseChan is nil. -func (c *CallContext[Req, Resp]) reportNodeError(nodeID uint32, err error) { +func (c *CallContext[Req, Resp]) reportNodeError(nodeID ID, err error) { if c.responseChan != nil { c.responseChan <- NodeResponse[*stream.Message]{NodeID: nodeID, Err: err} } diff --git a/internal/impl/call_context_test.go b/internal/impl/call_context_test.go index d3cf27d4..31868afe 100644 --- a/internal/impl/call_context_test.go +++ b/internal/impl/call_context_test.go @@ -19,7 +19,7 @@ type seqNoRecorder struct { seqNos []uint64 } -func (h *seqNoRecorder) HandleRequest(_ context.Context, msg *stream.Message, release func(), _ func(*stream.Message)) { +func (h *seqNoRecorder) HandleRequest(_ context.Context, _ ID, msg *stream.Message, release func(), _ func(*stream.Message)) { h.mu.Lock() h.seqNos = append(h.seqNos, msg.GetMessageSeqNo()) h.mu.Unlock() diff --git a/internal/impl/responses.go b/internal/impl/responses.go index ae176bc8..6904712b 100644 --- a/internal/impl/responses.go +++ b/internal/impl/responses.go @@ -71,8 +71,8 @@ func (seq ResponseSeq[Resp]) Filter(keep func(NodeResponse[Resp]) bool) Response // It returns early if n entries are collected or the iterator is exhausted. // When a node response carries an error, the zero value of Resp is stored for // that node ID; use [ResponseSeq.IgnoreErrors] to skip errored nodes entirely. -func (seq ResponseSeq[Resp]) CollectN(n int) map[uint32]Resp { - replies := make(map[uint32]Resp, n) +func (seq ResponseSeq[Resp]) CollectN(n int) map[ID]Resp { + replies := make(map[ID]Resp, n) for result := range seq { replies[result.NodeID] = result.Value if len(replies) >= n { @@ -85,8 +85,8 @@ func (seq ResponseSeq[Resp]) CollectN(n int) map[uint32]Resp { // CollectAll collects all values from the iterator into a map by node ID. // When a node response carries an error, the zero value of Resp is stored for // that node ID; use [ResponseSeq.IgnoreErrors] to skip errored nodes entirely. -func (seq ResponseSeq[Resp]) CollectAll() map[uint32]Resp { - replies := make(map[uint32]Resp) +func (seq ResponseSeq[Resp]) CollectAll() map[ID]Resp { + replies := make(map[ID]Resp) for result := range seq { replies[result.NodeID] = result.Value } diff --git a/internal/stream/channel.go b/internal/stream/channel.go index 36409eb5..0b2244f5 100644 --- a/internal/stream/channel.go +++ b/internal/stream/channel.go @@ -38,6 +38,10 @@ type Channel interface { // dispatching it to the appropriate method handler, as encoded in the // message's method field. Each call runs in its own goroutine. // +// The senderID is the ID of the channel that received the request: the +// peer's ID on a stream, 0 if the peer has no ID, and the local node's own +// ID on a [LocalChannel]. +// // The release function is idempotent. Calling it lets the next request from // the stream start before HandleRequest returns; otherwise the next request // starts when HandleRequest returns. @@ -49,12 +53,12 @@ type Channel interface { // one-way call types, the peer has no pending call to receive a response, so // it is dropped. type RequestHandler interface { - HandleRequest(ctx context.Context, msg *Message, release func(), send func(*Message)) + HandleRequest(ctx context.Context, senderID ID, msg *Message, release func(), send func(*Message)) } // endpoint holds the state that a stream channel shares across its streams. type endpoint struct { - id uint32 + id ID ctx context.Context // the channel's lifetime cancel context.CancelFunc queue *sendQueue @@ -74,7 +78,7 @@ type endpoint struct { // newEndpoint returns an endpoint that lives until parent ends or cancel is // called. A dispatchSize of 0 selects the default. -func newEndpoint(parent context.Context, id uint32, sendBufferSize, dispatchSize uint, handler RequestHandler, latency *Latency) endpoint { +func newEndpoint(parent context.Context, id ID, sendBufferSize, dispatchSize uint, handler RequestHandler, latency *Latency) endpoint { ctx, cancel := context.WithCancel(parent) return endpoint{ id: id, diff --git a/internal/stream/channel_test.go b/internal/stream/channel_test.go index 4ed466eb..e70f7bd8 100644 --- a/internal/stream/channel_test.go +++ b/internal/stream/channel_test.go @@ -1162,7 +1162,7 @@ type signalingRequestHandler struct { called chan *Message } -func (h *signalingRequestHandler) HandleRequest(_ context.Context, msg *Message, release func(), _ func(*Message)) { +func (h *signalingRequestHandler) HandleRequest(_ context.Context, _ ID, msg *Message, release func(), _ func(*Message)) { defer release() select { case h.called <- msg: diff --git a/internal/stream/inbound.go b/internal/stream/inbound.go index 015097a8..83b3c48e 100644 --- a/internal/stream/inbound.go +++ b/internal/stream/inbound.go @@ -35,7 +35,7 @@ type InboundChannel struct { // NewInboundChannel returns a channel over stream and starts its send loop. // ctx must be the stream's context; handlers receive contexts derived from it. // Call [InboundChannel.Serve] to receive from the stream. -func NewInboundChannel(ctx context.Context, id uint32, stream BidiStream, opts InboundOptions) *InboundChannel { +func NewInboundChannel(ctx context.Context, id ID, stream BidiStream, opts InboundOptions) *InboundChannel { c := &InboundChannel{ endpoint: newEndpoint(ctx, id, opts.SendBufferSize, opts.DispatchSize, opts.Handler, opts.Latency), } diff --git a/internal/stream/local.go b/internal/stream/local.go index c41cb113..4e576f01 100644 --- a/internal/stream/local.go +++ b/internal/stream/local.go @@ -9,13 +9,13 @@ import ( // bypassing the network. Handlers run one at a time, in request order, as on // a stream. type LocalChannel struct { - id uint32 + id ID handler RequestHandler requests *dispatcher } // NewLocalChannel returns a channel that serves requests with handler. -func NewLocalChannel(id uint32, handler RequestHandler) *LocalChannel { +func NewLocalChannel(id ID, handler RequestHandler) *LocalChannel { return &LocalChannel{id: id, handler: handler, requests: newDispatcher(nil, 0)} } @@ -41,7 +41,7 @@ func (c *LocalChannel) Enqueue(req Request) { req.deliver(response{NodeID: c.id, Value: msg, Err: msg.errorStatus()}) } } - run := func(release func()) { c.handler.HandleRequest(ctx, req.Msg, release, send) } + run := func(release func()) { c.handler.HandleRequest(ctx, c.id, req.Msg, release, send) } if req.wantServerResponse() { if !c.requests.tryPush(run) { req.sendErrorResponse(c.id, ErrSendQueueFull) diff --git a/internal/stream/outbound.go b/internal/stream/outbound.go index 52037998..63aa404c 100644 --- a/internal/stream/outbound.go +++ b/internal/stream/outbound.go @@ -66,7 +66,7 @@ type OutboundChannel struct { // NewOutboundChannel returns a channel to the server behind conn and starts // its goroutine. The channel lives until ctx ends or it is closed; closing it // also closes conn. -func NewOutboundChannel(ctx context.Context, id uint32, conn *grpc.ClientConn, opts OutboundOptions) *OutboundChannel { +func NewOutboundChannel(ctx context.Context, id ID, conn *grpc.ClientConn, opts OutboundOptions) *OutboundChannel { c := &OutboundChannel{ endpoint: newEndpoint(ctx, id, opts.SendBufferSize, 0, opts.Handler, opts.Latency), conn: conn, diff --git a/internal/stream/request.go b/internal/stream/request.go index 8eeee275..a9815010 100644 --- a/internal/stream/request.go +++ b/internal/stream/request.go @@ -46,15 +46,18 @@ func (r Request) deliver(resp response) bool { // sendErrorResponse sends a response with err on the request's response // channel, if it has one. -func (r Request) sendErrorResponse(nodeID uint32, err error) { +func (r Request) sendErrorResponse(nodeID ID, err error) { if r.ResponseChan != nil { r.deliver(response{NodeID: nodeID, Err: err}) } } +// ID identifies a node. A channel with ID 0 belongs to a peer that has no ID. +type ID = uint32 + // NodeResponse wraps a response value from node ID, and an error if any. type NodeResponse[T any] struct { - NodeID uint32 + NodeID ID Value T Err error } diff --git a/internal/stream/request_test.go b/internal/stream/request_test.go index c85a859f..ec11d5d8 100644 --- a/internal/stream/request_test.go +++ b/internal/stream/request_test.go @@ -7,7 +7,7 @@ import ( "time" ) -func (f requestHandlerFunc) HandleRequest(ctx context.Context, msg *Message, release func(), send func(*Message)) { +func (f requestHandlerFunc) HandleRequest(ctx context.Context, _ ID, msg *Message, release func(), send func(*Message)) { f(ctx, msg, release, send) } diff --git a/internal/stream/send_queue.go b/internal/stream/send_queue.go index 43a2e300..f5ab93ea 100644 --- a/internal/stream/send_queue.go +++ b/internal/stream/send_queue.go @@ -8,7 +8,7 @@ import ( // sendQueue is a bounded FIFO of requests awaiting a stream send. // Once closed, it fails every queued and later request with [ErrNodeClosed]. type sendQueue struct { - id uint32 + id ID ch chan Request done <-chan struct{} // ends waits for queue space @@ -20,7 +20,7 @@ type sendQueue struct { // newSendQueue returns a queue with the given capacity. Closing done ends // waits for queue space; [sendQueue.close] waits for those to end. -func newSendQueue(id uint32, capacity uint, done <-chan struct{}) *sendQueue { +func newSendQueue(id ID, capacity uint, done <-chan struct{}) *sendQueue { return &sendQueue{id: id, ch: make(chan Request, capacity), done: done} } diff --git a/internal/stream/session.go b/internal/stream/session.go index 31d26025..b3b00bfc 100644 --- a/internal/stream/session.go +++ b/internal/stream/session.go @@ -136,7 +136,7 @@ func (s *session) handle(msg *Message) { if s.handler != nil { ctx := msg.appendToIncomingContext(s.ctx) s.requests.push(s.ctx, func(release func()) { - s.handler.HandleRequest(ctx, msg, release, s.reply) + s.handler.HandleRequest(ctx, s.id, msg, release, s.reply) }) } return diff --git a/internal/stream/transport.go b/internal/stream/transport.go index 1d0513f5..3114b673 100644 --- a/internal/stream/transport.go +++ b/internal/stream/transport.go @@ -32,7 +32,7 @@ func (r *channelRef) store(ch Channel) { // shared channel reference changes as streams come and go. A shared transport // borrows these resources from an inbound peer node's transport. type Transport struct { - id uint32 + id ID channel *channelRef latency *Latency msgIDGen func() uint64 @@ -41,7 +41,7 @@ type Transport struct { // NewTransport returns an owned transport with no channel; attach one with // [Transport.StoreChannel]. -func NewTransport(id uint32, msgIDGen func() uint64) *Transport { +func NewTransport(id ID, msgIDGen func() uint64) *Transport { return &Transport{ id: id, channel: new(channelRef), @@ -99,7 +99,7 @@ func (t *Transport) StoreChannel(ch Channel) { func (t *Transport) Enqueue(req Request) { ch := t.LoadChannel() if ch == nil { - var id uint32 + var id ID if t != nil { id = t.id } diff --git a/local_servers.go b/local_servers.go index 6b4da93b..d5e5fe46 100644 --- a/local_servers.go +++ b/local_servers.go @@ -70,7 +70,7 @@ func NewLocalServers(n int, opts ...LocalServerOption) ([]*Server, func(), error } servers := make([]*Server, n) for i := range n { - myID := uint32(i + 1) + myID := ID(i + 1) serverOpts := append( []ServerOption{WithPeers(myID, nodeSource, localOpts.dialOpts...)}, localOpts.serverOpts..., diff --git a/server.go b/server.go index 7d1b2733..557263f2 100644 --- a/server.go +++ b/server.go @@ -43,7 +43,7 @@ type Server struct { // NodeID returns this server's own [Config] node ID, as configured with // [WithPeers]. It returns 0 if [WithPeers] was not used. -func (s *Server) NodeID() uint32 { +func (s *Server) NodeID() ID { return s.im.NodeID() } @@ -201,12 +201,13 @@ func chainInterceptors(final Handler, interceptors ...ServerInterceptor) Handler // [ServerContext.Release]. Otherwise the caller releases the request when // HandleRequest returns, as [stream.RequestHandler] specifies, so the stream's // next request can run on the same goroutine. -func (s *Server) HandleRequest(ctx context.Context, reqMsg *stream.Message, release func(), send func(*stream.Message)) { +func (s *Server) HandleRequest(ctx context.Context, senderID ID, reqMsg *stream.Message, release func(), send func(*stream.Message)) { srvCtx := ServerContext{ - Context: ctx, - release: release, - send: send, - srv: s, + Context: ctx, + senderID: senderID, + release: release, + send: send, + srv: s, } handler, ok := s.handlers[reqMsg.GetMethod()] @@ -364,9 +365,28 @@ func (s *Server) Stop() { // [ServerContext.Release] or returns. type ServerContext struct { context.Context - release func() - send func(*stream.Message) - srv *Server + senderID ID + release func() + send func(*stream.Message) + srv *Server +} + +// SenderID returns the ID of the node that sent the request, as this server +// identifies that node: +// - a peer configured with [WithPeers] has its configured ID; +// - a back-channel client has the ID that this server assigned to it, as in +// [Server.ConnectedClients]; the ID changes when the client reconnects; +// - a request on a connection that this server dialed has the dialed +// node's ID; +// - a request that this server sends to itself has this server's own ID. +// +// SenderID returns 0 for a client that announces no ID, or an ID that this +// server does not know. The sender is the last hop: when a message travels +// through several nodes, SenderID identifies the node that forwarded it. The +// sender asserts its ID when it connects, and the server does not +// authenticate it. +func (ctx *ServerContext) SenderID() ID { + return ctx.senderID } // Release lets the next request on this handler's stream start, concurrently diff --git a/server_options.go b/server_options.go index 9d26e17e..7e8cf8c8 100644 --- a/server_options.go +++ b/server_options.go @@ -15,7 +15,7 @@ type serverOptions struct { connectCallback func(context.Context) interceptors []ServerInterceptor // Peer management options - myID uint32 + myID ID peerNodes NodeSource // Peers this server tracks and calls; set by WithPeers. onConfigChange func(Config) // Callback registered via [WithPeerChange]; invoked after each connected-peer config change. listenAddr string // Listener address recorded by WithAddr; bound by ListenAndServe. @@ -89,7 +89,7 @@ func WithServerInterceptors(i ...ServerInterceptor) ServerOption { // The returned option only records the peer set; the [NewServer] call that // receives it panics if the node source is invalid, for example if it // contains a duplicate or malformed address. -func WithPeers(myID uint32, nodes NodeSource, opts ...DialOption) ServerOption { +func WithPeers(myID ID, nodes NodeSource, opts ...DialOption) ServerOption { return func(o *serverOptions) { o.myID = myID o.peerNodes = nodes diff --git a/server_test.go b/server_test.go index d82d34ca..7b433be2 100644 --- a/server_test.go +++ b/server_test.go @@ -314,7 +314,7 @@ func TestServerHandleRequestRelease(t *testing.T) { } req := stream.Message_builder{Method: tt.method, Payload: payload}.Build() releases, sends := 0, 0 - srv.HandleRequest(t.Context(), req, func() { releases++ }, func(*stream.Message) { sends++ }) + srv.HandleRequest(t.Context(), 0, req, func() { releases++ }, func(*stream.Message) { sends++ }) if releases != tt.wantReleases { t.Errorf("release called %d times, want %d", releases, tt.wantReleases) } @@ -1198,3 +1198,105 @@ func TestServerHandlerNestedCallBeforeRelease(t *testing.T) { t.Fatalf("server handler nested call: %v", err) } } + +// senderRecord is the sender ID that a receiving server's handler observed. +type senderRecord struct { + receiver gorums.ID + sender gorums.ID +} + +// senderRecorder returns a one-way handler that reports the request's +// [gorums.ServerContext.SenderID] as observed by receiver. +func senderRecorder(receiver gorums.ID, records chan<- senderRecord) gorums.Handler { + return func(ctx gorums.ServerContext, _ *gorums.Message) (*gorums.Message, error) { + records <- senderRecord{receiver: receiver, sender: ctx.SenderID()} + return nil, nil + } +} + +// multicastString sends a one-way multicast to cfg and fails the test on error. +func multicastString(t *testing.T, cfg gorums.Config, method string) { + t.Helper() + ctx := gorumstest.Context(t, 2*time.Second) + if err := gorumsimpl.Multicast(cfg.Context(ctx), pb.String("sender"), method).Send(); err != nil { + t.Fatalf("Multicast: %v", err) + } +} + +// TestServerSenderID verifies that a handler observes the sender of a request +// on every path a request can take to a server. +func TestServerSenderID(t *testing.T) { + // peerSenderIDs multicasts from every server to its peer configuration, + // which includes the server itself, and checks that every receiver + // observes the multicasting server's ID. + peerSenderIDs := func(t *testing.T, opts ...gorums.ServerOption) { + servers := gorumstest.LocalServers(t, 3, opts...) + records := make(chan senderRecord, len(servers)) + for _, srv := range servers { + srv.RegisterHandler(mock.StreamMethod, senderRecorder(srv.NodeID(), records)) + } + gorumstest.WaitForPeers(t, servers) + for _, srv := range servers { + multicastString(t, srv.PeerConfig(), mock.StreamMethod) + for _, r := range gorumstest.Collect(t, 2*time.Second, len(servers), records) { + if r.sender != srv.NodeID() { + t.Errorf("server %d: SenderID() = %d, want %d", r.receiver, r.sender, srv.NodeID()) + } + } + } + } + + // clientSenderID multicasts from a client that dials with opts and + // returns the sender ID that the server observed. + clientSenderID := func(t *testing.T, opts ...gorumstest.Option) gorums.ID { + records := make(chan senderRecord, 1) + srvFn := func(int) gorumstest.ServerIface { + srv := gorums.NewServer() + srv.RegisterHandler(mock.StreamMethod, senderRecorder(0, records)) + return srv + } + cfg := gorumstest.Config(t, 1, srvFn, opts...) + multicastString(t, cfg, mock.StreamMethod) + return gorumstest.Collect(t, 2*time.Second, 1, records)[0].sender + } + + // Inbound streams from known peers, and the in-process self node. + t.Run("KnownPeersAndSelf", func(t *testing.T) { peerSenderIDs(t) }) + // A higher-ID peer's requests arrive on the stream the receiver dialed. + t.Run("StreamDedup", func(t *testing.T) { peerSenderIDs(t, gorums.WithStreamDedup()) }) + + t.Run("RegularClient", func(t *testing.T) { + if got := clientSenderID(t); got != 0 { + t.Errorf("SenderID() = %d, want 0", got) + } + }) + t.Run("UnknownPeerID", func(t *testing.T) { + forged := gorums.WithMetadata(metadata.Pairs("gorums-node-id", "99")) + if got := clientSenderID(t, forged); got != 0 { + t.Errorf("SenderID() = %d, want 0", got) + } + }) + + t.Run("BackChannelClient", func(t *testing.T) { + srv, _, cfg := createServerAndClient(t) + records := make(chan senderRecord, 1) + srv.RegisterHandler(mock.StreamMethod, senderRecorder(0, records)) + awaitClientReady(t, srv, 1) + multicastString(t, cfg, mock.StreamMethod) + got := gorumstest.Collect(t, 2*time.Second, 1, records)[0].sender + if want := srv.ConnectedClients().NodeIDs()[0]; got != want { + t.Errorf("SenderID() = %d, want %d (the client's assigned ID)", got, want) + } + }) + t.Run("BackChannelServer", func(t *testing.T) { + srv, clientSrv, cfg := createServerAndClient(t) + records := make(chan senderRecord, 1) + clientSrv.RegisterHandler(mock.StreamMethod, senderRecorder(0, records)) + awaitClientReady(t, srv, 1) + multicastString(t, srv.ConnectedClients(), mock.StreamMethod) + got := gorumstest.Collect(t, 2*time.Second, 1, records)[0].sender + if want := cfg.NodeIDs()[0]; got != want { + t.Errorf("SenderID() = %d, want %d (the client's ID for the server)", got, want) + } + }) +}