From 1647302f6479a9e4e5f1bc5d32227aaf13086b33 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Thu, 9 Apr 2026 12:44:21 -0700 Subject: [PATCH 1/3] fix: gRPC appProtocol and EVM WebSocket routing Two networking fixes from expert validation: 1. gRPC: set appProtocol "kubernetes.io/h2c" on the gRPC ServicePort so Envoy uses HTTP/2 to the backend. Without this, Envoy defaults to HTTP/1.1 and gRPC requests fail. 2. EVM WebSocket: add a second rule to the EVM HTTPRoute with a header match for "Upgrade: websocket" routing to port 8546. seid serves JSON-RPC on 8545 and WebSocket on 8546 as separate ports. The first rule (default) routes HTTP to 8545, the second routes WebSocket upgrades to 8546. Both share the same hostname, matching the industry standard (Alchemy, Infura use one URL for HTTP + WS). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../controller/nodedeployment/networking.go | 56 +++++++++++---- .../nodedeployment/networking_test.go | 69 ++++++++++++++++++- 2 files changed, 111 insertions(+), 14 deletions(-) diff --git a/internal/controller/nodedeployment/networking.go b/internal/controller/nodedeployment/networking.go index d9c90d5..200297e 100644 --- a/internal/controller/nodedeployment/networking.go +++ b/internal/controller/nodedeployment/networking.go @@ -36,6 +36,7 @@ type effectiveRoute struct { Name string Hostnames []string Port int32 + WSPort int32 // non-zero when WebSocket requires a separate backend port } // hasExternalService returns true when the deployment has a LoadBalancer @@ -198,6 +199,10 @@ func portsForMode(mode seiconfig.NodeMode) []corev1.ServicePort { TargetPort: intstr.FromInt32(p.Port), Protocol: corev1.ProtocolTCP, } + if p.Name == "grpc" { + h2c := "kubernetes.io/h2c" + ports[i].AppProtocol = &h2c + } } return ports } @@ -242,11 +247,15 @@ func resolveEffectiveRoutes(group *seiv1alpha1.SeiNodeDeployment, domain string) if !isProtocolActiveForMode(proto.Prefix, activePorts) { continue } - routes = append(routes, effectiveRoute{ + er := effectiveRoute{ Name: fmt.Sprintf("%s-%s", group.Name, proto.Prefix), Hostnames: []string{fmt.Sprintf("%s.%s.%s", group.Name, proto.Prefix, domain)}, Port: proto.Port, - }) + } + if proto.Prefix == "evm" && activePorts["evm-ws"] { + er.WSPort = seiconfig.PortEVMWS + } + routes = append(routes, er) } return routes } @@ -331,6 +340,38 @@ func generateHTTPRoute(group *seiv1alpha1.SeiNodeDeployment, er effectiveRoute, "namespace": gatewayNamespace, } + rules := []any{ + map[string]any{ + "backendRefs": []any{ + map[string]any{ + "name": svcName, + "port": int64(er.Port), + }, + }, + }, + } + if er.WSPort != 0 { + rules = append(rules, map[string]any{ + "matches": []any{ + map[string]any{ + "headers": []any{ + map[string]any{ + "type": "Exact", + "name": "Upgrade", + "value": "websocket", + }, + }, + }, + }, + "backendRefs": []any{ + map[string]any{ + "name": svcName, + "port": int64(er.WSPort), + }, + }, + }) + } + route := &unstructured.Unstructured{ Object: map[string]any{ "apiVersion": "gateway.networking.k8s.io/v1", @@ -344,16 +385,7 @@ func generateHTTPRoute(group *seiv1alpha1.SeiNodeDeployment, er effectiveRoute, "spec": map[string]any{ "parentRefs": []any{parentRef}, "hostnames": hostnames, - "rules": []any{ - map[string]any{ - "backendRefs": []any{ - map[string]any{ - "name": svcName, - "port": int64(er.Port), - }, - }, - }, - }, + "rules": rules, }, }, } diff --git a/internal/controller/nodedeployment/networking_test.go b/internal/controller/nodedeployment/networking_test.go index cf42623..0e5bfdc 100644 --- a/internal/controller/nodedeployment/networking_test.go +++ b/internal/controller/nodedeployment/networking_test.go @@ -58,6 +58,24 @@ func TestGenerateExternalService_ValidatorModePorts(t *testing.T) { g.Expect(portNames).To(ConsistOf("p2p", "metrics")) } +func TestGenerateExternalService_GRPCAppProtocol(t *testing.T) { + g := NewWithT(t) + group := newTestGroup("pacific-1-rpc", "sei") + group.Spec.Networking = &seiv1alpha1.NetworkingConfig{ + Service: &seiv1alpha1.ExternalServiceConfig{}, + } + + svc := generateExternalService(group) + for _, p := range svc.Spec.Ports { + if p.Name == "grpc" { + g.Expect(p.AppProtocol).NotTo(BeNil()) + g.Expect(*p.AppProtocol).To(Equal("kubernetes.io/h2c")) + return + } + } + t.Fatal("grpc port not found") +} + func TestGenerateExternalService_Annotations(t *testing.T) { g := NewWithT(t) group := newTestGroup("archive-rpc", "sei") @@ -202,14 +220,17 @@ func TestGenerateHTTPRoute_EVMMerged(t *testing.T) { routes := resolveEffectiveRoutes(group, "prod.platform.sei.io") + var evmRoute effectiveRoute evmCount := 0 for _, r := range routes { if r.Name == "pacific-1-rpc-evm" { evmCount++ - g.Expect(r.Port).To(Equal(int32(8545))) + evmRoute = r } } g.Expect(evmCount).To(Equal(1), "expected exactly one merged EVM route") + g.Expect(evmRoute.Port).To(Equal(int32(8545))) + g.Expect(evmRoute.WSPort).To(Equal(int32(8546))) for _, r := range routes { g.Expect(r.Name).NotTo(ContainSubstring("evm-rpc")) @@ -217,6 +238,42 @@ func TestGenerateHTTPRoute_EVMMerged(t *testing.T) { } } +func TestGenerateHTTPRoute_EVMWebSocketRule(t *testing.T) { + g := NewWithT(t) + group := newTestGroup("pacific-1-rpc", "sei") + group.Spec.Networking = &seiv1alpha1.NetworkingConfig{ + Service: &seiv1alpha1.ExternalServiceConfig{}, + } + + routes := resolveEffectiveRoutes(group, "prod.platform.sei.io") + var evmRoute effectiveRoute + for _, r := range routes { + if r.Name == "pacific-1-rpc-evm" { + evmRoute = r + break + } + } + + httpRoute := generateHTTPRoute(group, evmRoute, "sei-gateway", "gateway") + spec := httpRoute.Object["spec"].(map[string]any) + rules := spec["rules"].([]any) + g.Expect(rules).To(HaveLen(2), "EVM route should have HTTP + WebSocket rules") + + httpRule := rules[0].(map[string]any) + httpBackend := httpRule["backendRefs"].([]any)[0].(map[string]any) + g.Expect(httpBackend["port"]).To(Equal(int64(8545))) + + wsRule := rules[1].(map[string]any) + wsMatches := wsRule["matches"].([]any) + wsHeaders := wsMatches[0].(map[string]any)["headers"].([]any) + wsHeader := wsHeaders[0].(map[string]any) + g.Expect(wsHeader["name"]).To(Equal("Upgrade")) + g.Expect(wsHeader["value"]).To(Equal("websocket")) + + wsBackend := wsRule["backendRefs"].([]any)[0].(map[string]any) + g.Expect(wsBackend["port"]).To(Equal(int64(8546))) +} + // --- HTTPRoute Generation --- func TestGenerateHTTPRoute_BasicFields(t *testing.T) { @@ -264,7 +321,15 @@ func TestGenerateHTTPRoute_BackendRef(t *testing.T) { } routes := resolveEffectiveRoutes(group, "prod.platform.sei.io") - route := generateHTTPRoute(group, routes[0], "sei-gateway", "istio-system") + // Use the RPC route (single rule, not EVM which has 2 rules) + var rpcRoute effectiveRoute + for _, r := range routes { + if r.Name == "archive-rpc-rpc" { + rpcRoute = r + break + } + } + route := generateHTTPRoute(group, rpcRoute, "sei-gateway", "istio-system") spec := route.Object["spec"].(map[string]any) rules := spec["rules"].([]any) From 23ccf0329b04f8c575909b30f6d30b1f5a81261d Mon Sep 17 00:00:00 2001 From: bdchatham Date: Thu, 9 Apr 2026 12:50:51 -0700 Subject: [PATCH 2/3] fix: set gRPC appProtocol on headless per-node Service Set appProtocol "kubernetes.io/h2c" on the gRPC port in the headless per-node Service (servicePorts), matching what's already set on the shared external Service. Ensures Istio/Envoy uses HTTP/2 for gRPC when routing directly to individual pods. Co-Authored-By: Claude Opus 4.6 (1M context) --- internal/controller/node/resources.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/controller/node/resources.go b/internal/controller/node/resources.go index 136c6a6..84c8b27 100644 --- a/internal/controller/node/resources.go +++ b/internal/controller/node/resources.go @@ -343,6 +343,10 @@ func servicePorts() []corev1.ServicePort { ports := make([]corev1.ServicePort, len(np)) for i, p := range np { ports[i] = corev1.ServicePort{Name: p.Name, Port: p.Port, TargetPort: intstr.FromInt32(p.Port), Protocol: corev1.ProtocolTCP} + if p.Name == "grpc" { + h2c := "kubernetes.io/h2c" + ports[i].AppProtocol = &h2c + } } return ports } From 74b1cc2d7a2328ca1f670f722e9132147dddf34d Mon Sep 17 00:00:00 2001 From: bdchatham Date: Thu, 9 Apr 2026 12:57:27 -0700 Subject: [PATCH 3/3] test: add edge case coverage and remove breadcrumb comment Add tests for: - Empty domain produces malformed hostnames (documents the invariant that platform.Validate() catches at startup) - Validator mode produces zero routes - Non-EVM routes have exactly one rule and zero WSPort Remove breadcrumb comment in BackendRef test. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../nodedeployment/networking_test.go | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/controller/nodedeployment/networking_test.go b/internal/controller/nodedeployment/networking_test.go index 0e5bfdc..58e1bd3 100644 --- a/internal/controller/nodedeployment/networking_test.go +++ b/internal/controller/nodedeployment/networking_test.go @@ -321,7 +321,6 @@ func TestGenerateHTTPRoute_BackendRef(t *testing.T) { } routes := resolveEffectiveRoutes(group, "prod.platform.sei.io") - // Use the RPC route (single rule, not EVM which has 2 rules) var rpcRoute effectiveRoute for _, r := range routes { if r.Name == "archive-rpc-rpc" { @@ -384,6 +383,53 @@ func TestIsProtocolActiveForMode_EVMMapping(t *testing.T) { g.Expect(isProtocolActiveForMode("grpc", activePorts)).To(BeFalse()) } +// --- Edge Cases --- + +func TestResolveEffectiveRoutes_EmptyDomain_MalformedHostnames(t *testing.T) { + g := NewWithT(t) + group := newTestGroup("pacific-1-rpc", "sei") + group.Spec.Networking = &seiv1alpha1.NetworkingConfig{ + Service: &seiv1alpha1.ExternalServiceConfig{}, + } + + routes := resolveEffectiveRoutes(group, "") + g.Expect(routes).To(HaveLen(4), "routes are still generated even with empty domain") + g.Expect(routes[0].Hostnames[0]).To(Equal("pacific-1-rpc.evm."), "empty domain produces trailing dot") +} + +func TestReconcileRoute_NoRoutesForValidatorMode(t *testing.T) { + g := NewWithT(t) + group := newTestGroup("pacific-1-val", "sei") + group.Spec.Template.Spec.Validator = &seiv1alpha1.ValidatorSpec{} + group.Spec.Networking = &seiv1alpha1.NetworkingConfig{ + Service: &seiv1alpha1.ExternalServiceConfig{}, + } + + routes := resolveEffectiveRoutes(group, "prod.platform.sei.io") + g.Expect(routes).To(BeEmpty(), "validator mode should produce zero routes") +} + +func TestGenerateHTTPRoute_NonEVMRoute_SingleRule(t *testing.T) { + g := NewWithT(t) + group := newTestGroup("pacific-1-rpc", "sei") + group.Spec.Networking = &seiv1alpha1.NetworkingConfig{ + Service: &seiv1alpha1.ExternalServiceConfig{}, + } + + routes := resolveEffectiveRoutes(group, "prod.platform.sei.io") + for _, r := range routes { + if r.Name == "pacific-1-rpc-rpc" { + httpRoute := generateHTTPRoute(group, r, "sei-gateway", "gateway") + spec := httpRoute.Object["spec"].(map[string]any) + rules := spec["rules"].([]any) + g.Expect(rules).To(HaveLen(1), "non-EVM routes should have exactly one rule") + g.Expect(r.WSPort).To(Equal(int32(0)), "non-EVM routes should have zero WSPort") + return + } + } + t.Fatal("rpc route not found") +} + // --- AuthorizationPolicy --- func TestGenerateAuthorizationPolicy_BasicStructure(t *testing.T) {