From 32361575c8a1995fdd49bc1bac9299d5e6d35110 Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 17 Nov 2026 11:31:15 +1100 Subject: [PATCH] docs(diagrams): apply the house diagram style Give the 22 Mermaid diagrams in docs/ the house frontmatter, HTML labels and role classes, so a component has one colour on every page. The enrichment diagram lists the reputation lookup first, which puts GeoIP on top. --- docs/README.md | 83 ++++++++--- docs/architecture.md | 223 ++++++++++++++++++++++++++---- docs/clickhouse/README.md | 76 ++++++++-- docs/clickhouse/ddl-directives.md | 74 ++++++++-- docs/clickhouse/insert-formats.md | 79 +++++++++-- docs/clickhouse/schema-cache.md | 79 +++++++++-- docs/clickhouse/tls.md | 74 ++++++++-- docs/configuration.md | 69 ++++++++- docs/deployment/publishing.md | 77 +++++++++-- docs/deployment/schemas.md | 75 ++++++++-- docs/performance/overview.md | 71 ++++++++-- docs/pipeline/README.md | 73 ++++++++-- docs/pipeline/capture-modes.md | 72 ++++++++-- docs/pipeline/enrichment.md | 70 ++++++++-- docs/pipeline/overview.md | 67 ++++++++- docs/pipeline/parallelism.md | 76 ++++++++-- docs/pipeline/routing.md | 78 +++++++++-- docs/transport/README.md | 77 +++++++++-- docs/transport/common-header.md | 74 ++++++++-- docs/transport/grpc.md | 77 +++++++++-- 20 files changed, 1407 insertions(+), 237 deletions(-) diff --git a/docs/README.md b/docs/README.md index 4838e2a..d35f84d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,37 +29,86 @@ Loads records from Kafka or a gRPC push into ClickHouse. Point it at a topic, gi Every record takes the same six steps whichever transport brought it, and normally leaves as a row in a batched ClickHouse insert or as a dead letter. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart TB accTitle: L3 Components -- dfe-loader accDescr: Records arrive from Kafka topics or gRPC senders. Inside the loader each record is parsed, routed to a table, has its columns extracted, is enriched and is buffered per table, and the buffers are inserted into ClickHouse in batches. Rows ClickHouse refuses for good and records that fail processing go to the DLQ. - kafka[("Kafka topics")] - grpc["gRPC senders"] + kafka[("Kafka topics")] + grpc["gRPC senders"] subgraph loader["dfe-loader"] direction LR - parse["Parse JSON"] --> route[["Route"]] --> extract["Extract columns"] --> enrich[["Enrich"]] --> buffer["Buffer per table"] --> insert[["Insert"]] + parse["Parse JSON"] --> route[["Route"]] --> extract["Extract columns"] --> enrich[["Enrich"]] --> buffer["Buffer per table"] --> insert[["Insert"]] end - clickhouse[("ClickHouse")] - dlq[("DLQ")] + clickhouse[("ClickHouse")] + dlq[("DLQ")] kafka -->|records| loader grpc -->|records| loader loader -->|rows| clickhouse loader -.->|dead letters| dlq - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef async fill:#F6E6EF,stroke:#A8517F,stroke-width:1px,color:#3D1529 - classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 - classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class kafka async - class grpc ext - class clickhouse store - class dlq error - class loader group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class kafka,dlq bus + class grpc ext + class parse,route,extract,enrich,buffer,insert part + class clickhouse store + style loader fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L3 Components -- dfe-loader. Double-bordered steps have a diagram each.* Up: top of dfe-loader. Down: [L4 Detail -- Loader table choice](pipeline/routing.md), [L4 Detail -- Enrichment](pipeline/enrichment.md), [L4 Detail -- Insert layer](clickhouse/README.md). diff --git a/docs/architecture.md b/docs/architecture.md index 0ec7a1a..726b8c5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,34 +22,91 @@ promote only schema columns, keep the full payload as a reference-counted The code sits in four layers. Orchestration calls the transform and the insert layers, and both call into the ClickHouse layer, never back up. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Loader layers accDescr: Orchestration holds the Orchestrator and the MessageProcessor. It calls the transform layer, the Router, the HeaderExtractor and the EnrichmentPipeline, and the buffer and insert layer, the BufferManager and the Inserter. Both call the ClickHouse layer, the query client, the schema cache, the dynamic insert and the ChClient. subgraph orchestration["1 Orchestration"] direction TB - orchestrator["Orchestrator"] ~~~ processor["MessageProcessor"] + orchestrator["Orchestrator"] ~~~ processor["MessageProcessor"] end subgraph transform["2 Transform"] direction TB - router["Router"] ~~~ extractor["HeaderExtractor"] ~~~ enrichment["EnrichmentPipeline"] + router["Router"] ~~~ extractor["HeaderExtractor"] ~~~ enrichment["EnrichmentPipeline"] end subgraph insertlayer["3 Buffer and insert"] direction TB - buffers["BufferManager"] ~~~ inserter["Inserter"] + buffers["BufferManager"] ~~~ inserter["Inserter"] end subgraph chlayer["4 ClickHouse layer"] direction TB - queryclient["ClickHouseQueryClient"] ~~~ schemacache["SchemaCache"] ~~~ dynamic["dfe_clickhouse
dynamic"] ~~~ chclient["ChClient"] + queryclient["ClickHouseQueryClient"] ~~~ schemacache["SchemaCache"] ~~~ dynamic["dfe_clickhouse
dynamic"] ~~~ chclient["ChClient"] end orchestration --> transform orchestration --> insertlayer transform --> chlayer insertlayer --> chlayer - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class orchestration,transform,insertlayer,chlayer group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class orchestrator,processor,router,extractor,enrichment,buffers,inserter,queryclient,schemacache,dynamic,chclient part + style orchestration fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 + style transform fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 + style insertlayer fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 + style chlayer fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Loader layers. Each layer calls only the layers to its right.* Up: [L3 Components -- dfe-loader](README.md). @@ -77,27 +134,80 @@ That dynamic layer comes from `dfe-clickhouse`'s `dynamic` module, on top of the `Client`. The rows are encoded against the schema read at runtime, then sent as RowBinary through the official client over HTTP, or as Native blocks through `dfe_clickhouse::TcpClient` over native. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Dynamic insert accDescr: The Inserter hands rows to dfe_clickhouse's dynamic module. It reads the table's schema from system.columns, encodes each row against it with DynamicRow, and DynamicInsert sends the RowBinary through the clickhouse crate's insert_formatted_with. - inserter["Inserter"] + inserter["Inserter"] subgraph dynamic["dfe_clickhouse::dynamic"] direction LR - schema["fetch_dynamic_schema"] --> encode["DynamicRow"] --> send["DynamicInsert"] + schema["fetch_dynamic_schema"] --> encode["DynamicRow"] --> send["DynamicInsert"] end - client["clickhouse::Client
insert_formatted_with"] + client["clickhouse::Client
insert_formatted_with"] inserter -->|rows| dynamic dynamic -->|RowBinary| client - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class client ext - class dynamic group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class inserter,schema,encode,send part + class client ext + style dynamic fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 2. L4 Detail -- Dynamic insert, over HTTP. The dashed box is the crates.io client.* Up: [L3 Components -- dfe-loader](README.md). @@ -111,17 +221,62 @@ Over HTTP every insert goes through `insert_formatted_with`: `FORMAT RowBinaryWi A failed RowBinary insert ends one of three ways: held for a later flush, retried against a fresh schema, or split so only its bad rows reach the DLQ. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Insert outcome accDescr: A successful insert lets its offsets join the flush cycle's commit. A timeout, a dropped connection or a server code on the retry list holds the batch. Any other refusal against a changed table is retried against the current schema, and one against an unchanged table goes to salvage and the DLQ. - ok{"Insert
answered ok?"} - retryable{"Retryable
failure?"} - changed{"Table
changed?"} - commit(["Offsets commit"]) - held(["Batch held"]) - retry(["Retry on
fresh schema"]) - salvage(["Salvage
to DLQ"]) + ok{"Insert
answered ok?
"} + retryable{"Retryable
failure?
"} + changed{"Table
changed?
"} + commit(["Offsets commit"]) + held(["Batch held"]) + retry(["Retry on
fresh schema
"]) + salvage(["Salvage
to DLQ
"]) ok -->|no| retryable ok -->|yes| commit @@ -129,12 +284,20 @@ flowchart LR retryable -->|yes| held changed -->|yes| retry changed -->|no| salvage - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 - classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 - class ok,retryable,changed decision - class salvage error + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class ok,retryable,changed decision + class commit,held,retry part + class salvage error ``` *Figure 3. L4 Detail -- Insert outcome. Orange is the end that dead-letters rows.* Up: [L3 Components -- dfe-loader](README.md). diff --git a/docs/clickhouse/README.md b/docs/clickhouse/README.md index 3ddff52..867fca7 100644 --- a/docs/clickhouse/README.md +++ b/docs/clickhouse/README.md @@ -17,31 +17,83 @@ straight to binary -- no compile-time `Row` types. The inserter takes each table's batch from its buffer and sends it through the dynamic insert and the client to ClickHouse, and the rows ClickHouse rejects for good are the ones that leave for the DLQ. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Insert layer accDescr: The per-table buffer hands batches to the insert layer. Inside, the Inserter passes rows to dfe_clickhouse's DynamicInsert, which writes them through the ChClient. Rows land in ClickHouse, and rows ClickHouse refuses for good go to the DLQ. - buffer["Buffer per table"] + buffer["Buffer per table"] subgraph insert["Insert"] direction LR - inserter["Inserter"] --> dynamic["DynamicInsert"] --> chclient["ChClient"] + inserter["Inserter"] --> dynamic["DynamicInsert"] --> chclient["ChClient"] end - clickhouse[("ClickHouse")] - dlq[("DLQ")] + clickhouse[("ClickHouse")] + dlq[("DLQ")] buffer -->|batches| insert insert -->|rows| clickhouse insert -.->|dead letters| dlq - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class clickhouse store - class dlq error - class insert group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class buffer,inserter,dynamic,chclient part + class clickhouse store + class dlq bus + style insert fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Insert layer. The dotted arrow is the salvage path.* Up: [L3 Components -- dfe-loader](../README.md). diff --git a/docs/clickhouse/ddl-directives.md b/docs/clickhouse/ddl-directives.md index 9167c12..4e4cdda 100644 --- a/docs/clickhouse/ddl-directives.md +++ b/docs/clickhouse/ddl-directives.md @@ -37,26 +37,80 @@ capture interacts with `_json` and `_raw`, see capture modes in A column reads the paths its `@source` or `@renamed` directive lists, else the field with the column's name, and `@computed` or `@default` fills it when no field is found. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Column directive resolution accDescr: A column marked skip is left out of the insert. Any other column reads the paths its source or renamed directive lists, else the field named like the column. A field found is written. A column with no field found takes its computed or default value. - skip{"@skip?"} - read["Listed paths
or column name"] - left(["Left out"]) - found{"Field
found?"} - value(["Field's value"]) - fill(["@computed or
@default value"]) + skip{"@skip?"} + read["Listed paths
or column name
"] + left(["Left out"]) + found{"Field
found?
"} + value(["Field's value"]) + fill(["@computed or
@default value
"]) skip -->|no| read skip -->|yes| left read --> found found -->|yes| value found -->|no| fill - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 - class skip,found decision + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class skip,found decision + class read,left,value,fill part ``` *Figure 1. L4 Detail -- Column directive resolution. `_org_id` and `_timestamp` read fixed fields and skip this path.* Up: [ClickHouse insert layer](README.md). diff --git a/docs/clickhouse/insert-formats.md b/docs/clickhouse/insert-formats.md index 70f8891..8756f78 100644 --- a/docs/clickhouse/insert-formats.md +++ b/docs/clickhouse/insert-formats.md @@ -15,17 +15,62 @@ dfe-loader has two knobs for how rows reach ClickHouse: `clickhouse.protocol` (h Three of the four pairs insert, and JSONEachRow over the native protocol stops the pipeline at start. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Insert format and protocol accDescr: A rowbinary insert goes as RowBinaryWithNamesAndTypes over HTTP or as Native blocks over the native protocol. A jsoneachrow insert goes as JSONEachRow over HTTP, and the pipeline refuses it at start over native. - format{"insert_format?"} - binproto{"protocol?"} - jsonproto{"protocol?"} - rowbinary(["RowBinaryWithNamesAndTypes
over HTTP"]) - native(["Native blocks
over TCP"]) - jsonrows(["JSONEachRow
over HTTP"]) - refused(["Refused at
pipeline start"]) + format{"insert_format?"} + binproto{"protocol?"} + jsonproto{"protocol?"} + rowbinary(["RowBinaryWithNamesAndTypes
over HTTP
"]) + native(["Native blocks
over TCP
"]) + jsonrows(["JSONEachRow
over HTTP
"]) + refused(["Refused at
pipeline start
"]) format -->|rowbinary| binproto format -->|jsoneachrow| jsonproto @@ -33,12 +78,20 @@ flowchart LR binproto -->|native| native jsonproto -->|http| jsonrows jsonproto -->|native| refused - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 - classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 - class format,binproto,jsonproto decision - class refused error + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class format,binproto,jsonproto decision + class rowbinary,native,jsonrows part + class refused error ``` *Figure 1. L4 Detail -- Insert format and protocol. Orange is the pair the loader rejects.* Up: [ClickHouse insert layer](README.md). diff --git a/docs/clickhouse/schema-cache.md b/docs/clickhouse/schema-cache.md index 8971e83..0078f11 100644 --- a/docs/clickhouse/schema-cache.md +++ b/docs/clickhouse/schema-cache.md @@ -15,17 +15,62 @@ The dynamic insert path needs to know a table's columns and their types before i A row is encoded against the cached schema until it expires, and with `schema.refresh_on_error` on, the default, a drift error drops the entry so the next attempt reads the table again. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Schema cache read and recovery accDescr: An insert uses the cached schema when it is fresh, else reads system.columns. The rows are encoded and inserted. A drift error invalidates the table's entry so the retry reads the schema again, and any other answer ends the attempt. - fresh{"Cached and
fresh?"} - cached["Cached schema"] - fetch["Read columns"] - insert["Encode, insert"] - drift{"Drift
error?"} - done(["Attempt ends"]) - invalidate(["Invalidate"]) + fresh{"Cached and
fresh?
"} + cached["Cached schema"] + fetch["Read columns"] + insert["Encode, insert"] + drift{"Drift
error?
"} + done(["Attempt ends"]) + invalidate(["Invalidate"]) fresh -->|yes| cached fresh -->|no| fetch @@ -34,12 +79,20 @@ flowchart LR insert --> drift drift -->|no| done drift -->|yes| invalidate - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 - class fresh,drift decision - class cached store + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class fresh,drift decision + class cached store + class fetch,insert,done,invalidate part ``` *Figure 1. L4 Detail -- Schema cache read and recovery, with `schema.refresh_on_error` on.* Up: [ClickHouse insert layer](README.md). diff --git a/docs/clickhouse/tls.md b/docs/clickhouse/tls.md index 96d3169..c20798b 100644 --- a/docs/clickhouse/tls.md +++ b/docs/clickhouse/tls.md @@ -15,26 +15,78 @@ With TLS on the loader speaks HTTPS (`clickhouse.protocol: http`, usually 8443) Turning TLS on is one key, and trust comes from the system store alone, so a private CA is added there rather than set in config. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- TLS trust accDescr: clickhouse.tls.enabled turns TLS on. The system trust store, the image bundle or the file SSL_CERT_FILE or SSL_CERT_DIR names, supplies the CA certificates. The loader verifies the server's certificate chain and host name against them before it opens HTTPS or native TLS to ClickHouse. - enabled["clickhouse.tls.enabled"] - store["System trust store"] - verify["Verify chain
and host name"] - conn["HTTPS or
native TLS"] - clickhouse[("ClickHouse")] + enabled["clickhouse.tls.enabled"] + store["System trust store"] + verify["Verify chain
and host name
"] + conn["HTTPS or
native TLS
"] + clickhouse[("ClickHouse")] enabled -->|on| verify store -->|CA certs| verify verify -->|verified| conn conn --> clickhouse - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - class verify core - class clickhouse store + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class enabled,verify,conn part + class store,clickhouse store ``` *Figure 1. L4 Detail -- TLS trust. The store is the image bundle, or what `SSL_CERT_FILE` or `SSL_CERT_DIR` names.* Up: [ClickHouse insert layer](README.md). diff --git a/docs/configuration.md b/docs/configuration.md index 2f3b647..8492138 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,19 +13,74 @@ dfe-loader reads its config in four layers, each applied over the one before, so a flat ENV variable beats a nested one, which beats the file, which beats the defaults. Every setting has a safe default, so a loader started with no config file boots. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Config cascade accDescr: The loader starts from its hard-coded defaults, applies the YAML file over them, then the nested ENV variables, then the flat ENV variables, and validates the result. - defaults["Hard-coded
defaults"] - file["YAML file"] - nested["Nested ENV"] - flat["Flat ENV"] - config(["Validated config"]) + defaults["Hard-coded
defaults
"] + file["YAML file"] + nested["Nested ENV"] + flat["Flat ENV"] + config(["Validated config"]) defaults --> file --> nested --> flat --> config - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class defaults,file,nested,flat,config part ``` *Figure 1. L4 Detail -- Config cascade. Each layer overrides the one to its left.* Up: [dfe-loader docs](README.md). diff --git a/docs/deployment/publishing.md b/docs/deployment/publishing.md index 266861a..a1f97c5 100644 --- a/docs/deployment/publishing.md +++ b/docs/deployment/publishing.md @@ -18,17 +18,62 @@ Both artefacts go to GHCR. The image is `ghcr.io/hyperi-io/dfe-loader` and the c One release build feeds both artefacts: the image wraps the built binary, and the chart comes from the deployment contract that binary emits. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Release artefacts accDescr: A release request starts a CI build of linux amd64 and arm64 binaries. The Dockerfile wraps the binaries into an image, which is pushed with its version tags. The binary's generate-artefacts emits the deployment contract, from which a thin chart is assembled and pushed. Both land in GHCR. - tag(["Release request"]) - build["Build binaries"] - image["Build image"] - contract["Emit contract"] - pushimage["Push image tags"] - chart["Assemble and
push chart"] - ghcr[("GHCR")] + tag(["Release request"]) + build["Build binaries"] + image["Build image"] + contract["Emit contract"] + pushimage["Push image tags"] + chart["Assemble and
push chart
"] + ghcr[("GHCR")] tag --> build build --> image @@ -37,10 +82,20 @@ flowchart LR contract --> chart pushimage --> ghcr chart --> ghcr - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - class ghcr store + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class tag ext + class build,image,contract,pushimage,chart part + class ghcr store ``` *Figure 1. L4 Detail -- Release artefacts. A release request is a merge with a `Release: true` trailer or a release dispatch.* Up: [Deployment](README.md). diff --git a/docs/deployment/schemas.md b/docs/deployment/schemas.md index 0f2390a..1ce43f2 100644 --- a/docs/deployment/schemas.md +++ b/docs/deployment/schemas.md @@ -21,26 +21,77 @@ loader by two steps, and nothing here needs a checkout of it. The schema reaches the loader through ClickHouse alone: dfe-engine applies the dfe-schemas manifest, and the loader reads the tables that result. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Where a table's schema comes from accDescr: dfe-schemas holds the YAML definitions. dfe-engine applies the manifest at boot with CREATE and ALTER statements. The loader reads the result from ClickHouse system.columns into its schema cache, again each time an entry expires. - yaml["dfe-schemas
YAML"] - engine["dfe-engine"] - clickhouse[("ClickHouse
system.columns")] - loader["dfe-loader
SchemaCache"] + yaml["dfe-schemas
YAML"] + engine["dfe-engine"] + clickhouse[("ClickHouse
system.columns")] + loader["dfe-loader
SchemaCache"] yaml -->|manifest| engine engine -->|CREATE, ALTER| clickhouse clickhouse -->|columns| loader - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 - class yaml,engine ext - class clickhouse store - class loader core + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class yaml,clickhouse store + class engine control + class loader load ``` *Figure 1. L4 Detail -- Where a table's schema comes from. Dashed boxes are other DFE repos.* Up: [Deployment](README.md). diff --git a/docs/performance/overview.md b/docs/performance/overview.md index 680f4c2..f7535f2 100644 --- a/docs/performance/overview.md +++ b/docs/performance/overview.md @@ -15,26 +15,81 @@ Performance work splits in two: build-time optimisations (allocator, LTO, PGO, B The build-time choices go into the binary once, and the batch settings tune the running loader without a rebuild. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Where performance is set accDescr: hyperi-ci builds the binary with jemalloc, LTO, and on release PGO then BOLT. The batch flush settings in config tune the running loader. subgraph build["Build-time, hyperi-ci"] direction LR - jemalloc["jemalloc"] ~~~ lto["LTO"] ~~~ pgo["PGO"] ~~~ bolt["BOLT"] + jemalloc["jemalloc"] ~~~ lto["LTO"] ~~~ pgo["PGO"] ~~~ bolt["BOLT"] end - binary["Optimised binary"] - batch["Batch settings
in config"] - loader(["Running loader"]) + binary["Optimised binary"] + batch["Batch settings
in config
"] + loader(["Running loader"]) build --> binary binary --> loader batch --> loader - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class build group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class jemalloc,lto,pgo,bolt,binary,batch part + class loader load + style build fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Where performance is set. BOLT runs on the PGO build.* Up: [Performance](README.md). diff --git a/docs/pipeline/README.md b/docs/pipeline/README.md index c345237..d58759e 100644 --- a/docs/pipeline/README.md +++ b/docs/pipeline/README.md @@ -13,29 +13,82 @@ The hot path: bytes in, ClickHouse row out. The pass runs per message, and the batching is per table: each table has a separate buffer, flushed as INSERTs of its rows. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Per-message pass, per-table batches accDescr: Each message goes through one pass that parses, routes, extracts and enriches it. The row lands in the buffer of the table it was routed to, such as dfe.main or dfe.auth. Each table's buffer flushes as INSERTs into ClickHouse. - pass["Per-message pass"] + pass["Per-message pass"] subgraph buffers["Per-table buffers"] direction TB - main["dfe.main"] ~~~ auth["dfe.auth"] ~~~ other["Each other table"] + main["dfe.main"] ~~~ auth["dfe.auth"] ~~~ other["Each other table"] end - insert["INSERTs
per table"] - clickhouse[("ClickHouse")] + insert["INSERTs
per table
"] + clickhouse[("ClickHouse")] pass -->|rows| buffers buffers -->|batches| insert insert --> clickhouse - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class clickhouse store - class buffers group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class pass,main,auth,other,insert part + class clickhouse store + style buffers fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Per-message pass, per-table batches. Each buffer flushes on a row, byte or age trigger.* Up: [L3 Components -- dfe-loader](../README.md). diff --git a/docs/pipeline/capture-modes.md b/docs/pipeline/capture-modes.md index 5ec1d09..be84689 100644 --- a/docs/pipeline/capture-modes.md +++ b/docs/pipeline/capture-modes.md @@ -38,24 +38,78 @@ A directive on the `_json` column itself, such as `@source: record`, makes `_jso Capture mode is resolved per table: the table's DDL tag wins, then a per-table config entry, then the global default. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Capture mode precedence accDescr: A table whose COMMENT carries a capture_mode tag uses that mode. Otherwise a metadata.table_capture_modes entry for the table decides. Otherwise metadata.capture_mode applies. - tag{"@capture_mode
tag?"} - entry{"table_capture_modes
entry?"} - tagmode(["The tag's mode"]) - entrymode(["The entry's mode"]) - global(["metadata.capture_mode"]) + tag{"@capture_mode
tag?
"} + entry{"table_capture_modes
entry?
"} + tagmode(["The tag's mode"]) + entrymode(["The entry's mode"]) + global(["metadata.capture_mode"]) tag -->|no| entry tag -->|yes| tagmode entry -->|yes| entrymode entry -->|no| global - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 - class tag,entry decision + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class tag,entry decision + class tagmode,entrymode,global part ``` *Figure 1. L4 Detail -- Capture mode precedence. The first question answered yes decides.* Up: [Pipeline](README.md). diff --git a/docs/pipeline/enrichment.md b/docs/pipeline/enrichment.md index 51d5712..ade746c 100644 --- a/docs/pipeline/enrichment.md +++ b/docs/pipeline/enrichment.md @@ -17,26 +17,80 @@ Enrichment adds columns to a row from the first IP field it has (`enrichment.ip_ The lookups key on the first IP field present, and risk scoring weighs what GeoIP and reputation found, so it adds nothing by itself. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Enrichment accDescr: The row from column extraction is looked up by its first IP field in GeoIP and in the reputation blocklists. Risk scoring weighs both results. The row goes on to its table's buffer with the geo, rep and risk columns added. - extract["Extract columns"] + extract["Extract columns"] subgraph lookups["Enrich"] direction LR - geo["GeoIP"] --> risk["Risk scoring"] - rep["Reputation
blocklists"] --> risk + rep["Reputation
blocklists
"] --> risk["Risk scoring"] + geo["GeoIP"] --> risk end - buffer["Buffer per table"] + buffer["Buffer per table"] extract -->|row| lookups lookups -->|row| buffer - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class lookups group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class extract,geo,rep,risk,buffer part + style lookups fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Enrichment. Each provider is off until its setting turns it on.* Up: [L3 Components -- dfe-loader](../README.md). diff --git a/docs/pipeline/overview.md b/docs/pipeline/overview.md index 64e3f36..641afb4 100644 --- a/docs/pipeline/overview.md +++ b/docs/pipeline/overview.md @@ -15,23 +15,78 @@ A message becomes a ClickHouse row in one pass. The hot path does the cheap work The pass is eight steps in two stages: the first decides which table a record goes to, and the second builds its row and writes it. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart TB accTitle: L4 Detail -- Loader pipeline accDescr: Two stages run in order. Read and route takes message bytes, splits batched messages, parses JSON and routes each record to a table. Extract and write extracts the header and schema columns, enriches, buffers per table, then inserts and commits offsets. subgraph s1["1 Read and route"] direction LR - bytes(["Message bytes"]) --> split["Split batches"] --> parse["Parse JSON"] --> route["Route to table"] + bytes(["Message bytes"]) --> split["Split batches"] --> parse["Parse JSON"] --> route["Route to table"] end subgraph s2["2 Extract and write"] direction LR - extract["Extract columns"] --> enrich["Enrich"] --> buffer["Buffer per table"] --> insert(["Insert and commit"]) + extract["Extract columns"] --> enrich["Enrich"] --> buffer["Buffer per table"] --> insert(["Insert and commit"]) end s1 --> s2 - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class s1,s2 group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class bytes,split,parse,route,extract,enrich,buffer,insert part + style s1 fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 + style s2 fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Loader pipeline. Read each stage left to right, then down.* Up: [Pipeline](README.md). diff --git a/docs/pipeline/parallelism.md b/docs/pipeline/parallelism.md index fffbbc4..84680f3 100644 --- a/docs/pipeline/parallelism.md +++ b/docs/pipeline/parallelism.md @@ -13,32 +13,86 @@ Each received batch goes through two phases in `src/pipeline/orchestrator.rs`: per-message work with no shared mutable state runs in parallel, and everything that mutates state runs after it, in one thread. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Batch phases accDescr: A received batch passes the BatchEngine's pre-route filters when a routing field is set. The worker pool runs MessageProcessor process on each message in parallel. The BatchCoordinator then walks the results in one thread, pushing each Ok row into its table's buffer, parking a message that waits on its schema and turning any other error into a dead letter. - recv(["Received batch"]) - preroute["Pre-route filters"] + recv(["Received batch"]) + preroute["Pre-route filters"] subgraph parallel["Parallel"] - process["MessageProcessor
process"] + process["MessageProcessor
process
"] end subgraph sequential["Sequential"] - push["Buffer push"] - dead["Dead letter"] + push["Buffer push"] + dead["Dead letter"] end recv --> preroute preroute --> process process -->|Ok| push process -.->|Err| dead - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class dead error - class parallel,sequential group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class recv,preroute,process,push part + class dead error + style parallel fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 + style sequential fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Batch phases. Pre-route runs only when the BatchEngine has a `routing_field`.* Up: [Pipeline](README.md). diff --git a/docs/pipeline/routing.md b/docs/pipeline/routing.md index 78110df..abd156d 100644 --- a/docs/pipeline/routing.md +++ b/docs/pipeline/routing.md @@ -15,17 +15,62 @@ Routing picks the `db.table` a message lands in, before any flattening, from fie The table is the first of four answers that holds: a `dlq_load` topic's table, a `table_fields` value, the source a topic names, then `default_table`. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Loader table choice accDescr: A message read from a dlq_load topic goes to dlq_load.table. Otherwise a table_fields value names the table. Otherwise a topic named for a source gives that source's table. Otherwise the message goes to default_table. - pin{"dlq_load
topic?"} - field{"table_fields
value?"} - topic{"Source
topic?"} - replay(["dlq_load.table"]) - value(["Value's table"]) - source(["Source's table"]) - fallback(["default_table"]) + pin{"dlq_load
topic?
"} + field{"table_fields
value?
"} + topic{"Source
topic?
"} + replay(["dlq_load.table"]) + value(["Value's table"]) + source(["Source's table"]) + fallback(["default_table"]) pin -->|no| field pin -->|yes| replay @@ -33,12 +78,19 @@ flowchart LR field -->|yes| value topic -->|yes| source topic -->|no| fallback - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 - class pin,field,topic decision - class source,fallback,value,replay store + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class pin,field,topic decision + class replay,value,source,fallback store ``` *Figure 1. L4 Detail -- Loader table choice. Green ends are tables.* Up: [L3 Components -- dfe-loader](../README.md). diff --git a/docs/transport/README.md b/docs/transport/README.md index a9faea9..f917635 100644 --- a/docs/transport/README.md +++ b/docs/transport/README.md @@ -15,31 +15,84 @@ The transports, Kafka and gRPC, are scalo's. The loader does not implement them: Each transport has an adapter, and both adapters hand the pipeline the same `KafkaMessage`, so nothing after them knows which transport a record came on. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Transport adapters accDescr: scalo's KafkaTransport feeds the loader's TransportAdapter and scalo's GrpcTransport feeds the GrpcTransportAdapter. Both adapters turn each record into a KafkaMessage, which goes to the JSON parse step. subgraph scalo["scalo transports"] - kafka["KafkaTransport"] - grpc["GrpcTransport"] + kafka["KafkaTransport"] + grpc["GrpcTransport"] end - kafkaadapter["TransportAdapter"] - grpcadapter["GrpcTransportAdapter"] - message["KafkaMessage"] - parse["Parse JSON"] + kafkaadapter["TransportAdapter"] + grpcadapter["GrpcTransportAdapter"] + message["KafkaMessage"] + parse["Parse JSON"] kafka --> kafkaadapter grpc --> grpcadapter kafkaadapter --> message grpcadapter --> message message --> parse - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class kafka,grpc ext - class scalo group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class kafka,grpc ext + class kafkaadapter,grpcadapter,message,parse part + style scalo fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Transport adapters. Dashed boxes are scalo's.* Up: [L3 Components -- dfe-loader](../README.md). diff --git a/docs/transport/common-header.md b/docs/transport/common-header.md index 131ed08..3d67ba5 100644 --- a/docs/transport/common-header.md +++ b/docs/transport/common-header.md @@ -19,30 +19,82 @@ The header's DDL belongs to dfe-schemas: its column types, ORDER BY, partitionin In the `timeseries` profile the landing table holds the common header and nothing else, and every other table holds the same header plus data columns. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- Common header against a complete table schema accDescr: The landing table dfe.main holds the common header alone. Every other table, such as dfe.auth, holds the same common header plus its data columns. subgraph landing["Landing table dfe.main"] - h1["Common header
_timestamp, _org_id, ..."] + h1["Common header
_timestamp, _org_id, ..."] end subgraph other["Non-default table dfe.auth"] direction LR - h2["Common header
_timestamp, _org_id, ..."] - d2["Data columns
user_id, action, ip, ..."] + h2["Common header
_timestamp, _org_id, ..."] + d2["Data columns
user_id, action, ip, ..."] h2 ~~~ d2 end landing ~~~ other - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class h1,h2 core - class d2 store - class landing,other group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class h1,h2 part + class d2 store + style landing fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 + style other fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- Common header against a complete table schema. The `timeseries` header (blue) is the same in each table, and non-default tables add data columns (green).* Up: [Transport](README.md). diff --git a/docs/transport/grpc.md b/docs/transport/grpc.md index 3002575..a3083d7 100644 --- a/docs/transport/grpc.md +++ b/docs/transport/grpc.md @@ -19,31 +19,82 @@ The wire is gRPC (Protobuf over HTTP/2) through `tonic`, at the version scalo-rs With acknowledgements on, the default, a Push is answered once its rows are placed, or with `UNAVAILABLE` when they cannot be, so the answer is the acknowledgement. ```mermaid +--- +config: + theme: base + look: neo + fontFamily: Inter, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif + flowchart: + curve: linear + nodeSpacing: 40 + rankSpacing: 64 + padding: 18 + wrappingWidth: 280 + htmlLabels: true + elk: + preset: modelOrder + themeVariables: + fontSize: 15px + lineColor: "#64748B" + primaryTextColor: "#0F172A" + clusterBorder: "#0072B2" + edgeLabelBackground: "#FFFFFF" + themeCSS: >- + p{margin:0} + .node[data-look] rect,.node[data-look] path,.node[data-look] polygon,.node[data-look] .outer-path{filter:drop-shadow(0 1px 2px rgba(15,23,42,.10)) drop-shadow(0 6px 14px rgba(15,23,42,.10))} + .node rect{rx:10px;ry:10px} + .cluster rect{rx:16px;ry:16px} + .node .label div{min-width:150px} + .node.decision .label div{min-width:0} + .cluster[data-look] rect{filter:none} + .nodeLabel{line-height:1.35} + .nodeLabel b{font-weight:600;font-size:15px;letter-spacing:-0.01em} + .nodeLabel span{font-size:12.5px;font-weight:400;color:#334155} + .cluster-label span.nodeLabel{display:inline-block;background:#FFFFFF;color:#0B2A3F;border:1.5px solid #0072B2;border-radius:999px;padding:2px 12px;font-size:11.5px;font-weight:700;letter-spacing:.09em;text-transform:uppercase} + .flowchart-link{stroke-width:1.75px} + .labelBkg{background:transparent!important} + span.edgeLabel{display:inline-block;background:#FFFFFF!important;color:#1E293B;border:1px solid #94A3B8;border-radius:999px;padding:2px 10px;font-size:12px;font-weight:500} + span.edgeLabel p{background:transparent!important} + .label-icon{width:20px;height:20px;vertical-align:-4px;margin-right:6px} + .ingest .label-icon{color:#0E8A7D} + .xform .label-icon{color:#7B4FB0} + .load .label-icon{color:#0072B2} + .bus .label-icon{color:#A8517F} + .store .label-icon{color:#00785A} + .ui .label-icon{color:#D9480F} + .ext .label-icon{color:#64748B} +--- flowchart LR accTitle: L4 Detail -- gRPC path accDescr: dfe-receiver pushes to the loader directly or through a transform, which is a gRPC server and client. The loader writes the rows to ClickHouse and answers each Push with a PushResponse. subgraph senders["gRPC senders"] direction LR - receiver["dfe-receiver"] -->|Push| transform["Transform"] + receiver["dfe-receiver"] -->|Push| transform["Transform"] end - loader["dfe-loader"] - clickhouse[("ClickHouse")] + loader["dfe-loader"] + clickhouse[("ClickHouse")] senders -->|Push| loader loader -->|rows| clickhouse loader -.->|PushResponse| senders - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class receiver,transform ext - class loader core - class clickhouse store - class senders group + classDef ext fill:#F4F6F8,stroke:#94A3B8,stroke-width:1.5px,stroke-dasharray:5 4,color:#0F172A + classDef ingest fill:#FFFFFF,stroke:#0E8A7D,stroke-width:3px,color:#0F172A + classDef xform fill:#FFFFFF,stroke:#7B4FB0,stroke-width:3px,color:#0F172A + classDef load fill:#FFFFFF,stroke:#0072B2,stroke-width:3px,color:#0F172A + classDef bus fill:#FFFFFF,stroke:#A8517F,stroke-width:3px,color:#0F172A + classDef store fill:#FFFFFF,stroke:#00785A,stroke-width:3px,color:#0F172A + classDef ui fill:#FFFFFF,stroke:#D9480F,stroke-width:3px,color:#0F172A + classDef decision fill:#FFFFFF,stroke:#9A6700,stroke-width:3px,color:#0F172A + classDef error fill:#FFFFFF,stroke:#C2410C,stroke-width:3px,color:#0F172A + classDef control fill:#FFFFFF,stroke:#334155,stroke-width:3px,color:#0F172A + classDef part fill:#FFFFFF,stroke:#94A3B8,stroke-width:1.5px,color:#0F172A + class receiver ingest + class transform xform + class loader load + class clickhouse store + style senders fill:none,stroke:#0072B2,stroke-width:1.5px,stroke-dasharray:6 4 ``` *Figure 1. L4 Detail -- gRPC path. The dotted arrow is the answer.* Up: [Transport](README.md).