Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -191,12 +191,12 @@ clickhousectl local server dotenv --user default --password secret --database my
When you also need a local Postgres alongside ClickHouse — e.g. for testing CDC pipelines or ingesting from Postgres — use `local postgres`. Each instance is keyed on `(name, major version)` so the same name can host multiple Postgres majors with isolated data: data lives at `.clickhouse/servers/<name>-pg<major>/data/`, metadata at `.clickhouse/servers/<name>-pg<major>.json`, and the container is `clickhousectl-pg-<name>-<major>`. ClickHouse paths (`<name>/data/`, `<name>.json`) stay separate, so a name can be used by both engines. Requires Docker to be installed and running.

```bash
# Pre-pull a Postgres image (optional; start will pull on demand). Supported: 16, 17, 18 (and any sub-tag like 16-alpine, 17.0, 18-bookworm).
clickhousectl local install postgres@16
# Pre-pull a Postgres image (optional; start will pull on demand). Supported: 17, 18 (and any sub-tag like 17-alpine, 17.0, 18-bookworm).
clickhousectl local install postgres@17

# Start a Postgres instance (defaults: postgres:18, port 5432, user "postgres", db "postgres")
clickhousectl local postgres start
clickhousectl local postgres start --name dev --version 16 --port 5433
clickhousectl local postgres start --name dev --version 17 --port 5433
clickhousectl local postgres start --user app --password s3cret --database myapp
clickhousectl local postgres start -e POSTGRES_INITDB_ARGS=--data-checksums

Expand All @@ -212,7 +212,7 @@ clickhousectl local postgres dotenv --name dev

# Stop / remove. Pass --version when more than one major shares a name.
clickhousectl local postgres stop dev
clickhousectl local postgres stop dev --version 16 # disambiguate
clickhousectl local postgres stop dev --version 17 # disambiguate
clickhousectl local postgres remove dev
```

Expand Down Expand Up @@ -540,7 +540,7 @@ clickhousectl cloud postgres switchover <pg-id>
| `--size` | Instance size, e.g. `m7i.2xlarge` (required; server-validated) |
| `--storage-gb` | Storage size in GB (required) |
| `--provider` | Cloud provider (default: `aws`) |
| `--pg-version` | Postgres major version: `18`, `17`, `16` |
| `--pg-version` | Postgres major version: `18`, `17` |
| `--ha-type` | High-availability: `none`, `async`, `sync` |
| `--tag` | Resource tag `key` or `key=value` (repeatable) |
| `--pg-config-file` | Path to JSON file with a `PgConfig` object |
Expand Down
3 changes: 0 additions & 3 deletions crates/clickhouse-cloud-api/src/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -560,8 +560,6 @@ pub enum PgVersion {
_18,
#[serde(rename = "17")]
_17,
#[serde(rename = "16")]
_16,
/// Catch-all for unknown or newly-added values.
#[serde(untagged)]
Unknown(String),
Expand All @@ -572,7 +570,6 @@ impl std::fmt::Display for PgVersion {
match self {
Self::_18 => write!(f, "18"),
Self::_17 => write!(f, "17"),
Self::_16 => write!(f, "16"),
Self::Unknown(s) => write!(f, "{s}"),
}
}
Expand Down
15 changes: 14 additions & 1 deletion crates/clickhousectl/src/cloud/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use std::path::{Path, PathBuf};
use tabled::{Table, Tabled, settings::Style};

const KNOWN_PG_PROVIDERS: &[&str] = &["aws"];
const KNOWN_PG_VERSIONS: &[&str] = &["18", "17", "16"];
const KNOWN_PG_VERSIONS: &[&str] = &["18", "17"];
const KNOWN_PG_HA_TYPES: &[&str] = &["none", "async", "sync"];

#[derive(Subcommand)]
Expand Down Expand Up @@ -1131,6 +1131,19 @@ mod tests {
assert!(err.to_string().contains("invalid value"));
}

#[test]
fn rejects_postgres_create_pg_version_16() {
let err = Cli::try_parse_from([
"clickhousectl", "cloud", "postgres", "create",
"--name", "pg1",
"--region", "us-east-1",
"--size", "m7i.2xlarge",
"--pg-version", "16",
])
.err().expect("expected parse error");
assert!(err.to_string().contains("invalid value"));
}

#[test]
fn parses_postgres_update_tag_diff_flags() {
let cmd = parse_postgres(&[
Expand Down
4 changes: 2 additions & 2 deletions crates/clickhousectl/src/local/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ CONTEXT FOR AGENTS:
Starts a named Postgres server backed by a Docker container.
Without --name, the first server is called \"default\"; if \"default\" is running,
a random name is generated (e.g. \"bold-crane\").
--version (-v) selects a postgres image tag (16, 17, or 18 — e.g. 16, 16-alpine, 17.0, 18-bookworm).
--version (-v) selects a postgres image tag (17 or 18 — e.g. 17, 17-alpine, 18.1, 18-bookworm).
Defaults to 18. Image is pulled if not already present locally.
--port defaults to 5432; if taken, a free port is auto-assigned.
Data persists at .clickhouse/servers/<name>/data/ and is bind-mounted into the container.
Expand All @@ -303,7 +303,7 @@ CONTEXT FOR AGENTS:
#[arg(long)]
name: Option<String>,

/// Postgres image tag (16, 17, or 18 — e.g. 16, 16-alpine, 17.0, 18-bookworm). Default: 18. Pulls if missing.
/// Postgres image tag (17 or 18 — e.g. 17, 17-alpine, 18.1, 18-bookworm). Default: 18. Pulls if missing.
#[arg(long, short = 'v')]
version: Option<String>,

Expand Down
4 changes: 2 additions & 2 deletions crates/clickhousectl/src/local/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -870,8 +870,8 @@ mod tests {

#[test]
fn parse_postgres_install_spec_recognizes_at_and_colon() {
assert_eq!(parse_postgres_install_spec("postgres@16"), Some("16"));
assert_eq!(parse_postgres_install_spec("postgres:16-alpine"), Some("16-alpine"));
assert_eq!(parse_postgres_install_spec("postgres@17"), Some("17"));
assert_eq!(parse_postgres_install_spec("postgres:17-alpine"), Some("17-alpine"));
assert_eq!(parse_postgres_install_spec("25.12"), None);
assert_eq!(parse_postgres_install_spec("stable"), None);
}
Expand Down
22 changes: 11 additions & 11 deletions crates/clickhousectl/src/local/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,25 @@ const DEFAULT_PG_PORT: u16 = 5432;
const DEFAULT_USER: &str = "postgres";
const DEFAULT_DATABASE: &str = "postgres";
/// Default image tag when `--version` is not given. Within the supported
/// range; users can override with any 16/17/18 tag (`16`, `16-alpine`, etc).
/// range; users can override with any 17/18 tag (`17`, `17.0`, `18-bookworm`, etc).
pub const DEFAULT_PG_TAG: &str = "18";

/// Extract the major-version digits from a Postgres image tag. `16-alpine` →
/// `"16"`, `17.0` → `"17"`, `18-bookworm` → `"18"`. Validation is the caller's
/// Extract the major-version digits from a Postgres image tag. `17-alpine` →
/// `"17"`, `17.0` → `"17"`, `18-bookworm` → `"18"`. Validation is the caller's
/// responsibility (`validate_pg_tag`) — this only parses.
pub(crate) fn pg_major_from_tag(tag: &str) -> String {
tag.chars().take_while(|c| c.is_ascii_digit()).collect()
}

/// Accept Postgres image tags whose major version is 16, 17, or 18 — anything
/// else is unsupported for now. Examples that pass: `16`, `16-alpine`, `17.0`,
/// `18-bookworm`, `16.4-alpine3.20`. Examples that fail: `latest`, `15`, `19`.
/// Accept Postgres image tags whose major version is 17 or 18 — anything
/// else is unsupported for now. Examples that pass: `17`, `17.0`, `17-alpine`,
/// `18-bookworm`, `18.1-alpine3.20`. Examples that fail: `latest`, `16`, `19`.
pub(crate) fn validate_pg_tag(tag: &str) -> Result<()> {
let major: String = tag.chars().take_while(|c| c.is_ascii_digit()).collect();
if !matches!(major.as_str(), "16" | "17" | "18") {
if !matches!(major.as_str(), "17" | "18") {
return Err(Error::Exec(format!(
"postgres version '{}' is not supported. Use a 16, 17, or 18 image tag \
(for example: 16, 16-alpine, 17.0, 18-bookworm).",
"postgres version '{}' is not supported. Use a 17 or 18 image tag \
(for example: 17, 17-alpine, 18.1, 18-bookworm).",
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PG16 disambiguation blocked locally

High Severity

Rejecting major 16 in validate_pg_tag also runs when stop, remove, client, and dotenv resolve --version. After two majors share a name, --version 16 is required to manage an existing PG16 instance but now errors, so teardown and client access for that instance can fail while PG17 remains.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 787b79c. Configure here.

tag
)));
}
Expand Down Expand Up @@ -703,14 +703,14 @@ mod tests {

#[test]
fn validate_pg_tag_accepts_supported_majors() {
for tag in ["16", "17", "18", "16-alpine", "17.0", "18-bookworm", "16.4-alpine3.20"] {
for tag in ["17", "18", "17-alpine", "17.0", "18-bookworm", "18.1-alpine3.20"] {
assert!(validate_pg_tag(tag).is_ok(), "expected `{}` to be accepted", tag);
}
}

#[test]
fn validate_pg_tag_rejects_unsupported() {
for tag in ["latest", "15", "19", "14-alpine", "alpine", ""] {
for tag in ["latest", "15", "16", "16-alpine", "19", "14-alpine", "alpine", ""] {
assert!(validate_pg_tag(tag).is_err(), "expected `{}` to be rejected", tag);
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/clickhousectl/src/local/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ mod tests {
let info = ServerInfo {
name: "dev".into(),
pid: 0,
version: "postgres:16".into(),
version: "postgres:17".into(),
http_port: 0,
tcp_port: 5432,
started_at: "1700000000".into(),
Expand Down
62 changes: 32 additions & 30 deletions scripts/test-postgres-integration.sh
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ die() { echo " -> $*"; return 1; }

# ── 1. Reuses existing container after stop/delete-metadata via discovery ──
case_orphan_recovery() {
"$CTL" local postgres start --name a --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
local meta=.clickhouse/servers/a-pg16.json
"$CTL" local postgres start --name a --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
local meta=.clickhouse/servers/a-pg18.json
local cid_before; cid_before=$(jq -r .container_id "$meta")
"$CTL" local postgres stop a >/dev/null 2>&1 || { die "stop"; return 1; }
rm "$meta"
Expand All @@ -86,28 +86,28 @@ case_orphan_recovery() {

# ── 2. start with externally-removed container errors with recovery guidance ──
case_externally_removed_container() {
"$CTL" local postgres start --name b --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
local cid; cid=$(jq -r .container_id .clickhouse/servers/b-pg16.json)
"$CTL" local postgres start --name b --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
local cid; cid=$(jq -r .container_id .clickhouse/servers/b-pg18.json)
docker rm -f "$cid" >/dev/null 2>&1 || { die "docker rm failed"; return 1; }
# Metadata still references the dead id. Should error with explicit
# recovery guidance, not silently recreate against potentially-corrupt PGDATA.
local out; out=$("$CTL" local postgres start --name b --version 16-alpine 2>&1) || true
local out; out=$("$CTL" local postgres start --name b --version 18-alpine 2>&1) || true
echo "$out" | grep -q "container is gone" || { die "no recovery message: $out"; return 1; }
# `local postgres remove` should clean it up.
"$CTL" local postgres remove b >/dev/null 2>&1 || { die "remove after orphan failed"; return 1; }
# Now start fresh should work.
"$CTL" local postgres start --name b --version 16-alpine >/dev/null 2>&1 || { die "fresh start after remove failed"; return 1; }
"$CTL" local postgres start --name b --version 18-alpine >/dev/null 2>&1 || { die "fresh start after remove failed"; return 1; }
"$CTL" local postgres stop b >/dev/null 2>&1
"$CTL" local postgres remove b >/dev/null 2>&1
}

# ── 3. Two named postgres servers coexist on different ports ──
case_two_concurrent_servers() {
"$CTL" local postgres start --name c1 --version 16-alpine >/dev/null 2>&1 || { die "start c1"; return 1; }
"$CTL" local postgres start --name c2 --version 16-alpine >/dev/null 2>&1 || { die "start c2"; return 1; }
"$CTL" local postgres start --name c1 --version 18-alpine >/dev/null 2>&1 || { die "start c1"; return 1; }
"$CTL" local postgres start --name c2 --version 18-alpine >/dev/null 2>&1 || { die "start c2"; return 1; }
local p1 p2
p1=$(jq -r .tcp_port .clickhouse/servers/c1-pg16.json)
p2=$(jq -r .tcp_port .clickhouse/servers/c2-pg16.json)
p1=$(jq -r .tcp_port .clickhouse/servers/c1-pg18.json)
p2=$(jq -r .tcp_port .clickhouse/servers/c2-pg18.json)
[[ "$p1" != "$p2" ]] || { die "ports collide: $p1 == $p2"; return 1; }
[[ "$p1" -gt 0 && "$p2" -gt 0 ]] || { die "ports invalid"; return 1; }
"$CTL" local postgres stop c1 >/dev/null 2>&1
Expand All @@ -118,24 +118,24 @@ case_two_concurrent_servers() {

# ── 4. Same name, two majors → two isolated instances ──
case_per_version_isolation() {
"$CTL" local postgres start --name d --version 16-alpine >/dev/null 2>&1 || { die "start 16"; return 1; }
"$CTL" local postgres stop d >/dev/null 2>&1
"$CTL" local postgres start --name d --version 17-alpine >/dev/null 2>&1 || { die "start 17"; return 1; }
[[ -f .clickhouse/servers/d-pg16.json ]] || { die "16 metadata vanished"; return 1; }
[[ -f .clickhouse/servers/d-pg17.json ]] || { die "17 metadata not created"; return 1; }
[[ -d .clickhouse/servers/d-pg16/data ]] || { die "16 data dir vanished"; return 1; }
[[ -d .clickhouse/servers/d-pg17/data ]] || { die "17 data dir not created"; return 1; }
"$CTL" local postgres stop d >/dev/null 2>&1
"$CTL" local postgres start --name d --version 18-alpine >/dev/null 2>&1 || { die "start 18"; return 1; }
[[ -f .clickhouse/servers/d-pg17.json ]] || { die "17 metadata vanished"; return 1; }
[[ -f .clickhouse/servers/d-pg18.json ]] || { die "18 metadata not created"; return 1; }
[[ -d .clickhouse/servers/d-pg17/data ]] || { die "17 data dir vanished"; return 1; }
[[ -d .clickhouse/servers/d-pg18/data ]] || { die "18 data dir not created"; return 1; }
# Bare `local postgres stop d` should ask for --version since multiple match.
local out; out=$("$CTL" local postgres stop d 2>&1) || true
echo "$out" | grep -q "pass --version" || { die "no disambiguation message: $out"; return 1; }
"$CTL" local postgres stop d --version 17 >/dev/null 2>&1 || { die "versioned stop failed"; return 1; }
"$CTL" local postgres stop d --version 18 >/dev/null 2>&1 || { die "versioned stop failed"; return 1; }
"$CTL" local postgres remove d --version 18 >/dev/null 2>&1
"$CTL" local postgres remove d --version 17 >/dev/null 2>&1
"$CTL" local postgres remove d --version 16 >/dev/null 2>&1
}

# ── 5. dotenv reflects the actual container password (post-restart) ──
case_dotenv_password_consistency() {
"$CTL" local postgres start --name e --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
"$CTL" local postgres start --name e --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
local pw_meta; pw_meta=$("$CTL" local postgres dotenv --name e 2>/dev/null \
| grep POSTGRES_PASSWORD= | cut -d= -f2)
[[ -n "$pw_meta" ]] || { die "no password emitted"; return 1; }
Expand Down Expand Up @@ -168,17 +168,17 @@ case_cross_engine_coexist() {
{"name":"shared","pid":99999,"version":"25.12.5.44","http_port":8123,"tcp_port":9000,"started_at":"0","cwd":"$PWD","engine":"clickhouse"}
EOF
# Starting Postgres "shared" should succeed — CH and PG live in different files.
"$CTL" local postgres start --name shared --version 16-alpine >/dev/null 2>&1 \
"$CTL" local postgres start --name shared --version 18-alpine >/dev/null 2>&1 \
|| { die "postgres start with same name failed"; return 1; }
[[ -f .clickhouse/servers/shared.json ]] || { die "CH metadata clobbered"; return 1; }
[[ -f .clickhouse/servers/shared-pg16.json ]] || { die "PG metadata not created"; return 1; }
[[ -f .clickhouse/servers/shared-pg18.json ]] || { die "PG metadata not created"; return 1; }
"$CTL" local postgres stop shared >/dev/null 2>&1
"$CTL" local postgres remove shared >/dev/null 2>&1
}

# ── 10. local server stop-all leaves Postgres running ──
case_stop_all_isolates_postgres() {
"$CTL" local postgres start --name p --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
"$CTL" local postgres start --name p --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
"$CTL" local server stop-all >/dev/null 2>&1
"$CTL" local server list 2>&1 | grep -E "^\| p " | grep -q running || { die "postgres got stopped"; return 1; }
"$CTL" local postgres stop p >/dev/null 2>&1
Expand All @@ -193,10 +193,10 @@ case_port_zero_rejected() {

# ── 12. Non-TTY query path returns query result ──
case_non_tty_query() {
"$CTL" local postgres start --name q --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
"$CTL" local postgres start --name q --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
# Wait for pg to be query-ready (up to ~5s); the first start already waits
# for the container, but pg itself takes a moment to accept queries.
local cid; cid=$(jq -r .container_id .clickhouse/servers/q-pg16.json)
local cid; cid=$(jq -r .container_id .clickhouse/servers/q-pg18.json)
# `pg_isready` without -h checks a unix socket dir that may not exist in
# alpine builds yet; force TCP to wait for actual query readiness.
for _ in {1..50}; do
Expand All @@ -212,7 +212,7 @@ case_non_tty_query() {

# ── 13. dotenv preserves unmanaged vars and replaces in-place ──
case_dotenv_preserves_other_vars() {
"$CTL" local postgres start --name r --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
"$CTL" local postgres start --name r --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
cat > .env <<EOF
DATABASE_URL=postgres://existing
CLICKHOUSE_HOST=ch.example.com
Expand All @@ -227,17 +227,17 @@ EOF

# ── 14. remove of running postgres is rejected ──
case_remove_running_rejected() {
"$CTL" local postgres start --name s --version 16-alpine >/dev/null 2>&1 || { die "start"; return 1; }
"$CTL" local postgres start --name s --version 18-alpine >/dev/null 2>&1 || { die "start"; return 1; }
local out; out=$("$CTL" local postgres remove s 2>&1) || true
echo "$out" | grep -qE "already running|running" || { die "no rejection: $out"; return 1; }
"$CTL" local postgres stop s >/dev/null 2>&1
"$CTL" local postgres remove s >/dev/null 2>&1
}

# ── 15a. Each supported major (16/17/18) starts and is query-ready ──
# ── 15a. Each supported major (17/18) starts and is query-ready ──
case_majors_start_and_serve() {
local tag fail=0
for tag in 16 17 18; do
for tag in 17 18; do
local n="m$tag"
if ! "$CTL" local postgres start --name "$n" --version "$tag" >/dev/null 2>&1; then
die "start postgres:$tag failed"
Expand All @@ -263,12 +263,14 @@ case_majors_start_and_serve() {
return $fail
}

# ── 16. validate_pg_tag rejects 19 and 14 ──
# ── 16. validate_pg_tag rejects 14, 16, and 19 ──
case_unsupported_majors() {
local o14 o19
local o14 o16 o19
o14=$("$CTL" local postgres start --name t --version 14-alpine 2>&1) || true
o16=$("$CTL" local postgres start --name t --version 16-alpine 2>&1) || true
o19=$("$CTL" local postgres start --name t --version 19 2>&1) || true
echo "$o14" | grep -q "not supported" || { die "14 not rejected: $o14"; return 1; }
echo "$o16" | grep -q "not supported" || { die "16 not rejected: $o16"; return 1; }
echo "$o19" | grep -q "not supported" || { die "19 not rejected: $o19"; return 1; }
}

Expand Down