From 71b79da11c306f56173459d51a49d546e44a3146 Mon Sep 17 00:00:00 2001 From: Prakriti Gupta Date: Thu, 5 Nov 2026 11:25:06 -0800 Subject: [PATCH 1/2] Add install.sh --local to install the hispec checkout without pulling, and make the broker host configurable via HISPEC_BROKER_HOST --- docs/install.md | 34 ++++++++++++++ docs/operations/systemd.md | 9 ++++ src/hispec/daemon.py | 9 ++-- systemd/install.sh | 95 +++++++++++++++++++++++++++----------- tests/test_daemon.py | 36 +++++++++++++++ 5 files changed, 154 insertions(+), 29 deletions(-) create mode 100644 tests/test_daemon.py diff --git a/docs/install.md b/docs/install.md index dc5c840..4ef3003 100644 --- a/docs/install.md +++ b/docs/install.md @@ -186,6 +186,40 @@ request from the clone; pushing needs a GitHub account with access, since the deploy key only reads. To go back to the GitHub build, remove the clone and re-run `install.sh`. +## Running a developer-managed checkout + +Sometimes a host has to run code that is not in a release yet, edited and +pulled by a developer working on that host. `--local` makes `install.sh` leave +the hispec checkout alone and install it as it is: + +```bash +sudo /opt/hispec/app/systemd/install.sh --local +``` + +In this mode `install.sh` never pulls. It runs no `git fetch`, `pull`, +`checkout` or `submodule update` in `/opt/hispec/app`, and it accepts +uncommitted edits. Bringing in new commits is up to the developer: + +```bash +git -C /opt/hispec/app pull +git -C /opt/hispec/app submodule update --init --recursive +``` + +Dependencies are still installed exactly: the venv is synced with +`uv sync --locked` from that checkout's `uv.lock`. hispec itself is installed +editable, so an edit under `src/hispec` takes effect at the next +`hispec restart `. Re-run `install.sh` only after `pyproject.toml` or +`uv.lock` changes. + +The choice is recorded in `/etc/hispec/source`, so later runs, such as +enrolling another operator, stay local without the flag. To go back to +releases, commit or discard the edits, then pass a ref, which also clears the +record: + +```bash +sudo /opt/hispec/app/systemd/install.sh --ref hispec-X.Y +``` + ## Development checkouts Your own clone for development is separate from all of this. Build it with diff --git a/docs/operations/systemd.md b/docs/operations/systemd.md index 91fb1b8..6307798 100644 --- a/docs/operations/systemd.md +++ b/docs/operations/systemd.md @@ -300,6 +300,15 @@ EOF Without them a daemon is refused with `ACCESS_REFUSED ... mechanism PLAIN`. +The broker is `hispec.caltech.edu` unless `HISPEC_BROKER_HOST` names another, +for a host with its own broker: + +```bash +echo HISPEC_BROKER_HOST=localhost | sudo tee -a /etc/hispec/secrets.env +``` + +The user and password above still apply to it. + Any top-level config key can also be replaced by a `LIBBY_` variable, so `LIBBY_RABBITMQ_URL` points one host at a different broker entirely. It replaces the whole URL, credentials included, and wins over the two variables diff --git a/src/hispec/daemon.py b/src/hispec/daemon.py index de18253..3db979b 100644 --- a/src/hispec/daemon.py +++ b/src/hispec/daemon.py @@ -7,8 +7,10 @@ from libby.daemon import LibbyDaemon # The shared broker, on hispec-new. Its credentials stay out of git: set these -# in /etc/hispec/secrets.env, which is root-only. +# in /etc/hispec/secrets.env, which is root-only. A host with its own broker +# (e.g. the Palomar cal machine) names it in HISPEC_BROKER_HOST. BROKER_HOST = "hispec.caltech.edu" +BROKER_HOST_ENV = "HISPEC_BROKER_HOST" BROKER_USER_ENV = "HISPEC_RABBITMQ_USER" BROKER_PASSWORD_ENV = "HISPEC_RABBITMQ_PASSWORD" @@ -17,10 +19,11 @@ def broker_url() -> str: """Return the shared broker URL, with credentials from the environment.""" user = os.environ.get(BROKER_USER_ENV) password = os.environ.get(BROKER_PASSWORD_ENV) + host = os.environ.get(BROKER_HOST_ENV) or BROKER_HOST if not (user and password): - return f"amqp://{BROKER_HOST}" + return f"amqp://{host}" # quote() so a password containing @ or / cannot break the URL apart - return f"amqp://{quote(user, safe='')}:{quote(password, safe='')}@{BROKER_HOST}" + return f"amqp://{quote(user, safe='')}:{quote(password, safe='')}@{host}" class HispecDaemon(LibbyDaemon): diff --git a/systemd/install.sh b/systemd/install.sh index a364eae..b0a0d72 100755 --- a/systemd/install.sh +++ b/systemd/install.sh @@ -7,6 +7,7 @@ # sudo HISPEC_OPS_USERS="alice bob" ./systemd/install.sh # same thing # sudo ./systemd/install.sh --extra scripts # ... with an optional extra # sudo ./systemd/install.sh --ref main # install a branch or tag instead +# sudo ./systemd/install.sh --local # install the checkout as it is # # A release is an annotated hispec-X.Y tag. The checkout moves to the newest # one, and the venv is synced from that release's uv.lock, so every git @@ -17,6 +18,12 @@ # camera module, which any other role is refused; feiserver and hispecserver # always get atlas, the viewer. # +# --local leaves the hispec checkout to whoever develops on this host: no +# fetch, pull, checkout or submodule update, and edits are allowed. It installs +# whatever is checked out, dependencies from that checkout's uv.lock. It is +# recorded in /etc/hispec/source, so later runs stay local until one passes +# --ref. See docs/install.md. +# # A clone in /opt/hispec/extra/ replaces the GitHub build of its extra, for # changing that code on this host; see docs/install.md. Remove the clone and # re-run to go back. @@ -40,6 +47,7 @@ REPO_DIR="${HISPEC_REPO_DIR:-/opt/hispec/app}" VENV_DIR="${HISPEC_VENV_DIR:-/opt/hispec/venv}" EXTRAS_FILE="${HISPEC_ETC_DIR:-/etc/hispec}/extras" HOST_FILE="${HISPEC_ETC_DIR:-/etc/hispec}/host" +SOURCE_FILE="${HISPEC_ETC_DIR:-/etc/hispec}/source" CAMERA_EXTRA=atcam CAMERA_ROLE=feiserver VIEWER_EXTRA=atlas @@ -83,9 +91,11 @@ if [[ -f "$EXTRAS_FILE" ]]; then fi ROLE_ARG="" REF="" +LOCAL_ARG="" while (($#)); do case $1 in --extra) EXTRAS+=("$2"); shift 2 ;; + --local) LOCAL_ARG=1; shift ;; --ref) REF=$2; shift 2 ;; --ref=*) REF=${1#*=}; shift ;; --host-role) ROLE_ARG=$2; INSTALL_ARGS+=("$1" "$2"); shift 2 ;; @@ -94,6 +104,22 @@ while (($#)); do esac done +# Local source: asked for now, or recorded by an earlier --local run. An +# explicit --ref goes back to installing releases. +if [[ -n ${HISPEC_LOCAL_SOURCE:-} ]]; then + LOCAL_ARG=1 +fi +if [[ -n $LOCAL_ARG && -n $REF ]]; then + echo "error: --local installs the checkout as it is; it cannot be combined with --ref." >&2 + exit 1 +fi +LOCAL="" +if [[ -n $LOCAL_ARG ]]; then + LOCAL=1 +elif [[ -z $REF && -f $SOURCE_FILE && $(tr -d '[:space:]' <"$SOURCE_FILE") == local ]]; then + LOCAL=1 +fi + # Same precedence as `hispec install`, which never overwrites a recorded role ROLE="" if [[ -f "$HOST_FILE" ]]; then @@ -141,35 +167,45 @@ if [[ -f "$DEPLOY_KEY" ]]; then export GIT_SSH_COMMAND="ssh -i $DEPLOY_KEY -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" fi -# Refuse to move a checkout with edits in it: checking out the release would -# carry them along or stop halfway. `hispec hotfix` saves them first. -if [[ -n $(git -C "$REPO_DIR" status --porcelain --untracked-files=no) ]]; then - echo "error: $REPO_DIR has uncommitted edits:" >&2 - git -C "$REPO_DIR" status --short --untracked-files=no >&2 - echo 'Save them with: hispec hotfix "what you fixed" (or discard: hispec hotfix --undo)' >&2 - exit 1 -fi -git -C "$REPO_DIR" fetch --quiet --tags --force origin -if [[ -z $REF ]]; then - REF=$(git -C "$REPO_DIR" tag --list 'hispec-*' --sort=-version:refname | head -n1) - if [[ -z $REF ]]; then - echo "error: no hispec-* release tag yet; pass --ref main to install the branch." >&2 - exit 1 +if [[ -n $LOCAL ]]; then + # The developer owns the checkout: no git command here changes it + LOCAL_DESC=$(git -C "$REPO_DIR" symbolic-ref --short -q HEAD || + git -C "$REPO_DIR" rev-parse --short HEAD) + if [[ -n $(git -C "$REPO_DIR" status --porcelain --untracked-files=no) ]]; then + LOCAL_DESC+=", modified" fi -fi -if git -C "$REPO_DIR" show-ref --verify --quiet "refs/remotes/origin/$REF"; then - # A branch: follow its tip on origin, refusing rather than dropping local commits - git -C "$REPO_DIR" checkout --quiet "$REF" - git -C "$REPO_DIR" merge --quiet --ff-only "origin/$REF" + echo "installing local checkout $REPO_DIR ($LOCAL_DESC); not pulling, that is up to you" else - git -C "$REPO_DIR" checkout --quiet --detach "$REF" -fi -git -C "$REPO_DIR" submodule update --quiet --init --recursive -# The checkout may have replaced this very script; carry on in the release's copy -if [[ -z ${HISPEC_INSTALL_CHECKED_OUT:-} ]]; then - HISPEC_INSTALL_CHECKED_OUT=1 exec "$REPO_DIR/systemd/install.sh" "${ORIGINAL_ARGS[@]}" --ref "$REF" + # Refuse to move a checkout with edits in it: checking out the release would + # carry them along or stop halfway. `hispec hotfix` saves them first. + if [[ -n $(git -C "$REPO_DIR" status --porcelain --untracked-files=no) ]]; then + echo "error: $REPO_DIR has uncommitted edits:" >&2 + git -C "$REPO_DIR" status --short --untracked-files=no >&2 + echo 'Save them with: hispec hotfix "what you fixed" (or discard: hispec hotfix --undo)' >&2 + exit 1 + fi + git -C "$REPO_DIR" fetch --quiet --tags --force origin + if [[ -z $REF ]]; then + REF=$(git -C "$REPO_DIR" tag --list 'hispec-*' --sort=-version:refname | head -n1) + if [[ -z $REF ]]; then + echo "error: no hispec-* release tag yet; pass --ref main to install the branch." >&2 + exit 1 + fi + fi + if git -C "$REPO_DIR" show-ref --verify --quiet "refs/remotes/origin/$REF"; then + # A branch: follow its tip on origin, refusing rather than dropping local commits + git -C "$REPO_DIR" checkout --quiet "$REF" + git -C "$REPO_DIR" merge --quiet --ff-only "origin/$REF" + else + git -C "$REPO_DIR" checkout --quiet --detach "$REF" + fi + git -C "$REPO_DIR" submodule update --quiet --init --recursive + # The checkout may have replaced this very script; carry on in the release's copy + if [[ -z ${HISPEC_INSTALL_CHECKED_OUT:-} ]]; then + HISPEC_INSTALL_CHECKED_OUT=1 exec "$REPO_DIR/systemd/install.sh" "${ORIGINAL_ARGS[@]}" --ref "$REF" + fi + echo "installing $REF ($(git -C "$REPO_DIR" rev-parse --short HEAD))" fi -echo "installing $REF ($(git -C "$REPO_DIR" rev-parse --short HEAD))" # Pinned, so re-running the installer cannot resolve with a different uv if [[ "$("$UV_BIN" --version 2>/dev/null)" != "uv $UV_VERSION"* ]]; then @@ -209,5 +245,12 @@ if ((${#EXTRAS[@]})); then mkdir -p "$(dirname "$EXTRAS_FILE")" printf '%s\n' "${EXTRAS[@]}" >"$EXTRAS_FILE" fi +# Remembered so a later plain run (enrolling an operator, say) stays local +if [[ -n $LOCAL ]]; then + mkdir -p "$(dirname "$SOURCE_FILE")" + echo local >"$SOURCE_FILE" +else + rm -f "$SOURCE_FILE" +fi exec "$VENV_DIR/bin/hispec" install "${INSTALL_ARGS[@]}" diff --git a/tests/test_daemon.py b/tests/test_daemon.py new file mode 100644 index 0000000..bf958d8 --- /dev/null +++ b/tests/test_daemon.py @@ -0,0 +1,36 @@ +"""Tests for the shared broker URL that HispecDaemon falls back to.""" +import pytest + +from hispec import daemon + + +@pytest.fixture(autouse=True) +def fixture_clean_env(monkeypatch): + """Start every test with none of the broker variables set.""" + for name in (daemon.BROKER_HOST_ENV, daemon.BROKER_USER_ENV, daemon.BROKER_PASSWORD_ENV): + monkeypatch.delenv(name, raising=False) + + +def test_default_host(): + """Without HISPEC_BROKER_HOST the shared broker is used.""" + assert daemon.broker_url() == f"amqp://{daemon.BROKER_HOST}" + + +def test_host_override(monkeypatch): + """HISPEC_BROKER_HOST replaces the host.""" + monkeypatch.setenv(daemon.BROKER_HOST_ENV, "localhost") + assert daemon.broker_url() == "amqp://localhost" + + +def test_host_override_with_credentials(monkeypatch): + """The credentials still apply to an overridden host, quoted.""" + monkeypatch.setenv(daemon.BROKER_HOST_ENV, "localhost") + monkeypatch.setenv(daemon.BROKER_USER_ENV, "ops") + monkeypatch.setenv(daemon.BROKER_PASSWORD_ENV, "p@ss/word") + assert daemon.broker_url() == "amqp://ops:p%40ss%2Fword@localhost" + + +def test_empty_host_falls_back(monkeypatch): + """An empty HISPEC_BROKER_HOST is treated as unset.""" + monkeypatch.setenv(daemon.BROKER_HOST_ENV, "") + assert daemon.broker_url() == f"amqp://{daemon.BROKER_HOST}" From 4db084e2ea01588d0d34b397e3439bcdfeadc1d4 Mon Sep 17 00:00:00 2001 From: Prakriti Gupta Date: Thu, 5 Nov 2026 11:52:07 -0800 Subject: [PATCH 2/2] Lock libby to libby-0.2.0 to match pyproject.toml --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 5bba011..12249a1 100644 --- a/uv.lock +++ b/uv.lock @@ -915,7 +915,7 @@ requires-dist = [ { name = "hardware-device-base", git = "https://github.com/COO-Utilities/hardware_device_base" }, { name = "instrumentctl", git = "https://github.com/CaltechOpticalObservatories/instrumentctl.git" }, { name = "keyring" }, - { name = "libby", extras = ["influxdb", "otel"], git = "https://github.com/CaltechOpticalObservatories/libby.git?rev=libby-0.1.0" }, + { name = "libby", extras = ["influxdb", "otel"], git = "https://github.com/CaltechOpticalObservatories/libby.git?rev=libby-0.2.0" }, { name = "libximc" }, { name = "matplotlib", marker = "extra == 'scripts'" }, { name = "paho-mqtt", specifier = ">=2" }, @@ -1266,8 +1266,8 @@ requires-dist = [{ name = "hardware-device-base", git = "https://github.com/COO- [[package]] name = "libby" -version = "0.1.0" -source = { git = "https://github.com/CaltechOpticalObservatories/libby.git?rev=libby-0.1.0#c88f7ac6e9137b6a306f3b1eb205ca529214245d" } +version = "0.2.0" +source = { git = "https://github.com/CaltechOpticalObservatories/libby.git?rev=libby-0.2.0#2529273fa2ddee5107b97a304cc99995fb29ab55" } dependencies = [ { name = "argcomplete" }, { name = "bamboo" },