diff --git a/README.md b/README.md index e2cad843..b8bfae23 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,42 @@ poetry run python manage.py runserver 0.0.0.0:8000 That last path will not start Celery/Redis for you. +#### Verified: run the training backend locally without AWS (2026-10-25) + +The Django + Celery + Redis stack **boots without AWS credentials**. AWS is only +reached at two runtime edges: Firebase auth via Secrets Manager (`authenticator.py` +→ `cli.py init_firebase()`) and saving results to S3 (`worker.py`). Booting and the +unauthenticated surface need neither. + +```sh +mkdir -p ~/.aws # lets the $HOME/.aws bind mount succeed even with no AWS +cd training +docker compose up --build +``` + +Verify it is up: + +```sh +curl http://localhost:8000/api/test # -> {"result": "200 Backend surface test successful"} +docker compose exec redis redis-cli ping # -> PONG +docker compose exec web mamba run -n dlp poetry run pytest tests +``` + +Expected: `/api/test` returns 200, Redis replies `PONG`, and pytest reports +`17 passed, 1 failed`. The single failure (`test_columns_endpoint[CALIFORNIA_HOUSING]`) +is an environment gap, not a pipeline bug: `/home/app` is root-owned while the +container runs as user `app`, so `fetch_california_housing` cannot write its download +cache. Run `pytest tests -k "not CALIFORNIA_HOUSING"` for an all-green run. + +Two fixes on a fresh checkout were needed to reach this state (both in this branch): +the unpinned `pytorch`/`torchvision` in `environment.yml` resolve to a build that is +incompatible with Python 3.9 (`import torch` → `ImportError: cannot import name +'TypeIs'`), so they are pinned to `pytorch=2.5.1` / `torchvision=0.20.1`; and the +training `Dockerfile`s needed `README.md` available before `poetry install`. + +A full end-to-end training job (`POST /api/tabular`) still requires AWS for Firebase +auth and the S3 result save, so it cannot complete on a no-AWS setup. + ### 3. Flask API (`backend/`) ```sh diff --git a/training/Dockerfile b/training/Dockerfile index f1b3c0ef..2126dbda 100644 --- a/training/Dockerfile +++ b/training/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /usr/src/build # install dependencies RUN mamba create --name dlp -y -COPY environment.yml pyproject.toml poetry.lock ./ +COPY environment.yml pyproject.toml poetry.lock README.md ./ RUN mamba run --live-stream -n dlp mamba env update --file environment.yml --prune # create directory for the app user @@ -25,7 +25,7 @@ RUN mkdir $APP_HOME FROM base AS development ENV ENVIRONMENT development -RUN mamba run --live-stream -n dlp poetry install +RUN mamba run --live-stream -n dlp poetry install --no-root # we previously mounted training/ to APP_HOME diff --git a/training/environment.yml b/training/environment.yml index e566214a..8b40727f 100644 --- a/training/environment.yml +++ b/training/environment.yml @@ -6,5 +6,5 @@ channels: dependencies: - python=3.9 - poetry - - pytorch - - torchvision \ No newline at end of file + - pytorch=2.5.1 + - torchvision=0.20.1 \ No newline at end of file diff --git a/training/training/core/celery/Dockerfile b/training/training/core/celery/Dockerfile index ece91a2b..ec2ad1dc 100644 --- a/training/training/core/celery/Dockerfile +++ b/training/training/core/celery/Dockerfile @@ -5,7 +5,9 @@ WORKDIR /usr/src/build # install dependencies RUN mamba create --name dlp -y -COPY environment.yml pyproject.toml poetry.lock ./ +COPY environment.yml pyproject.toml poetry.lock README.md ./ +# This is to just see if the readme file exists + RUN mamba run --live-stream -n dlp mamba env update --file environment.yml --prune # create directory for the app user @@ -20,12 +22,11 @@ ARG APP_HOME=/home/app/training RUN mkdir "$APP_HOME" - ######################## development target ######################## FROM base AS development ENV ENVIRONMENT development -RUN mamba run --live-stream -n dlp poetry install +RUN mamba run --live-stream -n dlp poetry install --no-root # we previously mounted training/ to APP_HOME