From 49e15031697547df867984e7d2d002ec51fb5431 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Sun, 15 Nov 2026 04:49:27 +0000 Subject: [PATCH 1/6] air: document runtime env vars in config help and --download-to limits `air run -h config` didn't say which environment variables the runtime sets, so agents fetched the external skill to learn about MLFLOW_RUN_ID. List them in the schema overview. Also note in `air logs --help` that --download-to can't be combined with --tail or --minutes. Co-authored-by: Isaac (cherry picked from commit e78ab34c2f4ec6fa33764698ac6d75b18d560ed9) --- .nextchanges/air/help-runtime-env-vars.md | 1 + acceptance/air/config-help/output.txt | 8 ++ acceptance/air/help/output.txt | 104 +++++++++++----------- cmd/air/logs.go | 2 +- cmd/air/runconfig.go | 10 ++- 5 files changed, 71 insertions(+), 54 deletions(-) create mode 100644 .nextchanges/air/help-runtime-env-vars.md diff --git a/.nextchanges/air/help-runtime-env-vars.md b/.nextchanges/air/help-runtime-env-vars.md new file mode 100644 index 00000000000..c69cc2fa6bb --- /dev/null +++ b/.nextchanges/air/help-runtime-env-vars.md @@ -0,0 +1 @@ +* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 94764d7752c..16803647103 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -60,6 +60,14 @@ config root_path: . command: python "$CODE_SOURCE_PATH/train.py" + Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this job; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set + Workload: experiment_name (required) Name of the experiment. compute (required) Which accelerators to run on and how many. diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index df1a20ce687..a20a7a60eab 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -28,13 +28,13 @@ Provisioned capacity: pools View provisioned GPU pools, if any (not required to run) Flags: - -h, --help[=true|false] help for air + -h, --help help for air Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) Use "databricks air [command] --help" for more information about a command. @@ -73,17 +73,17 @@ Examples: --filter user=person@example.com Flags: - --all-status[=true|false] Show runs in all states (default: active only) - --all-users[=true|false] Show runs from all users - --filter stringArray Filter runs by KEY=VALUE (repeatable) - -h, --help[=true|false] help for list - --limit int Maximum number of runs to show (default 20) + --all-status Show runs in all states (default: active only) + --all-users Show runs from all users + --filter stringArray Filter runs by KEY=VALUE (repeatable) + -h, --help help for list + --limit int Maximum number of runs to show (default 20) Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === get help @@ -96,13 +96,13 @@ Usage: databricks air get JOB_RUN_ID [flags] Flags: - -h, --help[=true|false] help for get + -h, --help help for get Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === pools help @@ -122,13 +122,13 @@ Available Commands: list List the GPU pools available to the current workspace Flags: - -h, --help[=true|false] help for pools + -h, --help help for pools Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) Use "databricks air pools [command] --help" for more information about a command. @@ -145,13 +145,13 @@ Available Commands: push *Preview* Push a container image to Databricks Artifact Registry Flags: - -h, --help[=true|false] help for images + -h, --help help for images Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) Use "databricks air images [command] --help" for more information about a command. @@ -176,18 +176,18 @@ Usage: databricks air images push [flags] Flags: - --artifact string Destination ARTIFACT[:TAG]; defaults to the source image name and tag (latest if untagged); maximum 255 characters for catalog.schema.artifact and 128 for the tag - --catalog string Destination Unity Catalog catalog; use lowercase letters and digits separated by one or two underscores or one or more hyphens - -h, --help[=true|false] help for push - --pull[=true|false] Pull the source image even when it is already available locally - --schema string Destination Unity Catalog schema; use lowercase letters and digits separated by one or two underscores or one or more hyphens - -s, --source string Source container image to push (NAME[:TAG] or NAME@DIGEST) + --artifact string Destination ARTIFACT[:TAG]; defaults to the source image name and tag (latest if untagged); maximum 255 characters for catalog.schema.artifact and 128 for the tag + --catalog string Destination Unity Catalog catalog; use lowercase letters and digits separated by one or two underscores or one or more hyphens + -h, --help help for push + --pull Pull the source image even when it is already available locally + --schema string Destination Unity Catalog schema; use lowercase letters and digits separated by one or two underscores or one or more hyphens + -s, --source string Source container image to push (NAME[:TAG] or NAME@DIGEST) Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === logs help @@ -206,18 +206,18 @@ Examples: databricks air logs 123456789 --download-to ./logs Flags: - --download-to string Download all logs to this directory instead of printing - -h, --help[=true|false] help for logs + --download-to string Download all logs to this directory instead of printing; not combinable with --tail or --minutes + -h, --help help for logs --minutes int Fetch only logs from the last N minutes --node int Fetch logs from this node --retry int View logs from a specific retry attempt; -1 means latest (default -1) --tail int Print the last N existing lines before following, or from a completed run (default 10000) Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === cancel help @@ -231,12 +231,12 @@ Usage: databricks air cancel [JOB_RUN_ID...] [flags] Flags: - --all[=true|false] Cancel all of your active runs - -h, --help[=true|false] help for cancel - -y, --yes[=true|false] Skip the confirmation prompt + --all Cancel all of your active runs + -h, --help help for cancel + -y, --yes Skip the confirmation prompt Global Flags: - --debug[=true|false] enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 33dd4d67366..949c08d13ef 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -45,7 +45,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, cmd.Flags().IntVar(&tail, "tail", 0, "Print the last N existing lines before following, or from a completed run (default 10000)") cmd.Flags().IntVar(&minutes, "minutes", 0, "Fetch only logs from the last N minutes") cmd.Flags().IntVar(&retry, "retry", -1, "View logs from a specific retry attempt; -1 means latest") - cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing") + cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing; not combinable with --tail or --minutes") cmd.Flags().BoolVar(&review, "review", false, "Download logs from all nodes and filter for error signatures") cmd.Flags().MarkHidden("review") diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index cdb60d486e8..c8667d7fd0d 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -667,7 +667,15 @@ func configSchema() configField { accelerator_type: GPU_1xA10 code_source: root_path: . - command: python "$CODE_SOURCE_PATH/train.py"`, + command: python "$CODE_SOURCE_PATH/train.py" + + Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this job; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set`, children: describeStruct(reflect.TypeFor[runConfig](), configHelpRoot), } } From 682b101e99966646a8c3565937a970fef2cfcb4e Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 04:26:26 +0000 Subject: [PATCH 2/6] Add PR link to changelog fragment Co-authored-by: Isaac --- .nextchanges/air/help-runtime-env-vars.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/help-runtime-env-vars.md b/.nextchanges/air/help-runtime-env-vars.md index c69cc2fa6bb..8d30030ca59 100644 --- a/.nextchanges/air/help-runtime-env-vars.md +++ b/.nextchanges/air/help-runtime-env-vars.md @@ -1 +1 @@ -* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. +* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. ([#7649](https://github.com/databricks/cli/pull/7649)) From 32a709d5a6de8063c33561ddc29ad4df8eb0666a Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 05:02:45 +0000 Subject: [PATCH 3/6] Point changelog fragment at #7655 Co-authored-by: Isaac --- .nextchanges/air/help-runtime-env-vars.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/help-runtime-env-vars.md b/.nextchanges/air/help-runtime-env-vars.md index 8d30030ca59..049426c5d30 100644 --- a/.nextchanges/air/help-runtime-env-vars.md +++ b/.nextchanges/air/help-runtime-env-vars.md @@ -1 +1 @@ -* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. ([#7649](https://github.com/databricks/cli/pull/7649)) +* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. ([#7655](https://github.com/databricks/cli/pull/7655)) From faed32c13f048a4e61a53a5d93ddffd927289420 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 07:52:38 +0000 Subject: [PATCH 4/6] air: note that the HF Trainer attaches to MLFLOW_RUN_ID itself With MLFLOW_RUN_ID set, transformers' MLflowCallback always calls mlflow.start_run(), so a script that also starts the run crashes at trainer.train() with "Run ... is already active". An agent lost a full queue-and-run cycle to this when fine-tuning DistilBERT. Co-authored-by: Isaac --- acceptance/air/config-help/output.txt | 3 +++ cmd/air/runconfig.go | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 16803647103..0fc694b188d 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -68,6 +68,9 @@ config MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun HYPERPARAMETERS_PATH The parameters YAML, when parameters is set + mlflow.start_run() attaches to MLFLOW_RUN_ID. The Hugging Face Trainer starts + it on its own, so don't also call mlflow.start_run() when you use the Trainer. + Workload: experiment_name (required) Name of the experiment. compute (required) Which accelerators to run on and how many. diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index c8667d7fd0d..efa4eb27ea6 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -675,7 +675,10 @@ func configSchema() configField { NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun - HYPERPARAMETERS_PATH The parameters YAML, when parameters is set`, + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set + + mlflow.start_run() attaches to MLFLOW_RUN_ID. The Hugging Face Trainer starts + it on its own, so don't also call mlflow.start_run() when you use the Trainer.`, children: describeStruct(reflect.TypeFor[runConfig](), configHelpRoot), } } From 109d90574dfdc43efd36241636d3e15ffb8f2e77 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 09:57:46 +0000 Subject: [PATCH 5/6] Regenerate air help output after the pflag bump Co-authored-by: Isaac --- acceptance/air/help/output.txt | 102 ++++++++++++++++----------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index a20a7a60eab..83c7659f07d 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -28,13 +28,13 @@ Provisioned capacity: pools View provisioned GPU pools, if any (not required to run) Flags: - -h, --help help for air + -h, --help[=true|false] help for air Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) Use "databricks air [command] --help" for more information about a command. @@ -73,17 +73,17 @@ Examples: --filter user=person@example.com Flags: - --all-status Show runs in all states (default: active only) - --all-users Show runs from all users - --filter stringArray Filter runs by KEY=VALUE (repeatable) - -h, --help help for list - --limit int Maximum number of runs to show (default 20) + --all-status[=true|false] Show runs in all states (default: active only) + --all-users[=true|false] Show runs from all users + --filter stringArray Filter runs by KEY=VALUE (repeatable) + -h, --help[=true|false] help for list + --limit int Maximum number of runs to show (default 20) Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === get help @@ -96,13 +96,13 @@ Usage: databricks air get JOB_RUN_ID [flags] Flags: - -h, --help help for get + -h, --help[=true|false] help for get Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === pools help @@ -122,13 +122,13 @@ Available Commands: list List the GPU pools available to the current workspace Flags: - -h, --help help for pools + -h, --help[=true|false] help for pools Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) Use "databricks air pools [command] --help" for more information about a command. @@ -145,13 +145,13 @@ Available Commands: push *Preview* Push a container image to Databricks Artifact Registry Flags: - -h, --help help for images + -h, --help[=true|false] help for images Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) Use "databricks air images [command] --help" for more information about a command. @@ -176,18 +176,18 @@ Usage: databricks air images push [flags] Flags: - --artifact string Destination ARTIFACT[:TAG]; defaults to the source image name and tag (latest if untagged); maximum 255 characters for catalog.schema.artifact and 128 for the tag - --catalog string Destination Unity Catalog catalog; use lowercase letters and digits separated by one or two underscores or one or more hyphens - -h, --help help for push - --pull Pull the source image even when it is already available locally - --schema string Destination Unity Catalog schema; use lowercase letters and digits separated by one or two underscores or one or more hyphens - -s, --source string Source container image to push (NAME[:TAG] or NAME@DIGEST) + --artifact string Destination ARTIFACT[:TAG]; defaults to the source image name and tag (latest if untagged); maximum 255 characters for catalog.schema.artifact and 128 for the tag + --catalog string Destination Unity Catalog catalog; use lowercase letters and digits separated by one or two underscores or one or more hyphens + -h, --help[=true|false] help for push + --pull[=true|false] Pull the source image even when it is already available locally + --schema string Destination Unity Catalog schema; use lowercase letters and digits separated by one or two underscores or one or more hyphens + -s, --source string Source container image to push (NAME[:TAG] or NAME@DIGEST) Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === logs help @@ -207,17 +207,17 @@ Examples: Flags: --download-to string Download all logs to this directory instead of printing; not combinable with --tail or --minutes - -h, --help help for logs + -h, --help[=true|false] help for logs --minutes int Fetch only logs from the last N minutes --node int Fetch logs from this node --retry int View logs from a specific retry attempt; -1 means latest (default -1) --tail int Print the last N existing lines before following, or from a completed run (default 10000) Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) === cancel help @@ -231,12 +231,12 @@ Usage: databricks air cancel [JOB_RUN_ID...] [flags] Flags: - --all Cancel all of your active runs - -h, --help help for cancel - -y, --yes Skip the confirmation prompt + --all[=true|false] Cancel all of your active runs + -h, --help[=true|false] help for cancel + -y, --yes[=true|false] Skip the confirmation prompt Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) + --debug[=true|false] enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) From 721ef04c6bda77eeebe4184f8da224ba6c004513 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 17:25:07 +0000 Subject: [PATCH 6/6] air: list runtime env vars after the config fields; drop the HF note Review feedback: the fields are the main reason to run `-h config`, so the runtime env vars follow them instead of preceding them, and MLFLOW_RUN_ID is described per attempt. The Hugging Face Trainer guidance is framework behavior, so it moves to the databricks-ai-runtime skill instead. Co-authored-by: Isaac --- acceptance/air/config-help/output.txt | 19 ++++++++----------- cmd/air/runconfig.go | 27 +++++++++++++++------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 0fc694b188d..5df22e08da8 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -60,17 +60,6 @@ config root_path: . command: python "$CODE_SOURCE_PATH/train.py" - Set on every node (use in command and training code): - CODE_SOURCE_PATH Extracted code_source directory - MLFLOW_RUN_ID MLflow run created for this job; log metrics and artifacts to it - NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) - WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node - MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun - HYPERPARAMETERS_PATH The parameters YAML, when parameters is set - - mlflow.start_run() attaches to MLFLOW_RUN_ID. The Hugging Face Trainer starts - it on its own, so don't also call mlflow.start_run() when you use the Trainer. - Workload: experiment_name (required) Name of the experiment. compute (required) Which accelerators to run on and how many. @@ -99,6 +88,14 @@ config For details on a field, run "databricks air run -h config.". +Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this attempt; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set + Let a coding agent write this YAML for you with the databricks-ai-runtime skill: databricks aitools install --skills-only --skills databricks-ai-runtime (add --scope project to install into the current project) diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index efa4eb27ea6..c1842f5ad0c 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -667,18 +667,7 @@ func configSchema() configField { accelerator_type: GPU_1xA10 code_source: root_path: . - command: python "$CODE_SOURCE_PATH/train.py" - - Set on every node (use in command and training code): - CODE_SOURCE_PATH Extracted code_source directory - MLFLOW_RUN_ID MLflow run created for this job; log metrics and artifacts to it - NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) - WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node - MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun - HYPERPARAMETERS_PATH The parameters YAML, when parameters is set - - mlflow.start_run() attaches to MLFLOW_RUN_ID. The Hugging Face Trainer starts - it on its own, so don't also call mlflow.start_run() when you use the Trainer.`, + command: python "$CODE_SOURCE_PATH/train.py"`, children: describeStruct(reflect.TypeFor[runConfig](), configHelpRoot), } } @@ -929,6 +918,7 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio } fmt.Fprintf(w, "\nFor details on a field, run \"databricks air run -h %s.\".\n", f.path) if f.path == configHelpRoot { + fmt.Fprint(w, runtimeEnvHelp) fmt.Fprint(w, agentSkillHelp) } } @@ -937,6 +927,19 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio // pages that point coding agents at the skill share it. const skillInstallCommand = "databricks aitools install --skills-only --skills databricks-ai-runtime" +// runtimeEnvHelp follows the fields in `-h config`: training code needs these +// variables (MLFLOW_RUN_ID especially, since the runtime has already created the +// MLflow run), and the YAML schema is where people and agents look for them. +const runtimeEnvHelp = ` +Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this attempt; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set +` + // agentSkillHelp closes `-h config`: users look up the schema while writing // YAML, which is when an agent skill that writes it for them is most useful. const agentSkillHelp = `