diff --git a/.nextchanges/air/help-runtime-env-vars.md b/.nextchanges/air/help-runtime-env-vars.md new file mode 100644 index 00000000000..8d30030ca59 --- /dev/null +++ b/.nextchanges/air/help-runtime-env-vars.md @@ -0,0 +1 @@ +* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. ([#7649](https://github.com/databricks/cli/pull/7649)) diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 1fae9e919f3..eb9fa6421cb 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -55,6 +55,14 @@ config root_path: . command: python "$CODE_SOURCE_PATH/train.py" + Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this job; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set + Workload: experiment_name (required) Name of the experiment. compute (required) Which accelerators to run on and how many. diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7b37a4c83ac..a20a7a60eab 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -206,7 +206,7 @@ Examples: databricks air logs 123456789 --download-to ./logs Flags: - --download-to string Download all logs to this directory instead of printing + --download-to string Download all logs to this directory instead of printing; not combinable with --tail or --minutes -h, --help help for logs --minutes int Fetch only logs from the last N minutes --node int Fetch logs from this node diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 33dd4d67366..949c08d13ef 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -45,7 +45,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, cmd.Flags().IntVar(&tail, "tail", 0, "Print the last N existing lines before following, or from a completed run (default 10000)") cmd.Flags().IntVar(&minutes, "minutes", 0, "Fetch only logs from the last N minutes") cmd.Flags().IntVar(&retry, "retry", -1, "View logs from a specific retry attempt; -1 means latest") - cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing") + cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing; not combinable with --tail or --minutes") cmd.Flags().BoolVar(&review, "review", false, "Download logs from all nodes and filter for error signatures") cmd.Flags().MarkHidden("review") diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index 78554b02183..a657d9c2a8f 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -667,7 +667,15 @@ func configSchema() configField { accelerator_type: GPU_1xA10 code_source: root_path: . - command: python "$CODE_SOURCE_PATH/train.py"`, + command: python "$CODE_SOURCE_PATH/train.py" + + Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this job; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set`, children: describeStruct(reflect.TypeFor[runConfig](), configHelpRoot), } }