diff --git a/.nextchanges/air/help-runtime-env-vars.md b/.nextchanges/air/help-runtime-env-vars.md new file mode 100644 index 00000000000..049426c5d30 --- /dev/null +++ b/.nextchanges/air/help-runtime-env-vars.md @@ -0,0 +1 @@ +* `air run -h config` now lists the environment variables AI Runtime sets on every node (such as `MLFLOW_RUN_ID` and `CODE_SOURCE_PATH`), and `air logs --help` notes that `--download-to` cannot be combined with `--tail` or `--minutes`. ([#7655](https://github.com/databricks/cli/pull/7655)) diff --git a/acceptance/air/config-help/output.txt b/acceptance/air/config-help/output.txt index 94764d7752c..5df22e08da8 100644 --- a/acceptance/air/config-help/output.txt +++ b/acceptance/air/config-help/output.txt @@ -88,6 +88,14 @@ config For details on a field, run "databricks air run -h config.". +Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this attempt; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set + Let a coding agent write this YAML for you with the databricks-ai-runtime skill: databricks aitools install --skills-only --skills databricks-ai-runtime (add --scope project to install into the current project) diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index df1a20ce687..83c7659f07d 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -206,7 +206,7 @@ Examples: databricks air logs 123456789 --download-to ./logs Flags: - --download-to string Download all logs to this directory instead of printing + --download-to string Download all logs to this directory instead of printing; not combinable with --tail or --minutes -h, --help[=true|false] help for logs --minutes int Fetch only logs from the last N minutes --node int Fetch logs from this node diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 33dd4d67366..949c08d13ef 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -45,7 +45,7 @@ JOB_RUN_ID is the job run ID returned by "databricks air run".`, cmd.Flags().IntVar(&tail, "tail", 0, "Print the last N existing lines before following, or from a completed run (default 10000)") cmd.Flags().IntVar(&minutes, "minutes", 0, "Fetch only logs from the last N minutes") cmd.Flags().IntVar(&retry, "retry", -1, "View logs from a specific retry attempt; -1 means latest") - cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing") + cmd.Flags().StringVar(&downloadTo, "download-to", "", "Download all logs to this directory instead of printing; not combinable with --tail or --minutes") cmd.Flags().BoolVar(&review, "review", false, "Download logs from all nodes and filter for error signatures") cmd.Flags().MarkHidden("review") diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index cdb60d486e8..c1842f5ad0c 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -918,6 +918,7 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio } fmt.Fprintf(w, "\nFor details on a field, run \"databricks air run -h %s.\".\n", f.path) if f.path == configHelpRoot { + fmt.Fprint(w, runtimeEnvHelp) fmt.Fprint(w, agentSkillHelp) } } @@ -926,6 +927,19 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio // pages that point coding agents at the skill share it. const skillInstallCommand = "databricks aitools install --skills-only --skills databricks-ai-runtime" +// runtimeEnvHelp follows the fields in `-h config`: training code needs these +// variables (MLFLOW_RUN_ID especially, since the runtime has already created the +// MLflow run), and the YAML schema is where people and agents look for them. +const runtimeEnvHelp = ` +Set on every node (use in command and training code): + CODE_SOURCE_PATH Extracted code_source directory + MLFLOW_RUN_ID MLflow run created for this attempt; log metrics and artifacts to it + NUM_NODES, NODE_RANK Node count and this node's rank (0 to NUM_NODES-1) + WORLD_SIZE, LOCAL_WORLD_SIZE GPUs across all nodes, and on this node + MASTER_ADDR, MASTER_PORT Rank-0 address for distributed launchers such as torchrun + HYPERPARAMETERS_PATH The parameters YAML, when parameters is set +` + // agentSkillHelp closes `-h config`: users look up the schema while writing // YAML, which is when an agent skill that writes it for them is most useful. const agentSkillHelp = `