diff --git a/.nextchanges/cli/ssh-pypi-fallback.md b/.nextchanges/cli/ssh-pypi-fallback.md new file mode 100644 index 00000000000..e79b5f42a23 --- /dev/null +++ b/.nextchanges/cli/ssh-pypi-fallback.md @@ -0,0 +1 @@ +* `databricks ssh connect` now downloads the Linux CLI binaries it uploads for the SSH server from the `databricks` package on PyPI when GitHub can't be reached. It uses `PIP_INDEX_URL` or `index-url` from the per-user pip config when set, else pypi.org, and checks each wheel against the SHA-256 digest the index lists. ([#7852](https://github.com/databricks/cli/pull/7852)) diff --git a/experimental/ssh/README.md b/experimental/ssh/README.md index 917f5579c02..c7b69f1fcb3 100644 --- a/experimental/ssh/README.md +++ b/experimental/ssh/README.md @@ -11,6 +11,22 @@ for terminal SSH sessions as well as for IDE Remote Development: the tunnel runs attaches as a single user. Standard access mode (`USER_ISOLATION`), Dedicated-to-a-group, and no-isolation clusters all fail with `cluster '' must be a dedicated single-user cluster`. +## Network Requirements + +The server is the Linux build of the CLI with the client's version. The client downloads it and +uploads it to `/Workspace/Users//.databricks/ssh-tunnel//` once per version, so the +machine running `ssh connect` needs to reach one of: + +- GitHub (`github.com` and `release-assets.githubusercontent.com`), which serves the CLI release + archives, or +- a Python package index that serves the CLI wheels of the `databricks` package: `PIP_INDEX_URL`, + else `index-url` in the per-user pip config (`pip.conf`, or `pip.ini` on Windows), else PyPI + (`pypi.org` and `files.pythonhosted.org`). System-wide pip config files are not read. The client + falls back to the index when the GitHub download fails, takes a wheel only if it matches the + SHA-256 digest the index lists, and sends credentials in the index URL only to the index host. + +Development builds are not published, so they need `--releases-dir` (see [Development](#development)). + ## Usage A. With local ssh config setup: ```shell diff --git a/experimental/ssh/internal/client/pypi.go b/experimental/ssh/internal/client/pypi.go new file mode 100644 index 00000000000..a0c6b7bc542 --- /dev/null +++ b/experimental/ssh/internal/client/pypi.go @@ -0,0 +1,288 @@ +package client + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/localenv" + "github.com/databricks/cli/libs/log" + "golang.org/x/net/html" +) + +// defaultPyPIIndexURL is the index pip uses when none is configured. +const defaultPyPIIndexURL = "https://pypi.org/simple" + +// maxIndexPageSize and maxWheelSize bound what is read into memory from the index. The +// databricks project page is about 20 KB and a CLI wheel about 20 MB. +const ( + maxIndexPageSize = 16 << 20 + maxWheelSize = 256 << 20 +) + +// pypiHTTPClient bounds each request, which http.DefaultClient does not: a proxy that +// accepts the connection and then stalls must not hang `ssh connect`. The limit leaves +// room to download a wheel over a slow link. +var pypiHTTPClient = &http.Client{Timeout: 10 * time.Minute, CheckRedirect: checkPyPIRedirect} + +// checkPyPIRedirect keeps the index credentials on the index origin, the exact scheme and +// host of the index URL, across redirects: net/http sends the userinfo of a URL only with +// requests to that URL, so an absolute redirect back to the index would go without it. +// Redirects elsewhere get neither the userinfo nor an Authorization header, which net/http +// keeps on redirects to subdomains. It also drops the Referer that net/http sets to the +// previous URL, whose query string can carry an index token. +func checkPyPIRedirect(req *http.Request, via []*http.Request) error { + // The limit of the default policy, which this one replaces. + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + req.Header.Del("Referer") + first := via[0].URL + if first.User != nil && first.Scheme == "https" && req.URL.Scheme == "http" { + return fmt.Errorf("refusing to follow a redirect from HTTPS to plain HTTP for an index URL with credentials, from %s to %s", first.Host, req.URL.Host) + } + if req.URL.Scheme != first.Scheme || req.URL.Host != first.Host { + req.Header.Del("Authorization") + req.URL.User = nil + return nil + } + if req.URL.User == nil { + req.URL.User = first.User + } + return nil +} + +// wheelMachines maps a release architecture to the machine in the Linux wheel platform tags. +var wheelMachines = map[string]string{ + "amd64": "x86_64", + "arm64": "aarch64", +} + +// getPyPIRelease builds the release archive for linux/ from the CLI's wheel +// on the package index that pypiIndexURL returns. +func getPyPIRelease(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + indexURL, err := url.Parse(pypiIndexURL(ctx)) + if err != nil { + // The parse error repeats the URL, which can carry credentials. + return nil, errors.New("the Python package index URL is not a valid URL") + } + wheelURL, wheelSHA256, err := findLinuxWheel(ctx, indexURL, architecture, version) + if err != nil { + return nil, err + } + wheel, err := downloadWheel(ctx, wheelURL, wheelSHA256) + if err != nil { + return nil, err + } + archive, err := releaseArchiveFromWheel(wheel, version) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(archive)), nil +} + +// pypiIndexURL returns the package index to download the CLI wheels from: PIP_INDEX_URL, +// else index-url in the per-user pip config, else PyPI. Networks that block PyPI point pip +// at a mirror of it there. System-wide pip config files are not read. +func pypiIndexURL(ctx context.Context) string { + // https://pip.pypa.io/en/stable/topics/configuration/#environment-variables + if indexURL := env.Get(ctx, "PIP_INDEX_URL"); indexURL != "" { + return indexURL + } + if indexURL := localenv.PipConfIndexURL(ctx); indexURL != "" { + return indexURL + } + return defaultPyPIIndexURL +} + +// findLinuxWheel returns the URL and SHA-256 digest of the CLI's Linux wheel for the +// version and architecture, read from the databricks project page of the index. +// https://peps.python.org/pep-0503/ +func findLinuxWheel(ctx context.Context, indexURL *url.URL, architecture, version string) (*url.URL, string, error) { + // Project pages live at //, trailing slash included. + pageURL := indexURL.JoinPath("databricks/") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL.String(), nil) + if err != nil { + return nil, "", fmt.Errorf("failed to fetch the databricks package from %s: %w", indexURL.Host, errWithoutURL(err)) + } + // Every index serves HTML; those that also serve JSON return HTML when asked for it. + // https://peps.python.org/pep-0691/#content-types + req.Header.Set("Accept", "text/html") + resp, err := pypiHTTPClient.Do(req) + if err != nil { + return nil, "", fmt.Errorf("failed to fetch the databricks package from %s: %w", indexURL.Host, errWithoutURL(err)) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("failed to fetch the databricks package from %s: HTTP %d", indexURL.Host, resp.StatusCode) + } + page, err := io.ReadAll(io.LimitReader(resp.Body, maxIndexPageSize+1)) + if err != nil { + return nil, "", fmt.Errorf("failed to read the databricks package page from %s: %w", indexURL.Host, err) + } + if len(page) > maxIndexPageSize { + return nil, "", fmt.Errorf("the databricks package page from %s is larger than %d bytes", indexURL.Host, maxIndexPageSize) + } + hrefs, err := anchorHrefs(bytes.NewReader(page)) + if err != nil { + return nil, "", fmt.Errorf("failed to read the databricks package page from %s: %w", indexURL.Host, err) + } + + machine := wheelMachines[architecture] + for _, href := range hrefs { + // Links are relative to the page, which redirects may have moved. + wheelURL, err := resp.Request.URL.Parse(href) + if err != nil || !isLinuxWheel(path.Base(wheelURL.Path), version, machine) { + continue + } + digest, ok := strings.CutPrefix(wheelURL.Fragment, "sha256=") + if !ok { + return nil, "", fmt.Errorf("%s lists no SHA-256 digest for %s, so it cannot be verified", indexURL.Host, path.Base(wheelURL.Path)) + } + // The index credentials belong to its host, as they do for pip: relative links + // inherit them, and absolute links back to the same host get them here. + if wheelURL.User == nil && wheelURL.Scheme == indexURL.Scheme && wheelURL.Host == indexURL.Host { + wheelURL.User = indexURL.User + } + return wheelURL, digest, nil + } + return nil, "", fmt.Errorf("%s has no Linux %s wheel of databricks %s", indexURL.Host, machine, version) +} + +// isLinuxWheel reports whether fileName is the CLI wheel for the version and Linux machine. +// Its platform part is a compressed tag set, such as +// manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64 (see tools/build_cli_wheels.py). +// https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention +func isLinuxWheel(fileName, version, machine string) bool { + platforms, ok := strings.CutPrefix(fileName, "databricks-"+version+"-py3-none-") + if !ok { + return false + } + platforms, ok = strings.CutSuffix(platforms, ".whl") + if !ok { + return false + } + for tag := range strings.SplitSeq(platforms, ".") { + if (strings.HasPrefix(tag, "manylinux") || strings.HasPrefix(tag, "musllinux")) && strings.HasSuffix(tag, "_"+machine) { + return true + } + } + return false +} + +// anchorHrefs returns the href of every anchor on an index page. +func anchorHrefs(page io.Reader) ([]string, error) { + var hrefs []string + tokenizer := html.NewTokenizer(page) + for { + tokenType := tokenizer.Next() + if tokenType == html.ErrorToken { + if errors.Is(tokenizer.Err(), io.EOF) { + return hrefs, nil + } + return nil, tokenizer.Err() + } + if tokenType != html.StartTagToken { + continue + } + name, hasAttr := tokenizer.TagName() + if string(name) != "a" { + continue + } + for hasAttr { + var key, value []byte + key, value, hasAttr = tokenizer.TagAttr() + if string(key) == "href" { + hrefs = append(hrefs, string(value)) + } + } + } +} + +// downloadWheel downloads the wheel and checks it against the SHA-256 digest the index lists. +func downloadWheel(ctx context.Context, wheelURL *url.URL, wantSHA256 string) ([]byte, error) { + fileName := path.Base(wheelURL.Path) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, wheelURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to download %s from %s: %w", fileName, wheelURL.Host, errWithoutURL(err)) + } + log.Infof(ctx, "Downloading %s from %s", fileName, wheelURL.Host) + resp, err := pypiHTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download %s from %s: %w", fileName, wheelURL.Host, errWithoutURL(err)) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to download %s from %s: HTTP %d", fileName, wheelURL.Host, resp.StatusCode) + } + wheel, err := io.ReadAll(io.LimitReader(resp.Body, maxWheelSize+1)) + if err != nil { + return nil, fmt.Errorf("failed to download %s from %s: %w", fileName, wheelURL.Host, err) + } + if len(wheel) > maxWheelSize { + return nil, fmt.Errorf("%s from %s is larger than %d bytes", fileName, wheelURL.Host, maxWheelSize) + } + sum := sha256.Sum256(wheel) + if got := hex.EncodeToString(sum[:]); !strings.EqualFold(got, wantSHA256) { + return nil, fmt.Errorf("%s from %s has SHA-256 digest %s, but the index lists %s", fileName, wheelURL.Host, got, wantSHA256) + } + return wheel, nil +} + +// releaseArchiveFromWheel returns a zip archive with the wheel's databricks binary at its +// root, the layout of the GitHub release archive that the workspace unpacks on upload. The +// binary is copied in its compressed form, as zip.Writer.Copy does, under its new name. +func releaseArchiveFromWheel(wheel []byte, version string) ([]byte, error) { + reader, err := zip.NewReader(bytes.NewReader(wheel), int64(len(wheel))) + if err != nil { + return nil, fmt.Errorf("failed to read the databricks %s wheel: %w", version, err) + } + // The wheel installs the binary through its scripts directory (see tools/build_cli_wheels.py). + binaryPath := "databricks-" + version + ".data/scripts/databricks" + for _, file := range reader.File { + if file.Name != binaryPath { + continue + } + compressed, err := file.OpenRaw() + if err != nil { + return nil, fmt.Errorf("failed to read %s from the databricks %s wheel: %w", binaryPath, version, err) + } + header := file.FileHeader + header.Name = "databricks" + + var archive bytes.Buffer + writer := zip.NewWriter(&archive) + entry, err := writer.CreateRaw(&header) + if err != nil { + return nil, fmt.Errorf("failed to build the release archive: %w", err) + } + if _, err := io.Copy(entry, compressed); err != nil { + return nil, fmt.Errorf("failed to build the release archive: %w", err) + } + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("failed to build the release archive: %w", err) + } + return archive.Bytes(), nil + } + return nil, fmt.Errorf("the databricks %s wheel has no %s", version, binaryPath) +} + +// errWithoutURL drops the request URL that net/http adds to its errors: the URL of a +// private index can carry credentials, which net/http masks only partially. +func errWithoutURL(err error) error { + if uerr, ok := errors.AsType[*url.Error](err); ok { + return uerr.Err + } + return err +} diff --git a/experimental/ssh/internal/client/pypi_test.go b/experimental/ssh/internal/client/pypi_test.go new file mode 100644 index 00000000000..175a68eb20a --- /dev/null +++ b/experimental/ssh/internal/client/pypi_test.go @@ -0,0 +1,592 @@ +package client + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "maps" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testWheelVersion = "1.27.0" + +// The platform tags of the CLI wheels, from tools/build_cli_wheels.py. +var testWheelPlatforms = map[string]string{ + "amd64": "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64", + "arm64": "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64", +} + +func testWheelName(version, platform string) string { + return "databricks-" + version + "-py3-none-" + platform + ".whl" +} + +// testWheel returns a wheel laid out as tools/build_cli_wheels.py lays out the CLI wheels. +func testWheel(t *testing.T, version string, binary []byte) []byte { + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + files := []struct { + name string + data []byte + mode fs.FileMode + }{ + {"databricks-" + version + ".data/scripts/databricks", binary, 0o755}, + {"databricks-" + version + ".dist-info/METADATA", []byte("Metadata-Version: 2.4\nName: databricks\nVersion: " + version + "\n"), 0o644}, + {"databricks-" + version + ".dist-info/WHEEL", []byte("Wheel-Version: 1.0\nRoot-Is-Purelib: false\n"), 0o644}, + {"databricks-" + version + ".dist-info/RECORD", []byte{}, 0o644}, + } + for _, f := range files { + header := &zip.FileHeader{Name: f.name, Method: zip.Deflate} + header.SetMode(f.mode) + w, err := zw.CreateHeader(header) + require.NoError(t, err) + _, err = w.Write(f.data) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// testBinaries returns a distinct stand-in binary per architecture. +func testBinaries() map[string][]byte { + return map[string][]byte{ + "amd64": bytes.Repeat([]byte("ELF amd64 databricks "), 1000), + "arm64": bytes.Repeat([]byte("ELF arm64 databricks "), 1000), + } +} + +// testIndexFiles returns the files of the databricks project: the Linux wheels of the +// version holding the binaries, and wheels and sdists that must not be picked. +func testIndexFiles(t *testing.T, binaries map[string][]byte) map[string][]byte { + files := map[string][]byte{ + "databricks-0.2.tar.gz": []byte("an unrelated old sdist"), + "databricks-0.2-py2.py3-none-any.whl": testWheel(t, "0.2", []byte("an unrelated old wheel")), + testWheelName(testWheelVersion, "macosx_13_0_arm64"): testWheel(t, testWheelVersion, []byte("Mach-O")), + testWheelName(testWheelVersion, "win_amd64"): testWheel(t, testWheelVersion, []byte("PE")), + } + for arch, binary := range binaries { + files[testWheelName(testWheelVersion, testWheelPlatforms[arch])] = testWheel(t, testWheelVersion, binary) + files[testWheelName("1.26.0", testWheelPlatforms[arch])] = testWheel(t, "1.26.0", []byte("the previous release")) + } + return files +} + +type testIndexOptions struct { + // prefix is the path the index is served under, such as /api/pypi/pypi-remote on Artifactory. + prefix string + // relativeLinks links files relative to the project page, as Artifactory and Nexus do, + // instead of with absolute URLs, as PyPI and other mirrors do. + relativeLinks bool + // filesURL, when set, is the server that absolute links point to instead of this one, as + // PyPI points to files.pythonhosted.org. + filesURL string + // digest renders the URL fragment of a file link; nil renders #sha256=. + digest func(sum string) string + // username and password, when set, are required on every request. + username, password string + // rejectCredentials fails every request that carries credentials, or a Referer, which + // can carry the query string of the URL it came from. + rejectCredentials bool + // redirectURL, when set, is the server that /moved redirects to instead of this one. + redirectURL string + // dropDownloads closes the connection of every file download instead of answering it. + dropDownloads bool +} + +// newTestIndex serves the PEP 503 project page of databricks at /simple/databricks/, +// the files it links to under /packages/, a relative redirect to the project page +// from /moved/simple/databricks/, and an absolute one from +// /moved-absolute/simple/databricks/. +func newTestIndex(t *testing.T, files map[string][]byte, opts testIndexOptions) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("GET "+opts.prefix+"/simple/databricks/", func(w http.ResponseWriter, r *http.Request) { + filesURL := "http://" + r.Host + opts.prefix + if opts.filesURL != "" { + filesURL = opts.filesURL + } + var page strings.Builder + page.WriteString("\n\n") + for _, name := range slices.Sorted(maps.Keys(files)) { + sum := sha256.Sum256(files[name]) + fragment := "#sha256=" + hex.EncodeToString(sum[:]) + if opts.digest != nil { + fragment = opts.digest(hex.EncodeToString(sum[:])) + } + href := filesURL + "/packages/" + name + fragment + if opts.relativeLinks { + href = "../../packages/" + name + fragment + } + fmt.Fprintf(&page, "%s
\n", href, name) + } + page.WriteString("\n") + _, _ = io.WriteString(w, page.String()) + }) + mux.HandleFunc("GET "+opts.prefix+"/packages/{name}", func(w http.ResponseWriter, r *http.Request) { + if opts.dropDownloads { + if conn, _, err := http.NewResponseController(w).Hijack(); err == nil { + conn.Close() + } + return + } + data, ok := files[r.PathValue("name")] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(data) + }) + mux.HandleFunc("GET /moved"+opts.prefix+"/simple/databricks/", func(w http.ResponseWriter, r *http.Request) { + target := opts.prefix + "/simple/databricks/" + if opts.redirectURL != "" { + target = opts.redirectURL + target + } + http.Redirect(w, r, target, http.StatusFound) + }) + mux.HandleFunc("GET /moved-absolute"+opts.prefix+"/simple/databricks/", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://"+r.Host+opts.prefix+"/simple/databricks/", http.StatusFound) + }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The mux redirects a path with a doubled slash to its clean form. An index need not, + // so such a request is a miss here. + if strings.Contains(r.URL.Path, "//") { + http.NotFound(w, r) + return + } + if opts.rejectCredentials && (r.Header.Get("Authorization") != "" || r.Header.Get("Referer") != "") { + http.Error(w, "this host must not receive credentials", http.StatusBadRequest) + return + } + if opts.username != "" { + if username, password, ok := r.BasicAuth(); !ok || username != opts.username || password != opts.password { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + } + mux.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + return server +} + +// withCredentials returns serverURL with the credentials as its userinfo. +func withCredentials(serverURL, username, password string) string { + return strings.Replace(serverURL, "http://", "http://"+username+":"+password+"@", 1) +} + +// pipConfigContext returns a context in which pip is configured only by PIP_INDEX_URL, when +// not empty, and by a per-user pip config with index-url set to pipConfIndexURL, when not empty. +func pipConfigContext(t *testing.T, pipIndexURL, pipConfIndexURL string) context.Context { + home := t.TempDir() + appData := filepath.Join(home, "AppData", "Roaming") + ctx := env.WithUserHomeDir(t.Context(), home) + ctx = env.Set(ctx, "APPDATA", appData) + ctx = env.Set(ctx, "PIP_INDEX_URL", pipIndexURL) + if pipConfIndexURL != "" { + conf := "[global]\nindex-url = " + pipConfIndexURL + "\n" + // The per-user pip config is ~/.config/pip/pip.conf on Linux and macOS, and + // %APPDATA%\pip\pip.ini on Windows. + for _, p := range []string{filepath.Join(home, ".config", "pip", "pip.conf"), filepath.Join(appData, "pip", "pip.ini")} { + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, []byte(conf), 0o644)) + } + } + return ctx +} + +// archiveEntries returns the files of a release archive with their contents. +func archiveEntries(t *testing.T, release io.Reader) map[string]string { + data, err := io.ReadAll(release) + require.NoError(t, err) + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + require.NoError(t, err) + entries := map[string]string{} + for _, f := range zr.File { + r, err := f.Open() + require.NoError(t, err) + content, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + entries[f.Name] = string(content) + } + return entries +} + +// requireReleases checks that the archives getPyPIRelease builds hold the binaries. +func requireReleases(t *testing.T, ctx context.Context, binaries map[string][]byte) { + for arch, binary := range binaries { + release, err := getPyPIRelease(ctx, arch, testWheelVersion, "") + require.NoError(t, err, arch) + assert.Equal(t, map[string]string{"databricks": string(binary)}, archiveEntries(t, release), arch) + require.NoError(t, release.Close()) + } +} + +func TestGetPyPIRelease(t *testing.T) { + tests := []struct { + name string + opts testIndexOptions + // indexURL renders the index URL from the server URL. + indexURL func(serverURL string) string + }{ + { + name: "absolute links", + indexURL: func(serverURL string) string { return serverURL + "/simple" }, + }, + { + name: "relative links and an index URL with a trailing slash", + opts: testIndexOptions{prefix: "/api/pypi/pypi-remote", relativeLinks: true}, + indexURL: func(serverURL string) string { return serverURL + "/api/pypi/pypi-remote/simple/" }, + }, + { + name: "relative links on a project page that redirects", + opts: testIndexOptions{prefix: "/api/pypi/pypi-remote", relativeLinks: true}, + indexURL: func(serverURL string) string { return serverURL + "/moved/api/pypi/pypi-remote/simple" }, + }, + { + name: "index credentials sent to the index host for absolute links", + opts: testIndexOptions{username: "user", password: "secret"}, + indexURL: func(serverURL string) string { + return withCredentials(serverURL, "user", "secret") + "/simple" + }, + }, + { + name: "index credentials sent to the index host for relative links", + opts: testIndexOptions{prefix: "/api/pypi/pypi-remote", relativeLinks: true, username: "user", password: "secret"}, + indexURL: func(serverURL string) string { + return withCredentials(serverURL, "user", "secret") + "/api/pypi/pypi-remote/simple" + }, + }, + { + name: "index credentials kept on an absolute redirect to the index host", + opts: testIndexOptions{prefix: "/api/pypi/pypi-remote", relativeLinks: true, username: "user", password: "secret"}, + indexURL: func(serverURL string) string { + return withCredentials(serverURL, "user", "secret") + "/moved-absolute/api/pypi/pypi-remote/simple" + }, + }, + { + name: "upper-case digest", + opts: testIndexOptions{digest: func(sum string) string { return "#sha256=" + strings.ToUpper(sum) }}, + indexURL: func(serverURL string) string { return serverURL + "/simple" }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + binaries := testBinaries() + index := newTestIndex(t, testIndexFiles(t, binaries), tt.opts) + requireReleases(t, pipConfigContext(t, tt.indexURL(index.URL), ""), binaries) + }) + } +} + +func TestGetPyPIReleaseSendsCredentialsOnlyToTheIndexHost(t *testing.T) { + // Same machine, other port: a different host, as pip and net/http see it. + binaries := testBinaries() + files := testIndexFiles(t, binaries) + filesHost := newTestIndex(t, files, testIndexOptions{rejectCredentials: true}) + index := newTestIndex(t, files, testIndexOptions{username: "user", password: "secret", filesURL: filesHost.URL}) + + requireReleases(t, pipConfigContext(t, withCredentials(index.URL, "user", "secret")+"/simple", ""), binaries) +} + +func TestGetPyPIReleaseKeepsCredentialsOffRedirectsToOtherHosts(t *testing.T) { + // The index redirects its project page to a mirror on another host, which must receive + // neither the index credentials nor, in a Referer, the token in the index URL. + binaries := testBinaries() + files := testIndexFiles(t, binaries) + mirror := newTestIndex(t, files, testIndexOptions{rejectCredentials: true}) + index := newTestIndex(t, files, testIndexOptions{username: "user", password: "secret", redirectURL: mirror.URL}) + + indexURL := withCredentials(index.URL, "user", "secret") + "/moved/simple?token=secret" + requireReleases(t, pipConfigContext(t, indexURL, ""), binaries) +} + +func TestCheckPyPIRedirect(t *testing.T) { + const indexURL = "https://user:secret@index.test/simple/databricks/?token=secret" + const authorization = "Basic dXNlcjpzZWNyZXQ=" + + tests := []struct { + name string + target string + // followed is the number of redirects followed before this one. + followed int + wantURL string + // wantAuthorization is the Authorization header left on the redirect, which starts + // with the one net/http copied from the previous request. + wantAuthorization string + wantErr string + }{ + { + name: "absolute URL on the index origin", + target: "https://index.test/packages/databricks.whl", + wantURL: "https://user:secret@index.test/packages/databricks.whl", + wantAuthorization: authorization, + }, + { + name: "other host", + target: "https://files.test/packages/databricks.whl", + wantURL: "https://files.test/packages/databricks.whl", + }, + { + name: "subdomain of the index host", + target: "https://files.index.test/packages/databricks.whl", + wantURL: "https://files.index.test/packages/databricks.whl", + }, + { + name: "other port", + target: "https://index.test:8443/packages/databricks.whl", + wantURL: "https://index.test:8443/packages/databricks.whl", + }, + { + name: "other host with the index credentials in the URL", + target: "https://user:secret@files.test/packages/databricks.whl", + wantURL: "https://files.test/packages/databricks.whl", + }, + { + name: "plain HTTP on the index host", + target: "http://index.test/packages/databricks.whl", + wantErr: "refusing to follow a redirect from HTTPS to plain HTTP for an index URL with credentials, from index.test to index.test", + }, + { + name: "plain HTTP on another host", + target: "http://files.test/packages/databricks.whl", + wantErr: "refusing to follow a redirect from HTTPS to plain HTTP for an index URL with credentials, from index.test to files.test", + }, + { + name: "too many redirects", + target: "https://index.test/packages/databricks.whl", + followed: 9, + wantErr: "stopped after 10 redirects", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + first := httptest.NewRequest(http.MethodGet, indexURL, nil) + via := []*http.Request{first} + for range tt.followed { + via = append(via, first) + } + req := httptest.NewRequest(http.MethodGet, tt.target, nil) + req.Header.Set("Referer", "https://index.test/simple/databricks/?token=secret") + req.Header.Set("Authorization", authorization) + + err := checkPyPIRedirect(req, via) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantURL, req.URL.String()) + assert.Equal(t, tt.wantAuthorization, req.Header.Get("Authorization")) + assert.Empty(t, req.Header.Get("Referer")) + }) + } +} + +func TestCheckPyPIRedirectWithoutCredentials(t *testing.T) { + // Without credentials in the index URL there is nothing to leak, so a redirect from + // HTTPS to plain HTTP is followed. + first := httptest.NewRequest(http.MethodGet, "https://index.test/simple/databricks/", nil) + req := httptest.NewRequest(http.MethodGet, "http://index.test/packages/databricks.whl", nil) + + require.NoError(t, checkPyPIRedirect(req, []*http.Request{first})) + assert.Equal(t, "http://index.test/packages/databricks.whl", req.URL.String()) +} + +func TestGetPyPIReleaseErrors(t *testing.T) { + binaries := testBinaries() + files := testIndexFiles(t, binaries) + + tests := []struct { + name string + files map[string][]byte + opts testIndexOptions + version string + // indexPath is the path of the index on the server. + indexPath string + wantErr string + }{ + { + name: "version not on the index", + files: files, + version: "1.28.0", + indexPath: "/simple", + wantErr: "has no Linux x86_64 wheel of databricks 1.28.0", + }, + { + name: "no project page", + files: files, + version: testWheelVersion, + indexPath: "/other-index", + wantErr: "failed to fetch the databricks package from 127.0.0.1:", + }, + { + name: "digest mismatch", + files: files, + opts: testIndexOptions{digest: func(string) string { return "#sha256=" + strings.Repeat("0", 64) }}, + version: testWheelVersion, + indexPath: "/simple", + wantErr: "but the index lists " + strings.Repeat("0", 64), + }, + { + name: "no digest", + files: files, + opts: testIndexOptions{digest: func(string) string { return "" }}, + version: testWheelVersion, + indexPath: "/simple", + wantErr: "lists no SHA-256 digest for " + testWheelName(testWheelVersion, testWheelPlatforms["amd64"]), + }, + { + name: "only an MD5 digest", + files: files, + opts: testIndexOptions{digest: func(string) string { return "#md5=0123456789abcdef0123456789abcdef" }}, + version: testWheelVersion, + indexPath: "/simple", + wantErr: "lists no SHA-256 digest", + }, + { + name: "wheel without the binary", + files: map[string][]byte{ + testWheelName(testWheelVersion, testWheelPlatforms["amd64"]): testWheel(t, "1.26.0", []byte("misnamed data directory")), + }, + version: testWheelVersion, + indexPath: "/simple", + wantErr: "the databricks " + testWheelVersion + " wheel has no databricks-" + testWheelVersion + ".data/scripts/databricks", + }, + { + name: "file that is not a wheel", + files: map[string][]byte{ + testWheelName(testWheelVersion, testWheelPlatforms["amd64"]): []byte("blocked by policy"), + }, + version: testWheelVersion, + indexPath: "/simple", + wantErr: "failed to read the databricks " + testWheelVersion + " wheel", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + index := newTestIndex(t, tt.files, tt.opts) + ctx := pipConfigContext(t, index.URL+tt.indexPath, "") + + _, err := getPyPIRelease(ctx, "amd64", tt.version, "") + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestGetPyPIReleaseKeepsCredentialsOutOfErrors(t *testing.T) { + files := testIndexFiles(t, testBinaries()) + index := newTestIndex(t, files, testIndexOptions{username: "index-user", password: "index-secret"}) + dropping := newTestIndex(t, files, testIndexOptions{relativeLinks: true, username: "index-user", password: "index-secret", dropDownloads: true}) + closed := httptest.NewServer(http.NotFoundHandler()) + closed.Close() + + tests := []struct { + name string + indexURL string + wantErr string + }{ + { + name: "rejected credentials", + indexURL: withCredentials(index.URL, "index-user", "not-the-secret") + "/simple", + wantErr: "HTTP 401", + }, + { + name: "unreachable index", + indexURL: withCredentials(closed.URL, "index-user", "index-secret") + "/simple", + wantErr: "failed to fetch the databricks package from " + strings.TrimPrefix(closed.URL, "http://"), + }, + { + name: "wheel download cut off", + indexURL: withCredentials(dropping.URL, "index-user", "index-secret") + "/simple", + wantErr: "from " + strings.TrimPrefix(dropping.URL, "http://"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := pipConfigContext(t, tt.indexURL, "") + _, err := getPyPIRelease(ctx, "amd64", testWheelVersion, "") + require.ErrorContains(t, err, tt.wantErr) + assert.NotContains(t, err.Error(), "index-user") + assert.NotContains(t, err.Error(), "secret") + }) + } +} + +func TestPyPIIndexURL(t *testing.T) { + tests := []struct { + name string + pipIndexURL string + pipConfIndexURL string + want string + }{ + { + name: "PyPI by default", + want: "https://pypi.org/simple", + }, + { + name: "index-url in the pip config", + pipConfIndexURL: "https://pypi-mirror.example/simple", + want: "https://pypi-mirror.example/simple", + }, + { + name: "PIP_INDEX_URL overrides the pip config", + pipIndexURL: "https://env-mirror.example/simple", + pipConfIndexURL: "https://pypi-mirror.example/simple", + want: "https://env-mirror.example/simple", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := pipConfigContext(t, tt.pipIndexURL, tt.pipConfIndexURL) + assert.Equal(t, tt.want, pypiIndexURL(ctx)) + }) + } +} + +func TestIsLinuxWheel(t *testing.T) { + tests := []struct { + fileName string + machine string + want bool + }{ + {testWheelName(testWheelVersion, testWheelPlatforms["amd64"]), "x86_64", true}, + {testWheelName(testWheelVersion, testWheelPlatforms["arm64"]), "aarch64", true}, + {testWheelName(testWheelVersion, "manylinux_2_28_x86_64"), "x86_64", true}, + {testWheelName(testWheelVersion, "musllinux_1_2_aarch64"), "aarch64", true}, + {testWheelName(testWheelVersion, testWheelPlatforms["arm64"]), "x86_64", false}, + {testWheelName(testWheelVersion, testWheelPlatforms["amd64"]), "aarch64", false}, + {testWheelName(testWheelVersion, "macosx_13_0_x86_64"), "x86_64", false}, + {testWheelName(testWheelVersion, "win_amd64"), "x86_64", false}, + {testWheelName(testWheelVersion+"1", testWheelPlatforms["amd64"]), "x86_64", false}, + {testWheelName(testWheelVersion+".1", testWheelPlatforms["amd64"]), "x86_64", false}, + {testWheelName("1.26.0", testWheelPlatforms["amd64"]), "x86_64", false}, + {"databricks-" + testWheelVersion + ".tar.gz", "x86_64", false}, + {"databricks_connect-" + testWheelVersion + "-py3-none-" + testWheelPlatforms["amd64"] + ".whl", "x86_64", false}, + } + + for _, tt := range tests { + t.Run(tt.fileName+" "+tt.machine, func(t *testing.T) { + assert.Equal(t, tt.want, isLinuxWheel(tt.fileName, testWheelVersion, tt.machine)) + }) + } +} diff --git a/experimental/ssh/internal/client/releases.go b/experimental/ssh/internal/client/releases.go index a38d6bff29b..66c6e8bf057 100644 --- a/experimental/ssh/internal/client/releases.go +++ b/experimental/ssh/internal/client/releases.go @@ -46,13 +46,39 @@ func UploadTunnelReleases(ctx context.Context, client *databricks.WorkspaceClien return fmt.Errorf("failed to create workspace files client: %w", err) } - getRelease := getGithubRelease + getRelease := withPyPIFallback(getGithubRelease, getPyPIRelease) if releasesDir != "" { getRelease = getLocalRelease } return uploadReleases(ctx, workspaceFiler, getRelease, version, releasesDir) } +// withPyPIFallback returns a provider that downloads the release from GitHub and, when that +// fails, from the CLI's wheel on PyPI: networks that block GitHub often allow PyPI or point pip +// at a mirror of it. Development versions are not published to PyPI, so their GitHub error is +// returned as it is. +func withPyPIFallback(github, pypi releaseProvider) releaseProvider { + return func(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + release, githubErr := github(ctx, architecture, version, releasesDir) + if githubErr == nil || build.IsDevelopmentVersion(version) { + return release, githubErr + } + // Expected on networks that block GitHub, so not a warning. If PyPI fails too, the + // error below carries the GitHub failure. + log.Infof(ctx, "Failed to download the CLI release from GitHub, trying PyPI: %v", githubErr) + release, pypiErr := pypi(ctx, architecture, version, releasesDir) + if pypiErr != nil { + return nil, fmt.Errorf("%w\n\nDownloading it from PyPI failed too: %w\n\n"+ + "The SSH tunnel uploads the Linux build of the Databricks CLI to your workspace. "+ + "Allow access to GitHub (github.com, release-assets.githubusercontent.com) or PyPI "+ + "(pypi.org, files.pythonhosted.org), or set PIP_INDEX_URL, or index-url in your per-user "+ + "pip config, to a PyPI mirror that serves the databricks package", + githubErr, pypiErr) + } + return release, nil + } +} + // newHTTP11WorkspaceClient returns a workspace client that is identical to one // built from src but negotiates HTTP/1.1 only, by setting an HTTP/1.1 transport // on its config. config.Config embeds a sync.Mutex and cannot be copied by value, diff --git a/experimental/ssh/internal/client/releases_test.go b/experimental/ssh/internal/client/releases_test.go index 51b6018b614..c49f532a311 100644 --- a/experimental/ssh/internal/client/releases_test.go +++ b/experimental/ssh/internal/client/releases_test.go @@ -225,3 +225,121 @@ func TestUploadReleasesSendsEachArchiveOnce(t *testing.T) { }) } } + +func TestWithPyPIFallback(t *testing.T) { + errGithub := errors.New("dial tcp: lookup github.com: no such host") + errPyPI := errors.New("failed to fetch the databricks package from pypi.org: HTTP 403") + + provider := func(name string, err error, calls *[]string) releaseProvider { + return func(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + *calls = append(*calls, name) + if err != nil { + return nil, err + } + return io.NopCloser(strings.NewReader(name)), nil + } + } + + tests := []struct { + name string + version string + githubErr error + pypiErr error + wantCalls []string + want string + wantErrs []error + }{ + { + name: "GitHub works", + version: testWheelVersion, + wantCalls: []string{"github"}, + want: "github", + }, + { + name: "GitHub fails", + version: testWheelVersion, + githubErr: errGithub, + wantCalls: []string{"github", "pypi"}, + want: "pypi", + }, + { + name: "both fail", + version: testWheelVersion, + githubErr: errGithub, + pypiErr: errPyPI, + wantCalls: []string{"github", "pypi"}, + wantErrs: []error{errGithub, errPyPI}, + }, + { + name: "development versions are not on PyPI", + version: "1.27.1-dev+abcdef123456", + githubErr: errGithub, + wantCalls: []string{"github"}, + wantErrs: []error{errGithub}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var calls []string + getRelease := withPyPIFallback(provider("github", tt.githubErr, &calls), provider("pypi", tt.pypiErr, &calls)) + + release, err := getRelease(t.Context(), "amd64", tt.version, "") + assert.Equal(t, tt.wantCalls, calls) + if tt.wantErrs != nil { + for _, wantErr := range tt.wantErrs { + assert.ErrorIs(t, err, wantErr) + } + return + } + require.NoError(t, err) + got, err := io.ReadAll(release) + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + }) + } +} + +func TestWithPyPIFallbackNamesWhatToAllow(t *testing.T) { + failing := func(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + return nil, errors.New("blocked") + } + _, err := withPyPIFallback(failing, failing)(t.Context(), "amd64", testWheelVersion, "") + assert.ErrorContains(t, err, "Allow access to GitHub (github.com, release-assets.githubusercontent.com) or PyPI "+ + "(pypi.org, files.pythonhosted.org), or set PIP_INDEX_URL, or index-url in your per-user pip config, to a PyPI mirror") +} + +// TestUploadReleasesFromPyPI uploads the archives built from the wheels to the fake workspace, +// which unpacks them as the import-file endpoint does, and reads back the binaries the +// bootstrap runs. +func TestUploadReleasesFromPyPI(t *testing.T) { + binaries := testBinaries() + index := newTestIndex(t, testIndexFiles(t, binaries), testIndexOptions{}) + ctx := pipConfigContext(t, index.URL+"/simple", "") + + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + workspaceFiler, err := filer.NewWorkspaceFilesClient(client, "/Workspace/Users/tester@databricks.test/.databricks/ssh-tunnel/"+testWheelVersion) + require.NoError(t, err) + + githubBlocked := func(ctx context.Context, architecture, version, releasesDir string) (io.ReadCloser, error) { + return nil, errors.New("failed to download: HTTP 403") + } + err = uploadReleases(ctx, workspaceFiler, withPyPIFallback(githubBlocked, getPyPIRelease), testWheelVersion, "") + require.NoError(t, err) + + for arch, binary := range binaries { + remoteBinaryPath := path.Join(strings.TrimSuffix(getReleaseName(arch, testWheelVersion), ".zip"), "databricks") + r, err := workspaceFiler.Read(ctx, remoteBinaryPath) + require.NoError(t, err) + got, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + assert.Equal(t, binary, got, arch) + } +} diff --git a/libs/localenv/provision_integration_test.go b/libs/localenv/provision_integration_test.go index ba26615b0ab..f0f28a5da61 100644 --- a/libs/localenv/provision_integration_test.go +++ b/libs/localenv/provision_integration_test.go @@ -41,7 +41,7 @@ func requireRealProvision(t *testing.T) { // but a corporate mirror is declared in pip.conf. func realProvisionCtx(t *testing.T) context.Context { ctx := env.Set(t.Context(), EnvAutoInstallUv, "1") - if idx := pipConfIndexURL(ctx); idx != "" { + if idx := PipConfIndexURL(ctx); idx != "" { ctx = env.Set(ctx, "UV_INDEX_URL", idx) } return ctx diff --git a/libs/localenv/uv.go b/libs/localenv/uv.go index 49d2eedc3ed..f8f6d70614a 100644 --- a/libs/localenv/uv.go +++ b/libs/localenv/uv.go @@ -357,7 +357,7 @@ func (m *uvManager) pipSeedArgs(venvPython string) []string { // pipIndexURLRe matches `index-url = ` lines in pip.conf. var pipIndexURLRe = regexp.MustCompile(`(?i)^\s*index-url\s*=\s*(\S+)`) -// pipConfIndexURL reads the user's pip config and returns the index-url value. +// PipConfIndexURL reads the user's pip config and returns the index-url value. // uv ignores pip.conf; on Databricks-managed machines pypi.org is blocked and // the corporate PyPI proxy is declared via pip.conf. Bridging the value through // UV_INDEX_URL lets uv reach the proxy. @@ -365,7 +365,7 @@ var pipIndexURLRe = regexp.MustCompile(`(?i)^\s*index-url\s*=\s*(\S+)`) // pip stores its per-user config in OS-specific locations, so we probe them in // order and return the first index-url found. // https://pip.pypa.io/en/stable/topics/configuration/#location -func pipConfIndexURL(ctx context.Context) string { +func PipConfIndexURL(ctx context.Context) string { for _, confPath := range pipConfPaths(ctx) { if url := indexURLFromFile(confPath); url != "" { return url @@ -433,7 +433,7 @@ func (m *uvManager) resolveIndexURL(ctx context.Context) string { log.Debugf(ctx, "uv: UV_INDEX_URL already set in environment, not overriding") return "" } - url := pipConfIndexURL(ctx) + url := PipConfIndexURL(ctx) if url != "" { // Redact any embedded credentials: private PyPI proxies often carry // userinfo (https://user:pass@host/simple) that must not reach debug logs. diff --git a/libs/localenv/uv_test.go b/libs/localenv/uv_test.go index 26f6786b421..a1e9aacfd2a 100644 --- a/libs/localenv/uv_test.go +++ b/libs/localenv/uv_test.go @@ -298,7 +298,7 @@ func TestPipConfIndexURLReadsOSSpecificPath(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Dir(primary), 0o755)) require.NoError(t, os.WriteFile(primary, []byte("[global]\nindex-url = https://proxy.example/simple\n"), 0o644)) - assert.Equal(t, "https://proxy.example/simple", pipConfIndexURL(ctx)) + assert.Equal(t, "https://proxy.example/simple", PipConfIndexURL(ctx)) } func TestRedactURLCredentials(t *testing.T) { @@ -328,26 +328,26 @@ func TestPipConfIndexURLReadsLegacyPath(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(legacy, "pip.conf"), []byte("[global]\nindex-url = https://legacy.example/simple\n"), 0o644)) ctx := env.WithUserHomeDir(t.Context(), tmp) - assert.Equal(t, "https://legacy.example/simple", pipConfIndexURL(ctx)) + assert.Equal(t, "https://legacy.example/simple", PipConfIndexURL(ctx)) } func TestPipConfIndexURL(t *testing.T) { t.Run("returns_url_from_pip_conf", func(t *testing.T) { ctx := writePipConf(t, "[global]\nindex-url = https://proxy.example/simple\n") - got := pipConfIndexURL(ctx) + got := PipConfIndexURL(ctx) assert.Equal(t, "https://proxy.example/simple", got) }) t.Run("returns_empty_when_no_pip_conf", func(t *testing.T) { tmp := t.TempDir() ctx := env.WithUserHomeDir(t.Context(), tmp) - got := pipConfIndexURL(ctx) + got := PipConfIndexURL(ctx) assert.Empty(t, got) }) t.Run("returns_empty_when_no_index_url_in_conf", func(t *testing.T) { ctx := writePipConf(t, "[global]\nextra-index-url = https://other.example/simple\n") - got := pipConfIndexURL(ctx) + got := PipConfIndexURL(ctx) assert.Empty(t, got) }) }