Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 208 additions & 3 deletions src/ucode/databricks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import configparser
import functools
import io
import json
import logging
import logging.handlers
Expand All @@ -15,6 +16,7 @@
import re
import shlex
import shutil
import ssl
import subprocess
import sys
import tempfile
Expand All @@ -27,12 +29,16 @@
from email.message import Message
from enum import Enum
from http.client import HTTPException
from http.cookiejar import CookieJar, DefaultCookiePolicy
from pathlib import Path
from typing import Any, Literal, NamedTuple, NoReturn, TypeGuard, cast, overload
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, NoReturn, TypeGuard, cast, overload
from urllib import error as urllib_error
from urllib import request as urllib_request
from urllib.parse import quote, urlencode, urlparse

if TYPE_CHECKING:
import httpx

from ucode import metrics, tls
from ucode.codex_config import catalog_slugs
from ucode.config_io import APP_DIR
Expand Down Expand Up @@ -376,6 +382,205 @@ def clear_workspace_org_id_cache() -> None:
_WORKSPACE_ORG_IDS.clear()


class _RejectCookiesPolicy(DefaultCookiePolicy):
def set_ok(self, cookie, request) -> bool:
return False


def _macos_system_proxies() -> dict[str, str] | None:
if sys.platform != "darwin" or any(
os.environ.get(name)
for name in (
"HTTP_PROXY",
"http_proxy",
"HTTPS_PROXY",
"https_proxy",
"ALL_PROXY",
"all_proxy",
)
):
return None
proxies = urllib_request.getproxies()
system_proxies: dict[str, str] = {}
for scheme in ("http", "https", "all"):
proxy = proxies.get(scheme)
if not proxy:
continue
if "://" not in proxy:
# macOS can report scheme-less values ("host:port"); urllib's proxy handler
# treats those as HTTP proxies, so make that explicit for httpx.
proxy = f"http://{proxy}"
system_proxies[scheme] = proxy
return system_proxies or None


def _urllib_proxy_for_request(scheme: str, host: str, proxies: dict[str, str]) -> str | None:
if urllib_request.proxy_bypass(host):
return None
return proxies.get(scheme) or proxies.get("all")


def _macos_proxy_transport(httpx, proxies, verify, limits):
class SystemProxyTransport(httpx.BaseTransport):
def __init__(self):
self.direct = httpx.HTTPTransport(verify=verify, trust_env=False, limits=limits)
self.proxied = {
proxy: httpx.HTTPTransport(
proxy=proxy, verify=verify, trust_env=False, limits=limits
)
for proxy in set(proxies.values())
}

def handle_request(self, request):
proxy = _urllib_proxy_for_request(request.url.scheme, request.url.host, proxies)
transport = self.proxied.get(proxy) if proxy else self.direct
return transport.handle_request(request)

def close(self):
self.direct.close()
for transport in self.proxied.values():
transport.close()

return SystemProxyTransport()


_HTTP_CLIENT: httpx.Client | None = None
_HTTP_CLIENT_PID: int | None = None
_HTTP_CLIENT_LOCK = threading.Lock()


def _reset_http_client_after_fork() -> None:
global _HTTP_CLIENT, _HTTP_CLIENT_PID, _HTTP_CLIENT_LOCK
_HTTP_CLIENT = None
_HTTP_CLIENT_PID = os.getpid()
_HTTP_CLIENT_LOCK = threading.Lock()


if hasattr(os, "register_at_fork"):
os.register_at_fork(after_in_child=_reset_http_client_after_fork)


def _get_http_client() -> httpx.Client:
import httpx

global _HTTP_CLIENT, _HTTP_CLIENT_PID
pid = os.getpid()
if _HTTP_CLIENT is not None and _HTTP_CLIENT_PID == pid:
return _HTTP_CLIENT
with _HTTP_CLIENT_LOCK:
if _HTTP_CLIENT is None or _HTTP_CLIENT_PID != pid:
# Before building the SSL context, so the pooled client verifies against the same
# trust store (os or python) as the urllib path did.
tls.apply()
verify = ssl.create_default_context()
limits = httpx.Limits(max_connections=100, max_keepalive_connections=20)
system_proxies = _macos_system_proxies()
transport = (
_macos_proxy_transport(httpx, system_proxies, verify, limits)
if system_proxies
else None
)
client = httpx.Client(
verify=verify,
trust_env=transport is None,
follow_redirects=True,
limits=limits,
cookies=CookieJar(policy=_RejectCookiesPolicy()),
**({"transport": transport} if transport is not None else {}),
)
_HTTP_CLIENT = client
_HTTP_CLIENT_PID = pid
return _HTTP_CLIENT


class _PooledResponse:
def __init__(self, response: httpx.Response):
self.status = response.status_code
self.headers = Message()
for name, value in response.headers.multi_items():
self.headers.add_header(name, value)
self._body = response.content
self._url = str(response.url)

def __enter__(self):
return self

def __exit__(self, *exc):
return False

def geturl(self) -> str:
return self._url

def read(self) -> bytes:
return self._body


def _http_open(request: urllib_request.Request, *, timeout: float) -> _PooledResponse:
import httpx

request_headers = dict(request.header_items())
header_names = {name.casefold() for name in request_headers}
if "accept-encoding" not in header_names:
request_headers["Accept-Encoding"] = "identity"
if "connection" not in header_names:
request_headers["Connection"] = "keep-alive"
if "user-agent" not in header_names:
# urllib's own default: '%d.%d' % sys.version_info[:2].
request_headers["User-Agent"] = f"Python-urllib/{sys.version_info[0]}.{sys.version_info[1]}"
httpx_request = httpx.Request(
request.get_method(),
request.full_url,
headers=request_headers,
# The only Request built with data is `_http_send_json`'s, which sends bytes or None.
content=cast("bytes | None", request.data),
extensions={"timeout": httpx.Timeout(timeout).as_dict()},
)
try:
response = _get_http_client().send(httpx_request, follow_redirects=True)
except httpx.TooManyRedirects as exc:
# urllib's redirect handler raises HTTPError when it exceeds its redirect budget, so
# callers see a status-shaped failure, not a network one. httpx drops the last response,
# so report the generic 310 (Too Many Redirects) status.
raise urllib_error.HTTPError(
request.full_url, 310, "Too Many Redirects", Message(), io.BytesIO(b"")
) from exc
except httpx.RequestError as exc:
pending: list[BaseException] = [exc]
seen: set[int] = set()
reason: BaseException = exc
while pending:
current = pending.pop()
if id(current) in seen:
continue
seen.add(id(current))
if isinstance(current, OSError):
reason = current
break
reason = current
if isinstance(current, BaseExceptionGroup):
pending.extend(current.exceptions)
if current.__cause__ is not None:
pending.append(current.__cause__)
elif current.__context__ is not None:
pending.append(current.__context__)
raise urllib_error.URLError(reason) from exc
except httpx.HTTPError as exc:
# Other transport-level httpx errors (the pool closed, a stream broke): urllib has no
# counterpart, so surface them as network errors like the RequestError mapping above.
raise urllib_error.URLError(exc) from exc

pooled_response = _PooledResponse(response)
if not 200 <= response.status_code < 300:
raise urllib_error.HTTPError(
pooled_response.geturl(),
response.status_code,
response.reason_phrase,
pooled_response.headers,
io.BytesIO(pooled_response.read()),
)
return pooled_response


@metrics.http_result
def _http_get_bytes(
url: str,
Expand Down Expand Up @@ -406,7 +611,7 @@ def _http_get_bytes(
if attempt:
metrics.record(metrics.HTTP_RETRY_COUNT, 1, api_operation=operation, http_method="GET")
try:
with urllib_request.urlopen(request, timeout=timeout) as response:
with _http_open(request, timeout=timeout) as response:
status = response.status
body = response.read()
failure = _workspace_response_failure(url, response)
Expand Down Expand Up @@ -537,7 +742,7 @@ def _http_send_json(
request_started = time.perf_counter()
status = None
try:
with urllib_request.urlopen(request, timeout=timeout) as response:
with _http_open(request, timeout=timeout) as response:
status = response.status
body = response.read().decode("utf-8")
failure = _workspace_response_failure(url, response)
Expand Down
6 changes: 5 additions & 1 deletion src/ucode/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,11 @@ def host_type(host: str) -> str | None:
)
result: str | None = None
try:
with urllib_request.urlopen(request, timeout=_HOST_TYPE_TIMEOUT_SECONDS) as response:
# Imported lazily: databricks imports this module. This unauthenticated request rides the
# same pooled HTTP client as every other workspace call.
from ucode.databricks import _http_open

with _http_open(request, timeout=_HOST_TYPE_TIMEOUT_SECONDS) as response:
payload = json.loads(response.read().decode("utf-8"))
value = payload.get("host_type") if isinstance(payload, dict) else None
result = value.lower() if isinstance(value, str) else None
Expand Down
Loading
Loading