Skip to content

Commit 66580de

Browse files
vuanhphungIsaac
andcommitted
perf: rebind pyarrow to the real module on first use
client, utils and result_set kept the stand-in as their pyarrow global after the first result, so every pyarrow.X on the query path went through a ModuleType subclass's slower attribute lookup. The first use now replaces the stand-in with the real module in each module that registered its globals. Co-authored-by: Isaac <no-reply@databricks.com> Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>
1 parent f77297c commit 66580de

5 files changed

Lines changed: 57 additions & 8 deletions

File tree

‎src/databricks/sql/client.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
if TYPE_CHECKING:
1919
import pyarrow
2020
else:
21-
pyarrow = optional_pyarrow()
21+
pyarrow = optional_pyarrow(globals())
2222
import json
2323
import os
2424
import decimal

‎src/databricks/sql/common/optional_pyarrow.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,29 @@
1414
import importlib.util
1515
import sys
1616
import types
17-
from typing import Any, Optional
17+
from typing import Any, Dict, List, Optional
18+
19+
# Module globals whose ``pyarrow`` is the stand-in, rebound on first use.
20+
_NAMESPACES: List[Dict[str, Any]] = []
1821

1922

2023
class _LazyPyarrow(types.ModuleType):
21-
"""Stands in for the ``pyarrow`` module until an attribute is used."""
24+
"""Stands in for the ``pyarrow`` module until an attribute is used.
25+
26+
The first use rebinds ``pyarrow`` to the real module in every registered
27+
namespace still holding the stand-in, so reads on the hot path afterwards
28+
are plain module attribute lookups.
29+
"""
2230

2331
def __getattr__(self, name: str) -> Any:
2432
try:
2533
module = importlib.import_module("pyarrow")
2634
except ImportError as e:
2735
# AttributeError, so hasattr() and getattr(..., default) still work.
2836
raise AttributeError(f"pyarrow could not be imported: {e}") from e
37+
for namespace in _NAMESPACES:
38+
if namespace.get("pyarrow") is self:
39+
namespace["pyarrow"] = module
2940
value = getattr(module, name)
3041
setattr(self, name, value)
3142
return value
@@ -34,12 +45,16 @@ def __getattr__(self, name: str) -> Any:
3445
_LAZY_PYARROW = _LazyPyarrow("pyarrow")
3546

3647

37-
def optional_pyarrow() -> Optional[types.ModuleType]:
48+
def optional_pyarrow(
49+
namespace: Optional[Dict[str, Any]] = None,
50+
) -> Optional[types.ModuleType]:
3851
"""``pyarrow``, or None when it isn't installed, without importing it.
3952
4053
Returns the real module if it is already imported, otherwise a stand-in that
4154
imports it on first attribute access. Whether pyarrow is installed is decided
42-
with ``find_spec``, so ``pyarrow is None`` checks behave as before.
55+
with ``find_spec``, so ``pyarrow is None`` checks behave as before. Pass the
56+
caller's ``globals()`` to have its ``pyarrow`` replaced by the real module
57+
once the stand-in is first used.
4358
"""
4459
module = sys.modules.get("pyarrow")
4560
if module is not None:
@@ -48,4 +63,8 @@ def optional_pyarrow() -> Optional[types.ModuleType]:
4863
installed = importlib.util.find_spec("pyarrow") is not None
4964
except (ImportError, ValueError):
5065
installed = False
51-
return _LAZY_PYARROW if installed else None
66+
if not installed:
67+
return None
68+
if namespace is not None:
69+
_NAMESPACES.append(namespace)
70+
return _LAZY_PYARROW

‎src/databricks/sql/result_set.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
if TYPE_CHECKING:
1111
import pyarrow
1212
else:
13-
pyarrow = optional_pyarrow()
13+
pyarrow = optional_pyarrow(globals())
1414

1515
if TYPE_CHECKING:
1616
from databricks.sql.backend.thrift_backend import ThriftDatabricksClient

‎src/databricks/sql/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
if TYPE_CHECKING:
3131
import pyarrow
3232
else:
33-
pyarrow = optional_pyarrow()
33+
pyarrow = optional_pyarrow(globals())
3434

3535
from databricks.sql import OperationalError
3636
from databricks.sql.exc import ProgrammingError

‎tests/unit/test_lazy_pyarrow_import.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,36 @@ def test_lazy_pyarrow_resolves_to_the_real_module():
121121
assert proc.returncode == 0, proc.stderr
122122

123123

124+
def test_first_use_rebinds_the_modules_to_the_real_pyarrow():
125+
# After the first read, hot-path reads go to the real module, not the
126+
# stand-in's __getattr__.
127+
proc = _run("""
128+
import databricks.sql.client as client
129+
import databricks.sql.result_set as result_set
130+
import databricks.sql.utils as utils
131+
132+
stand_in = utils.pyarrow
133+
assert "pyarrow" not in sys.modules
134+
assert all(m.pyarrow is stand_in for m in (client, result_set))
135+
stand_in.Table
136+
real = sys.modules["pyarrow"]
137+
assert all(m.pyarrow is real for m in (client, utils, result_set))
138+
assert stand_in.Table is real.Table
139+
""")
140+
assert proc.returncode == 0, proc.stderr
141+
142+
143+
def test_rebinding_leaves_a_replaced_global_alone(monkeypatch):
144+
stand_in = optional_pyarrow_module._LazyPyarrow("pyarrow")
145+
replaced, holding = {"pyarrow": "patched"}, {"pyarrow": stand_in}
146+
monkeypatch.setattr(optional_pyarrow_module, "_NAMESPACES", [replaced, holding])
147+
import pyarrow
148+
149+
assert stand_in.Table is pyarrow.Table
150+
assert holding["pyarrow"] is pyarrow
151+
assert replaced["pyarrow"] == "patched"
152+
153+
124154
def test_optional_pyarrow_is_none_when_not_installed(monkeypatch):
125155
monkeypatch.setitem(sys.modules, "pyarrow", None)
126156
monkeypatch.setattr(

0 commit comments

Comments
 (0)