Skip to content

Commit 517c4c8

Browse files
vuanhphungIsaac
andcommitted
fix: try a failed pyarrow import only once
When pyarrow is installed but can't be imported, every attribute read on the stand-in retried the import, rerunning pyarrow's and numpy's partial initialization. The first ImportError is now kept and later reads raise from it, as the eager import ran it once. Co-authored-by: Isaac <no-reply@databricks.com> Signed-off-by: Vu Anh Phung <vu.phung@databricks.com>
1 parent 66580de commit 517c4c8

2 files changed

Lines changed: 16 additions & 5 deletions

File tree

‎src/databricks/sql/common/optional_pyarrow.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,17 @@ class _LazyPyarrow(types.ModuleType):
2929
"""
3030

3131
def __getattr__(self, name: str) -> Any:
32-
try:
33-
module = importlib.import_module("pyarrow")
34-
except ImportError as e:
32+
# A failed import is tried once, as an eager import was: retrying
33+
# would rerun pyarrow's (and numpy's) partial initialization each time.
34+
error = self.__dict__.get("_import_error")
35+
if error is None:
36+
try:
37+
module = importlib.import_module("pyarrow")
38+
except ImportError as e:
39+
self.__dict__["_import_error"] = error = e
40+
if error is not None:
3541
# AttributeError, so hasattr() and getattr(..., default) still work.
36-
raise AttributeError(f"pyarrow could not be imported: {e}") from e
42+
raise AttributeError(f"pyarrow could not be imported: {error}") from error
3743
for namespace in _NAMESPACES:
3844
if namespace.get("pyarrow") is self:
3945
namespace["pyarrow"] = module

‎tests/unit/test_lazy_pyarrow_import.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,12 +160,17 @@ def test_optional_pyarrow_is_none_when_not_installed(monkeypatch):
160160

161161

162162
def test_unimportable_pyarrow_raises_attribute_error(monkeypatch):
163+
attempts = []
164+
163165
def broken(name):
166+
attempts.append(name)
164167
raise ImportError("numpy ABI mismatch")
165168

166169
monkeypatch.setattr(optional_pyarrow_module.importlib, "import_module", broken)
167170
lazy = optional_pyarrow_module._LazyPyarrow("pyarrow")
168171
assert not hasattr(lazy, "Table")
169172
assert getattr(lazy, "Table", None) is None
170-
with pytest.raises(AttributeError, match="numpy ABI mismatch"):
173+
with pytest.raises(AttributeError, match="numpy ABI mismatch") as raised:
171174
lazy.Table
175+
assert isinstance(raised.value.__cause__, ImportError)
176+
assert attempts == ["pyarrow"], "a failed import is not retried"

0 commit comments

Comments
 (0)