diff --git a/src/opensemantic/base/_dataset_controller_mixin.py b/src/opensemantic/base/_dataset_controller_mixin.py index f32a030..64142e0 100644 --- a/src/opensemantic/base/_dataset_controller_mixin.py +++ b/src/opensemantic/base/_dataset_controller_mixin.py @@ -146,6 +146,99 @@ class fails validation, so the layer the dataset belongs to decides. return RemoteFile +def _json_entries(payload: bytes) -> List[Any]: + """The row entries of a serialized dataset, list or ``{"data": [...]}``.""" + entries = json.loads(payload.decode("utf-8")) + if isinstance(entries, dict): + entries = entries.get("data", []) + return entries + + +def _pint_unit_for_iri(value_cls, iri: str) -> Optional[str]: + """The pint unit string a serialized unit IRI stands for.""" + unit_field = _fields_of(value_cls).get("unit") if _is_model_cls(value_cls) else None + if unit_field is None: + return None + annotation = getattr(unit_field, "annotation", None) or getattr( + unit_field, "type_", None + ) + members = getattr(_unwrap(annotation), "__members__", {}) + for name, member in members.items(): + if getattr(member, "value", None) == iri: + to_pint_str = getattr(value_cls, "get_pint_ureg_compatible_str", None) + return to_pint_str(name) if callable(to_pint_str) else name + return None + + +def _frame_from_entries(row_cls, entries: List[Any]): + """Serialized rows as a frame, column by column, without building rows. + + A serialized value is ``{"value": x, "unit": "Item:OSW..."}``. When a + column holds one unit - which is what a dataset written in one go looks + like - the magnitudes are read straight out and the unit is attached once. + A column that mixes units keeps its values and is left for the caller to + convert, since there is no single unit to give the column. + """ + import pandas as pd + + columns = {} + for field in _value_fields(row_cls): + value_cls = _field_type(row_cls, field) + raw = [ + entry.get(field) if isinstance(entry, dict) else getattr(entry, field, None) + for entry in entries + ] + iris = { + value.get("unit") + for value in raw + if isinstance(value, dict) and value.get("unit") is not None + } + magnitudes = [ + value.get("value") if isinstance(value, dict) else value for value in raw + ] + unit = _pint_unit_for_iri(value_cls, iris.pop()) if len(iris) == 1 else None + if unit is None: + unit = _declared_unit(row_cls, field) if not iris else None + if unit is None: + columns[field] = pd.Series(magnitudes, name=field, dtype="object") + else: + columns[field] = pd.Series(magnitudes, dtype=f"pint[{unit}]", name=field) + return pd.DataFrame(columns) + + +def _own_unit(value_cls, value) -> Optional[str]: + """The pint unit a single value is already expressed in, if readable.""" + name = getattr(getattr(value, "unit", None), "name", None) + if name is None: + return None + to_pint_str = getattr(value_cls, "get_pint_ureg_compatible_str", None) + return to_pint_str(name) if callable(to_pint_str) else name + + +def _column_magnitudes(value_cls, values: List[Any], unit: str) -> List[Any]: + """A column's magnitudes in ``unit``. + + Converting each value on its own costs a pint quantity, a unit parse and a + conversion per cell, which dominates a dataset of a few thousand rows. A + column almost always holds one unit, and when that unit is the target the + stored magnitudes already are the answer, so the whole column is read + without pint. A column that mixes units falls back to per-value + conversion. + """ + present = [value for value in values if value is not None] + units = {_own_unit(value_cls, value) for value in present} + if ( + len(units) == 1 + and units.pop() == unit + and all(hasattr(value, "value") for value in present) + ): + return [value.value if value is not None else None for value in values] + return [ + value.to_pint().to(unit).magnitude if value is not None else None + for value in values + ] + + def _is_missing(value) -> bool: """Whether a frame cell carries no value. @@ -281,10 +374,7 @@ def to_df(self): if unit is None: columns[field] = pd.Series(values, name=field, dtype="object") continue - magnitudes = [ - value.to_pint().to(unit).magnitude if value is not None else None - for value in values - ] + magnitudes = _column_magnitudes(_field_type(row_cls, field), values, unit) columns[field] = pd.Series(magnitudes, dtype=f"pint[{unit}]", name=field) return pd.DataFrame(columns) @@ -356,13 +446,13 @@ def _distribution_controller(self, distribution, **controller_kwargs): remote_cls(label=self.label, download_url=url), **controller_kwargs ) - def materialize_data(self, distribution=None, **controller_kwargs): - """Fill ``data`` from a distribution's file and return the rows. + def _read_distribution(self, parse, distribution=None, **controller_kwargs): + """Read the first distribution ``parse`` can make something of. - Without an explicit distribution the first one that yields rows is - used: a distribution whose file cannot be read or parsed is reported - and the next one is tried. The file is parsed by its ``media_type`` - (JSON or CSV). + A distribution whose file cannot be reached, read or parsed is + reported and the next one is tried; the error raised at the end + chains the last cause, so a column that will not parse does not read + as "there is no file". ``controller_kwargs`` are handed to the file controller, which is how a caller passes the session or credentials the file needs. @@ -372,9 +462,6 @@ def materialize_data(self, distribution=None, **controller_kwargs): if distribution is not None else list(getattr(self, "distributions", None) or []) ) - row_cls = row_class_of(type(self)) - if row_cls is None: - raise TypeError(f"{type(self).__name__}.data carries no row model to fill") last_error: Optional[Exception] = None for dist in distributions: try: @@ -388,22 +475,8 @@ def materialize_data(self, distribution=None, **controller_kwargs): payload = handle.read() if isinstance(payload, str): payload = payload.encode("utf-8") - media_type = getattr(dist, "media_type", None) or JSON_MEDIA_TYPE - if media_type == CSV_MEDIA_TYPE: - self.data = ( - type(self) - .from_df(_pint_csv.read_csv(payload), label=self.label) - .data - ) - else: - entries = json.loads(payload.decode("utf-8")) - if isinstance(entries, dict): - entries = entries.get("data", []) - self.data = [ - entry if isinstance(entry, row_cls) else row_cls(**entry) - for entry in entries - ] + return parse(payload, media_type) except Exception as error: # noqa: BLE001 - the next one may work last_error = error _logger.warning( @@ -413,13 +486,61 @@ def materialize_data(self, distribution=None, **controller_kwargs): error, ) continue - return self.data - # chained: without the cause, a column that cannot be parsed reads as - # "there is no file", and the error naming the field is lost raise ValueError( f"{type(self).__name__} has no distribution with a readable file" ) from last_error + def materialize_df(self, distribution=None, **controller_kwargs): + """A distribution's content as a DataFrame, without building rows. + + What a caller that only plots or aggregates needs. Building the typed + rows first and flattening them again costs a pydantic model per cell, + which is what makes a dataset of a hundred thousand rows take minutes; + read straight into a frame and it takes the time pandas needs. + + ``data`` is left alone. Use :meth:`materialize_data` for the rows. + """ + row_cls = row_class_of(type(self)) + + def parse(payload: bytes, media_type: str): + if media_type == CSV_MEDIA_TYPE: + return _pint_csv.read_csv(payload) + if row_cls is None: + raise TypeError( + f"{type(self).__name__}.data carries no row model to read" + ) + return _frame_from_entries(row_cls, _json_entries(payload)) + + return self._read_distribution(parse, distribution, **controller_kwargs) + + def materialize_data(self, distribution=None, **controller_kwargs): + """Fill ``data`` from a distribution's file and return the rows. + + Without an explicit distribution the first one that yields rows is + used. The file is parsed by its ``media_type`` (JSON or CSV). + + ``controller_kwargs`` are handed to the file controller, which is how + a caller passes the session or credentials the file needs. + """ + row_cls = row_class_of(type(self)) + if row_cls is None: + raise TypeError(f"{type(self).__name__}.data carries no row model to fill") + + def parse(payload: bytes, media_type: str): + if media_type == CSV_MEDIA_TYPE: + return ( + type(self) + .from_df(_pint_csv.read_csv(payload), label=self.label) + .data + ) + return [ + entry if isinstance(entry, row_cls) else row_cls(**entry) + for entry in _json_entries(payload) + ] + + self.data = self._read_distribution(parse, distribution, **controller_kwargs) + return self.data + def externalize_data(self, file, fmt: str = "json", keep_data: bool = False): """Write ``data`` to ``file``, attach a distribution and clear ``data``. diff --git a/tests/test_dataset_controller.py b/tests/test_dataset_controller.py index 7996b94..d5620d3 100644 --- a/tests/test_dataset_controller.py +++ b/tests/test_dataset_controller.py @@ -527,3 +527,65 @@ def test_a_foreign_csv_is_read_with_one_header_row(plain_ctx, tmp_path): rows = dataset.materialize_data() assert [(row.ID, row.city) for row in rows] == [(18, "Cambridge")] + + +# -- frames without building rows --------------------------------------------- + + +@pytest.mark.parametrize("fmt", ["json", "csv"]) +def test_materialize_df_matches_the_rows(ctx, dataset, fmt, tmp_path): + """Same numbers as the row path, without constructing a single row.""" + expected = dataset.to_df() + dataset.externalize_data(_local_file(ctx, tmp_path / f"rows.{fmt}"), fmt=fmt) + + frame = dataset.materialize_df() + + assert list(frame.columns) == list(expected.columns) + assert len(frame) == len(expected) + assert str(frame["temperature"].pint.units) == "kelvin" + assert frame["temperature"].pint.magnitude.tolist() == pytest.approx( + expected["temperature"].pint.magnitude.tolist() + ) + + +@pytest.mark.parametrize("fmt", ["json", "csv"]) +def test_materialize_df_leaves_data_alone(ctx, dataset, fmt, tmp_path): + dataset.externalize_data(_local_file(ctx, tmp_path / f"rows.{fmt}"), fmt=fmt) + assert dataset.data is None + + dataset.materialize_df() + + assert dataset.data is None + + +@pytest.mark.parametrize("fmt", ["json", "csv"]) +def test_materialize_df_carries_plain_columns(mixed_ctx, fmt, tmp_path): + """A row field that is not a quantity survives the frame path too.""" + dataset = _mixed_dataset(mixed_ctx) + dataset.externalize_data(_local_file(mixed_ctx, tmp_path / f"m.{fmt}"), fmt=fmt) + + frame = dataset.materialize_df() + + assert frame["note"].tolist() == ["start", "end"] + assert frame["cycle"].tolist() == [1, 2] + + +def test_materialize_df_reports_an_unreadable_distribution(ctx, dataset, tmp_path): + dataset.externalize_data(_local_file(ctx, tmp_path / "rows.json")) + (tmp_path / "rows.json").unlink() + + with pytest.raises(ValueError, match="no distribution"): + dataset.materialize_df() + + +def test_to_df_converts_a_column_that_mixes_units(ctx): + """The vectorized path only applies when a column agrees on one unit.""" + hot = ctx.Temperature(value=300.0) + cold = ctx.Temperature(value=30.0, unit=type(hot.unit)["Celsius"]) + data = [ctx.Row(temperature=hot), ctx.Row(temperature=cold)] + dataset = ctx.cls(label=[ctx.core.Label(text="Mixed units")], data=data) + + column = dataset.to_df()["temperature"] + + assert str(column.pint.units) == "kelvin" + assert column.pint.magnitude.tolist() == pytest.approx([300.0, 303.15])