Monthly Merge To production

DashAI/alembic/versions/3db684f4090a_merge_datafile_and_dataset_heads.py

@@ -0,0 +1,26 @@
+"""merge datafile and dataset heads
+
+Revision ID: 3db684f4090a
+Revises: a1f8e3b0c2d9, c3d7a1f05e8b
+Create Date: 2026-05-27 15:49:47.570864
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '3db684f4090a'
+down_revision: Union[str, None] = ('a1f8e3b0c2d9', 'c3d7a1f05e8b')
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass

DashAI/alembic/versions/a1c3e5f7b9d2_add_hub_download_table.py

@@ -0,0 +1,47 @@
+"""Add datafile table
+
+Revision ID: a1c3e5f7b9d2
+Revises: b4f9e70098e7
+Create Date: 2026-05-08 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "a1c3e5f7b9d2"
+down_revision: Union[str, None] = "b4f9e70098e7"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "datafile",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("source_name", sa.String(), nullable=False),
+        sa.Column("dataset_id", sa.String(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("local_path", sa.String(), nullable=True),
+        sa.Column(
+            "status",
+            sa.Enum("downloading", "ready", "error", name="hubdownloadstatus"),
+            nullable=False,
+        ),
+        sa.Column("error_message", sa.String(), nullable=True),
+        sa.Column("created", sa.DateTime(), nullable=False),
+        sa.Column("last_modified", sa.DateTime(), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_datafile")),
+        sa.UniqueConstraint(
+            "source_name",
+            "dataset_id",
+            name="uq_datafile_source_dataset",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("datafile")

DashAI/alembic/versions/a1f8e3b0c2d9_add_total_rows_columns_to_dataset.py

@@ -0,0 +1,27 @@
+"""Add total_rows and total_columns to dataset
+
+Revision ID: a1f8e3b0c2d9
+Revises: b4f9e70098e7
+Create Date: 2026-05-14 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "a1f8e3b0c2d9"
+down_revision: Union[str, None] = "b4f9e70098e7"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("dataset", sa.Column("total_rows", sa.Integer(), nullable=True))
+    op.add_column("dataset", sa.Column("total_columns", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("dataset", "total_columns")
+    op.drop_column("dataset", "total_rows")

DashAI/alembic/versions/c3d7a1f05e8b_add_metadata_to_datafile.py

@@ -0,0 +1,32 @@
+"""Add metadata columns to datafile table
+
+Revision ID: c3d7a1f05e8b
+Revises: a1c3e5f7b9d2
+Create Date: 2026-05-12 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "c3d7a1f05e8b"
+down_revision: Union[str, None] = "a1c3e5f7b9d2"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("datafile", sa.Column("size_bytes", sa.BigInteger(), nullable=True))
+    op.add_column("datafile", sa.Column("description", sa.Text(), nullable=True))
+    op.add_column("datafile", sa.Column("tags", sa.Text(), nullable=True))
+    op.add_column("datafile", sa.Column("source_url", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("datafile", "source_url")
+    op.drop_column("datafile", "tags")
+    op.drop_column("datafile", "description")
+    op.drop_column("datafile", "size_bytes")

DashAI/back/api/api_v1/api.py

@@ -2,6 +2,8 @@
 
 from DashAI.back.api.api_v1.endpoints.components import router as components
 from DashAI.back.api.api_v1.endpoints.converters import router as converters
+from DashAI.back.api.api_v1.endpoints.datafile import router as datafile_router
+from DashAI.back.api.api_v1.endpoints.dataset_source import router as dataset_source
 from DashAI.back.api.api_v1.endpoints.datasets import router as datasets
 from DashAI.back.api.api_v1.endpoints.explainers import router as explainers
 from DashAI.back.api.api_v1.endpoints.explorers import router as explorers
@@ -40,3 +42,5 @@
 api_router_v1.include_router(metrics, prefix="/metrics")
 api_router_v1.include_router(hardware, prefix="/hardware")
 api_router_v1.include_router(scoring, prefix="/scoring")
+api_router_v1.include_router(dataset_source, prefix="/dataset-source")
+api_router_v1.include_router(datafile_router, prefix="/datafile")

DashAI/back/api/api_v1/endpoints/components.py

@@ -353,33 +353,45 @@ async def get_component_image(
         .get("metadata", {})
         .get("image_preview", None)
     )
+    cache_headers = {"Cache-Control": "public, max-age=3600"}
+
     if not image_path:
         with open(local_path_image / "placeholder.svg", "rb") as image_file:
             return StreamingResponse(
-                io.BytesIO(image_file.read()), media_type="image/svg+xml"
+                io.BytesIO(image_file.read()),
+                media_type="image/svg+xml",
+                headers=cache_headers,
             )
 
     # If it is a URL, we obtain the image from the URL
     if image_path.startswith(("http://", "https://")):
         response = requests.get(image_path, timeout=5)
         if response.status_code == 200:
             return StreamingResponse(
-                io.BytesIO(response.content), media_type="image/png"
+                io.BytesIO(response.content),
+                media_type="image/png",
+                headers=cache_headers,
             )
         else:
             with open(local_path_image / "placeholder.svg", "rb") as image_file:
                 return StreamingResponse(
-                    io.BytesIO(image_file.read()), media_type="image/svg+xml"
+                    io.BytesIO(image_file.read()),
+                    media_type="image/svg+xml",
+                    headers=cache_headers,
                 )
 
     # Otherwise, we assume it is a local path
     try:
         with open(local_path_image / image_path, "rb") as image_file:
             return StreamingResponse(
-                io.BytesIO(image_file.read()), media_type="image/png"
+                io.BytesIO(image_file.read()),
+                media_type="image/png",
+                headers=cache_headers,
             )
     except FileNotFoundError:
         with open(local_path_image / "placeholder.svg", "rb") as image_file:
             return StreamingResponse(
-                io.BytesIO(image_file.read()), media_type="image/svg+xml"
+                io.BytesIO(image_file.read()),
+                media_type="image/svg+xml",
+                headers=cache_headers,
             )

DashAI/back/api/api_v1/endpoints/converters.py

@@ -229,15 +229,23 @@ async def delete_converter(
                 notebook.file_path,
                 dirs_exist_ok=True,
             )
+            # copytree preserves the source mtime, which may predate the API
+            # cache entry. Touch data.arrow so the cache invalidation detects
+            # the change even when no previous converters are re-run.
+            import os
+            import time
+
+            arrow_path = os.path.join(notebook.file_path, "dataset", "data.arrow")
+            if os.path.exists(arrow_path):
+                now = time.time()
+                os.utime(arrow_path, (now, now))
 
             # Enqueue all previous converters
             job_ids = []
             for converter in previous_converters:
-                # Crear instancia de ConverterJob y encolarlo directamente
                 job = ConverterJob(converter_id=converter.id)
-                job_queue.put(job)
-                if hasattr(job, "id"):
-                    job_ids.append(job.id)
+                enqueued = job_queue.put(job)
+                job_ids.append(enqueued.id)
 
             # Delete all the converters after the current one
             for converter in next_converters: