From 16f5855a0281229b06079dd4136a6166799fde02 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 4 Jun 2026 01:49:18 -0400 Subject: [PATCH] Update migrations 39-45 to perform the manifest data migration for new fields --- .../app/migrations/0039_manifest_data.py | 58 +++++++- .../0042_add_manifest_nature_field.py | 131 ++++++++++++++++++ ..._add_os_arch_image_size_manifest_fields.py | 6 +- ...45_alter_manifest_compressed_image_size.py | 73 +++++++++- 4 files changed, 256 insertions(+), 12 deletions(-) diff --git a/pulp_container/app/migrations/0039_manifest_data.py b/pulp_container/app/migrations/0039_manifest_data.py index c223c3bdc..b1464164d 100644 --- a/pulp_container/app/migrations/0039_manifest_data.py +++ b/pulp_container/app/migrations/0039_manifest_data.py @@ -1,15 +1,61 @@ # Generated by Django 4.2.10 on 2024-03-05 11:22 +import json import warnings from django.db import migrations, models +from django.core.files.storage import default_storage +from pulp_container.constants import ( + MEDIA_TYPE, +) -def print_warning_for_initializing_manifest_data(apps, schema_editor): - warnings.warn( - "Run 'pulpcore-manager container-handle-image-data' to move the manifests' " - "data from artifacts to the new 'data' database field." - ) +def get_content_data(artifact): + with default_storage.open(artifact.file.name, mode="rb") as file: + raw_data = file.read() + content_data = json.loads(raw_data) + return content_data, raw_data +def migrate_manifest_data(apps, schema_editor): + """ + Migrate the backing artifact manifest to the new 'data' field. + Also, initialize the 'annotations' and 'labels' fields. + + 'is_bootable' and 'is_flatpak' fields are not initialized since they require annotations and + labels to be initialized first. + """ + Manifest = apps.get_model("container", "Manifest") + to_update = [] + manifests = Manifest.objects.filter( + contentartifact__artifact__isnull=False + ).distinct().prefetch_related("_artifacts").select_related("config_blob") + for manifest in manifests.iterator(chunk_size=1000): + manifest_artifact = manifest._artifacts.get() + manifest_data, raw_bytes_data = get_content_data(manifest_artifact) + manifest.data = raw_bytes_data.decode("utf-8") + if not manifest.annotations: + init_annotations(manifest, manifest_data) + if not manifest.labels: + init_labels(manifest) + + to_update.append(manifest) + if len(to_update) > 1000: + Manifest.objects.bulk_update(to_update, ["data", "annotations", "labels"]) + to_update.clear() + if to_update: + Manifest.objects.bulk_update(to_update, ["data", "annotations", "labels"]) + +def init_annotations(manifest, manifest_data): + # annotations are part of OCI only + if manifest.media_type not in (MEDIA_TYPE.MANIFEST_OCI, MEDIA_TYPE.INDEX_OCI): + return False + + manifest.annotations = manifest_data.get("annotations", {}) + +def init_labels(manifest): + if manifest.config_blob: + config_artifact = manifest.config_blob._artifacts.get() + config_data, _ = get_content_data(config_artifact) + manifest.labels = config_data.get("config", {}).get("Labels") or {} class Migration(migrations.Migration): @@ -24,7 +70,7 @@ class Migration(migrations.Migration): field=models.TextField(null=True), ), migrations.RunPython( - print_warning_for_initializing_manifest_data, + migrate_manifest_data, reverse_code=migrations.RunPython.noop, elidable=True, ), diff --git a/pulp_container/app/migrations/0042_add_manifest_nature_field.py b/pulp_container/app/migrations/0042_add_manifest_nature_field.py index 73b3c304c..8b69f7f15 100644 --- a/pulp_container/app/migrations/0042_add_manifest_nature_field.py +++ b/pulp_container/app/migrations/0042_add_manifest_nature_field.py @@ -1,7 +1,137 @@ # Generated by Django 4.2.16 on 2024-10-21 19:14 +import json from django.db import migrations, models +from pulp_container.constants import MEDIA_TYPE, MANIFEST_TYPE, MANIFEST_MEDIA_TYPES, COSIGN_MEDIA_TYPES, COSIGN_MEDIA_TYPES_MANIFEST_TYPE_MAPPING + + +def migrate_manifest_nature(apps, schema_editor): + """ + Populate the `type` field, and also the `is_bootable` and `is_flatpak` fields even though they are deprecated. + """ + Manifest = apps.get_model("container", "Manifest") + to_update = [] + # First, update the manifests that are not manifest lists or index manifests. + manifests = Manifest.objects.filter(data__isnull=False).exclude(media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]) + for manifest in manifests.iterator(): + setattr(manifest, "json_manifest", json.loads(manifest.data)) + if init_manifest_nature(manifest): + to_update.append(manifest) + if len(to_update) > 1000: + Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"]) + to_update.clear() + if to_update: + Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"]) + to_update.clear() + # Then, update the manifest lists and index manifests. + manifest_lists = Manifest.objects.filter( + data__isnull=False, media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI] + ).prefetch_related("listed_manifests") + for manifest_list in manifest_lists.iterator(chunk_size=1000): + setattr(manifest_list, "json_manifest", json.loads(manifest_list.data)) + if init_manifest_list_nature(manifest_list): + to_update.append(manifest_list) + if len(to_update) > 1000: + Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"]) + to_update.clear() + if to_update: + Manifest.objects.bulk_update(to_update, ["type", "is_bootable", "is_flatpak"]) + to_update.clear() + +def init_manifest_list_nature(manifest_list): + updated_type = False + if not manifest_list.type: + manifest_list.type = MANIFEST_TYPE.INDEX + updated_type = True + + for manifest in manifest_list.listed_manifests.all(): + # it suffices just to have a single manifest of a specific nature; + # there is no case where the manifest is both bootable and flatpak-based + if manifest.type == MANIFEST_TYPE.BOOTABLE: + manifest_list.is_bootable = True + return True + elif manifest.type == MANIFEST_TYPE.FLATPAK: + manifest_list.is_flatpak = True + return True + + return updated_type + +def init_manifest_nature(manifest): + if is_bootable_image(manifest): + # DEPRECATED: is_bootable is deprecated and will be removed in a future release. + manifest.is_bootable = True + manifest.type = MANIFEST_TYPE.BOOTABLE + return True + elif is_flatpak_image(manifest): + # DEPRECATED: is_flatpak is deprecated and will be removed in a future release. + manifest.is_flatpak = True + manifest.type = MANIFEST_TYPE.FLATPAK + return True + elif is_helm_chart(manifest): + manifest.type = MANIFEST_TYPE.HELM + return True + elif media_type := is_cosign(manifest): + manifest.type = get_cosign_type(media_type) + return True + elif is_artifact(manifest): + manifest.type = MANIFEST_TYPE.ARTIFACT + return True + elif is_manifest_image(manifest): + manifest.type = MANIFEST_TYPE.IMAGE + return True + + return False + +def is_bootable_image(manifest): + return ( + manifest.annotations.get("containers.bootc") == "1" + or manifest.labels.get("containers.bootc") == "1" + ) + +def is_flatpak_image(manifest): + return True if manifest.labels.get("org.flatpak.ref") else False + +def is_manifest_image(manifest): + return manifest.media_type in MANIFEST_MEDIA_TYPES.IMAGE + +def is_cosign(manifest): + try: + # layers is not a mandatory field + layers = manifest.json_manifest["layers"] + except KeyError: + return False + + for layer in layers: + if layer["mediaType"] in COSIGN_MEDIA_TYPES: + return layer["mediaType"] + return False + +def get_cosign_type(media_type): + if media_type in MEDIA_TYPE.COSIGN_SBOM: + return MANIFEST_TYPE.COSIGN_SBOM + return COSIGN_MEDIA_TYPES_MANIFEST_TYPE_MAPPING.get(media_type, MANIFEST_TYPE.UNKNOWN) + +def is_helm_chart(manifest): + try: + return manifest.json_manifest["config"]["mediaType"] == MEDIA_TYPE.CONFIG_BLOB_HELM + except KeyError: + return False + +def is_artifact(manifest): + # artifact is valid only for OCI spec + if manifest.media_type != MEDIA_TYPE.MANIFEST_OCI: + return False + + if manifest.json_manifest.get("artifactType", None): + return True + + manifest_config_media_type = manifest.json_manifest["config"]["mediaType"] + return ( + manifest_config_media_type == MEDIA_TYPE.OCI_EMPTY_JSON + or manifest_config_media_type not in vars(MEDIA_TYPE).values() + ) + class Migration(migrations.Migration): @@ -15,4 +145,5 @@ class Migration(migrations.Migration): name='type', field=models.CharField(null=True), ), + migrations.RunPython(migrate_manifest_nature, reverse_code=migrations.RunPython.noop, elidable=True), ] diff --git a/pulp_container/app/migrations/0043_add_os_arch_image_size_manifest_fields.py b/pulp_container/app/migrations/0043_add_os_arch_image_size_manifest_fields.py index d4e20d4b2..461b2ed3b 100644 --- a/pulp_container/app/migrations/0043_add_os_arch_image_size_manifest_fields.py +++ b/pulp_container/app/migrations/0043_add_os_arch_image_size_manifest_fields.py @@ -41,9 +41,5 @@ class Migration(migrations.Migration): name='os', field=models.TextField(blank=True, default=''), ), - migrations.RunPython( - print_warning_for_updating_manifest_fields, - reverse_code=migrations.RunPython.noop, - elidable=True, - ), + # Will update the os, architecture, and compressed_image_size in migration 45 ] diff --git a/pulp_container/app/migrations/0045_alter_manifest_compressed_image_size.py b/pulp_container/app/migrations/0045_alter_manifest_compressed_image_size.py index 144f572fb..6170eb709 100644 --- a/pulp_container/app/migrations/0045_alter_manifest_compressed_image_size.py +++ b/pulp_container/app/migrations/0045_alter_manifest_compressed_image_size.py @@ -1,7 +1,73 @@ # Generated by Django 4.2.16 on 2025-03-06 22:37 - +import json +import warnings from django.db import migrations, models +from django.core.files.storage import default_storage + +from pulp_container.constants import MEDIA_TYPE + +def get_content_data(artifact): + with default_storage.open(artifact.file.name, mode="rb") as file: + raw_data = file.read() + content_data = json.loads(raw_data) + return content_data, raw_data + +def migrate_os_arch_image_size(apps, schema_editor): + """ + Populate the os, architecture, and compressed_image_size fields for the manifests. + + This migration can only handle manifests within the default domain. + """ + Manifest = apps.get_model("container", "Manifest") + to_update = [] + manifests = Manifest.objects.filter(data__isnull=False, pulp_domain__name="default").select_related("config_blob") + for manifest in manifests.iterator(): + if needs_os_arch_size_update(manifest): + manifest.json_manifest = json.loads(manifest.data) + init_architecture_and_os(manifest) + init_compressed_image_size(manifest) + to_update.append(manifest) + + if len(to_update) > 1000: + Manifest.objects.bulk_update(to_update, ["os", "architecture", "compressed_image_size"]) + to_update.clear() + if to_update: + Manifest.objects.bulk_update(to_update, ["os", "architecture", "compressed_image_size"]) + if Manifest.objects.exclude(pulp_domain__name="default").exists(): + warnings.warn( + "Run 'pulpcore-manager container-handle-image-data' to update the manifests' " + "os, architecture, and compressed_image_size fields." + ) + +def needs_os_arch_size_update(manifest): + return manifest.media_type not in [MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI] and not ( + manifest.architecture or manifest.os or manifest.compressed_image_size + ) + +def init_architecture_and_os(manifest): + # schema1 has the architecture/os definition in the Manifest (not in the ConfigBlob) + # and none of these fields are required + if manifest.json_manifest.get("architecture", None) or manifest.json_manifest.get("os", None): + manifest.architecture = manifest.json_manifest.get("architecture", None) + manifest.os = manifest.json_manifest.get("os", None) + elif manifest.config_blob: + config_artifact = manifest.config_blob._artifacts.get() + config_data, _ = get_content_data(config_artifact) + manifest.architecture = config_data.get("architecture", None) + manifest.os = config_data.get("os", None) + +def init_compressed_image_size(manifest): + # manifestv2 schema1 has only blobSum definition for each layer + if manifest.json_manifest.get("fsLayers", None): + manifest.compressed_image_size = 0 + return + + layers = manifest.json_manifest.get("layers") + compressed_size = 0 + for layer in layers: + compressed_size += layer.get("size") + manifest.compressed_image_size = compressed_size class Migration(migrations.Migration): @@ -15,4 +81,9 @@ class Migration(migrations.Migration): name="compressed_image_size", field=models.BigIntegerField(null=True), ), + migrations.RunPython( + migrate_os_arch_image_size, + reverse_code=migrations.RunPython.noop, + elidable=True, + ), ]