From fef68eb0054c6519b6eda2fb19b8b264d379e23e Mon Sep 17 00:00:00 2001 From: SKairinos Date: Mon, 18 May 2026 15:56:15 +0000 Subject: [PATCH 01/18] normalize the fields --- codeforlife/models/fields/__init__.py | 1 + codeforlife/models/fields/base_encrypted.py | 45 ++++++++---- codeforlife/models/fields/normalized.py | 40 +++++++++++ codeforlife/models/fields/sha256.py | 66 ++++++++++------- codeforlife/models/fields/sha256_test.py | 12 ++-- codeforlife/user/models/klass.py | 56 +++++++++++---- codeforlife/user/models/other.py | 78 +++++++++++++++------ codeforlife/user/models/school.py | 30 ++++++-- codeforlife/user/models/user/user.py | 72 ++++++++++++------- 9 files changed, 289 insertions(+), 111 deletions(-) create mode 100644 codeforlife/models/fields/normalized.py diff --git a/codeforlife/models/fields/__init__.py b/codeforlife/models/fields/__init__.py index 6da52f1b..a7df7331 100644 --- a/codeforlife/models/fields/__init__.py +++ b/codeforlife/models/fields/__init__.py @@ -7,4 +7,5 @@ from .data_encryption_key import DataEncryptionKeyField from .deferred_attribute import DeferredAttribute from .encrypted_text import EncryptedTextField +from .normalized import NormalizedField from .sha256 import Sha256Field diff --git a/codeforlife/models/fields/base_encrypted.py b/codeforlife/models/fields/base_encrypted.py index 673a3fc3..b95e698b 100644 --- a/codeforlife/models/fields/base_encrypted.py +++ b/codeforlife/models/fields/base_encrypted.py @@ -29,6 +29,7 @@ from ..encrypted import EncryptedModel from ..utils import is_real_model_class from .deferred_attribute import DeferredAttribute +from .normalized import Normalize, NormalizedField T = t.TypeVar("T") Ciphertext: t.TypeAlias = t.Union[bytes, memoryview] @@ -51,14 +52,21 @@ def __set__(self, instance, value): super().__set__(instance, value) -class BaseEncryptedField(BinaryField, t.Generic[T]): +class BaseEncryptedField( + NormalizedField[EncryptedModel, T], BinaryField, t.Generic[T] +): """Binary field base class for storing encrypted typed values.""" model: t.Type[EncryptedModel] descriptor_class = EncryptedAttribute - def __init__(self, associated_data: str, **kwargs): + def __init__( + self, + associated_data: str, + normalize: Normalize[T] = lambda x: x, + **kwargs, + ): if not associated_data: raise ValidationError( "Associated data cannot be empty.", @@ -66,7 +74,7 @@ def __init__(self, associated_data: str, **kwargs): ) self.associated_data = associated_data - super().__init__(**kwargs) + super().__init__(normalize=normalize, **kwargs) def deconstruct(self): name, path, args, kwargs = t.cast( @@ -182,18 +190,28 @@ def full_associated_data(self): def _decrypt(self, instance: EncryptedModel, ciphertext: bytes): """Decrypts a single value using the DEK and associated data.""" - data = instance.dek_aead.decrypt( - ciphertext=ciphertext, - associated_data=self.full_associated_data, + data = ( + b"" + if ciphertext == b"" + else instance.dek_aead.decrypt( + ciphertext=ciphertext, + associated_data=self.full_associated_data, + ) ) return self.bytes_to_value(data) - def _encrypt(self, instance: EncryptedModel, plaintext: T): + def _encrypt(self, instance: EncryptedModel, value: T): """Encrypts a single value using the DEK and associated data.""" - return instance.dek_aead.encrypt( - plaintext=self.value_to_bytes(plaintext), - associated_data=self.full_associated_data, + plaintext = self.value_to_bytes(value) + + return ( + b"" + if plaintext == b"" + else instance.dek_aead.encrypt( + plaintext=plaintext, + associated_data=self.full_associated_data, + ) ) @staticmethod @@ -239,8 +257,8 @@ def get(instance: EncryptedModel, field_name: str): return decrypted_value - @staticmethod - def set(instance: EncryptedModel, value: t.Optional[T], field_name: str): + @classmethod + def set(cls, instance, value, field_name, **kwargs): """Set a typed plaintext value for an encrypted field. The plaintext is staged in pending-encryption storage and encrypted at @@ -250,6 +268,7 @@ def set(instance: EncryptedModel, value: t.Optional[T], field_name: str): instance: The model instance on which to set the value. value: The plaintext value to set. If None, the field is cleared. field_name: The name of the encrypted field to set. + normalize: Whether to normalize the value before setting it. """ field = t.cast( BaseEncryptedField[T], instance._meta.get_field(field_name) @@ -259,6 +278,8 @@ def set(instance: EncryptedModel, value: t.Optional[T], field_name: str): if value is None: instance.__pending_encryption_values__.pop(field.attname, None) else: + if kwargs.get("normalize", True): + value = field.normalize(value) instance.__pending_encryption_values__[field.attname] = value # In all cases we need to clear the internal and cached-decrypted value. diff --git a/codeforlife/models/fields/normalized.py b/codeforlife/models/fields/normalized.py new file mode 100644 index 00000000..f8bccf93 --- /dev/null +++ b/codeforlife/models/fields/normalized.py @@ -0,0 +1,40 @@ +""" +© Ocado Group +Created on 18/05/2026 at 15:38:05(+01:00). +""" + +import typing as t + +from django.db.models import Field, Model + +AnyModel = t.TypeVar("AnyModel", bound=Model) +T = t.TypeVar("T") +Normalize: t.TypeAlias = t.Callable[[T], T] + + +class NormalizedField(Field, t.Generic[AnyModel, T]): + """A Django model field that normalizes values before saving.""" + + def __init__(self, normalize: Normalize[T], *args, **kwargs): + super().__init__(*args, **kwargs) + self.normalize = normalize + + @classmethod + def set( + cls, instance: AnyModel, value: t.Optional[T], field_name: str, **kwargs + ): + """ + Normalize and assign a value to a NormalizedField. + + Args: + instance: The model instance on which to set the value. + value: The value to normalize and set. + field_name: The name of the NormalizedField on the model. + """ + if value is not None: + value = t.cast( + NormalizedField[AnyModel, T], + instance._meta.get_field(field_name), + ).normalize(value) + + setattr(instance, field_name, value) diff --git a/codeforlife/models/fields/sha256.py b/codeforlife/models/fields/sha256.py index ddb2df7c..39bf020a 100644 --- a/codeforlife/models/fields/sha256.py +++ b/codeforlife/models/fields/sha256.py @@ -25,12 +25,15 @@ from django.core.exceptions import ValidationError from django.db.models import CharField, Model, lookups +from .normalized import Normalize, NormalizedField -class Sha256Field(CharField): + +class Sha256Field(NormalizedField[Model, str], CharField): """A CharField for deterministic, one-way hashed values.""" def __init__( self, + normalize: Normalize[str] = lambda x: x, editable: t.Literal[False] = False, max_length: t.Literal[64] = 64, # Length of SHA-256 hash in hexadecimal **kwargs, @@ -47,7 +50,12 @@ def __init__( code="max_length_not_64", ) - super().__init__(editable=editable, max_length=max_length, **kwargs) + super().__init__( + normalize=normalize, + editable=editable, + max_length=max_length, + **kwargs, + ) @staticmethod def hash(value: str): @@ -66,7 +74,7 @@ def hash(value: str): ).hexdigest() @classmethod - def set(cls, instance: Model, value: t.Optional[str], field_name: str): + def set(cls, instance, value, field_name, **kwargs): """ Hash and assign a plaintext value to a Sha256Field. @@ -74,15 +82,43 @@ def set(cls, instance: Model, value: t.Optional[str], field_name: str): instance: The model instance on which to set the value. value: The plaintext value to hash and set. field_name: The name of the Sha256Field on the model. + normalize: Whether to normalize the value before hashing. + hash: Whether to hash the value before setting it. """ - if value is not None: + if value is not None and kwargs.get("hash", True): + if kwargs.get("normalize", True): + value = t.cast( + Sha256Field, instance._meta.get_field(field_name) + ).normalize(value) value = cls.hash(value) setattr(instance, field_name, value) # pylint: disable-next=abstract-method -class Sha256ExactLookup(lookups.Exact): +class LookupMixin(lookups.Lookup): + """Mixin for lookups that hash the right-hand side value(s).""" + + rhs: None | str | t.Iterable[str] + + def process_rhs(self, compiler, connection): + sql, params = super().process_rhs(compiler, connection) + + field: Sha256Field = self.lhs.output_field + + if self.rhs is None: + return sql, params + + if isinstance(self.rhs, str): + return sql, [Sha256Field.hash(field.normalize(self.rhs))] + + return sql, [ + Sha256Field.hash(field.normalize(value)) for value in self.rhs + ] + + +# pylint: disable-next=abstract-method,too-many-ancestors +class Sha256ExactLookup(lookups.Exact, LookupMixin): """ A lookup that hashes a plaintext right-hand side value before comparing. @@ -90,15 +126,8 @@ class Sha256ExactLookup(lookups.Exact): `User.objects.filter(_email_hash__sha256="user@example.com")` """ - rhs: t.Optional[str] - lookup_name = "sha256" - def process_rhs(self, compiler, connection): - sql, params = super().process_rhs(compiler, connection) - - return sql, params if self.rhs is None else [Sha256Field.hash(self.rhs)] - def get_rhs_op(self, connection, rhs): """ Get the operator for the right-hand side of the expression. @@ -109,7 +138,7 @@ def get_rhs_op(self, connection, rhs): # pylint: disable-next=abstract-method,too-many-ancestors -class Sha256InLookup(lookups.In): +class Sha256InLookup(lookups.In, LookupMixin): """ A lookup that hashes plaintext right-hand side values before comparing. @@ -117,19 +146,8 @@ class Sha256InLookup(lookups.In): `User.objects.filter(_email_hash__sha256_in=["user@example.com"])` """ - rhs: t.Optional[t.Iterable[str]] - lookup_name = f"{Sha256ExactLookup.lookup_name}_in" - def process_rhs(self, compiler, connection): - sql, params = super().process_rhs(compiler, connection) - - return sql, ( - params - if self.rhs is None - else [Sha256Field.hash(value) for value in self.rhs] - ) - Sha256Field.register_lookup(Sha256ExactLookup) Sha256Field.register_lookup(Sha256InLookup) diff --git a/codeforlife/models/fields/sha256_test.py b/codeforlife/models/fields/sha256_test.py index 60dff338..412639e1 100644 --- a/codeforlife/models/fields/sha256_test.py +++ b/codeforlife/models/fields/sha256_test.py @@ -24,7 +24,7 @@ def test_init__max_length_not_64(self): def test_set__none(self): """Setting field to None sets to None.""" - user = User(_email_hash=None) + user = User(_email_hash=None) # type: ignore[misc] assert user.__dict__["_email_hash"] is None def test_hash(self): @@ -42,9 +42,10 @@ def test_lookup__sha256(self): """ user = User.objects.filter(_email_hash__isnull=False).first() assert user + email = user.email + " " # add whitespace to test value is normalized # pylint: disable-next=protected-access - assert user.email != user._email_hash - assert User.objects.get(_email_hash__sha256=user.email) == user + assert email != user._email_hash + assert User.objects.get(_email_hash__sha256=email) == user def test_lookup__sha256_in(self): """ @@ -53,6 +54,7 @@ def test_lookup__sha256_in(self): """ user = User.objects.filter(_email_hash__isnull=False).first() assert user + email = user.email + " " # add whitespace to test value is normalized # pylint: disable-next=protected-access - assert user.email != user._email_hash - assert User.objects.get(_email_hash__sha256_in=[user.email]) == user + assert email != user._email_hash + assert User.objects.get(_email_hash__sha256_in=[email]) == user diff --git a/codeforlife/user/models/klass.py b/codeforlife/user/models/klass.py index bf68ff42..f4dd59e6 100644 --- a/codeforlife/user/models/klass.py +++ b/codeforlife/user/models/klass.py @@ -48,6 +48,33 @@ class ClassModelManager(EncryptedModel.Manager["Class"]): """Manager for Class model.""" + @classmethod + def normalize_access_code(cls, access_code: str): + """Normalize a class' access code. + + The value is stripped and uppercased. + + Returns: + The normalized access code. + """ + return access_code.strip().upper() + + @classmethod + def normalize_name(cls, name: str, lower=True): + """Normalize a class' name. + + The value is stripped and optionally lowercased. + + Args: + name: The name to normalize. + lower: Whether to lowercase the name. + + Returns: + The normalized name. + """ + name = name.strip() + return name.lower() if lower else name + def get_original_queryset(self): """Get the original queryset without filtering.""" return super().get_queryset() @@ -79,29 +106,31 @@ class Class(EncryptedModel): _name_hash = Sha256Field( verbose_name=_("name hash"), - null=True, db_column="name_hash", + normalize=lambda name: ClassModelManager.normalize_name( + name, lower=True + ), ) _name_plain: str _name_plain = models.CharField(max_length=200) # type: ignore[assignment] _name_enc = EncryptedTextField( associated_data="name", db_column="name_enc", - null=True, verbose_name=_("name"), + normalize=lambda name: ClassModelManager.normalize_name( + name, lower=False + ), ) @property def name(self): """Get the name of the class.""" - if self._name_enc is not None: - return EncryptedTextField.get(self, "_name_enc") - return self._name_plain + return EncryptedTextField.get(self, "_name_enc") @name.setter def name(self, value: str): """Set the name of the class.""" - self._name_plain = value + self._name_plain = ClassModelManager.normalize_name(value, lower=False) EncryptedTextField.set(self, value, "_name_enc") Sha256Field.set(self, value, "_name_hash") @@ -120,32 +149,29 @@ def name(self, value: str): _access_code_hash = Sha256Field( verbose_name=_("access code hash"), - null=True, db_column="access_code_hash", + normalize=ClassModelManager.normalize_access_code, ) - _access_code_plain: t.Optional[str] + _access_code_plain: str _access_code_plain = models.CharField( # type: ignore[assignment] max_length=5, - null=True, ) _access_code_enc = EncryptedTextField( associated_data="access_code", - null=True, verbose_name=_("access code"), db_column="access_code_enc", + normalize=ClassModelManager.normalize_access_code, ) @property def access_code(self): """Get the access code for the class.""" - if self._access_code_enc is not None: - return EncryptedTextField.get(self, "_access_code_enc") - return self._access_code_plain + return EncryptedTextField.get(self, "_access_code_enc") @access_code.setter - def access_code(self, value: t.Optional[str]): + def access_code(self, value: str): """Set the access code for the class.""" - self._access_code_plain = value + self._access_code_plain = ClassModelManager.normalize_access_code(value) EncryptedTextField.set(self, value, "_access_code_enc") Sha256Field.set(self, value, "_access_code_hash") diff --git a/codeforlife/user/models/other.py b/codeforlife/user/models/other.py index 861e9da8..c7de5e5c 100644 --- a/codeforlife/user/models/other.py +++ b/codeforlife/user/models/other.py @@ -210,6 +210,43 @@ class SchoolTeacherInvitationModelManager( inactive invitations by default. """ + @classmethod + def normalize_first_name(cls, first_name: str, lower=True): + """Normalize a teacher's first name. + + The value is stripped and optionally lowercased. + + Args: + first_name: The first name to normalize. + lower: Whether to lowercase the first name. + + Returns: + The normalized first name. + """ + # The local import avoids circular imports. + # pylint: disable-next=import-outside-toplevel + from .user import SchoolTeacherUserManager + + return SchoolTeacherUserManager.normalize_first_name(first_name, lower) + + @classmethod + def normalize_email(cls, email: str | None): + """Normalize a user's email address. + + The value is stripped and lowercased. + + Args: + email: The email address to normalize. + + Returns: + The normalized email address. + """ + # The local import avoids circular imports. + # pylint: disable-next=import-outside-toplevel + from .user import SchoolTeacherUserManager + + return SchoolTeacherUserManager.normalize_email(email) + def get_original_queryset(self): """ Get the original queryset without filtering out inactive invitations. @@ -256,7 +293,6 @@ class SchoolTeacherInvitation(EncryptedModel): _token_hash = Sha256Field( verbose_name=_("token hash"), - null=True, unique=True, db_column="token_hash", ) @@ -264,7 +300,6 @@ class SchoolTeacherInvitation(EncryptedModel): _token_plain = models.CharField(max_length=88) # type: ignore[assignment] _token_enc = EncryptedTextField( associated_data="token", - null=True, verbose_name=_("token"), db_column="token_enc", ) @@ -272,9 +307,7 @@ class SchoolTeacherInvitation(EncryptedModel): @property def token(self): """Get the decrypted token value.""" - if self._token_enc is not None: - return EncryptedTextField.get(self, "_token_enc") - return self._token_plain + return EncryptedTextField.get(self, "_token_enc") @token.setter def token(self, value: str): @@ -312,24 +345,28 @@ def token(self, value: str): ) # Same as User model _invited_teacher_first_name_enc = EncryptedTextField( associated_data="invited_teacher_first_name", - null=True, verbose_name=_("invited teacher first name"), db_column="invited_teacher_first_name_enc", + normalize=lambda first_name: ( + SchoolTeacherInvitationModelManager.normalize_first_name( + first_name, lower=False + ) + ), ) @property def invited_teacher_first_name(self): """Get the decrypted invited teacher first name value.""" - if self._invited_teacher_first_name_enc is not None: - return EncryptedTextField.get( - self, "_invited_teacher_first_name_enc" - ) - return self._invited_teacher_first_name_plain + return EncryptedTextField.get(self, "_invited_teacher_first_name_enc") @invited_teacher_first_name.setter def invited_teacher_first_name(self, value: str): """Sets the invited teacher first name value.""" - self._invited_teacher_first_name_plain = value + self._invited_teacher_first_name_plain = ( + SchoolTeacherInvitationModelManager.normalize_first_name( + value, lower=False + ) + ) EncryptedTextField.set(self, value, "_invited_teacher_first_name_enc") # -------------------------------------------------------------------------- @@ -343,7 +380,6 @@ def invited_teacher_first_name(self, value: str): ) # Same as User model _invited_teacher_last_name_enc = EncryptedTextField( associated_data="invited_teacher_last_name", - null=True, verbose_name=_("invited teacher last name"), db_column="invited_teacher_last_name_enc", ) @@ -351,11 +387,7 @@ def invited_teacher_first_name(self, value: str): @property def invited_teacher_last_name(self): """Get the decrypted invited teacher last name value.""" - if self._invited_teacher_last_name_enc is not None: - return EncryptedTextField.get( - self, "_invited_teacher_last_name_enc" - ) - return self._invited_teacher_last_name_plain + return EncryptedTextField.get(self, "_invited_teacher_last_name_enc") @invited_teacher_last_name.setter def invited_teacher_last_name(self, value: str): @@ -374,22 +406,22 @@ def invited_teacher_last_name(self, value: str): ) # Same as User model _invited_teacher_email_enc = EncryptedTextField( associated_data="invited_teacher_email", - null=True, verbose_name=_("invited teacher email"), db_column="invited_teacher_email_enc", + normalize=SchoolTeacherInvitationModelManager.normalize_email, ) @property def invited_teacher_email(self): """Get the decrypted invited teacher email value.""" - if self._invited_teacher_email_enc is not None: - return EncryptedTextField.get(self, "_invited_teacher_email_enc") - return self._invited_teacher_email_plain + return EncryptedTextField.get(self, "_invited_teacher_email_enc") @invited_teacher_email.setter def invited_teacher_email(self, value: str): """Sets the invited teacher email value.""" - self._invited_teacher_email_plain = value + self._invited_teacher_email_plain = ( + SchoolTeacherInvitationModelManager.normalize_email(value) + ) EncryptedTextField.set(self, value, "_invited_teacher_email_enc") # -------------------------------------------------------------------------- diff --git a/codeforlife/user/models/school.py b/codeforlife/user/models/school.py index 61505499..9f41940a 100644 --- a/codeforlife/user/models/school.py +++ b/codeforlife/user/models/school.py @@ -31,6 +31,22 @@ class SchoolModelManager(DataEncryptionKeyModel.Manager["School"]): """Manager for School model.""" + @classmethod + def normalize_name(cls, name: str, lower=True): + """Normalize a school's name. + + The value is stripped and optionally lowercased. + + Args: + name: The name to normalize. + lower: Whether to lowercase the name. + + Returns: + The normalized name. + """ + name = name.strip() + return name.lower() if lower else name + def get_original_queryset(self): """Get the original queryset without filtering.""" return super().get_queryset() @@ -55,9 +71,11 @@ class School(DataEncryptionKeyModel): _name_hash = Sha256Field( verbose_name=_("name hash"), - null=True, unique=True, db_column="name_hash", + normalize=lambda name: SchoolModelManager.normalize_name( + name, lower=True + ), ) _name_plain: str _name_plain = models.CharField( # type: ignore[assignment] @@ -66,22 +84,22 @@ class School(DataEncryptionKeyModel): ) _name_enc = EncryptedTextField( associated_data="name", - null=True, verbose_name=_("name"), db_column="name_enc", + normalize=lambda name: SchoolModelManager.normalize_name( + name, lower=False + ), ) @property def name(self): """Get the school's name.""" - if self._name_enc is not None: - return EncryptedTextField.get(self, "_name_enc") - return self._name_plain + return EncryptedTextField.get(self, "_name_enc") @name.setter def name(self, value: str): """Set the school's name.""" - self._name_plain = value + self._name_plain = SchoolModelManager.normalize_name(value, lower=False) EncryptedTextField.set(self, value, "_name_enc") Sha256Field.set(self, value, "_name_hash") diff --git a/codeforlife/user/models/user/user.py b/codeforlife/user/models/user/user.py index 343e4819..bc4e6695 100644 --- a/codeforlife/user/models/user/user.py +++ b/codeforlife/user/models/user/user.py @@ -71,11 +71,37 @@ def _create_user_object( username=username, email=email, password=password, **extra_fields ) - # pylint: disable=missing-function-docstring + @classmethod + def normalize_email(cls, email): + """Normalize a user's email address. + + The value is stripped and lowercased. + + Args: + email: The email address to normalize. + + Returns: + The normalized email address. + """ + return super().normalize_email(email).lower() - # @classmethod - # def normalize_email(cls, email): - # return super().normalize_email(email).lower() + @classmethod + def normalize_first_name(cls, first_name: str, lower=True): + """Normalize a user's first name. + + The value is stripped. + + Args: + first_name: The first name to normalize. + lower: Whether to lowercase the first name. + + Returns: + The normalized first name. + """ + first_name = first_name.strip() + return first_name.lower() if lower else first_name + + # pylint: disable=missing-function-docstring # def get_by_natural_key(self, username): # return self.get(_username_hash__sha256=username) @@ -144,7 +170,6 @@ class User(AbstractBaseUser, PermissionsMixin, DataEncryptionKeyModel): verbose_name=_("username hash"), db_column="username_hash", unique=True, - null=True, ) _username_plain = models.CharField( _("username"), @@ -162,16 +187,13 @@ class User(AbstractBaseUser, PermissionsMixin, DataEncryptionKeyModel): _username_enc = EncryptedTextField( associated_data="username", db_column="username_enc", - null=True, verbose_name=_("username"), ) @property def username(self): """The user's username.""" - if self._username_enc is not None: - return EncryptedTextField.get(self, "_username_enc") - return self._username_plain + return EncryptedTextField.get(self, "_username_enc") @username.setter def username(self, value: str): @@ -187,7 +209,9 @@ def username(self, value: str): _first_name_hash = Sha256Field( verbose_name=_("first name hash"), db_column="first_name_hash", - null=True, + normalize=lambda first_name: UserManager.normalize_first_name( + first_name, lower=True + ), ) _first_name_plain = models.CharField( _("first name"), max_length=150, blank=True @@ -195,21 +219,23 @@ def username(self, value: str): _first_name_enc = EncryptedTextField( associated_data="first_name", db_column="first_name_enc", - null=True, verbose_name=_("first name"), + normalize=lambda first_name: UserManager.normalize_first_name( + first_name, lower=False + ), ) @property def first_name(self): """The user's first name.""" - if self._first_name_enc is not None: - return EncryptedTextField.get(self, "_first_name_enc") - return self._first_name_plain + return EncryptedTextField.get(self, "_first_name_enc") @first_name.setter def first_name(self, value: str): """Set the user's first name.""" - self._first_name_plain = value + self._first_name_plain = UserManager.normalize_first_name( + value, lower=False + ) EncryptedTextField.set(self, value, "_first_name_enc") Sha256Field.set(self, value, "_first_name_hash") @@ -223,16 +249,13 @@ def first_name(self, value: str): _last_name_enc = EncryptedTextField( associated_data="last_name", db_column="last_name_enc", - null=True, verbose_name=_("last name"), ) @property def last_name(self): """The user's last name.""" - if self._last_name_enc is not None: - return EncryptedTextField.get(self, "_last_name_enc") - return self._last_name_plain + return EncryptedTextField.get(self, "_last_name_enc") @last_name.setter def last_name(self, value: str): @@ -247,28 +270,25 @@ def last_name(self, value: str): _email_hash = Sha256Field( verbose_name=_("email hash"), db_column="email_hash", - null=True, + normalize=UserManager.normalize_email, ) _email_plain = models.EmailField(_("email address"), blank=True) _email_enc = EncryptedTextField( associated_data="email", db_column="email_enc", - null=True, verbose_name=_("email address"), + normalize=UserManager.normalize_email, ) @property def email(self): """The user's email address.""" - if self._email_enc is not None: - return EncryptedTextField.get(self, "_email_enc") - return self._email_plain + return EncryptedTextField.get(self, "_email_enc") @email.setter def email(self, value: str): """Set the user's email address.""" - value = self.__class__.objects.normalize_email(value) - self._email_plain = value + self._email_plain = self.__class__.objects.normalize_email(value) EncryptedTextField.set(self, value, "_email_enc") Sha256Field.set(self, value, "_email_hash") From bee7a2e32ce63a1d9006dfe07a51bf83709096ba Mon Sep 17 00:00:00 2001 From: SKairinos Date: Tue, 19 May 2026 09:16:40 +0000 Subject: [PATCH 02/18] set_field --- .../0005_client_side_encryption_part_3.py | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 codeforlife/user/migrations/0005_client_side_encryption_part_3.py diff --git a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py new file mode 100644 index 00000000..0f2c0e57 --- /dev/null +++ b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py @@ -0,0 +1,375 @@ +import typing as t + +from django.apps.registry import Apps +from django.db import migrations +from django.db.models import CharField, Model, Q + +from ...models.fields import EncryptedTextField, Sha256Field + + +def set_field(model_name: str, field_name: str, qs_filter: Q | None = None): + def forwards_func(apps: Apps, schema_editor): + # Get the model class from the apps registry. + model_class: t.Type[Model] = apps.get_model( + app_label="user", model_name=model_name + ) + + # Helper function to create a Q object that checks if a field is null or + # empty. + def null_or_empty(field_name: str, empty: t.Any): + return Q(**{f"{field_name}__isnull": True}) | Q( + **{f"{field_name}": empty} + ) + + # Define the names and default values for the plain, encrypted, and hash + # fields. + plain_field = f"_{field_name}_plain", "" + enc_field = f"_{field_name}_enc", b"" + hash_field = f"_{field_name}_hash", "" + + # Check if the encrypted and hash fields exist. + has_enc_field = enc_field[0] in model_class._meta.fields_map + has_hash_field = hash_field[0] in model_class._meta.fields_map + + # Build the queryset to filter instances where the plain field is not + # null or empty, and either the encrypted field or hash field is null or + # empty (if they exist). + queryset = model_class.objects.filter( # type: ignore[attr-defined] + ~null_or_empty(*plain_field) + & ( + null_or_empty(*enc_field) | null_or_empty(*hash_field) + if has_enc_field and has_hash_field + else ( + null_or_empty(*enc_field) + if has_enc_field + else null_or_empty(*hash_field) + ) + ) + ) + + # Additional filter if provided. + if qs_filter is not None: + queryset = queryset.filter(qs_filter) + + # Select fields. + fields = [plain_field[0]] + if has_enc_field: + fields.append(enc_field[0]) + if has_hash_field: + fields.append(hash_field[0]) + queryset = queryset.only(*fields) + + # Iterate over the queryset in chunks and save each instance. + for instance in queryset.iterator(chunk_size=1000): + # Get the plain value. + value = getattr(instance, plain_field[0]) + + # Set the plain, encrypted and hash values. + setattr(instance, field_name, value) + + # Save the instance, updating only the relevant fields. + instance.save(update_fields=fields) + + return migrations.RunPython(forwards_func) + + +def set_user_field(field_name: str): + return set_field( + model_name="user", field_name=field_name, qs_filter=Q(is_active=True) + ) + + +def set_class_field(field_name: str): + return set_field( + model_name="class", + field_name=field_name, + qs_filter=( + Q(is_active=True) + & Q(teacher__isnull=False) + & Q(teacher__school__isnull=False) + & Q(teacher__school__is_active=True) + ), + ) + + +def set_school_teacher_invitation_field(field_name: str): + return set_field( + model_name="schoolteacherinvitation", + field_name=field_name, + qs_filter=( + Q(is_active=True) + & Q(school__isnull=False) + & Q(school__is_active=True) + ), + ) + + +def set_school_field(field_name: str): + return set_field( + model_name="school", field_name=field_name, qs_filter=Q(is_active=True) + ) + + +user_migrations = [ + # Email + set_user_field(field_name="email"), + migrations.AlterField( + model_name="user", + name="_email_enc", + field=EncryptedTextField( + associated_data="email", + db_column="email_enc", + default="", + verbose_name="email address", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="user", + name="_email_hash", + field=Sha256Field( + db_column="email_hash", + default="", + editable=False, + max_length=64, + verbose_name="email hash", + ), + preserve_default=False, + ), + # First name + set_user_field(field_name="first_name"), + migrations.AlterField( + model_name="user", + name="_first_name_enc", + field=EncryptedTextField( + associated_data="first_name", + db_column="first_name_enc", + default="", + verbose_name="first name", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="user", + name="_first_name_hash", + field=Sha256Field( + db_column="first_name_hash", + default="", + editable=False, + max_length=64, + verbose_name="first name hash", + ), + preserve_default=False, + ), + # Last name + set_user_field(field_name="last_name"), + migrations.AlterField( + model_name="user", + name="_last_name_enc", + field=EncryptedTextField( + associated_data="last_name", + db_column="last_name_enc", + default="", + verbose_name="last name", + ), + preserve_default=False, + ), + # Username + set_user_field(field_name="username"), + migrations.AlterField( + model_name="user", + name="_username_enc", + field=EncryptedTextField( + associated_data="username", + db_column="username_enc", + default="", + verbose_name="username", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="user", + name="_username_hash", + field=Sha256Field( + db_column="username_hash", + default="", + editable=False, + max_length=64, + unique=True, + verbose_name="username hash", + ), + preserve_default=False, + ), +] + +class_migrations = [ + # Access code + set_class_field(field_name="access_code"), + migrations.AlterField( + model_name="class", + name="_access_code_enc", + field=EncryptedTextField( + associated_data="access_code", + db_column="access_code_enc", + default="", + verbose_name="access code", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="class", + name="_access_code_hash", + field=Sha256Field( + db_column="access_code_hash", + default="", + editable=False, + max_length=64, + verbose_name="access code hash", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="class", + name="_access_code_plain", + field=CharField(default="", max_length=5), + preserve_default=False, + ), + # Name + set_class_field(field_name="name"), + migrations.AlterField( + model_name="class", + name="_name_enc", + field=EncryptedTextField( + associated_data="name", + db_column="name_enc", + default="", + verbose_name="name", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="class", + name="_name_hash", + field=Sha256Field( + db_column="name_hash", + default="", + editable=False, + max_length=64, + verbose_name="name hash", + ), + preserve_default=False, + ), +] + +school_teacher_invitation_migrations = [ + # Email + set_school_teacher_invitation_field(field_name="invited_teacher_email"), + migrations.AlterField( + model_name="schoolteacherinvitation", + name="_invited_teacher_email_enc", + field=EncryptedTextField( + associated_data="invited_teacher_email", + db_column="invited_teacher_email_enc", + default="", + verbose_name="invited teacher email", + ), + preserve_default=False, + ), + # First name + set_school_teacher_invitation_field( + field_name="invited_teacher_first_name" + ), + migrations.AlterField( + model_name="schoolteacherinvitation", + name="_invited_teacher_first_name_enc", + field=EncryptedTextField( + associated_data="invited_teacher_first_name", + db_column="invited_teacher_first_name_enc", + default="", + verbose_name="invited teacher first name", + ), + preserve_default=False, + ), + # Last name + set_school_teacher_invitation_field(field_name="invited_teacher_last_name"), + migrations.AlterField( + model_name="schoolteacherinvitation", + name="_invited_teacher_last_name_enc", + field=EncryptedTextField( + associated_data="invited_teacher_last_name", + db_column="invited_teacher_last_name_enc", + default="", + verbose_name="invited teacher last name", + ), + preserve_default=False, + ), + # Token + set_school_teacher_invitation_field(field_name="token"), + migrations.AlterField( + model_name="schoolteacherinvitation", + name="_token_enc", + field=EncryptedTextField( + associated_data="token", + db_column="token_enc", + default="", + verbose_name="token", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="schoolteacherinvitation", + name="_token_hash", + field=Sha256Field( + db_column="token_hash", + default="", + editable=False, + max_length=64, + unique=True, + verbose_name="token hash", + ), + preserve_default=False, + ), +] + +school_migrations = [ + # Name + set_school_field(field_name="name"), + migrations.AlterField( + model_name="school", + name="_name_enc", + field=EncryptedTextField( + associated_data="name", + db_column="name_enc", + default="", + verbose_name="name", + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="school", + name="_name_hash", + field=Sha256Field( + db_column="name_hash", + default="", + editable=False, + max_length=64, + unique=True, + verbose_name="name hash", + ), + preserve_default=False, + ), +] + + +class Migration(migrations.Migration): + + dependencies = [ + ("user", "0004_client_side_encryption_part_2"), + ] + + operations = [ + *user_migrations, + *class_migrations, + *school_teacher_invitation_migrations, + *school_migrations, + ] From fd21cf84d84d76b0b2138c5c410729ba44fe6fa9 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Tue, 19 May 2026 10:55:42 +0000 Subject: [PATCH 03/18] allow bulk_update --- codeforlife/models/encrypted.py | 34 +++++++++++++++++++++++----- codeforlife/models/encrypted_test.py | 14 +++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/codeforlife/models/encrypted.py b/codeforlife/models/encrypted.py index 3ad8d571..5a4ccddf 100644 --- a/codeforlife/models/encrypted.py +++ b/codeforlife/models/encrypted.py @@ -76,12 +76,21 @@ class Manager( that would bypass field-level encryption. """ + def _is_encrypted_field(self, field_name: str): + return any( + field.name == field_name + for field in self.model.ENCRYPTED_FIELDS + ) + + def _is_none_or_empty(self, value: t.Any): + return value is None or value == b"" + def update(self, **kwargs): """Ensure encrypted fields are not updated via 'update()'.""" - for name in kwargs: - if any( - field.name == name for field in self.model.ENCRYPTED_FIELDS - ): + for name, value in kwargs.items(): + if self._is_encrypted_field( + name + ) and not self._is_none_or_empty(value): raise ValidationError( f"Cannot update encrypted field '{name}' via" " 'update()'. Set the property on each instance" @@ -91,9 +100,22 @@ def update(self, **kwargs): return super().update(**kwargs) + def bulk_update(self, objs, fields, batch_size=None): + """Ensure encrypted fields are not updated via 'bulk_update()'.""" + for name in fields: + if self._is_encrypted_field(name) and not all( + self._is_none_or_empty(getattr(obj, name)) for obj in objs + ): + raise ValidationError( + f"Cannot bulk update encrypted field '{name}' via" + " 'bulk_update()'. Set the property on each instance" + " instead.", + code="cannot_bulk_update", + ) + + return super().bulk_update(objs, fields, batch_size) + # Disable bulk operations that would bypass field-level encryption. - bulk_update: t.Never = None # type: ignore[assignment] - abulk_update: t.Never = None # type: ignore[assignment] bulk_create: t.Never = None # type: ignore[assignment] abulk_create: t.Never = None # type: ignore[assignment] in_bulk: t.Never = None # type: ignore[assignment] diff --git a/codeforlife/models/encrypted_test.py b/codeforlife/models/encrypted_test.py index 1986d962..6e42dba1 100644 --- a/codeforlife/models/encrypted_test.py +++ b/codeforlife/models/encrypted_test.py @@ -27,6 +27,8 @@ class Person(EncryptedModel): name = EncryptedTextField(associated_data="name") + objects: EncryptedModel.Manager["Person"] # type: ignore[assignment] + class Meta(TypedModelMeta): app_label = "codeforlife.user" @@ -37,13 +39,13 @@ def test_objects___update__cannot_update(self): with self.assert_raises_validation_error(code="cannot_update"): Person.objects.update(name="Alice") - def test_objects___bulk_update(self): + def test_objects___bulk_update__cannot_bulk_update(self): """Cannot bulk update encrypted field via objects.bulk_update().""" - assert Person.objects.bulk_update is None - - def test_objects___abulk_update(self): - """Cannot abulk_update encrypted field via objects.abulk_update().""" - assert Person.objects.abulk_update is None + with self.assert_raises_validation_error(code="cannot_bulk_update"): + Person.objects.bulk_update( + [Person(name="Alice"), Person(name="Bob")], + fields=["name"], + ) def test_objects___bulk_create(self): """Cannot bulk create encrypted field via objects.bulk_create().""" From c4cc4c3b8c831b7b4712036cd833dce104625a30 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Tue, 19 May 2026 13:33:51 +0000 Subject: [PATCH 04/18] quick save --- codeforlife/models/fields/sha256.py | 4 +- .../0005_client_side_encryption_part_3.py | 155 +++++++++++------- 2 files changed, 102 insertions(+), 57 deletions(-) diff --git a/codeforlife/models/fields/sha256.py b/codeforlife/models/fields/sha256.py index 39bf020a..eaf4f1d8 100644 --- a/codeforlife/models/fields/sha256.py +++ b/codeforlife/models/fields/sha256.py @@ -118,7 +118,7 @@ def process_rhs(self, compiler, connection): # pylint: disable-next=abstract-method,too-many-ancestors -class Sha256ExactLookup(lookups.Exact, LookupMixin): +class Sha256ExactLookup(LookupMixin, lookups.Exact): """ A lookup that hashes a plaintext right-hand side value before comparing. @@ -138,7 +138,7 @@ def get_rhs_op(self, connection, rhs): # pylint: disable-next=abstract-method,too-many-ancestors -class Sha256InLookup(lookups.In, LookupMixin): +class Sha256InLookup(LookupMixin, lookups.In): """ A lookup that hashes plaintext right-hand side values before comparing. diff --git a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py index 0f2c0e57..ec2fe23a 100644 --- a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py +++ b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py @@ -1,6 +1,7 @@ import typing as t from django.apps.registry import Apps +from django.core.exceptions import FieldDoesNotExist from django.db import migrations from django.db.models import CharField, Model, Q @@ -13,62 +14,106 @@ def forwards_func(apps: Apps, schema_editor): model_class: t.Type[Model] = apps.get_model( app_label="user", model_name=model_name ) + manager = model_class.objects # type: ignore[attr-defined] - # Helper function to create a Q object that checks if a field is null or - # empty. - def null_or_empty(field_name: str, empty: t.Any): - return Q(**{f"{field_name}__isnull": True}) | Q( - **{f"{field_name}": empty} - ) + # Generate the plain field name and the Q object to filter instances + # where the plain field is null or empty. + plain_field_name = f"_{field_name}_plain" + plain_field_is_null_or_empty = Q( + **{f"{plain_field_name}__isnull": True} + ) | Q(**{f"{plain_field_name}": ""}) - # Define the names and default values for the plain, encrypted, and hash - # fields. - plain_field = f"_{field_name}_plain", "" - enc_field = f"_{field_name}_enc", b"" - hash_field = f"_{field_name}_hash", "" - - # Check if the encrypted and hash fields exist. - has_enc_field = enc_field[0] in model_class._meta.fields_map - has_hash_field = hash_field[0] in model_class._meta.fields_map - - # Build the queryset to filter instances where the plain field is not - # null or empty, and either the encrypted field or hash field is null or - # empty (if they exist). - queryset = model_class.objects.filter( # type: ignore[attr-defined] - ~null_or_empty(*plain_field) - & ( - null_or_empty(*enc_field) | null_or_empty(*hash_field) - if has_enc_field and has_hash_field - else ( - null_or_empty(*enc_field) - if has_enc_field - else null_or_empty(*hash_field) - ) - ) + # Generate the encrypted and hash field names and check if they exist. + enc_field_name = f"_{field_name}_enc" + try: + model_class._meta.get_field(enc_field_name) + enc_field_exists = True + except FieldDoesNotExist: + enc_field_exists = False + hash_field_name = f"_{field_name}_hash" + try: + model_class._meta.get_field(hash_field_name) + hash_field_exists = True + except FieldDoesNotExist: + hash_field_exists = False + + # Update instances where the plain field is null or empty, setting the + # encrypted and hash fields to empty values (if they exist). + update_kwargs: dict[str, t.Any] = {} + if enc_field_exists: + update_kwargs[enc_field_name] = b"" + if hash_field_exists: + update_kwargs[hash_field_name] = "" + update_count = manager.filter(plain_field_is_null_or_empty).update( + **update_kwargs ) + print( + f"Updated {update_count} instances of {model_name} for field " + f"{field_name} where {plain_field_name} is null or empty." + ) + + # If the hash field does not exist, we don't need to do anything else. + if not hash_field_exists: + return - # Additional filter if provided. + # Build a queryset of instances where the plain field is not null or + # empty, and apply any additional filtering provided by qs_filter. + queryset = manager.filter(~plain_field_is_null_or_empty) if qs_filter is not None: queryset = queryset.filter(qs_filter) + count = queryset.count() + if count == 0: + print( + f"No instances of {model_name} found for field {field_name} " + f"where {plain_field_name} is not null or empty." + ) + return + print( + f"Hashing {count} instances of {model_name} for field " + f"{field_name}..." + ) + + # Set the chunk size for bulk updates and initialize an empty list to + # hold instances to be updated. + chunk_size = 1000 + instances: list[Model] = [] - # Select fields. - fields = [plain_field[0]] - if has_enc_field: - fields.append(enc_field[0]) - if has_hash_field: - fields.append(hash_field[0]) - queryset = queryset.only(*fields) + # Helper function to bulk update instances in chunks. + def bulk_update(i: int): + nonlocal instances + if not instances: + return + manager.bulk_update( + instances, fields=[hash_field_name], batch_size=chunk_size + ) + instances = [] + print(f"({i}/{count})") # Iterate over the queryset in chunks and save each instance. - for instance in queryset.iterator(chunk_size=1000): + i = 0 + for i, instance in enumerate( + queryset.only(plain_field_name, hash_field_name).iterator( + chunk_size + ), + start=1, + ): # Get the plain value. - value = getattr(instance, plain_field[0]) + value = getattr(instance, plain_field_name) + + # Set the hash value using the Sha256Field's set method, which will + # normalize and hash the value before setting it on the instance. + Sha256Field.set(instance, value, hash_field_name) + + # Append the instance to the list of instances to be bulk updated. + instances.append(instance) - # Set the plain, encrypted and hash values. - setattr(instance, field_name, value) + # Print progress every chunk_size instances. + if len(instances) == chunk_size: + bulk_update(i) - # Save the instance, updating only the relevant fields. - instance.save(update_fields=fields) + # Bulk update any remaining instances. + if len(instances) > 0: + bulk_update(count) return migrations.RunPython(forwards_func) @@ -119,7 +164,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="email", db_column="email_enc", - default="", + default=b"", verbose_name="email address", ), preserve_default=False, @@ -144,7 +189,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="first_name", db_column="first_name_enc", - default="", + default=b"", verbose_name="first name", ), preserve_default=False, @@ -169,7 +214,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="last_name", db_column="last_name_enc", - default="", + default=b"", verbose_name="last name", ), preserve_default=False, @@ -182,7 +227,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="username", db_column="username_enc", - default="", + default=b"", verbose_name="username", ), preserve_default=False, @@ -211,7 +256,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="access_code", db_column="access_code_enc", - default="", + default=b"", verbose_name="access code", ), preserve_default=False, @@ -242,7 +287,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="name", db_column="name_enc", - default="", + default=b"", verbose_name="name", ), preserve_default=False, @@ -270,7 +315,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="invited_teacher_email", db_column="invited_teacher_email_enc", - default="", + default=b"", verbose_name="invited teacher email", ), preserve_default=False, @@ -285,7 +330,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="invited_teacher_first_name", db_column="invited_teacher_first_name_enc", - default="", + default=b"", verbose_name="invited teacher first name", ), preserve_default=False, @@ -298,7 +343,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="invited_teacher_last_name", db_column="invited_teacher_last_name_enc", - default="", + default=b"", verbose_name="invited teacher last name", ), preserve_default=False, @@ -311,7 +356,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="token", db_column="token_enc", - default="", + default=b"", verbose_name="token", ), preserve_default=False, @@ -340,7 +385,7 @@ def set_school_field(field_name: str): field=EncryptedTextField( associated_data="name", db_column="name_enc", - default="", + default=b"", verbose_name="name", ), preserve_default=False, From 75f9047ef7a9ee1a99a8bc3d32e060920a09a5d7 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Tue, 19 May 2026 13:52:18 +0000 Subject: [PATCH 05/18] normalize hashes --- codeforlife/user/fixtures/google_users.json | 2 +- codeforlife/user/fixtures/independent.json | 4 +- codeforlife/user/fixtures/legacy.json | 54 +++++++++---------- .../user/fixtures/non_school_teacher.json | 4 +- codeforlife/user/fixtures/school_1.json | 16 +++--- codeforlife/user/fixtures/school_2.json | 10 ++-- codeforlife/user/fixtures/school_3.json | 6 +-- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/codeforlife/user/fixtures/google_users.json b/codeforlife/user/fixtures/google_users.json index c9d02b8b..561d8247 100644 --- a/codeforlife/user/fixtures/google_users.json +++ b/codeforlife/user/fixtures/google_users.json @@ -7,7 +7,7 @@ "_email_hash": "ee95f43c0012fa1a9d5771313a7034cf94af568b0588b34ca20c34c25701af78", "_email_plain": "google.teacher@noschool.com", "_first_name_enc": "ZmFrZV9lbmM6KXn5yJbvaIAq1O8qATyemFxIK7GJRkWh1tdv/QaBc/4PQw==", - "_first_name_hash": "5463f3f79e73077256f212d9e43a2752447cb7b238cda23004926bc0b9c5076f", + "_first_name_hash": "a7e4c63feb2b46212c35276010cfcc7a0a8a021f42aefab89765c211cc794870", "_first_name_plain": "Google", "_last_name_enc": "ZmFrZV9lbmM6DYRgnaINtnv2v6s09apDac1iGUCekmo7k4MuZ5TVKwhWdUE=", "_last_name_plain": "Teacher", diff --git a/codeforlife/user/fixtures/independent.json b/codeforlife/user/fixtures/independent.json index 67feca8b..fc1adccd 100644 --- a/codeforlife/user/fixtures/independent.json +++ b/codeforlife/user/fixtures/independent.json @@ -7,7 +7,7 @@ "_email_hash": "8efc7683cec3a31792d0c22f49a1cc374ba2ae2199b3cb8d228a7cbabe8370b2", "_email_plain": "indy.requester@email.com", "_first_name_enc": "ZmFrZV9lbmM6pCLIN/sWZNWKMxn/nDrhIxPqZJOvgjlMCMMbSP6LavQ=", - "_first_name_hash": "c90d2a9004eccdad8bec752f90ffe656955c44a81f984d3d32b827f17b473d7c", + "_first_name_hash": "fe1fe542767696689c8767d1b1e86734ce210252c07acc349c3a9f6175994e20", "_first_name_plain": "Indy", "_last_name_enc": "ZmFrZV9lbmM6/3o0FjosbmbS1oVTg10ezTJdBjsJBeuKxIHRFNBmZPASWT+9dA==", "_last_name_plain": "Requester", @@ -43,7 +43,7 @@ "_email_hash": "63e90930d7617f596f1006362382d81aecb8308c5a9893007f78c50e954e6d1a", "_email_plain": "indy@email.com", "_first_name_enc": "ZmFrZV9lbmM6P0Fsz+sx2cuXNgTPn0AoikMvFz67Uy2F6X+I2kCPTOg=", - "_first_name_hash": "c90d2a9004eccdad8bec752f90ffe656955c44a81f984d3d32b827f17b473d7c", + "_first_name_hash": "fe1fe542767696689c8767d1b1e86734ce210252c07acc349c3a9f6175994e20", "_first_name_plain": "Indy", "_last_name_enc": "ZmFrZV9lbmM6aOhhzg9mD3C0ROh1lCuDC1XKskZ6DYh6ajmv7jRUFFx4GyBY4w==", "_last_name_plain": "NoRequest", diff --git a/codeforlife/user/fixtures/legacy.json b/codeforlife/user/fixtures/legacy.json index 8b7e56ba..3f2f6f4d 100644 --- a/codeforlife/user/fixtures/legacy.json +++ b/codeforlife/user/fixtures/legacy.json @@ -235,7 +235,7 @@ "pk": 1, "fields": { "_name_enc": "ZmFrZV9lbmM6OZ5fpw/7lnlu6VCmZ1D9j/ypiKMWMpPS4xgpUVojgcT7kmoAI8VUeao08V7yLsHnE+SyTew=", - "_name_hash": "de785da99b1a6796945de0ead083eb3990fee70371a7d998682b450429ef8c1e", + "_name_hash": "6cb001965f12442bfcf1a8d9ecf5e31c6b2687bd5d05190cf56ecf648d9684f6", "_name_plain": "Swiss Federal Polytechnic", "country": "GB", "county": "nan", @@ -300,7 +300,7 @@ "_access_code_hash": "7dcbfe80fd523896acdad8310fe14d804dad5ae7e4dc4975815f3d5ff76aa963", "_access_code_plain": "AB123", "_name_enc": "ZmFrZV9lbmM6rfVJBo3PCRlQYodcqpqSFxZp7dYSoex1a8m6nuPs+pP1xkSkBg==", - "_name_hash": "9640c03014663d3047ca86523c62baf8e3cbb208a08c6f839ce649b1aac341d2", + "_name_hash": "d8857e685fc7cd512c000209b05e67d4c2341f7770e93671adafdcdc332b21e7", "_name_plain": "Class 101", "accept_requests_until": null, "always_accept_requests": true, @@ -319,7 +319,7 @@ "_access_code_hash": "ee16d5af3f31578ab0327da3257b212fe2beefcadef4f6b8ad6c992d153b90aa", "_access_code_plain": "AB124", "_name_enc": "ZmFrZV9lbmM6q8B+3LwD7ohrKPUED05T82/7CtGc2VslUmlVOG7o9tba5SG+Ag==", - "_name_hash": "d14218762d835f7a9c96377522b0c34da735efa982ea5a7d041bb5f0522cb3f3", + "_name_hash": "20a84b82906c60b52b71aebab89320cefd1dc47e0f84861664f9e44d0f70d594", "_name_plain": "Class 102", "accept_requests_until": null, "always_accept_requests": true, @@ -338,7 +338,7 @@ "_access_code_hash": "a895a482b72ec418311b8a9677bcf2c690425993f563f58d4a15b9586753b88b", "_access_code_plain": "AB125", "_name_enc": "ZmFrZV9lbmM60/Rt59QRAWw4AcoT0nKDqrFm25OgmMhKElgPi2A/e2oTsy9ZZg==", - "_name_hash": "e1782a43b71afd648ae245648d45e617665bc1ab9a65c6c05324eb1818bad644", + "_name_hash": "39710c33747315c8c6de8a7d5af97872303081fb0093cdb363aa4180223ac4a2", "_name_plain": "Class 103", "accept_requests_until": null, "always_accept_requests": true, @@ -357,7 +357,7 @@ "_access_code_hash": "38045227bf3c575c643dc1b34a1e70669f35b90225495bbeba0a53ae5ba82c81", "_access_code_plain": "RL123", "_name_enc": "ZmFrZV9lbmM61dGjSEe2fv5xma2w6WNAaxOJ4yJ28pd7TJkdh1eh9LqgDchS/3hDrY+j77I=", - "_name_hash": "cb7730d71c019fd442db0db09d27e49a1d84f01e79a99a82a949b0c6ed24d508", + "_name_hash": "6a09152d3d184ca20ad7e3e2981761e8dd3747c93744591bd632843ce96f5e76", "_name_plain": "Young Coders 101", "accept_requests_until": null, "always_accept_requests": true, @@ -376,7 +376,7 @@ "_access_code_hash": "732f2cd0915bdea50771ce7c92a4e1b8ea83f6d3323b95dd07619e5ea7f0fd06", "_access_code_plain": "PO123", "_name_enc": "ZmFrZV9lbmM6SwqcaZ2noGygc4dHOKUsHTL7lWU79/WrqGgK9XqIgAUQjQmUpJZKLGE9nTHRxQ0=", - "_name_hash": "72b62d2a95a4fcf7e4f7853953f0bc20858a57f9e7c2c9203203a293a13631a9", + "_name_hash": "d502bc6b827c3f1fe7f8f43cc431f672c3b4044ff17883f1e081e5ace66d39ee", "_name_plain": "Portaladmin's class", "accept_requests_until": null, "always_accept_requests": true, @@ -598,7 +598,7 @@ "_email_hash": "6229d653107a83cfe416912e62307e9b3bfa5185b06deb74db27576e291ba301", "_email_plain": "codeforlife-portal@ocado.com", "_first_name_enc": "ZmFrZV9lbmM6ZPlIinlRmmI7yo7cj16Hxm4N1WVBWgTkcKwRvxvB3Jfjmg==", - "_first_name_hash": "dc0c24809d18768f01857048c816eb3885cb627a759bd2dd5314970adafdaf3e", + "_first_name_hash": "b2d8ed60f78679e18127a64fcb5df09083d9499b15a61cdea646b7cf8847f9c0", "_first_name_plain": "Portal", "_last_name_enc": "ZmFrZV9lbmM6aRN4l2rGBRhy1m+Qh2RWRvNmADOqa+g4Sn/eUOwvUULe", "_last_name_plain": "Admin", @@ -624,7 +624,7 @@ "_email_hash": "9cb6a6152b4d3dfbad65576fb4f6688b56d73cbc65b36b2a7fbb313a1352d3c4", "_email_plain": "alberteinstein@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6dygqD/GL7Zx+cDPU0dlncOJFQhUZrEJkEWlTHALit+s5Xg==", - "_first_name_hash": "920e5149d016b9ef005052b8f52ec833b0974ecf7fb25dc8efd42d8cb1912bf2", + "_first_name_hash": "a28518a55f49b7810427f62b8cef8dcffde82aff3f8e16a17769f185caf8e11f", "_first_name_plain": "Albert", "_last_name_enc": "ZmFrZV9lbmM6AnE2DA2bdWH5SpZDB5F5PlypOzoZ07TeA2vLqfILyZbm9Y5c", "_last_name_plain": "Einstein", @@ -650,7 +650,7 @@ "_email_hash": "586984c6c88ed9d06cbb6d64e746ca1c7b71dc8782f68a075aece08fd2b5557a", "_email_plain": "maxplanck@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6Ago4UQZ3oxNQyigyT60Lb+fyVC87pZtkjk1MpUf41Q==", - "_first_name_hash": "337aa91f0cb585a80fee292721d7e78119da4aa52be054785bf1069302979e5e", + "_first_name_hash": "dc9a9f8e2d48f4db615efe61ad4f04906e28c93e37d4cf0ab13f692bcb86dfb1", "_first_name_plain": "Max", "_last_name_enc": "ZmFrZV9lbmM6ZdaW5MUAIckllddasnE72Xm6ZSxl9Z7lR3Hjyc8TDeeNhg==", "_last_name_plain": "Planck", @@ -676,7 +676,7 @@ "_email_hash": "6df8a9b30d61359c2f434a45e881fd4461899af9f5df0e05fd15ba57e5885878", "_email_plain": "ramleith@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6MP5pw/kzO6fiWqUFK0ZxqwsWrY66EuDvC1jVIAtulw==", - "_first_name_hash": "62ea6bdedfb21184d0fb2342a6109dfd8ecbe0ed5b80ec8db8b2b414ac01423c", + "_first_name_hash": "e46e1b8306b6d1c91eb05ee6dd95241517067d4a166b02e98f3148197340b542", "_first_name_plain": "Ram", "_last_name_enc": "ZmFrZV9lbmM6Jn1K31v4LzeQ/qZffGAGMZG0/J6biO5+Y1DEnEOI4CeK", "_last_name_plain": "Leith", @@ -702,7 +702,7 @@ "_email_hash": "55a63981ea426d7470560ecfaf3b53c5ecd90e036edd8eca2b1a33c84178908e", "_email_plain": "leonardodavinci@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6ZT6hCR6NYDU208WG5Zt2Q18itxlQtSKfLZ/3c5K+bxRw43K6", - "_first_name_hash": "9e84472777869ca6205d06421ed265c1c3cfcdfc35a7de158201227c3c541198", + "_first_name_hash": "858907074712f2743f5e3408525017a064aa8e717c4d47d913ff81c29ddde3c6", "_first_name_plain": "Leonardo", "_last_name_enc": "ZmFrZV9lbmM6Eiq20D36p1cMlSdqcy5Mtqq3hm22MTDChEcibQg985YsRXE=", "_last_name_plain": "DaVinci", @@ -728,7 +728,7 @@ "_email_hash": "1c5f6e85fe62be2048d5a3867eff53db48f9bcd6da889b92e68668a4a464afd2", "_email_plain": "galileogalilei@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6FGiykuz+Sx8x0lgoaRGOOBtiL2fRSKJDSIMLZoxttrQl4fo=", - "_first_name_hash": "cae481802ba39e14537dda2d0125c791bab6e0b4e323e9833591a5de6c50207e", + "_first_name_hash": "3cf019416da72602544d0127b388fdd9c36e40bb648aab0e5f267b707d85b4ae", "_first_name_plain": "Galileo", "_last_name_enc": "ZmFrZV9lbmM6zQGTj4h6SuBQ+GHF0VPEIZO+bu/PsWXhzhyuHeHIefBYmHE=", "_last_name_plain": "Galilei", @@ -754,7 +754,7 @@ "_email_hash": "4fc8d589140865b27bc27623cb547e2a24f9b985ab2e1dd0b5b5c035d367c4b6", "_email_plain": "isaacnewton@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6eCpZHFZkWzQ/hv8G9psJxOAqvezmqYDq0RSpDJlEptGK", - "_first_name_hash": "c49fb23f0176885dece511c58087b7f23019c343b9861e2d70926600577a4611", + "_first_name_hash": "81bdaa26501a8cb2ee69aec002df098c5025b75ae0494b4b9ff6ef003131a792", "_first_name_plain": "Isaac", "_last_name_enc": "ZmFrZV9lbmM6xN/iK8bQby0voDF9a35nJ83At3HgLkGbAptBWLtTNtnZzg==", "_last_name_plain": "Newton", @@ -780,7 +780,7 @@ "_email_hash": "6bd23daa66a426a27655016f084d3fa65a49e2e5d264701b6d84e550e415638d", "_email_plain": "richardfeynman@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6KD7poCabt24+OPDhCz9fiZcUqF+PrVEHoIT9gwFbShzh3JU=", - "_first_name_hash": "0a8a5deb9f8f9a8f0e96af7630e59b6fdc0a2528095a5a382c3d417da9c97f7e", + "_first_name_hash": "4997281c5f1ee586af59704d9790c82ed1260e035073cbf05ba9ef24fea9f4d4", "_first_name_plain": "Richard", "_last_name_enc": "ZmFrZV9lbmM6JgmcA+18EuPfUeWJSuXju16Z0h+HDj9m/Xn51LaSe+Ef2TA=", "_last_name_plain": "Feynman", @@ -806,7 +806,7 @@ "_email_hash": "1ab9d2f1ae8ef6a5e99cee8948c326e3404aceb89b8d72215e297a1cfc8393a6", "_email_plain": "alexanderflemming@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6ZVz+1VG3scf9MoEfVA/ShO5cJNBE9riNqQ/2I20l/OsroQMZrA==", - "_first_name_hash": "e78f1c3e180b8af2c8dfc79ec04d65ac565361c887584100e67890cdb70ab52b", + "_first_name_hash": "19e59a84a987c54abfd434eba28d49fa2c68b2be8a5649509562bd63b886e6d5", "_first_name_plain": "Alexander", "_last_name_enc": "ZmFrZV9lbmM6jxxUTKIDlEj/scNJ2jRVZujwmG8nYCfRqepcQi7lnBz/vYSI", "_last_name_plain": "Flemming", @@ -832,7 +832,7 @@ "_email_hash": "0c92dd15cc629b84ea9436c965f944543cfedd41ced6c8f2594a26ee50cf3ea5", "_email_plain": "danielbernoulli@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM65SWBLZni/laQtNAl64mXA79twEZho+651VCt311rAguFzA==", - "_first_name_hash": "6bc6d4fc17ad3131d3417aebb876bdec85a697de1a69734b9d113c800e72aaf3", + "_first_name_hash": "d8c5aba072c365b5ad92809ffa80dd21df8a77b7b28c2bb52d8e59492dc93f30", "_first_name_plain": "Daniel", "_last_name_enc": "ZmFrZV9lbmM60eUQSPe5OAsCeRt1gzrwLa7EVSar6AnRbZG3gHztcvdth8IrvA==", "_last_name_plain": "Bernoulli", @@ -858,7 +858,7 @@ "_email_hash": "1a93bc8026b39e9d71124c74cef2f072d4e41625ea3b12f4009f104dce962e2f", "_email_plain": "indianajones@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM62ghGxpAhcBGTgECH3XG5lhv3/nGPelk1SYi2on3HmiQnnEs=", - "_first_name_hash": "9859180db8b64c329244146e3cfda4e6c42fd38f8ef9fc0f272af81298c6c02a", + "_first_name_hash": "e63a00d61aa9887fd6e43a9a00545094b1dd9f9bfa902ee4e66bc1fe9e889bf7", "_first_name_plain": "Indiana", "_last_name_enc": "ZmFrZV9lbmM6excgvA44+7Tzd9kxwsurRR0/7JddJ4ONlEhYSqS+e1XH", "_last_name_plain": "Jones", @@ -881,7 +881,7 @@ "pk": 12, "fields": { "_first_name_enc": "ZmFrZV9lbmM6kjfxs345zoTZfqKXkDHtfcIY8poSjk8VFfhMq1knz/8=", - "_first_name_hash": "91886b94f7298f93affde2a4c4bb1e1ec836b5da66507834c964dc8db418fa14", + "_first_name_hash": "010d83330e905dd4464175fc41558206ba825a7428519c50b16b20607211ac21", "_first_name_plain": "Noah", "_last_name_enc": "ZmFrZV9lbmM6eHb/bRCUVrG5ESV5qHfbsiBvRelPu7wOHL8ci370OvxasRpm", "_last_name_plain": "Monaghan", @@ -904,7 +904,7 @@ "pk": 13, "fields": { "_first_name_enc": "ZmFrZV9lbmM6BWoSB7BYxH4FxpQhkd4TvieVMj0N3sPdLhCNohLeetFyrg==", - "_first_name_hash": "050fa7990069876e75ceb3fe387a3f10b5beec63612c5b4e500cfcf246daa312", + "_first_name_hash": "00ef09d6ae7627251b0673c48d9f00234537da9d3f38941b06726e6d902a5a66", "_first_name_plain": "Elliot", "_last_name_enc": "ZmFrZV9lbmM69YjNMSfcXFg3pGr58K77NGxqGG3nV21qsO+sSnFPRvbd", "_last_name_plain": "Sharp", @@ -927,7 +927,7 @@ "pk": 14, "fields": { "_first_name_enc": "ZmFrZV9lbmM663tdyyomEQ2QzpHpj3ZX8IldQVNNvwZoKn7SuywzsKUDvA==", - "_first_name_hash": "969bd503b94e99bb283caa34c5e28fcb8cebfb278111bff1f76ed1f8d08c2720", + "_first_name_hash": "5a012346d27f934d7196025377f84ef1c458929dfdd87419396780a4a2f5299d", "_first_name_plain": "Tajmae", "_last_name_enc": "ZmFrZV9lbmM6Hb+2sb0IFHydvEnx3koqRuZDYKraKhtD1gLI7XSs5hrM6g==", "_last_name_plain": "Joseph", @@ -950,7 +950,7 @@ "pk": 15, "fields": { "_first_name_enc": "ZmFrZV9lbmM6xiOckTwack87H3A7PBretwem+4On+TLQK7vQ40/0FpPNCX8=", - "_first_name_hash": "8e450f405264e612fb62f915830166cd43c03612b82c19c0c52aaee93183b0bb", + "_first_name_hash": "ab51446cbc56dc4eeddc680b90f3e81311601bdcff926bd9d1fce3e0b50da6d5", "_first_name_plain": "Carlton", "_last_name_enc": "ZmFrZV9lbmM69RSRhxmcba/7l7H8s49DQTlBtOLzTQUsgdggRgHLYzUELA==", "_last_name_plain": "Joseph", @@ -973,7 +973,7 @@ "pk": 16, "fields": { "_first_name_enc": "ZmFrZV9lbmM6OtOk2SsoMWAc38TD0Q41fmaYTkUR0+0nt2Rw/JuDkgb5", - "_first_name_hash": "8424a2115bf1ee8fa253b59cd46cd6487c1b53b51201e593c08a12c390d70ad4", + "_first_name_hash": "6b152865ddf094484e72b8277cc134df7c3a49947dc90a5438fdcf59a8e9c668", "_first_name_plain": "Nadal", "_last_name_enc": "ZmFrZV9lbmM6f8mE9Wc19ubz9l+FnWRJU4dXhOwbl3Vwc2ISVdXXF4GmrbeK+AMRwdZVwDI=", "_last_name_plain": "Spencer-Jennings", @@ -996,7 +996,7 @@ "pk": 17, "fields": { "_first_name_enc": "ZmFrZV9lbmM6j0YLTeDH49wQDBGAGGslQq+XvbkmjBOjZz/81chsQ6CgduE=", - "_first_name_hash": "8d81abd4d9f4dd116d4473915b068c9790f2229ab72fbcd974daff57ebf62717", + "_first_name_hash": "0e9530fe4c47d64bb62a31e31e730665ac69d3e8ba9f311523c20d09ea53ec70", "_first_name_plain": "Freddie", "_last_name_enc": "ZmFrZV9lbmM6qqw2DvU2GxSQXO2EqczZa2U8FVS4mRcyIZoDpbhoEeI=", "_last_name_plain": "Goff", @@ -1019,7 +1019,7 @@ "pk": 18, "fields": { "_first_name_enc": "ZmFrZV9lbmM6UerbiIlXuCiufRtvMUVZ+JauFHZ3y0Fp/GZl7ISGn/o=", - "_first_name_hash": "5081dca0f5b49f2fdfda5543c6bbeef8e733838de7b8e642064f9b8edc0e24b5", + "_first_name_hash": "496b23a98c8f4f069a4ee4ed4ec498ad6b41e2ca921686339b87cdcceb9ee210", "_first_name_plain": "Leon", "_last_name_enc": "ZmFrZV9lbmM6bxT3gxf8cicQPv5nGIKLHApI/oxYN3k15TGYe4vwZec0", "_last_name_plain": "Scott", @@ -1042,7 +1042,7 @@ "pk": 19, "fields": { "_first_name_enc": "ZmFrZV9lbmM609cs4YE+Kj0atwhR0SOrJyuO9vSA74sqShdgLD7KRQ+C", - "_first_name_hash": "c25e694e58d404373d63ef5dd2b2a413dab9c64aab028d735401c166813a87fa", + "_first_name_hash": "32b600fd12085215293ef30e867521520740cad567e21dc4c138c354d9189a1f", "_first_name_plain": "Betty", "_last_name_enc": "ZmFrZV9lbmM604sjV50gIcumJ2c/iDCvy3Siq/kVoTKllzvkyghd6A2hMoE=", "_last_name_plain": "Kessell", @@ -1065,7 +1065,7 @@ "pk": 20, "fields": { "_first_name_enc": "ZmFrZV9lbmM6kEW6JpB05mjWAPE/xrxMrbFNoyD6ZBNVl+YCZI/q3+LRROg=", - "_first_name_hash": "959e43f80903c037d87f933a2225789fbdbd71e0dea09fc1854a51e8d2f9b17e", + "_first_name_hash": "1dbd289e95b4925fc85eff99d7e879b2033f60182e05872f8665ff271d0d337c", "_first_name_plain": "Deleted", "_last_name_enc": "ZmFrZV9lbmM6gwxaGy9/aDQWDEVUwUverJ6wmk3ohaTkvF5GYq6koNg=", "_last_name_plain": "User", @@ -1091,7 +1091,7 @@ "_email_hash": "d10b776ab1e76347e63692528b0e4e97a5e3288b5f3c918476577a96b9255463", "_email_plain": "adminstudent@codeforlife.com", "_first_name_enc": "ZmFrZV9lbmM6WI6wHI7iEIUv0n0GxIsHsV75X85C0vRUlSEmAMnBr8aQOaDLa8RN", - "_first_name_hash": "689afa6b012cd1c17a5b32dbf80482ccb8049f87af317883a23e7144b84242b2", + "_first_name_hash": "a10cf8b1ffba3acb6bb68f941fde5b5226a0b0ab8e76390628114d73384792a7", "_first_name_plain": "Portaladmin", "_last_name_enc": "ZmFrZV9lbmM6++cVFuCbPR0sIbZk2qM4m46PJKlZleoblSe6PWeixQl61Qk=", "_last_name_plain": "Student", diff --git a/codeforlife/user/fixtures/non_school_teacher.json b/codeforlife/user/fixtures/non_school_teacher.json index 2615f6d2..0e4582e8 100644 --- a/codeforlife/user/fixtures/non_school_teacher.json +++ b/codeforlife/user/fixtures/non_school_teacher.json @@ -7,7 +7,7 @@ "_email_hash": "7cc99dd2f335a068ac421d2cec8f04687fa34d37d313718f56ebc6b6c3c09d2c", "_email_plain": "teacher@noschool.com", "_first_name_enc": "ZmFrZV9lbmM6naDBVRP/vWnfgWPmHiirA6COL71ARyfg0pnyeQ3DYlU=", - "_first_name_hash": "7554c6b4930e0ac2c4d7fb5e7e266797b0444953067a0d7a17e835b518229e84", + "_first_name_hash": "d0118d7d585c39fb03017af6692eb181443402f0cb216e1541ed2c4a219955e0", "_first_name_plain": "John", "_last_name_enc": "ZmFrZV9lbmM6BVE1wAneS3Mcvb4o3lxNbaV8U2yFpq7pBiP5ai8FWA==", "_last_name_plain": "Doe", @@ -42,7 +42,7 @@ "_email_hash": "7f34bbe92504e0ca5c2235f2f64bf2686a7d040690a45948074247cad2eb5c71", "_email_plain": "unverified.teacher@noschool.com", "_first_name_enc": "ZmFrZV9lbmM6TYaKDI0fEu4AoVsf35i4onELsjpPePS3+0aTi7bJ/cz6Uu6Cr3I=", - "_first_name_hash": "47c086ea8af2fcd653408088d2d3809d3d46fd2698755a4a208d5fb78ebc205b", + "_first_name_hash": "7b5f659dd23dbcb84e56645e523c2dc962e4a99120c99bc3e99d361f85028605", "_first_name_plain": "Unverified", "_last_name_enc": "ZmFrZV9lbmM6045ESPZ+3oNUHhw8P27QOxgaRxBEbAkT1nI9LGBZeTcrWqU=", "_last_name_plain": "Teacher", diff --git a/codeforlife/user/fixtures/school_1.json b/codeforlife/user/fixtures/school_1.json index 701f05c4..74b12e4b 100644 --- a/codeforlife/user/fixtures/school_1.json +++ b/codeforlife/user/fixtures/school_1.json @@ -4,7 +4,7 @@ "pk": 2, "fields": { "_name_enc": "ZmFrZV9lbmM67d4VVL/VNF+xT49FGteqYAb5nww953xETgWD4Dbq/Wx+Jtoe", - "_name_hash": "ae1a1256e88d98ce7870314ac36e23b7ab5163d4bdc6b72617b4c25a2324f256", + "_name_hash": "18e858c37975d85d482cfca4547d9a8abad8d1474b19cad80c703e51a37a51dd", "_name_plain": "School 1", "country": "GB", "county": "Hertfordshire", @@ -19,7 +19,7 @@ "_email_hash": "d23e2f365e919be1cf5e29a5472801d54bd75c6f360bce7c37674506bc63a9ce", "_email_plain": "teacher@school1.com", "_first_name_enc": "ZmFrZV9lbmM6KHAoomsgzlenkIf3O3B1HkeoBYLCHnmBSzdbpebUfr0=", - "_first_name_hash": "7554c6b4930e0ac2c4d7fb5e7e266797b0444953067a0d7a17e835b518229e84", + "_first_name_hash": "d0118d7d585c39fb03017af6692eb181443402f0cb216e1541ed2c4a219955e0", "_first_name_plain": "John", "_last_name_enc": "ZmFrZV9lbmM6UiD5C/IkRNEYEE7NgaGlAdMc+Le7fUhUCXNyEwEVYQ==", "_last_name_plain": "Doe", @@ -55,7 +55,7 @@ "_access_code_hash": "ca366870cf5e795a3c9ff340f119df2ff646128cf164d992b0167bbca8c247eb", "_access_code_plain": "ZZ111", "_name_enc": "ZmFrZV9lbmM6GE5ywQd6ChB38qHTDtALv40nO1RQu1ty7Azups4SMdlBAfNUh93EdK65vyHrMg==", - "_name_hash": "62e88a78e3a94e7b616d06399b55a3e667b85cf14386698df532d3c381244781", + "_name_hash": "bdd4dcac0ca92a55010f608038920319cc8a12441fe47e502cd4b78414cc65b5", "_name_plain": "Class 1 @ School 1", "accept_requests_until": "9999-02-09 20:26:08.298402+00:00", "teacher": 6 @@ -66,7 +66,7 @@ "pk": 27, "fields": { "_first_name_enc": "ZmFrZV9lbmM68LydJ3q0a8MRqB/aobe1wf3XPUOZZV/ibILf8bFjWeeTLVLY", - "_first_name_hash": "341d77ca3f20f1a3ddf8d9a06d076940c6e24b252a08c5dae066b43c2b236ed5", + "_first_name_hash": "8b8e193489d0dee3b05730e8a0ed7dc86f8517dca0bd3a72935f427612476037", "_first_name_plain": "Student1", "_username_enc": "ZmFrZV9lbmM6fgGMjMvNBd0UtOYl4ZheNodBdcSjPsVmLYrIES+t5PbSSw66LNdmCAXLWefCbQu3Nlf4m3ESwd/3cw==", "_username_hash": "1a3a828a1d7b20c8c9bd4d3677dcd8e5df6352c950b2d99e04a77cdc291fbfbe", @@ -100,7 +100,7 @@ "_email_hash": "01c1e0721b31cb0636da3f5b3dc1a107bdd44fd0185fa12dd87f6aad06c91a08", "_email_plain": "admin.teacher@school1.com", "_first_name_enc": "ZmFrZV9lbmM6ehSIW3xreQS4cUAXbBMnqq6OA7uS9iYEI9Y7iNOVDS4=", - "_first_name_hash": "a5945caf6491a22426f870debae98c823043af86691d336f95248daff7bb989a", + "_first_name_hash": "549014545102d32b3fd8280d7b5c1146c555b3ca09e02d71498a0d4d9c01283c", "_first_name_plain": "Jane", "_last_name_enc": "ZmFrZV9lbmM6uf/W7NPtVAiFRyiZCwzvPOWrYa1KeBZT/HcZhoO1MQ==", "_last_name_plain": "Doe", @@ -137,7 +137,7 @@ "_access_code_hash": "e90c6683965279b92914a9e2b18716f0bc826c0dccb5976130562ff88cbd7f01", "_access_code_plain": "ZZ222", "_name_enc": "ZmFrZV9lbmM6ghwPnoQhhjiDpIsfE2ufqAMOaXJAuiwUZ7nwuRLu208FZznOJkbXb3FKy9iE4Q==", - "_name_hash": "b385f80b0a7d2cbdff653027b2f280e492fdd8f3c8c968de11d593b2a790f3af", + "_name_hash": "f102f85bd156b35d38322042e7a9a9f0335b3eb39db766aa79d9ff9b4e43d968", "_name_plain": "Class 2 @ School 1", "teacher": 7 } @@ -147,7 +147,7 @@ "pk": 29, "fields": { "_first_name_enc": "ZmFrZV9lbmM6WTlhK3trdBVmqjtck1x0Jemd/5F1xq/hu5ghMYR+W+HRRsCj", - "_first_name_hash": "f7ecf0b3ebd27342b1a6d30a6ef300b6dc23518fd5c83ca25cdc2b7135927cf8", + "_first_name_hash": "0d082766ef1a725b9da1a94ea7436ba0cc58feb4682d708b973ae8bd183abf54", "_first_name_plain": "Student2", "_username_enc": "ZmFrZV9lbmM6r10EMezevJm9pQjvClWd/Rx06lfTdlIRYzsYEUcAq+V0yPYcafJ63+j+6jbHBcOR8qlXYoBw10mOHA==", "_username_hash": "69a7b577f2249a789e2a12ee80ee669c30ff20872c94153abeb3ebc05a61be08", @@ -181,7 +181,7 @@ "_access_code_hash": "3e3b96d4c7775a2c8d15999f6d662b164ccf3d66b8df93329bd5e426251d8cdb", "_access_code_plain": "ZZ333", "_name_enc": "ZmFrZV9lbmM6d6hSPPotg2fBEgOhd6IRzrvlWYQxFaSv4v/FS7uXG7YxNB2XBq2k0Y08ZrhFtQ==", - "_name_hash": "b6ff16f6d1e61c2f2845f3cde4cc5a67704b3d0b55e1d825c6813425fbbf5b8b", + "_name_hash": "a234c021eb820b72f6fb23a730d5d1b751767efdc374363826a99d220de21b34", "_name_plain": "Class 3 @ School 1", "accept_requests_until": "2023-02-09 20:26:08.298402+00:00", "teacher": 7 diff --git a/codeforlife/user/fixtures/school_2.json b/codeforlife/user/fixtures/school_2.json index cfbedf50..9a46be9c 100644 --- a/codeforlife/user/fixtures/school_2.json +++ b/codeforlife/user/fixtures/school_2.json @@ -4,7 +4,7 @@ "pk": 3, "fields": { "_name_enc": "ZmFrZV9lbmM6rDyP9kxRgaY6xKUohQdWebf8Gs94JU03UH5NCdROWyGOpUk5", - "_name_hash": "de23670d9df1c898501128f02d31ed462ffef10d0d884cd1807ebfba96c6751c", + "_name_hash": "b65117fdd735fc7d03e5ffcba2e616a2298b2042c23ffadb2e554c23ae488488", "_name_plain": "School 2", "country": "GB", "county": "Hertfordshire", @@ -19,7 +19,7 @@ "_email_hash": "8cdd687351ee31833adcb72d37e7b8f1c1c17c655c3fb182ac0127320af64284", "_email_plain": "teacher@school2.com", "_first_name_enc": "ZmFrZV9lbmM6SugTLn6zL+eLgD5bNHpO3WG1WS+ijY591LnSOvIxDic=", - "_first_name_hash": "7554c6b4930e0ac2c4d7fb5e7e266797b0444953067a0d7a17e835b518229e84", + "_first_name_hash": "d0118d7d585c39fb03017af6692eb181443402f0cb216e1541ed2c4a219955e0", "_first_name_plain": "John", "_last_name_enc": "ZmFrZV9lbmM6JG27G2EreOMHHuSC6ydDGwMO9sE3g0VQBKNLJDomIw==", "_last_name_plain": "Doe", @@ -144,7 +144,7 @@ "_access_code_hash": "0157477281b2aee7660889207dee1a14756bddd428162de41bada7a02c593ff6", "_access_code_plain": "XX111", "_name_enc": "ZmFrZV9lbmM6QS58FDOcIpwtt0KJ08wr+sKWQCVBucwv75ScfLpy53RQGS8dRIut/FEBoNUQ2g==", - "_name_hash": "9d2590b59d26bcb0df89c9102403bb583445831149f8569feb4a75cb006e71a0", + "_name_hash": "19de16ed89970e886445d7e01bae0cfe2bc38d8a296993f238d783f39f936cea", "_name_plain": "Class 1 @ School 2", "teacher": 8 } @@ -157,7 +157,7 @@ "_email_hash": "4841a8d3657918adffdab6cb4a0a6b64ad771aa56a53f80fed0ee8d074841ca3", "_email_plain": "admin.teacher@school2.com", "_first_name_enc": "ZmFrZV9lbmM6XAAtQFeiEPJbJe9g1zErH67kShgjeyTihaxID8eHsG4=", - "_first_name_hash": "a5945caf6491a22426f870debae98c823043af86691d336f95248daff7bb989a", + "_first_name_hash": "549014545102d32b3fd8280d7b5c1146c555b3ca09e02d71498a0d4d9c01283c", "_first_name_plain": "Jane", "_last_name_enc": "ZmFrZV9lbmM60OVDyC7hAJrEkLkpcJ7LvaWcl5ovVLPrafucEwvZsQ==", "_last_name_plain": "Doe", @@ -203,7 +203,7 @@ "_access_code_hash": "750df33e8a531e83fddf652344b680983a1555c6dcd8fda80f4af785357b444d", "_access_code_plain": "XX222", "_name_enc": "ZmFrZV9lbmM65DRSk4ioNDC3lTdBdMzCx8izq6MA3hx1Lv7bVmIB0klrcE/ytKj/2YAOqQF5uA==", - "_name_hash": "8e8c44adf55dce807796a63836973c83deca1c78995167d05a02bcac4e45c2e9", + "_name_hash": "88dd12eec5057abd68c74f5c32197e6110968cf135c00059132e2c5427c09f8a", "_name_plain": "Class 2 @ School 2", "teacher": 9 } diff --git a/codeforlife/user/fixtures/school_3.json b/codeforlife/user/fixtures/school_3.json index 54494a0c..81f123b1 100644 --- a/codeforlife/user/fixtures/school_3.json +++ b/codeforlife/user/fixtures/school_3.json @@ -4,7 +4,7 @@ "pk": 4, "fields": { "_name_enc": "ZmFrZV9lbmM6LQq31+NKOlsLWDpsatuQth2lvPldBNrjK/oLXHFMGZbSkZFh", - "_name_hash": "ccb652f4e608481591b5ab4d60348eed454b26aabf4ffed93a9748590e816a11", + "_name_hash": "54e5ecd0160f95541ac3975e488f242ab928383d1b974e49025ad9d4b87ec6de", "_name_plain": "School 3", "country": "GB", "county": "Hertfordshire", @@ -19,7 +19,7 @@ "_email_hash": "22a604d4828ae8429e8303d16c7d9967145b214b827a8ff1845b19e2a10b12c3", "_email_plain": "admin.teacher@school3.com", "_first_name_enc": "ZmFrZV9lbmM6X/pzRYxC3wSQmFiiPbeGouhsrbGoQviFfkpXbF4nutGt", - "_first_name_hash": "083ec2a4fd4004ed6f9bd61965b170a9b5db5d5873c7217f65bede117f004a79", + "_first_name_hash": "cf728fccc7b72ca0648afcb4821ff233814704282ea069b5ac16d2afd5f41217", "_first_name_plain": "Peter", "_last_name_enc": "ZmFrZV9lbmM6W9e5RMeFGyGqvF5EXOFm66Sa6OeiQT/wbub//DRcXxzjUA==", "_last_name_plain": "Parker", @@ -56,7 +56,7 @@ "_email_hash": "c66f480d5458f514966dd1a540e652d1c57850a11122a26b1760eb95d7833fa1", "_email_plain": "teacher@school3.com", "_first_name_enc": "ZmFrZV9lbmM6XZjkxcuqVijVuzm3HPh/C+wn9MbuGSI6FH+qbWghkHIRqg==", - "_first_name_hash": "be367a9fe2b4bf78b7f7bd98e888fac606cfafa10fc91f25745d4a60e167ba72", + "_first_name_hash": "0f17a087a4b3fa11dbe711f99d1278341c5603776dd1e58578f3b4279fb725f8", "_first_name_plain": "Doctor", "_last_name_enc": "ZmFrZV9lbmM6rZsTsYhs5sQaqfUSMArJQ7u7cb/ufIvxvUANzmjust+rIjw=", "_last_name_plain": "Octopus", From e22e15e022841b496c60ca3a99a270b346074cb3 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Tue, 19 May 2026 16:35:59 +0000 Subject: [PATCH 06/18] create command --- .../management/commands/normalize_fields.py | 203 ++++++++++++++++++ .../0005_client_side_encryption_part_3.py | 167 +------------- 2 files changed, 204 insertions(+), 166 deletions(-) create mode 100644 codeforlife/user/management/commands/normalize_fields.py diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py new file mode 100644 index 00000000..afd31ab8 --- /dev/null +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -0,0 +1,203 @@ +import typing as t + +from django.core.exceptions import FieldDoesNotExist +from django.core.management.base import BaseCommand +from django.db.models import Model, Q + +from ....models.fields import Sha256Field + + +class Command(BaseCommand): + """ + Django management command to set encrypted and hash field values for all models. + + This command replicates the logic from migration 0005_client_side_encryption_part_3 + and automatically processes all models in the codeforlife.user app that have + encrypted fields. + """ + + help = "Normalize encrypted and hash fields for all user models" + + # Define all models and their fields to process + MODELS_TO_PROCESS = { + "User": { + "fields": ["first_name", "last_name", "username", "email"], + "filter": Q(is_active=True), + }, + "School": { + "fields": ["name"], + "filter": Q(is_active=True), + }, + "Class": { + "fields": ["name", "access_code"], + "filter": Q( + is_active=True, + teacher__isnull=False, + teacher__school__isnull=False, + teacher__school__is_active=True, + ), + }, + "SchoolTeacherInvitation": { + "fields": [ + "token", + "invited_teacher_first_name", + "invited_teacher_last_name", + "invited_teacher_email", + ], + "filter": Q( + is_active=True, + school__isnull=False, + school__is_active=True, + ), + }, + } + + def handle(self, *args, **options): + self.stdout.write( + self.style.SUCCESS("Starting to normalize encrypted fields...") + ) + self.stdout.write("") + + for model_name, config in self.MODELS_TO_PROCESS.items(): + for field_name in config["fields"]: + self.stdout.write( + self.style.HTTP_INFO( + f"Processing {model_name}.{field_name}..." + ) + ) + self.set_field(model_name, field_name, config.get("filter")) + self.stdout.write("") + + self.stdout.write( + self.style.SUCCESS("Successfully normalized all encrypted fields!") + ) + + def set_field( + self, model_name: str, field_name: str, qs_filter: Q | None = None + ): + """Set encrypted and hash field values for a model field.""" + from codeforlife.user import models + + # Get the model class from the apps registry. + try: + model_class: t.Type[Model] = getattr(models, model_name) + except AttributeError: + self.stderr.write( + self.style.ERROR( + f"Model '{model_name}' not found in codeforlife.user.models" + ) + ) + return + + manager = model_class.objects # type: ignore[attr-defined] + + # Generate the plain field name and the Q object to filter instances + # where the plain field is null or empty. + plain_field_name = f"_{field_name}_plain" + plain_field_is_null_or_empty = Q( + **{f"{plain_field_name}__isnull": True} + ) | Q(**{f"{plain_field_name}": ""}) + + # Generate the encrypted and hash field names and check if they exist. + enc_field_name = f"_{field_name}_enc" + try: + model_class._meta.get_field(enc_field_name) + enc_field_exists = True + except FieldDoesNotExist: + enc_field_exists = False + + hash_field_name = f"_{field_name}_hash" + try: + model_class._meta.get_field(hash_field_name) + hash_field_exists = True + except FieldDoesNotExist: + hash_field_exists = False + + # If neither encrypted nor hash field exists, skip this field. + if not enc_field_exists and not hash_field_exists: + self.stdout.write( + self.style.WARNING( + f"Skipping {model_name}.{field_name}: no encrypted or hash field found" + ) + ) + return + + # Update instances where the plain field is null or empty, setting the + # encrypted and hash fields to empty values (if they exist). + update_kwargs: dict[str, t.Any] = {} + if enc_field_exists: + update_kwargs[enc_field_name] = b"" + if hash_field_exists: + update_kwargs[hash_field_name] = "" + + update_count = manager.filter(plain_field_is_null_or_empty).update( + **update_kwargs + ) + self.stdout.write( + f" Updated {update_count} instances with null/empty {plain_field_name}" + ) + + # If the hash field does not exist, we don't need to do anything else. + if not hash_field_exists: + return + + # Build a queryset of instances where the plain field is not null or + # empty, and apply any additional filtering provided by qs_filter. + queryset = manager.filter(~plain_field_is_null_or_empty) + if qs_filter is not None: + queryset = queryset.filter(qs_filter) + + count = queryset.count() + if count == 0: + self.stdout.write(f" No instances to hash for {plain_field_name}") + return + + self.stdout.write(f" Hashing {count} instances...") + + # Set the chunk size for bulk updates and initialize an empty list to + # hold instances to be updated. + chunk_size = 1000 + instances: list[Model] = [] + + # Helper function to bulk update instances in chunks. + def bulk_update(i: int): + nonlocal instances + if not instances: + return + manager.bulk_update( + instances, fields=[hash_field_name], batch_size=chunk_size + ) + instances = [] + self.stdout.write(f" Progress: {i}/{count}") + + # Iterate over the queryset in chunks and save each instance. + i = 0 + for i, instance in enumerate( + queryset.only(plain_field_name, hash_field_name).iterator( + chunk_size + ), + start=1, + ): + # Get the plain value. + value = getattr(instance, plain_field_name) + + # Set the hash value using the Sha256Field's set method, which will + # normalize and hash the value before setting it on the instance. + Sha256Field.set(instance, value, hash_field_name) + + # Append the instance to the list of instances to be bulk updated. + instances.append(instance) + + # Print progress every chunk_size instances. + if len(instances) == chunk_size: + bulk_update(i) + + # Bulk update any remaining instances. + if len(instances) > 0: + bulk_update(count) + + self.stdout.write( + self.style.SUCCESS( + f" Successfully hashed {count} instances for {field_name}" + ) + ) diff --git a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py index ec2fe23a..9e92fee7 100644 --- a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py +++ b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py @@ -1,163 +1,10 @@ -import typing as t - -from django.apps.registry import Apps -from django.core.exceptions import FieldDoesNotExist from django.db import migrations -from django.db.models import CharField, Model, Q +from django.db.models import CharField from ...models.fields import EncryptedTextField, Sha256Field - -def set_field(model_name: str, field_name: str, qs_filter: Q | None = None): - def forwards_func(apps: Apps, schema_editor): - # Get the model class from the apps registry. - model_class: t.Type[Model] = apps.get_model( - app_label="user", model_name=model_name - ) - manager = model_class.objects # type: ignore[attr-defined] - - # Generate the plain field name and the Q object to filter instances - # where the plain field is null or empty. - plain_field_name = f"_{field_name}_plain" - plain_field_is_null_or_empty = Q( - **{f"{plain_field_name}__isnull": True} - ) | Q(**{f"{plain_field_name}": ""}) - - # Generate the encrypted and hash field names and check if they exist. - enc_field_name = f"_{field_name}_enc" - try: - model_class._meta.get_field(enc_field_name) - enc_field_exists = True - except FieldDoesNotExist: - enc_field_exists = False - hash_field_name = f"_{field_name}_hash" - try: - model_class._meta.get_field(hash_field_name) - hash_field_exists = True - except FieldDoesNotExist: - hash_field_exists = False - - # Update instances where the plain field is null or empty, setting the - # encrypted and hash fields to empty values (if they exist). - update_kwargs: dict[str, t.Any] = {} - if enc_field_exists: - update_kwargs[enc_field_name] = b"" - if hash_field_exists: - update_kwargs[hash_field_name] = "" - update_count = manager.filter(plain_field_is_null_or_empty).update( - **update_kwargs - ) - print( - f"Updated {update_count} instances of {model_name} for field " - f"{field_name} where {plain_field_name} is null or empty." - ) - - # If the hash field does not exist, we don't need to do anything else. - if not hash_field_exists: - return - - # Build a queryset of instances where the plain field is not null or - # empty, and apply any additional filtering provided by qs_filter. - queryset = manager.filter(~plain_field_is_null_or_empty) - if qs_filter is not None: - queryset = queryset.filter(qs_filter) - count = queryset.count() - if count == 0: - print( - f"No instances of {model_name} found for field {field_name} " - f"where {plain_field_name} is not null or empty." - ) - return - print( - f"Hashing {count} instances of {model_name} for field " - f"{field_name}..." - ) - - # Set the chunk size for bulk updates and initialize an empty list to - # hold instances to be updated. - chunk_size = 1000 - instances: list[Model] = [] - - # Helper function to bulk update instances in chunks. - def bulk_update(i: int): - nonlocal instances - if not instances: - return - manager.bulk_update( - instances, fields=[hash_field_name], batch_size=chunk_size - ) - instances = [] - print(f"({i}/{count})") - - # Iterate over the queryset in chunks and save each instance. - i = 0 - for i, instance in enumerate( - queryset.only(plain_field_name, hash_field_name).iterator( - chunk_size - ), - start=1, - ): - # Get the plain value. - value = getattr(instance, plain_field_name) - - # Set the hash value using the Sha256Field's set method, which will - # normalize and hash the value before setting it on the instance. - Sha256Field.set(instance, value, hash_field_name) - - # Append the instance to the list of instances to be bulk updated. - instances.append(instance) - - # Print progress every chunk_size instances. - if len(instances) == chunk_size: - bulk_update(i) - - # Bulk update any remaining instances. - if len(instances) > 0: - bulk_update(count) - - return migrations.RunPython(forwards_func) - - -def set_user_field(field_name: str): - return set_field( - model_name="user", field_name=field_name, qs_filter=Q(is_active=True) - ) - - -def set_class_field(field_name: str): - return set_field( - model_name="class", - field_name=field_name, - qs_filter=( - Q(is_active=True) - & Q(teacher__isnull=False) - & Q(teacher__school__isnull=False) - & Q(teacher__school__is_active=True) - ), - ) - - -def set_school_teacher_invitation_field(field_name: str): - return set_field( - model_name="schoolteacherinvitation", - field_name=field_name, - qs_filter=( - Q(is_active=True) - & Q(school__isnull=False) - & Q(school__is_active=True) - ), - ) - - -def set_school_field(field_name: str): - return set_field( - model_name="school", field_name=field_name, qs_filter=Q(is_active=True) - ) - - user_migrations = [ # Email - set_user_field(field_name="email"), migrations.AlterField( model_name="user", name="_email_enc", @@ -182,7 +29,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # First name - set_user_field(field_name="first_name"), migrations.AlterField( model_name="user", name="_first_name_enc", @@ -207,7 +53,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # Last name - set_user_field(field_name="last_name"), migrations.AlterField( model_name="user", name="_last_name_enc", @@ -220,7 +65,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # Username - set_user_field(field_name="username"), migrations.AlterField( model_name="user", name="_username_enc", @@ -249,7 +93,6 @@ def set_school_field(field_name: str): class_migrations = [ # Access code - set_class_field(field_name="access_code"), migrations.AlterField( model_name="class", name="_access_code_enc", @@ -280,7 +123,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # Name - set_class_field(field_name="name"), migrations.AlterField( model_name="class", name="_name_enc", @@ -308,7 +150,6 @@ def set_school_field(field_name: str): school_teacher_invitation_migrations = [ # Email - set_school_teacher_invitation_field(field_name="invited_teacher_email"), migrations.AlterField( model_name="schoolteacherinvitation", name="_invited_teacher_email_enc", @@ -321,9 +162,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # First name - set_school_teacher_invitation_field( - field_name="invited_teacher_first_name" - ), migrations.AlterField( model_name="schoolteacherinvitation", name="_invited_teacher_first_name_enc", @@ -336,7 +174,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # Last name - set_school_teacher_invitation_field(field_name="invited_teacher_last_name"), migrations.AlterField( model_name="schoolteacherinvitation", name="_invited_teacher_last_name_enc", @@ -349,7 +186,6 @@ def set_school_field(field_name: str): preserve_default=False, ), # Token - set_school_teacher_invitation_field(field_name="token"), migrations.AlterField( model_name="schoolteacherinvitation", name="_token_enc", @@ -378,7 +214,6 @@ def set_school_field(field_name: str): school_migrations = [ # Name - set_school_field(field_name="name"), migrations.AlterField( model_name="school", name="_name_enc", From 7fbbe9d53376cbc1c4f7174ad243c393c8ae96d5 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Tue, 19 May 2026 16:47:10 +0000 Subject: [PATCH 07/18] add threading --- .../management/commands/normalize_fields.py | 205 ++++++++++++++++-- 1 file changed, 182 insertions(+), 23 deletions(-) diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index afd31ab8..ba490630 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -1,8 +1,11 @@ import typing as t +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from threading import Lock from django.core.exceptions import FieldDoesNotExist from django.core.management.base import BaseCommand -from django.db.models import Model, Q +from django.db import close_old_connections +from django.db.models import Model, Q, QuerySet from ....models.fields import Sha256Field @@ -18,6 +21,28 @@ class Command(BaseCommand): help = "Normalize encrypted and hash fields for all user models" + def add_arguments(self, parser): + parser.add_argument( + "--chunk-size", + type=int, + default=1000, + help="The number of records to process in each batch.", + ) + parser.add_argument( + "--enable-threading", + action="store_true", + help=( + "Enable threaded processing where each worker handles a " + "chunk of rows." + ), + ) + parser.add_argument( + "--max-workers", + type=int, + default=4, + help="Maximum thread workers when --enable-threading is used.", + ) + # Define all models and their fields to process MODELS_TO_PROCESS = { "User": { @@ -53,19 +78,42 @@ class Command(BaseCommand): } def handle(self, *args, **options): + chunk_size: int = options["chunk_size"] + enable_threading: bool = options["enable_threading"] + max_workers: int = options["max_workers"] + + if chunk_size < 1: + raise ValueError("--chunk-size must be at least 1.") + + if max_workers < 1: + raise ValueError("--max-workers must be at least 1.") + + if max_workers > 8: + raise ValueError("--max-workers must be <= 8.") + self.stdout.write( self.style.SUCCESS("Starting to normalize encrypted fields...") ) self.stdout.write("") for model_name, config in self.MODELS_TO_PROCESS.items(): - for field_name in config["fields"]: + fields = t.cast(list[str], config["fields"]) + qs_filter = t.cast(Q | None, config.get("filter")) + + for field_name in fields: self.stdout.write( self.style.HTTP_INFO( f"Processing {model_name}.{field_name}..." ) ) - self.set_field(model_name, field_name, config.get("filter")) + self.set_field( + model_name=model_name, + field_name=field_name, + qs_filter=qs_filter, + chunk_size=chunk_size, + enable_threading=enable_threading, + max_workers=max_workers, + ) self.stdout.write("") self.stdout.write( @@ -73,7 +121,13 @@ def handle(self, *args, **options): ) def set_field( - self, model_name: str, field_name: str, qs_filter: Q | None = None + self, + model_name: str, + field_name: str, + qs_filter: Q | None = None, + chunk_size: int = 1000, + enable_threading: bool = False, + max_workers: int = 4, ): """Set encrypted and hash field values for a model field.""" from codeforlife.user import models @@ -154,23 +208,89 @@ def set_field( self.stdout.write(f" Hashing {count} instances...") - # Set the chunk size for bulk updates and initialize an empty list to - # hold instances to be updated. - chunk_size = 1000 + if enable_threading: + self._hash_queryset_threaded( + manager=manager, + queryset=queryset, + plain_field_name=plain_field_name, + hash_field_name=hash_field_name, + count=count, + chunk_size=chunk_size, + max_workers=max_workers, + ) + else: + self._hash_queryset( + manager=manager, + queryset=queryset, + plain_field_name=plain_field_name, + hash_field_name=hash_field_name, + count=count, + chunk_size=chunk_size, + ) + + self.stdout.write( + self.style.SUCCESS( + f" Successfully hashed {count} instances for {field_name}" + ) + ) + + def _iter_model_batches(self, models: QuerySet[Model], chunk_size: int): + batch: list[Model] = [] + for model in models.iterator(chunk_size): + batch.append(model) + if len(batch) >= chunk_size: + yield batch + batch = [] + + if batch: + yield batch + + def _hash_batch( + self, + manager, + batch: list[Model], + plain_field_name: str, + hash_field_name: str, + chunk_size: int, + ) -> int: + # Ensure this thread has a valid Django DB connection state. + close_old_connections() + + for instance in batch: + value = getattr(instance, plain_field_name) + Sha256Field.set(instance, value, hash_field_name) + + manager.bulk_update( + batch, + fields=[hash_field_name], + batch_size=chunk_size, + ) + close_old_connections() + return len(batch) + + def _hash_queryset( + self, + manager, + queryset, + plain_field_name: str, + hash_field_name: str, + count: int, + chunk_size: int, + ): instances: list[Model] = [] - # Helper function to bulk update instances in chunks. def bulk_update(i: int): nonlocal instances if not instances: return manager.bulk_update( - instances, fields=[hash_field_name], batch_size=chunk_size + instances, + fields=[hash_field_name], + batch_size=chunk_size, ) instances = [] self.stdout.write(f" Progress: {i}/{count}") - # Iterate over the queryset in chunks and save each instance. i = 0 for i, instance in enumerate( queryset.only(plain_field_name, hash_field_name).iterator( @@ -178,26 +298,65 @@ def bulk_update(i: int): ), start=1, ): - # Get the plain value. value = getattr(instance, plain_field_name) - - # Set the hash value using the Sha256Field's set method, which will - # normalize and hash the value before setting it on the instance. Sha256Field.set(instance, value, hash_field_name) - - # Append the instance to the list of instances to be bulk updated. instances.append(instance) - # Print progress every chunk_size instances. if len(instances) == chunk_size: bulk_update(i) - # Bulk update any remaining instances. if len(instances) > 0: bulk_update(count) - self.stdout.write( - self.style.SUCCESS( - f" Successfully hashed {count} instances for {field_name}" - ) - ) + def _hash_queryset_threaded( + self, + manager, + queryset, + plain_field_name: str, + hash_field_name: str, + count: int, + chunk_size: int, + max_workers: int, + ): + progress_lock = Lock() + processed_count = 0 + submitted_batches = 0 + max_pending_futures = max_workers * 2 + + def complete_one(future: Future[int]): + nonlocal processed_count + processed_batch_size = future.result() + with progress_lock: + processed_count += processed_batch_size + self.stdout.write(f" Progress: {processed_count}/{count}") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + pending_futures: set[Future[int]] = set() + + models = queryset.only(plain_field_name, hash_field_name) + for batch in self._iter_model_batches(models, chunk_size): + pending_futures.add( + executor.submit( + self._hash_batch, + manager, + batch, + plain_field_name, + hash_field_name, + chunk_size, + ) + ) + submitted_batches += 1 + + if len(pending_futures) >= max_pending_futures: + done, pending_futures = wait( + pending_futures, + return_when=FIRST_COMPLETED, + ) + for future in done: + complete_one(future) + + if submitted_batches == 0: + return + + for future in pending_futures: + complete_one(future) From 49f5281d31bfeb74fcf3fad52202e7e8bd12bdd2 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Wed, 20 May 2026 11:06:47 +0000 Subject: [PATCH 08/18] quick save --- codeforlife/models/fields/sha256.py | 14 +- .../management/commands/normalize_fields.py | 139 ++++++++++++++++-- 2 files changed, 134 insertions(+), 19 deletions(-) diff --git a/codeforlife/models/fields/sha256.py b/codeforlife/models/fields/sha256.py index eaf4f1d8..0d11a5ae 100644 --- a/codeforlife/models/fields/sha256.py +++ b/codeforlife/models/fields/sha256.py @@ -67,11 +67,15 @@ def hash(value: str): Returns: A hash of the value salted with the Django secret key. """ - return hmac.new( - key=settings.SECRET_KEY.encode("utf-8"), - msg=value.encode("utf-8"), - digestmod=sha256, - ).hexdigest() + return ( + "" + if value == "" + else hmac.new( + key=settings.SECRET_KEY.encode("utf-8"), + msg=value.encode("utf-8"), + digestmod=sha256, + ).hexdigest() + ) @classmethod def set(cls, instance, value, field_name, **kwargs): diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index ba490630..ea6388df 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -1,5 +1,6 @@ import typing as t from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from dataclasses import dataclass from threading import Lock from django.core.exceptions import FieldDoesNotExist @@ -10,6 +11,12 @@ from ....models.fields import Sha256Field +@dataclass +class HashUniquenessState: + used_hashes: set[str] + suffix_counters: dict[str, int] + + class Command(BaseCommand): """ Django management command to set encrypted and hash field values for all models. @@ -162,9 +169,12 @@ def set_field( hash_field_name = f"_{field_name}_hash" try: - model_class._meta.get_field(hash_field_name) + hash_field = t.cast( + Sha256Field, model_class._meta.get_field(hash_field_name) + ) hash_field_exists = True except FieldDoesNotExist: + hash_field = None hash_field_exists = False # If neither encrypted nor hash field exists, skip this field. @@ -200,6 +210,7 @@ def set_field( queryset = manager.filter(~plain_field_is_null_or_empty) if qs_filter is not None: queryset = queryset.filter(qs_filter) + queryset = queryset.order_by("pk") count = queryset.count() if count == 0: @@ -207,6 +218,17 @@ def set_field( return self.stdout.write(f" Hashing {count} instances...") + unique_hash_field = bool(t.cast(Sha256Field, hash_field).unique) + + state: HashUniquenessState | None = None + if unique_hash_field: + # Ensure we avoid collisions with hashes that already exist in rows + # outside the queryset currently being normalized. + state = self._build_hash_uniqueness_state( + manager=manager, + queryset=queryset, + hash_field_name=hash_field_name, + ) if enable_threading: self._hash_queryset_threaded( @@ -214,9 +236,11 @@ def set_field( queryset=queryset, plain_field_name=plain_field_name, hash_field_name=hash_field_name, + hash_field=t.cast(Sha256Field, hash_field), count=count, chunk_size=chunk_size, max_workers=max_workers, + state=state, ) else: self._hash_queryset( @@ -224,8 +248,10 @@ def set_field( queryset=queryset, plain_field_name=plain_field_name, hash_field_name=hash_field_name, + hash_field=t.cast(Sha256Field, hash_field), count=count, chunk_size=chunk_size, + state=state, ) self.stdout.write( @@ -249,35 +275,88 @@ def _hash_batch( self, manager, batch: list[Model], - plain_field_name: str, - hash_field_name: str, + update_fields: list[str], chunk_size: int, ) -> int: # Ensure this thread has a valid Django DB connection state. close_old_connections() - for instance in batch: - value = getattr(instance, plain_field_name) - Sha256Field.set(instance, value, hash_field_name) - manager.bulk_update( batch, - fields=[hash_field_name], + fields=update_fields, batch_size=chunk_size, ) close_old_connections() return len(batch) + def _build_hash_uniqueness_state( + self, + manager, + queryset, + hash_field_name: str, + ) -> HashUniquenessState: + queryset_ids = queryset.values("pk") + existing_hashes = set( + manager.exclude(pk__in=queryset_ids) + .exclude(**{f"{hash_field_name}__isnull": True}) + .exclude(**{hash_field_name: ""}) + .values_list(hash_field_name, flat=True) + ) + return HashUniquenessState( + used_hashes=existing_hashes, + suffix_counters={}, + ) + + def _assign_unique_plain_and_hash( + self, + instance: Model, + plain_field_name: str, + hash_field_name: str, + hash_field: Sha256Field, + state: HashUniquenessState | None, + ): + value = t.cast(str, getattr(instance, plain_field_name)) + normalized_base = hash_field.normalize(value) + + if state is None: + setattr(instance, plain_field_name, normalized_base) + setattr( + instance, hash_field_name, Sha256Field.hash(normalized_base) + ) + return + + suffix = state.suffix_counters.get(normalized_base, 0) + while True: + suffix += 1 + candidate_plain = ( + normalized_base + if suffix == 1 + else f"{normalized_base} {suffix}" + ) + candidate_hash = Sha256Field.hash( + hash_field.normalize(candidate_plain) + ) + if candidate_hash not in state.used_hashes: + break + + state.suffix_counters[normalized_base] = suffix + state.used_hashes.add(candidate_hash) + setattr(instance, plain_field_name, candidate_plain) + setattr(instance, hash_field_name, candidate_hash) + def _hash_queryset( self, manager, queryset, plain_field_name: str, hash_field_name: str, + hash_field: Sha256Field, count: int, chunk_size: int, + state: HashUniquenessState | None, ): instances: list[Model] = [] + update_fields = [plain_field_name, hash_field_name] def bulk_update(i: int): nonlocal instances @@ -285,7 +364,7 @@ def bulk_update(i: int): return manager.bulk_update( instances, - fields=[hash_field_name], + fields=update_fields, batch_size=chunk_size, ) instances = [] @@ -298,8 +377,13 @@ def bulk_update(i: int): ), start=1, ): - value = getattr(instance, plain_field_name) - Sha256Field.set(instance, value, hash_field_name) + self._assign_unique_plain_and_hash( + instance=instance, + plain_field_name=plain_field_name, + hash_field_name=hash_field_name, + hash_field=hash_field, + state=state, + ) instances.append(instance) if len(instances) == chunk_size: @@ -314,14 +398,17 @@ def _hash_queryset_threaded( queryset, plain_field_name: str, hash_field_name: str, + hash_field: Sha256Field, count: int, chunk_size: int, max_workers: int, + state: HashUniquenessState | None, ): progress_lock = Lock() processed_count = 0 submitted_batches = 0 max_pending_futures = max_workers * 2 + update_fields = [plain_field_name, hash_field_name] def complete_one(future: Future[int]): nonlocal processed_count @@ -334,17 +421,29 @@ def complete_one(future: Future[int]): pending_futures: set[Future[int]] = set() models = queryset.only(plain_field_name, hash_field_name) - for batch in self._iter_model_batches(models, chunk_size): + batch: list[Model] = [] + for instance in models.iterator(chunk_size): + self._assign_unique_plain_and_hash( + instance=instance, + plain_field_name=plain_field_name, + hash_field_name=hash_field_name, + hash_field=hash_field, + state=state, + ) + batch.append(instance) + if len(batch) < chunk_size: + continue + pending_futures.add( executor.submit( self._hash_batch, manager, batch, - plain_field_name, - hash_field_name, + update_fields, chunk_size, ) ) + batch = [] submitted_batches += 1 if len(pending_futures) >= max_pending_futures: @@ -355,6 +454,18 @@ def complete_one(future: Future[int]): for future in done: complete_one(future) + if batch: + pending_futures.add( + executor.submit( + self._hash_batch, + manager, + batch, + update_fields, + chunk_size, + ) + ) + submitted_batches += 1 + if submitted_batches == 0: return From 3bcf849ce8fabd5eaacc129aa073b1907869e502 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Wed, 20 May 2026 13:49:13 +0000 Subject: [PATCH 09/18] refactor --- codeforlife/models/fields/base_encrypted.py | 8 +- .../models/fields/base_encrypted_test.py | 5 + .../management/commands/normalize_fields.py | 322 ++++++++++-------- 3 files changed, 200 insertions(+), 135 deletions(-) diff --git a/codeforlife/models/fields/base_encrypted.py b/codeforlife/models/fields/base_encrypted.py index b95e698b..81a37fa7 100644 --- a/codeforlife/models/fields/base_encrypted.py +++ b/codeforlife/models/fields/base_encrypted.py @@ -65,8 +65,14 @@ def __init__( self, associated_data: str, normalize: Normalize[T] = lambda x: x, + unique: t.Literal[False] = False, **kwargs, ): + if unique: + raise ValidationError( + f"{self.__class__.__name__} does not support unique=True.", + code="unique_not_supported", + ) if not associated_data: raise ValidationError( "Associated data cannot be empty.", @@ -74,7 +80,7 @@ def __init__( ) self.associated_data = associated_data - super().__init__(normalize=normalize, **kwargs) + super().__init__(normalize=normalize, unique=unique, **kwargs) def deconstruct(self): name, path, args, kwargs = t.cast( diff --git a/codeforlife/models/fields/base_encrypted_test.py b/codeforlife/models/fields/base_encrypted_test.py index ac81e02c..05609ea9 100644 --- a/codeforlife/models/fields/base_encrypted_test.py +++ b/codeforlife/models/fields/base_encrypted_test.py @@ -164,6 +164,11 @@ def test_init__no_associated_data(self): with self.assert_raises_validation_error(code="no_associated_data"): BaseEncryptedField(associated_data="") + def test_init__unique_not_supported(self): + """Cannot create BaseEncryptedField with unique=True.""" + with self.assert_raises_validation_error(code="unique_not_supported"): + BaseEncryptedField(associated_data="test", unique=True) + def test_init(self): """BaseEncryptedField is constructed correctly.""" assert self.field.associated_data == self.field_associated_data diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index ea6388df..7374f490 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -6,9 +6,15 @@ from django.core.exceptions import FieldDoesNotExist from django.core.management.base import BaseCommand from django.db import close_old_connections -from django.db.models import Model, Q, QuerySet +from django.db.models import Manager, Model, Q, QuerySet -from ....models.fields import Sha256Field +from ....models.fields import BaseEncryptedField, Sha256Field +from ....pprint import PrettyPrinter + +LogFn: t.TypeAlias = t.Callable[[str], None] +ModelClass: t.TypeAlias = t.Type[Model] +ModelManager: t.TypeAlias = Manager[Model] +ModelQuerySet: t.TypeAlias = QuerySet[Model] @dataclass @@ -19,11 +25,8 @@ class HashUniquenessState: class Command(BaseCommand): """ - Django management command to set encrypted and hash field values for all models. - - This command replicates the logic from migration 0005_client_side_encryption_part_3 - and automatically processes all models in the codeforlife.user app that have - encrypted fields. + Django management command to set encrypted and hash field values for all + models. """ help = "Normalize encrypted and hash fields for all user models" @@ -98,126 +101,174 @@ def handle(self, *args, **options): if max_workers > 8: raise ValueError("--max-workers must be <= 8.") - self.stdout.write( - self.style.SUCCESS("Starting to normalize encrypted fields...") - ) - self.stdout.write("") - - for model_name, config in self.MODELS_TO_PROCESS.items(): - fields = t.cast(list[str], config["fields"]) - qs_filter = t.cast(Q | None, config.get("filter")) - - for field_name in fields: - self.stdout.write( - self.style.HTTP_INFO( - f"Processing {model_name}.{field_name}..." - ) - ) - self.set_field( - model_name=model_name, - field_name=field_name, - qs_filter=qs_filter, - chunk_size=chunk_size, - enable_threading=enable_threading, - max_workers=max_workers, - ) - self.stdout.write("") + pprint = PrettyPrinter(write=self.stderr.write, name=self.__module__) + + with pprint.process("Normalizing encrypted fields") as root_pprint: + for model_name, config in self.MODELS_TO_PROCESS.items(): + fields = t.cast(list[str], config["fields"]) + qs_filter = t.cast(Q | None, config.get("filter")) + + with root_pprint.process( + f"Model: {root_pprint.notice.apply(model_name)}" + ) as model_pprint: + for field_name in fields: + with model_pprint.process( + "Field: " + model_pprint.notice.apply(field_name) + ) as field_pprint: + self._normalize_field_for_model( + model_name=model_name, + field_name=field_name, + qs_filter=qs_filter, + chunk_size=chunk_size, + enable_threading=enable_threading, + max_workers=max_workers, + log=field_pprint, + ) self.stdout.write( - self.style.SUCCESS("Successfully normalized all encrypted fields!") + self.style.SUCCESS("Successfully normalized all configured fields!") ) - def set_field( - self, - model_name: str, - field_name: str, - qs_filter: Q | None = None, - chunk_size: int = 1000, - enable_threading: bool = False, - max_workers: int = 4, - ): - """Set encrypted and hash field values for a model field.""" + def _get_model_class(self, model_name: str) -> ModelClass | None: from codeforlife.user import models - # Get the model class from the apps registry. try: - model_class: t.Type[Model] = getattr(models, model_name) + return t.cast(ModelClass, getattr(models, model_name)) except AttributeError: self.stderr.write( self.style.ERROR( f"Model '{model_name}' not found in codeforlife.user.models" ) ) - return - - manager = model_class.objects # type: ignore[attr-defined] + return None - # Generate the plain field name and the Q object to filter instances - # where the plain field is null or empty. + def _build_plain_field_filter(self, field_name: str) -> tuple[str, Q]: plain_field_name = f"_{field_name}_plain" plain_field_is_null_or_empty = Q( **{f"{plain_field_name}__isnull": True} ) | Q(**{f"{plain_field_name}": ""}) + return plain_field_name, plain_field_is_null_or_empty - # Generate the encrypted and hash field names and check if they exist. + def _discover_target_fields( + self, + model_class: ModelClass, + field_name: str, + ) -> tuple[BaseEncryptedField[t.Any] | None, Sha256Field | None]: + # Discover related encrypted/hash fields for this plaintext field. enc_field_name = f"_{field_name}_enc" try: - model_class._meta.get_field(enc_field_name) - enc_field_exists = True + enc_field = model_class._meta.get_field(enc_field_name) + assert isinstance(enc_field, BaseEncryptedField), ( + f"Expected '{model_class.__name__}.{enc_field_name}' to be a " + f"BaseEncryptedField, got {type(enc_field).__name__}." + ) + enc_field = t.cast(BaseEncryptedField[t.Any], enc_field) except FieldDoesNotExist: - enc_field_exists = False + enc_field = None hash_field_name = f"_{field_name}_hash" try: - hash_field = t.cast( - Sha256Field, model_class._meta.get_field(hash_field_name) + hash_field = model_class._meta.get_field(hash_field_name) + assert isinstance(hash_field, Sha256Field), ( + f"Expected '{model_class.__name__}.{hash_field_name}' to be " + f"a Sha256Field, got {type(hash_field).__name__}." ) - hash_field_exists = True + hash_field = t.cast(Sha256Field, hash_field) except FieldDoesNotExist: hash_field = None - hash_field_exists = False - # If neither encrypted nor hash field exists, skip this field. - if not enc_field_exists and not hash_field_exists: - self.stdout.write( - self.style.WARNING( - f"Skipping {model_name}.{field_name}: no encrypted or hash field found" - ) - ) - return + return (enc_field, hash_field) - # Update instances where the plain field is null or empty, setting the - # encrypted and hash fields to empty values (if they exist). + def _reset_missing_plain_values( + self, + model_manager: ModelManager, + plain_field_is_null_or_empty: Q, + enc_field: BaseEncryptedField[t.Any] | None, + hash_field: Sha256Field | None, + log: LogFn, + plain_field_name: str, + ) -> None: update_kwargs: dict[str, t.Any] = {} - if enc_field_exists: - update_kwargs[enc_field_name] = b"" - if hash_field_exists: - update_kwargs[hash_field_name] = "" + if enc_field is not None: + update_kwargs[enc_field.name] = b"" + if hash_field is not None: + update_kwargs[hash_field.name] = "" + + update_count = model_manager.filter( + plain_field_is_null_or_empty + ).update(**update_kwargs) + log(f"Updated {update_count} records with empty {plain_field_name}.") + + def _build_hash_queryset( + self, + model_manager: ModelManager, + plain_field_is_null_or_empty: Q, + qs_filter: Q | None, + ) -> ModelQuerySet: + queryset = model_manager.filter(~plain_field_is_null_or_empty) + if qs_filter is not None: + queryset = queryset.filter(qs_filter) + return t.cast(ModelQuerySet, queryset.order_by("pk")) - update_count = manager.filter(plain_field_is_null_or_empty).update( - **update_kwargs + def _normalize_field_for_model( + self, + model_name: str, + field_name: str, + qs_filter: Q | None = None, + chunk_size: int = 1000, + enable_threading: bool = False, + max_workers: int = 4, + log: LogFn | None = None, + ) -> None: + """Set encrypted and hash field values for a model field.""" + log = log or self.stdout.write + + model_class = self._get_model_class(model_name) + if model_class is None: + return + + model_manager = t.cast(ModelManager, model_class.objects) # type: ignore[attr-defined] + + plain_field_name, plain_field_is_null_or_empty = ( + self._build_plain_field_filter(field_name) ) - self.stdout.write( - f" Updated {update_count} instances with null/empty {plain_field_name}" + enc_field, hash_field = self._discover_target_fields( + model_class, field_name + ) + + # If neither encrypted nor hash field exists, skip this field. + if enc_field is None and hash_field is None: + log( + f"Skipping {model_name}.{field_name}: no encrypted or hash field found." + ) + return + + self._reset_missing_plain_values( + model_manager=model_manager, + plain_field_is_null_or_empty=plain_field_is_null_or_empty, + enc_field=enc_field, + hash_field=hash_field, + log=log, + plain_field_name=plain_field_name, ) # If the hash field does not exist, we don't need to do anything else. - if not hash_field_exists: + if hash_field is None: + log("No hash field found, skipping hash normalization.") return - # Build a queryset of instances where the plain field is not null or - # empty, and apply any additional filtering provided by qs_filter. - queryset = manager.filter(~plain_field_is_null_or_empty) - if qs_filter is not None: - queryset = queryset.filter(qs_filter) - queryset = queryset.order_by("pk") + queryset = self._build_hash_queryset( + model_manager=model_manager, + plain_field_is_null_or_empty=plain_field_is_null_or_empty, + qs_filter=qs_filter, + ) count = queryset.count() if count == 0: - self.stdout.write(f" No instances to hash for {plain_field_name}") + log(f"No records to hash for {plain_field_name}.") return - self.stdout.write(f" Hashing {count} instances...") + log(f"Hashing {count} records...") unique_hash_field = bool(t.cast(Sha256Field, hash_field).unique) state: HashUniquenessState | None = None @@ -225,44 +276,44 @@ def set_field( # Ensure we avoid collisions with hashes that already exist in rows # outside the queryset currently being normalized. state = self._build_hash_uniqueness_state( - manager=manager, - queryset=queryset, - hash_field_name=hash_field_name, + model_manager=model_manager, + model_queryset=queryset, + hash_field=t.cast(Sha256Field, hash_field), ) if enable_threading: - self._hash_queryset_threaded( - manager=manager, - queryset=queryset, + self._normalize_and_hash_queryset_threaded( + model_manager=model_manager, + model_queryset=queryset, plain_field_name=plain_field_name, - hash_field_name=hash_field_name, hash_field=t.cast(Sha256Field, hash_field), count=count, chunk_size=chunk_size, max_workers=max_workers, state=state, + log=log, ) else: - self._hash_queryset( - manager=manager, - queryset=queryset, + self._normalize_and_hash_queryset_sequential( + model_manager=model_manager, + model_queryset=queryset, plain_field_name=plain_field_name, - hash_field_name=hash_field_name, hash_field=t.cast(Sha256Field, hash_field), count=count, chunk_size=chunk_size, state=state, + log=log, ) - self.stdout.write( - self.style.SUCCESS( - f" Successfully hashed {count} instances for {field_name}" - ) - ) + log(f"Completed hashing for {field_name}: {count} records.") - def _iter_model_batches(self, models: QuerySet[Model], chunk_size: int): + def _iterate_queryset_batches( + self, + model_queryset: ModelQuerySet, + chunk_size: int, + ): batch: list[Model] = [] - for model in models.iterator(chunk_size): + for model in model_queryset.iterator(chunk_size): batch.append(model) if len(batch) >= chunk_size: yield batch @@ -271,9 +322,9 @@ def _iter_model_batches(self, models: QuerySet[Model], chunk_size: int): if batch: yield batch - def _hash_batch( + def _bulk_update_batch( self, - manager, + model_manager: ModelManager, batch: list[Model], update_fields: list[str], chunk_size: int, @@ -281,7 +332,7 @@ def _hash_batch( # Ensure this thread has a valid Django DB connection state. close_old_connections() - manager.bulk_update( + model_manager.bulk_update( batch, fields=update_fields, batch_size=chunk_size, @@ -291,13 +342,14 @@ def _hash_batch( def _build_hash_uniqueness_state( self, - manager, - queryset, - hash_field_name: str, + model_manager: ModelManager, + model_queryset: ModelQuerySet, + hash_field: Sha256Field, ) -> HashUniquenessState: - queryset_ids = queryset.values("pk") + hash_field_name = hash_field.name + queryset_ids = model_queryset.values("pk") existing_hashes = set( - manager.exclude(pk__in=queryset_ids) + model_manager.exclude(pk__in=queryset_ids) .exclude(**{f"{hash_field_name}__isnull": True}) .exclude(**{hash_field_name: ""}) .values_list(hash_field_name, flat=True) @@ -311,10 +363,10 @@ def _assign_unique_plain_and_hash( self, instance: Model, plain_field_name: str, - hash_field_name: str, hash_field: Sha256Field, state: HashUniquenessState | None, - ): + ) -> None: + hash_field_name = hash_field.name value = t.cast(str, getattr(instance, plain_field_name)) normalized_base = hash_field.normalize(value) @@ -344,35 +396,36 @@ def _assign_unique_plain_and_hash( setattr(instance, plain_field_name, candidate_plain) setattr(instance, hash_field_name, candidate_hash) - def _hash_queryset( + def _normalize_and_hash_queryset_sequential( self, - manager, - queryset, + model_manager: ModelManager, + model_queryset: ModelQuerySet, plain_field_name: str, - hash_field_name: str, hash_field: Sha256Field, count: int, chunk_size: int, state: HashUniquenessState | None, - ): + log: LogFn, + ) -> None: instances: list[Model] = [] + hash_field_name = hash_field.name update_fields = [plain_field_name, hash_field_name] def bulk_update(i: int): nonlocal instances if not instances: return - manager.bulk_update( + model_manager.bulk_update( instances, fields=update_fields, batch_size=chunk_size, ) instances = [] - self.stdout.write(f" Progress: {i}/{count}") + log(f"Progress: {i}/{count}") i = 0 for i, instance in enumerate( - queryset.only(plain_field_name, hash_field_name).iterator( + model_queryset.only(plain_field_name, hash_field_name).iterator( chunk_size ), start=1, @@ -380,7 +433,6 @@ def bulk_update(i: int): self._assign_unique_plain_and_hash( instance=instance, plain_field_name=plain_field_name, - hash_field_name=hash_field_name, hash_field=hash_field, state=state, ) @@ -392,22 +444,23 @@ def bulk_update(i: int): if len(instances) > 0: bulk_update(count) - def _hash_queryset_threaded( + def _normalize_and_hash_queryset_threaded( self, - manager, - queryset, + model_manager: ModelManager, + model_queryset: ModelQuerySet, plain_field_name: str, - hash_field_name: str, hash_field: Sha256Field, count: int, chunk_size: int, max_workers: int, state: HashUniquenessState | None, - ): + log: LogFn, + ) -> None: progress_lock = Lock() processed_count = 0 submitted_batches = 0 max_pending_futures = max_workers * 2 + hash_field_name = hash_field.name update_fields = [plain_field_name, hash_field_name] def complete_one(future: Future[int]): @@ -415,18 +468,19 @@ def complete_one(future: Future[int]): processed_batch_size = future.result() with progress_lock: processed_count += processed_batch_size - self.stdout.write(f" Progress: {processed_count}/{count}") + log(f"Progress: {processed_count}/{count}") with ThreadPoolExecutor(max_workers=max_workers) as executor: pending_futures: set[Future[int]] = set() - models = queryset.only(plain_field_name, hash_field_name) + ordered_queryset = model_queryset.only( + plain_field_name, hash_field_name + ) batch: list[Model] = [] - for instance in models.iterator(chunk_size): + for instance in ordered_queryset.iterator(chunk_size): self._assign_unique_plain_and_hash( instance=instance, plain_field_name=plain_field_name, - hash_field_name=hash_field_name, hash_field=hash_field, state=state, ) @@ -436,8 +490,8 @@ def complete_one(future: Future[int]): pending_futures.add( executor.submit( - self._hash_batch, - manager, + self._bulk_update_batch, + model_manager, batch, update_fields, chunk_size, @@ -457,8 +511,8 @@ def complete_one(future: Future[int]): if batch: pending_futures.add( executor.submit( - self._hash_batch, - manager, + self._bulk_update_batch, + model_manager, batch, update_fields, chunk_size, From c6a1b41d33c2a677144aaabcf02008cb397b53cc Mon Sep 17 00:00:00 2001 From: SKairinos Date: Wed, 20 May 2026 15:09:21 +0000 Subject: [PATCH 10/18] fixes --- .../management/commands/normalize_fields.py | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index 7374f490..35c15482 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -1,3 +1,8 @@ +""" +© Ocado Group +Created on 20/05/2026 at 15:44:33(+01:00). +""" + import typing as t from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait from dataclasses import dataclass @@ -19,6 +24,11 @@ @dataclass class HashUniquenessState: + """ + State to track used hashes and suffixes for ensuring uniqueness when + normalizing hash fields. + """ + used_hashes: set[str] suffix_counters: dict[str, int] @@ -52,6 +62,11 @@ def add_arguments(self, parser): default=4, help="Maximum thread workers when --enable-threading is used.", ) + parser.add_argument( + "--disable-styles", + action="store_true", + help="Disable styled output.", + ) # Define all models and their fields to process MODELS_TO_PROCESS = { @@ -91,6 +106,7 @@ def handle(self, *args, **options): chunk_size: int = options["chunk_size"] enable_threading: bool = options["enable_threading"] max_workers: int = options["max_workers"] + disable_styles: bool = options["disable_styles"] if chunk_size < 1: raise ValueError("--chunk-size must be at least 1.") @@ -101,7 +117,11 @@ def handle(self, *args, **options): if max_workers > 8: raise ValueError("--max-workers must be <= 8.") - pprint = PrettyPrinter(write=self.stderr.write, name=self.__module__) + pprint = PrettyPrinter( + write=self.stderr.write, + name=self.__module__, + disable_styles=disable_styles, + ) with pprint.process("Normalizing encrypted fields") as root_pprint: for model_name, config in self.MODELS_TO_PROCESS.items(): @@ -227,7 +247,9 @@ def _normalize_field_for_model( if model_class is None: return - model_manager = t.cast(ModelManager, model_class.objects) # type: ignore[attr-defined] + model_manager = t.cast( + ModelManager, model_class.objects # type: ignore[attr-defined] + ) plain_field_name, plain_field_is_null_or_empty = ( self._build_plain_field_filter(field_name) @@ -239,7 +261,8 @@ def _normalize_field_for_model( # If neither encrypted nor hash field exists, skip this field. if enc_field is None and hash_field is None: log( - f"Skipping {model_name}.{field_name}: no encrypted or hash field found." + f"Skipping {model_name}.{field_name}: no encrypted or hash" + " field found." ) return @@ -307,21 +330,6 @@ def _normalize_field_for_model( log(f"Completed hashing for {field_name}: {count} records.") - def _iterate_queryset_batches( - self, - model_queryset: ModelQuerySet, - chunk_size: int, - ): - batch: list[Model] = [] - for model in model_queryset.iterator(chunk_size): - batch.append(model) - if len(batch) >= chunk_size: - yield batch - batch = [] - - if batch: - yield batch - def _bulk_update_batch( self, model_manager: ModelManager, @@ -371,7 +379,6 @@ def _assign_unique_plain_and_hash( normalized_base = hash_field.normalize(value) if state is None: - setattr(instance, plain_field_name, normalized_base) setattr( instance, hash_field_name, Sha256Field.hash(normalized_base) ) @@ -393,7 +400,6 @@ def _assign_unique_plain_and_hash( state.suffix_counters[normalized_base] = suffix state.used_hashes.add(candidate_hash) - setattr(instance, plain_field_name, candidate_plain) setattr(instance, hash_field_name, candidate_hash) def _normalize_and_hash_queryset_sequential( @@ -409,7 +415,7 @@ def _normalize_and_hash_queryset_sequential( ) -> None: instances: list[Model] = [] hash_field_name = hash_field.name - update_fields = [plain_field_name, hash_field_name] + update_fields = [hash_field_name] def bulk_update(i: int): nonlocal instances @@ -461,7 +467,7 @@ def _normalize_and_hash_queryset_threaded( submitted_batches = 0 max_pending_futures = max_workers * 2 hash_field_name = hash_field.name - update_fields = [plain_field_name, hash_field_name] + update_fields = [hash_field_name] def complete_one(future: Future[int]): nonlocal processed_count From c29c03cab6ebdb4438818123843919d7e59da18b Mon Sep 17 00:00:00 2001 From: SKairinos Date: Wed, 20 May 2026 15:24:46 +0000 Subject: [PATCH 11/18] fixes --- Pipfile | 2 +- Pipfile.lock | 306 +++++++++++++++++++++++++-------------------------- 2 files changed, 154 insertions(+), 154 deletions(-) diff --git a/Pipfile b/Pipfile index 4e136cf5..4a9b4c5c 100644 --- a/Pipfile +++ b/Pipfile @@ -15,7 +15,7 @@ pyotp = "==2.9.0" psycopg2-binary = "==2.9.9" redis = {version = "==5.2.1", extras = ["hiredis"]} regex = "==2024.11.6" -requests = "==2.33.1" +requests = "==2.34.2" pyjwt = "==2.12.1" psutil = "==7.0.0" google-auth = "==2.48.0" diff --git a/Pipfile.lock b/Pipfile.lock index 75ff7d46..e424295e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "38ef0b0711d8176576bd1db76f82514c10cfd1fb0c84f1c1f779513ac21724a0" + "sha256": "9015568e47abbfc40dacbeba12151d641be8d9164dfdf2960adb1d4075c5bf71" }, "pipfile-spec": 6, "requires": { @@ -50,11 +50,11 @@ }, "certifi": { "hashes": [ - "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", - "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580" + "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", + "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d" ], "markers": "python_version >= '3.7'", - "version": "==2026.4.22" + "version": "==2026.5.20" }, "cffi": { "hashes": [ @@ -390,11 +390,11 @@ }, "django-formtools": { "hashes": [ - "sha256:47cb34552c6efca088863d693284d04fc36eaaf350eb21e1a1d935e0df523c93", - "sha256:bce9b64eda52cc1eef6961cc649cf75aacd1a707c2fff08d6c3efcbc8e7e761a" + "sha256:2848516dc66a7acde1cb595d225b30c1b837528948cad32bf0faafc6d8cc7af8", + "sha256:603ebc69a4b76b9bf38534f6e0245face007c7ed31bdfa073f8bf45cff5d7122" ], "markers": "python_version >= '3.8'", - "version": "==2.5.1" + "version": "==2.6.1" }, "django-import-export": { "hashes": [ @@ -732,11 +732,11 @@ }, "idna": { "hashes": [ - "sha256:466d810d7a2cc1022bea9b037c39728d51ae7dad40d480fc9b7d7ecf98ba8ee3", - "sha256:e677eaf072e290f7b725f9acf0b3a2bd55f9fd6f7c70abe5f0e34823d0accf69" + "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", + "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc" ], "markers": "python_version >= '3.8'", - "version": "==3.14" + "version": "==3.15" }, "libsass": { "hashes": [ @@ -762,81 +762,81 @@ }, "numpy": { "hashes": [ - "sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed", - "sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50", - "sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959", - "sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827", - "sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd", - "sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233", - "sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc", - "sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b", - "sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7", - "sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e", - "sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a", - "sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d", - "sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3", - "sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e", - "sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb", - "sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a", - "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", - "sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e", - "sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113", - "sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103", - "sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93", - "sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af", - "sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5", - "sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7", - "sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392", - "sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c", - "sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4", - "sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40", - "sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf", - "sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44", - "sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b", - "sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5", - "sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e", - "sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74", - "sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0", - "sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e", - "sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec", - "sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015", - "sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d", - "sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d", - "sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842", - "sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150", - "sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8", - "sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a", - "sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed", - "sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f", - "sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008", - "sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e", - "sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0", - "sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e", - "sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f", - "sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a", - "sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40", - "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", - "sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83", - "sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d", - "sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c", - "sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871", - "sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502", - "sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252", - "sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8", - "sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115", - "sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f", - "sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e", - "sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d", - "sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0", - "sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119", - "sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e", - "sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db", - "sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121", - "sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d", - "sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e" + "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", + "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", + "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", + "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", + "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", + "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", + "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", + "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", + "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", + "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", + "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", + "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", + "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", + "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", + "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", + "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", + "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", + "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", + "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", + "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", + "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", + "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", + "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", + "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", + "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", + "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", + "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", + "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", + "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", + "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", + "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", + "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", + "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", + "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", + "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", + "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", + "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", + "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", + "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", + "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", + "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", + "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", + "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", + "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", + "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", + "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", + "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", + "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", + "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", + "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", + "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", + "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", + "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", + "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", + "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", + "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", + "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", + "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", + "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", + "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", + "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", + "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", + "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", + "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", + "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", + "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", + "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", + "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", + "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", + "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", + "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", + "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20" ], "markers": "python_version >= '3.11'", - "version": "==2.4.4" + "version": "==2.4.6" }, "packaging": { "hashes": [ @@ -848,57 +848,57 @@ }, "pandas": { "hashes": [ - "sha256:01f31a546acd5574ef77fe199bc90b55527c225c20ccda6601cf6b0fd5ed597c", - "sha256:0555c5882688a39317179ab4a0ed41d3ebc8812ab14c69364bbee8fb7a3f6288", - "sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991", - "sha256:08504503f7101300107ecdc8df73658e4347586db5cfdadabc1592e9d7e7a0fd", - "sha256:0f48afd9bb13300ffb5a3316973324c787054ba6665cda0da3fbd67f451995db", - "sha256:140f0cffb1fa2524e874dde5b477d9defe10780d8e9e220d259b2c0874c89d9d", - "sha256:232a70ebb568c0c4d2db4584f338c1577d81e3af63292208d615907b698a0f18", - "sha256:32cc41f310ebd4a296d93515fcac312216adfedb1894e879303987b8f1e2b97d", - "sha256:339dda302bd8369dedeae979cb750e484d549b563c3f54f3922cb8ff4978c5eb", - "sha256:4544c7a54920de8eeacaa1466a6b7268ecfbc9bc64ab4dbb89c6bbe94d5e0660", - "sha256:4d888a5c678a419a5bb41a2a93818e8ed9fd3172246555c0b37b7cc27027effd", - "sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482", - "sha256:57a07209bebcbcf768d2d13c9b78b852f9a15978dac41b9e6421a81ad4cdd276", - "sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677", - "sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e", - "sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84", - "sha256:60a80bb4feacbef5e1447a3f82c33209c8b7e07f28d805cfd1fb951e5cb443aa", - "sha256:61c2fd96d72b983a9891b2598f286befd4ad262161a609c92dc1652544b46b76", - "sha256:63d141b56ef686f7f0d714cfb8de4e320475b86bf4b620aa0b7da89af8cbdbbb", - "sha256:6c4d8458b97a35717b62469a4ea0e85abd5ed8687277f5ccfc67f8a5126f8c53", - "sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9", - "sha256:734be7551687c00fbd760dc0522ed974f82ad230d4a10f54bf51b80d44a08702", - "sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0", - "sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1", - "sha256:970762605cff1ca0d3f71ed4f3a769ea8f85fc8e6348f6e110b8fea7e6eb5a14", - "sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39", - "sha256:a4785e1d6547d8427c5208b748ae2efb64659a21bd82bf440d4262d02bfa02a4", - "sha256:a727a73cbdba2f7458dc82449e2315899d5140b449015d822f515749a46cbbe0", - "sha256:ae37e833ff4fed0ba352f6bdd8b73ba3ab3256a85e54edfd1ab51ae40cca0af8", - "sha256:aff4e6f4d722e0652707d7bcb190c445fe58428500c6d16005b02401764b1b3d", - "sha256:b35d14bb5d8285d9494fe93815a9e9307c0876e10f1e8e89ac5b88f728ec8dcf", - "sha256:b444dc64c079e84df91baa8bf613d58405645461cabca929d9178f2cd392398d", - "sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172", - "sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3", - "sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d", - "sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d", - "sha256:c934008c733b8bbea273ea308b73b3156f0181e5b72960790b09c18a2794fe1e", - "sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7", - "sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668", - "sha256:db0dbfd2a6cdf3770aa60464d50333d8f3d9165b2f2671bcc299b72de5a6677b", - "sha256:dbbd4aa20ca51e63b53bbde6a0fa4254b1aaabb74d2f542df7a7959feb1d760c", - "sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235", - "sha256:deeca1b5a931fdf0c2212c8a659ade6d3b1edc21f0914ce71ef24456ca7a6535", - "sha256:ed72cb3f45190874eb579c64fa92d9df74e98fd63e2be7f62bce5ace0ade61df", - "sha256:ef8b27695c3d3dc78403c9a7d5e59a62d5464a7e1123b4e0042763f7104dc74f", - "sha256:f12b1a9e332c01e09510586f8ca9b108fd631fd656af82e452d7315ef6df5f9f", - "sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043", - "sha256:f8d68083e49e16b84734eb1a4dcae4259a75c90fb6e2251ab9a00b61120c06ab" + "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", + "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2", + "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", + "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", + "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", + "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824", + "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", + "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", + "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", + "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", + "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", + "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", + "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", + "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", + "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98", + "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", + "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", + "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639", + "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", + "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", + "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a", + "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", + "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea", + "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", + "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", + "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", + "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", + "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", + "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27", + "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", + "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", + "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", + "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", + "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938", + "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", + "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", + "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", + "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", + "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", + "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", + "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", + "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", + "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", + "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", + "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", + "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", + "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", + "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09" ], "markers": "python_version >= '3.11'", - "version": "==3.0.2" + "version": "==3.0.3" }, "pgeocode": { "hashes": [ @@ -1201,12 +1201,12 @@ }, "requests": { "hashes": [ - "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", - "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a" + "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", + "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==2.33.1" + "version": "==2.34.2" }, "rsa": { "hashes": [ @@ -1351,11 +1351,11 @@ }, "certifi": { "hashes": [ - "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", - "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580" + "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", + "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d" ], "markers": "python_version >= '3.7'", - "version": "==2026.4.22" + "version": "==2026.5.20" }, "charset-normalizer": { "hashes": [ @@ -1494,11 +1494,11 @@ }, "click": { "hashes": [ - "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", - "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613" + "sha256:40c50b7c6c6adac2823d411041ec84f3f103f1b280d5e9ce0d7f998995832f81", + "sha256:638f1338fe1235c8f4e008e4a8a254fb5c5fbdcbb40ece3c9142ebb78e792973" ], "markers": "python_version >= '3.10'", - "version": "==8.3.3" + "version": "==8.4.0" }, "coverage": { "extras": [ @@ -1690,11 +1690,11 @@ }, "idna": { "hashes": [ - "sha256:466d810d7a2cc1022bea9b037c39728d51ae7dad40d480fc9b7d7ecf98ba8ee3", - "sha256:e677eaf072e290f7b725f9acf0b3a2bd55f9fd6f7c70abe5f0e34823d0accf69" + "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", + "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc" ], "markers": "python_version >= '3.8'", - "version": "==3.14" + "version": "==3.15" }, "iniconfig": { "hashes": [ @@ -1909,12 +1909,12 @@ }, "requests": { "hashes": [ - "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", - "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a" + "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", + "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==2.33.1" + "version": "==2.34.2" }, "sqlparse": { "hashes": [ @@ -1952,11 +1952,11 @@ }, "types-pyyaml": { "hashes": [ - "sha256:09c1f1cb65a6eebea1e2e51ccf4918b8288e152909609a35cdb0d805efd125ad", - "sha256:3492eb9ba4d9d833473214c4d5736cccf5f37d93f5854059721e1c84f785309d" + "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", + "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466" ], "markers": "python_version >= '3.10'", - "version": "==6.0.12.20260510" + "version": "==6.0.12.20260518" }, "types-regex": { "hashes": [ @@ -1969,11 +1969,11 @@ }, "types-requests": { "hashes": [ - "sha256:81b2ae5f0d20967714a6aa5ef9284c05570d7cb06b7de8f2a77b918b63ddd411", - "sha256:fa01459cca184229713df03709db46a905325906d27e042cd4fd7ea3d15d3400" + "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0", + "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e" ], "markers": "python_version >= '3.10'", - "version": "==2.33.0.20260508" + "version": "==2.33.0.20260518" }, "typing-extensions": { "hashes": [ From bcb42f71fe4cbc3ce03e6dc1f6b72042df0da18f Mon Sep 17 00:00:00 2001 From: SKairinos Date: Wed, 20 May 2026 15:31:51 +0000 Subject: [PATCH 12/18] fic --- codeforlife/models/fields/base_encrypted_test.py | 5 ++++- codeforlife/user/management/commands/normalize_fields.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/codeforlife/models/fields/base_encrypted_test.py b/codeforlife/models/fields/base_encrypted_test.py index 05609ea9..1a4a665c 100644 --- a/codeforlife/models/fields/base_encrypted_test.py +++ b/codeforlife/models/fields/base_encrypted_test.py @@ -167,7 +167,10 @@ def test_init__no_associated_data(self): def test_init__unique_not_supported(self): """Cannot create BaseEncryptedField with unique=True.""" with self.assert_raises_validation_error(code="unique_not_supported"): - BaseEncryptedField(associated_data="test", unique=True) + BaseEncryptedField( + associated_data="test", + unique=True, # type: ignore[arg-type] + ) def test_init(self): """BaseEncryptedField is constructed correctly.""" diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index 35c15482..ba4fe1dd 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -21,6 +21,8 @@ ModelManager: t.TypeAlias = Manager[Model] ModelQuerySet: t.TypeAlias = QuerySet[Model] +# pylint: disable=duplicate-code,too-many-locals,import-outside-toplevel,too-many-positional-arguments,too-many-arguments + @dataclass class HashUniquenessState: From 6e4b1758c5dd44a3f9bfb13e302c7b28a657d126 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Thu, 21 May 2026 09:51:02 +0000 Subject: [PATCH 13/18] normalize is optional --- codeforlife/models/fields/base_encrypted.py | 4 +-- codeforlife/models/fields/normalized.py | 10 ++++--- codeforlife/models/fields/sha256.py | 33 +++++++++++++-------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/codeforlife/models/fields/base_encrypted.py b/codeforlife/models/fields/base_encrypted.py index 81a37fa7..99452947 100644 --- a/codeforlife/models/fields/base_encrypted.py +++ b/codeforlife/models/fields/base_encrypted.py @@ -64,7 +64,7 @@ class BaseEncryptedField( def __init__( self, associated_data: str, - normalize: Normalize[T] = lambda x: x, + normalize: None | Normalize[T] = None, unique: t.Literal[False] = False, **kwargs, ): @@ -284,7 +284,7 @@ def set(cls, instance, value, field_name, **kwargs): if value is None: instance.__pending_encryption_values__.pop(field.attname, None) else: - if kwargs.get("normalize", True): + if kwargs.get("normalize", True) and field.normalize is not None: value = field.normalize(value) instance.__pending_encryption_values__[field.attname] = value diff --git a/codeforlife/models/fields/normalized.py b/codeforlife/models/fields/normalized.py index f8bccf93..329f3cdd 100644 --- a/codeforlife/models/fields/normalized.py +++ b/codeforlife/models/fields/normalized.py @@ -15,13 +15,13 @@ class NormalizedField(Field, t.Generic[AnyModel, T]): """A Django model field that normalizes values before saving.""" - def __init__(self, normalize: Normalize[T], *args, **kwargs): + def __init__(self, normalize: None | Normalize[T], *args, **kwargs): super().__init__(*args, **kwargs) self.normalize = normalize @classmethod def set( - cls, instance: AnyModel, value: t.Optional[T], field_name: str, **kwargs + cls, instance: AnyModel, value: None | T, field_name: str, **kwargs ): """ Normalize and assign a value to a NormalizedField. @@ -32,9 +32,11 @@ def set( field_name: The name of the NormalizedField on the model. """ if value is not None: - value = t.cast( + field = t.cast( NormalizedField[AnyModel, T], instance._meta.get_field(field_name), - ).normalize(value) + ) + if field.normalize is not None: + value = field.normalize(value) setattr(instance, field_name, value) diff --git a/codeforlife/models/fields/sha256.py b/codeforlife/models/fields/sha256.py index 0d11a5ae..1edc5f66 100644 --- a/codeforlife/models/fields/sha256.py +++ b/codeforlife/models/fields/sha256.py @@ -33,7 +33,7 @@ class Sha256Field(NormalizedField[Model, str], CharField): def __init__( self, - normalize: Normalize[str] = lambda x: x, + normalize: None | Normalize[str] = None, editable: t.Literal[False] = False, max_length: t.Literal[64] = 64, # Length of SHA-256 hash in hexadecimal **kwargs, @@ -91,9 +91,11 @@ def set(cls, instance, value, field_name, **kwargs): """ if value is not None and kwargs.get("hash", True): if kwargs.get("normalize", True): - value = t.cast( + field = t.cast( Sha256Field, instance._meta.get_field(field_name) - ).normalize(value) + ) + if field.normalize is not None: + value = field.normalize(value) value = cls.hash(value) setattr(instance, field_name, value) @@ -110,15 +112,22 @@ def process_rhs(self, compiler, connection): field: Sha256Field = self.lhs.output_field - if self.rhs is None: - return sql, params - - if isinstance(self.rhs, str): - return sql, [Sha256Field.hash(field.normalize(self.rhs))] - - return sql, [ - Sha256Field.hash(field.normalize(value)) for value in self.rhs - ] + return sql, ( + params + if self.rhs is None + else ( + [ + Sha256Field.hash( + value + if field.normalize is None + else field.normalize(value) + ) + for value in ( + [self.rhs] if isinstance(self.rhs, str) else self.rhs + ) + ] + ) + ) # pylint: disable-next=abstract-method,too-many-ancestors From 91ec75bcc599d1a38de0b5826c60a6d7abcf8228 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Thu, 21 May 2026 09:51:27 +0000 Subject: [PATCH 14/18] unique access code --- .../migrations/0005_client_side_encryption_part_3.py | 10 +++++++++- codeforlife/user/models/klass.py | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py index 9e92fee7..885b9c46 100644 --- a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py +++ b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py @@ -1,5 +1,5 @@ from django.db import migrations -from django.db.models import CharField +from django.db.models import CharField, Q, UniqueConstraint from ...models.fields import EncryptedTextField, Sha256Field @@ -116,6 +116,14 @@ ), preserve_default=False, ), + migrations.AddConstraint( + model_name="class", + constraint=UniqueConstraint( + condition=Q(("_access_code_hash", ""), _negated=True), + fields=("_access_code_hash",), + name="unique_access_code_hash_non_empty", + ), + ), migrations.AlterField( model_name="class", name="_access_code_plain", diff --git a/codeforlife/user/models/klass.py b/codeforlife/user/models/klass.py index f4dd59e6..b47989aa 100644 --- a/codeforlife/user/models/klass.py +++ b/codeforlife/user/models/klass.py @@ -254,6 +254,13 @@ def anonymise(self): class Meta(TypedModelMeta): verbose_name_plural = "classes" + constraints = [ + models.UniqueConstraint( + fields=["_access_code_hash"], + condition=~models.Q(_access_code_hash=""), + name="unique_access_code_hash_non_empty", + ), + ] @property def dek_aead(self): From 337ad8d5ec052a62bad5f803c4a4808352c6cb80 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Thu, 21 May 2026 10:05:06 +0000 Subject: [PATCH 15/18] optimizations --- codeforlife/user/management/commands/normalize_fields.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index ba4fe1dd..446411b2 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -282,6 +282,11 @@ def _normalize_field_for_model( log("No hash field found, skipping hash normalization.") return + # If the hash field has no normalization method, there's nothing to do. + if hash_field.normalize is None: + log("No normalization method on hash field, skipping.") + return + queryset = self._build_hash_queryset( model_manager=model_manager, plain_field_is_null_or_empty=plain_field_is_null_or_empty, @@ -297,7 +302,7 @@ def _normalize_field_for_model( unique_hash_field = bool(t.cast(Sha256Field, hash_field).unique) state: HashUniquenessState | None = None - if unique_hash_field: + if unique_hash_field and model_name in ["School"]: # Ensure we avoid collisions with hashes that already exist in rows # outside the queryset currently being normalized. state = self._build_hash_uniqueness_state( @@ -376,6 +381,7 @@ def _assign_unique_plain_and_hash( hash_field: Sha256Field, state: HashUniquenessState | None, ) -> None: + assert hash_field.normalize is not None hash_field_name = hash_field.name value = t.cast(str, getattr(instance, plain_field_name)) normalized_base = hash_field.normalize(value) From cff1ec379267555d58ef487f0dc4dad36c5d07a3 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Thu, 21 May 2026 10:51:50 +0000 Subject: [PATCH 16/18] fix unique constraints --- .../0005_client_side_encryption_part_3.py | 18 ++++++++++++++++-- codeforlife/user/models/other.py | 10 +++++++++- codeforlife/user/models/user/user.py | 8 +++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py index 885b9c46..4b72d186 100644 --- a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py +++ b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py @@ -84,11 +84,18 @@ default="", editable=False, max_length=64, - unique=True, verbose_name="username hash", ), preserve_default=False, ), + migrations.AddConstraint( + model_name="user", + constraint=UniqueConstraint( + condition=Q(("_username_hash", ""), _negated=True), + fields=("_username_hash",), + name="unique_username_hash_non_empty", + ), + ), ] class_migrations = [ @@ -213,11 +220,18 @@ default="", editable=False, max_length=64, - unique=True, verbose_name="token hash", ), preserve_default=False, ), + migrations.AddConstraint( + model_name="schoolteacherinvitation", + constraint=UniqueConstraint( + condition=Q(("_token_hash", ""), _negated=True), + fields=("_token_hash",), + name="unique_token_hash_non_empty", + ), + ), ] school_migrations = [ diff --git a/codeforlife/user/models/other.py b/codeforlife/user/models/other.py index c7de5e5c..53f5c5e9 100644 --- a/codeforlife/user/models/other.py +++ b/codeforlife/user/models/other.py @@ -293,7 +293,6 @@ class SchoolTeacherInvitation(EncryptedModel): _token_hash = Sha256Field( verbose_name=_("token hash"), - unique=True, db_column="token_hash", ) _token_plain: str @@ -448,6 +447,15 @@ def invited_teacher_email(self, value: str): SchoolTeacherInvitationModelManager() # type: ignore[assignment] ) + class Meta: + constraints = [ + models.UniqueConstraint( + condition=~models.Q(_token_hash=""), + fields=["_token_hash"], + name="unique_token_hash_non_empty", + ), + ] + @property def is_expired(self): """Whether the invitation has expired based on the expiry datetime.""" diff --git a/codeforlife/user/models/user/user.py b/codeforlife/user/models/user/user.py index bc4e6695..1a6d0994 100644 --- a/codeforlife/user/models/user/user.py +++ b/codeforlife/user/models/user/user.py @@ -169,7 +169,6 @@ class User(AbstractBaseUser, PermissionsMixin, DataEncryptionKeyModel): _username_hash = Sha256Field( verbose_name=_("username hash"), db_column="username_hash", - unique=True, ) _username_plain = models.CharField( _("username"), @@ -318,6 +317,13 @@ def email(self, value: str): class Meta: verbose_name = _("user") verbose_name_plural = _("users") + constraints = [ + models.UniqueConstraint( + condition=~models.Q(_username_hash=""), + fields=["_username_hash"], + name="unique_username_hash_non_empty", + ), + ] # TODO: remove in new schema password: str # type: ignore[assignment] From 44ed1f2bd0b0f7672da058da4226a9623cf0049c Mon Sep 17 00:00:00 2001 From: SKairinos Date: Thu, 21 May 2026 11:05:31 +0000 Subject: [PATCH 17/18] fix unique school names --- .../0005_client_side_encryption_part_3.py | 9 ++++++++- codeforlife/user/models/school.py | 14 +++++++++++++- codeforlife/user/models/user/user.py | 6 +++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py index 4b72d186..380a6cee 100644 --- a/codeforlife/user/migrations/0005_client_side_encryption_part_3.py +++ b/codeforlife/user/migrations/0005_client_side_encryption_part_3.py @@ -255,11 +255,18 @@ default="", editable=False, max_length=64, - unique=True, verbose_name="name hash", ), preserve_default=False, ), + migrations.AddConstraint( + model_name="school", + constraint=UniqueConstraint( + condition=Q(("_name_hash", ""), _negated=True), + fields=("_name_hash",), + name="unique_name_hash_non_empty", + ), + ), ] diff --git a/codeforlife/user/models/school.py b/codeforlife/user/models/school.py index 9f41940a..4be1f781 100644 --- a/codeforlife/user/models/school.py +++ b/codeforlife/user/models/school.py @@ -18,6 +18,10 @@ if t.TYPE_CHECKING: # pragma: no cover from datetime import datetime + from django_stubs_ext.db.models import TypedModelMeta +else: + TypedModelMeta = object + # TODO: add to School.name field-validators in new schema. school_name_validators: Validators = [ @@ -71,7 +75,6 @@ class School(DataEncryptionKeyModel): _name_hash = Sha256Field( verbose_name=_("name hash"), - unique=True, db_column="name_hash", normalize=lambda name: SchoolModelManager.normalize_name( name, lower=True @@ -133,6 +136,15 @@ def name(self, value: str): SchoolModelManager() # type: ignore[assignment] ) + class Meta(TypedModelMeta): + constraints = [ + models.UniqueConstraint( + condition=~models.Q(_name_hash=""), + fields=["_name_hash"], + name="unique_name_hash_non_empty", + ), + ] + def __str__(self): return self.name diff --git a/codeforlife/user/models/user/user.py b/codeforlife/user/models/user/user.py index 1a6d0994..baa65127 100644 --- a/codeforlife/user/models/user/user.py +++ b/codeforlife/user/models/user/user.py @@ -24,11 +24,15 @@ from ....validators import UnicodeAlphanumericCharSetValidator if t.TYPE_CHECKING: # pragma: no cover + from django_stubs_ext.db.models import TypedModelMeta + from ..auth_factor import AuthFactor from ..otp_bypass_token import OtpBypassToken from ..session import Session from ..student import Student from ..teacher import Teacher +else: + TypedModelMeta = object # TODO: add to model validators in new schema. @@ -314,7 +318,7 @@ def email(self, value: str): date_joined = models.DateTimeField(_("date joined"), default=timezone.now) - class Meta: + class Meta(TypedModelMeta): verbose_name = _("user") verbose_name_plural = _("users") constraints = [ From 1540a898cca61f16146ce04ee876a04925517479 Mon Sep 17 00:00:00 2001 From: SKairinos Date: Thu, 21 May 2026 13:13:30 +0000 Subject: [PATCH 18/18] fix --- codeforlife/user/management/commands/normalize_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeforlife/user/management/commands/normalize_fields.py b/codeforlife/user/management/commands/normalize_fields.py index 446411b2..2dbe1074 100644 --- a/codeforlife/user/management/commands/normalize_fields.py +++ b/codeforlife/user/management/commands/normalize_fields.py @@ -302,7 +302,7 @@ def _normalize_field_for_model( unique_hash_field = bool(t.cast(Sha256Field, hash_field).unique) state: HashUniquenessState | None = None - if unique_hash_field and model_name in ["School"]: + if model_name in ["School"]: # Ensure we avoid collisions with hashes that already exist in rows # outside the queryset currently being normalized. state = self._build_hash_uniqueness_state(