diff --git a/apps/web/src/utils/__tests__/language.test.ts b/apps/web/src/utils/__tests__/language.test.ts index 22d77946f..6fe46cff9 100644 --- a/apps/web/src/utils/__tests__/language.test.ts +++ b/apps/web/src/utils/__tests__/language.test.ts @@ -51,9 +51,9 @@ describe('getValueForLanguage', () => { expect(getValueForLanguage({ en: 'Hello', es: ' ' }, 'es')).toBe('Hello'); }); - // Every `UploadError` in `../upload` is authored in English and French only, so the upload error + // Most `UploadError`s in `../upload` are authored in English and French only, so the upload error // page resolves through this rather than reading `description[resolvedLanguage]` directly. - it('should render an upload error in English for a Spanish reader, since none carry Spanish', () => { + it('should render an upload error in English for a Spanish reader when it carries no Spanish', () => { const { description } = new UploadError({ en: 'Record in the record array was left undefined', fr: `L'enregistrement dans le tableau d'enregistrements n'est pas défini` diff --git a/apps/web/src/utils/__tests__/upload.test.ts b/apps/web/src/utils/__tests__/upload.test.ts index 0a8b4cd4d..3b3301ef1 100644 --- a/apps/web/src/utils/__tests__/upload.test.ts +++ b/apps/web/src/utils/__tests__/upload.test.ts @@ -46,6 +46,9 @@ function expectRejection(promise: Promise, en: string) { return expect(promise).rejects.toMatchObject({ description: { en } }); } +/** A numeric enum as TypeScript compiles it: every value is also mapped back to its name. */ +const Severity = { 0: 'Mild', 1: 'Severe', Mild: 0, Severe: 1 } as const; + let consoleError: ReturnType; describe('Zod3', () => { @@ -110,6 +113,97 @@ describe('Zod3', () => { }); }); + it('should read a literal as its one allowed value', () => { + expect(Zod3.getZodTypeName(z3.literal(7))).toStrictEqual({ + enumValues: [7], + isOptional: false, + typeName: 'ZodLiteral' + }); + }); + + it('should read a union of literals as its allowed values, keeping their types', () => { + expect( + Zod3.getZodTypeName(z3.union([z3.literal(0), z3.literal('n/a'), z3.literal(true)]).optional()) + ).toStrictEqual({ enumValues: [0, 'n/a', true], isOptional: true, typeName: 'ZodUnion' }); + }); + + it('should read the values of a numeric native enum without its reverse mappings', () => { + expect(Zod3.getZodTypeName(z3.nativeEnum(Severity))).toStrictEqual({ + enumValues: [0, 1], + isOptional: false, + typeName: 'ZodNativeEnum' + }); + }); + + it('should read the values of a string native enum', () => { + expect(Zod3.getZodTypeName(z3.nativeEnum({ Left: 'left', Right: 'right' }))).toMatchObject({ + enumValues: ['left', 'right'] + }); + }); + + it('should reject a union with a member that is not a literal, since a cell could not say which member it is', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.union([z3.literal(1), z3.string()])))).toStrictEqual({ + en: 'Unsupported Zod union: every member must be a literal value', + es: 'Unión de Zod no compatible: cada miembro debe ser un valor literal', + fr: 'Union Zod non prise en charge : chaque membre doit être une valeur littérale' + }); + }); + + it('should reject a union with a member of a type it does not know', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.union([z3.literal(1), z3.null()])))).toMatchObject({ + en: 'Unsupported Zod union: every member must be a literal value' + }); + }); + + it('should reject a literal that a cell cannot express', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.literal(null)))).toStrictEqual({ + en: "Unsupported literal value 'null' of type 'object'", + es: "Valor literal no compatible 'null' de tipo 'object'", + fr: "Valeur littérale non prise en charge 'null' de type 'object'" + }); + }); + + it('should read a record keyed by an enum as its keys and the type of its values', () => { + const schema = z3.record(z3.enum(['monday', 'friday']), z3.union([z3.literal(1), z3.literal(2)])); + expect(Zod3.getZodTypeName(schema)).toStrictEqual({ + isOptional: false, + recordKeys: ['monday', 'friday'], + recordValue: { enumValues: [1, 2], isOptional: false, typeName: 'ZodUnion' }, + typeName: 'ZodRecord' + }); + }); + + it('should read a record keyed by any string as having no fixed keys', () => { + expect(Zod3.getZodTypeName(z3.record(z3.number()).optional())).toStrictEqual({ + isOptional: true, + recordKeys: null, + recordValue: { isOptional: false, typeName: 'ZodNumber' }, + typeName: 'ZodRecord' + }); + }); + + it('should reject a record whose keys are neither an enum nor strings', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.record(z3.number(), z3.number())))).toStrictEqual({ + en: 'Unsupported Zod record: keys must be an enum or strings', + es: 'Registro de Zod no compatible: las claves deben ser un enum o cadenas', + fr: 'Enregistrement Zod non pris en charge : les clés doivent être un enum ou des chaînes' + }); + }); + + it('should reject a record whose keys are of a type it does not know', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.record(z3.symbol(), z3.number())))).toMatchObject({ + en: 'Unsupported Zod record: keys must be an enum or strings' + }); + }); + + it('should reject a record whose values could contain the separators of a RECORD cell', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.record(z3.string())))).toStrictEqual({ + en: "Unsupported Zod record: values must be numbers or fixed choices, not 'ZodString'", + es: "Registro de Zod no compatible: los valores deben ser números u opciones fijas, no 'ZodString'", + fr: "Enregistrement Zod non pris en charge : les valeurs doivent être des nombres ou des choix fixes, pas 'ZodString'" + }); + }); + it('should parse a set of free-form values without enum values', () => { expect(Zod3.getZodTypeName(z3.set(z3.string()))).toStrictEqual({ isOptional: false, typeName: 'ZodSet' }); }); @@ -118,20 +212,31 @@ describe('Zod3', () => { expect(Zod3.getZodTypeName(z3.array(z3.string()))).toStrictEqual({ isOptional: false, typeName: 'ZodArray' }); }); - it('should reject a schema type that cannot be represented in a CSV', () => { - expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.literal('a')))).toMatchObject({ en: 'Unexpected Error' }); + it('should name the type of a schema that cannot be represented in a CSV', () => { + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.tuple([z3.string()])))).toStrictEqual({ + en: "Unsupported Zod type 'ZodTuple'", + es: "Tipo de Zod no compatible 'ZodTuple'", + fr: "Type Zod non pris en charge 'ZodTuple'" + }); + }); + + it('should reject a field that is not a Zod schema without naming a type', () => { + const notASchema = Object.assign(z3.string(), { _def: {} }); + expect(uploadErrorOf(() => Zod3.getZodTypeName(notASchema))).toMatchObject({ + en: "Unsupported Zod type 'undefined'" + }); }); it('should reject a set whose value type cannot be represented in a CSV', () => { - expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.set(z3.literal('a'))))).toMatchObject({ + expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.set(z3.tuple([z3.string()]))))).toMatchObject({ en: 'Invalid inner type: ZodSet value type must have a valid type definition' }); }); - it('should reject an array of objects with a field that cannot be represented in a CSV', () => { - expect(uploadErrorOf(() => Zod3.getZodTypeName(z3.array(z3.object({ a: z3.literal('x') }))))).toMatchObject({ - en: 'Unhandled case!' - }); + it('should name the type of a field in an array of objects that cannot be represented in a CSV', () => { + expect( + uploadErrorOf(() => Zod3.getZodTypeName(z3.array(z3.object({ a: z3.tuple([z3.string()]) })))) + ).toMatchObject({ en: "Unsupported Zod type 'ZodTuple'" }); }); it('should reject an array of objects without any fields', () => { @@ -185,10 +290,24 @@ describe('Zod3', () => { ); }); - it('should reject an enum without values', () => { - expect(uploadErrorOf(() => Zod3.generateSampleData({ isOptional: false, typeName: 'ZodEnum' }))).toMatchObject({ - en: 'Invalid Enum error' - }); + it.each([ + ['a literal', z3.literal('only'), 'only'], + ['a union of literals', z3.union([z3.literal(0), z3.literal(1), z3.literal(2)]), '0/1/2'], + ['a native enum', z3.nativeEnum(Severity), '0/1'], + ['an optional union of literals', z3.union([z3.literal(0), z3.literal(1)]).optional(), '0/1 (optional)'] + ])('should list the allowed values of %s', (_, schema, sample) => { + expect(Zod3.generateSampleData(Zod3.getZodTypeName(schema))).toBe(sample); + }); + + it('should show every key of a record keyed by an enum with the sample of its values', () => { + const schema = z3.record(z3.enum(['monday', 'friday']), z3.union([z3.literal(1), z3.literal(2)])); + expect(Zod3.generateSampleData(Zod3.getZodTypeName(schema))).toBe('RECORD(monday: 1/2, friday: 1/2)'); + }); + + it('should show a generic key for a record keyed by any string', () => { + expect(Zod3.generateSampleData(Zod3.getZodTypeName(z3.record(z3.number()).optional()))).toBe( + 'RECORD(key: number, ...) (optional)' + ); }); it('should describe every field of a record array, separated by commas', () => { @@ -254,6 +373,33 @@ describe('Zod3', () => { en: 'Validation schema for this instrument is invalid' }); }); + + it('should name the column and type of a field that cannot be represented in a CSV', () => { + const schema = z3.object({ pair: z3.tuple([z3.string()]), score: z3.number() }); + expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(schema)))).toStrictEqual({ + en: "Unsupported Zod type 'ZodTuple' at column name: 'pair'", + es: "Tipo de Zod no compatible 'ZodTuple' en la columna: 'pair'", + fr: "Type Zod non pris en charge 'ZodTuple' au nom de colonne : 'pair'" + }); + }); + + it('should locate an error only in the languages it was written in', () => { + const schema = z3.object({ tags: z3.set(z3.tuple([z3.string()])) }); + expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(schema)))).toStrictEqual({ + en: "Invalid inner type: ZodSet value type must have a valid type definition at column name: 'tags'", + fr: "Type interne invalide : le type de valeur ZodSet doit avoir une définition de type valide au nom de colonne : 'tags'" + }); + }); + + it('should not present an unexpected failure as an upload error', () => { + const failure = new Error('def is unreadable'); + const unreadable = Object.defineProperty(z3.string(), '_def', { + get(): never { + throw failure; + } + }); + expect(() => createUploadTemplateCSV(instrumentWith(z3.object({ unreadable })))).toThrow(failure); + }); }); describe('processInstrumentCSV', () => { @@ -480,32 +626,165 @@ describe('Zod3', () => { ]); await expectRejection( processInstrumentCSV(file, instrumentWith(z3.object({ score: z3.number().refine(Boolean) }))), - "Unexpected Zod type name 'ZodEffects' at column name: 'score' and row number '1'" + "Unsupported Zod type 'ZodEffects' at column name: 'score' and row number '1'" ); }); - it('should fall back to a generic error when an unsupported field schema cannot be serialized for logging', async () => { + it('should name the type of an unsupported field whose schema cannot be serialized to JSON', async () => { const file = csvFile([ ['subjectID', 'date', 'big'], ['subject1', '2024-01-01', '1'] ]); await expectRejection( processInstrumentCSV(file, instrumentWith(z3.object({ big: z3.literal(1n) }))), - 'Error parsing CSV' + "Unsupported literal value '1' of type 'bigint' at column name: 'big' and row number '1'" ); }); - it('should reject a row the instrument schema does not accept', async () => { + it('should convert a cell to the literal it names, with the type the schema declares', async () => { + const instrument = instrumentWith( + z3.object({ + putbacks: z3.union([z3.literal(0), z3.literal(1), z3.literal(2), z3.literal(3)]), + retries: z3.union([z3.literal(0), z3.literal(1)]).optional(), + severity: z3.nativeEnum(Severity), + side: z3.literal('left') + }) + ); + const file = csvFile([ + ['subjectID', 'date', 'putbacks', 'retries', 'severity', 'side'], + ['subject1', '2024-01-01', '2', '', '1', 'left'] + ]); + await expect(processInstrumentCSV(file, instrument)).resolves.toStrictEqual([ + { + date: new Date('2024-01-01'), + putbacks: 2, + retries: undefined, + severity: 1, + side: 'left', + subjectID: 'subject1' + } + ]); + }); + + it('should reject a value outside the allowed literals, naming them with the column and row', async () => { + const instrument = instrumentWith(z3.object({ putbacks: z3.union([z3.literal(0), z3.literal(1)]) })); + const file = csvFile([ + ['subjectID', 'date', 'putbacks'], + ['subject1', '2024-01-01', '0'], + ['subject2', '2024-01-01', '7'] + ]); + await expect(processInstrumentCSV(file, instrument)).rejects.toMatchObject({ + description: { + en: "Invalid value '7': expected one of 0/1 at column name: 'putbacks' and row number '2'", + es: "Valor no válido '7': se esperaba uno de 0/1 en la columna: 'putbacks' y el número de fila '2'", + fr: "Valeur invalide '7' : une des valeurs 0/1 était attendue au nom de colonne : 'putbacks' et numéro de ligne '2'" + } + }); + }); + + it('should reject an enum value outside its options by column and row, rather than as a schema failure', async () => { const file = csvFile([ ['subjectID', 'date', 'choice'], ['subject1', '2024-01-01', 'c'] ]); await expectRejection( processInstrumentCSV(file, instrumentWith(z3.object({ choice: z3.enum(['a', 'b']) }))), + "Invalid value 'c': expected one of a/b at column name: 'choice' and row number '1'" + ); + }); + + it('should fall back to a generic error when a value fails to convert for an unexpected reason', async () => { + const unreadable = Object.defineProperty(z3.string(), '_def', { + get(): never { + throw new Error('def is unreadable'); + } + }); + const file = csvFile([ + ['subjectID', 'date', 'unreadable'], + ['subject1', '2024-01-01', '1'] + ]); + await expectRejection(processInstrumentCSV(file, instrumentWith(z3.object({ unreadable }))), 'Error parsing CSV'); + }); + + it('should reject a row the instrument schema does not accept', async () => { + const file = csvFile([ + ['subjectID', 'date', 'score'], + ['subject1', '2024-01-01', '11'] + ]); + await expectRejection( + processInstrumentCSV(file, instrumentWith(z3.object({ score: z3.number().max(10) }))), 'Schema parsing failed: refer to the browser console for further details' ); }); + describe('records', () => { + const instrument = instrumentWith( + z3.object({ + ratings: z3.record(z3.enum(['monday', 'friday']), z3.union([z3.literal(1), z3.literal(2), z3.literal(3)])) + }) + ); + + const processRatings = (value: string) => { + return processInstrumentCSV( + csvFile([ + ['subjectID', 'date', 'ratings'], + ['subject1', '2024-01-01', value] + ]), + instrument + ); + }; + + it('should convert each value of a record with the type the schema declares', async () => { + await expect(processRatings('RECORD(monday: 3, friday: 1)')).resolves.toMatchObject([ + { ratings: { friday: 1, monday: 3 } } + ]); + }); + + it('should accept any key in a record keyed by any string', async () => { + const file = csvFile([ + ['subjectID', 'date', 'weights'], + ['subject1', '2024-01-01', 'RECORD(week1: 21.5, week2: 22)'] + ]); + await expect( + processInstrumentCSV(file, instrumentWith(z3.object({ weights: z3.record(z3.number()) }))) + ).resolves.toMatchObject([{ weights: { week1: 21.5, week2: 22 } }]); + }); + + it('should reject a key outside the keys of the record, naming them with the column and row', async () => { + await expect(processRatings('RECORD(monday: 3, sunday: 1)')).rejects.toMatchObject({ + description: { + en: "Unknown key 'sunday' in RECORD: expected one of monday/friday at column name: 'ratings' and row number '1'", + es: "Clave desconocida 'sunday' en RECORD: se esperaba una de monday/friday en la columna: 'ratings' y el número de fila '1'", + fr: "Clé inconnue 'sunday' dans RECORD : une des clés monday/friday était attendue au nom de colonne : 'ratings' et numéro de ligne '1'" + } + }); + }); + + it('should reject a value outside the options of the record', async () => { + await expectRejection( + processRatings('RECORD(monday: 9)'), + "Invalid value '9': expected one of 1/2/3 at column name: 'ratings' and row number '1'" + ); + }); + + it('should reject a cell that is not written as a RECORD', async () => { + await expect(processRatings('RECORD(monday: 3')).rejects.toMatchObject({ + description: { + en: "Syntax error in RECORD declaration: RECORD(monday: 3 at column name: 'ratings' and row number '1'", + es: "Error de sintaxis en la declaración RECORD: RECORD(monday: 3 en la columna: 'ratings' y el número de fila '1'", + fr: "Erreur de syntaxe dans la déclaration RECORD : RECORD(monday: 3 au nom de colonne : 'ratings' et numéro de ligne '1'" + } + }); + }); + + it('should reject a record with no fields', async () => { + await expectRejection( + processRatings('RECORD()'), + "One or more of the record fields was left empty at column name: 'ratings' and row number '1'" + ); + }); + }); + describe('record arrays', () => { const instrument = instrumentWith( z3.object({ items: z3.array(z3.object({ name: z3.string(), quantity: z3.number().optional() })) }) @@ -557,6 +836,19 @@ describe('Zod3', () => { "Malformed record at index 1 at column name: 'items' and row number '1'" ); }); + + it('should convert a literal field of a record to the literal it names', async () => { + const file = csvFile([ + ['subjectID', 'date', 'trials'], + ['subject1', '2024-01-01', 'RECORD_ARRAY(arena:2;arena:4;)'] + ]); + const schema = z3.object({ + trials: z3.array(z3.object({ arena: z3.union([z3.literal(1), z3.literal(2), z3.literal(4)]) })) + }); + await expect(processInstrumentCSV(file, instrumentWith(schema))).resolves.toMatchObject([ + { trials: [{ arena: 2 }, { arena: 4 }] } + ]); + }); }); }); }); @@ -641,8 +933,9 @@ describe('Zod4', () => { it('should reject a type name it has no sample for', () => { expect(uploadErrorOf(() => Zod4.generateSampleData({ isOptional: false, typeName: 'optional' }))).toEqual({ - en: "Unexpected Zod type name 'optional'", - fr: "Nom de type Zod inattendu 'optional'" + en: "Unsupported Zod type 'optional'", + es: "Tipo de Zod no compatible 'optional'", + fr: "Type Zod non pris en charge 'optional'" }); }); }); @@ -685,16 +978,72 @@ describe('Zod4', () => { }); }); + it('should list the allowed values of literal, literal union and native enum fields', () => { + const schema = z4.object({ + arena: z4.union([z4.literal(1), z4.literal(2), z4.literal([3, 4])]), + retries: z4.union([z4.literal(0), z4.literal(1)]).optional(), + severity: z4.nativeEnum(Severity), + side: z4.literal('left') + }); + expect(createUploadTemplateCSV(instrumentWith(schema)).content).toBe( + unparse([ + ['subjectID', 'date', 'arena', 'retries', 'severity', 'side'], + ['string', 'yyyy-mm-dd', '1/2/3/4', '0/1 (optional)', '0/1', 'left'] + ]) + ); + }); + + it('should show the keys of a record keyed by an enum, and a generic key for one keyed by any string', () => { + const schema = z4.object({ + ratings: z4.record(z4.enum(['monday', 'friday']), z4.union([z4.literal(1), z4.literal(2)])), + weights: z4.record(z4.string(), z4.number()).optional() + }); + expect(createUploadTemplateCSV(instrumentWith(schema)).content).toBe( + unparse([ + ['subjectID', 'date', 'ratings', 'weights'], + ['string', 'yyyy-mm-dd', 'RECORD(monday: 1/2, friday: 1/2)', 'RECORD(key: number, ...) (optional)'] + ]) + ); + }); + + it('should reject a record whose keys are neither an enum nor strings', () => { + const schema = z4.object({ byIndex: z4.record(z4.number(), z4.number()) }); + expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(schema)))).toMatchObject({ + en: "Unsupported Zod record: keys must be an enum or strings at column name: 'byIndex'" + }); + }); + + it('should reject a record whose values could contain the separators of a RECORD cell', () => { + const schema = z4.object({ notes: z4.record(z4.string(), z4.string()) }); + expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(schema)))).toMatchObject({ + en: "Unsupported Zod record: values must be numbers or fixed choices, not 'string' at column name: 'notes'" + }); + }); + + it('should reject a union with a member that is not a literal, since a cell could not say which member it is', () => { + const schema = z4.object({ mixed: z4.union([z4.literal(1), z4.string()]) }); + expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(schema)))).toMatchObject({ + en: "Unsupported Zod union: every member must be a literal value at column name: 'mixed'" + }); + }); + + it('should reject a literal that a cell cannot express', () => { + const schema = z4.object({ nothing: z4.literal(null) }); + expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(schema)))).toMatchObject({ + en: "Unsupported literal value 'null' of type 'object' at column name: 'nothing'" + }); + }); + it('should reject a schema that is not an object', () => { expect(uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(z4.string())))).toMatchObject({ en: 'Expected schema to be instance of ZodObject' }); }); - it('should reject a field type that cannot be represented in a CSV', () => { + it('should name the column and type of a field that cannot be represented in a CSV', () => { expect( - uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(z4.object({ kind: z4.literal('a') })))) - ).toMatchObject({ en: "Unexpected Zod type name 'literal'" }); + uploadErrorOf(() => createUploadTemplateCSV(instrumentWith(z4.object({ pair: z4.tuple([z4.string()]) })))) + ).toMatchObject({ en: "Unsupported Zod type 'tuple' at column name: 'pair'" }); }); }); @@ -772,6 +1121,74 @@ describe('Zod4', () => { expect(result[0]?.status).toBe('active'); }); + it('should convert a cell to the literal it names, with the type the schema declares', async () => { + const instrument = instrumentWith( + z4.object({ + arena: z4.union([z4.literal(1), z4.literal(2), z4.literal(3), z4.literal(4)]), + retries: z4.union([z4.literal(0), z4.literal(1)]).optional(), + severity: z4.nativeEnum(Severity), + side: z4.literal('left') + }) + ); + const file = csvFile([ + ['subjectID', 'date', 'arena', 'retries', 'severity', 'side'], + ['subject1', '2024-01-01', '3', '', '1', 'left'] + ]); + await expect(processInstrumentCSV(file, instrument)).resolves.toStrictEqual([ + { arena: 3, date: new Date('2024-01-01'), retries: undefined, severity: 1, side: 'left', subjectID: 'subject1' } + ]); + }); + + it('should reject a value outside the allowed literals, naming them with the column and row', async () => { + const file = csvFile([ + ['subjectID', 'date', 'arena'], + ['subject1', '2024-01-01', '5'] + ]); + await expectRejection( + processInstrumentCSV(file, instrumentWith(z4.object({ arena: z4.union([z4.literal(1), z4.literal(2)]) }))), + "Invalid value '5': expected one of 1/2 at column name: 'arena' and row number '1'" + ); + }); + + it('should reject an enum value outside its options by column and row, rather than as a schema failure', async () => { + const file = csvFile([ + ['subjectID', 'date', 'level'], + ['subject1', '2024-01-01', 'medium'] + ]); + await expectRejection( + processInstrumentCSV(file, instrumentWith(z4.object({ level: z4.enum(['low', 'high']) }))), + "Invalid value 'medium': expected one of low/high at column name: 'level' and row number '1'" + ); + }); + + it('should convert each value of a record with the type the schema declares', async () => { + const instrument = instrumentWith( + z4.object({ + ratings: z4.record(z4.enum(['monday', 'friday']), z4.union([z4.literal(1), z4.literal(2), z4.literal(3)])), + weights: z4.record(z4.string(), z4.number()) + }) + ); + const file = csvFile([ + ['subjectID', 'date', 'ratings', 'weights'], + ['subject1', '2024-01-01', 'RECORD(monday: 3, friday: 1)', 'RECORD(week1: 21.5)'] + ]); + await expect(processInstrumentCSV(file, instrument)).resolves.toMatchObject([ + { ratings: { friday: 1, monday: 3 }, weights: { week1: 21.5 } } + ]); + }); + + it('should reject a key outside the keys of a record, naming them with the column and row', async () => { + const instrument = instrumentWith(z4.object({ ratings: z4.record(z4.enum(['monday', 'friday']), z4.number()) })); + const file = csvFile([ + ['subjectID', 'date', 'ratings'], + ['subject1', '2024-01-01', 'RECORD(monday: 3, sunday: 1)'] + ]); + await expectRejection( + processInstrumentCSV(file, instrument), + "Unknown key 'sunday' in RECORD: expected one of monday/friday at column name: 'ratings' and row number '1'" + ); + }); + it('should process array of objects', async () => { const instrumentWithArray = instrumentWith( z4.object({ @@ -911,13 +1328,13 @@ describe('Zod4', () => { z4.array(z4.string()), "Unsupported type for innerType of array record 'string': must be 'object'" ], - ['nested', 'a', z4.object({ a: z4.string() }), "Unexpected Zod type name 'object'"], + ['nested', 'a', z4.object({ a: z4.string() }), "Unsupported Zod type 'object'"], ['items', 'items', z4.array(z4.object({ name: z4.string() })), 'Syntax error in RECORD_ARRAY declaration: items'], [ 'items', 'RECORD_ARRAY(name:a,,name:b)', z4.array(z4.object({ name: z4.string() })), - 'One or more of the record array fields was left empty' + 'One or more of the record fields was left empty' ], ['items', 'RECORD_ARRAY(:a)', z4.array(z4.object({ name: z4.string() })), 'Malformed record at index 0'] ])( @@ -947,17 +1364,145 @@ describe('Zod4', () => { it('should reject a row the instrument schema does not accept', async () => { const file = csvFile([ - ['subjectID', 'date', 'level'], - ['subject1', '2024-01-01', 'medium'] + ['subjectID', 'date', 'score'], + ['subject1', '2024-01-01', '11'] ]); await expectRejection( - processInstrumentCSV(file, instrumentWith(z4.object({ level: z4.enum(['low', 'high']) }))), + processInstrumentCSV(file, instrumentWith(z4.object({ score: z4.number().max(10) }))), 'Schema parsing failed: refer to the browser console for further details' ); }); }); }); +// The validation schema of the comprehensive form example in +// packages/instrument-guidelines/AGENTS.md §14.1, copied here: an instrument written from the guide +// has to be uploadable. +/* eslint-disable perfectionist/sort-objects -- kept in the guide's order, so the copy can be compared with its source line by line */ +const guideComprehensiveFormSchema = z4 + .object({ + agreedToTerms: z4.boolean(), + currentlyEmployed: z4.boolean(), + dateOfBirth: z4.date(), + overallHappiness: z4.number().int().min(1).max(10), + educationLevel: z4.union([z4.literal(1), z4.literal(2), z4.literal(3)]), + firstName: z4.string(), + preferredLanguage: z4.enum(['en', 'fr']), + additionalComments: z4.string().optional(), + preferredActivities: z4.set(z4.enum(['reading', 'sports', 'cooking'])), + weeklyMoodRatings: z4.record( + z4.enum(['monday', 'wednesday', 'friday']), + z4.union([z4.literal(1), z4.literal(2), z4.literal(3), z4.literal(4)]) + ), + medications: z4.array( + z4.object({ + medicationName: z4.string(), + dosageMg: z4.number() + }) + ), + hasChronicCondition: z4.boolean(), + chronicConditionName: z4.string().optional() + }) + .superRefine((data, ctx) => { + if (data.hasChronicCondition && !data.chronicConditionName) { + ctx.addIssue({ + code: 'custom', + path: ['chronicConditionName'], + message: 'This field is required / Ce champ est obligatoire.' + }); + } + }); +/* eslint-enable perfectionist/sort-objects */ + +const guideComprehensiveFormColumns = [ + 'subjectID', + 'date', + 'agreedToTerms', + 'currentlyEmployed', + 'dateOfBirth', + 'overallHappiness', + 'educationLevel', + 'firstName', + 'preferredLanguage', + 'additionalComments', + 'preferredActivities', + 'weeklyMoodRatings', + 'medications', + 'hasChronicCondition', + 'chronicConditionName' +]; + +describe('createUploadTemplateCSV', () => { + it("should describe every field of the instrument guide's comprehensive form example", () => { + expect(createUploadTemplateCSV(instrumentWith(guideComprehensiveFormSchema)).content).toBe( + unparse([ + guideComprehensiveFormColumns, + [ + 'string', + 'yyyy-mm-dd', + 'true/false', + 'true/false', + 'yyyy-mm-dd', + 'number', + '1/2/3', + 'string', + 'en/fr', + 'string (optional)', + 'SET(reading/sports/cooking)', + 'RECORD(monday: 1/2/3/4, wednesday: 1/2/3/4, friday: 1/2/3/4)', + 'RECORD_ARRAY(medicationName: string,dosageMg: number; ...)', + 'true/false', + 'string (optional)' + ] + ]) + ); + }); +}); + +describe('processInstrumentCSV', () => { + it("should import a row written for the instrument guide's comprehensive form example", async () => { + const file = csvFile([ + guideComprehensiveFormColumns, + [ + 'subject1', + '2024-01-01', + 'true', + 'false', + '1990-05-17', + '7', + '2', + 'Ada', + 'fr', + '', + 'SET(reading, cooking)', + 'RECORD(monday: 3, wednesday: 4, friday: 2)', + 'RECORD_ARRAY(medicationName: Aspirin, dosageMg: 81;)', + 'false', + '' + ] + ]); + await expect(processInstrumentCSV(file, instrumentWith(guideComprehensiveFormSchema))).resolves.toStrictEqual([ + { + additionalComments: undefined, + agreedToTerms: true, + chronicConditionName: undefined, + currentlyEmployed: false, + date: new Date('2024-01-01'), + dateOfBirth: new Date('1990-05-17'), + educationLevel: 2, + firstName: 'Ada', + hasChronicCondition: false, + medications: [{ dosageMg: 81, medicationName: 'Aspirin' }], + overallHappiness: 7, + preferredActivities: new Set(['cooking', 'reading']), + preferredLanguage: 'fr', + subjectID: 'subject1', + weeklyMoodRatings: { friday: 2, monday: 3, wednesday: 4 } + } + ]); + }); +}); + describe('reformatInstrumentData', () => { const group: Group = { accessibleInstrumentIds: [], diff --git a/apps/web/src/utils/upload.ts b/apps/web/src/utils/upload.ts index 9b29be455..766869dbe 100644 --- a/apps/web/src/utils/upload.ts +++ b/apps/web/src/utils/upload.ts @@ -13,6 +13,8 @@ import type { Merge } from 'type-fest'; import { z as z3 } from 'zod/v3'; import { z as z4 } from 'zod/v4'; +import { LANGUAGES } from './language'; + const INTERNAL_HEADERS = ['subjectID', 'date']; const INTERNAL_HEADERS_SAMPLE_DATA = ['string', 'yyyy-mm-dd']; @@ -93,31 +95,112 @@ const ZOD_TYPE_NAMES = [ 'ZodEnum', 'ZodArray', 'ZodObject', - 'ZodEffects' + 'ZodEffects', + 'ZodLiteral', + 'ZodNativeEnum', + 'ZodUnion', + 'ZodRecord' ] as const; +type UploadErrorDescription = { [L in Language]?: string }; + class UploadError extends Error { - description: { - [L in Language]?: string; - }; + description: UploadErrorDescription; - constructor(description: { [L in Language]?: string }) { + constructor(description: UploadErrorDescription) { super(); this.description = description; } } class UnexpectedZodTypeError extends UploadError { - private constructor(description: { [L in Language]?: string }) { + private constructor(description: UploadErrorDescription) { super(description); } static forTypeName(typeName: string | undefined) { return new this({ - en: `Unexpected Zod type name '${typeName}'`, - fr: `Nom de type Zod inattendu '${typeName}'` + en: `Unsupported Zod type '${typeName}'`, + es: `Tipo de Zod no compatible '${typeName}'`, + fr: `Type Zod non pris en charge '${typeName}'` + }); + } +} + +function locateUploadError(error: UploadError, { column, row }: { column: string; row?: number }): UploadError { + const location: { [L in Language]: string } = + row === undefined + ? { + en: `at column name: '${column}'`, + es: `en la columna: '${column}'`, + fr: `au nom de colonne : '${column}'` + } + : { + en: `at column name: '${column}' and row number '${row}'`, + es: `en la columna: '${column}' y el número de fila '${row}'`, + fr: `au nom de colonne : '${column}' et numéro de ligne '${row}'` + }; + return new UploadError( + Object.fromEntries( + LANGUAGES.flatMap((language) => { + const text = error.description[language]; + return text === undefined ? [] : [[language, `${text} ${location[language]}`]]; + }) + ) + ); +} + +type LiteralOption = boolean | number | string; + +function toLiteralOption(value: unknown): LiteralOption { + if (typeof value === 'boolean' || typeof value === 'number' || typeof value === 'string') { + return value; + } + throw new UploadError({ + en: `Unsupported literal value '${String(value)}' of type '${typeof value}'`, + es: `Valor literal no compatible '${String(value)}' de tipo '${typeof value}'`, + fr: `Valeur littérale non prise en charge '${String(value)}' de type '${typeof value}'` + }); +} + +// A numeric TypeScript enum also maps each value back to its name, and those reverse entries are not values. +function enumLikeValues(entries: { [key: string]: number | string }): (number | string)[] { + const numericValues = new Set(Object.values(entries).filter((value) => typeof value === 'number')); + return Object.entries(entries) + .filter(([key]) => !numericValues.has(Number(key))) + .map(([, value]) => value); +} + +function nonLiteralUnionError() { + return new UploadError({ + en: 'Unsupported Zod union: every member must be a literal value', + es: 'Unión de Zod no compatible: cada miembro debe ser un valor literal', + fr: 'Union Zod non prise en charge : chaque membre doit être une valeur littérale' + }); +} + +function formatOptions(options: readonly LiteralOption[]) { + return options.join('/'); +} + +function parseOptionEntry(entry: string, options: readonly LiteralOption[]): LiteralOption { + const selectedOption = options.find((option) => String(option) === entry); + if (selectedOption === undefined) { + throw new UploadError({ + en: `Invalid value '${entry}': expected one of ${formatOptions(options)}`, + es: `Valor no válido '${entry}': se esperaba uno de ${formatOptions(options)}`, + fr: `Valeur invalide '${entry}' : une des valeurs ${formatOptions(options)} était attendue` }); } + return selectedOption; +} + +function sampleDataForColumn(column: string, generateSampleData: () => string): string { + try { + return generateSampleData(); + } catch (error) { + throw error instanceof UploadError ? locateUploadError(error, { column }) : error; + } } function extractSetEntry(entry: string) { @@ -155,29 +238,78 @@ function extractRecordArrayEntry(entry: string): { [key: string]: string }[] { recordArrayDataList.pop(); } - return recordArrayDataList.map((listData) => { - const recordsList = listData.split(','); - if (recordsList.some((str) => str === '')) { + return recordArrayDataList.map(parseRecordFields); +} + +function parseRecordFields(fieldList: string): { [key: string]: string } { + const recordsList = fieldList.split(','); + if (recordsList.some((str) => str === '')) { + throw new UploadError({ + en: `One or more of the record fields was left empty`, + fr: `Un ou plusieurs champs de l'enregistrement ont été laissés vides` + }); + } + const records: { [key: string]: string } = {}; + recordsList.forEach((rawRecord, i) => { + const separatorIndex = rawRecord.indexOf(':'); + const recordKey = rawRecord.slice(0, separatorIndex).trim(); + const recordValue = rawRecord.slice(separatorIndex + 1).trim(); + if (separatorIndex === -1 || !recordKey) { throw new UploadError({ - en: `One or more of the record array fields was left empty`, - fr: `Un ou plusieurs champs du tableau d'enregistrements ont été laissés vides` + en: `Malformed record at index ${i}`, + fr: `Enregistrement mal formé à l'index ${i}` }); } - const records: { [key: string]: string } = {}; - recordsList.forEach((rawRecord, i) => { - const separatorIndex = rawRecord.indexOf(':'); - const recordKey = rawRecord.slice(0, separatorIndex).trim(); - const recordValue = rawRecord.slice(separatorIndex + 1).trim(); - if (separatorIndex === -1 || !recordKey) { - throw new UploadError({ - en: `Malformed record at index ${i}`, - fr: `Enregistrement mal formé à l'index ${i}` - }); - } - //recordValue can be empty if the value is optional - records[recordKey] = recordValue; + //recordValue can be empty if the value is optional + records[recordKey] = recordValue; + }); + return records; +} + +function parseRecordEntry( + entry: string, + keys: null | readonly string[], + parseValue: (value: string) => unknown +): { [key: string]: unknown } { + const opening = 'RECORD('; + const closing = ')'; + if (!entry.startsWith(opening) || !entry.endsWith(closing)) { + throw new UploadError({ + en: `Syntax error in RECORD declaration: ${entry}`, + es: `Error de sintaxis en la declaración RECORD: ${entry}`, + fr: `Erreur de syntaxe dans la déclaration RECORD : ${entry}` }); - return records; + } + return mapValues(parseRecordFields(entry.slice(opening.length, -closing.length)), (value, key) => { + if (keys && !keys.includes(key)) { + throw new UploadError({ + en: `Unknown key '${key}' in RECORD: expected one of ${keys.join('/')}`, + es: `Clave desconocida '${key}' en RECORD: se esperaba una de ${keys.join('/')}`, + fr: `Clé inconnue '${key}' dans RECORD : une des clés ${keys.join('/')} était attendue` + }); + } + return parseValue(value); + }); +} + +function formatRecordSample(keys: null | readonly string[], valueSample: string) { + const fields = keys ? keys.map((key) => `${key}: ${valueSample}`).join(', ') : `key: ${valueSample}, ...`; + return `RECORD(${fields})`; +} + +function unsupportedRecordKeyError() { + return new UploadError({ + en: 'Unsupported Zod record: keys must be an enum or strings', + es: 'Registro de Zod no compatible: las claves deben ser un enum o cadenas', + fr: 'Enregistrement Zod non pris en charge : les clés doivent être un enum ou des chaînes' + }); +} + +function unsupportedRecordValueError(typeName: string) { + return new UploadError({ + en: `Unsupported Zod record: values must be numbers or fixed choices, not '${typeName}'`, + es: `Registro de Zod no compatible: los valores deben ser números u opciones fijas, no '${typeName}'`, + fr: `Enregistrement Zod non pris en charge : les valeurs doivent être des nombres ou des choix fixes, pas '${typeName}'` }); } @@ -198,13 +330,35 @@ export namespace Zod3 { type ZodObjectArrayDef = z3.ZodArrayDef & { type: z3.AnyZodObject }; - export type ZodTypeNameResult = { - enumValues?: readonly string[]; - isOptional: boolean; - multiKeys?: string[]; - multiValues?: ZodTypeNameResult[]; - typeName: RequiredZodTypeName; - }; + type OptionZodTypeName = Extract; + + export type ZodTypeNameResult = + | { + enumValues: readonly LiteralOption[]; + isOptional: boolean; + typeName: OptionZodTypeName; + } + | { + enumValues?: readonly string[]; + isOptional: boolean; + multiKeys?: string[]; + multiValues?: ZodTypeNameResult[]; + typeName: Exclude; + } + | { + isOptional: boolean; + recordKeys: null | readonly string[]; + recordValue: ZodTypeNameResult; + typeName: 'ZodRecord'; + }; + + const RECORD_VALUE_TYPE_NAMES: readonly RequiredZodTypeName[] = [ + 'ZodEnum', + 'ZodLiteral', + 'ZodNativeEnum', + 'ZodNumber', + 'ZodUnion' + ]; function isZodEffects(value: unknown): value is z3.ZodEffects { return isObjectLike(value) && value.constructor.name === 'ZodEffects'; @@ -218,6 +372,10 @@ export namespace Zod3 { return isObjectLike(value) && value.constructor.name === 'ZodObject'; } + function hasTypeName(value: unknown): value is { typeName: string } { + return isPlainObject(value) && typeof value.typeName === 'string'; + } + function isZodTypeDef(value: unknown): value is AnyZodTypeDef { return isPlainObject(value) && ZOD_TYPE_NAMES.includes(value.typeName as ZodTypeName); } @@ -230,6 +388,26 @@ export namespace Zod3 { return def.typeName === z3.ZodFirstPartyTypeKind.ZodEnum; } + function isZodNativeEnumDef(def: AnyZodTypeDef): def is z3.ZodNativeEnumDef { + return def.typeName === z3.ZodFirstPartyTypeKind.ZodNativeEnum; + } + + function isZodLiteralDef(def: AnyZodTypeDef): def is z3.ZodLiteralDef { + return def.typeName === z3.ZodFirstPartyTypeKind.ZodLiteral; + } + + function isZodUnionDef(def: AnyZodTypeDef): def is z3.ZodUnionDef { + return def.typeName === z3.ZodFirstPartyTypeKind.ZodUnion; + } + + function isZodRecordDef(def: AnyZodTypeDef): def is z3.ZodRecordDef { + return def.typeName === z3.ZodFirstPartyTypeKind.ZodRecord; + } + + function isZodStringDef(def: AnyZodTypeDef): def is z3.ZodStringDef { + return def.typeName === z3.ZodFirstPartyTypeKind.ZodString; + } + function isZodSetDef(def: AnyZodTypeDef): def is z3.ZodSetDef { return def.typeName === z3.ZodFirstPartyTypeKind.ZodSet; } @@ -246,6 +424,32 @@ export namespace Zod3 { return def.typeName === z3.ZodFirstPartyTypeKind.ZodEffects; } + function literalValueOfUnionOption(option: z3.ZodTypeAny): LiteralOption { + const def: unknown = option._def; + if (!(isZodTypeDef(def) && isZodLiteralDef(def))) { + throw nonLiteralUnionError(); + } + return toLiteralOption(def.value); + } + + function recordKeysOf(keyType: z3.ZodTypeAny): null | readonly string[] { + const def: unknown = keyType._def; + if (isZodTypeDef(def) && isZodEnumDef(def)) { + return def.values; + } else if (isZodTypeDef(def) && isZodStringDef(def)) { + return null; + } + throw unsupportedRecordKeyError(); + } + + function recordValueOf(valueType: z3.ZodTypeAny): ZodTypeNameResult { + const result = getZodTypeName(valueType); + if (!RECORD_VALUE_TYPE_NAMES.includes(result.typeName)) { + throw unsupportedRecordValueError(result.typeName); + } + return result; + } + function interpretZodArray(def: ZodObjectArrayDef, isOptional?: boolean): ZodTypeNameResult { const listOfZodElements: ZodTypeNameResult[] = []; const listOfZodKeys: string[] = []; @@ -253,18 +457,8 @@ export namespace Zod3 { const shape = def.type.shape as { [key: string]: z3.ZodTypeAny }; for (const [key, insideType] of Object.entries(shape)) { - const def: unknown = insideType._def; - if (isZodTypeDef(def)) { - const innerTypeName = getZodTypeName(insideType); - listOfZodElements.push(innerTypeName); - listOfZodKeys.push(key); - } else { - console.error({ def }); - throw new UploadError({ - en: `Unhandled case!`, - fr: `Cas non géré !` - }); - } + listOfZodElements.push(getZodTypeName(insideType)); + listOfZodKeys.push(key); } if (listOfZodElements.length === 0) { throw new UploadError({ @@ -283,8 +477,7 @@ export namespace Zod3 { export function getZodTypeName(schema: z3.ZodTypeAny, isOptional?: boolean): ZodTypeNameResult { const def: unknown = schema._def; if (!isZodTypeDef(def)) { - console.error(`Cannot parse ZodType from schema: ${JSON.stringify(schema)}`); - throw new UploadError({ en: 'Unexpected Error', fr: 'Erreur inattendue' }); + throw UnexpectedZodTypeError.forTypeName(hasTypeName(def) ? def.typeName : undefined); } else if (isZodOptionalDef(def)) { return getZodTypeName(def.innerType, true); } else if (isZodEnumDef(def)) { @@ -293,6 +486,31 @@ export namespace Zod3 { isOptional: Boolean(isOptional), typeName: def.typeName }; + } else if (isZodNativeEnumDef(def)) { + return { + enumValues: enumLikeValues(def.values), + isOptional: Boolean(isOptional), + typeName: def.typeName + }; + } else if (isZodLiteralDef(def)) { + return { + enumValues: [toLiteralOption(def.value)], + isOptional: Boolean(isOptional), + typeName: def.typeName + }; + } else if (isZodUnionDef(def)) { + return { + enumValues: def.options.map(literalValueOfUnionOption), + isOptional: Boolean(isOptional), + typeName: def.typeName + }; + } else if (isZodRecordDef(def)) { + return { + isOptional: Boolean(isOptional), + recordKeys: recordKeysOf(def.keyType), + recordValue: recordValueOf(def.valueType), + typeName: def.typeName + }; } else if (isZodObjectArrayDef(def)) { return interpretZodArray(def, isOptional); } else if (isZodSetDef(def)) { @@ -313,28 +531,33 @@ export namespace Zod3 { } return { isOptional: Boolean(isOptional), - typeName: def.typeName satisfies ZodTypeName as RequiredZodTypeName + typeName: def.typeName satisfies ZodTypeName as Exclude }; } - export function generateSampleData({ - enumValues, - isOptional, - multiKeys, - multiValues, - typeName - }: ZodTypeNameResult): string { + export function generateSampleData(result: ZodTypeNameResult): string { + const { isOptional, typeName } = result; switch (typeName) { case 'ZodBoolean': return formatOptionalEntry('true/false', isOptional); case 'ZodDate': return formatOptionalEntry('yyyy-mm-dd', isOptional); + case 'ZodEnum': + case 'ZodLiteral': + case 'ZodNativeEnum': + case 'ZodUnion': + return formatOptionalEntry(formatOptions(result.enumValues), isOptional); case 'ZodNumber': return formatOptionalEntry('number', isOptional); + case 'ZodRecord': + return formatOptionalEntry( + formatRecordSample(result.recordKeys, generateSampleData(result.recordValue)), + isOptional + ); case 'ZodSet': try { - if (enumValues) { - return formatOptionalEntry(`SET(${enumValues.join('/')}, ...)`, isOptional); + if (result.enumValues) { + return formatOptionalEntry(`SET(${result.enumValues.join('/')}, ...)`, isOptional); } return formatOptionalEntry('SET(a,b,c)', isOptional); } catch { @@ -345,29 +568,10 @@ export namespace Zod3 { } case 'ZodString': return formatOptionalEntry('string', isOptional); - case 'ZodEnum': - try { - let possibleEnumOutputs = ''; - if (!enumValues) { - throw new UploadError({ - en: 'Enum values do not exist', - fr: `Values d'Enum n'existe pas` - }); - } - for (const val of enumValues) { - possibleEnumOutputs += `${val}/`; - } - possibleEnumOutputs = possibleEnumOutputs.slice(0, -1); - return formatOptionalEntry(possibleEnumOutputs, isOptional); - } catch { - throw new UploadError({ - en: 'Invalid Enum error', - fr: 'Erreur Enum invalide' - }); - } case 'ZodArray': case 'ZodObject': try { + const { multiKeys, multiValues } = result; let multiString = 'RECORD_ARRAY( '; if (!(multiValues && multiKeys)) { throw new UploadError({ @@ -405,10 +609,12 @@ export namespace Zod3 { if (entry === '' && convertResult.isOptional) { return undefined; } - switch (convertResult.typeName) { + const { typeName } = convertResult; + switch (typeName) { case 'ZodArray': - case 'ZodObject': - if (!convertResult.multiKeys || !convertResult.multiValues) { + case 'ZodObject': { + const { multiKeys, multiValues } = convertResult; + if (!multiKeys || !multiValues) { throw new UploadError({ en: 'Record Array keys or values do not exist', fr: "Les clés ou valeurs du tableau d'enregistrements n'existent pas" @@ -417,29 +623,37 @@ export namespace Zod3 { return extractRecordArrayEntry(entry).map((parsedRecord) => { const result: { [key: string]: unknown } = {}; - convertResult.multiKeys!.forEach((key, index) => { + multiKeys.forEach((key, index) => { const rawValue = parsedRecord[key]; if (rawValue !== undefined) { - result[key] = interpretZodConvertResult(convertResult.multiValues![index]!, rawValue); + result[key] = interpretZodConvertResult(multiValues[index]!, rawValue); } }); return result; }); + } case 'ZodBoolean': return parseBooleanEntry(entry); case 'ZodDate': return parseDateEntry(entry); case 'ZodEnum': - return entry; + case 'ZodLiteral': + case 'ZodNativeEnum': + case 'ZodUnion': + return parseOptionEntry(entry, convertResult.enumValues); case 'ZodNumber': return parseNumberEntry(entry); + case 'ZodRecord': + return parseRecordEntry(entry, convertResult.recordKeys, (value) => { + return interpretZodConvertResult(convertResult.recordValue, value); + }); case 'ZodSet': return parseSetEntry(entry); case 'ZodString': return entry; default: - throw UnexpectedZodTypeError.forTypeName(convertResult.typeName); + throw UnexpectedZodTypeError.forTypeName(typeName); } } @@ -463,8 +677,7 @@ export namespace Zod3 { const sampleData = [...INTERNAL_HEADERS_SAMPLE_DATA]; for (const col of columnNames) { - const typeNameResult = getZodTypeName(shape[col]!); - sampleData.push(generateSampleData(typeNameResult)); + sampleData.push(sampleDataForColumn(col, () => generateSampleData(getZodTypeName(shape[col]!)))); } return { @@ -592,12 +805,7 @@ export namespace Zod3 { jsonLine[headers[i]!] = result; } catch (error: unknown) { if (error instanceof UploadError) { - reject( - new UploadError({ - en: `${error.description.en} at column name: '${key}' and row number '${rowNumber}'`, - fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}'` - }) - ); + reject(locateUploadError(error, { column: key, row: rowNumber })); return; } console.error(error); @@ -635,12 +843,23 @@ export namespace Zod3 { export namespace Zod4 { type TypeName = Extract< z4.core.$ZodTypeDef['type'], - 'array' | 'boolean' | 'date' | 'enum' | 'number' | 'object' | 'optional' | 'set' | 'string' + | 'array' + | 'boolean' + | 'date' + | 'enum' + | 'literal' + | 'number' + | 'object' + | 'optional' + | 'record' + | 'set' + | 'string' + | 'union' >; type BaseZodConvertResult = { isOptional: boolean; - typeName: Exclude; + typeName: Exclude; }; type ArrayZodConvertResult = Merge< @@ -654,8 +873,8 @@ export namespace Zod4 { type EnumZodConvertResult = Merge< BaseZodConvertResult, { - enumValues: readonly string[]; - typeName: 'enum'; + enumValues: readonly LiteralOption[]; + typeName: 'enum' | 'literal' | 'union'; } >; @@ -669,6 +888,15 @@ export namespace Zod4 { } >; + type RecordZodConvertResult = Merge< + BaseZodConvertResult, + { + keys: null | readonly string[]; + typeName: 'record'; + valueType: ZodConvertResult; + } + >; + type SetZodConvertResult = Merge< BaseZodConvertResult, { @@ -678,7 +906,38 @@ export namespace Zod4 { >; type ZodConvertResult = - ArrayZodConvertResult | BaseZodConvertResult | EnumZodConvertResult | ObjectZodConvertResult | SetZodConvertResult; + | ArrayZodConvertResult + | BaseZodConvertResult + | EnumZodConvertResult + | ObjectZodConvertResult + | RecordZodConvertResult + | SetZodConvertResult; + + const RECORD_VALUE_TYPE_NAMES: readonly TypeName[] = ['enum', 'literal', 'number', 'union']; + + function recordKeysOf(keyType: unknown): null | readonly string[] { + if (keyType instanceof z4.ZodEnum) { + return enumLikeValues(keyType.def.entries).map(String); + } else if (keyType instanceof z4.ZodString) { + return null; + } + throw unsupportedRecordKeyError(); + } + + function recordValueOf(valueType: unknown): ZodConvertResult { + const result = parseZodSchema(valueType); + if (!RECORD_VALUE_TYPE_NAMES.includes(result.typeName)) { + throw unsupportedRecordValueError(result.typeName); + } + return result; + } + + function literalValuesOfUnionOption(option: unknown): LiteralOption[] { + if (!(option instanceof z4.ZodLiteral)) { + throw nonLiteralUnionError(); + } + return option.def.values.map(toLiteralOption); + } function parseZodSchema(schema: unknown, isOptional = false): ZodConvertResult { switch (true) { @@ -711,7 +970,19 @@ export namespace Zod4 { }; case schema instanceof z4.ZodEnum: return { - enumValues: schema.options.map(String), + enumValues: enumLikeValues(schema.def.entries), + isOptional, + typeName: schema.def.type + }; + case schema instanceof z4.ZodLiteral: + return { + enumValues: schema.def.values.map(toLiteralOption), + isOptional, + typeName: schema.def.type + }; + case schema instanceof z4.ZodUnion: + return { + enumValues: schema.def.options.flatMap(literalValuesOfUnionOption), isOptional, typeName: schema.def.type }; @@ -720,6 +991,13 @@ export namespace Zod4 { isOptional, typeName: schema.def.type }; + case schema instanceof z4.ZodRecord: + return { + isOptional, + keys: recordKeysOf(schema.def.keyType), + typeName: schema.def.type, + valueType: recordValueOf(schema.def.valueType) + }; case schema instanceof z4.ZodSet: return { innerType: parseZodSchema(schema.def.valueType), @@ -763,9 +1041,15 @@ export namespace Zod4 { case 'date': return parseDateEntry(entry); case 'enum': - return entry; + case 'literal': + case 'union': + return parseOptionEntry(entry, convertResult.enumValues); case 'number': return parseNumberEntry(entry); + case 'record': + return parseRecordEntry(entry, convertResult.keys, (value) => { + return interpretZodConvertResult(convertResult.valueType, value); + }); case 'set': return parseSetEntry(entry); case 'string': @@ -784,14 +1068,10 @@ export namespace Zod4 { return formatOptionalEntry('true/false', convertResult.isOptional); case 'date': return formatOptionalEntry('yyyy-mm-dd', convertResult.isOptional); - case 'enum': { - let possibleEnumOutputs = ''; - for (const val of convertResult.enumValues) { - possibleEnumOutputs += `${val}/`; - } - possibleEnumOutputs = possibleEnumOutputs.slice(0, -1); - return formatOptionalEntry(possibleEnumOutputs, convertResult.isOptional); - } + case 'enum': + case 'literal': + case 'union': + return formatOptionalEntry(formatOptions(convertResult.enumValues), convertResult.isOptional); case 'number': return formatOptionalEntry('number', convertResult.isOptional); case 'object': { @@ -805,6 +1085,11 @@ export namespace Zod4 { }); return output; } + case 'record': + return formatOptionalEntry( + formatRecordSample(convertResult.keys, generateSampleData(convertResult.valueType)), + convertResult.isOptional + ); case 'set': { const innerTypeSample = generateSampleData(convertResult.innerType); return formatOptionalEntry(`SET(${innerTypeSample})`, convertResult.isOptional); @@ -832,7 +1117,7 @@ export namespace Zod4 { Object.entries(instrumentSchema.shape).forEach(([key, subschema]) => { csvColumns.push(key); - sampleData.push(generateSampleData(parseZodSchema(subschema))); + sampleData.push(sampleDataForColumn(key, () => generateSampleData(parseZodSchema(subschema)))); }); return { @@ -949,12 +1234,7 @@ export namespace Zod4 { jsonLine[headers[i]!] = result; } catch (error: unknown) { if (error instanceof UploadError) { - reject( - new UploadError({ - en: `${error.description.en} at column name: '${key}' and row number '${rowNumber}'`, - fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}'` - }) - ); + reject(locateUploadError(error, { column: key, row: rowNumber })); return; } console.error(error); diff --git a/docs/en/3-guides/3.3-how-to-upload-data.mdx b/docs/en/3-guides/3.3-how-to-upload-data.mdx index 6b6048dd2..ac02a51f8 100644 --- a/docs/en/3-guides/3.3-how-to-upload-data.mdx +++ b/docs/en/3-guides/3.3-how-to-upload-data.mdx @@ -49,6 +49,14 @@ Upload is an experimental feature. The **Upload Data** item appears in the navig Some forms may have Zod field types that are challenging to represent in a standard CSV format. Therefore, specific notations must be used to ensure the upload feature can properly parse the data. + - **_Fixed-Choice Data_** + + If a form has a field that accepts only certain values, such as a score of 0, 1, 2 or 3, the template lists them separated by "/": + + - 0/1/2/3 + + Enter exactly one of the listed values, written as it appears in the template. Any other value is rejected, and the error names its column and row. + - **_Set Data_** If a form has a field that accepts a set of data, such as a multi-option checklist with options a, b, and c, the proper notation for the data would be: @@ -71,6 +79,14 @@ Upload is an experimental feature. The **Upload Data** item appears in the navig Note that every "," notates a new entry in the record, whilst every ";" notates a new record in the record array + - **_Record Data_** + + If a form has a field that asks the same question about a fixed list of items, such as a mood rating for monday, wednesday and friday, the proper notation gives each item and its value separated by ":": + + - RECORD(monday: 3, wednesday: 4, friday: 2) + + The template lists the items and the values each one accepts, for example `RECORD(monday: 1/2/3/4, wednesday: 1/2/3/4, friday: 1/2/3/4)`. An item that is not in the template is rejected. + - **_Optional Data_** Whenever data is optional within a form it can be left empty and still be properly submitted. Any optional data column in the csv template will have an "(optional)" tag attached to its sample data entry, which means any entry in the column can be left blank. This is usually done in that case of when one column value depends on another. For example, a sessionFailed column would be followed by an optional reasonForFailure column. diff --git a/testing/src/pages/_app/upload/$instrumentId.page.ts b/testing/src/pages/_app/upload/$instrumentId.page.ts index bbca041d1..6772cca6b 100644 --- a/testing/src/pages/_app/upload/$instrumentId.page.ts +++ b/testing/src/pages/_app/upload/$instrumentId.page.ts @@ -7,6 +7,8 @@ export class UploadInstrumentPage extends AppPage { readonly fileInput: Locator; readonly pageHeader: Locator; readonly submitButton: Locator; + readonly templateButton: Locator; + readonly templateErrorHeading: Locator; readonly tryAgainButton: Locator; constructor(page: Page) { @@ -16,6 +18,8 @@ export class UploadInstrumentPage extends AppPage { this.submitButton = page.getByRole('button', { name: 'Submit' }); this.errorHeading = page.getByRole('heading', { name: 'An error has happened within the request' }); this.tryAgainButton = page.getByRole('button', { name: 'Try Again' }); + this.templateButton = page.getByRole('button', { name: 'Template' }); + this.templateErrorHeading = page.getByRole('heading', { name: 'Error Occurred Downloading Sample Template' }); } async uploadFile(file: Parameters[0]) { diff --git a/testing/src/specs/upload.spec.ts b/testing/src/specs/upload.spec.ts index 1783e9027..15d6a88f3 100644 --- a/testing/src/specs/upload.spec.ts +++ b/testing/src/specs/upload.spec.ts @@ -1,3 +1,5 @@ +import { readFile } from 'node:fs/promises'; + import type { UploadInstrumentRecordsData } from '@opendatacapture/schemas/instrument-records'; import { UploadInstrumentPage } from '../pages/_app/upload/$instrumentId.page'; @@ -74,7 +76,23 @@ test.describe('upload', () => { uniqueId }) => { const title = `Record Array Upload ${uniqueId}`; - await api.createInstrument(recordArrayFormBundle({ name: `RECORD_ARRAY_UPLOAD_${uniqueId}`, title })); + await api.createInstrument( + formBundle({ + content: `{ + items: { + fieldset: { + name: { kind: 'string', label: 'Name', variant: 'input' }, + quantity: { kind: 'number', label: 'Quantity', variant: 'input' } + }, + kind: 'record-array', + label: 'Items' + } + }`, + name: `RECORD_ARRAY_UPLOAD_${uniqueId}`, + title, + validationSchema: 'z.object({ items: z.array(z.object({ name: z.string(), quantity: z.number() })) })' + }) + ); await isolatedGroupManager(); const csv = [ 'subjectID,date,items', @@ -99,6 +117,208 @@ test.describe('upload', () => { ).toBeVisible(); }); + // The template once failed with "Unexpected Error" for any field it could not describe, which named + // neither the field nor its type. + test('should name the column and Zod type of a field the template cannot describe', async ({ + api, + isolatedGroupManager, + page, + uniqueId + }) => { + const title = `Unsupported Field Upload ${uniqueId}`; + await api.createInstrument( + formBundle({ + content: `{ pair: { kind: 'string', label: 'Pair', variant: 'input' } }`, + name: `UNSUPPORTED_FIELD_UPLOAD_${uniqueId}`, + title, + validationSchema: 'z.object({ pair: z.tuple([z.string()]) })' + }) + ); + await isolatedGroupManager(); + + const uploadPage = new UploadPage(page); + await uploadPage.goto('/upload'); + await uploadPage.selectInstrument(title); + await page.waitForURL('**/upload/**'); + + const uploadInstrumentPage = new UploadInstrumentPage(page); + await expect(uploadInstrumentPage.pageHeader).toContainText(title); + await uploadInstrumentPage.templateButton.click(); + + await expect(uploadInstrumentPage.templateErrorHeading).toBeVisible(); + await expect(page.getByText("Unsupported Zod type 'ZodTuple' at column name: 'pair'")).toBeVisible(); + }); + + // A count recorded as a union of literals, or a native enum, once failed both the template and the + // import. Instruments reach the page built against either zod major, and each has its own path. + for (const zodEntry of ['index.js', 'v4.js'] as const) { + test.describe(`fixed-choice fields of a form written against zod@3.x/${zodEntry}`, () => { + let title: string; + + test.beforeEach(async ({ api, uniqueId }) => { + title = `Fixed Choice Upload ${uniqueId}`; + await api.createInstrument( + formBundle({ + content: `{ + putbacks: { kind: 'number', label: 'Putbacks', variant: 'input' }, + severity: { kind: 'number', label: 'Severity', variant: 'input' } + }`, + name: `FIXED_CHOICE_UPLOAD_${uniqueId}`, + title, + validationSchema: `z.object({ + putbacks: z.union([z.literal(0), z.literal(1), z.literal(2), z.literal(3)]), + severity: z.nativeEnum({ 0: 'Mild', 1: 'Severe', Mild: 0, Severe: 1 }) + })`, + zodEntry + }) + ); + }); + + test('should list the allowed values of each field in the template', async ({ isolatedGroupManager, page }) => { + await isolatedGroupManager(); + const uploadPage = new UploadPage(page); + await uploadPage.goto('/upload'); + await uploadPage.selectInstrument(title); + await page.waitForURL('**/upload/**'); + + const uploadInstrumentPage = new UploadInstrumentPage(page); + const downloadPromise = page.waitForEvent('download'); + await uploadInstrumentPage.templateButton.click(); + const download = await downloadPromise; + + const template = (await readFile(await download.path())).toString(); + expect(template).toBe('subjectID,date,putbacks,severity\r\nstring,yyyy-mm-dd,0/1/2/3,0/1'); + }); + + test('should store an uploaded value as the number its schema declares', async ({ + adminToken, + apiRequestContext, + isolatedGroupManager, + page, + uniqueId + }) => { + const group = await isolatedGroupManager(); + const csv = ['subjectID,date,putbacks,severity', `FixedChoice${uniqueId},2024-01-15,2,1`].join('\n'); + + const uploadPage = new UploadPage(page); + await uploadPage.goto('/upload'); + await uploadPage.selectInstrument(title); + await page.waitForURL('**/upload/**'); + + const uploadInstrumentPage = new UploadInstrumentPage(page); + await uploadInstrumentPage.uploadFile({ buffer: Buffer.from(csv), mimeType: 'text/csv', name: 'choices.csv' }); + await uploadInstrumentPage.submitButton.click(); + await expect(page.getByText('Success')).toBeVisible(); + + const records = await apiRequestContext.get(`/api/v1/instrument-records?groupId=${group.id}`, { + headers: { Authorization: `Bearer ${adminToken}` } + }); + expect(await records.json()).toMatchObject([{ data: { putbacks: 2, severity: 1 } }]); + }); + + test('should reject a value outside the allowed ones, naming its column and row', async ({ + isolatedGroupManager, + page, + uniqueId + }) => { + await isolatedGroupManager(); + const csv = ['subjectID,date,putbacks,severity', `FixedChoice${uniqueId},2024-01-15,7,1`].join('\n'); + + const uploadPage = new UploadPage(page); + await uploadPage.goto('/upload'); + await uploadPage.selectInstrument(title); + await page.waitForURL('**/upload/**'); + + const uploadInstrumentPage = new UploadInstrumentPage(page); + await uploadInstrumentPage.uploadFile({ buffer: Buffer.from(csv), mimeType: 'text/csv', name: 'choices.csv' }); + await uploadInstrumentPage.submitButton.click(); + + await expect(uploadInstrumentPage.errorHeading).toBeVisible(); + await expect( + page.getByText("Invalid value '7': expected one of 0/1/2/3 at column name: 'putbacks' and row number '1'") + ).toBeVisible(); + }); + }); + } + + // The instrument guide validates a number-record field with a record keyed by an enum, which the + // page could neither describe nor import. + for (const zodEntry of ['index.js', 'v4.js'] as const) { + test.describe(`record field of a form written against zod@3.x/${zodEntry}`, () => { + let title: string; + + test.beforeEach(async ({ api, uniqueId }) => { + title = `Record Upload ${uniqueId}`; + await api.createInstrument( + formBundle({ + content: `{ + ratings: { + items: { friday: { label: 'Friday' }, monday: { label: 'Monday' } }, + kind: 'number-record', + label: 'Ratings', + options: { 1: 'Poor', 2: 'Fair', 3: 'Good' }, + variant: 'likert' + } + }`, + name: `RECORD_UPLOAD_${uniqueId}`, + title, + validationSchema: `z.object({ + ratings: z.record(z.enum(['monday', 'friday']), z.union([z.literal(1), z.literal(2), z.literal(3)])) + })`, + zodEntry + }) + ); + }); + + test('should show every key of the record with its allowed values in the template', async ({ + isolatedGroupManager, + page + }) => { + await isolatedGroupManager(); + const uploadPage = new UploadPage(page); + await uploadPage.goto('/upload'); + await uploadPage.selectInstrument(title); + await page.waitForURL('**/upload/**'); + + const uploadInstrumentPage = new UploadInstrumentPage(page); + const downloadPromise = page.waitForEvent('download'); + await uploadInstrumentPage.templateButton.click(); + const download = await downloadPromise; + + const template = (await readFile(await download.path())).toString(); + expect(template).toBe('subjectID,date,ratings\r\nstring,yyyy-mm-dd,"RECORD(monday: 1/2/3, friday: 1/2/3)"'); + }); + + test('should store an uploaded RECORD cell as an object of numbers', async ({ + adminToken, + apiRequestContext, + isolatedGroupManager, + page, + uniqueId + }) => { + const group = await isolatedGroupManager(); + const csv = ['subjectID,date,ratings', `Record${uniqueId},2024-01-15,"RECORD(monday: 3, friday: 1)"`].join( + '\n' + ); + + const uploadPage = new UploadPage(page); + await uploadPage.goto('/upload'); + await uploadPage.selectInstrument(title); + await page.waitForURL('**/upload/**'); + + const uploadInstrumentPage = new UploadInstrumentPage(page); + await uploadInstrumentPage.uploadFile({ buffer: Buffer.from(csv), mimeType: 'text/csv', name: 'ratings.csv' }); + await uploadInstrumentPage.submitButton.click(); + await expect(page.getByText('Success')).toBeVisible(); + + const records = await apiRequestContext.get(`/api/v1/instrument-records?groupId=${group.id}`, { + headers: { Authorization: `Bearer ${adminToken}` } + }); + expect(await records.json()).toMatchObject([{ data: { ratings: { friday: 1, monday: 3 } } }]); + }); + }); + } + // The response was previously every record in the group for the instrument, so a second upload // leaked the first one's records back to the caller. An earlier upload to the same group and // instrument is what distinguishes the scoped response from the group-wide one. @@ -173,28 +393,33 @@ test.describe('upload instrument selection while loading', () => { }); /** - * A form with one record-array field, written as the bundle the api stores: an expression that - * evaluates to the instrument. Its runtime import resolves through the `__resolveImport` the api's - * sandbox provides, and against the web origin in the browser, as a bundled import would. + * A form written as the bundle the api stores: an expression that evaluates to the instrument. Its + * runtime import resolves through the `__resolveImport` the api's sandbox provides, and against the + * web origin in the browser, as a bundled import would. `content` and `validationSchema` are source + * text, the schema written against the `z` of `zodEntry`: `index.js` is the zod 3 API and `v4.js` + * the zod 4 one. */ -function recordArrayFormBundle({ name, title }: { name: string; title: string }) { +function formBundle({ + content, + name, + title, + validationSchema, + zodEntry = 'index.js' +}: { + content: string; + name: string; + title: string; + validationSchema: string; + zodEntry?: 'index.js' | 'v4.js'; +}) { return `(async () => { const runtimeImport = (specifier) => import(globalThis.__resolveImport?.(specifier) ?? specifier); - const { z } = await runtimeImport('/runtime/v1/zod@3.x/index.js'); + const { z } = await runtimeImport('/runtime/v1/zod@3.x/${zodEntry}'); return { __runtimeVersion: 1, - content: { - items: { - fieldset: { - name: { kind: 'string', label: 'Name', variant: 'input' }, - quantity: { kind: 'number', label: 'Quantity', variant: 'input' } - }, - kind: 'record-array', - label: 'Items' - } - }, + content: ${content}, details: { - description: 'A form with one record-array field', + description: 'A form seeded by the upload spec', license: 'Apache-2.0', title: ${JSON.stringify(title)} }, @@ -203,7 +428,7 @@ function recordArrayFormBundle({ name, title }: { name: string; title: string }) language: 'en', measures: null, tags: [], - validationSchema: z.object({ items: z.array(z.object({ name: z.string(), quantity: z.number() })) }) + validationSchema: ${validationSchema} }; })()`; }