chore: add zod4 process instrument csv and reformatInstrument data methods

david-roper · david-roper · commit 449c2ad5a4cc · 2025-09-09T11:24:03.000-04:00
diff --git a/apps/web/src/utils/upload2.ts b/apps/web/src/utils/upload2.ts
@@ -1,7 +1,11 @@
 /* eslint-disable @typescript-eslint/no-namespace */
 import { isNumberLike, isObjectLike, isPlainObject, isZodType, parseNumber } from '@douglasneuroinformatics/libjs';
 import type { Language } from '@douglasneuroinformatics/libui/i18n';
-import type { AnyUnilingualFormInstrument, FormTypes } from '@opendatacapture/runtime-core';
+import type { AnyUnilingualFormInstrument, FormTypes, Json } from '@opendatacapture/runtime-core';
+import type { Group } from '@opendatacapture/schemas/group';
+import type { UnilingualInstrumentInfo } from '@opendatacapture/schemas/instrument';
+import type { UploadInstrumentRecordsData } from '@opendatacapture/schemas/instrument-records';
+import { encodeScopedSubjectId } from '@opendatacapture/subject-utils';
 import { parse, unparse } from 'papaparse';
 import { z as z3 } from 'zod/v3';
 import { z as z4 } from 'zod/v4';
@@ -84,7 +88,7 @@ function extractRecordArrayEntry(entry: string) {
 }
 
 namespace Zod3 {
-  type ZodTypeName = Extract<`${z3.ZodFirstPartyTypeKind}`, (typeof ZOD_TYPE_NAMES)[number]>;
+  export type ZodTypeName = Extract<`${z3.ZodFirstPartyTypeKind}`, (typeof ZOD_TYPE_NAMES)[number]>;
 
   export type RequiredZodTypeName = Exclude<ZodTypeName, 'ZodEffects' | 'ZodOptional'>;
 
@@ -174,7 +178,7 @@ namespace Zod3 {
     };
   }
 
-  function getZodTypeName(schema: z3.ZodTypeAny, isOptional?: boolean): ZodTypeNameResult {
+  export function getZodTypeName(schema: z3.ZodTypeAny, isOptional?: boolean): ZodTypeNameResult {
     const def: unknown = schema._def;
     if (!isZodTypeDef(def)) {
       console.error(`Cannot parse ZodType from schema: ${JSON.stringify(schema)}`);
@@ -322,15 +326,23 @@ namespace Zod3 {
   }
 
   //new processInstrumentCSV code
-  export function processInstrumentCSV(input: File, instrument: AnyUnilingualFormInstrument) {
+  export function processInstrumentCSV(
+    input: File,
+    instrument: AnyUnilingualFormInstrument
+  ): Promise<UploadOperationResult<FormTypes.Data[]>> {
     const instrumentSchema = instrument.validationSchema as z3.AnyZodObject;
     let shape: { [key: string]: z3.ZodTypeAny } = {};
 
     let instrumentSchemaWithInternal: z3.AnyZodObject;
     const instrumentSchemaDef: unknown = instrumentSchema._def;
     if (isZodTypeDef(instrumentSchemaDef) && isZodEffectsDef(instrumentSchemaDef)) {
       if (!isZodObject(instrumentSchemaDef.schema)) {
-        return { message: { en: 'Invalid instrument schema', fr: "Schéma d'instrument invalide" }, success: false };
+        return new Promise<UploadOperationResult<FormTypes.Data[]>>((resolve) => {
+          return resolve({
+            message: { en: 'Invalid instrument schema', fr: "Schéma d'instrument invalide" },
+            success: false
+          });
+        });
       }
       instrumentSchemaWithInternal = instrumentSchemaDef.schema.extend({
         date: z3.coerce.date(),
@@ -799,8 +811,157 @@ namespace Zod4 {
     };
   }
   //to be filled
-  export function processInstrumentCSV(input: File, instrument: AnyUnilingualFormInstrument) {
-    return undefined;
+  export async function processInstrumentCSV(
+    input: File,
+    instrument: AnyUnilingualFormInstrument
+  ): Promise<UploadOperationResult<FormTypes.Data[]>> {
+    const instrumentSchema = instrument.validationSchema as z4.ZodObject;
+    let shape: { [key: string]: z3.ZodTypeAny } = {};
+    let instrumentSchemaWithInternal: z4.ZodObject;
+
+    if (instrumentSchema instanceof z4.ZodObject) {
+      instrumentSchemaWithInternal = instrumentSchema.extend({
+        date: z4.coerce.date(),
+        subjectID: z4.string().regex(SUBJECT_ID_REGEX)
+      });
+      shape = instrumentSchemaWithInternal.shape;
+    }
+
+    return new Promise<UploadOperationResult<FormTypes.Data[]>>((resolve) => {
+      const reader = new FileReader();
+      reader.onload = () => {
+        const text = reader.result as string;
+        const parseResultCsv = parse<string[]>(text, {
+          header: false,
+          skipEmptyLines: true
+        });
+
+        const [headers, ...dataLines] = parseResultCsv.data;
+
+        //remove sample data if included
+        if (dataLines[0]?.[0]?.startsWith(MONGOLIAN_VOWEL_SEPARATOR)) {
+          dataLines.shift();
+        }
+
+        if (dataLines.length === 0) {
+          return resolve({
+            message: { en: 'data lines is empty array', fr: 'les lignes de données sont un tableau vide' },
+            success: false
+          });
+        }
+
+        const result: FormTypes.Data[] = [];
+
+        if (!headers?.length) {
+          return resolve({
+            message: {
+              en: 'headers is undefined or empty array',
+              fr: 'les en-têtes ne sont pas définis ou constituent un tableau vide'
+            },
+            success: false
+          });
+        }
+        let rowNumber = 1;
+        for (const elements of dataLines) {
+          const jsonLine: { [key: string]: unknown } = {};
+          for (let j = 0; j < headers.length; j++) {
+            const key = headers[j]!.trim();
+            const rawValue = elements[j]!.trim();
+
+            if (rawValue === '\n') {
+              continue;
+            }
+            if (shape[key] === undefined) {
+              return resolve({
+                message: {
+                  en: `Schema value at column ${j} is not defined! Please check if Column has been edited/deleted from original template`,
+                  fr: `La valeur du schéma à la colonne ${j} n'est pas définie ! Veuillez vérifier si la colonne a été modifiée/supprimée du modèle original`
+                },
+                success: false
+              });
+            }
+            try {
+              const typeNameResult = Zod3.getZodTypeName(shape[key]);
+
+              let interpreterResult: UploadOperationResult<FormTypes.FieldValue> = {
+                message: {
+                  en: 'Could not interpret a correct value',
+                  fr: "Impossible d'interpréter une valeur correcte"
+                },
+                success: false
+              };
+
+              if (typeNameResult.typeName === 'ZodArray' || typeNameResult.typeName === 'ZodObject') {
+                // eslint-disable-next-line max-depth
+                if (typeNameResult.multiKeys && typeNameResult.multiValues) {
+                  interpreterResult = Zod3.interpretZodObjectValue(
+                    rawValue,
+                    typeNameResult.isOptional,
+                    typeNameResult.multiValues,
+                    typeNameResult.multiKeys
+                  );
+                  // TODO - what if this is not the case? Once generics are handled correctly should not be a problem
+                  // Dealt with via else statement for now
+                } else {
+                  interpreterResult.message = {
+                    en: 'Record Array keys do not exist',
+                    fr: "Les clés du tableau d'enregistrements n'existent pas"
+                  };
+                }
+              } else {
+                interpreterResult = Zod3.interpretZodValue(
+                  rawValue,
+                  typeNameResult.typeName,
+                  typeNameResult.isOptional
+                );
+              }
+              // if (!interpreterResult.success) {
+              //   return resolve({
+              //     message: {
+              //       en: `${interpreterResult.message.en} at column name: '${key}' and row number ${rowNumber}`,
+              //       fr: `${interpreterResult.message.fr} au nom de colonne : '${key}' et numéro de ligne ${rowNumber}`
+              //     },
+              //     success: false
+              //   });
+              // }
+              if (interpreterResult.success) jsonLine[headers[j]!] = interpreterResult.value;
+            } catch (error: unknown) {
+              if (error instanceof UploadError) {
+                return resolve({
+                  message: {
+                    en: `${error.description.en} at column name: '${key}' and row number '${rowNumber}'`,
+                    fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}`
+                  },
+                  success: false
+                });
+              } else {
+                return resolve({
+                  message: {
+                    en: `Error parsing CSV`,
+                    fr: `Erreur avec la CSV`
+                  },
+                  success: false
+                });
+              }
+            }
+          }
+
+          const zodCheck = instrumentSchemaWithInternal.safeParse(jsonLine);
+          if (!zodCheck.success) {
+            console.error(zodCheck.error.issues);
+            const zodIssues = zodCheck.error.issues.map((issue) => {
+              return `issue message: \n ${issue.message} \n path: ${issue.path.toString()}`;
+            });
+            console.error(`Failed to parse data: ${JSON.stringify(jsonLine)}`);
+            return resolve({ message: { en: zodIssues.join(), fr: zodIssues.join() }, success: false });
+          }
+          result.push(zodCheck.data as FormTypes.Data);
+          rowNumber++;
+        }
+        resolve({ success: true, value: result });
+      };
+      reader.readAsText(input);
+    });
   }
 }
 
@@ -819,3 +980,32 @@ export function processInstrumentCSV(input: File, instrument: AnyUnilingualFormI
   }
   return Zod3.processInstrumentCSV(input, instrument);
 }
+
+export function reformatInstrumentData({
+  currentGroup,
+  data,
+  instrument
+}: {
+  currentGroup: Group | null;
+  data: FormTypes.Data[];
+  instrument: UnilingualInstrumentInfo;
+}): UploadInstrumentRecordsData {
+  const recordsList: { data: Json; date: Date; subjectId: string }[] = [];
+  for (const dataInfo of data) {
+    const { date: dataDate, subjectID: dataSubjectId, ...restOfData } = dataInfo; // Destructure and extract the rest of the data
+    const createdRecord = {
+      data: restOfData as Json,
+      date: dataDate as Date,
+      subjectId: encodeScopedSubjectId(dataSubjectId as string, {
+        groupName: currentGroup?.name ?? 'root'
+      })
+    };
+    recordsList.push(createdRecord);
+  }
+  const reformatForSending: UploadInstrumentRecordsData = {
+    groupId: currentGroup?.id,
+    instrumentId: instrument.id,
+    records: recordsList
+  };
+  return reformatForSending;
+}
