Skip to content

Commit 06ee538

Browse files
david-roperdavid-roper
committed
feat: initial raw query and some cloning issues
1 parent e23bd8f commit 06ee538

1 file changed

Lines changed: 194 additions & 90 deletions

File tree

‎apps/api/src/instrument-records/instrument-records.service.ts‎

Lines changed: 194 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,13 @@ import { Worker } from 'worker_threads';
55

66
const __dirname = dirname(fileURLToPath(import.meta.url));
77

8-
import { replacer, reviver, yearsPassed } from '@douglasneuroinformatics/libjs';
8+
import { replacer, reviver } from '@douglasneuroinformatics/libjs';
99
import { InjectModel } from '@douglasneuroinformatics/libnest';
1010
import type { Model } from '@douglasneuroinformatics/libnest';
1111
import { linearRegression } from '@douglasneuroinformatics/libstats';
1212
import { BadRequestException, Injectable, NotFoundException, UnprocessableEntityException } from '@nestjs/common';
1313
import type { Json, ScalarInstrument } from '@opendatacapture/runtime-core';
1414
// import { DEFAULT_GROUP_NAME } from '@opendatacapture/schemas/core';
15-
import { $RecordArrayFieldValue } from '@opendatacapture/schemas/instrument';
1615
import type {
1716
CreateInstrumentRecordData,
1817
InstrumentRecord,
@@ -36,18 +35,19 @@ import { SubjectsService } from '@/subjects/subjects.service';
3635

3736
import { InstrumentMeasuresService } from './instrument-measures.service';
3837

39-
import type { InitData } from './thread-types';
38+
import type { InitData, RecordType } from './thread-types';
39+
import { P } from 'ts-pattern';
4040

41-
type ExpandDataType =
42-
| {
43-
measure: string;
44-
measureValue: boolean | Date | number | string | undefined;
45-
success: true;
46-
}
47-
| {
48-
message: string;
49-
success: false;
50-
};
41+
// type ExpandDataType =
42+
// | {
43+
// measure: string;
44+
// measureValue: boolean | Date | number | string | undefined;
45+
// success: true;
46+
// }
47+
// | {
48+
// message: string;
49+
// success: false;
50+
// };
5151

5252
type WorkerMessage = { data: InstrumentRecordsExport; success: true } | { error: string; success: false };
5353

@@ -145,36 +145,82 @@ export class InstrumentRecordsService {
145145
return this.instrumentRecordModel.exists(where);
146146
}
147147

148-
async exportRecords({ groupId }: { groupId?: string } = {}, { ability }: EntityOperationOptions = {}) {
148+
async exportRecords({ groupId: _ }: { groupId?: string } = {}, { ability: __ }: EntityOperationOptions = {}) {
149149
//separate this into seperate queries that are done within the thread (ie find session and subject info in thread instead with prisma model)
150-
const records = await this.instrumentRecordModel.findMany({
151-
include: {
152-
session: {
153-
select: {
154-
date: true,
155-
id: true,
156-
type: true,
157-
user: { select: { username: true } }
158-
}
159-
},
160-
subject: {
161-
select: {
162-
dateOfBirth: true,
163-
groupIds: true,
164-
id: true,
165-
sex: true
166-
}
167-
}
168-
},
169-
where: {
170-
AND: [
171-
{
172-
subject: groupId ? { groupIds: { has: groupId } } : {}
173-
},
174-
accessibleQuery(ability, 'read', 'InstrumentRecord')
175-
]
150+
// const records = await this.instrumentRecordModel.findMany({
151+
// include: {
152+
// session: {
153+
// select: {
154+
// date: true,
155+
// id: true,
156+
// type: true,
157+
// user: { select: { username: true } }
158+
// }
159+
// },
160+
// subject: {
161+
// select: {
162+
// dateOfBirth: true,
163+
// groupIds: true,
164+
// id: true,
165+
// sex: true
166+
// }
167+
// }
168+
// },
169+
// where: {
170+
// AND: [
171+
// {
172+
// subject: groupId ? { groupIds: { has: groupId } } : {}
173+
// },
174+
// accessibleQuery(ability, 'read', 'InstrumentRecord')
175+
// ]
176+
// }
177+
// });
178+
179+
// TBD IMPORTANT - add permissions
180+
181+
const records = await this.queryRecordsRaw();
182+
183+
// console.log(records[0]
184+
// records.forEach((record) => {
185+
// for (const key in record) {
186+
// try {
187+
// structuredClone(record[key])
188+
// } catch (err) {
189+
// console.log(key, record[key], record)
190+
// throw err
191+
// }
192+
// }
193+
// })
194+
// records.map((record) => {
195+
// try{
196+
// structuredClone(record)
197+
// }
198+
// catch {
199+
// console.log(record)
200+
// console.log(Object.getPrototypeOf(record) === Object.prototype)
201+
// console.log(record.computedMeasures)
202+
// throw new Error()
203+
// }
204+
205+
for (let i = 0; i < records.length; i++) {
206+
const record = records[i];
207+
if (Object.getPrototypeOf(record) !== Object.prototype) {
208+
console.log(record);
209+
throw new Error('Bad prototype');
176210
}
177-
});
211+
// for (const key in record) {
212+
// structuredClone(record[key])
213+
// }
214+
records[i] = {
215+
...record
216+
};
217+
}
218+
219+
console.log(records[0]);
220+
221+
// throw new Error("NULL")
222+
structuredClone(records);
223+
178224
const instrumentIds = [...new Set(records.map((r) => r.instrumentId))];
179225

180226
const instrumentsArray = await Promise.all(
@@ -183,35 +229,35 @@ export class InstrumentRecordsService {
183229

184230
const instruments = new Map(instrumentsArray.map((instrument) => [instrument.id, instrument]));
185231

186-
const convertRecords = records.map((record) => {
187-
return {
188-
computedMeasures: record.computedMeasures,
189-
date: record.date.toISOString(),
190-
id: record.id,
191-
instrumentId: record.instrumentId,
192-
session: {
193-
date: record.session.date.toISOString(),
194-
id: record.session.id,
195-
type: record.session.type,
196-
user: {
197-
username: record.session.user?.username
198-
}
199-
},
200-
subject: {
201-
age: record.subject.dateOfBirth ? yearsPassed(record.subject.dateOfBirth) : null,
202-
groupIds: record.subject.groupIds,
203-
id: record.subject.id,
204-
sex: record.subject.sex
205-
}
206-
};
207-
});
232+
// const convertRecords = records.map((record) => {
233+
// return {
234+
// computedMeasures: record.computedMeasures,
235+
// date: record.date.toISOString(),
236+
// id: record.id,
237+
// instrumentId: record.instrumentId,
238+
// session: {
239+
// date: record.session.date.toISOString(),
240+
// id: record.session.id,
241+
// type: record.session.type,
242+
// user: {
243+
// username: record.session.user?.username
244+
// }
245+
// },
246+
// subject: {
247+
// age: record.subject.dateOfBirth ? yearsPassed(record.subject.dateOfBirth) : null,
248+
// groupIds: record.subject.groupIds,
249+
// id: record.subject.id,
250+
// sex: record.subject.sex
251+
// }
252+
// };
253+
// });
208254

209255
const numWorkers = Math.min(cpus().length, Math.ceil(records.length / 100)); // Use up to CPU count, chunk size 100
210-
const chunkSize = Math.ceil(convertRecords.length / numWorkers);
256+
const chunkSize = Math.ceil(records.length / numWorkers);
211257
const chunks = [];
212258

213-
for (let i = 0; i < convertRecords.length; i += chunkSize) {
214-
chunks.push(convertRecords.slice(i, i + chunkSize));
259+
for (let i = 0; i < records.length; i += chunkSize) {
260+
chunks.push(records.slice(i, i + chunkSize));
215261
}
216262

217263
const availableInstrumentArray: InitData = instruments
@@ -468,30 +514,6 @@ export class InstrumentRecordsService {
468514
}
469515
}
470516

471-
private expandData(listEntry: any[]): ExpandDataType[] {
472-
const validRecordArrayList: ExpandDataType[] = [];
473-
if (listEntry.length < 1) {
474-
throw new Error('Record Array is Empty');
475-
}
476-
for (const objectEntry of Object.values(listEntry)) {
477-
for (const [dataKey, dataValue] of Object.entries(objectEntry as { [key: string]: any })) {
478-
const parseResult = $RecordArrayFieldValue.safeParse(dataValue);
479-
if (!parseResult.success) {
480-
validRecordArrayList.push({
481-
message: `Error interpreting value ${dataValue} and record array key ${dataKey}`,
482-
success: false
483-
});
484-
}
485-
validRecordArrayList.push({
486-
measure: dataKey,
487-
measureValue: parseResult.data,
488-
success: true
489-
});
490-
}
491-
}
492-
return validRecordArrayList;
493-
}
494-
495517
private getInstrumentById(instrumentId: string) {
496518
return this.instrumentsService
497519
.findById(instrumentId)
@@ -502,6 +524,88 @@ export class InstrumentRecordsService {
502524
return JSON.parse(JSON.stringify(data), reviver) as unknown;
503525
}
504526

527+
private async queryRecordsRaw() {
528+
const pipeline = [
529+
// {
530+
// $match: {
531+
// $and: [
532+
// // Filter by groupId if provided
533+
// ...(groupId ? [{ groupIds: { $in: [groupId] } }] : []),
534+
// //permissionFilter
535+
// ]
536+
// }
537+
// },
538+
{
539+
// Join with Session collection
540+
$lookup: {
541+
as: 'session',
542+
foreignField: '_id',
543+
from: 'SessionModel',
544+
localField: 'sessionId' // Ensure this matches your @map or field name in Prisma
545+
}
546+
},
547+
{ $unwind: { path: '$session', preserveNullAndEmptyArrays: true } },
548+
{
549+
// Join with Subject collection
550+
$lookup: {
551+
as: 'subject',
552+
foreignField: '_id',
553+
from: 'SubjectModel',
554+
localField: 'subjectId'
555+
}
556+
},
557+
{ $unwind: { path: '$subject', preserveNullAndEmptyArrays: true } },
558+
{
559+
$project: {
560+
computedMeasures: 1,
561+
date: {
562+
$dateToString: {
563+
date: '$createdAt',
564+
format: '%Y-%m-%d'
565+
}
566+
},
567+
id: {
568+
$toString: '$_id'
569+
},
570+
instrumentId: 1,
571+
session: {
572+
date: {
573+
$dateToString: {
574+
date: '$session.date',
575+
format: '%Y-%m-%d'
576+
}
577+
},
578+
id: {
579+
$toString: '$session._id'
580+
},
581+
type: '$session.type',
582+
user: { username: '$session.user.username' } // TBD test this works
583+
},
584+
// sessionId: 1,
585+
subject: {
586+
age: {
587+
$dateDiff: {
588+
endDate: '$$NOW',
589+
startDate: '$subject.dateOfBirth',
590+
unit: 'year'
591+
}
592+
},
593+
dateOfBirth: '$subject.dateOfBirth',
594+
groupIds: '$subject.groupIds', // TBD make sure groupIds is string array
595+
id: {
596+
$toString: '$subject._id'
597+
},
598+
sex: '$subject.sex'
599+
}
600+
}
601+
}
602+
];
603+
604+
const records = await this.instrumentRecordModel.aggregateRaw({ pipeline });
605+
606+
return JSON.parse(JSON.stringify(records)) as unknown as RecordType[];
607+
}
608+
505609
private serializeData(data: unknown) {
506610
return JSON.parse(JSON.stringify(data, replacer)) as unknown;
507611
}

0 commit comments

Comments
 (0)