Merge pull request #1269 from joshunrau/fix-build

use msgpackr to encode exports as optimized buffer rather than massive JSON

Author: joshunrau

apps/api/package.json

@@ -43,8 +43,10 @@
     "@prisma/client": "catalog:",
     "axios": "catalog:",
     "express": "^5.0.1",
+    "fastify": "^5.7.1",
     "lodash-es": "workspace:lodash-es__4.x@*",
     "mongodb": "^6.15.0",
+    "msgpackr": "catalog:",
     "neverthrow": "catalog:",
     "passport": "^0.7.0",
     "passport-jwt": "^4.0.1",

apps/api/src/core/interceptors/msgpack.interceptor.ts

@@ -0,0 +1,24 @@
+import { Injectable } from '@nestjs/common';
+import type { CallHandler, ExecutionContext, NestInterceptor } from '@nestjs/common';
+import type { FastifyReply } from 'fastify';
+import { Packr } from 'msgpackr';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+@Injectable()
+export class MsgpackInterceptor implements NestInterceptor {
+  private packr = new Packr({
+    useRecords: true
+  });
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
+    const response = context.switchToHttp().getResponse<FastifyReply>();
+
+    return next.handle().pipe(
+      map((data) => {
+        response.header('Content-Type', 'application/x-msgpack');
+        return this.packr.pack(data);
+      })
+    );
+  }
+}

apps/api/src/instrument-records/instrument-records.controller.ts

@@ -1,7 +1,19 @@
 /* eslint-disable perfectionist/sort-classes */
 
 import { CurrentUser, ParseSchemaPipe, ValidObjectIdPipe } from '@douglasneuroinformatics/libnest';
-import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Patch, Post, Query } from '@nestjs/common';
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpCode,
+  HttpStatus,
+  Param,
+  Patch,
+  Post,
+  Query,
+  UseInterceptors
+} from '@nestjs/common';
 import { ApiOperation, ApiTags } from '@nestjs/swagger';
 import type { InstrumentKind } from '@opendatacapture/runtime-core';
 import {
@@ -13,6 +25,7 @@ import { z } from 'zod/v4';
 
 import type { AppAbility } from '@/auth/auth.types';
 import { RouteAccess } from '@/core/decorators/route-access.decorator';
+import { MsgpackInterceptor } from '@/core/interceptors/msgpack.interceptor';
 
 import { InstrumentRecordsService } from './instrument-records.service';
 
@@ -67,6 +80,7 @@ export class InstrumentRecordsController {
   @ApiOperation({ summary: 'Export Records' })
   @Get('export')
   @RouteAccess({ action: 'read', subject: 'InstrumentRecord' })
+  @UseInterceptors(MsgpackInterceptor)
   exportRecords(@CurrentUser('ability') ability: AppAbility, @Query('groupId') groupId?: string) {
     return this.instrumentRecordsService.exportRecords({ groupId }, { ability });
   }

apps/api/src/instrument-records/instrument-records.service.ts

@@ -132,7 +132,10 @@ export class InstrumentRecordsService {
     return this.instrumentRecordModel.exists(where);
   }
 
-  async exportRecords({ groupId }: { groupId?: string } = {}, { ability }: Required<EntityOperationOptions>) {
+  async exportRecords(
+    { groupId }: { groupId?: string } = {},
+    { ability }: Required<EntityOperationOptions>
+  ): Promise<InstrumentRecordsExport> {
     const records = await this.queryRecordsRaw(ability, groupId);
 
     const instrumentIds = new Set(records.map((r) => r.instrumentId));

apps/web/package.json

@@ -45,6 +45,7 @@
     "lodash-es": "workspace:lodash-es__4.x@*",
     "lucide-react": "^0.507.0",
     "motion": "catalog:",
+    "msgpackr": "catalog:",
     "papaparse": "workspace:papaparse__5.x@*",
     "qrcode": "^1.5.4",
     "react": "workspace:react__19.x@*",

apps/web/src/routes/_app/datahub/index.tsx

@@ -10,6 +10,7 @@ import { removeSubjectIdScope } from '@opendatacapture/subject-utils';
 import { createFileRoute, useNavigate } from '@tanstack/react-router';
 import axios from 'axios';
 import { UserSearchIcon } from 'lucide-react';
+import { unpack } from 'msgpackr/unpack';
 import { unparse } from 'papaparse';
 
 import { IdentificationForm } from '@/components/IdentificationForm';
@@ -50,12 +51,13 @@ const Toggles: React.FC<{ table: TanstackTable.Table<Subject> }> = ({ table }) =
   };
 
   const getExportRecords = async () => {
-    const response = await axios.get<InstrumentRecordsExport>('/v1/instrument-records/export', {
+    const response = await axios.get<ArrayBuffer>('/v1/instrument-records/export', {
       params: {
         groupId: currentGroup?.id
-      }
+      },
+      responseType: 'arraybuffer'
     });
-    return response.data;
+    return unpack(new Uint8Array(response.data)) as InstrumentRecordsExport;
   };
 
   const handleExportSelection = (option: 'CSV' | 'Excel' | 'JSON') => {

apps/web/src/services/axios.ts

@@ -10,7 +10,7 @@ axios.defaults.baseURL = config.setup.apiBaseUrl;
 axios.interceptors.request.use((config) => {
   const accessToken = useAppStore.getState().accessToken;
 
-  config.headers.setAccept('application/json');
+  config.headers.setAccept(['application/json', 'application/x-msgpack']);
 
   // Do not set timeout for setup (can be CPU intensive, especially on slow server)
   if (

package.json

@@ -87,6 +87,7 @@
       "@tailwindcss/oxide",
       "esbuild",
       "mongodb-memory-server",
+      "msgpackr-extract",
       "msw",
       "oxc-resolver",
       "prisma",

pnpm-lock.yaml

@@ -20,6 +20,7 @@ catalog:
   esbuild: '0.23.x'
   esbuild-wasm: '0.23.x'
   happy-dom: '^20.0.2'
+  msgpackr: ^1.11.8
   motion: '11.15.0'
   neverthrow: ^8.2.0
   nodemon: '^3.1.9'