feat: use binary encoding to reduce network latency for large exports

Author: joshunrau

apps/api/src/core/interceptors/msgpack.interceptor.ts

@@ -0,0 +1,24 @@
+import { Injectable } from '@nestjs/common';
+import type { CallHandler, ExecutionContext, NestInterceptor } from '@nestjs/common';
+import type { FastifyReply } from 'fastify';
+import { Packr } from 'msgpackr';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
+
+@Injectable()
+export class MsgpackInterceptor implements NestInterceptor {
+  private packr = new Packr({
+    useRecords: true
+  });
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
+    const response = context.switchToHttp().getResponse<FastifyReply>();
+
+    return next.handle().pipe(
+      map((data) => {
+        response.header('Content-Type', 'application/x-msgpack');
+        return this.packr.pack(data);
+      })
+    );
+  }
+}

apps/api/src/instrument-records/instrument-records.controller.ts

@@ -1,7 +1,19 @@
 /* eslint-disable perfectionist/sort-classes */
 
 import { CurrentUser, ParseSchemaPipe, ValidObjectIdPipe } from '@douglasneuroinformatics/libnest';
-import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, Patch, Post, Query } from '@nestjs/common';
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpCode,
+  HttpStatus,
+  Param,
+  Patch,
+  Post,
+  Query,
+  UseInterceptors
+} from '@nestjs/common';
 import { ApiOperation, ApiTags } from '@nestjs/swagger';
 import type { InstrumentKind } from '@opendatacapture/runtime-core';
 import {
@@ -13,6 +25,7 @@ import { z } from 'zod/v4';
 
 import type { AppAbility } from '@/auth/auth.types';
 import { RouteAccess } from '@/core/decorators/route-access.decorator';
+import { MsgpackInterceptor } from '@/core/interceptors/msgpack.interceptor';
 
 import { InstrumentRecordsService } from './instrument-records.service';
 
@@ -67,6 +80,7 @@ export class InstrumentRecordsController {
   @ApiOperation({ summary: 'Export Records' })
   @Get('export')
   @RouteAccess({ action: 'read', subject: 'InstrumentRecord' })
+  @UseInterceptors(MsgpackInterceptor)
   exportRecords(@CurrentUser('ability') ability: AppAbility, @Query('groupId') groupId?: string) {
     return this.instrumentRecordsService.exportRecords({ groupId }, { ability });
   }

apps/web/src/routes/_app/datahub/index.tsx

@@ -10,6 +10,7 @@ import { removeSubjectIdScope } from '@opendatacapture/subject-utils';
 import { createFileRoute, useNavigate } from '@tanstack/react-router';
 import axios from 'axios';
 import { UserSearchIcon } from 'lucide-react';
+import { unpack } from 'msgpackr/unpack';
 import { unparse } from 'papaparse';
 
 import { IdentificationForm } from '@/components/IdentificationForm';
@@ -50,12 +51,13 @@ const Toggles: React.FC<{ table: TanstackTable.Table<Subject> }> = ({ table }) =
   };
 
   const getExportRecords = async () => {
-    const response = await axios.get<InstrumentRecordsExport>('/v1/instrument-records/export', {
+    const response = await axios.get<ArrayBuffer>('/v1/instrument-records/export', {
       params: {
         groupId: currentGroup?.id
-      }
+      },
+      responseType: 'arraybuffer'
     });
-    return response.data;
+    return unpack(new Uint8Array(response.data)) as InstrumentRecordsExport;
   };
 
   const handleExportSelection = (option: 'CSV' | 'Excel' | 'JSON') => {

apps/web/src/services/axios.ts

@@ -10,7 +10,7 @@ axios.defaults.baseURL = config.setup.apiBaseUrl;
 axios.interceptors.request.use((config) => {
   const accessToken = useAppStore.getState().accessToken;
 
-  config.headers.setAccept('application/json');
+  config.headers.setAccept(['application/json', 'application/x-msgpack']);
 
   // Do not set timeout for setup (can be CPU intensive, especially on slow server)
   if (