feat(knowledge-base): implement shared-table mode for knowledge bases

- Introduced shared-table mode allowing multiple knowledge bases to share the same underlying storage tables, reducing table proliferation.
- Added `ScopedTabularStorage` and `ScopedVectorStorage` wrappers to manage data access scoped by `kb_id`.
- Updated registration process to support shared table names and added utility functions for schema management.
- Enhanced documentation to include new shared-table mode features and usage examples.

Author: sroussey

bun.lock

@@ -149,13 +149,14 @@ When defining task input schemas, you can use `format` annotations to enable aut
 
 The system supports several format annotations out of the box:
 
-| Format                         | Description                         | Helper Function               |
-| ------------------------------ | ----------------------------------- | ----------------------------- |
-| `model`                        | Any AI model configuration          | `TypeModel()`                 |
-| `model:TaskName`               | Model compatible with specific task | —                             |
-| `storage:tabular`              | Tabular data dataset                | `TypeTabularStorage()`        |
-| `dataset:document-node-vector` | Vector storage dataset              | `TypeChunkVectorRepository()` |
-| `dataset:document`             | Document dataset                    | `TypeDocumentRepository()`    |
+| Format                            | Description                         | Helper Function               |
+| --------------------------------- | ----------------------------------- | ----------------------------- |
+| `model`                           | Any AI model configuration          | `TypeModel()`                 |
+| `model:TaskName`                  | Model compatible with specific task | —                             |
+| `storage:tabular`                 | Tabular data storage                | `TypeTabularStorage()`        |
+| `knowledge-base`                  | Knowledge base instance             | `TypeKnowledgeBase()`         |
+| `credential`                      | Credential from credential store    | —                             |
+| `tasks`                           | Task class from task registry       | —                             |
 
 ### Example: Using Format Annotations
 

docs/developers/03_extending.md

@@ -47,8 +47,8 @@
     "@inkjs/ui": "^2.0.0",
     "chalk": "^5.6.2",
     "commander": "^14.0.3",
-    "ink": "^6.8.0",
-    "react": "^19.2.4",
+    "ink": "^7.0.0",
+    "react": "^19.2.5",
     "smol-toml": "^1.6.1"
   },
   "devDependencies": {

examples/cli/package.json

@@ -33,8 +33,8 @@
     "@uiw/react-codemirror": "^4.25.9",
     "@xyflow/react": "^12.10.2",
     "clsx": "^2.1.1",
-    "react": "^19.2.4",
-    "react-dom": "^19.2.4",
+    "react": "^19.2.5",
+    "react-dom": "^19.2.5",
     "react-icons": "^5.6.0",
     "react-resizable-panels": "^4.8.0",
     "tailwind-merge": "^3.3.0"

examples/web/package.json

@@ -46,15 +46,15 @@
     "publish-workspaces": "bun ./scripts/publish-workspaces.ts"
   },
   "dependencies": {
-    "caniuse-lite": "^1.0.30001786"
+    "caniuse-lite": "^1.0.30001787"
   },
   "catalog": {
     "@huggingface/transformers": "^4.0.1",
     "@mediapipe/tasks-text": "^0.10.34",
     "@mediapipe/tasks-vision": "^0.10.34",
     "@mediapipe/tasks-audio": "^0.10.34",
     "@mediapipe/tasks-genai": "^0.10.27",
-    "@anthropic-ai/sdk": "^0.82.0",
+    "@anthropic-ai/sdk": "^0.86.1",
     "@google/generative-ai": "^0.24.1",
     "node-llama-cpp": "^3.18.1",
     "@huggingface/inference": "^4.13.15",
@@ -65,19 +65,19 @@
     "@modelcontextprotocol/sdk": "^1.29.0",
     "pg": "^8.20.0",
     "@electric-sql/pglite": "^0.4.3",
-    "@supabase/supabase-js": "^2.101.1",
+    "@supabase/supabase-js": "^2.102.1",
     "@sqlite.org/sqlite-wasm": "^3.51.2-build8",
     "better-sqlite3": "^12.8.0"
   },
   "devDependencies": {
-    "@sqliteai/sqlite-vector": "^0.9.94",
+    "@sqliteai/sqlite-vector": "^0.9.95",
     "@types/bun": "^1.3.11",
     "@types/node": "^24.12.2",
-    "@typescript-eslint/eslint-plugin": "^8.58.0",
-    "@typescript-eslint/parser": "^8.58.0",
-    "@typescript/native-preview": "=7.0.0-dev.20260406.1",
-    "@vitest/coverage-v8": "^4.1.2",
-    "@vitest/ui": "4.1.2",
+    "@typescript-eslint/eslint-plugin": "^8.58.1",
+    "@typescript-eslint/parser": "^8.58.1",
+    "@typescript/native-preview": "=7.0.0-dev.20260408.1",
+    "@vitest/coverage-v8": "^4.1.3",
+    "@vitest/ui": "4.1.3",
     "bunset": "^1.0.10",
     "concurrently": "^9.2.1",
     "eslint": "^10.2.0",
@@ -88,9 +88,9 @@
     "globals": "^17.4.0",
     "nyc": "^18.0.0",
     "prettier": "^3.8.1",
-    "turbo": "^2.9.4",
+    "turbo": "^2.9.5",
     "typescript": "6.0.2",
-    "vitest": "^4.1.2"
+    "vitest": "^4.1.3"
   },
   "engines": {
     "bun": "^1.3.11"

package.json

@@ -722,13 +722,14 @@ This resolution is handled by the input resolver system, which inspects schema `
 
 ### Supported Format Annotations
 
-| Format                            | Description                              | Resolver                   |
-| --------------------------------- | ---------------------------------------- | -------------------------- |
-| `model`                           | Any AI model configuration               | ModelRepository            |
-| `model:TaskName`                  | Model compatible with specific task type | ModelRepository            |
-| `repository:tabular`              | Tabular data repository                  | TabularStorageRegistry     |
-| `repository:document-node-vector` | Vector storage repository                | VectorRepositoryRegistry   |
-| `repository:document`             | Document repository                      | DocumentRepositoryRegistry |
+| Format            | Description                              | Resolver                |
+| ----------------- | ---------------------------------------- | ----------------------- |
+| `model`           | Any AI model configuration               | ModelRepository         |
+| `model:TaskName`  | Model compatible with specific task type | ModelRepository         |
+| `storage:tabular` | Tabular data storage                     | TabularStorageRegistry  |
+| `knowledge-base`  | Knowledge base instance                  | KnowledgeBaseRegistry   |
+| `credential`      | Credential from credential store         | CredentialStoreRegistry |
+| `tasks`           | Task class from task registry            | TaskRegistry            |
 
 ### Custom Model Validation
 

packages/ai/README.md

@@ -21,13 +21,22 @@ Document management, hierarchical chunking, and knowledge base infrastructure fo
   - [Tree Traversal](#tree-traversal)
   - [Lifecycle Management](#lifecycle-management)
   - [Registry](#registry)
+- [Shared-Table Mode](#shared-table-mode)
+  - [Overview](#overview-1)
+  - [Setting Up Shared Storage](#setting-up-shared-storage)
+  - [Scoped Wrappers](#scoped-wrappers)
+  - [Registering with Shared Tables](#registering-with-shared-tables)
+  - [Schemas and Indexes](#schemas-and-indexes)
+  - [When to Use Shared Tables](#when-to-use-shared-tables)
 - [Data Flow](#data-flow)
   - [Ingestion Pipeline](#ingestion-pipeline)
   - [Retrieval Pipeline](#retrieval-pipeline)
 - [API Reference](#api-reference)
   - [Document](#document)
   - [KnowledgeBase](#knowledgebase-1)
   - [createKnowledgeBase](#createknowledgebase)
+  - [ScopedTabularStorage](#scopedtabularstorage)
+  - [ScopedVectorStorage](#scopedvectorstorage)
   - [StructuralParser](#structuralparser)
   - [Type Helpers](#type-helpers)
 - [License](#license)
@@ -442,6 +451,134 @@ await task.run({ knowledgeBase: kb }); // Direct instance
 await task.run({ knowledgeBase: "my-kb" }); // Resolved from registry
 ```
 
+## Shared-Table Mode
+
+### Overview
+
+By default, each `KnowledgeBase` gets its own document table and chunk table. **Shared-table mode** lets multiple knowledge bases share the same underlying storage tables, partitioned by a `kb_id` column. This is useful when you have many knowledge bases and want to reduce table proliferation in your database.
+
+```
+Default mode (per-KB tables):          Shared-table mode:
+┌──────────────────────┐               ┌──────────────────────────┐
+│ kb_docs_my_kb        │               │ shared_documents         │
+│ (doc_id, data)       │               │ (doc_id, kb_id, data)    │
+├──────────────────────┤               │  ├─ kb_id = "kb-1" rows  │
+│ kb_chunks_my_kb      │               │  └─ kb_id = "kb-2" rows  │
+│ (chunk_id, vector..) │               ├──────────────────────────┤
+├──────────────────────┤               │ shared_chunks            │
+│ kb_docs_other_kb     │               │ (chunk_id, kb_id, vec..) │
+│ (doc_id, data)       │               │  ├─ kb_id = "kb-1" rows  │
+├──────────────────────┤               │  └─ kb_id = "kb-2" rows  │
+│ kb_chunks_other_kb   │               └──────────────────────────┘
+│ (chunk_id, vector..) │
+└──────────────────────┘
+```
+
+The `KnowledgeBase` class itself is unchanged — shared-table mode is implemented via thin wrapper classes (`ScopedTabularStorage`, `ScopedVectorStorage`) that inject `kb_id` on writes and filter by `kb_id` on reads.
+
+### Setting Up Shared Storage
+
+Create the shared storage instances once, globally:
+
+```typescript
+import { InMemoryTabularStorage, InMemoryVectorStorage } from "@workglow/storage";
+import {
+  SharedDocumentStorageSchema,
+  SharedChunkVectorStorageSchema,
+  SharedDocumentIndexes,
+  SharedChunkIndexes,
+  SHARED_DOCUMENT_TABLE,
+  SHARED_CHUNK_TABLE,
+  DocumentStorageKey,
+  ChunkVectorPrimaryKey,
+} from "@workglow/knowledge-base";
+
+const sharedDocStorage = new InMemoryTabularStorage(
+  SharedDocumentStorageSchema,
+  DocumentStorageKey,
+  SharedDocumentIndexes
+);
+
+const sharedChunkStorage = new InMemoryVectorStorage(
+  SharedChunkVectorStorageSchema,
+  ChunkVectorPrimaryKey,
+  SharedChunkIndexes,
+  1024 // vector dimensions
+);
+```
+
+For SQL backends (SQLite, PostgreSQL), replace `InMemoryTabularStorage` / `InMemoryVectorStorage` with the appropriate implementations. The shared schemas include indexes on `kb_id` and `[kb_id, doc_id]` for efficient scoped queries.
+
+### Scoped Wrappers
+
+For each knowledge base, create scoped wrappers that filter to that KB's data:
+
+```typescript
+import {
+  ScopedTabularStorage,
+  ScopedVectorStorage,
+  KnowledgeBase,
+} from "@workglow/knowledge-base";
+
+// KB 1
+const scopedDocs1 = new ScopedTabularStorage(sharedDocStorage, "kb-1");
+const scopedChunks1 = new ScopedVectorStorage(sharedChunkStorage, "kb-1");
+const kb1 = new KnowledgeBase("kb-1", scopedDocs1, scopedChunks1);
+
+// KB 2
+const scopedDocs2 = new ScopedTabularStorage(sharedDocStorage, "kb-2");
+const scopedChunks2 = new ScopedVectorStorage(sharedChunkStorage, "kb-2");
+const kb2 = new KnowledgeBase("kb-2", scopedDocs2, scopedChunks2);
+```
+
+Each `KnowledgeBase` instance works exactly the same as in default mode — all CRUD, search, and lifecycle operations are transparently scoped to the KB's data.
+
+### Registering with Shared Tables
+
+Pass `{ sharedTables: true }` when registering so that the metadata record uses the shared table names:
+
+```typescript
+import { registerKnowledgeBase } from "@workglow/knowledge-base";
+
+await registerKnowledgeBase("kb-1", kb1, { sharedTables: true });
+await registerKnowledgeBase("kb-2", kb2, { sharedTables: true });
+```
+
+You can check whether a persisted record uses shared tables with the `isSharedTableMode` helper:
+
+```typescript
+import { isSharedTableMode } from "@workglow/knowledge-base";
+
+const record = await repo.getKnowledgeBase("kb-1");
+if (isSharedTableMode(record)) {
+  // reconstruct using scoped wrappers
+}
+```
+
+### Schemas and Indexes
+
+The shared schemas augment the standard schemas with a `kb_id` column:
+
+| Schema                          | Base Schema                | Added Column |
+| ------------------------------- | -------------------------- | ------------ |
+| `SharedDocumentStorageSchema`   | `DocumentStorageSchema`    | `kb_id: string` |
+| `SharedChunkVectorStorageSchema`| `ChunkVectorStorageSchema` | `kb_id: string` |
+
+Default shared table names: `SHARED_DOCUMENT_TABLE = "shared_documents"`, `SHARED_CHUNK_TABLE = "shared_chunks"`.
+
+Pre-defined index arrays for efficient queries:
+- `SharedDocumentIndexes` — `[["kb_id"]]`
+- `SharedChunkIndexes` — `[["kb_id"], ["kb_id", "doc_id"]]`
+
+### When to Use Shared Tables
+
+| Scenario | Recommendation |
+| --- | --- |
+| Few knowledge bases, each large | Default (per-KB tables) — simpler, no `kb_id` overhead |
+| Many knowledge bases (e.g., per-user, per-tenant) | Shared tables — avoids table proliferation |
+| Need cross-KB queries | Shared tables — query the shared storage directly |
+| Using managed databases with table limits | Shared tables |
+
 ## Data Flow
 
 ### Ingestion Pipeline
@@ -640,6 +777,35 @@ interface CreateKnowledgeBaseOptions {
 }
 ```
 
+### ScopedTabularStorage
+
+```typescript
+class ScopedTabularStorage<Schema, PrimaryKeyNames, Entity, PrimaryKey, InsertType>
+  implements ITabularStorage<Schema, PrimaryKeyNames, Entity, PrimaryKey, InsertType>
+{
+  constructor(inner: AnyTabularStorage, kbId: string);
+
+  // All ITabularStorage methods are implemented.
+  // Writes inject kb_id, reads filter by kb_id, results strip kb_id.
+  // setupDatabase() and destroy() are no-ops (shared storage lifecycle is external).
+}
+```
+
+### ScopedVectorStorage
+
+```typescript
+class ScopedVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames>
+  extends ScopedTabularStorage<Schema, PrimaryKeyNames, Entity>
+  implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames>
+{
+  constructor(inner: AnyVectorStorage, kbId: string);
+
+  getVectorDimensions(): number; // Delegates to inner
+  similaritySearch(query, options?): Promise<(Entity & { score })[]>; // Post-filters by kb_id
+  hybridSearch?(query, options): Promise<(Entity & { score })[]>; // Post-filters by kb_id
+}
+```
+
 ### StructuralParser
 
 ```typescript

packages/knowledge-base/README.md

@@ -12,6 +12,9 @@ export * from "./knowledge-base/KnowledgeBaseRepository";
 export * from "./knowledge-base/InMemoryKnowledgeBaseRepository";
 export * from "./knowledge-base/KnowledgeBaseRegistry";
 export * from "./knowledge-base/createKnowledgeBase";
+export * from "./knowledge-base/ScopedTabularStorage";
+export * from "./knowledge-base/ScopedVectorStorage";
+export * from "./knowledge-base/SharedTableSchemas";
 
 export * from "./util/DatasetSchema";
 export * from "./document/Document";

packages/knowledge-base/src/common.ts

@@ -14,8 +14,9 @@ import {
 import { InMemoryKnowledgeBaseRepository } from "./InMemoryKnowledgeBaseRepository";
 import type { KnowledgeBase } from "./KnowledgeBase";
 import { KnowledgeBaseRepository } from "./KnowledgeBaseRepository";
-import { knowledgeBaseTableNames } from "./KnowledgeBaseSchema";
 import type { KnowledgeBaseRecord } from "./KnowledgeBaseSchema";
+import { knowledgeBaseTableNames } from "./KnowledgeBaseSchema";
+import { SHARED_CHUNK_TABLE, SHARED_DOCUMENT_TABLE } from "./SharedTableSchemas";
 
 /**
  * Service token for the knowledge base registry
@@ -70,15 +71,28 @@ export function setGlobalKnowledgeBaseRepository(repository: KnowledgeBaseReposi
   globalServiceRegistry.registerInstance(KNOWLEDGE_BASE_REPOSITORY, repository);
 }
 
+export interface RegisterKnowledgeBaseOptions {
+  /** When true, record uses shared table names instead of per-KB table names. */
+  readonly sharedTables?: boolean;
+}
+
 /**
  * Registers a knowledge base globally by ID.
  * Adds to both the live Map and the persistent repository.
  */
-export async function registerKnowledgeBase(id: string, kb: KnowledgeBase): Promise<void> {
+
+export async function registerKnowledgeBase(
+  id: string,
+  kb: KnowledgeBase,
+  options?: RegisterKnowledgeBaseOptions
+): Promise<void> {
   const kbs = getGlobalKnowledgeBases();
 
   const now = new Date().toISOString();
-  const tableNames = knowledgeBaseTableNames(id);
+  const useShared = options?.sharedTables === true;
+  const tableNames = useShared
+    ? { documentTable: SHARED_DOCUMENT_TABLE, chunkTable: SHARED_CHUNK_TABLE }
+    : knowledgeBaseTableNames(id);
   const record: KnowledgeBaseRecord = {
     kb_id: id,
     title: kb.title,

packages/knowledge-base/src/knowledge-base/KnowledgeBaseRegistry.ts

@@ -5,6 +5,7 @@
  */
 
 import type { DataPortSchemaObject, FromSchema } from "@workglow/util/schema";
+import { SHARED_CHUNK_TABLE, SHARED_DOCUMENT_TABLE } from "./SharedTableSchemas";
 
 /**
  * Schema for persisting KnowledgeBase metadata to tabular storage.
@@ -50,3 +51,10 @@ export function knowledgeBaseTableNames(kbId: string): {
     chunkTable: `kb_chunks_${safe}`,
   };
 }
+
+/**
+ * Checks whether a KnowledgeBaseRecord uses shared-table mode.
+ */
+export function isSharedTableMode(record: KnowledgeBaseRecord): boolean {
+  return record.document_table === SHARED_DOCUMENT_TABLE && record.chunk_table === SHARED_CHUNK_TABLE;
+}