Chunking utilities

import {
  splitTextRecursively,
  splitJsonRecursively,
  TChunkDocument,
} from "@ai_kit/core";

const chunks = splitTextRecursively(longArticle, {
  chunkSize: 500,
  chunkOverlap: 50,
});

// Generate embeddings
await vectorStore.embed(chunks.map(chunk => chunk.content));

// Build a quick summary
const summary = chunks.map(chunk => chunk.content.split("\n")[0]).join("\n");

// Attach unique identifiers
const passages = chunks.map(chunk => ({
  id: `article-${chunk.index}`,
  text: chunk.content,
  start: chunk.start,
  end: chunk.end,
}));

const data = { foo: "bar", nested: { value: 42 } };

const chunks = splitJsonRecursively(data, {
  chunkSize: 300,
  format: "pretty",
  metadata: { source: "example" },
});

const doc = TChunkDocument.fromJSON(myJson, { dataset: "customers" });
const chunks = doc.chunk({
  chunkSize: 256,
  chunkOverlap: 32,
  metadata: { stage: "training" },
});

const normalized = doc.toString("pretty");