Vector Data Manager

'use client';/** * @file vector-data-manager.tsx * @description Core-only knowledge-base data manager over an owned corpus: 4-format vector import (Pinecone/ChromaDB/CSV/JSONL + native JSON) with auto-detect + preview + re-embed toggle, native JSON/CSV/JSONL export (round-trippable), embedding-drift banner + cancellable reindex, and storage + vector observability with adaptive batch sizing. * @constraint Core engine only — never imports @localmode/langchain, @localmode/wllama, or @localmode/pdfjs; getLanguageModel is lazy and never invoked (this block never generates). */import { useEffect, useRef, useState, type ReactNode } from 'react';import {  AlertTriangle,  CheckCircle2,  ClipboardPaste,  FileText,  Play,  Plus,  RefreshCw,  RotateCcw,  Square,  Upload,} from 'lucide-react';import {  downloadBlob,  useAdaptiveBatchSize,  useImportExport,  useKnowledgeBase,  useStorageQuota,  type AnyLoadProgress,} from '@localmode/react';import {  createKnowledgeBaseEngine,  exportToCSV,  exportToJSONL,  isWebGPUSupported,} from '@localmode/core';import type {  ChunkingMode,  DocumentSource,  EngineStats,  ImportRecord,  KnowledgeBaseEngine,  RawDocument,} from '@localmode/core';import { transformers, isModelCached } from '@localmode/transformers';import {  VectorImportFlow,  type ImportProgressLike,  type PreviewRecord as FlowPreviewRecord,} from '@/components/vector-import-flow';import { VectorExportPanel } from '@/components/vector-export-panel';import { FormatDetectionBadge } from '@/components/format-detection-badge';import { StorageMeter } from '@/components/storage-meter';import { VectorStorageObservability } from '@/components/vector-storage-observability';import {  EmbeddingDriftBanner,  type ReindexProgressLike,} from '@/components/embedding-drift-banner';import { AdaptiveBatchCard } from '@/components/adaptive-batch-card';import { ModelDownloader } from '@/components/model-downloader';import { cn } from '@/lib/utils';const DEFAULT_EMBEDDING_MODEL_ID = 'Xenova/bge-small-en-v1.5';const EMBEDDING_MODEL_META: Record<string, { name: string; size: string }> = {  'Xenova/bge-small-en-v1.5': { name: 'BGE Small EN v1.5', size: '34 MB' },  'Xenova/all-MiniLM-L6-v2': { name: 'all-MiniLM-L6-v2', size: '23 MB' },};const EMBEDDING_MODEL_IDS = Object.keys(EMBEDDING_MODEL_META);const ANSWER_MODEL_ID = 'onnx-community/granite-4.0-350m-ONNX-web';const MAX_FILE_SIZE = 50 * 1024 * 1024;const SUPPORTED_EXTENSIONS = ['.json', '.jsonl', '.csv'];const MAX_PREVIEW_RECORDS = 10;const DEFAULT_DIMENSIONS = 384;const NATIVE_MARKER = 'localmode-kb';const NATIVE_FORMAT_ID = 'native-json';const FORMAT_LABELS: Record<string, string> = {  pinecone: 'Pinecone JSON',  chroma: 'ChromaDB JSON',  csv: 'CSV',  jsonl: 'JSONL',  [NATIVE_FORMAT_ID]: 'Native JSON (knowledge base)',};const FORMAT_COLOR_MAP = {  [NATIVE_FORMAT_ID]: {    badge: 'bg-primary/10 text-primary border-primary/30',    dot: 'bg-primary',  },};const EXPORT_FORMATS = [  {    id: NATIVE_FORMAT_ID,    label: 'Native JSON',    description:      'Lossless document array (titles, text, sources, categories, pages, metadata) - round-trip re-importable.',  },  {    id: 'csv',    label: 'CSV',    description: 'id + text + metadata columns (no vectors) for spreadsheets.',  },  {    id: 'jsonl',    label: 'JSONL',    description: 'One id + text + metadata record per line (no vectors).',  },];const VALID_SOURCES: DocumentSource[] = ['text', 'sample', 'pdf', 'ocr', 'import'];const SAMPLE_NOTE =  'Privacy and encryption on device: encrypting personal data with AES-GCM keys derived on the device keeps private information confidential, and no plaintext ever leaves the browser. Local-first vector search then runs entirely offline over the encrypted-at-rest corpus.';interface Session {  engine: KnowledgeBaseEngine;  engineKind: 'core';  documents: RawDocument[];  addDocuments: (docs: Array<Omit<RawDocument, 'id' | 'addedAt'>>) => Promise<void>;  removeDocument: (docId: string) => Promise<void>;  clearAll: () => Promise<void>;  chunking: ChunkingMode;  setChunking: (mode: ChunkingMode) => void;  chunkSize: number;  setChunkSize: (n: number) => void;  embeddingModelId: string;  setEmbeddingModelId: (id: string) => void;  busy: boolean;  error: string | null;}type DocDraft = Omit<RawDocument, 'id' | 'addedAt'>;interface NativeExport {  format: typeof NATIVE_MARKER;  version: 1;  exportedAt: string;  embeddingModelId: string;  engine: { kind: 'core' | 'langchain'; documents: number; chunks: number; dimensions: number };  documents: RawDocument[];}interface ImportLanes {  direct: DocDraft[];  needsEmbed: DocDraft[];  vectorOnly: number;  vectorOnlyMismatched: number;  empty: number;}interface ImportPlan {  formatId: string;  fileName: string | null;  totalRecords: number;  recordsWithVectors: number;  recordsWithTextOnly: number;  dimensions: number | null;  lanes: ImportLanes;  previewRows: FlowPreviewRecord[];}interface ImportStats {  imported: number;  skipped: number;  reEmbedded: number;  totalParsed: number;  formatId: string;  durationMs: number;  cancelled: boolean;}interface IngestFingerprint {  modelId: string;  docsKey: string;}function firstLine(text: string) {  return text.split('\n').find((l) => l.trim().length > 0)?.trim().slice(0, 80) ?? '';}function truncate(text: string, max = 80) {  return text.length > max ? `${text.slice(0, max)}…` : text;}function formatDuration(ms: number) {  return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.round(ms)}ms`;}function toDraft(doc: RawDocument): DocDraft {  return {    title: doc.title,    text: doc.text,    source: doc.source,    ...(doc.category !== undefined ? { category: doc.category } : {}),    ...(doc.meta !== undefined ? { meta: doc.meta } : {}),    ...(doc.pages !== undefined ? { pages: doc.pages } : {}),  };}function coerceNativeDocument(raw: unknown, index: number): RawDocument {  if (raw === null || typeof raw !== 'object') {    throw new Error(`Native export document #${index + 1} is not an object.`);  }  const obj = raw as Record<string, unknown>;  if (typeof obj.title !== 'string' || typeof obj.text !== 'string') {    throw new Error(      `Native export document #${index + 1} is missing its "title"/"text" string fields.`,    );  }  const source = VALID_SOURCES.includes(obj.source as DocumentSource)    ? (obj.source as DocumentSource)    : 'import';  const meta: Record<string, string | number> = {};  if (obj.meta !== null && typeof obj.meta === 'object') {    for (const [k, v] of Object.entries(obj.meta as Record<string, unknown>)) {      if (typeof v === 'string' || typeof v === 'number') meta[k] = v;    }  }  const pages = Array.isArray(obj.pages)    ? obj.pages.filter(        (p): p is { page: number; text: string } =>          p !== null &&          typeof p === 'object' &&          typeof (p as { page?: unknown }).page === 'number' &&          typeof (p as { text?: unknown }).text === 'string',      )    : undefined;  return {    id: typeof obj.id === 'string' && obj.id ? obj.id : `import-${index}`,    title: obj.title,    text: obj.text,    source,    ...(typeof obj.category === 'string' ? { category: obj.category } : {}),    ...(Object.keys(meta).length > 0 ? { meta } : {}),    ...(pages && pages.length > 0 ? { pages } : {}),    addedAt: typeof obj.addedAt === 'number' ? obj.addedAt : Date.now(),  };}function parseNativeExport(content: string): NativeExport | null {  const trimmed = content.trim();  if (!trimmed.startsWith('{')) return null;  let parsed: unknown;  try {    parsed = JSON.parse(trimmed);  } catch {    return null;  }  if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return null;  const obj = parsed as Record<string, unknown>;  if (obj.format !== NATIVE_MARKER) return null;  if (!Array.isArray(obj.documents)) {    throw new Error('Native knowledge-base export is malformed: expected a "documents" array.');  }  const documents = obj.documents.map(coerceNativeDocument);  const engine =    obj.engine !== null && typeof obj.engine === 'object'      ? (obj.engine as Record<string, unknown>)      : {};  return {    format: NATIVE_MARKER,    version: 1,    exportedAt: typeof obj.exportedAt === 'string' ? obj.exportedAt : '',    embeddingModelId: typeof obj.embeddingModelId === 'string' ? obj.embeddingModelId : '',    engine: {      kind: engine.kind === 'langchain' ? 'langchain' : 'core',      documents: typeof engine.documents === 'number' ? engine.documents : documents.length,      chunks: typeof engine.chunks === 'number' ? engine.chunks : 0,      dimensions: typeof engine.dimensions === 'number' ? engine.dimensions : 0,    },    documents,  };}function externalRecordToDraft(record: ImportRecord, formatId: string): DocDraft {  const md = record.metadata ?? {};  const title =    typeof md.title === 'string' && md.title.trim().length > 0      ? md.title      : firstLine(record.text ?? '') || record.id;  const meta: Record<string, string | number> = {    importFormat: formatId,    importId: record.id,  };  for (const [k, v] of Object.entries(md)) {    if (k === 'title' || k === 'category') continue;    if (typeof v === 'string' || typeof v === 'number') meta[k] = v;  }  return {    title,    text: record.text ?? '',    source: 'import',    ...(typeof md.category === 'string' ? { category: md.category } : {}),    meta,  };}function buildExternalLanes(  records: ImportRecord[],  formatId: string,  targetDims: number | null,): ImportLanes {  const direct: DocDraft[] = [];  const needsEmbed: DocDraft[] = [];  let vectorOnly = 0;  let vectorOnlyMismatched = 0;  let empty = 0;  for (const r of records) {    const hasText = typeof r.text === 'string' && r.text.trim().length > 0;    const dimsUsable = r.vector != null && (targetDims === null || r.vector.length === targetDims);    if (hasText && dimsUsable) {      direct.push(externalRecordToDraft(r, formatId));    } else if (hasText) {      needsEmbed.push(externalRecordToDraft(r, formatId));    } else if (r.vector != null) {      vectorOnly++;      if (targetDims !== null && r.vector.length !== targetDims) vectorOnlyMismatched++;    } else {      empty++;    }  }  return { direct, needsEmbed, vectorOnly, vectorOnlyMismatched, empty };}function docToImportRecord(doc: RawDocument): ImportRecord {  return {    id: doc.id,    text: doc.text,    metadata: {      title: doc.title,      source: doc.source,      ...(doc.category !== undefined ? { category: doc.category } : {}),      ...(doc.meta ?? {}),    },  };}function countBy(docs: RawDocument[], key: (d: RawDocument) => string) {  const counts: Record<string, number> = {};  for (const d of docs) {    const k = key(d);    counts[k] = (counts[k] ?? 0) + 1;  }  return Object.entries(counts).sort((a, b) => b[1] - a[1]);}function SectionCard({ title, children }: { title: string; children: ReactNode }) {  return (    <section className="flex flex-col gap-3 rounded-xl border border-border bg-card p-4 text-card-foreground">      <h2 className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">        {title}      </h2>      {children}    </section>  );}function PreviewStat({ label, value }: { label: string; value: string }) {  return (    <div className="flex flex-col gap-0.5 rounded-md border border-border bg-background px-2 py-1.5">      <span className="text-[10px] uppercase tracking-wide text-muted-foreground">{label}</span>      <span className="text-sm font-semibold tabular-nums">{value}</span>    </div>  );}function DataManager({ session }: { session: Session }) {  const { quota, refresh: refreshQuota } = useStorageQuota();  const [mounted, setMounted] = useState(false);  useEffect(() => setMounted(true), []);  const [engineStats, setEngineStats] = useState<EngineStats | null>(null);  useEffect(() => {    if (session.busy) return;    let alive = true;    void session.engine      .stats()      .then((s) => {        if (alive) setEngineStats(s);      })      .catch(() => {        if (alive) setEngineStats(null);      });    return () => {      alive = false;    };  }, [session.engine, session.busy, session.documents]);  const targetDims = engineStats && engineStats.dimensions > 0 ? engineStats.dimensions : null;  const batch = useAdaptiveBatchSize({    taskType: 'ingestion',    modelDimensions: targetDims ?? DEFAULT_DIMENSIONS,  });  const {    parsePreview,    isParsing,    error: parseError,    reset: resetParse,  } = useImportExport({    db: { dimensions: targetDims ?? DEFAULT_DIMENSIONS, addMany: async () => {} },  });  const [stage, setStage] = useState<'idle' | 'preview' | 'importing' | 'complete'>('idle');  const [plan, setPlan] = useState<ImportPlan | null>(null);  const [reEmbed, setReEmbed] = useState(false);  const [pasteText, setPasteText] = useState('');  const [importProgress, setImportProgress] = useState<ImportProgressLike | null>(null);  const [importStats, setImportStats] = useState<ImportStats | null>(null);  const [importError, setImportError] = useState<string | null>(null);  const importAbortRef = useRef<AbortController | null>(null);  const fileInputRef = useRef<HTMLInputElement>(null);  const [exportingFormat, setExportingFormat] = useState<string | null>(null);  const [lastExport, setLastExport] = useState<{    formatId: string;    records: number;    bytes: number;    at: string;  } | null>(null);  const [lastIngest, setLastIngest] = useState<IngestFingerprint | null>(null);  const [reindexRunning, setReindexRunning] = useState(false);  const [reindexProgress, setReindexProgress] = useState<ReindexProgressLike | null>(null);  const [reindexError, setReindexError] = useState<string | null>(null);  const reindexAbortRef = useRef<AbortController | null>(null);  const prevBusyRef = useRef(session.busy);  const docsKey = session.documents.map((d) => d.id).join('|');  useEffect(() => {    const wasBusy = prevBusyRef.current;    prevBusyRef.current = session.busy;    if (session.busy) return;    if (session.documents.length === 0) {      setLastIngest(null);      return;    }    if (session.error) return;    if (      wasBusy ||      lastIngest === null ||      (lastIngest.docsKey !== docsKey && lastIngest.modelId === session.embeddingModelId)    ) {      setLastIngest({ modelId: session.embeddingModelId, docsKey });    }  }, [    session.busy,    session.error,    session.documents.length,    session.embeddingModelId,    docsKey,    lastIngest,  ]);  const drift =    lastIngest !== null &&    session.documents.length > 0 &&    lastIngest.modelId !== session.embeddingModelId;  const importing = stage === 'importing';  const busyAny =    session.busy || importing || reindexRunning || exportingFormat !== null || isParsing;  const resetImport = () => {    importAbortRef.current?.abort();    resetParse();    setStage('idle');    setPlan(null);    setReEmbed(false);    setImportProgress(null);    setImportStats(null);    setImportError(null);    if (fileInputRef.current) fileInputRef.current.value = '';  };  const handleContent = async (content: string, fileName: string | null) => {    resetParse();    setImportError(null);    setImportStats(null);    setImportProgress(null);    setPlan(null);    if (!content.trim()) {      setImportError('The provided content is empty.');      return;    }    let native: NativeExport | null = null;    try {      native = parseNativeExport(content);    } catch (err) {      setImportError(err instanceof Error ? err.message : String(err));      return;    }    if (native) {      if (native.documents.length === 0) {        setImportError('Native knowledge-base export contains no documents.');        return;      }      setPlan({        formatId: NATIVE_FORMAT_ID,        fileName,        totalRecords: native.documents.length,        recordsWithVectors: 0,        recordsWithTextOnly: native.documents.length,        dimensions: native.engine.dimensions > 0 ? native.engine.dimensions : null,        lanes: {          direct: native.documents.map(toDraft),          needsEmbed: [],          vectorOnly: 0,          vectorOnlyMismatched: 0,          empty: 0,        },        previewRows: native.documents.slice(0, MAX_PREVIEW_RECORDS).map((d) => ({          id: d.id,          text: truncate(`${d.title} - ${d.text}`),          hasVector: false,        })),      });      setStage('preview');      return;    }    const result = await parsePreview({ content });    if (!result) return;    setPlan({      formatId: result.format,      fileName,      totalRecords: result.totalRecords,      recordsWithVectors: result.recordsWithVectors,      recordsWithTextOnly: result.recordsWithTextOnly,      dimensions: result.dimensions,      lanes: buildExternalLanes(result.records, result.format, targetDims),      previewRows: result.records.slice(0, MAX_PREVIEW_RECORDS).map((r) => ({        id: r.id,        ...(r.text ? { text: truncate(r.text) } : {}),        hasVector: r.vector != null,      })),    });    setStage('preview');  };  const handleFile = async (file: File) => {    if (file.size > MAX_FILE_SIZE) {      setImportError(        `File is ${(file.size / (1024 * 1024)).toFixed(1)} MB - the import limit is 50 MB.`,      );      return;    }    const content = await file.text();    await handleContent(content, file.name);  };  const runImport = async () => {    if (!plan || stage !== 'preview' || session.busy || reindexRunning) return;    const docs = reEmbed ? [...plan.lanes.direct, ...plan.lanes.needsEmbed] : plan.lanes.direct;    if (docs.length === 0) return;    const controller = new AbortController();    importAbortRef.current = controller;    setStage('importing');    setImportError(null);    setImportProgress({ phase: 'validating', overallCompleted: 0, overallTotal: docs.length });    const started = performance.now();    const batchSize = Math.max(1, batch.batchSize);    let imported = 0;    try {      for (let i = 0; i < docs.length; i += batchSize) {        if (controller.signal.aborted) break;        setImportProgress({          phase: 'embedding',          overallCompleted: imported,          overallTotal: docs.length,        });        await session.addDocuments(docs.slice(i, i + batchSize));        imported += Math.min(batchSize, docs.length - i);        setImportProgress({          phase: 'importing',          overallCompleted: imported,          overallTotal: docs.length,        });      }      setImportStats({        imported,        skipped: plan.totalRecords - imported,        reEmbedded: reEmbed ? Math.max(0, imported - plan.lanes.direct.length) : 0,        totalParsed: plan.totalRecords,        formatId: plan.formatId,        durationMs: performance.now() - started,        cancelled: controller.signal.aborted,      });      setStage('complete');    } catch (err) {      setImportError(err instanceof Error ? err.message : String(err));      setStage('preview');    } finally {      setImportProgress(null);      importAbortRef.current = null;      void refreshQuota();    }  };  const cancelImport = () => importAbortRef.current?.abort();  const exportDisabled = busyAny || session.documents.length === 0;  const handleExport = (formatId: string) => {    if (exportDisabled) return;    setExportingFormat(formatId);    try {      let content: string;      let filename: string;      let mime: string;      if (formatId === NATIVE_FORMAT_ID) {        const envelope: NativeExport = {          format: NATIVE_MARKER,          version: 1,          exportedAt: new Date().toISOString(),          embeddingModelId: session.embeddingModelId,          engine: {            kind: session.engineKind,            documents: session.documents.length,            chunks: engineStats?.chunks ?? 0,            dimensions: engineStats?.dimensions ?? 0,          },          documents: session.documents,        };        content = JSON.stringify(envelope, null, 2);        filename = 'knowledge-base-export.json';        mime = 'application/json';      } else {        const records = session.documents.map(docToImportRecord);        if (formatId === 'csv') {          content = exportToCSV(records, { includeVectors: false });          filename = 'knowledge-base-export.csv';          mime = 'text/csv';        } else {          content = exportToJSONL(records, { includeVectors: false });          filename = 'knowledge-base-export.jsonl';          mime = 'application/jsonl';        }      }      downloadBlob(content, filename, mime);      setLastExport({        formatId,        records: session.documents.length,        bytes: new Blob([content]).size,        at: new Date().toISOString(),      });    } catch (err) {      setImportError(err instanceof Error ? err.message : String(err));    } finally {      setExportingFormat(null);    }  };  const reindex = async () => {    if (session.busy || reindexRunning || importing || session.documents.length === 0) return;    const controller = new AbortController();    reindexAbortRef.current = controller;    setReindexRunning(true);    setReindexError(null);    const snapshot = session.documents.map(toDraft);    setReindexProgress({ completed: 0, total: snapshot.length, phase: 'embedding' });    try {      session.setEmbeddingModelId(session.embeddingModelId);      await session.clearAll();      const batchSize = Math.max(1, batch.batchSize);      let completed = 0;      for (let i = 0; i < snapshot.length; i += batchSize) {        if (controller.signal.aborted) break;        await session.addDocuments(snapshot.slice(i, i + batchSize));        completed += Math.min(batchSize, snapshot.length - i);        setReindexProgress({          completed,          total: snapshot.length,          phase: completed === snapshot.length ? 'indexing' : 'embedding',        });      }      if (!controller.signal.aborted) {        setLastIngest({ modelId: session.embeddingModelId, docsKey: '' });      }    } catch (err) {      setReindexError(err instanceof Error ? err.message : String(err));    } finally {      setReindexRunning(false);      setReindexProgress(null);      reindexAbortRef.current = null;      void refreshQuota();    }  };  const cancelReindex = () => reindexAbortRef.current?.abort();  const docsToImport = plan    ? reEmbed      ? plan.lanes.direct.length + plan.lanes.needsEmbed.length      : plan.lanes.direct.length    : 0;  const importErrorMessage = importError ?? parseError?.message ?? null;  const dimensionMismatch =    plan !== null &&    plan.dimensions !== null &&    targetDims !== null &&    plan.dimensions !== targetDims;  const chunkCount = engineStats?.chunks ?? 0;  const dims = engineStats?.dimensions ?? 0;  const estimatedBytes = chunkCount * dims * 4;  const sourceCounts = countBy(session.documents, (d) => d.source);  const categoryCounts = countBy(session.documents, (d) => d.category ?? 'uncategorized');  return (    <div className="flex flex-col gap-4">      {}      {drift && lastIngest && (        <div          data-stored-model={lastIngest.modelId}          data-current-model={session.embeddingModelId}          data-affected-docs={session.documents.length}          data-affected-chunks={chunkCount}          className="flex flex-col gap-1.5"        >          <EmbeddingDriftBanner            className="max-w-none"            storedModelId={lastIngest.modelId}            currentModelId={session.embeddingModelId}            isReindexing={reindexRunning}            progress={reindexProgress}            onReindex={() => void reindex()}            onCancel={cancelReindex}          />          <p className="text-xs text-muted-foreground">            Affected: {session.documents.length} document            {session.documents.length === 1 ? '' : 's'} · {chunkCount} chunk            {chunkCount === 1 ? '' : 's'}            {session.busy && ' · corpus ingest in progress - reindex is available when idle'}          </p>        </div>      )}      <div className="grid items-start gap-4 lg:grid-cols-2">        {}        <div className="flex min-w-0 flex-col gap-4">          <SectionCard title="Import vectors">            <div className="flex flex-wrap items-center gap-2">              <span className="text-xs text-muted-foreground">                Pinecone JSON · ChromaDB JSON · CSV · JSONL · Native JSON (≤50{' '}MB)              </span>              {(plan || isParsing) && (                <span                  role="group"                  aria-label="Detected import format"                  className="inline-flex"                >                  <FormatDetectionBadge format={plan?.formatId ?? null} colorMap={FORMAT_COLOR_MAP} />                </span>              )}            </div>            {importErrorMessage && (              <p                className="flex items-start gap-1.5 rounded-lg border border-destructive/30 bg-destructive/5 px-3 py-2 text-xs text-destructive"              >                <AlertTriangle className="mt-0.5 size-3.5 shrink-0" aria-hidden="true" />                <span className="min-w-0 flex-1">{importErrorMessage}</span>              </p>            )}            {}            {stage === 'idle' && (              <div className="flex flex-col gap-3">                <div                  role="button"                  aria-label="Import file upload"                  tabIndex={0}                  onClick={() => fileInputRef.current?.click()}                  onKeyDown={(e) => {                    if (e.key === 'Enter' || e.key === ' ') fileInputRef.current?.click();                  }}                  onDragOver={(e) => e.preventDefault()}                  onDrop={(e) => {                    e.preventDefault();                    const file = e.dataTransfer.files[0];                    if (file) void handleFile(file);                  }}                  className="flex cursor-pointer flex-col items-center gap-2 rounded-xl border-2 border-dashed border-border p-6 text-center transition-colors hover:border-primary/50"                >                  <Upload className="size-6 text-muted-foreground" aria-hidden="true" />                  <p className="text-sm font-medium">Drop an export file or click to browse</p>                  <p className="text-xs text-muted-foreground">                    {SUPPORTED_EXTENSIONS.join(', ')} - the format is auto-detected from content                  </p>                  <input                    ref={fileInputRef}                    type="file"                    accept={SUPPORTED_EXTENSIONS.join(',')}                    className="hidden"                    onChange={(e) => {                      const file = e.target.files?.[0];                      if (file) void handleFile(file);                    }}                  />                </div>                <div className="flex flex-col gap-1.5">                  <label                    htmlFor="vdm-import-paste-input"                    className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground"                  >                    <ClipboardPaste className="size-3.5" aria-hidden="true" />                    Or paste export content                  </label>                  <textarea                    id="vdm-import-paste-input"                    value={pasteText}                    onChange={(e) => setPasteText(e.target.value)}                    rows={3}                    placeholder='{"vectors": [...]} · {"ids": [...]} · CSV · JSONL'                    className="w-full resize-y rounded-lg border border-border bg-background px-3 py-2 font-mono text-xs outline-none focus-visible:ring-[3px] focus-visible:ring-ring/40"                  />                  <button                    type="button"                    disabled={!pasteText.trim() || isParsing}                    onClick={() => void handleContent(pasteText, null)}                    className="self-end rounded-md border border-border bg-background px-3 py-1.5 text-xs font-medium transition-colors hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:pointer-events-none disabled:opacity-50"                  >                    {isParsing ? 'Parsing…' : 'Parse pasted content'}                  </button>                </div>              </div>            )}            {}            {plan && (stage === 'preview' || stage === 'importing') && (              <div                role="group"                aria-label="Import preview"                data-total-records={plan.totalRecords}                data-with-vectors={plan.recordsWithVectors}                data-text-only={plan.recordsWithTextOnly}                data-dimensions={plan.dimensions ?? ''}                data-importable={docsToImport}                className="flex min-w-0 flex-col gap-3 overflow-x-auto"              >                <p className="flex items-center gap-1.5 text-xs text-muted-foreground">                  <FileText className="size-3.5 shrink-0" aria-hidden="true" />                  <span className="min-w-0 truncate font-medium text-foreground">                    {plan.fileName ?? 'Pasted content'}                  </span>                  <span>· {FORMAT_LABELS[plan.formatId] ?? plan.formatId}</span>                </p>                <dl className="grid grid-cols-2 gap-2 text-xs sm:grid-cols-4">                  <PreviewStat label="Total" value={plan.totalRecords.toLocaleString()} />                  <PreviewStat label="With vectors" value={plan.recordsWithVectors.toLocaleString()} />                  <PreviewStat label="Text-only" value={plan.recordsWithTextOnly.toLocaleString()} />                  <PreviewStat                    label="Dimensions"                    value={plan.dimensions !== null ? String(plan.dimensions) : '-'}                  />                </dl>                {dimensionMismatch && (                  <p className="flex items-start gap-1.5 text-xs text-amber-600 dark:text-amber-400">                    <AlertTriangle className="mt-0.5 size-3.5 shrink-0" aria-hidden="true" />                    <span>                      Dimension mismatch: source {plan.dimensions}d vs current index {targetDims}d.                      Text-bearing records are re-embedded with the session model.                    </span>                  </p>                )}                {}                {plan.lanes.vectorOnly > 0 && (                  <p className="flex items-start gap-1.5 text-xs text-amber-600 dark:text-amber-400">                    <AlertTriangle className="mt-0.5 size-3.5 shrink-0" aria-hidden="true" />                    <span>                      {plan.lanes.vectorOnly} vector-only record                      {plan.lanes.vectorOnly === 1 ? '' : 's'} will be skipped: the corpus only                      accepts text documents (raw-vector insert is not part of the engine's ingest                      API).                      {plan.lanes.vectorOnlyMismatched > 0 &&                        ` ${plan.lanes.vectorOnlyMismatched} of them also mismatch the current index dimensions.`}                    </span>                  </p>                )}                {plan.recordsWithVectors > 0 && plan.lanes.direct.length > 0 && (                  <p className="text-xs text-muted-foreground">                    Text-bearing records import as documents and are re-embedded by the engine -                    their source vectors are not inserted.                  </p>                )}                {}                {plan.lanes.needsEmbed.length > 0 && (                  <div className="flex items-center justify-between gap-3 rounded-lg border border-border bg-background px-3 py-2">                    <div className="min-w-0">                      <p className="text-xs font-medium">Re-embed records without usable vectors</p>                      <p className="text-xs text-muted-foreground">                        {plan.lanes.needsEmbed.length} record                        {plan.lanes.needsEmbed.length === 1 ? ' has' : 's have'} text but a missing                        or dimension-mismatched vector. Enable to embed them with the session model (                        {session.embeddingModelId}); otherwise they are skipped.                      </p>                    </div>                    <button                      type="button"                      role="switch"                      aria-checked={reEmbed}                      disabled={importing}                      onClick={() => setReEmbed((v) => !v)}                      className={`relative h-5 w-9 shrink-0 rounded-full transition-colors focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 ${                        reEmbed ? 'bg-primary' : 'bg-muted'                      }`}                    >                      <span                        className={`absolute top-0.5 size-4 rounded-full bg-background shadow transition-all ${                          reEmbed ? 'left-[1.125rem]' : 'left-0.5'                        }`}                      />                      <span className="sr-only">Re-embed records without usable vectors</span>                    </button>                  </div>                )}                {}                <VectorImportFlow                  className="max-w-none"                  records={plan.previewRows}                  progress={importProgress}                  isImporting={importing}                />                {}                {importing && importProgress && (                  <span                    className="sr-only"                    data-phase={importProgress.phase}                    data-completed={importProgress.overallCompleted}                    data-total={importProgress.overallTotal}                  >                    {importProgress.phase} {importProgress.overallCompleted}/                    {importProgress.overallTotal}                  </span>                )}                <div className="flex items-center justify-end gap-2">                  <button                    type="button"                    onClick={resetImport}                    disabled={importing}                    className="rounded-md border border-border bg-background px-3 py-1.5 text-xs font-medium transition-colors hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:pointer-events-none disabled:opacity-50"                  >                    Change file                  </button>                  {importing ? (                    <button                      type="button"                      onClick={cancelImport}                      className="inline-flex items-center gap-1.5 rounded-md border border-destructive/40 bg-destructive/5 px-3 py-1.5 text-xs font-medium text-destructive transition-colors hover:bg-destructive/10 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                    >                      <Square className="size-3.5" aria-hidden="true" />                      Cancel import                    </button>                  ) : (                    <button                      type="button"                      disabled={docsToImport === 0 || session.busy || reindexRunning}                      onClick={() => void runImport()}                      className="inline-flex items-center gap-1.5 rounded-md bg-primary px-3 py-1.5 text-xs font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:pointer-events-none disabled:opacity-50"                    >                      <Play className="size-3.5" aria-hidden="true" />                      Import {docsToImport.toLocaleString()} of {plan.totalRecords.toLocaleString()}{' '}                      records                    </button>                  )}                </div>              </div>            )}            {}            {stage === 'complete' && importStats && (              <div                role="group"                aria-label="Import result"                data-imported={importStats.imported}                data-skipped={importStats.skipped}                data-reembedded={importStats.reEmbedded}                data-total={importStats.totalParsed}                data-format={importStats.formatId}                data-cancelled={importStats.cancelled ? 'true' : 'false'}                data-duration-ms={Math.round(importStats.durationMs)}                className="flex flex-col gap-2"              >                <p className="flex items-center gap-1.5 text-sm font-medium">                  {importStats.cancelled ? (                    <>                      <AlertTriangle className="size-4 shrink-0 text-amber-500" aria-hidden="true" />                      Import cancelled                    </>                  ) : (                    <>                      <CheckCircle2 className="size-4 shrink-0 text-emerald-500" aria-hidden="true" />                      Import complete                    </>                  )}                </p>                <p className="text-xs text-muted-foreground">                  Imported {importStats.imported.toLocaleString()} · Skipped{' '}                  {importStats.skipped.toLocaleString()} · Re-embedded{' '}                  {importStats.reEmbedded.toLocaleString()} ·{' '}                  {FORMAT_LABELS[importStats.formatId] ?? importStats.formatId} ·{' '}                  {formatDuration(importStats.durationMs)}                </p>                <button                  type="button"                  onClick={resetImport}                  className="inline-flex items-center gap-1.5 self-start rounded-md border border-border bg-background px-3 py-1.5 text-xs font-medium transition-colors hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                >                  <RotateCcw className="size-3.5" aria-hidden="true" />                  Import another file                </button>              </div>            )}          </SectionCard>          <SectionCard title="Export corpus">            <div              role="group"              aria-label="Export panel"              data-record-count={session.documents.length}              data-exporting={exportingFormat ?? undefined}              data-last-format={lastExport?.formatId ?? undefined}              data-last-records={lastExport?.records ?? undefined}              data-last-bytes={lastExport?.bytes ?? undefined}              className="min-w-0 overflow-x-auto"            >              <VectorExportPanel                className="max-w-none"                formats={EXPORT_FORMATS}                recordCount={session.documents.length}                exporting={exportingFormat ?? false}                {...(lastExport ? { lastExport } : {})}                onExport={handleExport}                disabled={exportDisabled}              />            </div>            {}            <div className="sr-only">              {EXPORT_FORMATS.map((f) => (                <button                  key={f.id}                  type="button"                  disabled={exportDisabled}                  onClick={() => handleExport(f.id)}                  className="focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                >                  Export {f.label}                </button>              ))}            </div>            <p className="text-xs text-muted-foreground">              Exports carry the raw documents and metadata. Stored vectors are regenerated by              re-embedding on import. Native JSON is lossless and round-trip re-importable; CSV/JSONL              round-trip text + metadata.            </p>          </SectionCard>        </div>        {}        <div className="flex min-w-0 flex-col gap-4">          <SectionCard title="Storage">            <div              role="group"              aria-label="Storage usage"              data-used-bytes={quota?.usedBytes ?? undefined}              data-quota-bytes={quota?.quotaBytes ?? undefined}              className="min-w-0 overflow-x-auto"            >              <StorageMeter                className="max-w-none"                quota={                  quota && quota.quotaBytes > 0                    ? { usedBytes: quota.usedBytes, quotaBytes: quota.quotaBytes }                    : undefined                }              />            </div>          </SectionCard>          <SectionCard title="Vector storage">            <div              role="group"              aria-label="Vector storage stats"              data-docs={session.documents.length}              data-chunks={chunkCount}              data-dimensions={dims}              className="flex min-w-0 flex-col gap-3 overflow-x-auto"            >              <VectorStorageObservability                className="max-w-none"                stats={{                  ratio: 1,                  originalSizeBytes: estimatedBytes,                  compressedSizeBytes: estimatedBytes,                  vectorCount: chunkCount,                }}                tier="raw"              />              <p className="text-xs text-muted-foreground">                {session.documents.length} document{session.documents.length === 1 ? '' : 's'} ·{' '}                {chunkCount} chunk{chunkCount === 1 ? '' : 's'} ·{' '}                {dims > 0 ? `${dims}d vectors` : 'dimensions unknown'} (engine: {session.engineKind})              </p>              {(sourceCounts.length > 0 || categoryCounts.length > 0) && (                <dl className="flex flex-col gap-1 text-xs text-muted-foreground">                  {sourceCounts.length > 0 && (                    <div className="flex flex-wrap gap-x-3 gap-y-1">                      <dt className="font-medium text-foreground">By source:</dt>                      {sourceCounts.map(([source, count]) => (                        <dd key={source} className="tabular-nums">                          {source} · {count}                        </dd>                      ))}                    </div>                  )}                  {categoryCounts.length > 0 && (                    <div className="flex flex-wrap gap-x-3 gap-y-1">                      <dt className="font-medium text-foreground">By category:</dt>                      {categoryCounts.map(([category, count]) => (                        <dd key={category} className="tabular-nums">                          {category} · {count}                        </dd>                      ))}                    </div>                  )}                </dl>              )}              {}              <p className="text-xs text-muted-foreground">                Sizes are estimated as chunks × dimensions × 4 bytes (Raw F32). Live compression                stats via <code className="font-mono">getCompressionStats()</code> need the                underlying VectorDB, which the core engine encapsulates; per-document chunk counts                are likewise unavailable (<code className="font-mono">engine.stats()</code> reports                totals only).              </p>            </div>          </SectionCard>          <SectionCard title="Maintenance">            <div className="flex items-center justify-between gap-3">              <div className="min-w-0">                <p className="text-xs font-medium">Reindex corpus</p>                <p className="text-xs text-muted-foreground">                  Re-embeds all {session.documents.length} document                  {session.documents.length === 1 ? '' : 's'} ({chunkCount} chunk                  {chunkCount === 1 ? '' : 's'}) with the active model and chunking mode (                  {session.chunking}).                </p>              </div>              {reindexRunning ? (                <button                  type="button"                  onClick={cancelReindex}                  className="inline-flex shrink-0 items-center gap-1.5 rounded-md border border-destructive/40 bg-destructive/5 px-3 py-1.5 text-xs font-medium text-destructive transition-colors hover:bg-destructive/10 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                >                  <Square className="size-3.5" aria-hidden="true" />                  Cancel                </button>              ) : (                <button                  type="button"                  disabled={session.busy || importing || session.documents.length === 0}                  onClick={() => void reindex()}                  className="inline-flex shrink-0 items-center gap-1.5 rounded-md bg-primary px-3 py-1.5 text-xs font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:pointer-events-none disabled:opacity-50"                >                  <RefreshCw className="size-3.5" aria-hidden="true" />                  Reindex                </button>              )}            </div>            {reindexRunning && reindexProgress && (              <p className="text-xs tabular-nums text-muted-foreground">                {reindexProgress.phase === 'indexing' ? 'Rebuilding index' : 'Re-embedding'} -{' '}                {reindexProgress.completed}/{reindexProgress.total} documents              </p>            )}            {reindexError && (              <p className="flex items-start gap-1.5 text-xs text-destructive">                <AlertTriangle className="mt-0.5 size-3.5 shrink-0" aria-hidden="true" />                {reindexError}              </p>            )}            <p className="text-xs text-muted-foreground">              Reindex snapshots the documents, clears the index, and re-adds them in batches of{' '}              {Math.max(1, batch.batchSize)}. Cancelling stops between batches - documents from the              snapshot not yet re-added are not restored.            </p>          </SectionCard>          <div            data-batch-size={batch.batchSize}            className="flex flex-col gap-1.5"          >            {mounted && <AdaptiveBatchCard className="max-w-none" result={batch} />}            <p className="text-xs text-muted-foreground">              Used as the ingest batch size for imports and reindexing on this device.            </p>          </div>        </div>      </div>    </div>  );}export function VectorDataManagerBlock() {  const idRef = useRef(DEFAULT_EMBEDDING_MODEL_ID);  const kb = useKnowledgeBase({    embeddingModelId: DEFAULT_EMBEDDING_MODEL_ID,    engineKind: 'core',    createEmbeddingModel: (id, onProgress) =>      transformers.embedding(id, { onProgress: (p) => onProgress(p as AnyLoadProgress) }),    isModelCached: (id) => isModelCached(id),    createEngine: () => {      const embeddingModel = transformers.embedding(idRef.current);      const getLanguageModel = async () => {        const device = (await isWebGPUSupported()) ? 'webgpu' : 'wasm';        return transformers.languageModel(ANSWER_MODEL_ID, { device });      };      return createKnowledgeBaseEngine({ embeddingModel, getLanguageModel });    },  });  idRef.current = kb.embeddingModelId;  const [seedText, setSeedText] = useState('');  const addSeed = async () => {    const text = seedText.trim();    if (!text || !kb.engine || kb.busy) return;    await kb.addDocuments([{ title: firstLine(text) || 'Untitled note', text, source: 'text' }]);    setSeedText('');  };  const session: Session | null = kb.engine    ? {        engine: kb.engine,        engineKind: 'core',        documents: kb.documents,        addDocuments: kb.addDocuments,        removeDocument: kb.removeDocument,        clearAll: kb.clearAll,        chunking: kb.chunking,        setChunking: kb.setChunking,        chunkSize: kb.chunkSize,        setChunkSize: kb.setChunkSize,        embeddingModelId: kb.embeddingModelId,        setEmbeddingModelId: kb.setEmbeddingModelId,        busy: kb.busy,        error: kb.error,      }    : null;  const modelMeta = EMBEDDING_MODEL_META[kb.embeddingModelId] ?? {    name: kb.embeddingModelId,    size: '',  };  const statusText = kb.busy    ? kb.modelStatus === 'loading'      ? `loading embedding model… ${Math.round(kb.modelProgress * 100)}%`      : kb.switching        ? `re-embedding ${kb.documents.length} docs through ${modelMeta.name}…`        : 'working…'    : kb.error      ? 'error'      : kb.documents.length > 0        ? `ready - ${kb.documents.length} docs · ${kb.stats?.chunks ?? 0} chunks`        : 'idle - import an export file or add a note to build the corpus';  return (    <div className="mx-auto flex max-w-5xl flex-col gap-4 p-4">      {}      <p        role="status"        aria-live="polite"        className="text-xs text-muted-foreground"      >        {statusText}      </p>      {kb.error && (        <p className="text-xs text-destructive">          {kb.error}        </p>      )}      {}      <div        role="group"        aria-label="Embedding model status"        data-status={kb.modelStatus}        data-model-id={kb.embeddingModelId}        className="flex flex-col gap-2 rounded-xl border border-border bg-card p-4"      >        <div className="flex flex-wrap items-center gap-2">          <span className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">            Embedding model          </span>          <div            role="group"            aria-label="Embedding model"            className="inline-flex items-center rounded-md border border-border bg-muted/40 p-0.5"          >            {EMBEDDING_MODEL_IDS.map((id) => (              <button                key={id}                type="button"                aria-pressed={kb.embeddingModelId === id}                disabled={kb.busy || !kb.engine || kb.embeddingModelId === id}                onClick={() => kb.setEmbeddingModelId(id)}                className={cn(                  'inline-flex h-7 items-center rounded px-2.5 text-xs font-medium transition-colors focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50',                  kb.embeddingModelId === id                    ? 'bg-background text-foreground shadow-sm'                    : 'text-muted-foreground hover:text-foreground',                  (kb.busy || !kb.engine) && kb.embeddingModelId !== id && 'opacity-60',                )}              >                {EMBEDDING_MODEL_META[id].name}              </button>            ))}          </div>        </div>        {kb.modelStatus === 'idle' ? (          <div className="flex flex-wrap items-center gap-2">            <p className="text-xs text-muted-foreground">              <span className="font-medium">{modelMeta.name}</span>              {modelMeta.size ? ` (${modelMeta.size})` : ''} - not loaded. It downloads on Load, the              first import/seed, or a model switch.            </p>            <button              type="button"              disabled={!kb.engine || kb.busy}              onClick={() => void kb.loadModel()}              className="inline-flex h-7 items-center rounded-md bg-primary px-3 text-xs font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:opacity-50"            >              Load embedding model            </button>          </div>        ) : (          <ModelDownloader            name={modelMeta.name}            size={modelMeta.size || undefined}            category="Embedding"            progress={kb.modelProgressValue}            cached={kb.modelCached}            ready={kb.modelReady}            className="max-w-sm"          />        )}        <p className="text-xs text-muted-foreground">          Switching the model re-embeds the whole corpus through the new space (drift banner appears          until the re-ingest settles).        </p>      </div>      {}      <div className="flex flex-col gap-1.5 rounded-xl border border-border bg-card p-4">        <div className="flex items-center justify-between">          <span className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">            Seed the corpus          </span>          <button            type="button"            onClick={() => setSeedText(SAMPLE_NOTE)}            className="inline-flex h-7 items-center rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground transition-colors hover:bg-accent hover:text-foreground focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"          >            Load sample          </button>        </div>        <textarea          aria-label="Seed text"          value={seedText}          onChange={(e) => setSeedText(e.target.value)}          rows={3}          placeholder="Paste a note or passage to add it as a document…"          className="w-full resize-y rounded-lg border border-border bg-background px-3 py-2 text-sm outline-none focus-visible:ring-[3px] focus-visible:ring-ring/40"        />        <button          type="button"          disabled={!kb.engine || kb.busy || !seedText.trim()}          onClick={() => void addSeed()}          className="inline-flex h-8 items-center gap-1.5 self-start rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:opacity-50"        >          <Plus className="size-3.5" aria-hidden="true" />          {kb.busy ? 'Adding…' : 'Add note'}        </button>      </div>      {}      {session ? (        <DataManager session={session} />      ) : (        <p className="p-4 text-sm text-muted-foreground">Preparing engine…</p>      )}    </div>  );}