Data Extractor

'use client';/** * @file data-extractor.tsx * @description Self-sufficient Data Extractor block — its own WebGPU-only WebLLM load plus schema-validated JSON extraction (5 zod templates, retry/self-correction) docked into the artifacts canvas (sortable table + chart). */import { useState } from 'react';import { AlertTriangle } from 'lucide-react';import { jsonSchema, type LanguageModel, type ObjectSchema } from '@localmode/core';import { useGenerateObject, useModelLoad } from '@localmode/react';import { webllm, WEBLLM_MODELS, isModelCached as webllmIsModelCached } from '@localmode/webllm';import { z } from 'zod';import { ModelSelector, type SelectableModel } from '@/components/model-selector';import { ModelDownloader } from '@/components/model-downloader';import { CapabilityGate } from '@/components/capability-gate';import { StructuredOutputViewer } from '@/components/structured-output-viewer';import {  Artifact,  ArtifactHeader,  ArtifactTitle,  ArtifactDescription,  ArtifactActions,  ArtifactAction,  ArtifactContent,} from '@/components/artifact';import { DataTableArtifact } from '@/components/data-table-artifact';import { ChartArtifact } from '@/components/chart-artifact';import { InMessageError } from '@/components/in-message-error';import { useCapabilities } from '@/lib/use-environment';import { cn } from '@/lib/utils';export const DEFAULT_MODEL_ID = 'Qwen3-1.7B-q4f16_1-MLC';export type ModelTier = 'medium' | 'large';export interface AgentModelEntry {  id: string;  name: string;  size: string;  sizeBytes: number;  tier: ModelTier;  contextLength: number;  description?: string;}const CURATED_MODEL_IDS: readonly string[] = [  'Qwen3-1.7B-q4f16_1-MLC',  'Qwen2.5-3B-Instruct-q4f16_1-MLC',  'Llama-3.2-3B-Instruct-q4f16_1-MLC',  'Hermes-3-Llama-3.2-3B-q4f16_1-MLC',  'Phi-3.5-mini-instruct-q4f16_1-MLC',  'Phi-3-mini-4k-instruct-q4f16_1-MLC',  'Qwen3-4B-q4f16_1-MLC',  'Mistral-7B-Instruct-v0.3-q4f16_1-MLC',  'Qwen2.5-7B-Instruct-q4f16_1-MLC',  'DeepSeek-R1-Distill-Qwen-7B-q4f16_1-MLC',  'DeepSeek-R1-Distill-Llama-8B-q4f16_1-MLC',  'Hermes-3-Llama-3.1-8B-q4f16_1-MLC',  'Llama-3.1-8B-Instruct-q4f16_1-MLC',  'Qwen3-8B-q4f16_1-MLC',];interface WebLLMCatalogEntry {  name: string;  contextLength: number;  sizeBytes: number;  size: string;  description: string;}const WEBLLM_ENTRIES = WEBLLM_MODELS as Record<string, WebLLMCatalogEntry>;const LARGE_TIER_BYTES = 2.2 * 1024 * 1024 * 1024;let cachedCatalog: AgentModelEntry[] | null = null;export function getAgentModelCatalog(): AgentModelEntry[] {  if (cachedCatalog) return cachedCatalog;  const entries: AgentModelEntry[] = [];  for (const id of CURATED_MODEL_IDS) {    const entry = WEBLLM_ENTRIES[id];    if (!entry) continue;    entries.push({      id,      name: entry.name,      size: entry.size,      sizeBytes: entry.sizeBytes,      tier: entry.sizeBytes >= LARGE_TIER_BYTES ? 'large' : 'medium',      contextLength: entry.contextLength,      description: entry.description,    });  }  cachedCatalog = entries;  return entries;}export function getModelEntry(id: string): AgentModelEntry {  const found = getAgentModelCatalog().find((m) => m.id === id);  if (found) return found;  const entry = WEBLLM_ENTRIES[id];  return {    id,    name: entry?.name ?? id,    size: entry?.size ?? '',    sizeBytes: entry?.sizeBytes ?? 0,    tier: (entry?.sizeBytes ?? 0) >= LARGE_TIER_BYTES ? 'large' : 'medium',    contextLength: entry?.contextLength ?? 4096,    description: entry?.description,  };}export function createAgentModel(  modelId: string,  onProgress?: (p: unknown) => void,): LanguageModel {  return webllm.languageModel(modelId, { onProgress });}export function isAgentModelCached(modelId: string): Promise<boolean> {  return webllmIsModelCached(modelId);}export type TemplateId = 'contact' | 'event' | 'review' | 'recipe' | 'job';export interface ExtractionTemplate {  id: TemplateId;  name: string;  schema: ObjectSchema<unknown>;  schemaDisplay: string;  sampleText: string;}const contactSchema = z.object({  name: z.string().describe('Full name'),  email: z.string().describe('Email address'),  phone: z.string().optional().describe('Phone number'),  company: z.string().optional().describe('Company name'),});const eventSchema = z.object({  title: z.string().describe('Event title'),  date: z.string().describe('Event date'),  location: z.string().describe('Event location'),  description: z.string().optional().describe('Brief description'),});const reviewSchema = z.object({  product: z.string().describe('Product name'),  rating: z.number().describe('Rating from 1 to 5'),  pros: z.array(z.string()).describe('List of positive aspects'),  cons: z.array(z.string()).describe('List of negative aspects'),});const recipeSchema = z.object({  name: z.string().describe('Recipe name'),  servings: z.number().describe('Number of servings'),  ingredients: z    .array(      z.object({        item: z.string().describe('Ingredient name'),        amount: z.string().describe('Amount with unit'),      }),    )    .describe('List of ingredients'),  steps: z.array(z.string()).describe('Cooking steps'),});const jobSchema = z.object({  title: z.string().describe('Job title'),  company: z.string().describe('Company name'),  salary: z.string().optional().describe('Salary range'),  requirements: z.array(z.string()).describe('Job requirements'),  location: z.string().describe('Job location'),});export const DEFAULT_TEMPLATE_ID: TemplateId = 'contact';export const TEMPLATES: readonly ExtractionTemplate[] = [  {    id: 'contact',    name: 'Contact Info',    schema: jsonSchema(contactSchema),    schemaDisplay: '{ name, email, phone?, company? }',    sampleText:      'Hi, my name is Sarah Chen. You can reach me at sarah.chen@acme.co or call 555-0147. I work at Acme Corporation as a Senior Engineer.',  },  {    id: 'event',    name: 'Event Details',    schema: jsonSchema(eventSchema),    schemaDisplay: '{ title, date, location, description? }',    sampleText:      'Join us for the Annual Tech Summit on March 15, 2027 at the SF Convention Center. This year we focus on AI, privacy, and the future of local-first computing.',  },  {    id: 'review',    name: 'Product Review',    schema: jsonSchema(reviewSchema),    schemaDisplay: '{ product, rating, pros[], cons[] }',    sampleText:      "I bought the NovaPhone X200 last month. It's fantastic: the camera is incredible, battery lasts two days, and the display is gorgeous. However, it's quite heavy and the price is steep at $1200. I'd give it 4 out of 5 stars.",  },  {    id: 'recipe',    name: 'Recipe',    schema: jsonSchema(recipeSchema),    schemaDisplay: '{ name, servings, ingredients[{item,amount}], steps[] }',    sampleText:      'Classic Pancakes (serves 4): Mix 1.5 cups flour, 3.5 tsp baking powder, 1 tbsp sugar, and a pinch of salt. In another bowl, combine 1.25 cups milk, 1 egg, and 3 tbsp melted butter. Mix wet into dry until smooth. Pour 1/4 cup batter onto a hot griddle. Cook until bubbles form, flip, cook until golden. Serve with maple syrup.',  },  {    id: 'job',    name: 'Job Posting',    schema: jsonSchema(jobSchema),    schemaDisplay: '{ title, company, salary?, requirements[], location }',    sampleText:      'We are hiring a Senior Frontend Engineer at CloudTech Inc. The position is based in Austin, TX with a salary range of $150K-$190K. Requirements: 5+ years of React experience, TypeScript proficiency, experience with Next.js, and familiarity with CI/CD pipelines.',  },];export function getTemplate(id: TemplateId): ExtractionTemplate {  return TEMPLATES.find((t) => t.id === id) ?? TEMPLATES[0];}export interface DerivedColumn {  key: string;  header: string;}export interface DerivedTable {  columns: DerivedColumn[];  rows: Record<string, unknown>[];}export interface DerivedChart {  type: 'gauge' | 'bar' | 'line' | 'area' | 'scatter' | 'radar';  data: Array<{ x?: number; y?: number; label?: string; value?: number }>;  max?: number;  title: string;}export interface DerivedArtifacts {  table: DerivedTable;  chart: DerivedChart | null;  chartEmptyReason?: string;}function asText(value: unknown): string {  if (value == null) return '';  if (typeof value === 'string') return value;  if (typeof value === 'number' || typeof value === 'boolean') return String(value);  return JSON.stringify(value);}function parseSalaries(text: string): number[] {  const out: number[] = [];  const re = /\$?\s?([\d,]+(?:\.\d+)?)\s?([kmKM])?/g;  let match: RegExpExecArray | null;  while ((match = re.exec(text)) !== null) {    const base = Number.parseFloat(match[1].replace(/,/g, ''));    if (!Number.isFinite(base)) continue;    const suffix = (match[2] ?? '').toLowerCase();    const mult = suffix === 'k' ? 1_000 : suffix === 'm' ? 1_000_000 : 1;    out.push(base * mult);  }  return out;}export function deriveArtifacts(templateId: TemplateId, object: unknown): DerivedArtifacts {  const obj = (object ?? {}) as Record<string, unknown>;  switch (templateId) {    case 'review': {      const pros = Array.isArray(obj.pros) ? obj.pros : [];      const cons = Array.isArray(obj.cons) ? obj.cons : [];      const rows: Record<string, unknown>[] = [        ...pros.map((p) => ({ aspect: asText(p), sentiment: 'pro' })),        ...cons.map((c) => ({ aspect: asText(c), sentiment: 'con' })),      ];      const rating = typeof obj.rating === 'number' ? obj.rating : Number.NaN;      const chart: DerivedChart | null = Number.isFinite(rating)        ? { type: 'gauge', data: [{ value: rating }], max: 5, title: 'Rating (out of 5)' }        : null;      return {        table: {          columns: [            { key: 'aspect', header: 'Aspect' },            { key: 'sentiment', header: 'Sentiment' },          ],          rows,        },        chart,        ...(chart ? {} : { chartEmptyReason: 'No numeric rating was extracted.' }),      };    }    case 'recipe': {      const ingredients = Array.isArray(obj.ingredients) ? obj.ingredients : [];      const steps = Array.isArray(obj.steps) ? obj.steps : [];      const rows: Record<string, unknown>[] = [        ...ingredients.map((ing) => {          const record = (ing ?? {}) as Record<string, unknown>;          return { entry: asText(record.item), detail: asText(record.amount), kind: 'ingredient' };        }),        ...steps.map((s, i) => ({ entry: `Step ${i + 1}`, detail: asText(s), kind: 'step' })),      ];      const servings = typeof obj.servings === 'number' ? obj.servings : Number.NaN;      const chart: DerivedChart = {        type: 'bar',        data: [          { label: 'Servings', value: Number.isFinite(servings) ? servings : 0 },          { label: 'Ingredients', value: ingredients.length },          { label: 'Steps', value: steps.length },        ],        title: 'Recipe counts',      };      return {        table: {          columns: [            { key: 'entry', header: 'Item / Step' },            { key: 'detail', header: 'Detail' },            { key: 'kind', header: 'Kind' },          ],          rows,        },        chart,      };    }    case 'job': {      const requirements = Array.isArray(obj.requirements) ? obj.requirements : [];      const rows: Record<string, unknown>[] = requirements.map((r, i) => ({        index: i + 1,        requirement: asText(r),      }));      const salaries = typeof obj.salary === 'string' ? parseSalaries(obj.salary) : [];      let chart: DerivedChart | null = null;      if (salaries.length >= 2) {        chart = {          type: 'bar',          data: [            { label: 'Min', value: Math.min(...salaries) },            { label: 'Max', value: Math.max(...salaries) },          ],          title: 'Salary range',        };      } else if (salaries.length === 1) {        chart = { type: 'bar', data: [{ label: 'Salary', value: salaries[0] }], title: 'Salary' };      }      return {        table: {          columns: [            { key: 'index', header: '#' },            { key: 'requirement', header: 'Requirement' },          ],          rows,        },        chart,        ...(chart ? {} : { chartEmptyReason: 'No parseable salary figure was extracted.' }),      };    }    case 'contact':    case 'event':    default: {      const rows: Record<string, unknown>[] = Object.entries(obj).map(([field, value]) => ({        field,        value: asText(value),      }));      return {        table: {          columns: [            { key: 'field', header: 'Field' },            { key: 'value', header: 'Value' },          ],          rows,        },        chart: null,        chartEmptyReason: 'This template extracts no numeric fields.',      };    }  }}function errorHint(error: unknown): string | null {  if (error && typeof error === 'object' && 'hint' in error) {    const hint = (error as { hint?: unknown }).hint;    if (typeof hint === 'string' && hint.length > 0) return hint;  }  return null;}export function DataExtractorBlock() {  const [modelId, setModelId] = useState(DEFAULT_MODEL_ID);  const { capabilities } = useCapabilities();  const hasWebGPU = Boolean(capabilities?.features.webgpu);  const webgpuUnsupported = capabilities != null && !hasWebGPU;  const modelEntry = getModelEntry(modelId);  const catalog = getAgentModelCatalog();  const {    model,    status,    progressValue,    cached,    error: modelError,    load,  } = useModelLoad<LanguageModel>({    key: `webllm:${modelId}`,    create: (onProgress) => createAgentModel(modelId, (p) => onProgress(p as never)),    isCached: () => isAgentModelCached(modelId),  });  const modelReady = status === 'ready';  const selectorModels: SelectableModel[] = catalog.map((m) => ({    id: m.id,    name: m.name,    backend: 'webgpu',    category: m.tier === 'large' ? 'Large (2.2GB+)' : 'Medium (1-2.2GB)',    size: m.size,    cached: m.id === modelId ? cached : undefined,  }));  const selectModel = (id: string) => {    if (id === modelId || status === 'loading') return;    setModelId(id);  };  const statusText =    status === 'ready'      ? `ready - ${modelEntry.name}`      : status === 'loading'        ? `loading ${modelEntry.name}… ${Math.round(progressValue.percent * 100)}%`        : status === 'error'          ? 'error'          : webgpuUnsupported            ? 'WebGPU required - this device cannot run these models'            : 'idle - select a model and click Load';  const [templateId, setTemplateId] = useState<TemplateId>(DEFAULT_TEMPLATE_ID);  const [input, setInput] = useState('');  const template = getTemplate(templateId);  const { data, error, isLoading, execute, cancel, reset } = useGenerateObject<unknown>({    model: model as LanguageModel,    schema: template.schema,    mode: 'json',    temperature: 0,    maxRetries: 3,    providerOptions: {      webllm: {        response_format: {          type: 'json_object',          schema: JSON.stringify(template.schema.jsonSchema),        },      },    },  });  const hasInput = input.trim().length > 0;  const canExtract = hasInput && modelReady && !isLoading;  const extract = () => {    if (!canExtract) return;    void execute(input);  };  const switchTemplate = (id: TemplateId) => {    if (id === templateId) return;    cancel();    reset();    setTemplateId(id);  };  const loadSample = () => setInput(template.sampleText);  const object = data?.object;  const attempts = data?.attempts ?? 0;  const usage = data?.usage;  const derived = data ? deriveArtifacts(templateId, object) : null;  const jsonText = data ? JSON.stringify(object, null, 2) : '';  const webgpuGate = (    <div      role="status"      className="flex flex-col gap-2 rounded-lg border border-amber-500/40 bg-amber-500/10 px-4 py-3 text-sm"    >      <span className="flex items-center gap-2 font-semibold text-amber-700 dark:text-amber-400">        <AlertTriangle className="size-4 shrink-0" aria-hidden="true" />        WebGPU required      </span>      <span className="text-muted-foreground">        This block runs WebLLM models, which need WebGPU - a modern GPU plus a        recent Chrome or Edge build. This browser or device exposes no WebGPU        adapter, so these models cannot load here. On a supported device, a model        selector and a Load button appear in this spot.      </span>    </div>  );  const loadArea = (    <div className="flex flex-col gap-2">      <p className="text-sm text-muted-foreground">        <span className="font-medium text-foreground">{modelEntry.name}</span>        {modelEntry.size ? ` (${modelEntry.size})` : ''} - not loaded. It downloads only when you        click Load.      </p>      <button        type="button"        onClick={() => void load()}        className="inline-flex h-9 w-fit items-center rounded-md bg-primary px-4 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"      >        Load model      </button>    </div>  );  const modelStatusGroup = (    <div      role="group"      aria-label="Model status"      data-status={status}      data-model-id={modelId}      className="flex flex-col gap-2"    >      {status === 'idle' ? (        <CapabilityGate requires="webgpu" fallback={webgpuGate}>          {loadArea}        </CapabilityGate>      ) : (        <ModelDownloader          name={modelEntry.name}          size={modelEntry.size || undefined}          contextLength={modelEntry.contextLength}          category="Chat"          progress={progressValue}          cached={cached}          ready={modelReady}          className="max-w-md"        />      )}    </div>  );  return (    <div className="mx-auto flex max-w-5xl flex-col gap-4 p-4">      {}      <p        role="status"        aria-live="polite"        data-state={status}        className="text-xs text-muted-foreground"      >        {statusText}      </p>      {modelError && (        <p role="status" className="text-xs text-destructive">          {modelError.message}        </p>      )}      {}      {status === 'idle' && webgpuUnsupported ? (        modelStatusGroup      ) : (        <div className="grid gap-4 md:grid-cols-[minmax(0,20rem)_1fr]">          <div className="flex flex-col gap-3">            <ModelSelector              models={selectorModels}              selectedId={modelId}              hasWebGPU={hasWebGPU}              onSelect={selectModel}            />          </div>          {modelStatusGroup}        </div>      )}      {}        <div className="min-h-48">          <div className="flex flex-col gap-4">            {}            <div className="flex flex-col gap-2">              <div                className="flex flex-wrap gap-1.5"                role="group"                aria-label="Extraction template"                data-template={templateId}              >                {TEMPLATES.map((t) => (                  <button                    key={t.id}                    type="button"                    aria-pressed={t.id === templateId}                    onClick={() => switchTemplate(t.id)}                    className={cn(                      'inline-flex h-8 items-center rounded-md border px-3 text-xs font-medium transition-colors focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50',                      t.id === templateId                        ? 'border-primary bg-primary/10 text-primary'                        : 'border-border bg-background text-muted-foreground hover:text-foreground',                    )}                  >                    {t.name}                  </button>                ))}              </div>              <div className="flex items-center gap-2 text-xs text-muted-foreground">                <span className="font-medium">Schema</span>                <code                  className="rounded bg-muted px-1.5 py-0.5 font-mono text-[11px]"                >                  {template.schemaDisplay}                </code>              </div>            </div>            {}            <div className="flex flex-col gap-2">              <textarea                aria-label="Text to extract from"                value={input}                onChange={(e) => setInput(e.target.value)}                onKeyDown={(e) => {                  if (e.key === 'Enter' && (e.metaKey || e.ctrlKey) && canExtract) {                    e.preventDefault();                    extract();                  }                }}                rows={5}                placeholder={                  modelReady                    ? 'Paste free text to extract structured data from…'                    : 'Load the model to start extracting…'                }                className="w-full resize-y rounded-lg border border-border bg-background p-3 text-sm focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"              />              <div className="flex flex-wrap items-center gap-2">                <button                  type="button"                  onClick={loadSample}                  className="inline-flex h-8 items-center rounded-md border border-border px-3 text-xs font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                >                  Load Sample                </button>                {isLoading ? (                  <button                    type="button"                    onClick={cancel}                    className="inline-flex h-8 items-center rounded-md border border-border px-3 text-xs font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                  >                    Cancel                  </button>                ) : (                  <button                    type="button"                    onClick={extract}                    disabled={!canExtract}                    className="inline-flex h-8 items-center rounded-md bg-primary px-3 text-xs font-medium text-primary-foreground transition-colors hover:bg-primary/90 disabled:opacity-50 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                  >                    Extract                  </button>                )}                <span className="text-xs text-muted-foreground">                  {isLoading ? 'Extracting…' : 'Cmd/Ctrl+Enter to extract'}                </span>              </div>            </div>            {}            <div className="sr-only">              <div                aria-label="Extraction state"                data-loading={isLoading ? 'loading' : 'idle'}                data-has-result={data ? 'true' : 'false'}                data-attempts={attempts}                data-template={templateId}                data-object={jsonText}              />            </div>            {}            {!data && !error && (              <p                role="status"                className="rounded-lg border border-dashed border-border p-6 text-center text-sm text-muted-foreground"              >                {isLoading                  ? 'Extracting structured data…'                  : 'Pick a template, load or paste text, and extract to see the validated JSON here.'}              </p>            )}            {data && (              <>                {}                <div className="flex items-center gap-2">                  <span                    data-attempts={attempts}                    className="inline-flex items-center rounded-full border border-emerald-500/30 bg-emerald-500/10 px-2.5 py-0.5 text-xs font-medium text-emerald-600 dark:text-emerald-400"                  >                    Attempt {attempts}/3                  </span>                  {attempts > 1 && (                    <span className="text-xs text-muted-foreground">                      Recovered via schema self-correction                    </span>                  )}                </div>                {}                <div role="group" aria-label="Extracted data">                  <StructuredOutputViewer                    object={object}                    usage={usage}                    durationMs={usage?.durationMs}                    attempts={attempts}                  />                </div>                {}                {derived && (                  <Artifact className="w-full">                    <ArtifactHeader>                      <div className="min-w-0">                        <ArtifactTitle>{template.name}</ArtifactTitle>                        <ArtifactDescription>                          Extracted on-device • {derived.table.rows.length} row                          {derived.table.rows.length === 1 ? '' : 's'}                        </ArtifactDescription>                      </div>                      <ArtifactActions>                        <ArtifactAction                          label="Copy JSON"                          content={jsonText}                        />                        <ArtifactAction                          label="Download JSON"                          content={jsonText}                          fileName={`${templateId}.json`}                        />                      </ArtifactActions>                    </ArtifactHeader>                    <ArtifactContent>                      <div className="flex flex-col gap-4">                        <div                          data-rows={derived.table.rows.length}                        >                          <DataTableArtifact                            rows={derived.table.rows}                            columns={derived.table.columns}                          />                        </div>                        {derived.chart ? (                          <div                            data-chart-type={derived.chart.type}                          >                            <ChartArtifact                              type={derived.chart.type}                              data={derived.chart.data}                              max={derived.chart.max}                              title={derived.chart.title}                            />                          </div>                        ) : (                          <p                            className="rounded-lg border border-dashed border-border p-4 text-center text-xs text-muted-foreground"                          >                            No chart -{' '}                            {derived.chartEmptyReason ?? 'this extraction has no numeric data.'}                          </p>                        )}                      </div>                    </ArtifactContent>                  </Artifact>                )}              </>            )}            {}            {error && (              <div className="flex flex-col gap-2">                <InMessageError error={error} onRetry={hasInput ? extract : undefined} />                {errorHint(error) && (                  <pre className="overflow-x-auto rounded-md border border-border bg-muted/40 p-2 text-[11px] text-muted-foreground">                    <code>{errorHint(error)}</code>                  </pre>                )}              </div>            )}          </div>        </div>    </div>  );}