# Model Evaluator

Measure how accurate a text classifier is on a labeled set. Get accuracy along with precision, recall, and F1, a color-coded confusion matrix, and one-click JSON export of the results. The model loads only when you press Run.

**Install**

```bash
npx shadcn@latest add @localmode/ui/blocks/text-insights/model-evaluator
```

**Full block (all files):** https://localmode.ai/r/ui/blocks/text-insights/model-evaluator.json

```tsx
'use client';

/**
 * @file model-evaluator.tsx
 * @description Model Evaluator block — classifier evaluation over labeled datasets: radio model/dataset selectors, `useEvaluateModel` with completed/total progress + cancel, accuracy + macro P/R/F1, run duration, a color-coded confusion matrix, and JSON export. Model download gated behind Run.
 */
import { useState } from 'react';
import { Download, Play, Square, Trash2 } from 'lucide-react';
import { useEvaluateModel, useModelLoad, toAppError } from '@localmode/react';
import {
  classify,
  accuracy,
  precision,
  recall,
  f1Score,
  confusionMatrix,
  type ClassificationModel,
  type ConfusionMatrix,
} from '@localmode/core';
import { transformers, isModelCached } from '@localmode/transformers';

import { EvaluationMetricsDashboard } from '@/components/evaluation-metrics-dashboard';
import { ModelLoadingPanel } from '@/components/model-loading-panel';
import { CacheBadge } from '@/components/cache-badge';
import { ErrorAlert } from '@/components/error-alert';
import { ModeErrorBoundary } from '@/components/mode-error-boundary';
import { cn } from '@/lib/utils';

interface ModelOption {
  id: string;
  name: string;
  description: string;
  size: string;
}

const CLASSIFIER_MODELS: ModelOption[] = [
  {
    id: 'Xenova/distilbert-base-uncased-finetuned-sst-2-english',
    name: 'DistilBERT Sentiment',
    description: 'Fine-tuned for binary sentiment classification (POSITIVE / NEGATIVE)',
    size: '67 MB',
  },
  {
    id: 'Xenova/mobilebert-uncased-mnli',
    name: 'MobileBERT Zero-Shot',
    description: 'Zero-shot classification via natural language inference',
    size: '27 MB',
  },
];

interface DatasetEntry {
  input: string;
  expected: string;
}

interface SampleDataset {
  id: string;
  name: string;
  description: string;
  entries: DatasetEntry[];
}

const SAMPLE_DATASETS: SampleDataset[] = [
  {
    id: 'sentiment',
    name: 'Sentiment Analysis',
    description: 'Product reviews labeled as POSITIVE or NEGATIVE',
    entries: [
      { input: 'This product is amazing! Best purchase I ever made.', expected: 'POSITIVE' },
      { input: 'Terrible quality. Broke after one day of use.', expected: 'NEGATIVE' },
      { input: 'I love how easy this is to set up. Highly recommend!', expected: 'POSITIVE' },
      { input: 'Waste of money. Very disappointed with this item.', expected: 'NEGATIVE' },
      { input: 'Great value for the price. Works perfectly.', expected: 'POSITIVE' },
      { input: 'Awful customer service and the product is defective.', expected: 'NEGATIVE' },
      { input: 'Exceeded my expectations. Beautiful design and build.', expected: 'POSITIVE' },
      { input: 'Cheap materials, poor construction. Do not buy.', expected: 'NEGATIVE' },
      { input: 'Fast shipping and excellent packaging. Very happy!', expected: 'POSITIVE' },
      { input: 'The worst purchase I have ever made. Total scam.', expected: 'NEGATIVE' },
      { input: 'Absolutely fantastic! My whole family loves it.', expected: 'POSITIVE' },
      { input: 'Returned immediately. Nothing like the description.', expected: 'NEGATIVE' },
      { input: 'Perfect gift idea. Arrived on time and looks great.', expected: 'POSITIVE' },
      { input: 'Flimsy and cheaply made. Falls apart easily.', expected: 'NEGATIVE' },
      { input: 'Outstanding performance. Best in its class.', expected: 'POSITIVE' },
      { input: 'Overpriced for what you get. Not worth it.', expected: 'NEGATIVE' },
      { input: 'So glad I bought this. Life-changing product!', expected: 'POSITIVE' },
      { input: 'Stopped working after a week. No refund offered.', expected: 'NEGATIVE' },
      { input: 'Sleek design and works as advertised. Five stars.', expected: 'POSITIVE' },
      { input: 'Unbelievably bad. Save your money and avoid this.', expected: 'NEGATIVE' },
      { input: 'My favorite purchase this year. Truly impressed.', expected: 'POSITIVE' },
      { input: 'Misleading product images. Very poor quality.', expected: 'NEGATIVE' },
      { input: 'Incredible sound quality for the price. Love it!', expected: 'POSITIVE' },
      { input: 'Complete garbage. Threw it away after one use.', expected: 'NEGATIVE' },
    ],
  },
  {
    id: 'topic',
    name: 'News Topic Classification',
    description: 'News headlines labeled by category',
    entries: [
      { input: 'Stocks rally as Fed signals rate cuts ahead', expected: 'business' },
      { input: 'Lakers defeat Celtics in overtime thriller', expected: 'sports' },
      { input: 'New AI chip promises 10x faster inference', expected: 'technology' },
      { input: 'Senate passes bipartisan infrastructure bill', expected: 'politics' },
      { input: 'Tesla reports record quarterly earnings', expected: 'business' },
      { input: 'World Cup final draws 1 billion viewers', expected: 'sports' },
      { input: 'Apple unveils next-generation MacBook Pro', expected: 'technology' },
      { input: 'President signs executive order on climate', expected: 'politics' },
      { input: 'Inflation falls to lowest level in two years', expected: 'business' },
      { input: 'Olympic swimmer breaks three world records', expected: 'sports' },
      { input: 'Google launches open-source language model', expected: 'technology' },
      { input: 'Election results spark debate over voting laws', expected: 'politics' },
      { input: 'Amazon acquires streaming platform for $5B', expected: 'business' },
      { input: 'Champions League draw reveals exciting matchups', expected: 'sports' },
      { input: 'Quantum computer solves protein folding puzzle', expected: 'technology' },
      { input: 'Supreme Court rules on digital privacy case', expected: 'politics' },
      { input: 'Startup raises $200M in Series C funding round', expected: 'business' },
      { input: 'Tennis star announces retirement after 20 seasons', expected: 'sports' },
      { input: 'SpaceX successfully lands reusable rocket booster', expected: 'technology' },
      { input: 'Governor proposes sweeping education reform plan', expected: 'politics' },
    ],
  },
];

function formatScore(score: number) {
  return `${(score * 100).toFixed(1)}%`;
}

function formatDuration(ms: number) {
  if (ms < 1000) return `${Math.round(ms)}ms`;
  return `${(ms / 1000).toFixed(1)}s`;
}

interface RichOption {
  id: string;
  name: string;
  description: string;
  meta?: string;
}

interface OptionCardListProps {
  options: RichOption[];
  selectedId: string;
  onSelect: (id: string) => void;
  label: string;
  disabled?: boolean;
}

function OptionCardList({
  options,
  selectedId,
  onSelect,
  label,
  disabled,
}: OptionCardListProps) {
  return (
    <div role="radiogroup" aria-label={label} className="flex flex-col gap-2">
      {options.map((option) => {
        const active = option.id === selectedId;
        return (
          <button
            key={option.id}
            type="button"
            role="radio"
            aria-checked={active}
            disabled={disabled}
            data-active={active}
            onClick={() => onSelect(option.id)}
            className={cn(
              'flex items-start gap-3 rounded-lg border p-3 text-left transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-60',
              active ? 'border-primary bg-primary/5' : 'border-border hover:bg-muted',
            )}
          >
            <span
              className={cn(
                'mt-0.5 flex h-4 w-4 shrink-0 items-center justify-center rounded-full border',
                active ? 'border-primary' : 'border-muted-foreground/50',
              )}
              aria-hidden
            >
              {active && <span className="h-2 w-2 rounded-full bg-primary" />}
            </span>
            <span className="min-w-0 flex-1">
              <span className="flex flex-wrap items-center gap-2">
                <span className="text-sm font-medium text-foreground">{option.name}</span>
                {option.meta && (
                  <span className="rounded-full bg-muted px-1.5 py-0.5 font-mono text-[10px] text-muted-foreground">
                    {option.meta}
                  </span>
                )}
              </span>
              <span className="mt-0.5 block text-xs text-muted-foreground">{option.description}</span>
            </span>
          </button>
        );
      })}
    </div>
  );
}

interface EvalResults {
  accuracy: number;
  precision: number;
  recall: number;
  f1: number;
  predictions: string[];
  expected: string[];
  matrix: ConfusionMatrix;
  datasetSize: number;
  durationMs: number;
  modelId: string;
  datasetName: string;
}

function downloadJson(data: object, filename: string) {
  const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
  const url = URL.createObjectURL(blob);
  const a = document.createElement('a');
  a.href = url;
  a.download = filename;
  document.body.appendChild(a);
  a.click();
  document.body.removeChild(a);
  URL.revokeObjectURL(url);
}

const MODEL_OPTIONS: RichOption[] = CLASSIFIER_MODELS.map((m) => ({
  id: m.id,
  name: m.name,
  description: m.description,
  meta: m.size,
}));
const DATASET_OPTIONS: RichOption[] = SAMPLE_DATASETS.map((d) => ({
  id: d.id,
  name: d.name,
  description: d.description,
  meta: `${d.entries.length} items`,
}));

export function ModelEvaluatorBlock() {
  return (
    <div className="mx-auto flex max-w-4xl flex-col gap-4 p-4">
      <p className="text-xs text-muted-foreground">
        Model Evaluator: classifier evaluation over labeled datasets. Model loads only behind an
        explicit action.
      </p>
      <ModeErrorBoundary>
        <EvaluateInner />
      </ModeErrorBoundary>
    </div>
  );
}

function EvaluateInner() {
  const [modelId, setModelId] = useState(CLASSIFIER_MODELS[0].id);
  const [datasetId, setDatasetId] = useState(SAMPLE_DATASETS[0].id);
  const [results, setResults] = useState<EvalResults | null>(null);
  const [progress, setProgress] = useState<{ completed: number; total: number } | null>(null);

  const { isLoading, error, execute, cancel, reset } = useEvaluateModel<string, string>();

  const load = useModelLoad<ClassificationModel>({
    key: `text-insights-eval:${modelId}`,
    create: (onProgress) =>
      transformers.classifier(modelId, {
        onProgress: (p) => onProgress(p as Parameters<typeof onProgress>[0]),
      }),
    warmup: (model) => classify({ model, text: 'ready' }),
    isCached: () => isModelCached(modelId),
  });

  const appErr = toAppError(error) ?? (load.error ? toAppError(load.error) : null);
  const modelMeta = CLASSIFIER_MODELS.find((m) => m.id === modelId);

  const resetResults = () => {
    setResults(null);
    setProgress(null);
    reset();
  };

  const selectModel = (id: string) => {
    setModelId(id);
    resetResults();
  };
  const selectDataset = (id: string) => {
    setDatasetId(id);
    resetResults();
  };

  const run = async () => {
    const dataset = SAMPLE_DATASETS.find((d) => d.id === datasetId);
    if (!dataset || isLoading) return;

    setResults(null);
    setProgress({ completed: 0, total: dataset.entries.length });

    try {
      await load.load();
    } catch {
      setProgress(null);
      return;
    }
    const model = load.model;
    if (!model) {
      setProgress(null);
      return;
    }

    const inputs = dataset.entries.map((e) => e.input);
    const expected = dataset.entries.map((e) => e.expected);

    const evalResult = await execute({
      dataset: { inputs, expected },
      predict: async (text: string, signal: AbortSignal) => {
        const r = await classify({ model, text, abortSignal: signal });
        return r.label;
      },
      metric: accuracy,
      onProgress: (completed: number, total: number) => setProgress({ completed, total }),
    });

    if (evalResult) {
      const preds = evalResult.predictions;
      setResults({
        accuracy: evalResult.score,
        precision: precision(preds, expected),
        recall: recall(preds, expected),
        f1: f1Score(preds, expected),
        predictions: preds,
        expected,
        matrix: confusionMatrix(preds, expected),
        datasetSize: evalResult.datasetSize,
        durationMs: evalResult.durationMs,
        modelId,
        datasetName: dataset.name,
      });
    }
    setProgress(null);
  };

  const exportJson = () => {
    if (!results) return;
    downloadJson(
      {
        modelId: results.modelId,
        datasetName: results.datasetName,
        datasetSize: results.datasetSize,
        durationMs: results.durationMs,
        metrics: {
          accuracy: results.accuracy,
          precision: results.precision,
          recall: results.recall,
          f1: results.f1,
        },
        predictions: results.predictions,
        expected: results.expected,
      },
      'evaluation-results.json',
    );
  };

  return (
    <div className="flex flex-col gap-4">
      <div className="grid gap-4 sm:grid-cols-2">
        <div>
          <p className="mb-1.5 text-xs font-medium text-muted-foreground">Model</p>
          <OptionCardList
            options={MODEL_OPTIONS}
            selectedId={modelId}
            onSelect={selectModel}
            label="Classifier model"
            disabled={isLoading}
          />
        </div>
        <div>
          <p className="mb-1.5 text-xs font-medium text-muted-foreground">Dataset</p>
          <OptionCardList
            options={DATASET_OPTIONS}
            selectedId={datasetId}
            onSelect={selectDataset}
            label="Labeled dataset"
            disabled={isLoading}
          />
        </div>
      </div>

      <div className="flex flex-wrap items-center gap-2">
        <button
          type="button"
          onClick={() => void run()}
          disabled={isLoading}
          className="inline-flex h-8 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:opacity-50"
        >
          <Play className="h-3.5 w-3.5" aria-hidden /> Evaluate
        </button>
        {isLoading && (
          <button
            type="button"
            onClick={cancel}
            className="inline-flex h-8 items-center gap-1.5 rounded-md border border-border px-3 text-sm font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
          >
            <Square className="h-3.5 w-3.5" aria-hidden /> Stop
          </button>
        )}
        {results && (
          <>
            <button
              type="button"
              onClick={exportJson}
              className="inline-flex h-8 items-center gap-1.5 rounded-md border border-border px-3 text-sm font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
            >
              <Download className="h-3.5 w-3.5" aria-hidden /> Export JSON
            </button>
            <button
              type="button"
              onClick={resetResults}
              className="inline-flex h-8 items-center gap-1.5 rounded-md border border-border px-3 text-sm font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
            >
              <Trash2 className="h-3.5 w-3.5" aria-hidden /> Clear
            </button>
          </>
        )}
        {load.cached === true && (
          <span>
            <CacheBadge cached label="model cached" />
          </span>
        )}
      </div>

      {load.status === 'loading' && (
        <div>
          <ModelLoadingPanel
            name={modelMeta?.name ?? modelId}
            size={modelMeta?.size}
            progress={load.progressValue}
            cached={load.cached === true}
          />
        </div>
      )}

      {appErr && (
        <span>
          <ErrorAlert message={appErr.message} onRetry={() => void run()} onDismiss={reset} />
        </span>
      )}

      {progress && (
        <div className="flex flex-col gap-1.5 rounded-md border border-border p-3">
          <span
            data-completed={progress.completed}
            data-total={progress.total}
            className="text-xs text-muted-foreground"
          >
            {progress.completed} / {progress.total} evaluated
          </span>
          <div className="h-1.5 w-full overflow-hidden rounded-full bg-muted">
            <div
              className="h-full rounded-full bg-primary transition-all"
              style={{ width: `${progress.total > 0 ? (progress.completed / progress.total) * 100 : 0}%` }}
            />
          </div>
        </div>
      )}

      {results && (
        <div className="flex flex-col gap-3">
          {}
          <span
            role="status"
            aria-label="Evaluation metrics"
            data-accuracy={results.accuracy.toFixed(6)}
            data-precision={results.precision.toFixed(6)}
            data-recall={results.recall.toFixed(6)}
            data-f1={results.f1.toFixed(6)}
            className="sr-only"
          />
          <span
            role="status"
            aria-label="Confusion matrix data"
            data-labels={JSON.stringify(results.matrix.labels)}
            data-matrix={JSON.stringify(results.matrix.matrix)}
            className="sr-only"
          />
          <p className="text-xs text-muted-foreground">
            Ran in {formatDuration(results.durationMs)} · {results.datasetSize} items ·{' '}
            {formatScore(results.accuracy)} accuracy
          </p>
          <EvaluationMetricsDashboard
            stats={[
              { label: 'Dataset size', value: results.datasetSize },
              { label: 'Duration', value: formatDuration(results.durationMs) },
            ]}
            metrics={[
              { label: 'Accuracy', value: results.accuracy },
              { label: 'Precision', value: results.precision },
              { label: 'Recall', value: results.recall },
              { label: 'F1', value: results.f1 },
            ]}
            confusionMatrix={{ labels: results.matrix.labels, matrix: results.matrix.matrix }}
          />
        </div>
      )}
    </div>
  );
}
```