# Vision

2 on-device blocks in the Vision category — each installs and runs on its own, entirely in the browser.

## Object Detector

Find and label objects in a photo, drawing a colored box around each one with its name and confidence. Use an uploaded image, the built-in sample, or a still from your webcam, plus a live camera mode that tracks faces in real time. Everything runs in your browser, and the camera asks permission before it starts.

**Install**

```bash
npx shadcn@latest add @localmode/ui/blocks/vision/object-detector
```

**Full block (all files):** https://localmode.ai/r/ui/blocks/vision/object-detector.json

```tsx
'use client';

/**
 * @file object-detector.tsx
 * @description Vision — Object Detector: one-shot DETR (Xenova/detr-resnet-50) object detection on upload/sample/webcam-still plus a live BlazeFace webcam face loop, fully on-device.
 */

import { useEffect, useRef, useState } from 'react';
import { Camera, RotateCcw, X } from 'lucide-react';
import { useDetectObjects, useModelLoad, type UseModelLoadReturn } from '@localmode/react';
import {
  detectFace,
  type FaceDetectionResultItem,
  type ObjectDetectionModel,
} from '@localmode/core';
import { transformers } from '@localmode/transformers';
import { mediapipe } from '@localmode/mediapipe';

import { VideoCanvas, type VideoCanvasHandle } from '@/components/video-canvas';
import {
  BoundingBoxOverlay,
  DetectionLabelLegend,
} from '@/components/bounding-box-overlay';
import { ImageProcessingOverlay } from '@/components/image-processing-overlay';
import { MediaDropzone } from '@/components/media-dropzone';
import { ScoredResultBarList } from '@/components/scored-result-bar-list';
import { CapabilityGate } from '@/components/capability-gate';
import { readFileAsDataUrl } from '@/lib/browser-utils';

import { useWebcam } from '@/hooks/use-webcam';

const DETR_MODEL_ID = 'Xenova/detr-resnet-50';
const SAMPLE_SRC = '/test-assets/portrait.jpg';
const ACCEPTED_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/webp', 'image/gif'];
const FACE_DETECT_INTERVAL_MS = 1500;

let faceModel: ReturnType<typeof mediapipe.faceDetector> | null = null;
const getFaceModel = () => (faceModel ??= mediapipe.faceDetector());

interface SubjectImage {
  src: string;
  width: number;
  height: number;
  origin: 'sample' | 'upload' | 'webcam';
  name: string;
}

interface CapturedStill {
  src: string;
  width: number;
  height: number;
}

function getImageDimensions(src: string) {
  return new Promise<{ width: number; height: number }>((resolve, reject) => {
    const img = new Image();
    img.onload = () => resolve({ width: img.naturalWidth, height: img.naturalHeight });
    img.onerror = () => reject(new Error('Failed to load the image.'));
    img.src = src;
  });
}

export function ObjectDetectorBlock() {
  const detr = useModelLoad<ObjectDetectionModel>({
    key: DETR_MODEL_ID,
    create: (onProgress) => transformers.objectDetector(DETR_MODEL_ID, { onProgress }),
  });

  if (!detr.model) return null;
  return <DetectSurface detr={detr} detrModel={detr.model} />;
}

function DetectSurface({
  detr,
  detrModel,
}: {
  detr: UseModelLoadReturn<ObjectDetectionModel>;
  detrModel: ObjectDetectionModel;
}) {
  const [subject, setSubject] = useState<SubjectImage | null>(null);
  const [faces, setFaces] = useState<FaceDetectionResultItem[] | null>(null);
  const [still, setStill] = useState<CapturedStill | null>(null);
  const [localError, setLocalError] = useState<string | null>(null);

  const webcam = useWebcam({ width: 640, height: 480 });
  const videoCanvasRef = useRef<VideoCanvasHandle>(null);
  const detectBusyRef = useRef(false);

  const objects = useDetectObjects({ model: detrModel });

  const startCamera = async () => {
    if (webcam.isActive) return;
    setLocalError(null);
    await webcam.start();
  };

  useEffect(() => {
    const stream = webcam.stream;
    if (!stream) return;

    const detectOnFrame = async () => {
      if (detectBusyRef.current) return;
      const video = videoCanvasRef.current?.video;
      const overlay = videoCanvasRef.current?.canvas;
      if (!video || !overlay || video.videoWidth === 0) return;

      detectBusyRef.current = true;
      try {
        const frame = document.createElement('canvas');
        frame.width = video.videoWidth;
        frame.height = video.videoHeight;
        const frameCtx = frame.getContext('2d');
        if (!frameCtx) return;
        frameCtx.drawImage(video, 0, 0);
        const imageData = frameCtx.getImageData(0, 0, frame.width, frame.height);

        const result = await detectFace({ model: getFaceModel(), image: imageData });
        setFaces(result.faces);
        setStill({
          src: frame.toDataURL('image/jpeg', 0.85),
          width: frame.width,
          height: frame.height,
        });

        const ctx = overlay.getContext('2d');
        if (ctx) {
          ctx.clearRect(0, 0, overlay.width, overlay.height);
          ctx.lineWidth = 3;
          ctx.strokeStyle = '#10b981';
          ctx.fillStyle = '#10b981';
          ctx.font = '14px sans-serif';
          for (const face of result.faces) {
            ctx.strokeRect(face.box.x, face.box.y, face.box.width, face.box.height);
            ctx.fillText(
              `face ${(face.score * 100).toFixed(0)}%`,
              face.box.x,
              Math.max(14, face.box.y - 4),
            );
            for (const kp of face.keypoints) {
              ctx.beginPath();
              ctx.arc(kp.x * overlay.width, kp.y * overlay.height, 3, 0, Math.PI * 2);
              ctx.fill();
            }
          }
        }
      } catch (err) {
        setLocalError(err instanceof Error ? err.message : String(err));
      } finally {
        detectBusyRef.current = false;
      }
    };

    const id = window.setInterval(() => void detectOnFrame(), FACE_DETECT_INTERVAL_MS);
    void detectOnFrame();
    return () => window.clearInterval(id);
  }, [webcam.stream]);

  const detectOn = async (next: SubjectImage) => {
    setLocalError(null);
    setSubject(next);
    await objects.execute(next.src);
  };

  const detectObjects = async () => {
    setLocalError(null);
    try {
      if (subject) {
        await objects.execute(subject.src);
        return;
      }
      const url = new URL(SAMPLE_SRC, window.location.origin).toString();
      const dims = await getImageDimensions(url);
      await detectOn({ ...dims, src: url, origin: 'sample', name: 'sample image' });
    } catch (err) {
      setLocalError(err instanceof Error ? err.message : String(err));
    }
  };

  const handleFiles = async (files: File[]) => {
    const file = files[0];
    if (!file) return;
    setLocalError(null);
    try {
      const dataUrl = await readFileAsDataUrl(file);
      const dims = await getImageDimensions(dataUrl);
      await detectOn({ ...dims, src: dataUrl, origin: 'upload', name: file.name });
    } catch (err) {
      setLocalError(err instanceof Error ? err.message : String(err));
    }
  };

  const captureStill = async () => {
    const video = videoCanvasRef.current?.video;
    if (!video || video.videoWidth === 0) return;
    setLocalError(null);
    const frame = document.createElement('canvas');
    frame.width = video.videoWidth;
    frame.height = video.videoHeight;
    frame.getContext('2d')?.drawImage(video, 0, 0);
    await detectOn({
      src: frame.toDataURL('image/jpeg', 0.9),
      width: frame.width,
      height: frame.height,
      origin: 'webcam',
      name: 'webcam capture',
    });
  };

  const clear = () => {
    setSubject(null);
    setLocalError(null);
    objects.reset();
  };

  const detections = objects.data?.objects ?? [];
  const objectsError = objects.error?.message ?? null;
  const errorText = localError ?? webcam.error?.message ?? objectsError;

  const detrDownloading = objects.isLoading && detr.progress > 0 && detr.progress < 1;

  const statusText = objects.isLoading
    ? 'detecting objects'
    : errorText
      ? 'error'
      : webcam.isActive
        ? faces
          ? `camera on - ${faces.length} face(s)`
          : 'camera on - detecting faces'
        : objects.data
          ? 'objects detected'
          : 'idle';

  return (
    <div className="flex flex-col gap-4 p-4">
      {}
      <p role="status" aria-live="polite" aria-label="Detector status" className="text-xs text-muted-foreground">
        <span className="font-medium text-foreground">Status:</span> {statusText}
      </p>
      {errorText && (
        <div role="alert" className="flex flex-wrap items-center gap-2">
          <p className="text-xs text-destructive">{errorText}</p>
          <button
            type="button"
            onClick={() => {
              webcam.clearError();
              objects.reset();
              if (webcam.error) void startCamera();
              else void detectObjects();
            }}
            className="inline-flex h-6 items-center gap-1 rounded-md border border-border px-2 text-xs font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
          >
            <RotateCcw className="h-3 w-3" aria-hidden />
            Retry
          </button>
        </div>
      )}

      {}
      <section aria-labelledby="od-detect-heading" className="flex flex-col gap-3 rounded-lg border border-border p-4">
        <h2 id="od-detect-heading" className="sr-only">
          Object detection
        </h2>
        <div className="flex flex-wrap items-center gap-3">
          <button
            type="button"
            onClick={() => void detectObjects()}
            disabled={objects.isLoading}
            className="inline-flex h-8 items-center rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
          >
            {objects.isLoading
              ? 'Detecting…'
              : subject
                ? 'Detect objects'
                : 'Detect objects (sample image)'}
          </button>
          <span className="text-sm">
            Objects found: <span className="tabular-nums">{objects.data ? detections.length : '-'}</span>
          </span>
          {subject && (
            <button
              type="button"
              onClick={clear}
              disabled={objects.isLoading}
              className="inline-flex h-8 items-center gap-1 rounded-md border border-border px-3 text-sm font-medium hover:bg-muted disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
            >
              <X className="h-3.5 w-3.5" aria-hidden />
              Clear
            </button>
          )}
        </div>

        {!subject && (
          <div className="max-w-xl">
            <MediaDropzone
              accept={ACCEPTED_IMAGE_TYPES}
              multiple={false}
              processing={objects.isLoading}
              processingLabel="Detecting objects…"
              title="Drop an image to detect objects"
              subtitle="or click to browse: PNG, JPEG, WebP, GIF"
              onFiles={(files) => void handleFiles(files)}
              onReject={(rejections) =>
                setLocalError(rejections[0]?.reason ?? 'That file type is not supported.')
              }
            />
          </div>
        )}

        {subject && (
          <div className="flex flex-col gap-3">
            <div
              role="group"
              aria-label="Detection subject"
              data-origin={subject.origin}
              className="relative max-w-xl"
            >
              {}
              <img src={subject.src} alt={`Detection subject (${subject.name})`} className="w-full rounded-md" />
              <BoundingBoxOverlay
                detections={detections}
                naturalWidth={subject.width}
                naturalHeight={subject.height}
              />
              <ImageProcessingOverlay
                processing={objects.isLoading}
                variant="scan"
                status={
                  detrDownloading
                    ? `Downloading model… ${(detr.progress * 100).toFixed(0)}%`
                    : 'Detecting objects…'
                }
                detail={DETR_MODEL_ID}
                onCancel={objects.cancel}
              />
            </div>

            {objects.data && detections.length > 0 && (
              <div role="group" aria-label="Detected object labels">
                <DetectionLabelLegend labels={detections.map((d) => d.label)} />
              </div>
            )}

            {objects.data && (
              <div role="group" aria-label="Detection results" className="max-w-xl">
                <ScoredResultBarList
                  results={detections.map((d) => ({ label: d.label, score: d.score }))}
                  emptyState="No objects detected"
                />
              </div>
            )}
          </div>
        )}
      </section>

      {}
      <CapabilityGate requires="wasm">
        <CapabilityGate requires="camera">
          <section aria-labelledby="od-camera-heading" className="flex flex-col gap-3 rounded-lg border border-border p-4">
            <h2 id="od-camera-heading" className="sr-only">
              Live webcam face tracking
            </h2>
            <div className="flex flex-wrap items-center gap-3">
              <button
                type="button"
                onClick={() => void startCamera()}
                disabled={webcam.isActive}
                className="inline-flex h-8 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
              >
                <Camera className="h-3.5 w-3.5" aria-hidden />
                {webcam.isActive ? 'Camera running' : 'Start camera'}
              </button>
              <span className="text-sm">
                Faces detected: <span className="tabular-nums">{faces ? faces.length : '-'}</span>
              </span>
              {webcam.isActive && (
                <button
                  type="button"
                  onClick={() => void captureStill()}
                  disabled={objects.isLoading}
                  className="inline-flex h-8 items-center rounded-md border border-border px-3 text-sm font-medium hover:bg-muted disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
                >
                  Capture still → detect objects
                </button>
              )}
            </div>

            <div className="max-w-xl">
              <VideoCanvas ref={videoCanvasRef} stream={webcam.stream} mirrored={false} hideFps />
            </div>

            {still && faces && (
              <div>
                <p className="mb-1 text-xs text-muted-foreground">
                  Captured still + BoundingBoxOverlay (same detections)
                </p>
                <div className="relative w-80">
                  {}
                  <img src={still.src} alt="Captured webcam still" className="w-full rounded-md" />
                  <BoundingBoxOverlay
                    detections={faces.map((f) => ({
                      label: 'face',
                      score: f.score,
                      box: f.box,
                    }))}
                    naturalWidth={still.width}
                    naturalHeight={still.height}
                  />
                </div>
              </div>
            )}
          </section>
        </CapabilityGate>
      </CapabilityGate>
    </div>
  );
}
```

## Live Tracker

Track your body in real time through the webcam. Pick from four live modes: hand skeletons, full-body pose, a detailed face mesh with expressions, and hand-gesture recognition, all drawn on the video with a live frame rate. Everything runs in your browser, and the camera asks permission before it starts.

**Install**

```bash
npx shadcn@latest add @localmode/ui/blocks/vision/live-tracker
```

**Full block (all files):** https://localmode.ai/r/ui/blocks/vision/live-tracker.json

```tsx
'use client';

/**
 * @file live-tracker.tsx
 * @description Vision — Live Tracker: one block, a 4-mode picker (Hands 21-pt / Pose 33-pt / Face 478-pt mesh + blendshapes / Gestures) over real-time MediaPipe streaming trackers on live webcam video.
 * @constraint Exactly one tracker/WASM task alive at a time — each sub-mode is exclusively mounted (React key per mode) and useStreamingTracker.close()s the previous on unmount, sidestepping the MediaPipe audio↔vision WASM concurrency conflict (mediapipe#4737).
 */

import { useRef, useState, type ReactNode } from 'react';
import { RotateCcw, Video, VideoOff } from 'lucide-react';
import { useStreamingTracker } from '@localmode/react';
import {
  FACE_CONNECTIONS,
  HAND_CONNECTIONS,
  POSE_CONNECTIONS,
  type FaceLandmarkResultItem,
  type GestureResultItem,
  type HandLandmarkResultItem,
  type PoseLandmarkResultItem,
} from '@localmode/core';
import {
  createFaceTracker,
  createGestureTracker,
  createHandTracker,
  createPoseTracker,
  type TrackerInstance,
} from '@localmode/mediapipe';

import { VideoCanvas, type VideoCanvasHandle } from '@/components/video-canvas';
import { CapabilityGate } from '@/components/capability-gate';
import { ConfidenceScoreBadge } from '@/components/confidence-score-badge';
import { ScoredResultBarList } from '@/components/scored-result-bar-list';
import { SegmentedModePicker } from '@/components/segmented-mode-picker';

import { useWebcam } from '@/hooks/use-webcam';

interface DrawableLandmark {
  x: number;
  y: number;
}

const FINGER_COLORS = {
  palm: '#e5e7eb',
  thumb: '#ef4444',
  index: '#f59e0b',
  middle: '#22c55e',
  ring: '#3b82f6',
  pinky: '#a855f7',
} as const;

const FINGER_RANGES: ReadonlyArray<{ color: string; indices: readonly number[] }> = [
  { color: FINGER_COLORS.palm, indices: [0] },
  { color: FINGER_COLORS.thumb, indices: [1, 2, 3, 4] },
  { color: FINGER_COLORS.index, indices: [5, 6, 7, 8] },
  { color: FINGER_COLORS.middle, indices: [9, 10, 11, 12] },
  { color: FINGER_COLORS.ring, indices: [13, 14, 15, 16] },
  { color: FINGER_COLORS.pinky, indices: [17, 18, 19, 20] },
];

function clearCanvas(ctx: CanvasRenderingContext2D): void {
  ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
}

function drawConnections(
  ctx: CanvasRenderingContext2D,
  landmarks: readonly DrawableLandmark[],
  connections: ReadonlyArray<readonly [number, number]>,
  color: string,
  lineWidth = 2,
): void {
  const { width, height } = ctx.canvas;
  ctx.strokeStyle = color;
  ctx.lineWidth = lineWidth;
  ctx.beginPath();
  for (const [from, to] of connections) {
    const a = landmarks[from];
    const b = landmarks[to];
    if (!a || !b) continue;
    ctx.moveTo(a.x * width, a.y * height);
    ctx.lineTo(b.x * width, b.y * height);
  }
  ctx.stroke();
}

function drawPoints(
  ctx: CanvasRenderingContext2D,
  landmarks: readonly DrawableLandmark[],
  color: string,
  radius = 4,
): void {
  const { width, height } = ctx.canvas;
  ctx.fillStyle = color;
  for (const landmark of landmarks) {
    ctx.beginPath();
    ctx.arc(landmark.x * width, landmark.y * height, radius, 0, Math.PI * 2);
    ctx.fill();
  }
}

function drawHand(
  ctx: CanvasRenderingContext2D,
  landmarks: readonly DrawableLandmark[],
): void {
  drawConnections(ctx, landmarks, HAND_CONNECTIONS, '#ffffff');
  for (const { color, indices } of FINGER_RANGES) {
    drawPoints(
      ctx,
      indices.map((i) => landmarks[i]).filter((l): l is DrawableLandmark => l != null),
      color,
    );
  }
}

type TrackMode = 'hands' | 'pose' | 'face' | 'gestures';

const MODES: Array<{ id: TrackMode; label: string }> = [
  { id: 'hands', label: 'Hands' },
  { id: 'pose', label: 'Pose' },
  { id: 'face', label: 'Face' },
  { id: 'gestures', label: 'Gestures' },
];

const GESTURE_LABELS: Record<string, string> = {
  None: 'No gesture',
  Closed_Fist: 'Closed Fist',
  Open_Palm: 'Open Palm',
  Pointing_Up: 'Pointing Up',
  Thumb_Down: 'Thumbs Down',
  Thumb_Up: 'Thumbs Up',
  Victory: 'Victory',
  ILoveYou: 'I Love You',
};

const POSE_COLOR = '#22d3ee';
const FACE_COLOR = '#f472b6';

export function LiveTrackerBlock() {
  const [mode, setMode] = useState<TrackMode>('hands');
  const [showBlendshapes, setShowBlendshapes] = useState(true);

  return (
    <div className="flex flex-col gap-3 p-4">
      {}
      <p role="status" aria-live="polite" className="sr-only">
        {mode} tracker selected
      </p>

      <SegmentedModePicker<TrackMode>
        items={MODES}
        selectedId={mode}
        onSelect={setMode}
        aria-label="Streaming tracker"
      />

      <CapabilityGate requires="wasm">
        <CapabilityGate requires="camera">
          {}
          {mode === 'hands' && (
            <TrackerSurface<HandLandmarkResultItem>
              key="hands"
              create={({ video, onResults, onError }) =>
                createHandTracker({ video, onResults, onError })
              }
              draw={(ctx, hands) => {
                for (const hand of hands) drawHand(ctx, hand.landmarks);
              }}
              landmarkCount={(hands) => hands[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and hold up a hand: 21 landmarks per hand, up to 2 hands."
              panel={(hands) => (
                <div className="flex flex-col gap-2">
                  <p className="text-sm font-medium">Detected hands</p>
                  {hands.length === 0 ? (
                    <p className="text-sm text-muted-foreground">No hands detected yet.</p>
                  ) : (
                    hands.map((hand, i) => (
                      <div
                        key={i}
                        className="flex items-center justify-between gap-2 rounded-md border border-border px-3 py-2"
                      >
                        <span className="text-sm">{hand.handedness} hand</span>
                        <ConfidenceScoreBadge score={hand.score} />
                      </div>
                    ))
                  )}
                </div>
              )}
            />
          )}

          {mode === 'pose' && (
            <TrackerSurface<PoseLandmarkResultItem>
              key="pose"
              create={({ video, onResults, onError }) =>
                createPoseTracker({ video, onResults, onError })
              }
              draw={(ctx, poses) => {
                for (const pose of poses) {
                  drawConnections(ctx, pose.landmarks, POSE_CONNECTIONS, POSE_COLOR);
                  drawPoints(ctx, pose.landmarks, '#ffffff', 3);
                }
              }}
              landmarkCount={(poses) => poses[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and step back until your upper body is in frame."
              panel={(poses) => (
                <div className="flex flex-col gap-2">
                  <p className="text-sm font-medium">Pose tracking</p>
                  <p className="text-sm text-muted-foreground">
                    {poses.length > 0
                      ? `Tracking ${poses.length} person(s): 33 body landmarks each.`
                      : 'No pose detected yet, stand back so more of your body is visible.'}
                  </p>
                </div>
              )}
            />
          )}

          {mode === 'face' && (
            <TrackerSurface<FaceLandmarkResultItem>
              key="face"
              create={({ video, onResults, onError }) =>
                createFaceTracker({ video, onResults, onError, outputBlendshapes: true })
              }
              draw={(ctx, faces) => {
                for (const face of faces) {
                  drawConnections(ctx, face.landmarks, FACE_CONNECTIONS, FACE_COLOR);
                  drawPoints(ctx, face.landmarks, 'rgba(255,255,255,0.5)', 1);
                }
              }}
              landmarkCount={(faces) => faces[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and face it: a 478-point mesh with optional expression blendshapes."
              panel={(faces) => {
                const blendshapes = faces[0]?.blendshapes ?? [];
                return (
                  <div className="flex flex-col gap-2">
                    <label className="flex items-center gap-2 text-sm font-medium">
                      <input
                        type="checkbox"
                        checked={showBlendshapes}
                        onChange={(e) => setShowBlendshapes(e.target.checked)}
                        className="h-4 w-4 accent-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
                      />
                      Show expression blendshapes
                    </label>
                    {showBlendshapes && (
                      <div role="group" aria-label="Expression blendshapes">
                        <ScoredResultBarList
                          results={blendshapes.map((b) => ({
                            label: b.categoryName,
                            score: b.score,
                          }))}
                          limit={8}
                          emptyState="No face in view yet."
                        />
                      </div>
                    )}
                  </div>
                );
              }}
            />
          )}

          {mode === 'gestures' && (
            <TrackerSurface<GestureResultItem>
              key="gestures"
              create={({ video, onResults, onError }) =>
                createGestureTracker({ video, onResults, onError })
              }
              draw={(ctx, gestures) => {
                for (const gesture of gestures) drawHand(ctx, gesture.landmarks);
              }}
              landmarkCount={(gestures) => gestures[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and make a gesture: thumbs up, victory, open palm, fist…"
              badge={(gestures) => {
                const top = topGesture(gestures);
                return (
                  <span
                    role="status"
                    aria-label="Recognized gesture"
                    data-gesture={top?.gesture ?? ''}
                    className="rounded-md bg-black/70 px-2 py-1 text-xs font-medium text-white"
                  >
                    {top ? (GESTURE_LABELS[top.gesture] ?? top.gesture) : 'No gesture yet'}
                  </span>
                );
              }}
              panel={(gestures) => {
                const top = topGesture(gestures);
                return (
                  <div className="flex flex-col gap-2">
                    <p className="text-sm font-medium">Recognized gesture</p>
                    {top ? (
                      <div className="flex flex-col gap-1 rounded-md border border-border px-3 py-2">
                        <span className="text-xl font-semibold">
                          {GESTURE_LABELS[top.gesture] ?? top.gesture}
                        </span>
                        <span className="text-sm text-muted-foreground">
                          {(top.score * 100).toFixed(1)}% confidence · {top.handedness} hand
                        </span>
                      </div>
                    ) : (
                      <p className="text-sm text-muted-foreground">
                        Make a gesture: thumbs up, victory, open palm, fist…
                      </p>
                    )}
                  </div>
                );
              }}
            />
          )}
        </CapabilityGate>
      </CapabilityGate>
    </div>
  );
}

function topGesture(gestures: GestureResultItem[]) {
  return gestures.find((g) => g.gesture !== 'None') ?? gestures[0] ?? null;
}


interface TrackerCreateContext<T> {
  video: HTMLVideoElement;
  onResults: (results: T[], timestampMs: number) => void;
  onError: (error: Error) => void;
}

interface TrackerSurfaceProps<T> {
  create: (ctx: TrackerCreateContext<T>) => TrackerInstance;
  draw: (ctx: CanvasRenderingContext2D, results: T[]) => void;
  landmarkCount: (results: T[]) => number;
  panel: (results: T[]) => ReactNode;
  badge?: (results: T[]) => ReactNode;
  idleHint: string;
}

function TrackerSurface<T>({
  create,
  draw,
  landmarkCount,
  panel,
  badge,
  idleHint,
}: TrackerSurfaceProps<T>) {
  const videoCanvasRef = useRef<VideoCanvasHandle>(null);
  const webcam = useWebcam({ width: 1280, height: 720 });

  const tracker = useStreamingTracker<T[]>({
    video: () => videoCanvasRef.current?.video ?? null,
    create: (ctx) => create(ctx),
    onResults: (results) => {
      const canvas = videoCanvasRef.current?.canvas;
      const ctx = canvas?.getContext('2d');
      if (!canvas || !ctx) return;
      clearCanvas(ctx);
      draw(ctx, results);
    },
  });

  const running = tracker.status === 'running';
  const starting = tracker.status === 'starting';
  const results = tracker.results ?? [];

  const toggle = async () => {
    if (webcam.isActive || running || starting) {
      tracker.stop();
      webcam.stop();
      const ctx = videoCanvasRef.current?.canvas?.getContext('2d');
      if (ctx) clearCanvas(ctx);
      return;
    }
    webcam.clearError();
    const stream = await webcam.start();
    if (!stream) return;
    await tracker.start();
  };

  const errorText = webcam.error?.message ?? tracker.error?.message ?? null;
  const statusValue = errorText
    ? 'error'
    : starting
      ? 'loading'
      : running
        ? tracker.results
          ? 'running'
          : 'waiting'
        : 'idle';

  return (
    <div className="grid gap-3 lg:grid-cols-[2fr_1fr]">
      <div className="flex flex-col gap-2">
        <div className="flex flex-wrap items-center gap-3">
          <button
            type="button"
            onClick={() => void toggle()}
            className="inline-flex h-8 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
          >
            {webcam.isActive ? (
              <VideoOff className="h-3.5 w-3.5" aria-hidden />
            ) : (
              <Video className="h-3.5 w-3.5" aria-hidden />
            )}
            {webcam.isActive ? 'Stop camera' : 'Start camera'}
          </button>
          <p
            role="status"
            aria-live="polite"
            aria-label="Tracker status"
            data-status={statusValue}
            className="text-xs text-muted-foreground"
          >
            {errorText
              ? 'error'
              : starting
                ? 'loading tracker model…'
                : running
                  ? tracker.results
                    ? 'tracking'
                    : 'waiting for first results…'
                  : 'idle'}
          </p>
          <span
            role="status"
            aria-label="Tracker frames per second"
            data-fps={tracker.fps}
            className="text-xs tabular-nums text-muted-foreground"
          >
            {running ? `${tracker.fps} fps` : ''}
          </span>
          <span role="status" aria-label="Tracked item count" data-count={results.length} className="sr-only">
            {results.length}
          </span>
          <span role="status" aria-label="Landmark count" data-points={landmarkCount(results)} className="sr-only">
            {landmarkCount(results)}
          </span>
        </div>

        {errorText && (
          <div role="alert" className="flex flex-wrap items-center gap-2">
            <p className="text-xs text-destructive">{errorText}</p>
            <button
              type="button"
              onClick={() => void toggle()}
              className="inline-flex h-6 items-center gap-1 rounded-md border border-border px-2 text-xs font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
            >
              <RotateCcw className="h-3 w-3" aria-hidden />
              Retry
            </button>
          </div>
        )}

        <VideoCanvas
          ref={videoCanvasRef}
          stream={webcam.stream}
          fps={running ? tracker.fps : undefined}
        >
          {badge && webcam.isActive ? badge(results) : null}
        </VideoCanvas>

        {!webcam.isActive && <p className="text-xs text-muted-foreground">{idleHint}</p>}
        {starting && (
          <p className="text-xs text-muted-foreground">
            Loading the tracker model… it downloads once, then runs fully on-device.
          </p>
        )}
      </div>

      <div role="group" aria-label="Tracker output" className="rounded-lg border border-border p-3">
        {panel(results)}
      </div>
    </div>
  );
}
```