# Live Tracker

Track your body in real time through the webcam. Pick from four live modes: hand skeletons, full-body pose, a detailed face mesh with expressions, and hand-gesture recognition, all drawn on the video with a live frame rate. Everything runs in your browser, and the camera asks permission before it starts.

**Install**

```bash
npx shadcn@latest add @localmode/ui/blocks/vision/live-tracker
```

**Full block (all files):** https://localmode.ai/r/ui/blocks/vision/live-tracker.json

```tsx
'use client';

/**
 * @file live-tracker.tsx
 * @description Vision — Live Tracker: one block, a 4-mode picker (Hands 21-pt / Pose 33-pt / Face 478-pt mesh + blendshapes / Gestures) over real-time MediaPipe streaming trackers on live webcam video.
 * @constraint Exactly one tracker/WASM task alive at a time — each sub-mode is exclusively mounted (React key per mode) and useStreamingTracker.close()s the previous on unmount, sidestepping the MediaPipe audio↔vision WASM concurrency conflict (mediapipe#4737).
 */

import { useRef, useState, type ReactNode } from 'react';
import { RotateCcw, Video, VideoOff } from 'lucide-react';
import { useStreamingTracker } from '@localmode/react';
import {
  FACE_CONNECTIONS,
  HAND_CONNECTIONS,
  POSE_CONNECTIONS,
  type FaceLandmarkResultItem,
  type GestureResultItem,
  type HandLandmarkResultItem,
  type PoseLandmarkResultItem,
} from '@localmode/core';
import {
  createFaceTracker,
  createGestureTracker,
  createHandTracker,
  createPoseTracker,
  type TrackerInstance,
} from '@localmode/mediapipe';

import { VideoCanvas, type VideoCanvasHandle } from '@/components/video-canvas';
import { CapabilityGate } from '@/components/capability-gate';
import { ConfidenceScoreBadge } from '@/components/confidence-score-badge';
import { ScoredResultBarList } from '@/components/scored-result-bar-list';
import { SegmentedModePicker } from '@/components/segmented-mode-picker';

import { useWebcam } from '@/hooks/use-webcam';

interface DrawableLandmark {
  x: number;
  y: number;
}

const FINGER_COLORS = {
  palm: '#e5e7eb',
  thumb: '#ef4444',
  index: '#f59e0b',
  middle: '#22c55e',
  ring: '#3b82f6',
  pinky: '#a855f7',
} as const;

const FINGER_RANGES: ReadonlyArray<{ color: string; indices: readonly number[] }> = [
  { color: FINGER_COLORS.palm, indices: [0] },
  { color: FINGER_COLORS.thumb, indices: [1, 2, 3, 4] },
  { color: FINGER_COLORS.index, indices: [5, 6, 7, 8] },
  { color: FINGER_COLORS.middle, indices: [9, 10, 11, 12] },
  { color: FINGER_COLORS.ring, indices: [13, 14, 15, 16] },
  { color: FINGER_COLORS.pinky, indices: [17, 18, 19, 20] },
];

function clearCanvas(ctx: CanvasRenderingContext2D): void {
  ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
}

function drawConnections(
  ctx: CanvasRenderingContext2D,
  landmarks: readonly DrawableLandmark[],
  connections: ReadonlyArray<readonly [number, number]>,
  color: string,
  lineWidth = 2,
): void {
  const { width, height } = ctx.canvas;
  ctx.strokeStyle = color;
  ctx.lineWidth = lineWidth;
  ctx.beginPath();
  for (const [from, to] of connections) {
    const a = landmarks[from];
    const b = landmarks[to];
    if (!a || !b) continue;
    ctx.moveTo(a.x * width, a.y * height);
    ctx.lineTo(b.x * width, b.y * height);
  }
  ctx.stroke();
}

function drawPoints(
  ctx: CanvasRenderingContext2D,
  landmarks: readonly DrawableLandmark[],
  color: string,
  radius = 4,
): void {
  const { width, height } = ctx.canvas;
  ctx.fillStyle = color;
  for (const landmark of landmarks) {
    ctx.beginPath();
    ctx.arc(landmark.x * width, landmark.y * height, radius, 0, Math.PI * 2);
    ctx.fill();
  }
}

function drawHand(
  ctx: CanvasRenderingContext2D,
  landmarks: readonly DrawableLandmark[],
): void {
  drawConnections(ctx, landmarks, HAND_CONNECTIONS, '#ffffff');
  for (const { color, indices } of FINGER_RANGES) {
    drawPoints(
      ctx,
      indices.map((i) => landmarks[i]).filter((l): l is DrawableLandmark => l != null),
      color,
    );
  }
}

type TrackMode = 'hands' | 'pose' | 'face' | 'gestures';

const MODES: Array<{ id: TrackMode; label: string }> = [
  { id: 'hands', label: 'Hands' },
  { id: 'pose', label: 'Pose' },
  { id: 'face', label: 'Face' },
  { id: 'gestures', label: 'Gestures' },
];

const GESTURE_LABELS: Record<string, string> = {
  None: 'No gesture',
  Closed_Fist: 'Closed Fist',
  Open_Palm: 'Open Palm',
  Pointing_Up: 'Pointing Up',
  Thumb_Down: 'Thumbs Down',
  Thumb_Up: 'Thumbs Up',
  Victory: 'Victory',
  ILoveYou: 'I Love You',
};

const POSE_COLOR = '#22d3ee';
const FACE_COLOR = '#f472b6';

export function LiveTrackerBlock() {
  const [mode, setMode] = useState<TrackMode>('hands');
  const [showBlendshapes, setShowBlendshapes] = useState(true);

  return (
    <div className="flex flex-col gap-3 p-4">
      {}
      <p role="status" aria-live="polite" className="sr-only">
        {mode} tracker selected
      </p>

      <SegmentedModePicker<TrackMode>
        items={MODES}
        selectedId={mode}
        onSelect={setMode}
        aria-label="Streaming tracker"
      />

      <CapabilityGate requires="wasm">
        <CapabilityGate requires="camera">
          {}
          {mode === 'hands' && (
            <TrackerSurface<HandLandmarkResultItem>
              key="hands"
              create={({ video, onResults, onError }) =>
                createHandTracker({ video, onResults, onError })
              }
              draw={(ctx, hands) => {
                for (const hand of hands) drawHand(ctx, hand.landmarks);
              }}
              landmarkCount={(hands) => hands[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and hold up a hand: 21 landmarks per hand, up to 2 hands."
              panel={(hands) => (
                <div className="flex flex-col gap-2">
                  <p className="text-sm font-medium">Detected hands</p>
                  {hands.length === 0 ? (
                    <p className="text-sm text-muted-foreground">No hands detected yet.</p>
                  ) : (
                    hands.map((hand, i) => (
                      <div
                        key={i}
                        className="flex items-center justify-between gap-2 rounded-md border border-border px-3 py-2"
                      >
                        <span className="text-sm">{hand.handedness} hand</span>
                        <ConfidenceScoreBadge score={hand.score} />
                      </div>
                    ))
                  )}
                </div>
              )}
            />
          )}

          {mode === 'pose' && (
            <TrackerSurface<PoseLandmarkResultItem>
              key="pose"
              create={({ video, onResults, onError }) =>
                createPoseTracker({ video, onResults, onError })
              }
              draw={(ctx, poses) => {
                for (const pose of poses) {
                  drawConnections(ctx, pose.landmarks, POSE_CONNECTIONS, POSE_COLOR);
                  drawPoints(ctx, pose.landmarks, '#ffffff', 3);
                }
              }}
              landmarkCount={(poses) => poses[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and step back until your upper body is in frame."
              panel={(poses) => (
                <div className="flex flex-col gap-2">
                  <p className="text-sm font-medium">Pose tracking</p>
                  <p className="text-sm text-muted-foreground">
                    {poses.length > 0
                      ? `Tracking ${poses.length} person(s): 33 body landmarks each.`
                      : 'No pose detected yet, stand back so more of your body is visible.'}
                  </p>
                </div>
              )}
            />
          )}

          {mode === 'face' && (
            <TrackerSurface<FaceLandmarkResultItem>
              key="face"
              create={({ video, onResults, onError }) =>
                createFaceTracker({ video, onResults, onError, outputBlendshapes: true })
              }
              draw={(ctx, faces) => {
                for (const face of faces) {
                  drawConnections(ctx, face.landmarks, FACE_CONNECTIONS, FACE_COLOR);
                  drawPoints(ctx, face.landmarks, 'rgba(255,255,255,0.5)', 1);
                }
              }}
              landmarkCount={(faces) => faces[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and face it: a 478-point mesh with optional expression blendshapes."
              panel={(faces) => {
                const blendshapes = faces[0]?.blendshapes ?? [];
                return (
                  <div className="flex flex-col gap-2">
                    <label className="flex items-center gap-2 text-sm font-medium">
                      <input
                        type="checkbox"
                        checked={showBlendshapes}
                        onChange={(e) => setShowBlendshapes(e.target.checked)}
                        className="h-4 w-4 accent-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
                      />
                      Show expression blendshapes
                    </label>
                    {showBlendshapes && (
                      <div role="group" aria-label="Expression blendshapes">
                        <ScoredResultBarList
                          results={blendshapes.map((b) => ({
                            label: b.categoryName,
                            score: b.score,
                          }))}
                          limit={8}
                          emptyState="No face in view yet."
                        />
                      </div>
                    )}
                  </div>
                );
              }}
            />
          )}

          {mode === 'gestures' && (
            <TrackerSurface<GestureResultItem>
              key="gestures"
              create={({ video, onResults, onError }) =>
                createGestureTracker({ video, onResults, onError })
              }
              draw={(ctx, gestures) => {
                for (const gesture of gestures) drawHand(ctx, gesture.landmarks);
              }}
              landmarkCount={(gestures) => gestures[0]?.landmarks.length ?? 0}
              idleHint="Start the camera and make a gesture: thumbs up, victory, open palm, fist…"
              badge={(gestures) => {
                const top = topGesture(gestures);
                return (
                  <span
                    role="status"
                    aria-label="Recognized gesture"
                    data-gesture={top?.gesture ?? ''}
                    className="rounded-md bg-black/70 px-2 py-1 text-xs font-medium text-white"
                  >
                    {top ? (GESTURE_LABELS[top.gesture] ?? top.gesture) : 'No gesture yet'}
                  </span>
                );
              }}
              panel={(gestures) => {
                const top = topGesture(gestures);
                return (
                  <div className="flex flex-col gap-2">
                    <p className="text-sm font-medium">Recognized gesture</p>
                    {top ? (
                      <div className="flex flex-col gap-1 rounded-md border border-border px-3 py-2">
                        <span className="text-xl font-semibold">
                          {GESTURE_LABELS[top.gesture] ?? top.gesture}
                        </span>
                        <span className="text-sm text-muted-foreground">
                          {(top.score * 100).toFixed(1)}% confidence · {top.handedness} hand
                        </span>
                      </div>
                    ) : (
                      <p className="text-sm text-muted-foreground">
                        Make a gesture: thumbs up, victory, open palm, fist…
                      </p>
                    )}
                  </div>
                );
              }}
            />
          )}
        </CapabilityGate>
      </CapabilityGate>
    </div>
  );
}

function topGesture(gestures: GestureResultItem[]) {
  return gestures.find((g) => g.gesture !== 'None') ?? gestures[0] ?? null;
}


interface TrackerCreateContext<T> {
  video: HTMLVideoElement;
  onResults: (results: T[], timestampMs: number) => void;
  onError: (error: Error) => void;
}

interface TrackerSurfaceProps<T> {
  create: (ctx: TrackerCreateContext<T>) => TrackerInstance;
  draw: (ctx: CanvasRenderingContext2D, results: T[]) => void;
  landmarkCount: (results: T[]) => number;
  panel: (results: T[]) => ReactNode;
  badge?: (results: T[]) => ReactNode;
  idleHint: string;
}

function TrackerSurface<T>({
  create,
  draw,
  landmarkCount,
  panel,
  badge,
  idleHint,
}: TrackerSurfaceProps<T>) {
  const videoCanvasRef = useRef<VideoCanvasHandle>(null);
  const webcam = useWebcam({ width: 1280, height: 720 });

  const tracker = useStreamingTracker<T[]>({
    video: () => videoCanvasRef.current?.video ?? null,
    create: (ctx) => create(ctx),
    onResults: (results) => {
      const canvas = videoCanvasRef.current?.canvas;
      const ctx = canvas?.getContext('2d');
      if (!canvas || !ctx) return;
      clearCanvas(ctx);
      draw(ctx, results);
    },
  });

  const running = tracker.status === 'running';
  const starting = tracker.status === 'starting';
  const results = tracker.results ?? [];

  const toggle = async () => {
    if (webcam.isActive || running || starting) {
      tracker.stop();
      webcam.stop();
      const ctx = videoCanvasRef.current?.canvas?.getContext('2d');
      if (ctx) clearCanvas(ctx);
      return;
    }
    webcam.clearError();
    const stream = await webcam.start();
    if (!stream) return;
    await tracker.start();
  };

  const errorText = webcam.error?.message ?? tracker.error?.message ?? null;
  const statusValue = errorText
    ? 'error'
    : starting
      ? 'loading'
      : running
        ? tracker.results
          ? 'running'
          : 'waiting'
        : 'idle';

  return (
    <div className="grid gap-3 lg:grid-cols-[2fr_1fr]">
      <div className="flex flex-col gap-2">
        <div className="flex flex-wrap items-center gap-3">
          <button
            type="button"
            onClick={() => void toggle()}
            className="inline-flex h-8 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
          >
            {webcam.isActive ? (
              <VideoOff className="h-3.5 w-3.5" aria-hidden />
            ) : (
              <Video className="h-3.5 w-3.5" aria-hidden />
            )}
            {webcam.isActive ? 'Stop camera' : 'Start camera'}
          </button>
          <p
            role="status"
            aria-live="polite"
            aria-label="Tracker status"
            data-status={statusValue}
            className="text-xs text-muted-foreground"
          >
            {errorText
              ? 'error'
              : starting
                ? 'loading tracker model…'
                : running
                  ? tracker.results
                    ? 'tracking'
                    : 'waiting for first results…'
                  : 'idle'}
          </p>
          <span
            role="status"
            aria-label="Tracker frames per second"
            data-fps={tracker.fps}
            className="text-xs tabular-nums text-muted-foreground"
          >
            {running ? `${tracker.fps} fps` : ''}
          </span>
          <span role="status" aria-label="Tracked item count" data-count={results.length} className="sr-only">
            {results.length}
          </span>
          <span role="status" aria-label="Landmark count" data-points={landmarkCount(results)} className="sr-only">
            {landmarkCount(results)}
          </span>
        </div>

        {errorText && (
          <div role="alert" className="flex flex-wrap items-center gap-2">
            <p className="text-xs text-destructive">{errorText}</p>
            <button
              type="button"
              onClick={() => void toggle()}
              className="inline-flex h-6 items-center gap-1 rounded-md border border-border px-2 text-xs font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
            >
              <RotateCcw className="h-3 w-3" aria-hidden />
              Retry
            </button>
          </div>
        )}

        <VideoCanvas
          ref={videoCanvasRef}
          stream={webcam.stream}
          fps={running ? tracker.fps : undefined}
        >
          {badge && webcam.isActive ? badge(results) : null}
        </VideoCanvas>

        {!webcam.isActive && <p className="text-xs text-muted-foreground">{idleHint}</p>}
        {starting && (
          <p className="text-xs text-muted-foreground">
            Loading the tracker model… it downloads once, then runs fully on-device.
          </p>
        )}
      </div>

      <div role="group" aria-label="Tracker output" className="rounded-lg border border-border p-3">
        {panel(results)}
      </div>
    </div>
  );
}
```