# Object Detector

Find and label objects in a photo, drawing a colored box around each one with its name and confidence. Use an uploaded image, the built-in sample, or a still from your webcam, plus a live camera mode that tracks faces in real time. Everything runs in your browser, and the camera asks permission before it starts.

**Install**

```bash
npx shadcn@latest add @localmode/ui/blocks/vision/object-detector
```

**Full block (all files):** https://localmode.ai/r/ui/blocks/vision/object-detector.json

```tsx
'use client';

/**
 * @file object-detector.tsx
 * @description Vision — Object Detector: one-shot DETR (Xenova/detr-resnet-50) object detection on upload/sample/webcam-still plus a live BlazeFace webcam face loop, fully on-device.
 */

import { useEffect, useRef, useState } from 'react';
import { Camera, RotateCcw, X } from 'lucide-react';
import { useDetectObjects, useModelLoad, type UseModelLoadReturn } from '@localmode/react';
import {
  detectFace,
  type FaceDetectionResultItem,
  type ObjectDetectionModel,
} from '@localmode/core';
import { transformers } from '@localmode/transformers';
import { mediapipe } from '@localmode/mediapipe';

import { VideoCanvas, type VideoCanvasHandle } from '@/components/video-canvas';
import {
  BoundingBoxOverlay,
  DetectionLabelLegend,
} from '@/components/bounding-box-overlay';
import { ImageProcessingOverlay } from '@/components/image-processing-overlay';
import { MediaDropzone } from '@/components/media-dropzone';
import { ScoredResultBarList } from '@/components/scored-result-bar-list';
import { CapabilityGate } from '@/components/capability-gate';
import { readFileAsDataUrl } from '@/lib/browser-utils';

import { useWebcam } from '@/hooks/use-webcam';

const DETR_MODEL_ID = 'Xenova/detr-resnet-50';
const SAMPLE_SRC = '/test-assets/portrait.jpg';
const ACCEPTED_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/webp', 'image/gif'];
const FACE_DETECT_INTERVAL_MS = 1500;

let faceModel: ReturnType<typeof mediapipe.faceDetector> | null = null;
const getFaceModel = () => (faceModel ??= mediapipe.faceDetector());

interface SubjectImage {
  src: string;
  width: number;
  height: number;
  origin: 'sample' | 'upload' | 'webcam';
  name: string;
}

interface CapturedStill {
  src: string;
  width: number;
  height: number;
}

function getImageDimensions(src: string) {
  return new Promise<{ width: number; height: number }>((resolve, reject) => {
    const img = new Image();
    img.onload = () => resolve({ width: img.naturalWidth, height: img.naturalHeight });
    img.onerror = () => reject(new Error('Failed to load the image.'));
    img.src = src;
  });
}

export function ObjectDetectorBlock() {
  const detr = useModelLoad<ObjectDetectionModel>({
    key: DETR_MODEL_ID,
    create: (onProgress) => transformers.objectDetector(DETR_MODEL_ID, { onProgress }),
  });

  if (!detr.model) return null;
  return <DetectSurface detr={detr} detrModel={detr.model} />;
}

function DetectSurface({
  detr,
  detrModel,
}: {
  detr: UseModelLoadReturn<ObjectDetectionModel>;
  detrModel: ObjectDetectionModel;
}) {
  const [subject, setSubject] = useState<SubjectImage | null>(null);
  const [faces, setFaces] = useState<FaceDetectionResultItem[] | null>(null);
  const [still, setStill] = useState<CapturedStill | null>(null);
  const [localError, setLocalError] = useState<string | null>(null);

  const webcam = useWebcam({ width: 640, height: 480 });
  const videoCanvasRef = useRef<VideoCanvasHandle>(null);
  const detectBusyRef = useRef(false);

  const objects = useDetectObjects({ model: detrModel });

  const startCamera = async () => {
    if (webcam.isActive) return;
    setLocalError(null);
    await webcam.start();
  };

  useEffect(() => {
    const stream = webcam.stream;
    if (!stream) return;

    const detectOnFrame = async () => {
      if (detectBusyRef.current) return;
      const video = videoCanvasRef.current?.video;
      const overlay = videoCanvasRef.current?.canvas;
      if (!video || !overlay || video.videoWidth === 0) return;

      detectBusyRef.current = true;
      try {
        const frame = document.createElement('canvas');
        frame.width = video.videoWidth;
        frame.height = video.videoHeight;
        const frameCtx = frame.getContext('2d');
        if (!frameCtx) return;
        frameCtx.drawImage(video, 0, 0);
        const imageData = frameCtx.getImageData(0, 0, frame.width, frame.height);

        const result = await detectFace({ model: getFaceModel(), image: imageData });
        setFaces(result.faces);
        setStill({
          src: frame.toDataURL('image/jpeg', 0.85),
          width: frame.width,
          height: frame.height,
        });

        const ctx = overlay.getContext('2d');
        if (ctx) {
          ctx.clearRect(0, 0, overlay.width, overlay.height);
          ctx.lineWidth = 3;
          ctx.strokeStyle = '#10b981';
          ctx.fillStyle = '#10b981';
          ctx.font = '14px sans-serif';
          for (const face of result.faces) {
            ctx.strokeRect(face.box.x, face.box.y, face.box.width, face.box.height);
            ctx.fillText(
              `face ${(face.score * 100).toFixed(0)}%`,
              face.box.x,
              Math.max(14, face.box.y - 4),
            );
            for (const kp of face.keypoints) {
              ctx.beginPath();
              ctx.arc(kp.x * overlay.width, kp.y * overlay.height, 3, 0, Math.PI * 2);
              ctx.fill();
            }
          }
        }
      } catch (err) {
        setLocalError(err instanceof Error ? err.message : String(err));
      } finally {
        detectBusyRef.current = false;
      }
    };

    const id = window.setInterval(() => void detectOnFrame(), FACE_DETECT_INTERVAL_MS);
    void detectOnFrame();
    return () => window.clearInterval(id);
  }, [webcam.stream]);

  const detectOn = async (next: SubjectImage) => {
    setLocalError(null);
    setSubject(next);
    await objects.execute(next.src);
  };

  const detectObjects = async () => {
    setLocalError(null);
    try {
      if (subject) {
        await objects.execute(subject.src);
        return;
      }
      const url = new URL(SAMPLE_SRC, window.location.origin).toString();
      const dims = await getImageDimensions(url);
      await detectOn({ ...dims, src: url, origin: 'sample', name: 'sample image' });
    } catch (err) {
      setLocalError(err instanceof Error ? err.message : String(err));
    }
  };

  const handleFiles = async (files: File[]) => {
    const file = files[0];
    if (!file) return;
    setLocalError(null);
    try {
      const dataUrl = await readFileAsDataUrl(file);
      const dims = await getImageDimensions(dataUrl);
      await detectOn({ ...dims, src: dataUrl, origin: 'upload', name: file.name });
    } catch (err) {
      setLocalError(err instanceof Error ? err.message : String(err));
    }
  };

  const captureStill = async () => {
    const video = videoCanvasRef.current?.video;
    if (!video || video.videoWidth === 0) return;
    setLocalError(null);
    const frame = document.createElement('canvas');
    frame.width = video.videoWidth;
    frame.height = video.videoHeight;
    frame.getContext('2d')?.drawImage(video, 0, 0);
    await detectOn({
      src: frame.toDataURL('image/jpeg', 0.9),
      width: frame.width,
      height: frame.height,
      origin: 'webcam',
      name: 'webcam capture',
    });
  };

  const clear = () => {
    setSubject(null);
    setLocalError(null);
    objects.reset();
  };

  const detections = objects.data?.objects ?? [];
  const objectsError = objects.error?.message ?? null;
  const errorText = localError ?? webcam.error?.message ?? objectsError;

  const detrDownloading = objects.isLoading && detr.progress > 0 && detr.progress < 1;

  const statusText = objects.isLoading
    ? 'detecting objects'
    : errorText
      ? 'error'
      : webcam.isActive
        ? faces
          ? `camera on - ${faces.length} face(s)`
          : 'camera on - detecting faces'
        : objects.data
          ? 'objects detected'
          : 'idle';

  return (
    <div className="flex flex-col gap-4 p-4">
      {}
      <p role="status" aria-live="polite" aria-label="Detector status" className="text-xs text-muted-foreground">
        <span className="font-medium text-foreground">Status:</span> {statusText}
      </p>
      {errorText && (
        <div role="alert" className="flex flex-wrap items-center gap-2">
          <p className="text-xs text-destructive">{errorText}</p>
          <button
            type="button"
            onClick={() => {
              webcam.clearError();
              objects.reset();
              if (webcam.error) void startCamera();
              else void detectObjects();
            }}
            className="inline-flex h-6 items-center gap-1 rounded-md border border-border px-2 text-xs font-medium hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
          >
            <RotateCcw className="h-3 w-3" aria-hidden />
            Retry
          </button>
        </div>
      )}

      {}
      <section aria-labelledby="od-detect-heading" className="flex flex-col gap-3 rounded-lg border border-border p-4">
        <h2 id="od-detect-heading" className="sr-only">
          Object detection
        </h2>
        <div className="flex flex-wrap items-center gap-3">
          <button
            type="button"
            onClick={() => void detectObjects()}
            disabled={objects.isLoading}
            className="inline-flex h-8 items-center rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
          >
            {objects.isLoading
              ? 'Detecting…'
              : subject
                ? 'Detect objects'
                : 'Detect objects (sample image)'}
          </button>
          <span className="text-sm">
            Objects found: <span className="tabular-nums">{objects.data ? detections.length : '-'}</span>
          </span>
          {subject && (
            <button
              type="button"
              onClick={clear}
              disabled={objects.isLoading}
              className="inline-flex h-8 items-center gap-1 rounded-md border border-border px-3 text-sm font-medium hover:bg-muted disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
            >
              <X className="h-3.5 w-3.5" aria-hidden />
              Clear
            </button>
          )}
        </div>

        {!subject && (
          <div className="max-w-xl">
            <MediaDropzone
              accept={ACCEPTED_IMAGE_TYPES}
              multiple={false}
              processing={objects.isLoading}
              processingLabel="Detecting objects…"
              title="Drop an image to detect objects"
              subtitle="or click to browse: PNG, JPEG, WebP, GIF"
              onFiles={(files) => void handleFiles(files)}
              onReject={(rejections) =>
                setLocalError(rejections[0]?.reason ?? 'That file type is not supported.')
              }
            />
          </div>
        )}

        {subject && (
          <div className="flex flex-col gap-3">
            <div
              role="group"
              aria-label="Detection subject"
              data-origin={subject.origin}
              className="relative max-w-xl"
            >
              {}
              <img src={subject.src} alt={`Detection subject (${subject.name})`} className="w-full rounded-md" />
              <BoundingBoxOverlay
                detections={detections}
                naturalWidth={subject.width}
                naturalHeight={subject.height}
              />
              <ImageProcessingOverlay
                processing={objects.isLoading}
                variant="scan"
                status={
                  detrDownloading
                    ? `Downloading model… ${(detr.progress * 100).toFixed(0)}%`
                    : 'Detecting objects…'
                }
                detail={DETR_MODEL_ID}
                onCancel={objects.cancel}
              />
            </div>

            {objects.data && detections.length > 0 && (
              <div role="group" aria-label="Detected object labels">
                <DetectionLabelLegend labels={detections.map((d) => d.label)} />
              </div>
            )}

            {objects.data && (
              <div role="group" aria-label="Detection results" className="max-w-xl">
                <ScoredResultBarList
                  results={detections.map((d) => ({ label: d.label, score: d.score }))}
                  emptyState="No objects detected"
                />
              </div>
            )}
          </div>
        )}
      </section>

      {}
      <CapabilityGate requires="wasm">
        <CapabilityGate requires="camera">
          <section aria-labelledby="od-camera-heading" className="flex flex-col gap-3 rounded-lg border border-border p-4">
            <h2 id="od-camera-heading" className="sr-only">
              Live webcam face tracking
            </h2>
            <div className="flex flex-wrap items-center gap-3">
              <button
                type="button"
                onClick={() => void startCamera()}
                disabled={webcam.isActive}
                className="inline-flex h-8 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background"
              >
                <Camera className="h-3.5 w-3.5" aria-hidden />
                {webcam.isActive ? 'Camera running' : 'Start camera'}
              </button>
              <span className="text-sm">
                Faces detected: <span className="tabular-nums">{faces ? faces.length : '-'}</span>
              </span>
              {webcam.isActive && (
                <button
                  type="button"
                  onClick={() => void captureStill()}
                  disabled={objects.isLoading}
                  className="inline-flex h-8 items-center rounded-md border border-border px-3 text-sm font-medium hover:bg-muted disabled:opacity-50 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
                >
                  Capture still → detect objects
                </button>
              )}
            </div>

            <div className="max-w-xl">
              <VideoCanvas ref={videoCanvasRef} stream={webcam.stream} mirrored={false} hideFps />
            </div>

            {still && faces && (
              <div>
                <p className="mb-1 text-xs text-muted-foreground">
                  Captured still + BoundingBoxOverlay (same detections)
                </p>
                <div className="relative w-80">
                  {}
                  <img src={still.src} alt="Captured webcam still" className="w-full rounded-md" />
                  <BoundingBoxOverlay
                    detections={faces.map((f) => ({
                      label: 'face',
                      score: f.score,
                      box: f.box,
                    }))}
                    naturalWidth={still.width}
                    naturalHeight={still.height}
                  />
                </div>
              </div>
            )}
          </section>
        </CapabilityGate>
      </CapabilityGate>
    </div>
  );
}
```