Image Search

Search a photo library on your device by typing what you are looking for, or by dropping in a reference image. Both search the same photos, with a control for how many results to show and a minimum match threshold. Nothing downloads until you load a model.

Install this block

npx shadcn@latest add @localmode/ui/blocks/photo/image-search

idle - load a model to start

Multimodal (CLIP)

CLIP ViT-B/32 (~350 MB) - not loaded. It powers both search embeddings and categorization. Nothing downloads until you press Load.

1 · Build a library

Load a model to start

PNG, JPEG or WebP - or click to browse

PNG · JPEG · WebP · up to 10 MB

2 · Search the shared vector space

Minimum similarity: 20%

Results (top-K)20

Ingest photos above first, then search them.

'use client';/** * @file image-search.tsx * @description Self-sufficient CLIP image-search block — ingest a photo library, then run text→image and image→image search over one shared multimodal vector space. */import { useState } from 'react';import { Loader2, Search, Trash2, X } from 'lucide-react';import {  usePhotoLibrary,  readFileAsDataUrl,  type PhotoEntry,  type RankedHit,  type PhotoLibrary,} from '@localmode/react';import { transformers, isModelCached } from '@localmode/transformers';import { cn } from '@/lib/utils';import { ModelSelector, type SelectableModel } from '@/components/model-selector';import { ModelDownloader } from '@/components/model-downloader';import { SegmentedModePicker } from '@/components/segmented-mode-picker';import { ParameterSlider } from '@/components/parameter-slider';import { MediaDropzone } from '@/components/media-dropzone';import { ImageResultGallery, type ImageResultCard } from '@/components/image-result-gallery';import { ScoredResultBarList } from '@/components/scored-result-bar-list';interface PhotoSearchModel {  id: string;  name: string;  size: string;  dimensions: number;}const DEFAULT_MODEL_ID = 'Xenova/clip-vit-base-patch32';const MODEL_CATALOG: PhotoSearchModel[] = [  { id: 'Xenova/clip-vit-base-patch32', name: 'CLIP ViT-B/32', size: '~350 MB', dimensions: 512 },  { id: 'Xenova/siglip-base-patch16-224', name: 'SigLIP Base', size: '~400 MB', dimensions: 768 },];function getModel(id: string): PhotoSearchModel {  return MODEL_CATALOG.find((m) => m.id === id) ?? MODEL_CATALOG[0];}const PHOTO_LABELS = [  'nature',  'people',  'animals',  'food',  'architecture',  'vehicles',  'art',  'technology',  'sports',  'other',];const PRODUCT_LABELS = [  'Electronics',  'Clothing',  'Home & Garden',  'Toys',  'Food & Beverage',  'Sports',  'Books',  'Automotive',  'Health',  'Other',];const LABEL_PRESETS: Record<string, { labels: string[] }> = {  photo: { labels: PHOTO_LABELS },  product: { labels: PRODUCT_LABELS },};const DEFAULT_TOP_K = 20;const MIN_SIMILARITY_FALLBACK = 0.2;function formatScore(score: number): string {  return `${Math.round(score * 100)}%`;}function scoreTone(score: number): 'strong' | 'medium' | 'weak' {  if (score >= 0.35) return 'strong';  if (score >= 0.2) return 'medium';  return 'weak';}type SearchMode = 'text' | 'image';const ACCEPTED = ['image/png', 'image/jpeg', 'image/webp'];const SELECTABLE_MODELS: SelectableModel[] = MODEL_CATALOG.map((m) => ({  id: m.id,  name: m.name,  backend: 'onnx',  category: 'Multimodal (CLIP)',  size: m.size,  vision: true,}));function toLibraryCard(photo: PhotoEntry): ImageResultCard {  const category = photo.processing    ? 'Analyzing…'    : photo.category      ? `${photo.category} · ${photo.similarCount} similar`      : undefined;  return {    id: photo.id,    src: photo.src,    label: photo.filename,    category,    score: photo.processing || photo.confidence === 0 ? undefined : photo.confidence,    processing: photo.processing,  };}export function ImageSearchBlock() {  const lib: PhotoLibrary = usePhotoLibrary({    modelId: DEFAULT_MODEL_ID,    createEmbeddingModel: (id, onProgress) =>      transformers.multimodalEmbedding(id, { onProgress: (p) => onProgress(p as any) }),    createZeroShotClassifier: (id) => transformers.zeroShotImageClassifier(id),    isModelCached: (id) => isModelCached(id),    labelPresets: { photo: { labels: LABEL_PRESETS.photo.labels }, product: { labels: LABEL_PRESETS.product.labels } },    getModelDimensions: (id) => getModel(id).dimensions,    defaultTopK: DEFAULT_TOP_K,    minSimilarityFallback: MIN_SIMILARITY_FALLBACK,  });  const model = getModel(lib.activeModelId);  const statusText = lib.switching    ? lib.reindexProgress      ? `re-indexing ${lib.reindexProgress.completed}/${lib.reindexProgress.total}…`      : 're-indexing library…'    : lib.modelStatus === 'loading'      ? `loading ${model.name}… ${Math.round(lib.modelProgress * 100)}%`      : lib.ingestProgress        ? `embedding ${lib.ingestProgress.completed}/${lib.ingestProgress.total}…`        : lib.error          ? 'error'          : lib.modelReady            ? `ready - ${lib.photos.length} photo${lib.photos.length === 1 ? '' : 's'} indexed`            : 'idle - load a model to start';  const [mode, setMode] = useState<SearchMode>('text');  const [query, setQuery] = useState('');  const [hits, setHits] = useState<RankedHit[] | null>(null);  const [refImage, setRefImage] = useState<string | null>(null);  const [searching, setSearching] = useState(false);  const searchDisabled = !lib.modelReady || lib.switching || lib.photos.length === 0;  const ingestDisabled = !lib.modelReady || lib.busy || lib.switching;  const hasPhotos = lib.photos.length > 0;  const clearSearch = () => {    setHits(null);    setQuery('');    setRefImage(null);  };  const runTextSearch = async () => {    const q = query.trim();    if (!q || searchDisabled) return;    setSearching(true);    try {      setHits(await lib.searchByText(q));    } finally {      setSearching(false);    }  };  const runImageSearch = async (file: File) => {    if (searchDisabled) return;    const dataUrl = await readFileAsDataUrl(file);    setRefImage(dataUrl);    setSearching(true);    try {      setHits(await lib.searchByImage(dataUrl));    } finally {      setSearching(false);    }  };  const cards: ImageResultCard[] = [];  for (const hit of hits ?? []) {    const photo = lib.getPhoto(hit.id);    if (!photo) continue;    cards.push({      id: photo.id,      src: photo.src,      label: photo.filename,      category: photo.category || undefined,      score: hit.score,    });  }  const scored = cards.map((c) => ({ label: c.label ?? c.id, score: c.score ?? 0 }));  const topFilename = cards[0]?.label ?? '';  return (    <div className="mx-auto flex max-w-5xl flex-col gap-4 p-4">      {}      <p        role="status"        aria-live="polite"        className="text-xs text-muted-foreground"      >        {statusText}      </p>      {lib.error && (        <div          className="flex items-center justify-between gap-3 rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-xs text-destructive"        >          <span>{lib.error}</span>          <button            type="button"            onClick={lib.clearError}            className="rounded px-2 py-0.5 font-medium hover:bg-destructive/20 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"          >            Dismiss          </button>        </div>      )}      {}      <div        data-status={lib.modelStatus}        data-model-id={lib.activeModelId}        role="group"        aria-label="CLIP model status"        className="flex flex-col gap-3 rounded-xl border border-border bg-muted/40 p-4 sm:flex-row sm:items-start"      >        <div className="w-full sm:max-w-sm">          <ModelSelector            models={SELECTABLE_MODELS}            selectedId={lib.activeModelId}            onSelect={(id) => lib.requestModel(id)}          />        </div>        <div className="flex min-w-0 flex-1 flex-col gap-2">          {lib.modelStatus === 'idle' ? (            <>              <p className="text-sm text-muted-foreground">                <span className="font-medium text-foreground">{model.name}</span> ({model.size}) -                not loaded. It powers both search embeddings and categorization. Nothing downloads                until you press Load.              </p>              <button                type="button"                onClick={() => void lib.loadModel()}                className="inline-flex h-9 w-fit items-center rounded-md bg-primary px-4 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background"              >                Load {model.name}              </button>            </>          ) : (            <div>              <ModelDownloader                name={model.name}                size={model.size}                category="Multimodal (CLIP)"                progress={lib.modelProgressValue}                cached={lib.modelCached}                ready={lib.modelReady && !lib.switching}                className="max-w-sm"              />            </div>          )}          {}          {lib.pendingModelId && (            <div              className="flex flex-col gap-2 rounded-lg border border-amber-500/40 bg-amber-500/10 p-3 text-xs"            >              <span className="text-foreground">                Switch to <span className="font-medium">{getModel(lib.pendingModelId).name}</span>?                The {getModel(lib.pendingModelId).dimensions}-dim vector space is incompatible - all{' '}                {lib.photos.length} photos will be re-embedded and re-categorized.              </span>              <div className="flex items-center gap-2">                <button                  type="button"                  onClick={lib.confirmModelSwitch}                  className="inline-flex h-7 items-center rounded-md bg-primary px-3 font-medium text-primary-foreground hover:bg-primary/90 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background"                >                  Confirm switch                </button>                <button                  type="button"                  onClick={lib.cancelModelSwitch}                  className="inline-flex h-7 items-center rounded-md border border-border px-3 font-medium hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                >                  Cancel                </button>              </div>            </div>          )}          {}          {lib.switching && lib.reindexProgress && (            <div              data-completed={lib.reindexProgress.completed}              data-total={lib.reindexProgress.total}              role="status"              aria-live="polite"              aria-label="Re-index progress"              className="text-xs tabular-nums text-muted-foreground"            >              Re-indexing {lib.reindexProgress.completed}/{lib.reindexProgress.total} photos through{' '}              {model.name}…            </div>          )}        </div>      </div>      {}      <section className="flex flex-col gap-3">        <h2 className="text-sm font-semibold text-foreground">1 · Build a library</h2>        <div role="group" aria-label="Photo library upload">          <MediaDropzone            accept={ACCEPTED}            multiple            disabled={ingestDisabled}            addAnother={hasPhotos}            processing={lib.ingestProgress != null}            processingLabel={              lib.ingestProgress                ? `Embedding ${lib.ingestProgress.completed}/${lib.ingestProgress.total}…`                : 'Processing…'            }            title={lib.modelReady ? 'Drop photos here' : 'Load a model to start'}            subtitle="PNG, JPEG or WebP - or click to browse"            onFiles={(files) => void lib.ingest(files)}            onReject={(rejections) =>              lib.setRejection({                filename: rejections[0].file.name,                reason: rejections[0].reason,              })            }          />        </div>        {}        {lib.rejection && (          <div            className="flex items-center justify-between gap-3 rounded-md border border-destructive/40 bg-destructive/10 px-3 py-2 text-sm text-destructive"          >            <span>              Rejected <span className="font-medium">{lib.rejection.filename}</span>:{' '}              {lib.rejection.reason}            </span>            <button              type="button"              onClick={() => lib.setRejection(null)}              className="rounded px-2 py-0.5 text-xs font-medium hover:bg-destructive/20 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"            >              Dismiss            </button>          </div>        )}        {}        {lib.ingestProgress && (          <div            data-completed={lib.ingestProgress.completed}            data-total={lib.ingestProgress.total}            role="status"            aria-live="polite"            aria-label="Ingest progress"            className="flex items-center justify-between gap-3 rounded-lg border border-border bg-card px-3 py-2 text-sm"          >            <span className="tabular-nums text-muted-foreground">              Embedding {lib.ingestProgress.completed}/{lib.ingestProgress.total}…            </span>            <button              type="button"              onClick={lib.cancelIngest}              className="rounded-md border border-border px-2.5 py-1 text-xs font-medium hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"            >              Cancel            </button>          </div>        )}        {}        {hasPhotos && (          <div className="flex flex-wrap items-center gap-2">            <span className="text-sm font-medium tabular-nums">              {lib.photos.length} photo{lib.photos.length === 1 ? '' : 's'} indexed            </span>            <button              type="button"              onClick={lib.clearAll}              disabled={lib.busy || lib.switching}              className="ml-auto inline-flex h-7 items-center gap-1 rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground transition-colors hover:border-destructive hover:text-destructive disabled:opacity-50 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"            >              <Trash2 className="size-3.5" />              Clear all            </button>          </div>        )}        {}        {hasPhotos ? (          <div>            <ImageResultGallery              cards={lib.photos.map(toLibraryCard)}              layout="grid"              onDelete={(id) => lib.deletePhoto(id)}              scoreThresholds={{ high: 0.35, medium: 0.2 }}            />          </div>        ) : lib.modelReady ? (          <p            className="rounded-lg border border-dashed border-border px-4 py-8 text-center text-sm text-muted-foreground"          >            No photos yet. Drop some images above to build a searchable library.          </p>        ) : null}        {}        <ul aria-label="Indexed photos" className="sr-only">          {lib.photos.map((photo) => (            <li              key={photo.id}              data-id={photo.id}              data-filename={photo.filename}              data-category={photo.category}              data-confidence={photo.confidence.toFixed(4)}              data-embedded={photo.embedding !== null}              data-processing={photo.processing}            >              {photo.filename}: {photo.category} ({formatScore(photo.confidence)})            </li>          ))}        </ul>      </section>      {}      <section className="flex flex-col gap-4 border-t border-border pt-4">        <h2 className="text-sm font-semibold text-foreground">          2 · Search the shared vector space        </h2>        {}        <div className="flex flex-wrap items-center gap-3">          <div data-mode={mode}>            <SegmentedModePicker<SearchMode>              items={[                { id: 'text', label: 'Text' },                { id: 'image', label: 'Image' },              ]}              selectedId={mode}              onSelect={(m) => {                setMode(m);                clearSearch();              }}              aria-label="Search mode"            />          </div>          <span            data-threshold={lib.minSimilarity}            className="ml-auto text-xs text-muted-foreground"          >            Minimum similarity: {formatScore(lib.minSimilarity)}          </span>        </div>        {}        <div data-value={lib.topK} className="max-w-xs">          <ParameterSlider            label="Results (top-K)"            value={lib.topK}            onChange={lib.setTopK}            min={1}            max={50}            step={1}          />        </div>        {}        {mode === 'text' && (          <div className="flex flex-wrap items-center gap-2">            <div className="relative flex-1">              <Search className="pointer-events-none absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" />              <input                type="text"                aria-label="Search query"                value={query}                disabled={searchDisabled}                placeholder="Describe a photo, e.g. “a photo of a dog”"                onChange={(e) => setQuery(e.target.value)}                onKeyDown={(e) => {                  if (e.key === 'Enter') void runTextSearch();                }}                className="h-9 w-full rounded-md border border-border bg-background pl-8 pr-3 text-sm outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:opacity-50"              />            </div>            <button              type="button"              onClick={() => void runTextSearch()}              disabled={searchDisabled || !query.trim() || searching}              className="inline-flex h-9 items-center gap-1.5 rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground transition-colors hover:bg-primary/90 disabled:opacity-50 focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 focus-visible:ring-offset-2 focus-visible:ring-offset-background"            >              {searching ? <Loader2 className="size-4 animate-spin" /> : <Search className="size-4" />}              Search            </button>            {hits && (              <button                type="button"                onClick={clearSearch}                className="inline-flex h-9 items-center gap-1 rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"              >                <X className="size-3.5" />                Clear              </button>            )}          </div>        )}        {}        {mode === 'image' && (          <div className="flex flex-col gap-3 sm:flex-row sm:items-start">            <div              className="w-full sm:max-w-xs"              role="group"              aria-label="Reference image upload"            >              <MediaDropzone                accept={ACCEPTED}                multiple={false}                disabled={searchDisabled}                processing={searching}                processingLabel="Searching…"                title="Drop a reference image"                subtitle="Find visually similar photos"                onFiles={(files) => void runImageSearch(files[0])}              />            </div>            {refImage && (              <div className="flex items-center gap-2">                {}                <img src={refImage} alt="Reference" className="size-20 rounded-md object-cover" />                <button                  type="button"                  onClick={clearSearch}                  className="inline-flex h-8 items-center gap-1 rounded-md border border-border px-2.5 text-xs font-medium text-muted-foreground hover:bg-accent focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"                >                  <X className="size-3.5" />                  Clear                </button>              </div>            )}          </div>        )}        {}        {hits !== null &&          (cards.length > 0 ? (            <div className="flex flex-col gap-4">              <div                data-count={cards.length}                data-top={topFilename}                role="region"                aria-label="Search results"              >                <ImageResultGallery                  cards={cards}                  layout="grid"                  scoreThresholds={{ high: 0.35, medium: 0.2 }}                />              </div>              <ScoredResultBarList results={scored} sort={false} limit={10} />              {}              <ol aria-label="Search results ranking" className="sr-only">                {cards.map((c, i) => (                  <li                    key={c.id}                    data-rank={i}                    data-id={c.id}                    data-filename={c.label}                    data-score={(c.score ?? 0).toFixed(4)}                    data-tone={scoreTone(c.score ?? 0)}                  >                    {c.label}: {formatScore(c.score ?? 0)}                  </li>                ))}              </ol>            </div>          ) : (            <p              className="rounded-lg border border-dashed border-border px-4 py-10 text-center text-sm text-muted-foreground"            >              No photos at or above the {formatScore(lib.minSimilarity)} similarity threshold.            </p>          ))}        {searchDisabled && lib.photos.length === 0 && (          <p className="text-sm text-muted-foreground">Ingest photos above first, then search them.</p>        )}      </section>    </div>  );}