Voice Notes

'use client';/** * @file voice-notes.tsx * @description Voice Notes — record or upload → Whisper/Moonshine transcription → saved notes with synced word replay and block-local semantic search; owns its STT selector. */import { useEffect, useRef, useState, type Dispatch, type SetStateAction } from 'react';import {  useModelLoad,  useTranscribe,  useVoiceRecorder,  type UseModelLoadReturn,} from '@localmode/react';import type { EmbeddingModel, SpeechToTextModel, VectorDB } from '@localmode/core';import { transcribe, createVectorDB, embed } from '@localmode/core';import { transformers, isModelCached } from '@localmode/transformers';import { VoiceButton, type VoiceButtonState } from '@/components/voice-button';import { MicSelector } from '@/components/mic-selector';import { WaveformActivityBars } from '@/components/waveform-activity-bars';import { FileDropzone } from '@/components/file-dropzone';import { TranscribedNoteCard } from '@/components/transcribed-note-card';import { SyncedTranscriptViewer } from '@/components/synced-transcript-viewer';import { ScoredResultBarList, type ScoredResult } from '@/components/scored-result-bar-list';import { ModelSelector } from '@/components/model-selector';import { ModelLoadingPanel } from '@/components/model-loading-panel';import { ErrorAlert } from '@/components/error-alert';export interface NoteSearchHit {  noteId: string;  score: number;}interface IndexableNote {  id: string;  text: string;}export function useNoteIndex() {  const dbPromiseRef = useRef<Promise<VectorDB<{ text: string }>> | null>(null);  const indexedIdsRef = useRef<Set<string>>(new Set());  const getDb = (dimensions: number): Promise<VectorDB<{ text: string }>> => {    if (!dbPromiseRef.current) {      const promise = createVectorDB<{ text: string }>({        name: 'voice-notes-search',        dimensions,        storage: 'memory',      });      promise.catch(() => {        if (dbPromiseRef.current === promise) dbPromiseRef.current = null;      });      dbPromiseRef.current = promise;    }    return dbPromiseRef.current;  };  useEffect(() => {    return () => {      const pending = dbPromiseRef.current;      dbPromiseRef.current = null;      indexedIdsRef.current.clear();      if (pending) void pending.then((db) => db.close()).catch(() => {});    };  }, []);  const search = async (    model: EmbeddingModel,    notes: readonly IndexableNote[],    query: string,    abortSignal?: AbortSignal,  ): Promise<NoteSearchHit[]> => {    abortSignal?.throwIfAborted();    const { embedding: queryVector } = await embed({      model,      value: query,      ...(abortSignal ? { abortSignal } : {}),    });    const db = await getDb(queryVector.length);    const indexedIds = indexedIdsRef.current;    const liveIds = new Set(notes.map((n) => n.id));    for (const id of [...indexedIds]) {      if (!liveIds.has(id)) {        abortSignal?.throwIfAborted();        await db.delete(id);        indexedIds.delete(id);      }    }    for (const note of notes) {      if (indexedIds.has(note.id)) continue;      abortSignal?.throwIfAborted();      const { embedding } = await embed({        model,        value: note.text,        ...(abortSignal ? { abortSignal } : {}),      });      await db.add({ id: note.id, vector: embedding, metadata: { text: note.text } });      indexedIds.add(note.id);    }    abortSignal?.throwIfAborted();    const results = await db.search(queryVector, { k: Math.max(notes.length, 1) });    return results.map((r) => ({ noteId: r.id, score: r.score }));  };  return { search };}interface SttModelEntry {  id: string;  name: string;  size: string;  timestamps: boolean;}const STT_MODELS: readonly SttModelEntry[] = [  { id: 'Xenova/whisper-tiny.en', name: 'Whisper Tiny EN', size: '~40MB', timestamps: true },  { id: 'onnx-community/moonshine-tiny-ONNX', name: 'Moonshine Tiny', size: '~50MB', timestamps: false },  { id: 'onnx-community/moonshine-base-ONNX', name: 'Moonshine Base', size: '~237MB', timestamps: false },];const DEFAULT_STT_MODEL_ID = STT_MODELS[0].id;function sttSupportsTimestamps(modelId: string): boolean {  return STT_MODELS.find((m) => m.id === modelId)?.timestamps ?? false;}const EMBEDDING_MODEL_ID = 'Xenova/bge-small-en-v1.5';const EMBEDDING_MODEL_SIZE = '~34MB';const NO_SPEECH_TEXT = '[No speech detected]';const ACCEPTED_AUDIO_MIME_TYPES = [  'audio/wav',  'audio/x-wav',  'audio/wave',  'audio/mp3',  'audio/mpeg',  'audio/webm',  'audio/ogg',  'audio/mp4',  'audio/x-m4a',  'audio/m4a',  'audio/aac',  'video/mp4',];interface NoteWord {  text: string;  start: number;  end: number;}interface SavedNote {  id: string;  text: string;  audio: Blob;  timestamp: Date;  words: NoteWord[];}export function VoiceNotesBlock() {  const [sttModelId, setSttModelId] = useState(DEFAULT_STT_MODEL_ID);  const [notes, setNotes] = useState<SavedNote[]>([]);  return (    <div className="flex flex-col gap-4 p-4">      <NotesSession        key={sttModelId}        sttModelId={sttModelId}        onSttModelIdChange={setSttModelId}        notes={notes}        onNotesChange={setNotes}      />    </div>  );}interface NotesSessionProps {  sttModelId: string;  onSttModelIdChange: (id: string) => void;  notes: SavedNote[];  onNotesChange: Dispatch<SetStateAction<SavedNote[]>>;}function NotesSession({ sttModelId, onSttModelIdChange, notes, onNotesChange }: NotesSessionProps) {  const stt = useModelLoad<SpeechToTextModel>({    key: `voice-notes-stt:${sttModelId}`,    create: (onProgress) =>      transformers.speechToText(sttModelId, {        onProgress: (p) => onProgress(p as Parameters<typeof onProgress>[0]),      }),    warmup: (model) => transcribe({ model, audio: new Float32Array(16_000) }),    isCached: () => isModelCached(sttModelId),  });  const sttEntry = STT_MODELS.find((m) => m.id === sttModelId) ?? STT_MODELS[0];  const supportsTimestamps = sttSupportsTimestamps(sttModelId);  const addNote = (note: SavedNote) => onNotesChange((prev) => [note, ...prev]);  const deleteNote = (id: string) => onNotesChange((prev) => prev.filter((n) => n.id !== id));  return (    <>      {}      <section className="flex flex-col gap-2">        <div data-model-id={sttModelId}>          <p className="mb-1 text-xs font-medium text-muted-foreground">            Speech-to-text model - downloads only when you transcribe, upload audio, or press its            download action.          </p>          <ModelSelector            models={STT_MODELS.map((m) => ({              id: m.id,              name: m.name,              backend: 'onnx' as const,              category: m.timestamps ? 'STT · timestamps' : 'STT',              size: m.size,            }))}            selectedId={sttModelId}            busyIds={stt.status === 'loading' ? new Set([sttModelId]) : undefined}            onSelect={onSttModelIdChange}            onDownload={(id) => {              if (id === sttModelId) void stt.load().catch(() => {});              else onSttModelIdChange(id);            }}          />        </div>        {stt.status === 'loading' && (          <ModelLoadingPanel            name={sttEntry.name}            size={sttEntry.size}            category="Speech-to-text"            progress={stt.progressValue}            cached={stt.cached === true}          />        )}      </section>      {}      {stt.model ? (        <NotesSurface          stt={stt as UseModelLoadReturn<SpeechToTextModel>}          sttModel={stt.model}          supportsTimestamps={supportsTimestamps}          notes={notes}          onAddNote={addNote}          onDeleteNote={deleteNote}        />      ) : (        <p className="p-4 text-sm text-muted-foreground">Preparing…</p>      )}    </>  );}interface NotesSurfaceProps {  stt: UseModelLoadReturn<SpeechToTextModel>;  sttModel: SpeechToTextModel;  supportsTimestamps: boolean;  notes: SavedNote[];  onAddNote: (note: SavedNote) => void;  onDeleteNote: (id: string) => void;}function NotesSurface({  stt,  sttModel,  supportsTimestamps,  notes,  onAddNote,  onDeleteNote,}: NotesSurfaceProps) {  const [micDeviceId, setMicDeviceId] = useState<string | undefined>();  const recorder = useVoiceRecorder({ deviceId: micDeviceId });  const transcriber = useTranscribe({ model: sttModel, returnTimestamps: supportsTimestamps });  const [voiceState, setVoiceState] = useState<VoiceButtonState>('idle');  const [micVolume, setMicVolume] = useState(0);  const [dismissedError, setDismissedError] = useState<string | null>(null);  const noteIndex = useNoteIndex();  const embedLoad = useModelLoad<EmbeddingModel>({    key: `voice-notes-embedding:${EMBEDDING_MODEL_ID}`,    create: (onProgress) =>      transformers.embedding(EMBEDDING_MODEL_ID, {        onProgress: (p) => onProgress(p as Parameters<typeof onProgress>[0]),      }),    isCached: () => isModelCached(EMBEDDING_MODEL_ID),  });  const [searchQuery, setSearchQuery] = useState('');  const [searchResults, setSearchResults] = useState<ScoredResult[] | null>(null);  const [searching, setSearching] = useState(false);  const [searchError, setSearchError] = useState<string | null>(null);  const recordStartedAtRef = useRef(0);  const buttonState: VoiceButtonState =    voiceState === 'recording' && recorder.error ? 'error' : voiceState;  useEffect(() => {    if (buttonState !== 'recording') {      setMicVolume(0);      return;    }    const id = window.setInterval(() => setMicVolume(Math.min(1, recorder.getVolume() * 4)), 100);    return () => window.clearInterval(id);  }, [buttonState]);  const startRecording = async () => {    setVoiceState('recording');    recordStartedAtRef.current = Date.now();    await recorder.startRecording();  };  const transcribeToNote = async (blob: Blob) => {    setVoiceState('processing');    setDismissedError(null);    try {      await stt.load();    } catch {      setVoiceState('error');      return;    }    const result = await transcriber.execute(blob);    if (!result) {      setVoiceState('idle');      return;    }    onAddNote({      id: crypto.randomUUID(),      text: result.text.trim() || NO_SPEECH_TEXT,      audio: blob,      timestamp: new Date(),      words: supportsTimestamps        ? (result.segments ?? [])            .filter((s) => Number.isFinite(s.start) && Number.isFinite(s.end))            .map((s) => ({ text: s.text.trim(), start: s.start, end: s.end }))        : [],    });    setVoiceState('success');  };  const stopAndTranscribe = async () => {    if (buttonState !== 'recording') return;    if (Date.now() - recordStartedAtRef.current < 500) return;    const blob = await recorder.stopRecording();    if (!blob) {      setVoiceState('error');      return;    }    await transcribeToNote(blob);  };  const runSearch = async () => {    const query = searchQuery.trim();    if (!query || searching || notes.length === 0) return;    setSearching(true);    setSearchError(null);    try {      await embedLoad.load();      const model = embedLoad.model;      if (!model) throw new Error('Embedding model unavailable');      const hits = await noteIndex.search(model, notes, query);      const byId = new Map(notes.map((n) => [n.id, n]));      setSearchResults(        hits          .filter((h) => byId.has(h.noteId))          .map((h) => ({ label: byId.get(h.noteId)!.text, score: h.score })),      );    } catch (err) {      setSearchError(err instanceof Error ? err.message : String(err));    } finally {      setSearching(false);    }  };  const latestTimestamped = notes.find((n) => n.words.length > 0);  const latestText = notes[0]?.text ?? '';  const rawErrorText =    recorder.error?.message ??    transcriber.error?.message ??    searchError ??    (buttonState === 'error' ? 'Recording failed' : null);  const errorText = rawErrorText && rawErrorText !== dismissedError ? rawErrorText : null;  const statusText =    buttonState === 'recording'      ? 'recording'      : buttonState === 'processing'        ? 'transcribing'        : searching          ? 'searching'          : errorText            ? 'error'            : notes.length > 0              ? 'ready'              : 'idle';  const statusLabel =    statusText === 'recording'      ? 'Recording…'      : statusText === 'transcribing'        ? 'Transcribing…'        : statusText === 'searching'          ? 'Searching…'          : statusText === 'error'            ? 'Error'            : statusText === 'ready'              ? 'Ready'              : 'Idle';  return (    <div className="flex flex-col gap-4">      <p        data-status={statusText}        role="status"        aria-live="polite"        aria-label="Status"        className="text-xs text-muted-foreground"      >        {statusLabel}      </p>      {errorText && (        <ErrorAlert          message={errorText}          onDismiss={() => {            const startFailed = !!recorder.error && voiceState === 'recording';            setDismissedError(rawErrorText);            setSearchError(null);            recorder.clearError();            transcriber.reset();            if (voiceState === 'error' || startFailed) setVoiceState('idle');          }}        />      )}      {}      <section className="flex flex-col gap-3 rounded-lg border border-border p-4">        <h2 className="text-sm font-medium">Record a note</h2>        <div className="flex flex-wrap items-center gap-4">          <span>            <VoiceButton              state={buttonState}              onStart={() => void startRecording()}              onStop={() => void stopAndTranscribe()}              volume={micVolume}            />          </span>          <button            type="button"            onClick={() => void stopAndTranscribe()}            disabled={buttonState !== 'recording'}            className="inline-flex h-8 items-center rounded-md border border-border px-3 text-sm focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:opacity-50"          >            Stop &amp; transcribe          </button>          <WaveformActivityBars            active={buttonState === 'recording'}            volume={micVolume}            height={28}            barCount={9}          />          <span>            <MicSelector value={micDeviceId} onValueChange={setMicDeviceId} />          </span>        </div>        {latestText && (          <div            role="region"            aria-label="Latest transcript"            className="min-h-6 rounded-md bg-muted/50 p-2 text-sm whitespace-pre-wrap"          >            {latestText}          </div>        )}        {buttonState === 'processing' && (          <div className="flex items-center gap-2">            <TranscribedNoteCard transcribing className="flex-1" />            <button              type="button"              onClick={() => {                transcriber.cancel();                setVoiceState('idle');              }}              className="inline-flex h-8 items-center rounded-md border border-border px-3 text-sm focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50"            >              Cancel            </button>          </div>        )}      </section>      {}      <section aria-label="Or upload audio" className="flex flex-col gap-3 rounded-lg border border-border p-4">        <h2 className="text-sm font-medium">Or upload audio</h2>        <div>          <FileDropzone            onUpload={(files) => {              const file = files[0];              if (file) void transcribeToNote(file);            }}            accept={ACCEPTED_AUDIO_MIME_TYPES}            multiple={false}            disabled={buttonState === 'processing' || buttonState === 'recording'}            processing={buttonState === 'processing'}            processingLabel="Transcribing…"            label="Drop an audio file or click to browse"            hint=".wav, .mp3, .webm, .m4a, .ogg, .mp4"          />        </div>      </section>      {}      {supportsTimestamps ? (        latestTimestamped && (          <section            aria-label="Synced transcript"            className="flex flex-col gap-2 rounded-lg border border-border p-4"          >            <p className="text-xs text-muted-foreground">              Synced transcript - real Whisper SEGMENT timestamps (each highlighted span is one              segment, not a single word). Click a span to seek.            </p>            <SyncedTranscriptViewer words={latestTimestamped.words} audio={latestTimestamped.audio} />          </section>        )      ) : (        <p className="rounded-lg border border-dashed border-border p-3 text-xs text-muted-foreground">          Synced replay is disabled - segment timestamps are only validated on Whisper Tiny EN.          Plain transcripts still work with the selected Moonshine model.        </p>      )}      {}      <section className="flex flex-col gap-3 rounded-lg border border-border p-4">        <h2 className="text-sm font-medium">Search notes</h2>        <div className="flex flex-wrap items-center gap-2">          <input            value={searchQuery}            onChange={(e) => setSearchQuery(e.target.value)}            onKeyDown={(e) => {              if (e.key === 'Enter') void runSearch();            }}            placeholder={notes.length === 0 ? 'Save a note first…' : 'Semantic search…'}            disabled={notes.length === 0}            aria-label="Search notes"            className="h-8 min-w-56 flex-1 rounded-md border border-input bg-background px-3 text-sm focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:opacity-50"          />          <button            type="button"            onClick={() => void runSearch()}            disabled={!searchQuery.trim() || searching || notes.length === 0}            className="inline-flex h-8 items-center rounded-md bg-primary px-3 text-sm font-medium text-primary-foreground focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:opacity-50"          >            {searching ? 'Searching…' : 'Search'}          </button>        </div>        {embedLoad.status === 'loading' && (          <ModelLoadingPanel            name="BGE Small EN v1.5"            size={EMBEDDING_MODEL_SIZE}            category="Embedding"            progress={embedLoad.progressValue}            cached={embedLoad.cached === true}          />        )}        {searchResults && (          <div            data-count={searchResults.length}            data-top-label={searchResults[0]?.label ?? ''}            role="region"            aria-label="Note search results"          >            <ScoredResultBarList              results={searchResults}              isLoading={searching}              emptyState="No matching notes"            />          </div>        )}      </section>      {}      <section className="flex flex-col gap-3 rounded-lg border border-border p-4">        <div className="flex items-center justify-between">          <h2 className="text-sm font-medium">Saved notes</h2>          <span className="text-xs tabular-nums text-muted-foreground">            {notes.length} note{notes.length === 1 ? '' : 's'}          </span>        </div>        {notes.length === 0 ? (          <p className="text-sm text-muted-foreground">            No notes yet - record or upload audio to create your first note.          </p>        ) : (          <ul aria-label="Saved notes" className="flex flex-col gap-2">            {notes.map((note) => (              <li                key={note.id}                data-note-id={note.id}                className="group/note flex flex-col gap-1"              >                <TranscribedNoteCard                  text={note.text}                  timestamp={note.timestamp}                  audio={note.audio}                  onDelete={() => onDeleteNote(note.id)}                />              </li>            ))}          </ul>        )}      </section>    </div>  );}