Add voice notes: on-device transcription, multi-language, watch capture

- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud - AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline with enabled-language set and confidence-based auto-pick, re-transcribe menu - Settings > Transcription > Languages with asset download/reserve handling - watchOS app + accessory complication: one-button recorder, WCSession transferFile to phone, WatchInbox staging, direct-upsert ingestion - Player UI, transcription status chips, quick-capture mic in notes list - Version 0.2, changelog + README updated
2026-07-15 13:30:43 -04:00
parent 6a93cedaa6
commit 86d74806c9
44 changed files with 3329 additions and 123 deletions
@@ -40,7 +40,10 @@ struct CaptureContext: Codable, Sendable, Equatable {
 /// One note = one JSON file in the iCloud container, filed under
 /// `Records/YYYY/MM/<ULID>.json` (UTC-bucketed by the ULID's timestamp).
 struct Note: SyncDocument, VersionedDocument, Equatable {
-    static let currentSchemaVersion = 1
+    /// v2 added `Payload.audio` for voice notes. Because it is optional, a v1
+    /// file decodes unchanged (missing key ⇒ nil) and `migrateIfNeeded` only
+    /// bumps the stamped version — no field back-fill is needed.
+    static let currentSchemaVersion = 2

    struct Meta: Codable, Sendable, Equatable {
        var id: ULID
@@ -49,18 +52,43 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
        var modifiedAt: Date
    }

-    /// How the note's text came to be. `transcribed` is reserved for the
-    /// planned voice-capture path.
+    /// How the note's text came to be. `transcribed` marks a voice note whose
+    /// text is (or will be) produced by on-device transcription.
    enum Source: String, Codable, Sendable {
        case typed
        case transcribed
    }

+    /// Where a voice note is in its transcription lifecycle.
+    enum TranscriptionStatus: String, Codable, Sendable {
+        case pending
+        case done
+        case failed
+    }
+
+    /// The audio-note sidecar metadata. Present only on voice notes; the audio
+    /// bytes themselves live in a `.m4a` next to the note JSON (path derived,
+    /// never stored — see `audioRelativePath`).
+    struct AudioInfo: Codable, Sendable, Equatable {
+        var duration: TimeInterval
+        /// Per-install UUID of the device that owns pending transcription. Only
+        /// the device whose `InstallIdentity.installID` matches this transcribes;
+        /// every other device renders status only. Re-transcribe reassigns it.
+        var transcriberDeviceID: String
+        var transcriptionStatus: TranscriptionStatus
+        /// BCP-47 locale the transcript was produced in; nil while pending.
+        var transcriptionLocaleID: String?
+        var transcribedAt: Date?
+    }
+
    struct Payload: Codable, Sendable, Equatable {
        var text: String
        var source: Source
        var tags: [String]
        var context: CaptureContext
+        /// Voice-note sidecar metadata, nil for a plain typed note. Optional so
+        /// a v1 document (no `audio` key) decodes as a text note unchanged.
+        var audio: AudioInfo?
    }

    var meta: Meta
@@ -75,16 +103,67 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
        TimeBucketedLayout.relativePath(for: id)
    }

+    /// The `.m4a` sidecar path for a note, derived by swapping `.json` → `.m4a`
+    /// on the note's **actual** JSON path. The audio path is never stored — it
+    /// is always this swap — so it cannot drift from the note it belongs to, and
+    /// deleting a note whose file sits under a legacy (pre-fix) month bucket
+    /// still targets the sidecar that shares that bucket.
+    static func audioRelativePath(forJSONPath jsonPath: String) -> String {
+        guard jsonPath.hasSuffix(".json") else { return jsonPath + ".m4a" }
+        return String(jsonPath.dropLast(".json".count)) + ".m4a"
+    }
+
+    /// The `.m4a` sidecar path recomputed from a note id, sharing the note's
+    /// UTC-bucketed layout. Safe for load/warm paths because audio exists only
+    /// on v2+ records, which always file under the current UTC bucket — the
+    /// legacy local-month concern applies solely to v1 text notes (no audio).
+    static func audioRelativePath(forID id: ULID) -> String {
+        audioRelativePath(forJSONPath: relativePath(forID: id))
+    }
+
    static func create(
        text: String,
        source: Source = .typed,
        tags: [String] = [],
-        context: CaptureContext
+        context: CaptureContext,
+        audio: AudioInfo? = nil
    ) -> Note {
        let now = Date()
        return Note(
            meta: Meta(id: ULID(), schemaVersion: currentSchemaVersion, createdAt: now, modifiedAt: now),
-            payload: Payload(text: text, source: source, tags: tags, context: context)
+            payload: Payload(text: text, source: source, tags: tags, context: context, audio: audio)
+        )
+    }
+
+    /// Factory for a note ingested from another device's recording (the watch,
+    /// which has no iCloud of its own). The audio was captured elsewhere; this
+    /// device only stores and transcribes it. `createdAt` is the ULID's embedded
+    /// timestamp — the record-start instant on the capturing device — so the note
+    /// buckets under the month it was *recorded*, not the (possibly much later)
+    /// month it was ingested. Text is empty until transcription lands.
+    static func createIngested(
+        id: ULID,
+        duration: TimeInterval,
+        transcriberDeviceID: String,
+        tags: [String] = [],
+        context: CaptureContext
+    ) -> Note {
+        let recordedAt = id.timestamp
+        return Note(
+            meta: Meta(id: id, schemaVersion: currentSchemaVersion, createdAt: recordedAt, modifiedAt: recordedAt),
+            payload: Payload(
+                text: "",
+                source: .transcribed,
+                tags: tags,
+                context: context,
+                audio: AudioInfo(
+                    duration: duration,
+                    transcriberDeviceID: transcriberDeviceID,
+                    transcriptionStatus: .pending,
+                    transcriptionLocaleID: nil,
+                    transcribedAt: nil
+                )
+            )
        )
    }

@@ -30,6 +30,17 @@ final class NoteEntity {
    var timeZoneID: String?
    var device: String?

+    // Flattened AudioInfo — nil columns mean a plain text note. `hasAudio` is
+    // stored rather than derived from `audioDuration` so a zero-length recording
+    // is still recognizably a voice note. `transcribedAt` is deliberately not
+    // cached (the file is authoritative for that timestamp), so a cache-only
+    // reconstruction of a completed note has a nil `transcribedAt`.
+    var hasAudio: Bool = false
+    var audioDuration: Double?
+    var transcriptionStatusRaw: String?
+    var transcriptionLocaleID: String?
+    var transcriberDeviceID: String?
+
    init(id: String, jsonRelativePath: String) {
        self.id = id
        self.jsonRelativePath = jsonRelativePath
@@ -70,6 +81,31 @@ extension NoteEntity {
        Note.Source(rawValue: sourceRaw) ?? .typed
    }

+    /// Reconstructs `AudioInfo` from the flattened columns (nil for a text
+    /// note). `transcribedAt` is not cached, so it comes back nil even for a
+    /// completed transcription — acceptable for the cache-only offline fallback,
+    /// where the file (which carries it) simply wasn't reachable.
+    var audio: Note.AudioInfo? {
+        get {
+            guard hasAudio else { return nil }
+            return Note.AudioInfo(
+                duration: audioDuration ?? 0,
+                transcriberDeviceID: transcriberDeviceID ?? "",
+                transcriptionStatus: transcriptionStatusRaw
+                    .flatMap(Note.TranscriptionStatus.init(rawValue:)) ?? .pending,
+                transcriptionLocaleID: transcriptionLocaleID,
+                transcribedAt: nil
+            )
+        }
+        set {
+            hasAudio = newValue != nil
+            audioDuration = newValue?.duration
+            transcriberDeviceID = newValue?.transcriberDeviceID
+            transcriptionStatusRaw = newValue?.transcriptionStatus.rawValue
+            transcriptionLocaleID = newValue?.transcriptionLocaleID
+        }
+    }
+
    /// First line of the text, used as the display title.
    var title: String {
        text.split(separator: "\n", omittingEmptySubsequences: true)
@@ -35,6 +35,7 @@ enum NoteMapper {
        entity.modifiedAt = note.meta.modifiedAt
        entity.jsonRelativePath = relativePath
        entity.context = note.payload.context
+        entity.audio = note.payload.audio
    }

    /// Near-lossless reconstruction from the cache, used only as the offline
@@ -52,7 +53,8 @@ enum NoteMapper {
                text: entity.text,
                source: entity.source,
                tags: entity.tags,
-                context: entity.context
+                context: entity.context,
+                audio: entity.audio
            )
        )
    }
@@ -6,7 +6,8 @@ import SwiftData
 /// from the files (an empty cache is the entire contract between the two).
 enum NotesModelContainer {
    /// Bump whenever the cache schema changes — wipes and rebuilds from files.
-    static let cacheSchemaVersion = 1
+    /// v2 added the flattened audio columns on `NoteEntity`.
+    static let cacheSchemaVersion = 2
    private static let cacheSchemaVersionKey = "Notes.cacheSchemaVersion"
    private static let identityTokenKey = "Notes.iCloudIdentityToken"