Add voice notes: on-device transcription, multi-language, watch capture
- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud - AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline with enabled-language set and confidence-based auto-pick, re-transcribe menu - Settings > Transcription > Languages with asset download/reserve handling - watchOS app + accessory complication: one-button recorder, WCSession transferFile to phone, WatchInbox staging, direct-upsert ingestion - Player UI, transcription status chips, quick-capture mic in notes list - Version 0.2, changelog + README updated
This commit is contained in:
@@ -37,6 +37,12 @@ final class SyncEngine {
|
||||
/// the next successful write or rebuild.
|
||||
private(set) var lastSyncError: String?
|
||||
|
||||
/// Fired after a watch recording is ingested, carrying the new note's id, so
|
||||
/// the composition root can enqueue on-device transcription. Wired by
|
||||
/// `AppServices`; nil until then, so ingestion is safe before the
|
||||
/// transcription stack exists.
|
||||
var onAudioNoteIngested: ((ULID) -> Void)?
|
||||
|
||||
static let recordsDirectory = "Records"
|
||||
static let stubsDirectory = "Stubs"
|
||||
|
||||
@@ -54,6 +60,10 @@ final class SyncEngine {
|
||||
private var dataMonitorTask: Task<Void, Never>?
|
||||
private var stubMonitorTask: Task<Void, Never>?
|
||||
|
||||
/// Guards `ingestStagedWatchRecordings` against overlapping triggers (connect,
|
||||
/// foreground reconcile, receiver callback) double-ingesting the same inbox.
|
||||
private var isDrainingInbox = false
|
||||
|
||||
private var cacheContext: ModelContext { modelContainer.mainContext }
|
||||
|
||||
init(modelContainer: ModelContainer) {
|
||||
@@ -142,6 +152,12 @@ final class SyncEngine {
|
||||
}
|
||||
let path = Note.relativePath(forID: id)
|
||||
try await tombstones.softDelete(dataRelativePath: Self.dataPath(path), at: path)
|
||||
// Drop the audio sidecar too. Best-effort and harmless when the note had
|
||||
// no audio (nothing to remove); the sidecar never gets its own stub, so
|
||||
// this is the only thing that reaps it.
|
||||
if let store {
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
}
|
||||
lastSyncError = nil
|
||||
// Cache cleanup happens via the monitors (file removal + stub
|
||||
// appearance both drop the row), but do it eagerly too so the UI
|
||||
@@ -163,6 +179,273 @@ final class SyncEngine {
|
||||
return NoteMapper.note(from: entity)
|
||||
}
|
||||
|
||||
// MARK: - Audio notes
|
||||
|
||||
/// Create a voice note: write the `.m4a` sidecar **before** the JSON, so a
|
||||
/// note that claims audio can never sync to another device ahead of its
|
||||
/// bytes (which would surface a pending audio note whose `.m4a` doesn't yet
|
||||
/// exist). The note starts with empty (or provisional) text and a `.pending`
|
||||
/// transcription owned by `transcriberDeviceID`; the transcript arrives later
|
||||
/// through `applyTranscription`. Like every other write, the cache catches up
|
||||
/// via the monitor — no direct upsert here.
|
||||
@discardableResult
|
||||
func createAudioNote(
|
||||
audioFileURL: URL,
|
||||
duration: TimeInterval,
|
||||
text: String = "",
|
||||
tags: [String] = [],
|
||||
context: CaptureContext,
|
||||
transcriberDeviceID: String
|
||||
) async throws -> Note {
|
||||
guard let store else {
|
||||
report("Note not saved — iCloud not connected")
|
||||
throw SyncError.iCloudUnavailable
|
||||
}
|
||||
let audio = Note.AudioInfo(
|
||||
duration: duration,
|
||||
transcriberDeviceID: transcriberDeviceID,
|
||||
transcriptionStatus: .pending,
|
||||
transcriptionLocaleID: nil,
|
||||
transcribedAt: nil
|
||||
)
|
||||
let note = Note.create(text: text, source: .transcribed, tags: tags, context: context, audio: audio)
|
||||
do {
|
||||
let audioData = try Data(contentsOf: audioFileURL)
|
||||
try await store.writeData(audioData, to: Self.dataPath(Note.audioRelativePath(forID: note.id)))
|
||||
_ = try await store.write(note, to: Self.dataPath(note.relativePath))
|
||||
} catch {
|
||||
report("Failed to save audio note", error)
|
||||
throw error
|
||||
}
|
||||
lastSyncError = nil
|
||||
return note
|
||||
}
|
||||
|
||||
/// Load a note's audio bytes for playback. Fail-fast (`.requestOnly`): an
|
||||
/// evicted file kicks off a download and returns nil immediately rather than
|
||||
/// stalling the player ~30s — the UI shows a "downloading" retry state and
|
||||
/// the bytes land on a later attempt (or via `requestAudioDownload`).
|
||||
func loadAudioData(for id: ULID) async -> Data? {
|
||||
guard let store else { return nil }
|
||||
return try? await store.readData(from: audioPath(for: id), downloadPolicy: .requestOnly)
|
||||
}
|
||||
|
||||
/// Load a note's audio with a bounded wait for an evicted file to download —
|
||||
/// for re-transcription, where blocking to fetch the source is acceptable
|
||||
/// (unlike interactive playback, which uses `loadAudioData`). Throws on a
|
||||
/// download timeout rather than yielding a dataless placeholder.
|
||||
func loadAudioDataWaiting(for id: ULID) async throws -> Data {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
return try await store.readData(from: audioPath(for: id), downloadPolicy: .waitBounded)
|
||||
}
|
||||
|
||||
/// Warm an evicted audio file ahead of an anticipated playback, so a later
|
||||
/// `loadAudioData` finds the bytes already local. Fire-and-forget.
|
||||
func requestAudioDownload(for id: ULID) async {
|
||||
guard let store else { return }
|
||||
await store.requestDownload(at: audioPath(for: id))
|
||||
}
|
||||
|
||||
/// Store-rooted `.m4a` sidecar path for a note. Audio exists only on v2+
|
||||
/// records, which file under the current UTC bucket, so recomputing the path
|
||||
/// from the id is always correct here.
|
||||
private func audioPath(for id: ULID) -> String {
|
||||
Self.dataPath(Note.audioRelativePath(forID: id))
|
||||
}
|
||||
|
||||
// MARK: - Transcription
|
||||
|
||||
/// Land a completed transcription onto a note. Re-reads the file (the
|
||||
/// authority) so a concurrent edit isn't clobbered, then **appends** rather
|
||||
/// than replaces when the user typed while the note was pending — non-empty
|
||||
/// text whose `modifiedAt` moved past the note's `createdAt`. Otherwise the
|
||||
/// transcript replaces the placeholder-empty text. The cache updates through
|
||||
/// the monitor as usual (via `updateNote`).
|
||||
func applyTranscription(id: ULID, text: String, localeID: String) async throws {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
|
||||
note.isReadable, var audio = note.payload.audio else { return }
|
||||
|
||||
let userEdited = !note.payload.text.isEmpty && note.meta.modifiedAt > note.meta.createdAt
|
||||
note.payload.text = userEdited ? note.payload.text + "\n\n" + text : text
|
||||
audio.transcriptionStatus = .done
|
||||
audio.transcriptionLocaleID = localeID
|
||||
audio.transcribedAt = Date()
|
||||
note.payload.audio = audio
|
||||
try await updateNote(note)
|
||||
}
|
||||
|
||||
/// Flag a note's transcription as failed (retryable only by an explicit
|
||||
/// re-transcribe). Re-reads first so it can't resurrect a since-deleted or
|
||||
/// non-audio note.
|
||||
func markTranscriptionFailed(id: ULID) async throws {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
|
||||
note.isReadable, var audio = note.payload.audio else { return }
|
||||
audio.transcriptionStatus = .failed
|
||||
note.payload.audio = audio
|
||||
try await updateNote(note)
|
||||
}
|
||||
|
||||
/// Reassign a note's transcription to *this* device and reset it to
|
||||
/// `.pending`, optionally forcing a language (`localeID`, else auto-detect).
|
||||
/// Claiming `transcriberDeviceID` is also the recovery path when the original
|
||||
/// recording device is gone — any device can take a stuck note over.
|
||||
func requestRetranscription(id: ULID, localeID: String?) async throws {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
|
||||
note.isReadable, var audio = note.payload.audio else { return }
|
||||
audio.transcriptionStatus = .pending
|
||||
audio.transcriberDeviceID = InstallIdentity.installID
|
||||
audio.transcriptionLocaleID = localeID
|
||||
note.payload.audio = audio
|
||||
try await updateNote(note)
|
||||
}
|
||||
|
||||
// MARK: - Watch ingestion
|
||||
|
||||
/// Local directory where the phone receiver (`PhoneWatchReceiver`, a later
|
||||
/// stage) stages recordings handed over from the watch before they are
|
||||
/// ingested. It sits outside the iCloud container on purpose: the watch has
|
||||
/// no iCloud, so a delivered recording becomes durable only once
|
||||
/// `ingestWatchRecording` writes it into the container. Each staged
|
||||
/// recording is a `<ULID>.m4a` plus a `<ULID>.meta.plist` sidecar carrying
|
||||
/// `id` / `recordedAt` / `duration` / `timeZoneID`.
|
||||
/// `nonisolated` so the phone receiver can stage a delivery synchronously in
|
||||
/// its nonisolated `WCSessionDelegate` callback (the system reclaims the inbox
|
||||
/// file the moment that returns). Pure — no actor state — so it is safe to
|
||||
/// share this one source of truth for the inbox path with the drain.
|
||||
nonisolated static var watchInboxURL: URL {
|
||||
URL.applicationSupportDirectory.appending(path: "WatchInbox", directoryHint: .isDirectory)
|
||||
}
|
||||
|
||||
/// Idempotent drain of the watch staging inbox. Called after `connect()`, on
|
||||
/// foreground reconcile, and (a later stage) on the receiver callback, so a
|
||||
/// recording delivered while the app was dead still lands. Serialized against
|
||||
/// itself so overlapping triggers can't double-ingest.
|
||||
func ingestStagedWatchRecordings() async {
|
||||
guard store != nil, !isDrainingInbox else { return }
|
||||
isDrainingInbox = true
|
||||
defer { isDrainingInbox = false }
|
||||
|
||||
for staged in Self.stagedWatchRecordings() {
|
||||
await ingestWatchRecording(
|
||||
audioURL: staged.audioURL,
|
||||
id: staged.id,
|
||||
recordedAt: staged.recordedAt,
|
||||
duration: staged.duration,
|
||||
context: staged.context
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Ingest one recording handed over from the watch — the single sanctioned
|
||||
/// direct-cache-upsert path (icloud-sync-engine §9). The phone is where the
|
||||
/// note first becomes durable, so: veto if the note was already deleted
|
||||
/// (tombstone) or already ingested (duplicate redelivery), then write audio
|
||||
/// **before** JSON (same ordering invariant as `createAudioNote`), upsert the
|
||||
/// cache directly (a WatchConnectivity delivery has no other live trigger to
|
||||
/// refresh an open list), remove the staged files, and fire
|
||||
/// `onAudioNoteIngested` so transcription can enqueue. Returns whether a new
|
||||
/// note was created.
|
||||
@discardableResult
|
||||
func ingestWatchRecording(
|
||||
audioURL: URL,
|
||||
id: ULID,
|
||||
recordedAt: Date,
|
||||
duration: TimeInterval,
|
||||
context: CaptureContext
|
||||
) async -> Bool {
|
||||
guard let store, let tombstones else { return false }
|
||||
|
||||
// Permanent skips — clear the stale staged copy so it stops re-draining.
|
||||
if await tombstones.stubExists(id: id.stringValue) {
|
||||
Self.removeStagedFiles(audioURL: audioURL)
|
||||
return false
|
||||
}
|
||||
let jsonPath = Note.relativePath(forID: id)
|
||||
if await store.fileExists(Self.dataPath(jsonPath)) {
|
||||
Self.removeStagedFiles(audioURL: audioURL)
|
||||
return false
|
||||
}
|
||||
|
||||
let note = Note.createIngested(
|
||||
id: id,
|
||||
duration: duration,
|
||||
transcriberDeviceID: InstallIdentity.installID,
|
||||
context: context
|
||||
)
|
||||
do {
|
||||
let audioData = try Data(contentsOf: audioURL)
|
||||
try await store.writeData(audioData, to: Self.dataPath(Note.audioRelativePath(forID: id)))
|
||||
try await store.write(note, to: Self.dataPath(jsonPath))
|
||||
} catch {
|
||||
// Transient (iCloud write, or the staged file vanished mid-drain):
|
||||
// leave the staged files for the next drain to retry.
|
||||
report("Failed to ingest watch recording \(id.stringValue)", error)
|
||||
return false
|
||||
}
|
||||
|
||||
NoteMapper.upsert(note, relativePath: jsonPath, into: cacheContext)
|
||||
try? cacheContext.save()
|
||||
Self.removeStagedFiles(audioURL: audioURL)
|
||||
lastSyncError = nil
|
||||
|
||||
print("[Sync] Ingested watch recording \(id.stringValue) recorded \(recordedAt)")
|
||||
onAudioNoteIngested?(id)
|
||||
return true
|
||||
}
|
||||
|
||||
/// One recording staged in the watch inbox: the `.m4a` and the metadata the
|
||||
/// receiver wrote in its `<ULID>.meta.plist` sidecar.
|
||||
private struct StagedRecording {
|
||||
var id: ULID
|
||||
var recordedAt: Date
|
||||
var duration: TimeInterval
|
||||
var context: CaptureContext
|
||||
var audioURL: URL
|
||||
}
|
||||
|
||||
/// Enumerate the staging inbox, pairing each `<ULID>.meta.plist` with its
|
||||
/// `.m4a`. A sidecar whose id doesn't parse, whose required fields are
|
||||
/// missing, or whose audio is absent is skipped (a half-delivered pair can't
|
||||
/// be ingested; the next full delivery re-stages it).
|
||||
private static func stagedWatchRecordings() -> [StagedRecording] {
|
||||
let fm = FileManager.default
|
||||
guard let entries = try? fm.contentsOfDirectory(
|
||||
at: watchInboxURL, includingPropertiesForKeys: nil) else { return [] }
|
||||
|
||||
var staged: [StagedRecording] = []
|
||||
for metaURL in entries where metaURL.pathExtension == "plist" {
|
||||
// `<ULID>.meta.plist` → drop `.plist` then `.meta` to the ULID base.
|
||||
let base = metaURL.deletingPathExtension().deletingPathExtension().lastPathComponent
|
||||
guard let id = ULID(string: base),
|
||||
let data = try? Data(contentsOf: metaURL),
|
||||
let dict = try? PropertyListSerialization.propertyList(
|
||||
from: data, options: [], format: nil) as? [String: Any],
|
||||
let duration = dict["duration"] as? Double,
|
||||
let recordedAt = dict["recordedAt"] as? Date else { continue }
|
||||
|
||||
let audioURL = watchInboxURL.appending(path: "\(base).m4a")
|
||||
guard fm.fileExists(atPath: audioURL.path) else { continue }
|
||||
|
||||
var context = CaptureContext.empty
|
||||
context.device = "Watch"
|
||||
context.timeZoneID = dict["timeZoneID"] as? String
|
||||
staged.append(StagedRecording(
|
||||
id: id, recordedAt: recordedAt, duration: duration,
|
||||
context: context, audioURL: audioURL))
|
||||
}
|
||||
return staged
|
||||
}
|
||||
|
||||
/// Remove a staged recording's `.m4a` and its `<ULID>.meta.plist` sibling.
|
||||
private static func removeStagedFiles(audioURL: URL) {
|
||||
let fm = FileManager.default
|
||||
try? fm.removeItem(at: audioURL)
|
||||
try? fm.removeItem(at: audioURL.deletingPathExtension().appendingPathExtension("meta.plist"))
|
||||
}
|
||||
|
||||
// MARK: - Persistence plumbing
|
||||
|
||||
/// Writes touch the file layer ONLY. iCloud JSON is the source of truth:
|
||||
@@ -263,6 +546,7 @@ final class SyncEngine {
|
||||
let id = DocumentFileStore.documentID(fromRelativePath: path)
|
||||
if await tombstones.stubExists(id: id) {
|
||||
try? await store.remove(at: Self.dataPath(path))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
return
|
||||
}
|
||||
await importRecord(at: path, using: store)
|
||||
@@ -275,6 +559,7 @@ final class SyncEngine {
|
||||
private func handleStubAdded(relativePath: String) async {
|
||||
if let store {
|
||||
try? await store.remove(at: Self.dataPath(relativePath))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: relativePath)))
|
||||
}
|
||||
removeFromCache(jsonRelativePath: relativePath)
|
||||
}
|
||||
@@ -344,9 +629,11 @@ final class SyncEngine {
|
||||
isSyncing = true
|
||||
defer { isSyncing = false }
|
||||
|
||||
// Clean up files a stub has tombstoned (mirrors the rebuild path).
|
||||
// Clean up files a stub has tombstoned (mirrors the rebuild path),
|
||||
// including each record's audio sidecar sibling.
|
||||
for path in orphanedPaths {
|
||||
try? await store.remove(at: Self.dataPath(path))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
}
|
||||
|
||||
removeFromCache(ids: toRemove)
|
||||
@@ -382,6 +669,7 @@ final class SyncEngine {
|
||||
for path in await listRecordPaths(in: store) {
|
||||
if stubIDs.contains(DocumentFileStore.documentID(fromRelativePath: path)) {
|
||||
try? await store.remove(at: Self.dataPath(path))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
continue
|
||||
}
|
||||
await importRecord(at: path, using: store)
|
||||
|
||||
Reference in New Issue
Block a user