Decouple transcripts from note text; end-to-end watch transfer acks

Schema v3: transcription attempts accumulate on AudioInfo (text, locale,
confidence, forced flag) — latest attempt is displayed, payload.text is
purely user writing, v2 notes migrate at read time. Editor shows the
transcript in its own live-updating section with insert-into-note; search
matches transcripts weighted by confidence. Watch recordings are now
deleted only on the phone's durable-ingest ack (transferUserInfo), closing
every phone-side loss window; failed sidecar writes unstage instead of
orphaning, and unreadable inbox recordings are surfaced in logs.

Claude-Session: https://claude.ai/code/session_014esDWi42URLEC6Cj17hGQ3
This commit is contained in:
2026-07-16 12:29:49 -04:00
parent c29d25748a
commit c5c90e9441
15 changed files with 546 additions and 55 deletions
+22 -2
View File
@@ -37,6 +37,15 @@ final class PhoneWatchReceiver: NSObject {
session.activate()
self.session = session
}
/// Tell the watch a recording reached a terminal state here, so it can
/// delete its local copy. `transferUserInfo` queues durably the ack
/// arrives even if the watch app is closed right now. Wired to
/// `SyncEngine.onWatchRecordingHandled` by `NotesApp`.
func acknowledgeIngest(id: String) {
guard let session, session.activationState == .activated else { return }
session.transferUserInfo(IngestAck(id: id).userInfo())
}
}
// MARK: - WCSessionDelegate
@@ -95,6 +104,9 @@ extension PhoneWatchReceiver: WCSessionDelegate {
do {
try fm.moveItem(at: file.fileURL, to: audioDest)
} catch {
// Bytes lost on this delivery safe because the watch keeps its
// copy until the ingest ack, which this recording never gets; the
// next watch reconcile redelivers it.
log.error("failed to stage recording \(base, privacy: .public): \(error, privacy: .public)")
return
}
@@ -105,8 +117,16 @@ extension PhoneWatchReceiver: WCSessionDelegate {
"duration": payload.duration,
]
if let tz = payload.timeZoneID { meta["timeZoneID"] = tz }
if let data = try? PropertyListSerialization.data(fromPropertyList: meta, format: .binary, options: 0) {
try? data.write(to: metaDest)
do {
let data = try PropertyListSerialization.data(fromPropertyList: meta, format: .binary, options: 0)
try data.write(to: metaDest)
} catch {
// A sidecar-less m4a would sit in the inbox forever (the drain
// can't ingest it). Remove the audio too and let the un-acked
// watch copy drive a clean redelivery instead.
try? fm.removeItem(at: audioDest)
log.error("failed to write sidecar for \(base, privacy: .public), unstaging: \(error, privacy: .public)")
return
}
log.info("staged watch recording \(base, privacy: .public) for ingest")
}
+73 -6
View File
@@ -40,10 +40,13 @@ struct CaptureContext: Codable, Sendable, Equatable {
/// One note = one JSON file in the iCloud container, filed under
/// `Records/YYYY/MM/<ULID>.json` (UTC-bucketed by the ULID's timestamp).
struct Note: SyncDocument, VersionedDocument, Equatable {
/// v2 added `Payload.audio` for voice notes. Because it is optional, a v1
/// file decodes unchanged (missing key nil) and `migrateIfNeeded` only
/// bumps the stamped version no field back-fill is needed.
static let currentSchemaVersion = 2
/// v2 added `Payload.audio` for voice notes. v3 moved transcripts out of
/// `payload.text` into `AudioInfo.attempts`: text is now purely the user's
/// own writing, and every transcription attempt is preserved. v1/v2 files
/// decode unchanged (missing keys nil/empty); `migrateIfNeeded` back-fills
/// v2 voice notes by moving a completed transcript from `text` into
/// `attempts` at read time.
static let currentSchemaVersion = 3
struct Meta: Codable, Sendable, Equatable {
var id: ULID
@@ -66,6 +69,24 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
case failed
}
/// One finished transcription of a voice note. Attempts accumulate a
/// re-transcription appends rather than overwrites, so no machine output is
/// ever lost and the user can be shown alternatives.
struct TranscriptionAttempt: Codable, Sendable, Equatable {
var text: String
/// BCP-47 locale the transcript was produced in.
var localeID: String
/// Character-weighted mean confidence 01, or nil when the engine
/// emitted none (`DictationTranscriber` locales need not report it).
/// Confidences are not comparable across locales/engines display and
/// search-weighting metadata, never a cross-attempt selector.
var confidence: Double?
var transcribedAt: Date
/// Whether the user forced this language (re-transcribe menu) rather
/// than auto-detection picking it.
var languageWasForced: Bool
}
/// The audio-note sidecar metadata. Present only on voice notes; the audio
/// bytes themselves live in a `.m4a` next to the note JSON (path derived,
/// never stored see `audioRelativePath`).
@@ -79,6 +100,45 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
/// BCP-47 locale the transcript was produced in; nil while pending.
var transcriptionLocaleID: String?
var transcribedAt: Date?
/// Every finished transcription, oldest first. The **latest** attempt is
/// the displayed one a forced re-transcribe is a statement of intent,
/// so recency wins, not confidence (which isn't cross-locale comparable).
var attempts: [TranscriptionAttempt] = []
/// The attempt currently shown for this note.
var latestAttempt: TranscriptionAttempt? { attempts.last }
private enum CodingKeys: String, CodingKey {
case duration, transcriberDeviceID, transcriptionStatus,
transcriptionLocaleID, transcribedAt, attempts
}
init(
duration: TimeInterval,
transcriberDeviceID: String,
transcriptionStatus: TranscriptionStatus,
transcriptionLocaleID: String? = nil,
transcribedAt: Date? = nil,
attempts: [TranscriptionAttempt] = []
) {
self.duration = duration
self.transcriberDeviceID = transcriberDeviceID
self.transcriptionStatus = transcriptionStatus
self.transcriptionLocaleID = transcriptionLocaleID
self.transcribedAt = transcribedAt
self.attempts = attempts
}
init(from decoder: any Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
duration = try container.decode(TimeInterval.self, forKey: .duration)
transcriberDeviceID = try container.decode(String.self, forKey: .transcriberDeviceID)
transcriptionStatus = try container.decode(TranscriptionStatus.self, forKey: .transcriptionStatus)
transcriptionLocaleID = try container.decodeIfPresent(String.self, forKey: .transcriptionLocaleID)
transcribedAt = try container.decodeIfPresent(Date.self, forKey: .transcribedAt)
// Absent on v2 files decode to empty, back-filled by migration.
attempts = try container.decodeIfPresent([TranscriptionAttempt].self, forKey: .attempts) ?? []
}
}
struct Payload: Codable, Sendable, Equatable {
@@ -167,9 +227,16 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
)
}
/// First line of the text, used as the display title.
/// What a reader should see as this note's body: the user's own text when
/// they wrote any, else the latest transcript. Transcripts live in
/// `audio.attempts`, never in `payload.text`.
var displayText: String {
payload.text.isEmpty ? (payload.audio?.latestAttempt?.text ?? "") : payload.text
}
/// First line of the display text, used as the display title.
var title: String {
payload.text
displayText
.split(separator: "\n", omittingEmptySubsequences: true)
.first.map(String.init) ?? ""
}
+30 -6
View File
@@ -40,6 +40,10 @@ final class NoteEntity {
var transcriptionStatusRaw: String?
var transcriptionLocaleID: String?
var transcriberDeviceID: String?
// Latest transcription attempt, cached flat for list display and search.
// The full attempt history lives only in the file (like `transcribedAt`).
var transcriptText: String?
var transcriptConfidence: Double?
init(id: String, jsonRelativePath: String) {
self.id = id
@@ -82,19 +86,31 @@ extension NoteEntity {
}
/// Reconstructs `AudioInfo` from the flattened columns (nil for a text
/// note). `transcribedAt` is not cached, so it comes back nil even for a
/// completed transcription acceptable for the cache-only offline fallback,
/// where the file (which carries it) simply wasn't reachable.
/// note). `transcribedAt` and the attempt history are not fully cached
/// only the latest attempt survives (with a placeholder date), acceptable
/// for the cache-only offline fallback, where the file (which carries the
/// full history) simply wasn't reachable.
var audio: Note.AudioInfo? {
get {
guard hasAudio else { return nil }
var attempts: [Note.TranscriptionAttempt] = []
if let transcriptText {
attempts = [Note.TranscriptionAttempt(
text: transcriptText,
localeID: transcriptionLocaleID ?? "",
confidence: transcriptConfidence,
transcribedAt: modifiedAt,
languageWasForced: false
)]
}
return Note.AudioInfo(
duration: audioDuration ?? 0,
transcriberDeviceID: transcriberDeviceID ?? "",
transcriptionStatus: transcriptionStatusRaw
.flatMap(Note.TranscriptionStatus.init(rawValue:)) ?? .pending,
transcriptionLocaleID: transcriptionLocaleID,
transcribedAt: nil
transcribedAt: nil,
attempts: attempts
)
}
set {
@@ -103,12 +119,20 @@ extension NoteEntity {
transcriberDeviceID = newValue?.transcriberDeviceID
transcriptionStatusRaw = newValue?.transcriptionStatus.rawValue
transcriptionLocaleID = newValue?.transcriptionLocaleID
transcriptText = newValue?.latestAttempt?.text
transcriptConfidence = newValue?.latestAttempt?.confidence
}
}
/// First line of the text, used as the display title.
/// What a reader should see as this note's body: the user's own text when
/// they wrote any, else the latest transcript.
var displayText: String {
text.isEmpty ? (transcriptText ?? "") : text
}
/// First line of the display text, used as the display title.
var title: String {
text.split(separator: "\n", omittingEmptySubsequences: true)
displayText.split(separator: "\n", omittingEmptySubsequences: true)
.first.map(String.init) ?? ""
}
}
+7 -2
View File
@@ -33,8 +33,13 @@ struct NotesApp: App {
appServices = services
#if os(iOS)
// Hand the (already-activated) watch receiver the live sync
// engine so a delivery arriving now drains immediately.
appDelegate.watchReceiver.syncEngine = services.syncEngine
// engine so a delivery arriving now drains immediately, and
// route ingest acks back so the watch can free its copies.
let receiver = appDelegate.watchReceiver
receiver.syncEngine = services.syncEngine
services.syncEngine.onWatchRecordingHandled = { [weak receiver] id in
receiver?.acknowledgeIngest(id: id)
}
#endif
await services.startDeferredServices()
if let pending = pendingBackupURL {
+6 -1
View File
@@ -117,7 +117,12 @@ final class TranscriptionService {
try await syncEngine.applyTranscription(
id: job.noteID,
text: text,
localeID: TranscriptionSettings.canonicalID(locale)
localeID: TranscriptionSettings.canonicalID(locale),
// Real mean confidence when the engine attributed any runs; nil for
// a confidence-less engine so it's never mistaken for a measured 0.
confidence: LanguagePicker.hasConfidence(of: transcript)
? LanguagePicker.meanConfidence(of: transcript) : nil,
languageWasForced: job.forcedLocaleID != nil
)
}
+82 -18
View File
@@ -43,6 +43,13 @@ final class SyncEngine {
/// transcription stack exists.
var onAudioNoteIngested: ((ULID) -> Void)?
/// Fired when a staged watch recording reaches a **terminal** state on this
/// device durably ingested, or permanently vetoed (tombstoned / already
/// present). The composition root wires this to send the ingest ack that
/// lets the watch delete its local copy; the watch keeps (and redelivers)
/// anything never acked, so transient failures stay silent here on purpose.
var onWatchRecordingHandled: ((String) -> Void)?
static let recordsDirectory = "Records"
static let stubsDirectory = "Stubs"
@@ -255,22 +262,35 @@ final class SyncEngine {
// MARK: - Transcription
/// Land a completed transcription onto a note. Re-reads the file (the
/// authority) so a concurrent edit isn't clobbered, then **appends** rather
/// than replaces when the user typed while the note was pending non-empty
/// text whose `modifiedAt` moved past the note's `createdAt`. Otherwise the
/// transcript replaces the placeholder-empty text. The cache updates through
/// the monitor as usual (via `updateNote`).
func applyTranscription(id: ULID, text: String, localeID: String) async throws {
/// Land a completed transcription onto a note as a new attempt. Re-reads
/// the file (the authority) so a concurrent edit isn't clobbered.
/// `payload.text` the user's own writing is never touched: transcripts
/// accumulate in `audio.attempts` and the latest one is what the UI shows.
/// The cache updates through the monitor as usual (via `updateNote`).
func applyTranscription(
id: ULID,
text: String,
localeID: String,
confidence: Double? = nil,
languageWasForced: Bool = false
) async throws {
guard let store else { throw SyncError.iCloudUnavailable }
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
note.isReadable, var audio = note.payload.audio else { return }
guard let raw = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
raw.isReadable else { return }
var note = migrateIfNeeded(raw)
guard var audio = note.payload.audio else { return }
let userEdited = !note.payload.text.isEmpty && note.meta.modifiedAt > note.meta.createdAt
note.payload.text = userEdited ? note.payload.text + "\n\n" + text : text
let now = Date()
audio.attempts.append(Note.TranscriptionAttempt(
text: text,
localeID: localeID,
confidence: confidence,
transcribedAt: now,
languageWasForced: languageWasForced
))
audio.transcriptionStatus = .done
audio.transcriptionLocaleID = localeID
audio.transcribedAt = Date()
audio.transcribedAt = now
note.payload.audio = audio
try await updateNote(note)
}
@@ -337,6 +357,11 @@ final class SyncEngine {
context: staged.context
)
}
let orphans = Self.watchInboxOrphanCount()
if orphans > 0 {
print("[Sync] Watch inbox holds \(orphans) recording(s) without readable metadata — not ingestible by this build")
}
}
/// Ingest one recording handed over from the watch the single sanctioned
@@ -358,14 +383,17 @@ final class SyncEngine {
) async -> Bool {
guard let store, let tombstones else { return false }
// Permanent skips clear the stale staged copy so it stops re-draining.
// Permanent skips clear the stale staged copy so it stops re-draining,
// and ack so the watch drops its copy too (redelivery would just loop).
if await tombstones.stubExists(id: id.stringValue) {
Self.removeStagedFiles(audioURL: audioURL)
onWatchRecordingHandled?(id.stringValue)
return false
}
let jsonPath = Note.relativePath(forID: id)
if await store.fileExists(Self.dataPath(jsonPath)) {
Self.removeStagedFiles(audioURL: audioURL)
onWatchRecordingHandled?(id.stringValue)
return false
}
@@ -392,10 +420,24 @@ final class SyncEngine {
lastSyncError = nil
print("[Sync] Ingested watch recording \(id.stringValue) recorded \(recordedAt)")
onWatchRecordingHandled?(id.stringValue)
onAudioNoteIngested?(id)
return true
}
/// `.m4a` files sitting in the watch inbox with no readable metadata sidecar
/// quarantined forward-gate deliveries or the residue of a failed staging.
/// The drain can never ingest them; surfaced so they're at least visible.
nonisolated static func watchInboxOrphanCount() -> Int {
let fm = FileManager.default
let entries = (try? fm.contentsOfDirectory(at: watchInboxURL, includingPropertiesForKeys: nil)) ?? []
let sidecarBases = Set(entries.filter { $0.lastPathComponent.hasSuffix(".meta.plist") }
.map { $0.lastPathComponent.replacingOccurrences(of: ".meta.plist", with: "") })
return entries.filter {
$0.pathExtension == "m4a" && !sidecarBases.contains($0.deletingPathExtension().lastPathComponent)
}.count
}
/// One recording staged in the watch inbox: the `.m4a` and the metadata the
/// receiver wrote in its `<ULID>.meta.plist` sidecar.
private struct StagedRecording {
@@ -711,13 +753,35 @@ final class SyncEngine {
// MARK: - Schema migration
/// Read-time upgrade for documents behind the current schema. A no-op
/// while `currentSchemaVersion` is 1, but wired into both read paths so
/// the first schema bump migrates instead of silently skipping. Write-back
/// happens lazily on the next save never from live monitor events.
private func migrateIfNeeded(_ note: Note) -> Note {
/// Read-time upgrade for documents behind the current schema. Wired into
/// both read paths so a schema bump migrates instead of silently skipping.
/// Write-back happens lazily on the next save never from live monitor
/// events.
///
/// v2 v3: transcripts moved from `payload.text` into `audio.attempts`.
/// A v2 voice note whose transcription completed holds its transcript (or
/// the transcript merged with user typing indistinguishable by then) in
/// `text`; move that whole text into the first attempt so the v3 invariant
/// "text is user writing only" holds. Confidence was never recorded nil.
/// Internal (not private) so the migration policy is unit-testable.
func migrateIfNeeded(_ note: Note) -> Note {
guard note.meta.schemaVersion < Note.currentSchemaVersion else { return note }
var migrated = note
if note.meta.schemaVersion < 3,
var audio = migrated.payload.audio,
audio.transcriptionStatus == .done,
audio.attempts.isEmpty,
!migrated.payload.text.isEmpty {
audio.attempts = [Note.TranscriptionAttempt(
text: migrated.payload.text,
localeID: audio.transcriptionLocaleID ?? "",
confidence: nil,
transcribedAt: audio.transcribedAt ?? migrated.meta.modifiedAt,
languageWasForced: false
)]
migrated.payload.text = ""
migrated.payload.audio = audio
}
migrated.meta.schemaVersion = Note.currentSchemaVersion
return migrated
}
+33
View File
@@ -44,6 +44,10 @@ struct NoteEditorView: View {
@Environment(\.dismiss) private var dismiss
@State private var text = ""
/// The note text as last loaded from the entity, so an external change
/// (a transcription landing while this editor is open) can be told apart
/// from the user's own typing: refresh only while `text == loadedText`.
@State private var loadedText = ""
@State private var tagsText = ""
@State private var saveError: String?
@State private var recordedAudio: AudioRecorderService.Recording?
@@ -115,9 +119,18 @@ struct NoteEditorView: View {
await contextService.refresh()
case .edit(let note):
text = note.text
loadedText = note.text
tagsText = note.tags.joined(separator: ", ")
}
}
// A transcription finishing while this editor is open updates the
// entity underneath us; mirror it into the text field as long as the
// user hasn't typed (their edits always win over a live refresh).
.onChange(of: editedNote?.text) { _, newValue in
guard let newValue, text == loadedText else { return }
text = newValue
loadedText = newValue
}
#if os(macOS)
.frame(minWidth: 480, minHeight: 420)
#endif
@@ -190,6 +203,21 @@ struct NoteEditorView: View {
transcriptionStatusRow(for: note)
retranscribeMenu(for: note)
}
// The transcript is machine output, kept apart from the user's own
// text: shown read-only here (it updates live as attempts land) with
// a one-tap copy into the editable note body.
if let transcript = note.transcriptText, !transcript.isEmpty {
Section("Transcript") {
Text(transcript)
.textSelection(.enabled)
.foregroundStyle(.secondary)
Button {
text = text.isEmpty ? transcript : text + "\n\n" + transcript
} label: {
Label("Insert into Note", systemImage: "text.insert")
}
}
}
}
@ViewBuilder
@@ -263,6 +291,11 @@ struct NoteEditorView: View {
return false
}
private var editedNote: NoteEntity? {
if case .edit(let note) = mode { return note }
return nil
}
private var canSave: Bool {
if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return true }
if recordedAudio != nil { return true }
+24 -7
View File
@@ -56,7 +56,12 @@ struct SearchView: View {
private var results: [NoteEntity] {
guard !searchText.isEmpty else { return [] }
return notes.filter { $0.matches(search: searchText) }
return notes
.compactMap { note in note.searchScore(for: searchText).map { (note, $0) } }
.sorted { lhs, rhs in
lhs.1 != rhs.1 ? lhs.1 > rhs.1 : lhs.0.createdAt > rhs.0.createdAt
}
.map(\.0)
}
private func delete(_ note: NoteEntity) {
@@ -68,12 +73,24 @@ struct SearchView: View {
}
extension NoteEntity {
/// Case-insensitive match against text, tags, and capture location.
func matches(search: String) -> Bool {
/// Case-insensitive relevance of this note to a search, or nil when nothing
/// matches. The score is the strongest matching field: the user's own text
/// and tags outrank transcript matches, and a transcript match is weighted
/// by the engine's confidence in it a shaky transcription can still be
/// found, it just ranks below solid matches. Confidence-less engines get a
/// mid-scale weight rather than sinking to the bottom.
func searchScore(for search: String) -> Double? {
let needle = search.lowercased()
return text.lowercased().contains(needle)
|| tags.contains { $0.lowercased().contains(needle) }
|| (placeName?.lowercased().contains(needle) ?? false)
|| (locality?.lowercased().contains(needle) ?? false)
var score = 0.0
if text.lowercased().contains(needle) { score = max(score, 1.0) }
if tags.contains(where: { $0.lowercased().contains(needle) }) { score = max(score, 0.9) }
if let transcript = transcriptText, transcript.lowercased().contains(needle) {
score = max(score, 0.5 + 0.4 * (transcriptConfidence ?? 0.5))
}
if placeName?.lowercased().contains(needle) == true
|| locality?.lowercased().contains(needle) == true {
score = max(score, 0.4)
}
return score > 0 ? score : nil
}
}