diff --git a/CHANGELOG.md b/CHANGELOG.md index a5ecb3d..645006f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ **July 2026** +Transcripts now appear in their own section of a voice note, separate from your own text, with a button to insert them into it + +Re-transcribing a note in another language now works, and every previous transcript is kept + +Search now also finds words inside voice note transcripts + +Watch recordings are now kept on the watch until the iPhone confirms they're safely stored, so a hiccup during handoff can no longer lose them + Fixed watch voice notes arriving on the iPhone without audio, so they couldn't be played or transcribed Transcription now supports many more languages, including Russian, Ukrainian, and Polish diff --git a/Notes Watch App/Connectivity/WatchTransferService.swift b/Notes Watch App/Connectivity/WatchTransferService.swift index 5bb79e2..9f2e8b4 100644 --- a/Notes Watch App/Connectivity/WatchTransferService.swift +++ b/Notes Watch App/Connectivity/WatchTransferService.swift @@ -8,11 +8,14 @@ import WatchConnectivity /// /// Each queued recording is a `.m4a` plus a `.meta.plist` sidecar in /// `Documents/Recordings/`. `transferFile` hands the audio to WatchConnectivity -/// with an `AudioNotePayload` as metadata. Local files are deleted **only** in -/// `didFinish` with a nil error — so a delivery we never confirm (app killed -/// mid-flight, transfer failed) is re-enqueued by `reconcileQueue()` from the -/// persisted sidecar on the next activation. `pendingCount` (files still on disk) -/// drives the "N waiting for iPhone" footer. +/// with an `AudioNotePayload` as metadata. Local files are deleted **only** when +/// the phone sends an `IngestAck` — the end-to-end confirmation that the +/// recording is durably in the iCloud container (or permanently vetoed there). +/// WC-level `didFinish` success is not enough: the phone can still fail after +/// delivery (staging, iCloud write), and this local copy is the only recovery. +/// Anything never acked is re-enqueued by `reconcileQueue()` from the persisted +/// sidecar on the next activation/foreground; the phone dedupes and re-acks. +/// `pendingCount` (files still on disk) drives the "N waiting for iPhone" footer. /// /// Strict concurrency: the delegate callbacks are nonisolated and run on /// WatchConnectivity's background queue — they extract Sendable values (a file @@ -30,6 +33,13 @@ final class WatchTransferService: NSObject { private var session: WCSession? + /// Recordings WC reported delivered this session but the phone hasn't yet + /// acked as ingested. Skipped by `reconcileQueue` so a foreground pass right + /// after delivery doesn't re-send a whole audio file the phone is busy + /// ingesting. In-memory only: after a relaunch these fall back into the + /// reconcile pool, and a redundant redelivery is harmless (phone dedupes). + private var deliveredAwaitingAck: Set = [] + func activate() { guard WCSession.isSupported() else { return } let session = WCSession.default @@ -62,6 +72,7 @@ final class WatchTransferService: NSObject { let entries = (try? fm.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)) ?? [] for m4a in entries where m4a.pathExtension == "m4a" { if inFlight.contains(m4a.lastPathComponent) { continue } + if deliveredAwaitingAck.contains(m4a.lastPathComponent) { continue } let base = m4a.deletingPathExtension().lastPathComponent guard ULID(string: base) != nil else { continue } let meta = readSidecar(base: base) @@ -93,13 +104,20 @@ final class WatchTransferService: NSObject { session.transferFile(url, metadata: payload.metadata()) } - /// Delete a delivered recording and its sidecar (called only on a confirmed - /// `didFinish` with no error). + /// Delete an **acked** recording and its sidecar — called only when the + /// phone's `IngestAck` arrives, never on mere WC delivery success. private func completeTransfer(fileName: String) { let dir = WatchAudioRecorder.recordingsDirectory let base = (fileName as NSString).deletingPathExtension try? FileManager.default.removeItem(at: dir.appending(path: fileName)) try? FileManager.default.removeItem(at: sidecarURL(base: base)) + deliveredAwaitingAck.remove(fileName) + refreshPendingCount() + } + + /// WC delivered the file to the phone; park it until the ingest ack. + private func markDelivered(fileName: String) { + deliveredAwaitingAck.insert(fileName) refreshPendingCount() } @@ -160,6 +178,14 @@ extension WatchTransferService: WCSessionDelegate { Task { @MainActor in self.refreshPendingCount() } return } - Task { @MainActor in self.completeTransfer(fileName: fileName) } + // Delivered ≠ ingested: keep the local copy until the phone's ack. + Task { @MainActor in self.markDelivered(fileName: fileName) } + } + + nonisolated func session(_ session: WCSession, didReceiveUserInfo userInfo: [String: Any] = [:]) { + // The phone's ingest ack — the recording is durable in iCloud (or + // permanently vetoed); the local copy has done its job. + guard let ack = IngestAck(userInfo: userInfo) else { return } + Task { @MainActor in self.completeTransfer(fileName: "\(ack.id).m4a") } } } diff --git a/Notes/Connectivity/PhoneWatchReceiver.swift b/Notes/Connectivity/PhoneWatchReceiver.swift index bcd0fed..931c491 100644 --- a/Notes/Connectivity/PhoneWatchReceiver.swift +++ b/Notes/Connectivity/PhoneWatchReceiver.swift @@ -37,6 +37,15 @@ final class PhoneWatchReceiver: NSObject { session.activate() self.session = session } + + /// Tell the watch a recording reached a terminal state here, so it can + /// delete its local copy. `transferUserInfo` queues durably — the ack + /// arrives even if the watch app is closed right now. Wired to + /// `SyncEngine.onWatchRecordingHandled` by `NotesApp`. + func acknowledgeIngest(id: String) { + guard let session, session.activationState == .activated else { return } + session.transferUserInfo(IngestAck(id: id).userInfo()) + } } // MARK: - WCSessionDelegate @@ -95,6 +104,9 @@ extension PhoneWatchReceiver: WCSessionDelegate { do { try fm.moveItem(at: file.fileURL, to: audioDest) } catch { + // Bytes lost on this delivery — safe because the watch keeps its + // copy until the ingest ack, which this recording never gets; the + // next watch reconcile redelivers it. log.error("failed to stage recording \(base, privacy: .public): \(error, privacy: .public)") return } @@ -105,8 +117,16 @@ extension PhoneWatchReceiver: WCSessionDelegate { "duration": payload.duration, ] if let tz = payload.timeZoneID { meta["timeZoneID"] = tz } - if let data = try? PropertyListSerialization.data(fromPropertyList: meta, format: .binary, options: 0) { - try? data.write(to: metaDest) + do { + let data = try PropertyListSerialization.data(fromPropertyList: meta, format: .binary, options: 0) + try data.write(to: metaDest) + } catch { + // A sidecar-less m4a would sit in the inbox forever (the drain + // can't ingest it). Remove the audio too and let the un-acked + // watch copy drive a clean redelivery instead. + try? fm.removeItem(at: audioDest) + log.error("failed to write sidecar for \(base, privacy: .public), unstaging: \(error, privacy: .public)") + return } log.info("staged watch recording \(base, privacy: .public) for ingest") } diff --git a/Notes/Model/Note.swift b/Notes/Model/Note.swift index f579d23..271568c 100644 --- a/Notes/Model/Note.swift +++ b/Notes/Model/Note.swift @@ -40,10 +40,13 @@ struct CaptureContext: Codable, Sendable, Equatable { /// One note = one JSON file in the iCloud container, filed under /// `Records/YYYY/MM/.json` (UTC-bucketed by the ULID's timestamp). struct Note: SyncDocument, VersionedDocument, Equatable { - /// v2 added `Payload.audio` for voice notes. Because it is optional, a v1 - /// file decodes unchanged (missing key ⇒ nil) and `migrateIfNeeded` only - /// bumps the stamped version — no field back-fill is needed. - static let currentSchemaVersion = 2 + /// v2 added `Payload.audio` for voice notes. v3 moved transcripts out of + /// `payload.text` into `AudioInfo.attempts`: text is now purely the user's + /// own writing, and every transcription attempt is preserved. v1/v2 files + /// decode unchanged (missing keys ⇒ nil/empty); `migrateIfNeeded` back-fills + /// v2 voice notes by moving a completed transcript from `text` into + /// `attempts` at read time. + static let currentSchemaVersion = 3 struct Meta: Codable, Sendable, Equatable { var id: ULID @@ -66,6 +69,24 @@ struct Note: SyncDocument, VersionedDocument, Equatable { case failed } + /// One finished transcription of a voice note. Attempts accumulate — a + /// re-transcription appends rather than overwrites, so no machine output is + /// ever lost and the user can be shown alternatives. + struct TranscriptionAttempt: Codable, Sendable, Equatable { + var text: String + /// BCP-47 locale the transcript was produced in. + var localeID: String + /// Character-weighted mean confidence 0…1, or nil when the engine + /// emitted none (`DictationTranscriber` locales need not report it). + /// Confidences are not comparable across locales/engines — display and + /// search-weighting metadata, never a cross-attempt selector. + var confidence: Double? + var transcribedAt: Date + /// Whether the user forced this language (re-transcribe menu) rather + /// than auto-detection picking it. + var languageWasForced: Bool + } + /// The audio-note sidecar metadata. Present only on voice notes; the audio /// bytes themselves live in a `.m4a` next to the note JSON (path derived, /// never stored — see `audioRelativePath`). @@ -79,6 +100,45 @@ struct Note: SyncDocument, VersionedDocument, Equatable { /// BCP-47 locale the transcript was produced in; nil while pending. var transcriptionLocaleID: String? var transcribedAt: Date? + /// Every finished transcription, oldest first. The **latest** attempt is + /// the displayed one — a forced re-transcribe is a statement of intent, + /// so recency wins, not confidence (which isn't cross-locale comparable). + var attempts: [TranscriptionAttempt] = [] + + /// The attempt currently shown for this note. + var latestAttempt: TranscriptionAttempt? { attempts.last } + + private enum CodingKeys: String, CodingKey { + case duration, transcriberDeviceID, transcriptionStatus, + transcriptionLocaleID, transcribedAt, attempts + } + + init( + duration: TimeInterval, + transcriberDeviceID: String, + transcriptionStatus: TranscriptionStatus, + transcriptionLocaleID: String? = nil, + transcribedAt: Date? = nil, + attempts: [TranscriptionAttempt] = [] + ) { + self.duration = duration + self.transcriberDeviceID = transcriberDeviceID + self.transcriptionStatus = transcriptionStatus + self.transcriptionLocaleID = transcriptionLocaleID + self.transcribedAt = transcribedAt + self.attempts = attempts + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + duration = try container.decode(TimeInterval.self, forKey: .duration) + transcriberDeviceID = try container.decode(String.self, forKey: .transcriberDeviceID) + transcriptionStatus = try container.decode(TranscriptionStatus.self, forKey: .transcriptionStatus) + transcriptionLocaleID = try container.decodeIfPresent(String.self, forKey: .transcriptionLocaleID) + transcribedAt = try container.decodeIfPresent(Date.self, forKey: .transcribedAt) + // Absent on v2 files — decode to empty, back-filled by migration. + attempts = try container.decodeIfPresent([TranscriptionAttempt].self, forKey: .attempts) ?? [] + } } struct Payload: Codable, Sendable, Equatable { @@ -167,9 +227,16 @@ struct Note: SyncDocument, VersionedDocument, Equatable { ) } - /// First line of the text, used as the display title. + /// What a reader should see as this note's body: the user's own text when + /// they wrote any, else the latest transcript. Transcripts live in + /// `audio.attempts`, never in `payload.text`. + var displayText: String { + payload.text.isEmpty ? (payload.audio?.latestAttempt?.text ?? "") : payload.text + } + + /// First line of the display text, used as the display title. var title: String { - payload.text + displayText .split(separator: "\n", omittingEmptySubsequences: true) .first.map(String.init) ?? "" } diff --git a/Notes/Model/NoteEntity.swift b/Notes/Model/NoteEntity.swift index 7de6e13..bd6e57b 100644 --- a/Notes/Model/NoteEntity.swift +++ b/Notes/Model/NoteEntity.swift @@ -40,6 +40,10 @@ final class NoteEntity { var transcriptionStatusRaw: String? var transcriptionLocaleID: String? var transcriberDeviceID: String? + // Latest transcription attempt, cached flat for list display and search. + // The full attempt history lives only in the file (like `transcribedAt`). + var transcriptText: String? + var transcriptConfidence: Double? init(id: String, jsonRelativePath: String) { self.id = id @@ -82,19 +86,31 @@ extension NoteEntity { } /// Reconstructs `AudioInfo` from the flattened columns (nil for a text - /// note). `transcribedAt` is not cached, so it comes back nil even for a - /// completed transcription — acceptable for the cache-only offline fallback, - /// where the file (which carries it) simply wasn't reachable. + /// note). `transcribedAt` and the attempt history are not fully cached — + /// only the latest attempt survives (with a placeholder date), acceptable + /// for the cache-only offline fallback, where the file (which carries the + /// full history) simply wasn't reachable. var audio: Note.AudioInfo? { get { guard hasAudio else { return nil } + var attempts: [Note.TranscriptionAttempt] = [] + if let transcriptText { + attempts = [Note.TranscriptionAttempt( + text: transcriptText, + localeID: transcriptionLocaleID ?? "", + confidence: transcriptConfidence, + transcribedAt: modifiedAt, + languageWasForced: false + )] + } return Note.AudioInfo( duration: audioDuration ?? 0, transcriberDeviceID: transcriberDeviceID ?? "", transcriptionStatus: transcriptionStatusRaw .flatMap(Note.TranscriptionStatus.init(rawValue:)) ?? .pending, transcriptionLocaleID: transcriptionLocaleID, - transcribedAt: nil + transcribedAt: nil, + attempts: attempts ) } set { @@ -103,12 +119,20 @@ extension NoteEntity { transcriberDeviceID = newValue?.transcriberDeviceID transcriptionStatusRaw = newValue?.transcriptionStatus.rawValue transcriptionLocaleID = newValue?.transcriptionLocaleID + transcriptText = newValue?.latestAttempt?.text + transcriptConfidence = newValue?.latestAttempt?.confidence } } - /// First line of the text, used as the display title. + /// What a reader should see as this note's body: the user's own text when + /// they wrote any, else the latest transcript. + var displayText: String { + text.isEmpty ? (transcriptText ?? "") : text + } + + /// First line of the display text, used as the display title. var title: String { - text.split(separator: "\n", omittingEmptySubsequences: true) + displayText.split(separator: "\n", omittingEmptySubsequences: true) .first.map(String.init) ?? "" } } diff --git a/Notes/NotesApp.swift b/Notes/NotesApp.swift index a393fad..0dce945 100644 --- a/Notes/NotesApp.swift +++ b/Notes/NotesApp.swift @@ -33,8 +33,13 @@ struct NotesApp: App { appServices = services #if os(iOS) // Hand the (already-activated) watch receiver the live sync - // engine so a delivery arriving now drains immediately. - appDelegate.watchReceiver.syncEngine = services.syncEngine + // engine so a delivery arriving now drains immediately, and + // route ingest acks back so the watch can free its copies. + let receiver = appDelegate.watchReceiver + receiver.syncEngine = services.syncEngine + services.syncEngine.onWatchRecordingHandled = { [weak receiver] id in + receiver?.acknowledgeIngest(id: id) + } #endif await services.startDeferredServices() if let pending = pendingBackupURL { diff --git a/Notes/Services/TranscriptionService.swift b/Notes/Services/TranscriptionService.swift index 0714bd9..ceba58d 100644 --- a/Notes/Services/TranscriptionService.swift +++ b/Notes/Services/TranscriptionService.swift @@ -117,7 +117,12 @@ final class TranscriptionService { try await syncEngine.applyTranscription( id: job.noteID, text: text, - localeID: TranscriptionSettings.canonicalID(locale) + localeID: TranscriptionSettings.canonicalID(locale), + // Real mean confidence when the engine attributed any runs; nil for + // a confidence-less engine so it's never mistaken for a measured 0. + confidence: LanguagePicker.hasConfidence(of: transcript) + ? LanguagePicker.meanConfidence(of: transcript) : nil, + languageWasForced: job.forcedLocaleID != nil ) } diff --git a/Notes/Storage/SyncEngine.swift b/Notes/Storage/SyncEngine.swift index 7d150ec..b7bfb69 100644 --- a/Notes/Storage/SyncEngine.swift +++ b/Notes/Storage/SyncEngine.swift @@ -43,6 +43,13 @@ final class SyncEngine { /// transcription stack exists. var onAudioNoteIngested: ((ULID) -> Void)? + /// Fired when a staged watch recording reaches a **terminal** state on this + /// device — durably ingested, or permanently vetoed (tombstoned / already + /// present). The composition root wires this to send the ingest ack that + /// lets the watch delete its local copy; the watch keeps (and redelivers) + /// anything never acked, so transient failures stay silent here on purpose. + var onWatchRecordingHandled: ((String) -> Void)? + static let recordsDirectory = "Records" static let stubsDirectory = "Stubs" @@ -255,22 +262,35 @@ final class SyncEngine { // MARK: - Transcription - /// Land a completed transcription onto a note. Re-reads the file (the - /// authority) so a concurrent edit isn't clobbered, then **appends** rather - /// than replaces when the user typed while the note was pending — non-empty - /// text whose `modifiedAt` moved past the note's `createdAt`. Otherwise the - /// transcript replaces the placeholder-empty text. The cache updates through - /// the monitor as usual (via `updateNote`). - func applyTranscription(id: ULID, text: String, localeID: String) async throws { + /// Land a completed transcription onto a note as a new attempt. Re-reads + /// the file (the authority) so a concurrent edit isn't clobbered. + /// `payload.text` — the user's own writing — is never touched: transcripts + /// accumulate in `audio.attempts` and the latest one is what the UI shows. + /// The cache updates through the monitor as usual (via `updateNote`). + func applyTranscription( + id: ULID, + text: String, + localeID: String, + confidence: Double? = nil, + languageWasForced: Bool = false + ) async throws { guard let store else { throw SyncError.iCloudUnavailable } - guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))), - note.isReadable, var audio = note.payload.audio else { return } + guard let raw = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))), + raw.isReadable else { return } + var note = migrateIfNeeded(raw) + guard var audio = note.payload.audio else { return } - let userEdited = !note.payload.text.isEmpty && note.meta.modifiedAt > note.meta.createdAt - note.payload.text = userEdited ? note.payload.text + "\n\n" + text : text + let now = Date() + audio.attempts.append(Note.TranscriptionAttempt( + text: text, + localeID: localeID, + confidence: confidence, + transcribedAt: now, + languageWasForced: languageWasForced + )) audio.transcriptionStatus = .done audio.transcriptionLocaleID = localeID - audio.transcribedAt = Date() + audio.transcribedAt = now note.payload.audio = audio try await updateNote(note) } @@ -337,6 +357,11 @@ final class SyncEngine { context: staged.context ) } + + let orphans = Self.watchInboxOrphanCount() + if orphans > 0 { + print("[Sync] Watch inbox holds \(orphans) recording(s) without readable metadata — not ingestible by this build") + } } /// Ingest one recording handed over from the watch — the single sanctioned @@ -358,14 +383,17 @@ final class SyncEngine { ) async -> Bool { guard let store, let tombstones else { return false } - // Permanent skips — clear the stale staged copy so it stops re-draining. + // Permanent skips — clear the stale staged copy so it stops re-draining, + // and ack so the watch drops its copy too (redelivery would just loop). if await tombstones.stubExists(id: id.stringValue) { Self.removeStagedFiles(audioURL: audioURL) + onWatchRecordingHandled?(id.stringValue) return false } let jsonPath = Note.relativePath(forID: id) if await store.fileExists(Self.dataPath(jsonPath)) { Self.removeStagedFiles(audioURL: audioURL) + onWatchRecordingHandled?(id.stringValue) return false } @@ -392,10 +420,24 @@ final class SyncEngine { lastSyncError = nil print("[Sync] Ingested watch recording \(id.stringValue) recorded \(recordedAt)") + onWatchRecordingHandled?(id.stringValue) onAudioNoteIngested?(id) return true } + /// `.m4a` files sitting in the watch inbox with no readable metadata sidecar + /// — quarantined forward-gate deliveries or the residue of a failed staging. + /// The drain can never ingest them; surfaced so they're at least visible. + nonisolated static func watchInboxOrphanCount() -> Int { + let fm = FileManager.default + let entries = (try? fm.contentsOfDirectory(at: watchInboxURL, includingPropertiesForKeys: nil)) ?? [] + let sidecarBases = Set(entries.filter { $0.lastPathComponent.hasSuffix(".meta.plist") } + .map { $0.lastPathComponent.replacingOccurrences(of: ".meta.plist", with: "") }) + return entries.filter { + $0.pathExtension == "m4a" && !sidecarBases.contains($0.deletingPathExtension().lastPathComponent) + }.count + } + /// One recording staged in the watch inbox: the `.m4a` and the metadata the /// receiver wrote in its `.meta.plist` sidecar. private struct StagedRecording { @@ -711,13 +753,35 @@ final class SyncEngine { // MARK: - Schema migration - /// Read-time upgrade for documents behind the current schema. A no-op - /// while `currentSchemaVersion` is 1, but wired into both read paths so - /// the first schema bump migrates instead of silently skipping. Write-back - /// happens lazily on the next save — never from live monitor events. - private func migrateIfNeeded(_ note: Note) -> Note { + /// Read-time upgrade for documents behind the current schema. Wired into + /// both read paths so a schema bump migrates instead of silently skipping. + /// Write-back happens lazily on the next save — never from live monitor + /// events. + /// + /// v2 → v3: transcripts moved from `payload.text` into `audio.attempts`. + /// A v2 voice note whose transcription completed holds its transcript (or + /// the transcript merged with user typing — indistinguishable by then) in + /// `text`; move that whole text into the first attempt so the v3 invariant + /// "text is user writing only" holds. Confidence was never recorded ⇒ nil. + /// Internal (not private) so the migration policy is unit-testable. + func migrateIfNeeded(_ note: Note) -> Note { guard note.meta.schemaVersion < Note.currentSchemaVersion else { return note } var migrated = note + if note.meta.schemaVersion < 3, + var audio = migrated.payload.audio, + audio.transcriptionStatus == .done, + audio.attempts.isEmpty, + !migrated.payload.text.isEmpty { + audio.attempts = [Note.TranscriptionAttempt( + text: migrated.payload.text, + localeID: audio.transcriptionLocaleID ?? "", + confidence: nil, + transcribedAt: audio.transcribedAt ?? migrated.meta.modifiedAt, + languageWasForced: false + )] + migrated.payload.text = "" + migrated.payload.audio = audio + } migrated.meta.schemaVersion = Note.currentSchemaVersion return migrated } diff --git a/Notes/Views/Notes/NoteEditorView.swift b/Notes/Views/Notes/NoteEditorView.swift index 807d655..b6eeee9 100644 --- a/Notes/Views/Notes/NoteEditorView.swift +++ b/Notes/Views/Notes/NoteEditorView.swift @@ -44,6 +44,10 @@ struct NoteEditorView: View { @Environment(\.dismiss) private var dismiss @State private var text = "" + /// The note text as last loaded from the entity, so an external change + /// (a transcription landing while this editor is open) can be told apart + /// from the user's own typing: refresh only while `text == loadedText`. + @State private var loadedText = "" @State private var tagsText = "" @State private var saveError: String? @State private var recordedAudio: AudioRecorderService.Recording? @@ -115,9 +119,18 @@ struct NoteEditorView: View { await contextService.refresh() case .edit(let note): text = note.text + loadedText = note.text tagsText = note.tags.joined(separator: ", ") } } + // A transcription finishing while this editor is open updates the + // entity underneath us; mirror it into the text field as long as the + // user hasn't typed (their edits always win over a live refresh). + .onChange(of: editedNote?.text) { _, newValue in + guard let newValue, text == loadedText else { return } + text = newValue + loadedText = newValue + } #if os(macOS) .frame(minWidth: 480, minHeight: 420) #endif @@ -190,6 +203,21 @@ struct NoteEditorView: View { transcriptionStatusRow(for: note) retranscribeMenu(for: note) } + // The transcript is machine output, kept apart from the user's own + // text: shown read-only here (it updates live as attempts land) with + // a one-tap copy into the editable note body. + if let transcript = note.transcriptText, !transcript.isEmpty { + Section("Transcript") { + Text(transcript) + .textSelection(.enabled) + .foregroundStyle(.secondary) + Button { + text = text.isEmpty ? transcript : text + "\n\n" + transcript + } label: { + Label("Insert into Note", systemImage: "text.insert") + } + } + } } @ViewBuilder @@ -263,6 +291,11 @@ struct NoteEditorView: View { return false } + private var editedNote: NoteEntity? { + if case .edit(let note) = mode { return note } + return nil + } + private var canSave: Bool { if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return true } if recordedAudio != nil { return true } diff --git a/Notes/Views/Notes/SearchView.swift b/Notes/Views/Notes/SearchView.swift index d23db11..ef83f16 100644 --- a/Notes/Views/Notes/SearchView.swift +++ b/Notes/Views/Notes/SearchView.swift @@ -56,7 +56,12 @@ struct SearchView: View { private var results: [NoteEntity] { guard !searchText.isEmpty else { return [] } - return notes.filter { $0.matches(search: searchText) } + return notes + .compactMap { note in note.searchScore(for: searchText).map { (note, $0) } } + .sorted { lhs, rhs in + lhs.1 != rhs.1 ? lhs.1 > rhs.1 : lhs.0.createdAt > rhs.0.createdAt + } + .map(\.0) } private func delete(_ note: NoteEntity) { @@ -68,12 +73,24 @@ struct SearchView: View { } extension NoteEntity { - /// Case-insensitive match against text, tags, and capture location. - func matches(search: String) -> Bool { + /// Case-insensitive relevance of this note to a search, or nil when nothing + /// matches. The score is the strongest matching field: the user's own text + /// and tags outrank transcript matches, and a transcript match is weighted + /// by the engine's confidence in it — a shaky transcription can still be + /// found, it just ranks below solid matches. Confidence-less engines get a + /// mid-scale weight rather than sinking to the bottom. + func searchScore(for search: String) -> Double? { let needle = search.lowercased() - return text.lowercased().contains(needle) - || tags.contains { $0.lowercased().contains(needle) } - || (placeName?.lowercased().contains(needle) ?? false) - || (locality?.lowercased().contains(needle) ?? false) + var score = 0.0 + if text.lowercased().contains(needle) { score = max(score, 1.0) } + if tags.contains(where: { $0.lowercased().contains(needle) }) { score = max(score, 0.9) } + if let transcript = transcriptText, transcript.lowercased().contains(needle) { + score = max(score, 0.5 + 0.4 * (transcriptConfidence ?? 0.5)) + } + if placeName?.lowercased().contains(needle) == true + || locality?.lowercased().contains(needle) == true { + score = max(score, 0.4) + } + return score > 0 ? score : nil } } diff --git a/NotesTests/AudioNotePayloadTests.swift b/NotesTests/AudioNotePayloadTests.swift index c5ae7d1..c44ce3e 100644 --- a/NotesTests/AudioNotePayloadTests.swift +++ b/NotesTests/AudioNotePayloadTests.swift @@ -80,4 +80,20 @@ struct AudioNotePayloadTests { @Test func decodeRejectsEmptyMetadata() { #expect(AudioNotePayload(metadata: [:]) == nil) } + + // MARK: - IngestAck + + @Test func ackRoundTrips() { + let ack = IngestAck(id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E") + let decoded = IngestAck(userInfo: ack.userInfo()) + #expect(decoded == ack) + } + + @Test func ackRejectsNewerVersionAndForeignUserInfo() { + var dict = IngestAck(id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E").userInfo() + dict["ackV"] = IngestAck.currentVersion + 1 + #expect(IngestAck(userInfo: dict) == nil) + #expect(IngestAck(userInfo: [:]) == nil) + #expect(IngestAck(userInfo: ["unrelated": true]) == nil) + } } diff --git a/NotesTests/NoteDocumentTests.swift b/NotesTests/NoteDocumentTests.swift index 27e7c01..75f6ffb 100644 --- a/NotesTests/NoteDocumentTests.swift +++ b/NotesTests/NoteDocumentTests.swift @@ -77,13 +77,13 @@ struct NoteDocumentTests { /// A file stamped with a newer schema (and carrying a field this build /// doesn't model) must be quarantined by the gate — Codable silently drops /// the unknown key, so partial-decoding then rewriting would downgrade it. - @Test func forwardGateQuarantinesV3Fixture() throws { + @Test func forwardGateQuarantinesNewerFixture() throws { let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 3, randomLo: 4) let json = """ { "meta": { "id": "\(id.stringValue)", - "schemaVersion": 3, + "schemaVersion": \(Note.currentSchemaVersion + 1), "createdAt": "2023-11-14T22:13:20Z", "modifiedAt": "2023-11-14T22:13:20Z" }, @@ -98,7 +98,7 @@ struct NoteDocumentTests { } """ let note = try DocumentCoder.decode(Note.self, from: Data(json.utf8)) - #expect(note.meta.schemaVersion == 3) + #expect(note.meta.schemaVersion == Note.currentSchemaVersion + 1) #expect(!note.isReadable) } diff --git a/NotesTests/TranscriptionAttemptTests.swift b/NotesTests/TranscriptionAttemptTests.swift new file mode 100644 index 0000000..4a1e9c0 --- /dev/null +++ b/NotesTests/TranscriptionAttemptTests.swift @@ -0,0 +1,167 @@ +import Foundation +import IndieSync +import SwiftData +import Testing +@testable import Notes + +/// The v3 transcript model: attempts accumulate on `AudioInfo`, the latest one +/// is displayed, `payload.text` stays purely user writing, and v2 files whose +/// transcript was baked into `text` migrate at read time. +@MainActor +struct TranscriptionAttemptTests { + private func makeEngine() throws -> (engine: SyncEngine, container: ModelContainer) { + let config = ModelConfiguration(isStoredInMemoryOnly: true, cloudKitDatabase: .none) + let container = try ModelContainer(for: NoteEntity.self, configurations: config) + return (SyncEngine(modelContainer: container), container) + } + + private func attempt(_ text: String, locale: String = "en-US", confidence: Double? = 0.9) -> Note.TranscriptionAttempt { + Note.TranscriptionAttempt( + text: text, + localeID: locale, + confidence: confidence, + transcribedAt: Date(timeIntervalSince1970: 1_700_000_100), + languageWasForced: false + ) + } + + // MARK: - Model + + @Test func latestAttemptWins() { + var audio = Note.AudioInfo(duration: 5, transcriberDeviceID: "d", transcriptionStatus: .done) + audio.attempts = [attempt("first"), attempt("второй", locale: "ru-RU")] + #expect(audio.latestAttempt?.text == "второй") + } + + @Test func displayTextPrefersUserTextThenTranscript() { + var audio = Note.AudioInfo(duration: 5, transcriberDeviceID: "d", transcriptionStatus: .done) + audio.attempts = [attempt("spoken words")] + var note = Note.create(text: "", source: .transcribed, context: .empty, audio: audio) + #expect(note.displayText == "spoken words") + #expect(note.title == "spoken words") + note.payload.text = "my own words" + #expect(note.displayText == "my own words") + } + + /// A v2 file (no `attempts` key) must decode with empty attempts, not fail. + @Test func v2AudioInfoDecodesWithoutAttemptsKey() throws { + let json = """ + {"duration": 5, "transcriberDeviceID": "d", "transcriptionStatus": "done", "transcriptionLocaleID": "en-US"} + """ + let audio = try JSONDecoder().decode(Note.AudioInfo.self, from: Data(json.utf8)) + #expect(audio.attempts.isEmpty) + #expect(audio.transcriptionStatus == .done) + } + + @Test func attemptsSurviveCodableRoundTrip() throws { + var audio = Note.AudioInfo(duration: 5, transcriberDeviceID: "d", transcriptionStatus: .done) + audio.attempts = [attempt("hello", confidence: nil), attempt("привет", locale: "ru-RU")] + let note = Note.create(text: "", source: .transcribed, context: .empty, audio: audio) + let decoded = try DocumentCoder.decode(Note.self, from: DocumentCoder.encode(note)) + #expect(decoded.payload.audio?.attempts.count == 2) + #expect(decoded.payload.audio?.latestAttempt?.localeID == "ru-RU") + #expect(decoded.payload.audio?.attempts.first?.confidence == nil) + } + + // MARK: - v2 → v3 migration + + @Test func migrationMovesCompletedTranscriptIntoAttempts() throws { + let (engine, container) = try makeEngine() + defer { withExtendedLifetime(container) {} } + var note = Note.create( + text: "the old transcript", + source: .transcribed, + context: .empty, + audio: Note.AudioInfo( + duration: 5, + transcriberDeviceID: "d", + transcriptionStatus: .done, + transcriptionLocaleID: "en-US", + transcribedAt: Date(timeIntervalSince1970: 1_700_000_100) + ) + ) + note.meta.schemaVersion = 2 + + let migrated = engine.migrateIfNeeded(note) + #expect(migrated.meta.schemaVersion == Note.currentSchemaVersion) + #expect(migrated.payload.text.isEmpty) + #expect(migrated.payload.audio?.attempts.count == 1) + #expect(migrated.payload.audio?.latestAttempt?.text == "the old transcript") + #expect(migrated.payload.audio?.latestAttempt?.localeID == "en-US") + #expect(migrated.payload.audio?.latestAttempt?.confidence == nil) + #expect(migrated.displayText == "the old transcript") + } + + @Test func migrationLeavesPendingAndTextNotesAlone() throws { + let (engine, container) = try makeEngine() + defer { withExtendedLifetime(container) {} } + + var pending = Note.create( + text: "", + source: .transcribed, + context: .empty, + audio: Note.AudioInfo(duration: 5, transcriberDeviceID: "d", transcriptionStatus: .pending) + ) + pending.meta.schemaVersion = 2 + let migratedPending = engine.migrateIfNeeded(pending) + #expect(migratedPending.payload.audio?.attempts.isEmpty == true) + #expect(migratedPending.meta.schemaVersion == Note.currentSchemaVersion) + + var textNote = Note.create(text: "just typing", context: .empty) + textNote.meta.schemaVersion = 1 + let migratedText = engine.migrateIfNeeded(textNote) + #expect(migratedText.payload.text == "just typing") + #expect(migratedText.meta.schemaVersion == Note.currentSchemaVersion) + } + + // MARK: - Cache columns + + @Test func mapperCachesLatestAttempt() throws { + let config = ModelConfiguration(isStoredInMemoryOnly: true, cloudKitDatabase: .none) + let container = try ModelContainer(for: NoteEntity.self, configurations: config) + defer { withExtendedLifetime(container) {} } + let context = container.mainContext + + var audio = Note.AudioInfo(duration: 5, transcriberDeviceID: "d", transcriptionStatus: .done, transcriptionLocaleID: "ru-RU") + audio.attempts = [attempt("first draft"), attempt("окончательный", locale: "ru-RU", confidence: 0.8)] + let note = Note.create(text: "", source: .transcribed, context: .empty, audio: audio) + NoteMapper.upsert(note, relativePath: note.relativePath, into: context) + + let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context)) + #expect(entity.transcriptText == "окончательный") + #expect(entity.transcriptConfidence == 0.8) + #expect(entity.displayText == "окончательный") + #expect(entity.title == "окончательный") + } + + // MARK: - Search scoring + + @Test func searchRanksUserTextAboveTranscript() throws { + let config = ModelConfiguration(isStoredInMemoryOnly: true, cloudKitDatabase: .none) + let container = try ModelContainer(for: NoteEntity.self, configurations: config) + defer { withExtendedLifetime(container) {} } + let context = container.mainContext + + let typed = Note.create(text: "coffee order", context: .empty) + var audio = Note.AudioInfo(duration: 5, transcriberDeviceID: "d", transcriptionStatus: .done) + audio.attempts = [attempt("coffee with sam", confidence: 0.9)] + let voice = Note.create(text: "", source: .transcribed, context: .empty, audio: audio) + NoteMapper.upsert(typed, relativePath: typed.relativePath, into: context) + NoteMapper.upsert(voice, relativePath: voice.relativePath, into: context) + + let typedEntity = try #require(NoteMapper.fetch(id: typed.meta.id.stringValue, in: context)) + let voiceEntity = try #require(NoteMapper.fetch(id: voice.meta.id.stringValue, in: context)) + + let typedScore = try #require(typedEntity.searchScore(for: "coffee")) + let voiceScore = try #require(voiceEntity.searchScore(for: "coffee")) + #expect(typedScore > voiceScore) + #expect(typedEntity.searchScore(for: "zzz") == nil) + + // Higher-confidence transcripts outrank shakier ones. + var lowAudio = audio + lowAudio.attempts = [attempt("coffee maybe", confidence: 0.2)] + voiceEntity.audio = lowAudio + let lowScore = try #require(voiceEntity.searchScore(for: "coffee")) + #expect(voiceScore > lowScore) + } +} diff --git a/README.md b/README.md index b4d73f3..17ed282 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,12 @@ the top of the list. - **Quick text notes** — jot something down in two taps; first line doubles as the title - **Voice notes** — record a memo (iPhone, Mac, or Apple Watch); the audio is kept and transcribed on-device afterwards, with a per-note "re-transcribe in another language" override and a settings screen to enable transcription languages -- **Apple Watch capture** — a watch-face complication opens a one-button recorder; the recording transfers to your iPhone over WatchConnectivity, which ingests and transcribes it (no iCloud on the watch) +- **Transcripts kept separate from your text** — every transcription attempt is preserved on the note (latest shown, with language and confidence); your own typing is never mixed with or overwritten by machine output, and a transcript can be inserted into the note body on demand +- **Apple Watch capture** — a watch-face complication opens a one-button recorder; the recording transfers to your iPhone over WatchConnectivity, which ingests and transcribes it (no iCloud on the watch); the watch keeps its copy until the iPhone acknowledges durable ingest, so a failed handoff never loses audio - **Automatic context capture** — location, place name (reverse-geocoded), time zone, and device are stamped onto every note, best-effort and privacy-friendly (all data stays in your iCloud) - **"Right here, right now" recall** — a ranked shelf of notes whose capture context matches your current place and time, above the regular newest-first list - **Tags** — manual organization alongside the automatic context; browse notes by tag -- **Search** — full-text over note text, tags, and captured place names +- **Search** — full-text over note text, voice transcripts (weighted by transcription confidence), tags, and captured place names - **iCloud sync** — notes live as JSON files in your iCloud Drive (visible in Files.app) and sync across iPhone and Mac; the local database is just a rebuildable cache - **iOS + macOS + watchOS** — same app and data across iPhone and Mac, with an Apple Watch companion for voice capture - **Backups** — one-tap local backup and restore of all notes (IndieBackup); backup files can be shared and imported across devices diff --git a/Shared/Connectivity/AudioNotePayload.swift b/Shared/Connectivity/AudioNotePayload.swift index 5773cb2..ec03a2c 100644 --- a/Shared/Connectivity/AudioNotePayload.swift +++ b/Shared/Connectivity/AudioNotePayload.swift @@ -71,3 +71,41 @@ struct AudioNotePayload: Equatable, Sendable { self.timeZoneID = metadata[Key.timeZone] as? String } } + +/// Phone → watch acknowledgment that a recording reached a terminal state on +/// the phone (durably ingested into the iCloud container, or permanently +/// vetoed). Sent over `transferUserInfo` — queued and durable, so an ack +/// survives both apps being closed. **Only** this ack lets the watch delete +/// its local copy of a recording; WatchConnectivity-level delivery success is +/// deliberately not trusted, because the phone can still fail after `didReceive` +/// returns (a staging or iCloud write failure would otherwise lose the audio +/// forever — the watch's copy is the last line of defense). +struct IngestAck: Equatable, Sendable { + static let currentVersion = 1 + + /// ULID string of the acknowledged recording. + let id: String + + private enum Key { + static let version = "ackV" + static let id = "ackID" + } + + init(id: String) { + self.id = id + } + + /// A plist-serializable dictionary for `WCSession.transferUserInfo(_:)`. + func userInfo() -> [String: Any] { + [Key.version: Self.currentVersion, Key.id: id] + } + + /// Decode a received userInfo dictionary; nil when it isn't an ack this + /// build understands (same forward-gate shape as `AudioNotePayload`). + init?(userInfo: [String: Any]) { + guard let version = userInfo[Key.version] as? Int, + version >= 1, version <= Self.currentVersion, + let id = userInfo[Key.id] as? String else { return nil } + self.id = id + } +}