Add voice notes: on-device transcription, multi-language, watch capture
- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud - AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline with enabled-language set and confidence-based auto-pick, re-transcribe menu - Settings > Transcription > Languages with asset download/reserve handling - watchOS app + accessory complication: one-button recorder, WCSession transferFile to phone, WatchInbox staging, direct-upsert ingestion - Player UI, transcription status chips, quick-capture mic in notes list - Version 0.2, changelog + README updated
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
#if os(iOS)
|
||||
import UIKit
|
||||
|
||||
/// iOS application delegate. Its sole job is to stand up the WatchConnectivity
|
||||
/// receiver in `didFinishLaunching` — which runs even on a background launch the
|
||||
/// system makes purely to deliver a watch file transfer, unlike a SwiftUI
|
||||
/// `.task` that may not run until the UI is built.
|
||||
///
|
||||
/// The receiver is created and its `WCSession` activated here immediately, so a
|
||||
/// delivered recording is staged to the inbox even before `AppServices` exists.
|
||||
/// `NotesApp` wires the receiver's `syncEngine` in once services are ready, so a
|
||||
/// live delivery can drain right away; a delivery that arrives before then is
|
||||
/// drained on the next foreground connect.
|
||||
@MainActor
|
||||
final class AppDelegate: NSObject, UIApplicationDelegate {
|
||||
let watchReceiver = PhoneWatchReceiver()
|
||||
|
||||
func application(
|
||||
_ application: UIApplication,
|
||||
didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
|
||||
) -> Bool {
|
||||
watchReceiver.activate()
|
||||
return true
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -11,6 +11,9 @@ final class AppServices {
|
||||
let syncEngine: SyncEngine
|
||||
let contextService: ContextService
|
||||
let backupController: BackupController
|
||||
let audioRecorder: AudioRecorderService
|
||||
let transcriptionSettings: TranscriptionSettings
|
||||
let transcriptionService: TranscriptionService
|
||||
|
||||
init() async {
|
||||
let container = NotesModelContainer.make()
|
||||
@@ -21,6 +24,20 @@ final class AppServices {
|
||||
self.backupController = BackupController(
|
||||
configuration: NotesBackupConfiguration(syncEngine: engine)
|
||||
)
|
||||
self.audioRecorder = AudioRecorderService()
|
||||
let settings = TranscriptionSettings()
|
||||
self.transcriptionSettings = settings
|
||||
let transcription = TranscriptionService(
|
||||
syncEngine: engine,
|
||||
settings: settings,
|
||||
modelContainer: container
|
||||
)
|
||||
self.transcriptionService = transcription
|
||||
|
||||
// A watch recording, once ingested, is this device's to transcribe.
|
||||
engine.onAudioNoteIngested = { [weak transcription] id in
|
||||
transcription?.enqueue(noteID: id)
|
||||
}
|
||||
}
|
||||
|
||||
/// The file extension this app owns for backup documents. Used to route
|
||||
@@ -30,5 +47,12 @@ final class AppServices {
|
||||
/// Slow startup work (iCloud container discovery) — runs after the UI is up.
|
||||
func startDeferredServices() async {
|
||||
await syncEngine.connect()
|
||||
// Drain any watch recordings delivered while the app was closed — the
|
||||
// container is connected now, so they can be written and ingested.
|
||||
await syncEngine.ingestStagedWatchRecordings()
|
||||
// Load the supported languages / reservation cap, then pick up any of
|
||||
// this device's notes that were still pending when it last quit.
|
||||
await transcriptionSettings.prepare()
|
||||
transcriptionService.resumePendingWork()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
#if os(iOS)
|
||||
import Foundation
|
||||
import os
|
||||
import WatchConnectivity
|
||||
|
||||
/// Phone side of the watch recording hand-off. Receives a `WCSessionFile` the
|
||||
/// watch sent via `transferFile`, stages it into the local `WatchInbox` (with the
|
||||
/// `<ULID>.m4a` + `<ULID>.meta.plist` layout `SyncEngine.ingestStagedWatchRecordings`
|
||||
/// expects), then asks the sync engine to drain the inbox.
|
||||
///
|
||||
/// The move happens **synchronously in the nonisolated delegate callback**: the
|
||||
/// system deletes the received inbox file the instant `didReceive` returns, so
|
||||
/// deferring the move to a `MainActor` hop would race the bytes away. Only after
|
||||
/// staging do we hop to the main actor to trigger ingest.
|
||||
///
|
||||
/// Activated from the `UIApplicationDelegate` (not a SwiftUI `.task`) so a
|
||||
/// background launch the system makes purely to deliver a transfer still stages
|
||||
/// the file. `syncEngine` is wired in later, once `AppServices` exists; when it's
|
||||
/// nil (a background launch with no UI yet) the staged file is drained on the
|
||||
/// next foreground connect instead.
|
||||
@MainActor
|
||||
final class PhoneWatchReceiver: NSObject {
|
||||
private nonisolated static let log = Logger(
|
||||
subsystem: "dev.rzen.indie.Notes", category: "phone-watch-receiver")
|
||||
|
||||
/// The live sync engine, set once `AppServices` is built. Weak: `AppServices`
|
||||
/// owns it for the app's lifetime; the receiver only borrows it to trigger a
|
||||
/// drain when one is available.
|
||||
weak var syncEngine: SyncEngine?
|
||||
|
||||
private var session: WCSession?
|
||||
|
||||
func activate() {
|
||||
guard WCSession.isSupported() else { return }
|
||||
let session = WCSession.default
|
||||
session.delegate = self
|
||||
session.activate()
|
||||
self.session = session
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - WCSessionDelegate
|
||||
|
||||
extension PhoneWatchReceiver: WCSessionDelegate {
|
||||
nonisolated func session(_ session: WCSession, activationDidCompleteWith activationState: WCSessionActivationState, error: (any Error)?) {
|
||||
if let error { Self.log.error("phone WC activation failed: \(error, privacy: .public)") }
|
||||
}
|
||||
|
||||
nonisolated func sessionDidBecomeInactive(_ session: WCSession) {}
|
||||
|
||||
nonisolated func sessionDidDeactivate(_ session: WCSession) {
|
||||
// Reactivate for a switched watch.
|
||||
session.activate()
|
||||
}
|
||||
|
||||
nonisolated func session(_ session: WCSession, didReceive file: WCSessionFile) {
|
||||
// Synchronous move — the inbox file is deleted the moment this returns.
|
||||
Self.stage(file: file)
|
||||
// Drain if the engine is live; otherwise the next foreground connect drains.
|
||||
Task { @MainActor in await self.syncEngine?.ingestStagedWatchRecordings() }
|
||||
}
|
||||
|
||||
/// Move the received audio into `WatchInbox` and write the metadata sidecar
|
||||
/// `SyncEngine`'s drain reads. On a decodable payload this stages a full
|
||||
/// `<ULID>.m4a` + `<ULID>.meta.plist` pair. On an unknown/newer payload
|
||||
/// version (the forward gate) the bytes are still quarantined in the inbox —
|
||||
/// but without a sidecar, so the drain skips them and a future build can pick
|
||||
/// them up.
|
||||
nonisolated private static func stage(file: WCSessionFile) {
|
||||
let fm = FileManager.default
|
||||
let inbox = SyncEngine.watchInboxURL
|
||||
try? fm.createDirectory(at: inbox, withIntermediateDirectories: true)
|
||||
|
||||
let metadata = file.metadata ?? [:]
|
||||
|
||||
guard let payload = AudioNotePayload(metadata: metadata) else {
|
||||
// Forward gate: keep the bytes but don't make them ingestible.
|
||||
let base = (metadata["id"] as? String) ?? UUID().uuidString
|
||||
let dest = inbox.appending(path: "\(base).m4a")
|
||||
try? fm.removeItem(at: dest)
|
||||
do {
|
||||
try fm.moveItem(at: file.fileURL, to: dest)
|
||||
log.error("staged watch recording \(base, privacy: .public) with unknown payload version — skipping ingest")
|
||||
} catch {
|
||||
log.error("failed to stage unknown-version recording: \(error, privacy: .public)")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
let base = payload.id
|
||||
let audioDest = inbox.appending(path: "\(base).m4a")
|
||||
let metaDest = inbox.appending(path: "\(base).meta.plist")
|
||||
// Idempotent for a redelivery of the same recording.
|
||||
try? fm.removeItem(at: audioDest)
|
||||
do {
|
||||
try fm.moveItem(at: file.fileURL, to: audioDest)
|
||||
} catch {
|
||||
log.error("failed to stage recording \(base, privacy: .public): \(error, privacy: .public)")
|
||||
return
|
||||
}
|
||||
|
||||
var meta: [String: Any] = [
|
||||
"id": payload.id,
|
||||
"recordedAt": payload.recordedAt,
|
||||
"duration": payload.duration,
|
||||
]
|
||||
if let tz = payload.timeZoneID { meta["timeZoneID"] = tz }
|
||||
if let data = try? PropertyListSerialization.data(fromPropertyList: meta, format: .binary, options: 0) {
|
||||
try? data.write(to: metaDest)
|
||||
}
|
||||
log.info("staged watch recording \(base, privacy: .public) for ingest")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
+84
-5
@@ -40,7 +40,10 @@ struct CaptureContext: Codable, Sendable, Equatable {
|
||||
/// One note = one JSON file in the iCloud container, filed under
|
||||
/// `Records/YYYY/MM/<ULID>.json` (UTC-bucketed by the ULID's timestamp).
|
||||
struct Note: SyncDocument, VersionedDocument, Equatable {
|
||||
static let currentSchemaVersion = 1
|
||||
/// v2 added `Payload.audio` for voice notes. Because it is optional, a v1
|
||||
/// file decodes unchanged (missing key ⇒ nil) and `migrateIfNeeded` only
|
||||
/// bumps the stamped version — no field back-fill is needed.
|
||||
static let currentSchemaVersion = 2
|
||||
|
||||
struct Meta: Codable, Sendable, Equatable {
|
||||
var id: ULID
|
||||
@@ -49,18 +52,43 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
|
||||
var modifiedAt: Date
|
||||
}
|
||||
|
||||
/// How the note's text came to be. `transcribed` is reserved for the
|
||||
/// planned voice-capture path.
|
||||
/// How the note's text came to be. `transcribed` marks a voice note whose
|
||||
/// text is (or will be) produced by on-device transcription.
|
||||
enum Source: String, Codable, Sendable {
|
||||
case typed
|
||||
case transcribed
|
||||
}
|
||||
|
||||
/// Where a voice note is in its transcription lifecycle.
|
||||
enum TranscriptionStatus: String, Codable, Sendable {
|
||||
case pending
|
||||
case done
|
||||
case failed
|
||||
}
|
||||
|
||||
/// The audio-note sidecar metadata. Present only on voice notes; the audio
|
||||
/// bytes themselves live in a `.m4a` next to the note JSON (path derived,
|
||||
/// never stored — see `audioRelativePath`).
|
||||
struct AudioInfo: Codable, Sendable, Equatable {
|
||||
var duration: TimeInterval
|
||||
/// Per-install UUID of the device that owns pending transcription. Only
|
||||
/// the device whose `InstallIdentity.installID` matches this transcribes;
|
||||
/// every other device renders status only. Re-transcribe reassigns it.
|
||||
var transcriberDeviceID: String
|
||||
var transcriptionStatus: TranscriptionStatus
|
||||
/// BCP-47 locale the transcript was produced in; nil while pending.
|
||||
var transcriptionLocaleID: String?
|
||||
var transcribedAt: Date?
|
||||
}
|
||||
|
||||
struct Payload: Codable, Sendable, Equatable {
|
||||
var text: String
|
||||
var source: Source
|
||||
var tags: [String]
|
||||
var context: CaptureContext
|
||||
/// Voice-note sidecar metadata, nil for a plain typed note. Optional so
|
||||
/// a v1 document (no `audio` key) decodes as a text note unchanged.
|
||||
var audio: AudioInfo?
|
||||
}
|
||||
|
||||
var meta: Meta
|
||||
@@ -75,16 +103,67 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
|
||||
TimeBucketedLayout.relativePath(for: id)
|
||||
}
|
||||
|
||||
/// The `.m4a` sidecar path for a note, derived by swapping `.json` → `.m4a`
|
||||
/// on the note's **actual** JSON path. The audio path is never stored — it
|
||||
/// is always this swap — so it cannot drift from the note it belongs to, and
|
||||
/// deleting a note whose file sits under a legacy (pre-fix) month bucket
|
||||
/// still targets the sidecar that shares that bucket.
|
||||
static func audioRelativePath(forJSONPath jsonPath: String) -> String {
|
||||
guard jsonPath.hasSuffix(".json") else { return jsonPath + ".m4a" }
|
||||
return String(jsonPath.dropLast(".json".count)) + ".m4a"
|
||||
}
|
||||
|
||||
/// The `.m4a` sidecar path recomputed from a note id, sharing the note's
|
||||
/// UTC-bucketed layout. Safe for load/warm paths because audio exists only
|
||||
/// on v2+ records, which always file under the current UTC bucket — the
|
||||
/// legacy local-month concern applies solely to v1 text notes (no audio).
|
||||
static func audioRelativePath(forID id: ULID) -> String {
|
||||
audioRelativePath(forJSONPath: relativePath(forID: id))
|
||||
}
|
||||
|
||||
static func create(
|
||||
text: String,
|
||||
source: Source = .typed,
|
||||
tags: [String] = [],
|
||||
context: CaptureContext
|
||||
context: CaptureContext,
|
||||
audio: AudioInfo? = nil
|
||||
) -> Note {
|
||||
let now = Date()
|
||||
return Note(
|
||||
meta: Meta(id: ULID(), schemaVersion: currentSchemaVersion, createdAt: now, modifiedAt: now),
|
||||
payload: Payload(text: text, source: source, tags: tags, context: context)
|
||||
payload: Payload(text: text, source: source, tags: tags, context: context, audio: audio)
|
||||
)
|
||||
}
|
||||
|
||||
/// Factory for a note ingested from another device's recording (the watch,
|
||||
/// which has no iCloud of its own). The audio was captured elsewhere; this
|
||||
/// device only stores and transcribes it. `createdAt` is the ULID's embedded
|
||||
/// timestamp — the record-start instant on the capturing device — so the note
|
||||
/// buckets under the month it was *recorded*, not the (possibly much later)
|
||||
/// month it was ingested. Text is empty until transcription lands.
|
||||
static func createIngested(
|
||||
id: ULID,
|
||||
duration: TimeInterval,
|
||||
transcriberDeviceID: String,
|
||||
tags: [String] = [],
|
||||
context: CaptureContext
|
||||
) -> Note {
|
||||
let recordedAt = id.timestamp
|
||||
return Note(
|
||||
meta: Meta(id: id, schemaVersion: currentSchemaVersion, createdAt: recordedAt, modifiedAt: recordedAt),
|
||||
payload: Payload(
|
||||
text: "",
|
||||
source: .transcribed,
|
||||
tags: tags,
|
||||
context: context,
|
||||
audio: AudioInfo(
|
||||
duration: duration,
|
||||
transcriberDeviceID: transcriberDeviceID,
|
||||
transcriptionStatus: .pending,
|
||||
transcriptionLocaleID: nil,
|
||||
transcribedAt: nil
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,17 @@ final class NoteEntity {
|
||||
var timeZoneID: String?
|
||||
var device: String?
|
||||
|
||||
// Flattened AudioInfo — nil columns mean a plain text note. `hasAudio` is
|
||||
// stored rather than derived from `audioDuration` so a zero-length recording
|
||||
// is still recognizably a voice note. `transcribedAt` is deliberately not
|
||||
// cached (the file is authoritative for that timestamp), so a cache-only
|
||||
// reconstruction of a completed note has a nil `transcribedAt`.
|
||||
var hasAudio: Bool = false
|
||||
var audioDuration: Double?
|
||||
var transcriptionStatusRaw: String?
|
||||
var transcriptionLocaleID: String?
|
||||
var transcriberDeviceID: String?
|
||||
|
||||
init(id: String, jsonRelativePath: String) {
|
||||
self.id = id
|
||||
self.jsonRelativePath = jsonRelativePath
|
||||
@@ -70,6 +81,31 @@ extension NoteEntity {
|
||||
Note.Source(rawValue: sourceRaw) ?? .typed
|
||||
}
|
||||
|
||||
/// Reconstructs `AudioInfo` from the flattened columns (nil for a text
|
||||
/// note). `transcribedAt` is not cached, so it comes back nil even for a
|
||||
/// completed transcription — acceptable for the cache-only offline fallback,
|
||||
/// where the file (which carries it) simply wasn't reachable.
|
||||
var audio: Note.AudioInfo? {
|
||||
get {
|
||||
guard hasAudio else { return nil }
|
||||
return Note.AudioInfo(
|
||||
duration: audioDuration ?? 0,
|
||||
transcriberDeviceID: transcriberDeviceID ?? "",
|
||||
transcriptionStatus: transcriptionStatusRaw
|
||||
.flatMap(Note.TranscriptionStatus.init(rawValue:)) ?? .pending,
|
||||
transcriptionLocaleID: transcriptionLocaleID,
|
||||
transcribedAt: nil
|
||||
)
|
||||
}
|
||||
set {
|
||||
hasAudio = newValue != nil
|
||||
audioDuration = newValue?.duration
|
||||
transcriberDeviceID = newValue?.transcriberDeviceID
|
||||
transcriptionStatusRaw = newValue?.transcriptionStatus.rawValue
|
||||
transcriptionLocaleID = newValue?.transcriptionLocaleID
|
||||
}
|
||||
}
|
||||
|
||||
/// First line of the text, used as the display title.
|
||||
var title: String {
|
||||
text.split(separator: "\n", omittingEmptySubsequences: true)
|
||||
|
||||
@@ -35,6 +35,7 @@ enum NoteMapper {
|
||||
entity.modifiedAt = note.meta.modifiedAt
|
||||
entity.jsonRelativePath = relativePath
|
||||
entity.context = note.payload.context
|
||||
entity.audio = note.payload.audio
|
||||
}
|
||||
|
||||
/// Near-lossless reconstruction from the cache, used only as the offline
|
||||
@@ -52,7 +53,8 @@ enum NoteMapper {
|
||||
text: entity.text,
|
||||
source: entity.source,
|
||||
tags: entity.tags,
|
||||
context: entity.context
|
||||
context: entity.context,
|
||||
audio: entity.audio
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@ import SwiftData
|
||||
/// from the files (an empty cache is the entire contract between the two).
|
||||
enum NotesModelContainer {
|
||||
/// Bump whenever the cache schema changes — wipes and rebuilds from files.
|
||||
static let cacheSchemaVersion = 1
|
||||
/// v2 added the flattened audio columns on `NoteEntity`.
|
||||
static let cacheSchemaVersion = 2
|
||||
private static let cacheSchemaVersionKey = "Notes.cacheSchemaVersion"
|
||||
private static let identityTokenKey = "Notes.iCloudIdentityToken"
|
||||
|
||||
|
||||
@@ -4,6 +4,9 @@ import SwiftUI
|
||||
|
||||
@main
|
||||
struct NotesApp: App {
|
||||
#if os(iOS)
|
||||
@UIApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate
|
||||
#endif
|
||||
@State private var appServices: AppServices?
|
||||
@State private var pendingBackupURL: URL?
|
||||
@Environment(\.scenePhase) private var scenePhase
|
||||
@@ -16,6 +19,9 @@ struct NotesApp: App {
|
||||
.modelContainer(appServices.modelContainer)
|
||||
.environment(appServices.syncEngine)
|
||||
.environment(appServices.contextService)
|
||||
.environment(appServices.audioRecorder)
|
||||
.environment(appServices.transcriptionSettings)
|
||||
.environment(appServices.transcriptionService)
|
||||
.environmentObject(appServices.backupController)
|
||||
} else {
|
||||
ProgressView()
|
||||
@@ -25,6 +31,11 @@ struct NotesApp: App {
|
||||
if appServices == nil {
|
||||
let services = await AppServices()
|
||||
appServices = services
|
||||
#if os(iOS)
|
||||
// Hand the (already-activated) watch receiver the live sync
|
||||
// engine so a delivery arriving now drains immediately.
|
||||
appDelegate.watchReceiver.syncEngine = services.syncEngine
|
||||
#endif
|
||||
await services.startDeferredServices()
|
||||
if let pending = pendingBackupURL {
|
||||
pendingBackupURL = nil
|
||||
@@ -58,6 +69,9 @@ struct NotesApp: App {
|
||||
// file swap would import a half-restored state.
|
||||
if !appServices.backupController.isRestoring {
|
||||
await syncEngine.reconcile()
|
||||
// Ingest any watch recordings staged since we last
|
||||
// looked (delivered while backgrounded).
|
||||
await syncEngine.ingestStagedWatchRecordings()
|
||||
}
|
||||
case .checking:
|
||||
break
|
||||
|
||||
@@ -6,51 +6,6 @@
|
||||
<string>$(DEVELOPMENT_LANGUAGE)</string>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Contextful</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>$(MARKETING_VERSION)</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>$(CURRENT_PROJECT_VERSION)</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
<true/>
|
||||
<key>UILaunchScreen</key>
|
||||
<dict/>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations~ipad</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
|
||||
<key>NSUbiquitousContainers</key>
|
||||
<dict>
|
||||
<key>iCloud.dev.rzen.indie.Notes</key>
|
||||
<dict>
|
||||
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
|
||||
<true/>
|
||||
<key>NSUbiquitousContainerSupportedFolderLevels</key>
|
||||
<string>Any</string>
|
||||
<key>NSUbiquitousContainerName</key>
|
||||
<string>Notes</string>
|
||||
</dict>
|
||||
</dict>
|
||||
<key>LSSupportsOpeningDocumentsInPlace</key>
|
||||
<true/>
|
||||
<key>CFBundleDocumentTypes</key>
|
||||
<array>
|
||||
<dict>
|
||||
@@ -66,18 +21,67 @@
|
||||
</array>
|
||||
</dict>
|
||||
</array>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>$(MARKETING_VERSION)</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>$(CURRENT_PROJECT_VERSION)</string>
|
||||
<key>ITSAppUsesNonExemptEncryption</key>
|
||||
<false/>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
<true/>
|
||||
<key>LSSupportsOpeningDocumentsInPlace</key>
|
||||
<true/>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Notes records voice memos and transcribes them on-device, so you can capture a thought by speaking.</string>
|
||||
<key>NSUbiquitousContainers</key>
|
||||
<dict>
|
||||
<key>iCloud.dev.rzen.indie.Notes</key>
|
||||
<dict>
|
||||
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
|
||||
<true/>
|
||||
<key>NSUbiquitousContainerName</key>
|
||||
<string>Notes</string>
|
||||
<key>NSUbiquitousContainerSupportedFolderLevels</key>
|
||||
<string>Any</string>
|
||||
</dict>
|
||||
</dict>
|
||||
<key>UILaunchScreen</key>
|
||||
<dict/>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations~ipad</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>UTExportedTypeDeclarations</key>
|
||||
<array>
|
||||
<dict>
|
||||
<key>UTTypeIdentifier</key>
|
||||
<string>dev.rzen.indie.Notes.backup</string>
|
||||
<key>UTTypeDescription</key>
|
||||
<string>Notes Backup</string>
|
||||
<key>UTTypeConformsTo</key>
|
||||
<array>
|
||||
<string>public.data</string>
|
||||
<string>public.archive</string>
|
||||
</array>
|
||||
<key>UTTypeDescription</key>
|
||||
<string>Notes Backup</string>
|
||||
<key>UTTypeIdentifier</key>
|
||||
<string>dev.rzen.indie.Notes.backup</string>
|
||||
<key>UTTypeTagSpecification</key>
|
||||
<dict>
|
||||
<key>public.filename-extension</key>
|
||||
|
||||
@@ -6,40 +6,6 @@
|
||||
<string>$(DEVELOPMENT_LANGUAGE)</string>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Contextful</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>$(MARKETING_VERSION)</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>$(CURRENT_PROJECT_VERSION)</string>
|
||||
<key>LSMinimumSystemVersion</key>
|
||||
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
|
||||
<key>LSApplicationCategoryType</key>
|
||||
<string>public.app-category.productivity</string>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
|
||||
<key>NSLocationUsageDescription</key>
|
||||
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
|
||||
<key>NSUbiquitousContainers</key>
|
||||
<dict>
|
||||
<key>iCloud.dev.rzen.indie.Notes</key>
|
||||
<dict>
|
||||
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
|
||||
<true/>
|
||||
<key>NSUbiquitousContainerSupportedFolderLevels</key>
|
||||
<string>Any</string>
|
||||
<key>NSUbiquitousContainerName</key>
|
||||
<string>Notes</string>
|
||||
</dict>
|
||||
</dict>
|
||||
<key>CFBundleDocumentTypes</key>
|
||||
<array>
|
||||
<dict>
|
||||
@@ -55,18 +21,56 @@
|
||||
</array>
|
||||
</dict>
|
||||
</array>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>$(PRODUCT_NAME)</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>$(MARKETING_VERSION)</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>$(CURRENT_PROJECT_VERSION)</string>
|
||||
<key>ITSAppUsesNonExemptEncryption</key>
|
||||
<false/>
|
||||
<key>LSApplicationCategoryType</key>
|
||||
<string>public.app-category.productivity</string>
|
||||
<key>LSMinimumSystemVersion</key>
|
||||
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
|
||||
<key>NSLocationUsageDescription</key>
|
||||
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Notes records voice memos and transcribes them on-device, so you can capture a thought by speaking.</string>
|
||||
<key>NSUbiquitousContainers</key>
|
||||
<dict>
|
||||
<key>iCloud.dev.rzen.indie.Notes</key>
|
||||
<dict>
|
||||
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
|
||||
<true/>
|
||||
<key>NSUbiquitousContainerName</key>
|
||||
<string>Notes</string>
|
||||
<key>NSUbiquitousContainerSupportedFolderLevels</key>
|
||||
<string>Any</string>
|
||||
</dict>
|
||||
</dict>
|
||||
<key>UTExportedTypeDeclarations</key>
|
||||
<array>
|
||||
<dict>
|
||||
<key>UTTypeIdentifier</key>
|
||||
<string>dev.rzen.indie.Notes.backup</string>
|
||||
<key>UTTypeDescription</key>
|
||||
<string>Notes Backup</string>
|
||||
<key>UTTypeConformsTo</key>
|
||||
<array>
|
||||
<string>public.data</string>
|
||||
<string>public.archive</string>
|
||||
</array>
|
||||
<key>UTTypeDescription</key>
|
||||
<string>Notes Backup</string>
|
||||
<key>UTTypeIdentifier</key>
|
||||
<string>dev.rzen.indie.Notes.backup</string>
|
||||
<key>UTTypeTagSpecification</key>
|
||||
<dict>
|
||||
<key>public.filename-extension</key>
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
<dict>
|
||||
<key>com.apple.security.app-sandbox</key>
|
||||
<true/>
|
||||
<key>com.apple.security.device.audio-input</key>
|
||||
<true/>
|
||||
<key>com.apple.security.files.user-selected.read-write</key>
|
||||
<true/>
|
||||
<key>com.apple.security.network.client</key>
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import IndieSync
|
||||
|
||||
/// Records a single voice note into a temporary `<ULID>.m4a` (AAC mono
|
||||
/// 44.1 kHz ~48 kbps). Modeled on `ContextService`: `@Observable @MainActor`,
|
||||
/// so the UI observes `isRecording` / `elapsed` / `level` directly.
|
||||
///
|
||||
/// Never records into the iCloud container — the file is written to the temp
|
||||
/// directory and handed up as a URL; `SyncEngine.createAudioNote` is what copies
|
||||
/// the finished bytes into the container (audio before JSON). `AVAudioRecorder`
|
||||
/// is not Sendable, so it stays confined to the main actor for its whole life.
|
||||
@Observable
|
||||
@MainActor
|
||||
final class AudioRecorderService {
|
||||
/// One finished recording: the temp file and how long it runs. The caller
|
||||
/// owns the file from here on (moves its bytes into the container, then
|
||||
/// discards the temp copy).
|
||||
struct Recording: Equatable {
|
||||
let url: URL
|
||||
let duration: TimeInterval
|
||||
}
|
||||
|
||||
private(set) var isRecording = false
|
||||
/// Seconds recorded so far, updated by the metering loop while recording.
|
||||
private(set) var elapsed: TimeInterval = 0
|
||||
/// Normalized input level (0…1) for the level-bar UI; 0 when idle.
|
||||
private(set) var level: Float = 0
|
||||
|
||||
/// Fired when a system interruption (phone call, Siri) forces recording to
|
||||
/// stop. The partial file is finalized and handed up rather than lost, so the
|
||||
/// UI can move straight to the "recorded" state. nil unless a recorder wires
|
||||
/// it up for the duration of a take.
|
||||
var onInterrupted: ((Recording) -> Void)?
|
||||
|
||||
private var recorder: AVAudioRecorder?
|
||||
private var meterTask: Task<Void, Never>?
|
||||
private var currentURL: URL?
|
||||
|
||||
#if os(iOS)
|
||||
private var interruptionObserver: (any NSObjectProtocol)?
|
||||
#endif
|
||||
|
||||
/// Ask for microphone access, returning whether it was granted. Safe to call
|
||||
/// every time recording starts — the system only prompts once.
|
||||
func requestPermission() async -> Bool {
|
||||
await AVAudioApplication.requestRecordPermission()
|
||||
}
|
||||
|
||||
/// Whether the mic has been explicitly denied (so the UI can show a jump to
|
||||
/// Settings instead of a dead record button).
|
||||
var permissionDenied: Bool {
|
||||
AVAudioApplication.shared.recordPermission == .denied
|
||||
}
|
||||
|
||||
/// Begin recording into a fresh temp file. Throws if the audio session or
|
||||
/// recorder can't be configured (surfaced by the UI). A no-op if already
|
||||
/// recording.
|
||||
func startRecording() throws {
|
||||
guard !isRecording else { return }
|
||||
|
||||
#if os(iOS)
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try session.setCategory(.playAndRecord, mode: .spokenAudio, options: [.duckOthers])
|
||||
try session.setActive(true)
|
||||
observeInterruptions()
|
||||
#endif
|
||||
|
||||
let url = FileManager.default.temporaryDirectory
|
||||
.appending(path: "\(ULID().stringValue).m4a")
|
||||
let settings: [String: Any] = [
|
||||
AVFormatIDKey: kAudioFormatMPEG4AAC,
|
||||
AVSampleRateKey: 44_100.0,
|
||||
AVNumberOfChannelsKey: 1,
|
||||
AVEncoderBitRateKey: 48_000,
|
||||
AVEncoderAudioQualityKey: AVAudioQuality.medium.rawValue,
|
||||
]
|
||||
|
||||
let recorder = try AVAudioRecorder(url: url, settings: settings)
|
||||
recorder.isMeteringEnabled = true
|
||||
guard recorder.record() else {
|
||||
throw RecorderError.couldNotStart
|
||||
}
|
||||
self.recorder = recorder
|
||||
self.currentURL = url
|
||||
isRecording = true
|
||||
elapsed = 0
|
||||
level = 0
|
||||
startMetering()
|
||||
}
|
||||
|
||||
/// Stop recording and return the finished file plus its duration, or nil if
|
||||
/// nothing was recording. Deactivates the audio session on iOS.
|
||||
@discardableResult
|
||||
func stopRecording() -> Recording? {
|
||||
guard let recorder, let url = currentURL else { return nil }
|
||||
let duration = recorder.currentTime
|
||||
recorder.stop()
|
||||
teardown()
|
||||
return Recording(url: url, duration: duration)
|
||||
}
|
||||
|
||||
/// Abandon the in-progress (or just-finished) recording and delete its temp
|
||||
/// file. Used by "Discard" and when the composer is dismissed unsaved.
|
||||
func discard() {
|
||||
recorder?.stop()
|
||||
if let url = currentURL {
|
||||
try? FileManager.default.removeItem(at: url)
|
||||
}
|
||||
teardown()
|
||||
}
|
||||
|
||||
// MARK: - Metering
|
||||
|
||||
/// ~30 Hz loop that refreshes `elapsed` and `level` while recording. Runs on
|
||||
/// the main actor (a plain `Task` inherits this actor), so it can touch the
|
||||
/// non-Sendable recorder safely.
|
||||
private func startMetering() {
|
||||
meterTask?.cancel()
|
||||
meterTask = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self, let recorder = self.recorder, recorder.isRecording else { return }
|
||||
recorder.updateMeters()
|
||||
self.elapsed = recorder.currentTime
|
||||
self.level = Self.normalizedLevel(recorder.averagePower(forChannel: 0))
|
||||
try? await Task.sleep(for: .milliseconds(33))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a dBFS reading (roughly -60…0) to a 0…1 bar height.
|
||||
private static func normalizedLevel(_ power: Float) -> Float {
|
||||
let floorDB: Float = -50
|
||||
guard power > floorDB else { return 0 }
|
||||
return min(1, (power - floorDB) / -floorDB)
|
||||
}
|
||||
|
||||
private func teardown() {
|
||||
meterTask?.cancel()
|
||||
meterTask = nil
|
||||
recorder = nil
|
||||
currentURL = nil
|
||||
isRecording = false
|
||||
elapsed = 0
|
||||
level = 0
|
||||
#if os(iOS)
|
||||
if let interruptionObserver {
|
||||
NotificationCenter.default.removeObserver(interruptionObserver)
|
||||
self.interruptionObserver = nil
|
||||
}
|
||||
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||
#endif
|
||||
}
|
||||
|
||||
// MARK: - Interruptions (iOS)
|
||||
|
||||
#if os(iOS)
|
||||
private func observeInterruptions() {
|
||||
guard interruptionObserver == nil else { return }
|
||||
interruptionObserver = NotificationCenter.default.addObserver(
|
||||
forName: AVAudioSession.interruptionNotification,
|
||||
object: AVAudioSession.sharedInstance(),
|
||||
queue: .main
|
||||
) { [weak self] note in
|
||||
// Extract the Sendable interruption type before hopping — the
|
||||
// Notification itself isn't Sendable. Delivered on the main queue, so
|
||||
// assuming main-actor isolation is sound.
|
||||
let began = (note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt)
|
||||
.flatMap(AVAudioSession.InterruptionType.init(rawValue:)) == .began
|
||||
guard began else { return }
|
||||
MainActor.assumeIsolated {
|
||||
self?.handleInterruptionBegan()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleInterruptionBegan() {
|
||||
guard isRecording else { return }
|
||||
// Finalize rather than drop the take — the partial recording is still a
|
||||
// valid note.
|
||||
if let finished = stopRecording() {
|
||||
onInterrupted?(finished)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
enum RecorderError: Error {
|
||||
case couldNotStart
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import Foundation
|
||||
|
||||
/// Stable per-install identifier — a UUID minted once on first access and kept
|
||||
/// in UserDefaults for the life of the install. Distinct from the iCloud
|
||||
/// account: two devices signed into the same account get different install IDs.
|
||||
///
|
||||
/// It answers "which device owns a pending transcription?" A voice note stamps
|
||||
/// the recording device's `installID` into `AudioInfo.transcriberDeviceID`; only
|
||||
/// the device whose `installID` matches transcribes it, while the others render
|
||||
/// status only. Re-transcribing on any device reassigns the id to itself, which
|
||||
/// is also the recovery path when the original recording device is gone.
|
||||
enum InstallIdentity {
|
||||
private static let key = "Notes.installID"
|
||||
|
||||
/// The per-install UUID, created lazily on first read and reused forever.
|
||||
static var installID: String {
|
||||
if let existing = UserDefaults.standard.string(forKey: key) {
|
||||
return existing
|
||||
}
|
||||
let fresh = UUID().uuidString
|
||||
UserDefaults.standard.set(fresh, forKey: key)
|
||||
return fresh
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import Foundation
|
||||
import Speech
|
||||
|
||||
/// One language's showing in the sampling pass: how well `SpeechTranscriber`
|
||||
/// transcribed the first few seconds of audio in that locale.
|
||||
struct LanguageCandidate: Equatable {
|
||||
/// BCP-47 identifier of the locale (e.g. `en-US`, `ru-RU`).
|
||||
let localeID: String
|
||||
/// Character-count-weighted mean of the per-run transcription confidence,
|
||||
/// 0…1. A locale that produced no text scores 0.
|
||||
let meanConfidence: Double
|
||||
/// Number of characters the transcript came out to — the tie-breaker and the
|
||||
/// reason an empty transcript can't win.
|
||||
let transcriptLength: Int
|
||||
}
|
||||
|
||||
/// Pure, testable language selection for a voice note. No `SpeechAnalyzer` calls
|
||||
/// live here: the service transcribes a ~15 s sample in each enabled language,
|
||||
/// turns each into a `LanguageCandidate`, and this picks the winner. Keeping it
|
||||
/// side-effect-free is what makes the (undocumented, cross-locale) confidence
|
||||
/// calibration unit-testable.
|
||||
enum LanguagePicker {
|
||||
/// Character-count-weighted mean confidence over an attributed transcript.
|
||||
/// Only runs that actually carry a `transcriptionConfidence` attribute
|
||||
/// contribute, weighted by their character length; a transcript with no
|
||||
/// confidence anywhere (or no characters) scores 0.
|
||||
static func meanConfidence(of attributed: AttributedString) -> Double {
|
||||
var weighted = 0.0
|
||||
var totalCharacters = 0
|
||||
for run in attributed.runs {
|
||||
guard let confidence = run.transcriptionConfidence else { continue }
|
||||
let length = attributed[run.range].characters.count
|
||||
guard length > 0 else { continue }
|
||||
weighted += confidence * Double(length)
|
||||
totalCharacters += length
|
||||
}
|
||||
return totalCharacters > 0 ? weighted / Double(totalCharacters) : 0
|
||||
}
|
||||
|
||||
/// The best candidate, or nil when given nothing. Argmax on `meanConfidence`;
|
||||
/// a zero-length transcript can only win if every candidate is empty. Ties
|
||||
/// break deterministically — higher confidence, then longer transcript, then
|
||||
/// lexicographically smaller `localeID` — so the same samples always yield the
|
||||
/// same pick.
|
||||
static func pick(from candidates: [LanguageCandidate]) -> LanguageCandidate? {
|
||||
candidates.max { lhs, rhs in
|
||||
if lhs.meanConfidence != rhs.meanConfidence {
|
||||
return lhs.meanConfidence < rhs.meanConfidence
|
||||
}
|
||||
if lhs.transcriptLength != rhs.transcriptLength {
|
||||
return lhs.transcriptLength < rhs.transcriptLength
|
||||
}
|
||||
// `max` keeps the greater element; invert so the smaller localeID wins.
|
||||
return lhs.localeID > rhs.localeID
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import IndieSync
|
||||
import Speech
|
||||
import SwiftData
|
||||
|
||||
/// On-device, after-the-fact transcription of voice notes. `@Observable
|
||||
/// @MainActor` with a FIFO one-at-a-time queue: a `SpeechAnalyzer` is memory- and
|
||||
/// model-heavy, so notes are transcribed strictly one after another. The analyzer
|
||||
/// work is awaited (the analyzer is an actor), so it doesn't block the main actor
|
||||
/// despite this class being main-actor confined.
|
||||
///
|
||||
/// Only this device's own pending notes are ever enqueued — ownership is decided
|
||||
/// by `AudioInfo.transcriberDeviceID` upstream (`SyncEngine`), which this service
|
||||
/// trusts. A failed job is terminal until the user asks to retry.
|
||||
@Observable
|
||||
@MainActor
|
||||
final class TranscriptionService {
|
||||
private let syncEngine: SyncEngine
|
||||
private let settings: TranscriptionSettings
|
||||
private let modelContainer: ModelContainer
|
||||
|
||||
/// Seconds of audio the sampling pass listens to when choosing a language.
|
||||
private static let sampleSeconds: Double = 15
|
||||
|
||||
private struct Job {
|
||||
let noteID: ULID
|
||||
/// A fresh recording's temp file, passed on enqueue for the fast path; nil
|
||||
/// for re-transcription, which pulls the audio back down from the container.
|
||||
let localAudioURL: URL?
|
||||
/// A forced language (re-transcribe menu), else auto-select.
|
||||
let forcedLocaleID: String?
|
||||
}
|
||||
|
||||
private var queue: [Job] = []
|
||||
/// Ids currently queued or in flight, so overlapping triggers (launch resume,
|
||||
/// watch ingest, a re-transcribe) can't enqueue the same note twice.
|
||||
private var inFlightIDs: Set<String> = []
|
||||
private var pumpTask: Task<Void, Never>?
|
||||
|
||||
init(syncEngine: SyncEngine, settings: TranscriptionSettings, modelContainer: ModelContainer) {
|
||||
self.syncEngine = syncEngine
|
||||
self.settings = settings
|
||||
self.modelContainer = modelContainer
|
||||
}
|
||||
|
||||
// MARK: - Queue
|
||||
|
||||
/// Queue a note for transcription. `localAudioURL` is the fresh-recording fast
|
||||
/// path (skips the container round-trip); `forcedLocaleID` skips language
|
||||
/// auto-selection. Duplicate enqueues of an in-flight note are ignored.
|
||||
func enqueue(noteID: ULID, localAudioURL: URL? = nil, forcedLocaleID: String? = nil) {
|
||||
let key = noteID.stringValue
|
||||
guard !inFlightIDs.contains(key) else { return }
|
||||
inFlightIDs.insert(key)
|
||||
queue.append(Job(noteID: noteID, localAudioURL: localAudioURL, forcedLocaleID: forcedLocaleID))
|
||||
pump()
|
||||
}
|
||||
|
||||
/// On launch, re-enqueue every note this device owns that is still pending —
|
||||
/// recovering work that a crash or kill interrupted mid-transcription. Reads
|
||||
/// the cache (already rebuilt by the sync engine's connect).
|
||||
func resumePendingWork() {
|
||||
let context = modelContainer.mainContext
|
||||
let installID = TranscriptionSettings.installID
|
||||
let pendingRaw = Note.TranscriptionStatus.pending.rawValue
|
||||
let descriptor = FetchDescriptor<NoteEntity>(predicate: #Predicate {
|
||||
$0.hasAudio
|
||||
&& $0.transcriptionStatusRaw == pendingRaw
|
||||
&& $0.transcriberDeviceID == installID
|
||||
})
|
||||
let rows = (try? context.fetch(descriptor)) ?? []
|
||||
for row in rows {
|
||||
if let id = ULID(string: row.id) { enqueue(noteID: id) }
|
||||
}
|
||||
}
|
||||
|
||||
private func pump() {
|
||||
guard pumpTask == nil else { return }
|
||||
pumpTask = Task { [weak self] in
|
||||
while let self, let job = self.dequeue() {
|
||||
await self.process(job)
|
||||
self.inFlightIDs.remove(job.noteID.stringValue)
|
||||
}
|
||||
self?.pumpTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
private func dequeue() -> Job? {
|
||||
queue.isEmpty ? nil : queue.removeFirst()
|
||||
}
|
||||
|
||||
// MARK: - Processing
|
||||
|
||||
/// Transcribe one job, retrying once before marking the note failed. A failed
|
||||
/// note is left for the user to retry explicitly (re-transcribe / Retry).
|
||||
private func process(_ job: Job) async {
|
||||
do {
|
||||
try await transcribe(job)
|
||||
} catch {
|
||||
do {
|
||||
try await transcribe(job)
|
||||
} catch {
|
||||
print("[Transcription] \(job.noteID.stringValue) failed: \(error)")
|
||||
try? await syncEngine.markTranscriptionFailed(id: job.noteID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func transcribe(_ job: Job) async throws {
|
||||
let audio = try await resolveAudioFile(for: job)
|
||||
defer { cleanUp(audio: audio, job: job) }
|
||||
|
||||
let locale = try await selectLocale(fileURL: audio.url, forcedLocaleID: job.forcedLocaleID)
|
||||
let transcript = try await runTranscription(fileURL: audio.url, locale: locale)
|
||||
let text = String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
try await syncEngine.applyTranscription(
|
||||
id: job.noteID,
|
||||
text: text,
|
||||
localeID: TranscriptionSettings.canonicalID(locale)
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Audio acquisition
|
||||
|
||||
private struct AudioFile {
|
||||
let url: URL
|
||||
/// Whether we wrote a temp copy that must be cleaned up after the job.
|
||||
let isTemporary: Bool
|
||||
}
|
||||
|
||||
/// The fresh recording if it's still on disk, else a temp copy pulled back
|
||||
/// from the container (bounded wait for an evicted file).
|
||||
private func resolveAudioFile(for job: Job) async throws -> AudioFile {
|
||||
if let local = job.localAudioURL, FileManager.default.fileExists(atPath: local.path) {
|
||||
return AudioFile(url: local, isTemporary: false)
|
||||
}
|
||||
let data = try await syncEngine.loadAudioDataWaiting(for: job.noteID)
|
||||
let url = FileManager.default.temporaryDirectory
|
||||
.appending(path: "transcribe-\(job.noteID.stringValue).m4a")
|
||||
try data.write(to: url)
|
||||
return AudioFile(url: url, isTemporary: true)
|
||||
}
|
||||
|
||||
private func cleanUp(audio: AudioFile, job: Job) {
|
||||
let fm = FileManager.default
|
||||
if audio.isTemporary { try? fm.removeItem(at: audio.url) }
|
||||
// A fresh recording's temp file is no longer needed once the container
|
||||
// holds the bytes; drop it to keep the temp dir tidy.
|
||||
if let local = job.localAudioURL,
|
||||
local.path.hasPrefix(fm.temporaryDirectory.path) {
|
||||
try? fm.removeItem(at: local)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Language selection
|
||||
|
||||
private func selectLocale(fileURL: URL, forcedLocaleID: String?) async throws -> Locale {
|
||||
if let forcedLocaleID {
|
||||
return Locale(identifier: forcedLocaleID)
|
||||
}
|
||||
let candidates = await settings.enabledInstalledLocales()
|
||||
guard !candidates.isEmpty else { throw TranscriptionError.noInstalledLanguage }
|
||||
if candidates.count == 1 { return candidates[0] }
|
||||
|
||||
// Sampling pass: transcribe a short clip in each language and pick the
|
||||
// most confident. Run sequentially — one analyzer at a time keeps the
|
||||
// memory ceiling low.
|
||||
let sample = try makeSampleClip(from: fileURL, seconds: Self.sampleSeconds)
|
||||
defer { if sample.isTemporary { try? FileManager.default.removeItem(at: sample.url) } }
|
||||
|
||||
var scored: [LanguageCandidate] = []
|
||||
for locale in candidates {
|
||||
let transcript = (try? await runTranscription(fileURL: sample.url, locale: locale)) ?? AttributedString()
|
||||
scored.append(LanguageCandidate(
|
||||
localeID: TranscriptionSettings.canonicalID(locale),
|
||||
meanConfidence: LanguagePicker.meanConfidence(of: transcript),
|
||||
transcriptLength: transcript.characters.count
|
||||
))
|
||||
}
|
||||
guard let winner = LanguagePicker.pick(from: scored) else { return candidates[0] }
|
||||
return Locale(identifier: winner.localeID)
|
||||
}
|
||||
|
||||
/// The first `seconds` of the recording as its own file, so the sampling pass
|
||||
/// isn't run over a whole long note. Returns the original file (no copy) when
|
||||
/// the recording is already shorter than the sample window.
|
||||
private func makeSampleClip(from fileURL: URL, seconds: Double) throws -> AudioFile {
|
||||
let source = try AVAudioFile(forReading: fileURL)
|
||||
let format = source.processingFormat
|
||||
let wantFrames = format.sampleRate * seconds
|
||||
guard Double(source.length) > wantFrames else {
|
||||
return AudioFile(url: fileURL, isTemporary: false)
|
||||
}
|
||||
let frameCount = AVAudioFrameCount(wantFrames)
|
||||
guard frameCount > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) else {
|
||||
throw TranscriptionError.audioUnreadable
|
||||
}
|
||||
try source.read(into: buffer, frameCount: frameCount)
|
||||
let clipURL = FileManager.default.temporaryDirectory
|
||||
.appending(path: "sample-\(UUID().uuidString).caf")
|
||||
let clip = try AVAudioFile(forWriting: clipURL, settings: format.settings)
|
||||
try clip.write(from: buffer)
|
||||
return AudioFile(url: clipURL, isTemporary: true)
|
||||
}
|
||||
|
||||
// MARK: - Analyzer
|
||||
|
||||
/// Transcribe a whole file in one locale and return the finalized attributed
|
||||
/// transcript (carrying per-run confidence). Follows the documented pattern:
|
||||
/// start a collector over `transcriber.results`, feed the file, finalize.
|
||||
private func runTranscription(fileURL: URL, locale: Locale) async throws -> AttributedString {
|
||||
let transcriber = SpeechTranscriber(
|
||||
locale: locale,
|
||||
transcriptionOptions: [],
|
||||
reportingOptions: [],
|
||||
attributeOptions: [.transcriptionConfidence]
|
||||
)
|
||||
let analyzer = SpeechAnalyzer(
|
||||
modules: [transcriber],
|
||||
options: SpeechAnalyzer.Options(priority: .utility, modelRetention: .whileInUse)
|
||||
)
|
||||
let file = try AVAudioFile(forReading: fileURL)
|
||||
|
||||
// Collect finalized results concurrently; `results` terminates when the
|
||||
// analyzer finishes, ending the loop.
|
||||
let collector = Task {
|
||||
var full = AttributedString()
|
||||
for try await result in transcriber.results where result.isFinal {
|
||||
full.append(result.text)
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
do {
|
||||
if let lastSample = try await analyzer.analyzeSequence(from: file) {
|
||||
try await analyzer.finalizeAndFinish(through: lastSample)
|
||||
} else {
|
||||
await analyzer.cancelAndFinishNow()
|
||||
}
|
||||
} catch {
|
||||
collector.cancel()
|
||||
throw error
|
||||
}
|
||||
return try await collector.value
|
||||
}
|
||||
|
||||
enum TranscriptionError: Error {
|
||||
case audioUnreadable
|
||||
case noInstalledLanguage
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
import Foundation
|
||||
import Speech
|
||||
|
||||
/// The user's transcription-language configuration and the Speech asset state
|
||||
/// behind it. `@Observable @MainActor` so the Settings UI binds to it directly.
|
||||
///
|
||||
/// There is no automatic language detection in `SpeechAnalyzer`, so the user
|
||||
/// enables a small set of languages; a new voice note is sampled in each and the
|
||||
/// highest-confidence one wins (see `TranscriptionService`). Enabling a language
|
||||
/// *reserves* it with `AssetInventory` — which is both what keeps its model from
|
||||
/// being purged and what the system caps at `AssetInventory.maximumReservedLocales`
|
||||
/// — so "enabled" and "reserved" are kept in lock-step and the UI enforces the cap.
|
||||
@Observable
|
||||
@MainActor
|
||||
final class TranscriptionSettings {
|
||||
private static let enabledKey = "Notes.enabledTranscriptionLocales"
|
||||
private static let seededKey = "Notes.transcriptionLocalesSeeded"
|
||||
|
||||
/// Per-install id of the transcription owner, reused from `InstallIdentity`
|
||||
/// — a voice note stamps this into `AudioInfo.transcriberDeviceID` so only the
|
||||
/// recording device transcribes it.
|
||||
static var installID: String { InstallIdentity.installID }
|
||||
|
||||
/// Canonical BCP-47 ids of the enabled languages, persisted in UserDefaults.
|
||||
private(set) var enabledLocaleIDs: [String]
|
||||
|
||||
/// Every locale `SpeechTranscriber` can transcribe (36+), loaded once by
|
||||
/// `prepare()`. Empty until then.
|
||||
private(set) var supportedLocales: [Locale] = []
|
||||
|
||||
/// Runtime cap on how many languages can be enabled at once — read from the
|
||||
/// system, not hard-coded, because it can change.
|
||||
private(set) var maximumReserved = 0
|
||||
|
||||
/// Per-language asset state, keyed by canonical BCP-47 id, for the row UI.
|
||||
private(set) var assetStates: [String: AssetState] = [:]
|
||||
|
||||
/// Where a language's on-device model sits.
|
||||
enum AssetState: Equatable {
|
||||
case unsupported
|
||||
case notInstalled
|
||||
/// Downloading; the fraction is nil until progress is known.
|
||||
case downloading(Double?)
|
||||
case installed
|
||||
}
|
||||
|
||||
private let defaults: UserDefaults
|
||||
|
||||
init(defaults: UserDefaults = .standard) {
|
||||
self.defaults = defaults
|
||||
self.enabledLocaleIDs = defaults.stringArray(forKey: Self.enabledKey) ?? []
|
||||
}
|
||||
|
||||
// MARK: - Canonical ids
|
||||
|
||||
/// The stable key for a locale everywhere in the app: its BCP-47 identifier
|
||||
/// (`en-US`), so it round-trips through `Locale(identifier:)`.
|
||||
static func canonicalID(_ locale: Locale) -> String { locale.identifier(.bcp47) }
|
||||
|
||||
/// The enabled languages as `Locale`s, in the order the user has them.
|
||||
var enabledLocales: [Locale] { enabledLocaleIDs.map(Locale.init(identifier:)) }
|
||||
|
||||
/// Localized display name for a locale ("English (United States)"), falling
|
||||
/// back to the raw identifier.
|
||||
static func displayName(for locale: Locale) -> String {
|
||||
Locale.current.localizedString(forIdentifier: locale.identifier(.bcp47))
|
||||
?? locale.identifier(.bcp47)
|
||||
}
|
||||
|
||||
// MARK: - Loading
|
||||
|
||||
/// One-time load of the supported-locale list and the reservation cap, plus a
|
||||
/// first-run seed of the enabled set from the user's preferred languages. Idempotent
|
||||
/// and cheap to call again — it re-fetches the list but only seeds once.
|
||||
func prepare() async {
|
||||
maximumReserved = AssetInventory.maximumReservedLocales
|
||||
supportedLocales = await SpeechTranscriber.supportedLocales
|
||||
.sorted { Self.displayName(for: $0) < Self.displayName(for: $1) }
|
||||
|
||||
if !defaults.bool(forKey: Self.seededKey) {
|
||||
seedDefaults()
|
||||
defaults.set(true, forKey: Self.seededKey)
|
||||
}
|
||||
await refreshStates()
|
||||
}
|
||||
|
||||
/// Seed the enabled set from `Locale.preferredLanguages ∩ supportedLocales`,
|
||||
/// matched by language code and capped at the reservation limit. Falls back to
|
||||
/// English (or the first supported locale) so a note recorded before the user
|
||||
/// visits Settings still has something to transcribe with.
|
||||
private func seedDefaults() {
|
||||
guard enabledLocaleIDs.isEmpty else { return }
|
||||
var seed: [String] = []
|
||||
for preferred in Locale.preferredLanguages.map(Locale.init(identifier:)) {
|
||||
guard let match = supportedLocales.first(where: {
|
||||
$0.language.languageCode == preferred.language.languageCode
|
||||
}) else { continue }
|
||||
let id = Self.canonicalID(match)
|
||||
if !seed.contains(id) { seed.append(id) }
|
||||
}
|
||||
if seed.isEmpty {
|
||||
let fallback = supportedLocales.first { $0.language.languageCode?.identifier == "en" }
|
||||
?? supportedLocales.first
|
||||
if let fallback { seed = [Self.canonicalID(fallback)] }
|
||||
}
|
||||
enabledLocaleIDs = Array(seed.prefix(max(1, maximumReserved)))
|
||||
persist()
|
||||
// Reserve the seeded languages so their models are protected from purge.
|
||||
Task { await self.reserveEnabled() }
|
||||
}
|
||||
|
||||
private func reserveEnabled() async {
|
||||
for locale in enabledLocales {
|
||||
_ = try? await AssetInventory.reserve(locale: locale)
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh every supported locale's asset state (installed / downloadable /
|
||||
/// unsupported). Runs sequentially — `status(forModules:)` is cheap but there
|
||||
/// are dozens of locales, and there's no need to fan out.
|
||||
func refreshStates() async {
|
||||
for locale in supportedLocales {
|
||||
let id = Self.canonicalID(locale)
|
||||
// Preserve an in-flight download's fraction rather than stomping it.
|
||||
if case .downloading = assetStates[id] { continue }
|
||||
assetStates[id] = await currentState(for: locale)
|
||||
}
|
||||
}
|
||||
|
||||
private func currentState(for locale: Locale) async -> AssetState {
|
||||
switch await AssetInventory.status(forModules: [Self.transcriber(for: locale)]) {
|
||||
case .unsupported: return .unsupported
|
||||
case .supported: return .notInstalled
|
||||
case .downloading: return .downloading(nil)
|
||||
case .installed: return .installed
|
||||
@unknown default: return .notInstalled
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Enable / disable
|
||||
|
||||
var isAtCapacity: Bool { enabledLocaleIDs.count >= maximumReserved }
|
||||
|
||||
func isEnabled(_ locale: Locale) -> Bool {
|
||||
enabledLocaleIDs.contains(Self.canonicalID(locale))
|
||||
}
|
||||
|
||||
/// Enable a language: reserve it, mark it enabled, and download its model if
|
||||
/// needed (surfacing progress through `assetStates`). Throws `atCapacity` when
|
||||
/// already at the reservation cap.
|
||||
func enable(_ locale: Locale) async throws {
|
||||
let id = Self.canonicalID(locale)
|
||||
guard !enabledLocaleIDs.contains(id) else { return }
|
||||
guard !isAtCapacity else { throw SettingsError.atCapacity }
|
||||
|
||||
_ = try await AssetInventory.reserve(locale: locale)
|
||||
enabledLocaleIDs.append(id)
|
||||
persist()
|
||||
try await installIfNeeded(locale)
|
||||
}
|
||||
|
||||
/// Disable a language: drop it from the enabled set and release its
|
||||
/// reservation (letting the system reclaim the model later). Best-effort.
|
||||
func disable(_ locale: Locale) async {
|
||||
let id = Self.canonicalID(locale)
|
||||
enabledLocaleIDs.removeAll { $0 == id }
|
||||
persist()
|
||||
_ = await AssetInventory.release(reservedLocale: locale)
|
||||
assetStates[id] = await currentState(for: locale)
|
||||
}
|
||||
|
||||
private func installIfNeeded(_ locale: Locale) async throws {
|
||||
let id = Self.canonicalID(locale)
|
||||
let transcriber = Self.transcriber(for: locale)
|
||||
guard await AssetInventory.status(forModules: [transcriber]) != .installed else {
|
||||
assetStates[id] = .installed
|
||||
return
|
||||
}
|
||||
guard let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else {
|
||||
assetStates[id] = await currentState(for: locale)
|
||||
return
|
||||
}
|
||||
let progress = request.progress
|
||||
assetStates[id] = .downloading(0)
|
||||
// Poll the Progress so the @Observable dict updates the row; Progress KVO
|
||||
// doesn't drive SwiftUI on its own.
|
||||
let poll = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(for: .milliseconds(300))
|
||||
guard let self else { return }
|
||||
self.assetStates[id] = .downloading(progress.fractionCompleted)
|
||||
if progress.isFinished { return }
|
||||
}
|
||||
}
|
||||
defer { poll.cancel() }
|
||||
try await request.downloadAndInstall()
|
||||
assetStates[id] = await currentState(for: locale)
|
||||
}
|
||||
|
||||
/// The subset of enabled languages whose models are installed — the only ones
|
||||
/// `TranscriptionService` can actually run.
|
||||
func enabledInstalledLocales() async -> [Locale] {
|
||||
var installed: [Locale] = []
|
||||
for locale in enabledLocales
|
||||
where await AssetInventory.status(forModules: [Self.transcriber(for: locale)]) == .installed {
|
||||
installed.append(locale)
|
||||
}
|
||||
return installed
|
||||
}
|
||||
|
||||
// MARK: - Helpers
|
||||
|
||||
/// A minimal transcriber for a locale, used purely for asset queries. The
|
||||
/// attribute options don't affect asset status, so they're omitted here.
|
||||
static func transcriber(for locale: Locale) -> SpeechTranscriber {
|
||||
SpeechTranscriber(
|
||||
locale: locale,
|
||||
transcriptionOptions: [],
|
||||
reportingOptions: [],
|
||||
attributeOptions: []
|
||||
)
|
||||
}
|
||||
|
||||
private func persist() {
|
||||
defaults.set(enabledLocaleIDs, forKey: Self.enabledKey)
|
||||
}
|
||||
|
||||
enum SettingsError: Error {
|
||||
case atCapacity
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,12 @@ final class SyncEngine {
|
||||
/// the next successful write or rebuild.
|
||||
private(set) var lastSyncError: String?
|
||||
|
||||
/// Fired after a watch recording is ingested, carrying the new note's id, so
|
||||
/// the composition root can enqueue on-device transcription. Wired by
|
||||
/// `AppServices`; nil until then, so ingestion is safe before the
|
||||
/// transcription stack exists.
|
||||
var onAudioNoteIngested: ((ULID) -> Void)?
|
||||
|
||||
static let recordsDirectory = "Records"
|
||||
static let stubsDirectory = "Stubs"
|
||||
|
||||
@@ -54,6 +60,10 @@ final class SyncEngine {
|
||||
private var dataMonitorTask: Task<Void, Never>?
|
||||
private var stubMonitorTask: Task<Void, Never>?
|
||||
|
||||
/// Guards `ingestStagedWatchRecordings` against overlapping triggers (connect,
|
||||
/// foreground reconcile, receiver callback) double-ingesting the same inbox.
|
||||
private var isDrainingInbox = false
|
||||
|
||||
private var cacheContext: ModelContext { modelContainer.mainContext }
|
||||
|
||||
init(modelContainer: ModelContainer) {
|
||||
@@ -142,6 +152,12 @@ final class SyncEngine {
|
||||
}
|
||||
let path = Note.relativePath(forID: id)
|
||||
try await tombstones.softDelete(dataRelativePath: Self.dataPath(path), at: path)
|
||||
// Drop the audio sidecar too. Best-effort and harmless when the note had
|
||||
// no audio (nothing to remove); the sidecar never gets its own stub, so
|
||||
// this is the only thing that reaps it.
|
||||
if let store {
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
}
|
||||
lastSyncError = nil
|
||||
// Cache cleanup happens via the monitors (file removal + stub
|
||||
// appearance both drop the row), but do it eagerly too so the UI
|
||||
@@ -163,6 +179,273 @@ final class SyncEngine {
|
||||
return NoteMapper.note(from: entity)
|
||||
}
|
||||
|
||||
// MARK: - Audio notes
|
||||
|
||||
/// Create a voice note: write the `.m4a` sidecar **before** the JSON, so a
|
||||
/// note that claims audio can never sync to another device ahead of its
|
||||
/// bytes (which would surface a pending audio note whose `.m4a` doesn't yet
|
||||
/// exist). The note starts with empty (or provisional) text and a `.pending`
|
||||
/// transcription owned by `transcriberDeviceID`; the transcript arrives later
|
||||
/// through `applyTranscription`. Like every other write, the cache catches up
|
||||
/// via the monitor — no direct upsert here.
|
||||
@discardableResult
|
||||
func createAudioNote(
|
||||
audioFileURL: URL,
|
||||
duration: TimeInterval,
|
||||
text: String = "",
|
||||
tags: [String] = [],
|
||||
context: CaptureContext,
|
||||
transcriberDeviceID: String
|
||||
) async throws -> Note {
|
||||
guard let store else {
|
||||
report("Note not saved — iCloud not connected")
|
||||
throw SyncError.iCloudUnavailable
|
||||
}
|
||||
let audio = Note.AudioInfo(
|
||||
duration: duration,
|
||||
transcriberDeviceID: transcriberDeviceID,
|
||||
transcriptionStatus: .pending,
|
||||
transcriptionLocaleID: nil,
|
||||
transcribedAt: nil
|
||||
)
|
||||
let note = Note.create(text: text, source: .transcribed, tags: tags, context: context, audio: audio)
|
||||
do {
|
||||
let audioData = try Data(contentsOf: audioFileURL)
|
||||
try await store.writeData(audioData, to: Self.dataPath(Note.audioRelativePath(forID: note.id)))
|
||||
_ = try await store.write(note, to: Self.dataPath(note.relativePath))
|
||||
} catch {
|
||||
report("Failed to save audio note", error)
|
||||
throw error
|
||||
}
|
||||
lastSyncError = nil
|
||||
return note
|
||||
}
|
||||
|
||||
/// Load a note's audio bytes for playback. Fail-fast (`.requestOnly`): an
|
||||
/// evicted file kicks off a download and returns nil immediately rather than
|
||||
/// stalling the player ~30s — the UI shows a "downloading" retry state and
|
||||
/// the bytes land on a later attempt (or via `requestAudioDownload`).
|
||||
func loadAudioData(for id: ULID) async -> Data? {
|
||||
guard let store else { return nil }
|
||||
return try? await store.readData(from: audioPath(for: id), downloadPolicy: .requestOnly)
|
||||
}
|
||||
|
||||
/// Load a note's audio with a bounded wait for an evicted file to download —
|
||||
/// for re-transcription, where blocking to fetch the source is acceptable
|
||||
/// (unlike interactive playback, which uses `loadAudioData`). Throws on a
|
||||
/// download timeout rather than yielding a dataless placeholder.
|
||||
func loadAudioDataWaiting(for id: ULID) async throws -> Data {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
return try await store.readData(from: audioPath(for: id), downloadPolicy: .waitBounded)
|
||||
}
|
||||
|
||||
/// Warm an evicted audio file ahead of an anticipated playback, so a later
|
||||
/// `loadAudioData` finds the bytes already local. Fire-and-forget.
|
||||
func requestAudioDownload(for id: ULID) async {
|
||||
guard let store else { return }
|
||||
await store.requestDownload(at: audioPath(for: id))
|
||||
}
|
||||
|
||||
/// Store-rooted `.m4a` sidecar path for a note. Audio exists only on v2+
|
||||
/// records, which file under the current UTC bucket, so recomputing the path
|
||||
/// from the id is always correct here.
|
||||
private func audioPath(for id: ULID) -> String {
|
||||
Self.dataPath(Note.audioRelativePath(forID: id))
|
||||
}
|
||||
|
||||
// MARK: - Transcription
|
||||
|
||||
/// Land a completed transcription onto a note. Re-reads the file (the
|
||||
/// authority) so a concurrent edit isn't clobbered, then **appends** rather
|
||||
/// than replaces when the user typed while the note was pending — non-empty
|
||||
/// text whose `modifiedAt` moved past the note's `createdAt`. Otherwise the
|
||||
/// transcript replaces the placeholder-empty text. The cache updates through
|
||||
/// the monitor as usual (via `updateNote`).
|
||||
func applyTranscription(id: ULID, text: String, localeID: String) async throws {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
|
||||
note.isReadable, var audio = note.payload.audio else { return }
|
||||
|
||||
let userEdited = !note.payload.text.isEmpty && note.meta.modifiedAt > note.meta.createdAt
|
||||
note.payload.text = userEdited ? note.payload.text + "\n\n" + text : text
|
||||
audio.transcriptionStatus = .done
|
||||
audio.transcriptionLocaleID = localeID
|
||||
audio.transcribedAt = Date()
|
||||
note.payload.audio = audio
|
||||
try await updateNote(note)
|
||||
}
|
||||
|
||||
/// Flag a note's transcription as failed (retryable only by an explicit
|
||||
/// re-transcribe). Re-reads first so it can't resurrect a since-deleted or
|
||||
/// non-audio note.
|
||||
func markTranscriptionFailed(id: ULID) async throws {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
|
||||
note.isReadable, var audio = note.payload.audio else { return }
|
||||
audio.transcriptionStatus = .failed
|
||||
note.payload.audio = audio
|
||||
try await updateNote(note)
|
||||
}
|
||||
|
||||
/// Reassign a note's transcription to *this* device and reset it to
|
||||
/// `.pending`, optionally forcing a language (`localeID`, else auto-detect).
|
||||
/// Claiming `transcriberDeviceID` is also the recovery path when the original
|
||||
/// recording device is gone — any device can take a stuck note over.
|
||||
func requestRetranscription(id: ULID, localeID: String?) async throws {
|
||||
guard let store else { throw SyncError.iCloudUnavailable }
|
||||
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
|
||||
note.isReadable, var audio = note.payload.audio else { return }
|
||||
audio.transcriptionStatus = .pending
|
||||
audio.transcriberDeviceID = InstallIdentity.installID
|
||||
audio.transcriptionLocaleID = localeID
|
||||
note.payload.audio = audio
|
||||
try await updateNote(note)
|
||||
}
|
||||
|
||||
// MARK: - Watch ingestion
|
||||
|
||||
/// Local directory where the phone receiver (`PhoneWatchReceiver`, a later
|
||||
/// stage) stages recordings handed over from the watch before they are
|
||||
/// ingested. It sits outside the iCloud container on purpose: the watch has
|
||||
/// no iCloud, so a delivered recording becomes durable only once
|
||||
/// `ingestWatchRecording` writes it into the container. Each staged
|
||||
/// recording is a `<ULID>.m4a` plus a `<ULID>.meta.plist` sidecar carrying
|
||||
/// `id` / `recordedAt` / `duration` / `timeZoneID`.
|
||||
/// `nonisolated` so the phone receiver can stage a delivery synchronously in
|
||||
/// its nonisolated `WCSessionDelegate` callback (the system reclaims the inbox
|
||||
/// file the moment that returns). Pure — no actor state — so it is safe to
|
||||
/// share this one source of truth for the inbox path with the drain.
|
||||
nonisolated static var watchInboxURL: URL {
|
||||
URL.applicationSupportDirectory.appending(path: "WatchInbox", directoryHint: .isDirectory)
|
||||
}
|
||||
|
||||
/// Idempotent drain of the watch staging inbox. Called after `connect()`, on
|
||||
/// foreground reconcile, and (a later stage) on the receiver callback, so a
|
||||
/// recording delivered while the app was dead still lands. Serialized against
|
||||
/// itself so overlapping triggers can't double-ingest.
|
||||
func ingestStagedWatchRecordings() async {
|
||||
guard store != nil, !isDrainingInbox else { return }
|
||||
isDrainingInbox = true
|
||||
defer { isDrainingInbox = false }
|
||||
|
||||
for staged in Self.stagedWatchRecordings() {
|
||||
await ingestWatchRecording(
|
||||
audioURL: staged.audioURL,
|
||||
id: staged.id,
|
||||
recordedAt: staged.recordedAt,
|
||||
duration: staged.duration,
|
||||
context: staged.context
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Ingest one recording handed over from the watch — the single sanctioned
|
||||
/// direct-cache-upsert path (icloud-sync-engine §9). The phone is where the
|
||||
/// note first becomes durable, so: veto if the note was already deleted
|
||||
/// (tombstone) or already ingested (duplicate redelivery), then write audio
|
||||
/// **before** JSON (same ordering invariant as `createAudioNote`), upsert the
|
||||
/// cache directly (a WatchConnectivity delivery has no other live trigger to
|
||||
/// refresh an open list), remove the staged files, and fire
|
||||
/// `onAudioNoteIngested` so transcription can enqueue. Returns whether a new
|
||||
/// note was created.
|
||||
@discardableResult
|
||||
func ingestWatchRecording(
|
||||
audioURL: URL,
|
||||
id: ULID,
|
||||
recordedAt: Date,
|
||||
duration: TimeInterval,
|
||||
context: CaptureContext
|
||||
) async -> Bool {
|
||||
guard let store, let tombstones else { return false }
|
||||
|
||||
// Permanent skips — clear the stale staged copy so it stops re-draining.
|
||||
if await tombstones.stubExists(id: id.stringValue) {
|
||||
Self.removeStagedFiles(audioURL: audioURL)
|
||||
return false
|
||||
}
|
||||
let jsonPath = Note.relativePath(forID: id)
|
||||
if await store.fileExists(Self.dataPath(jsonPath)) {
|
||||
Self.removeStagedFiles(audioURL: audioURL)
|
||||
return false
|
||||
}
|
||||
|
||||
let note = Note.createIngested(
|
||||
id: id,
|
||||
duration: duration,
|
||||
transcriberDeviceID: InstallIdentity.installID,
|
||||
context: context
|
||||
)
|
||||
do {
|
||||
let audioData = try Data(contentsOf: audioURL)
|
||||
try await store.writeData(audioData, to: Self.dataPath(Note.audioRelativePath(forID: id)))
|
||||
try await store.write(note, to: Self.dataPath(jsonPath))
|
||||
} catch {
|
||||
// Transient (iCloud write, or the staged file vanished mid-drain):
|
||||
// leave the staged files for the next drain to retry.
|
||||
report("Failed to ingest watch recording \(id.stringValue)", error)
|
||||
return false
|
||||
}
|
||||
|
||||
NoteMapper.upsert(note, relativePath: jsonPath, into: cacheContext)
|
||||
try? cacheContext.save()
|
||||
Self.removeStagedFiles(audioURL: audioURL)
|
||||
lastSyncError = nil
|
||||
|
||||
print("[Sync] Ingested watch recording \(id.stringValue) recorded \(recordedAt)")
|
||||
onAudioNoteIngested?(id)
|
||||
return true
|
||||
}
|
||||
|
||||
/// One recording staged in the watch inbox: the `.m4a` and the metadata the
|
||||
/// receiver wrote in its `<ULID>.meta.plist` sidecar.
|
||||
private struct StagedRecording {
|
||||
var id: ULID
|
||||
var recordedAt: Date
|
||||
var duration: TimeInterval
|
||||
var context: CaptureContext
|
||||
var audioURL: URL
|
||||
}
|
||||
|
||||
/// Enumerate the staging inbox, pairing each `<ULID>.meta.plist` with its
|
||||
/// `.m4a`. A sidecar whose id doesn't parse, whose required fields are
|
||||
/// missing, or whose audio is absent is skipped (a half-delivered pair can't
|
||||
/// be ingested; the next full delivery re-stages it).
|
||||
private static func stagedWatchRecordings() -> [StagedRecording] {
|
||||
let fm = FileManager.default
|
||||
guard let entries = try? fm.contentsOfDirectory(
|
||||
at: watchInboxURL, includingPropertiesForKeys: nil) else { return [] }
|
||||
|
||||
var staged: [StagedRecording] = []
|
||||
for metaURL in entries where metaURL.pathExtension == "plist" {
|
||||
// `<ULID>.meta.plist` → drop `.plist` then `.meta` to the ULID base.
|
||||
let base = metaURL.deletingPathExtension().deletingPathExtension().lastPathComponent
|
||||
guard let id = ULID(string: base),
|
||||
let data = try? Data(contentsOf: metaURL),
|
||||
let dict = try? PropertyListSerialization.propertyList(
|
||||
from: data, options: [], format: nil) as? [String: Any],
|
||||
let duration = dict["duration"] as? Double,
|
||||
let recordedAt = dict["recordedAt"] as? Date else { continue }
|
||||
|
||||
let audioURL = watchInboxURL.appending(path: "\(base).m4a")
|
||||
guard fm.fileExists(atPath: audioURL.path) else { continue }
|
||||
|
||||
var context = CaptureContext.empty
|
||||
context.device = "Watch"
|
||||
context.timeZoneID = dict["timeZoneID"] as? String
|
||||
staged.append(StagedRecording(
|
||||
id: id, recordedAt: recordedAt, duration: duration,
|
||||
context: context, audioURL: audioURL))
|
||||
}
|
||||
return staged
|
||||
}
|
||||
|
||||
/// Remove a staged recording's `.m4a` and its `<ULID>.meta.plist` sibling.
|
||||
private static func removeStagedFiles(audioURL: URL) {
|
||||
let fm = FileManager.default
|
||||
try? fm.removeItem(at: audioURL)
|
||||
try? fm.removeItem(at: audioURL.deletingPathExtension().appendingPathExtension("meta.plist"))
|
||||
}
|
||||
|
||||
// MARK: - Persistence plumbing
|
||||
|
||||
/// Writes touch the file layer ONLY. iCloud JSON is the source of truth:
|
||||
@@ -263,6 +546,7 @@ final class SyncEngine {
|
||||
let id = DocumentFileStore.documentID(fromRelativePath: path)
|
||||
if await tombstones.stubExists(id: id) {
|
||||
try? await store.remove(at: Self.dataPath(path))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
return
|
||||
}
|
||||
await importRecord(at: path, using: store)
|
||||
@@ -275,6 +559,7 @@ final class SyncEngine {
|
||||
private func handleStubAdded(relativePath: String) async {
|
||||
if let store {
|
||||
try? await store.remove(at: Self.dataPath(relativePath))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: relativePath)))
|
||||
}
|
||||
removeFromCache(jsonRelativePath: relativePath)
|
||||
}
|
||||
@@ -344,9 +629,11 @@ final class SyncEngine {
|
||||
isSyncing = true
|
||||
defer { isSyncing = false }
|
||||
|
||||
// Clean up files a stub has tombstoned (mirrors the rebuild path).
|
||||
// Clean up files a stub has tombstoned (mirrors the rebuild path),
|
||||
// including each record's audio sidecar sibling.
|
||||
for path in orphanedPaths {
|
||||
try? await store.remove(at: Self.dataPath(path))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
}
|
||||
|
||||
removeFromCache(ids: toRemove)
|
||||
@@ -382,6 +669,7 @@ final class SyncEngine {
|
||||
for path in await listRecordPaths(in: store) {
|
||||
if stubIDs.contains(DocumentFileStore.documentID(fromRelativePath: path)) {
|
||||
try? await store.remove(at: Self.dataPath(path))
|
||||
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
|
||||
continue
|
||||
}
|
||||
await importRecord(at: path, using: store)
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import AVFoundation
|
||||
import IndieSync
|
||||
import SwiftUI
|
||||
|
||||
/// Plays back a voice note's audio. Two sources: a fresh recording still on disk
|
||||
/// (`localURL`, used by the composer) or a saved note whose bytes come back
|
||||
/// through `SyncEngine.loadAudioData` (fail-fast, so an evicted file shows a
|
||||
/// "downloading" retry state instead of stalling). `AVAudioPlayer` is not
|
||||
/// Sendable, so the playback model stays main-actor confined.
|
||||
struct AudioPlayerView: View {
|
||||
enum Source: Equatable {
|
||||
case localURL(URL)
|
||||
case note(id: ULID, duration: TimeInterval)
|
||||
}
|
||||
|
||||
let source: Source
|
||||
|
||||
@Environment(SyncEngine.self) private var syncEngine
|
||||
@State private var model = AudioPlaybackModel()
|
||||
|
||||
var body: some View {
|
||||
Group {
|
||||
switch model.loadState {
|
||||
case .idle, .loading:
|
||||
HStack {
|
||||
ProgressView().controlSize(.small)
|
||||
Text("Loading audio…").foregroundStyle(.secondary)
|
||||
}
|
||||
.font(.callout)
|
||||
|
||||
case .downloading:
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
Label("Downloading from iCloud…", systemImage: "icloud.and.arrow.down")
|
||||
.foregroundStyle(.secondary)
|
||||
Button("Retry") { Task { await load() } }
|
||||
.buttonStyle(.bordered)
|
||||
.controlSize(.small)
|
||||
}
|
||||
.font(.callout)
|
||||
|
||||
case .failed:
|
||||
Label("Couldn't play audio", systemImage: "exclamationmark.triangle")
|
||||
.foregroundStyle(.secondary)
|
||||
.font(.callout)
|
||||
|
||||
case .ready:
|
||||
player
|
||||
}
|
||||
}
|
||||
.task { await load() }
|
||||
.onDisappear { model.stop() }
|
||||
}
|
||||
|
||||
private var player: some View {
|
||||
HStack(spacing: 12) {
|
||||
Button {
|
||||
model.togglePlay()
|
||||
} label: {
|
||||
Image(systemName: model.isPlaying ? "pause.circle.fill" : "play.circle.fill")
|
||||
.font(.largeTitle)
|
||||
.foregroundStyle(.tint)
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
|
||||
VStack(spacing: 2) {
|
||||
Slider(
|
||||
value: Binding(get: { model.progress }, set: { model.seek(toFraction: $0) }),
|
||||
in: 0...1
|
||||
)
|
||||
HStack {
|
||||
Text(Self.timeString(model.currentTime))
|
||||
Spacer()
|
||||
Text(Self.timeString(model.duration))
|
||||
}
|
||||
.font(.caption2.monospacedDigit())
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func load() async {
|
||||
switch source {
|
||||
case .localURL(let url):
|
||||
model.loadLocal(url)
|
||||
case .note(let id, let duration):
|
||||
await model.loadNote(id: id, fallbackDuration: duration, syncEngine: syncEngine)
|
||||
}
|
||||
}
|
||||
|
||||
static func timeString(_ interval: TimeInterval) -> String {
|
||||
let total = Int(interval.rounded())
|
||||
return String(format: "%d:%02d", total / 60, total % 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns the `AVAudioPlayer` and a lightweight progress ticker. Main-actor
|
||||
/// confined because `AVAudioPlayer` isn't Sendable.
|
||||
@Observable
|
||||
@MainActor
|
||||
final class AudioPlaybackModel {
|
||||
enum LoadState: Equatable { case idle, loading, ready, downloading, failed }
|
||||
|
||||
private(set) var loadState: LoadState = .idle
|
||||
private(set) var isPlaying = false
|
||||
/// 0…1 position, bound to the scrubber.
|
||||
private(set) var progress: Double = 0
|
||||
private(set) var duration: TimeInterval = 0
|
||||
|
||||
var currentTime: TimeInterval { duration * progress }
|
||||
|
||||
private var player: AVAudioPlayer?
|
||||
private var ticker: Task<Void, Never>?
|
||||
|
||||
func loadLocal(_ url: URL) {
|
||||
loadState = .loading
|
||||
do {
|
||||
let player = try AVAudioPlayer(contentsOf: url)
|
||||
player.prepareToPlay()
|
||||
adopt(player)
|
||||
} catch {
|
||||
loadState = .failed
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a saved note's audio (fail-fast). A miss means the file is evicted:
|
||||
/// show the downloading state and warm it up for the retry.
|
||||
func loadNote(id: ULID, fallbackDuration: TimeInterval, syncEngine: SyncEngine) async {
|
||||
loadState = .loading
|
||||
duration = fallbackDuration
|
||||
guard let data = await syncEngine.loadAudioData(for: id) else {
|
||||
loadState = .downloading
|
||||
await syncEngine.requestAudioDownload(for: id)
|
||||
return
|
||||
}
|
||||
do {
|
||||
let player = try AVAudioPlayer(data: data)
|
||||
player.prepareToPlay()
|
||||
adopt(player)
|
||||
} catch {
|
||||
loadState = .failed
|
||||
}
|
||||
}
|
||||
|
||||
private func adopt(_ player: AVAudioPlayer) {
|
||||
self.player = player
|
||||
duration = player.duration
|
||||
progress = 0
|
||||
loadState = .ready
|
||||
}
|
||||
|
||||
func togglePlay() {
|
||||
guard let player else { return }
|
||||
if player.isPlaying {
|
||||
player.pause()
|
||||
isPlaying = false
|
||||
ticker?.cancel()
|
||||
} else {
|
||||
activatePlaybackSession()
|
||||
player.play()
|
||||
isPlaying = true
|
||||
startTicker()
|
||||
}
|
||||
}
|
||||
|
||||
func seek(toFraction fraction: Double) {
|
||||
guard let player else { return }
|
||||
progress = min(max(fraction, 0), 1)
|
||||
player.currentTime = progress * duration
|
||||
}
|
||||
|
||||
func stop() {
|
||||
player?.stop()
|
||||
isPlaying = false
|
||||
ticker?.cancel()
|
||||
ticker = nil
|
||||
#if os(iOS)
|
||||
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
||||
#endif
|
||||
}
|
||||
|
||||
private func startTicker() {
|
||||
ticker?.cancel()
|
||||
ticker = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
guard let self, let player = self.player else { return }
|
||||
if player.isPlaying {
|
||||
self.progress = self.duration > 0 ? player.currentTime / self.duration : 0
|
||||
} else {
|
||||
// Reached the end (or was paused elsewhere): reset to the start.
|
||||
self.isPlaying = false
|
||||
self.progress = 0
|
||||
return
|
||||
}
|
||||
try? await Task.sleep(for: .milliseconds(50))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func activatePlaybackSession() {
|
||||
#if os(iOS)
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
try? session.setCategory(.playback, mode: .spokenAudio)
|
||||
try? session.setActive(true)
|
||||
#endif
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import SwiftUI
|
||||
|
||||
/// The record surface in the composer: a mic button that, once tapped (or
|
||||
/// auto-started), shows the elapsed time and a live level meter with Stop /
|
||||
/// Discard. When recording ends it hands the finished `(url, duration)` up via
|
||||
/// `onFinish`; the composer owns everything after that.
|
||||
struct AudioRecordingControl: View {
|
||||
@Environment(AudioRecorderService.self) private var recorder
|
||||
|
||||
/// Begin recording as soon as the view appears (the toolbar "record a note"
|
||||
/// entry point), after the permission prompt resolves.
|
||||
var autostart = false
|
||||
let onFinish: (URL, TimeInterval) -> Void
|
||||
|
||||
@State private var permissionDenied = false
|
||||
@State private var startError: String?
|
||||
@State private var didAutostart = false
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 12) {
|
||||
if permissionDenied {
|
||||
deniedState
|
||||
} else if recorder.isRecording {
|
||||
recordingState
|
||||
} else {
|
||||
idleState
|
||||
}
|
||||
|
||||
if let startError {
|
||||
Text(startError)
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.red)
|
||||
}
|
||||
}
|
||||
.frame(maxWidth: .infinity)
|
||||
.task {
|
||||
recorder.onInterrupted = { recording in onFinish(recording.url, recording.duration) }
|
||||
if autostart, !didAutostart {
|
||||
didAutostart = true
|
||||
await beginRecording()
|
||||
}
|
||||
}
|
||||
.onDisappear {
|
||||
recorder.onInterrupted = nil
|
||||
if recorder.isRecording { recorder.discard() }
|
||||
}
|
||||
}
|
||||
|
||||
private var idleState: some View {
|
||||
Button {
|
||||
Task { await beginRecording() }
|
||||
} label: {
|
||||
VStack(spacing: 6) {
|
||||
Image(systemName: "mic.circle.fill")
|
||||
.font(.system(size: 52))
|
||||
.foregroundStyle(.tint)
|
||||
Text("Record")
|
||||
.font(.callout)
|
||||
}
|
||||
}
|
||||
.buttonStyle(.plain)
|
||||
}
|
||||
|
||||
private var recordingState: some View {
|
||||
VStack(spacing: 14) {
|
||||
LevelMeter(level: recorder.level)
|
||||
|
||||
Text(AudioPlayerView.timeString(recorder.elapsed))
|
||||
.font(.title2.monospacedDigit())
|
||||
.foregroundStyle(.primary)
|
||||
|
||||
HStack(spacing: 24) {
|
||||
Button(role: .destructive) {
|
||||
recorder.discard()
|
||||
} label: {
|
||||
Label("Discard", systemImage: "trash")
|
||||
}
|
||||
.buttonStyle(.bordered)
|
||||
|
||||
Button {
|
||||
if let finished = recorder.stopRecording() {
|
||||
onFinish(finished.url, finished.duration)
|
||||
}
|
||||
} label: {
|
||||
Label("Stop", systemImage: "stop.circle.fill")
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var deniedState: some View {
|
||||
VStack(spacing: 8) {
|
||||
Image(systemName: "mic.slash.circle.fill")
|
||||
.font(.system(size: 44))
|
||||
.foregroundStyle(.secondary)
|
||||
Text("Microphone access is off. Enable it for Contextful in Settings to record voice notes.")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.secondary)
|
||||
.multilineTextAlignment(.center)
|
||||
}
|
||||
}
|
||||
|
||||
private func beginRecording() async {
|
||||
startError = nil
|
||||
guard await recorder.requestPermission() else {
|
||||
permissionDenied = true
|
||||
return
|
||||
}
|
||||
permissionDenied = false
|
||||
do {
|
||||
try recorder.startRecording()
|
||||
} catch {
|
||||
startError = "Couldn't start recording: \(error.localizedDescription)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A small row of bars whose height rides the current input level — enough to
|
||||
/// show the mic is live without pretending to be a real waveform.
|
||||
private struct LevelMeter: View {
|
||||
let level: Float
|
||||
|
||||
private let barCount = 9
|
||||
/// Fixed per-bar weights so the meter has a stable, mic-like shape.
|
||||
private let weights: [CGFloat] = [0.35, 0.55, 0.75, 0.9, 1.0, 0.9, 0.75, 0.55, 0.35]
|
||||
|
||||
var body: some View {
|
||||
HStack(alignment: .center, spacing: 4) {
|
||||
ForEach(0..<barCount, id: \.self) { index in
|
||||
Capsule()
|
||||
.fill(.tint)
|
||||
.frame(width: 4, height: height(index))
|
||||
}
|
||||
}
|
||||
.frame(height: 40)
|
||||
.animation(.easeOut(duration: 0.08), value: level)
|
||||
}
|
||||
|
||||
private func height(_ index: Int) -> CGFloat {
|
||||
let base: CGFloat = 4
|
||||
let peak: CGFloat = 36
|
||||
return base + peak * CGFloat(level) * weights[index]
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,10 @@ import SwiftUI
|
||||
/// Composer for a new note and editor for an existing one. On create it
|
||||
/// snapshots the current context (refreshing location in the background);
|
||||
/// on edit the original capture context is preserved untouched.
|
||||
///
|
||||
/// A note can carry audio: create mode can record one (optionally auto-starting
|
||||
/// from the toolbar mic), and edit mode plays it back, shows transcription
|
||||
/// status, and offers a per-note "Re-transcribe" override.
|
||||
struct NoteEditorView: View {
|
||||
enum Mode {
|
||||
case create
|
||||
@@ -11,25 +15,38 @@ struct NoteEditorView: View {
|
||||
}
|
||||
|
||||
let mode: Mode
|
||||
/// Begin recording the moment the composer appears (toolbar mic entry point).
|
||||
var startRecording = false
|
||||
|
||||
@Environment(SyncEngine.self) private var syncEngine
|
||||
@Environment(ContextService.self) private var contextService
|
||||
@Environment(TranscriptionService.self) private var transcriptionService
|
||||
@Environment(TranscriptionSettings.self) private var transcriptionSettings
|
||||
@Environment(\.dismiss) private var dismiss
|
||||
|
||||
@State private var text = ""
|
||||
@State private var tagsText = ""
|
||||
@State private var saveError: String?
|
||||
@State private var recordedAudio: AudioRecorderService.Recording?
|
||||
@FocusState private var textFocused: Bool
|
||||
|
||||
var body: some View {
|
||||
NavigationStack {
|
||||
Form {
|
||||
if isCreating {
|
||||
audioComposeSection
|
||||
}
|
||||
|
||||
Section {
|
||||
TextEditor(text: $text)
|
||||
.focused($textFocused)
|
||||
.frame(minHeight: 160)
|
||||
}
|
||||
|
||||
if case .edit(let note) = mode, note.hasAudio {
|
||||
audioPlaybackSection(for: note)
|
||||
}
|
||||
|
||||
Section("Tags") {
|
||||
TextField("coffee, people, ideas…", text: $tagsText)
|
||||
.autocorrectionDisabled()
|
||||
@@ -65,14 +82,15 @@ struct NoteEditorView: View {
|
||||
}
|
||||
ToolbarItem(placement: .confirmationAction) {
|
||||
Button("Save") { save() }
|
||||
.disabled(text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
|
||||
.disabled(!canSave)
|
||||
}
|
||||
}
|
||||
}
|
||||
.task {
|
||||
switch mode {
|
||||
case .create:
|
||||
textFocused = true
|
||||
// Don't steal focus from the mic entry point.
|
||||
if !startRecording { textFocused = true }
|
||||
await contextService.refresh()
|
||||
case .edit(let note):
|
||||
text = note.text
|
||||
@@ -84,11 +102,119 @@ struct NoteEditorView: View {
|
||||
#endif
|
||||
}
|
||||
|
||||
// MARK: - Audio (create)
|
||||
|
||||
@ViewBuilder
|
||||
private var audioComposeSection: some View {
|
||||
Section("Voice") {
|
||||
if let recordedAudio {
|
||||
AudioPlayerView(source: .localURL(recordedAudio.url))
|
||||
Button(role: .destructive) {
|
||||
try? FileManager.default.removeItem(at: recordedAudio.url)
|
||||
self.recordedAudio = nil
|
||||
} label: {
|
||||
Label("Discard Recording", systemImage: "trash")
|
||||
}
|
||||
} else {
|
||||
AudioRecordingControl(autostart: startRecording) { url, duration in
|
||||
recordedAudio = AudioRecorderService.Recording(url: url, duration: duration)
|
||||
}
|
||||
.padding(.vertical, 4)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Audio (edit)
|
||||
|
||||
@ViewBuilder
|
||||
private func audioPlaybackSection(for note: NoteEntity) -> some View {
|
||||
Section("Voice") {
|
||||
if let id = ULID(string: note.id) {
|
||||
AudioPlayerView(source: .note(id: id, duration: note.audioDuration ?? 0))
|
||||
}
|
||||
transcriptionStatusRow(for: note)
|
||||
retranscribeMenu(for: note)
|
||||
}
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private func transcriptionStatusRow(for note: NoteEntity) -> some View {
|
||||
switch note.audio?.transcriptionStatus {
|
||||
case .pending:
|
||||
if note.transcriberDeviceID == TranscriptionSettings.installID {
|
||||
Label {
|
||||
Text("Transcribing…")
|
||||
} icon: {
|
||||
ProgressView().controlSize(.small)
|
||||
}
|
||||
.foregroundStyle(.secondary)
|
||||
} else {
|
||||
Label("Waiting to transcribe on another device", systemImage: "hourglass")
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
case .failed:
|
||||
HStack {
|
||||
Label("Transcription failed", systemImage: "exclamationmark.triangle")
|
||||
.foregroundStyle(.secondary)
|
||||
Spacer()
|
||||
Button("Retry") { retranscribe(note: note, localeID: nil) }
|
||||
.buttonStyle(.bordered)
|
||||
.controlSize(.small)
|
||||
}
|
||||
case .done:
|
||||
if let localeID = note.audio?.transcriptionLocaleID {
|
||||
Label(
|
||||
"Transcribed · \(TranscriptionSettings.displayName(for: Locale(identifier: localeID)))",
|
||||
systemImage: "checkmark.circle"
|
||||
)
|
||||
.foregroundStyle(.secondary)
|
||||
.font(.footnote)
|
||||
}
|
||||
case .none:
|
||||
EmptyView()
|
||||
}
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private func retranscribeMenu(for note: NoteEntity) -> some View {
|
||||
let locales = transcriptionSettings.enabledLocales
|
||||
if !locales.isEmpty {
|
||||
Menu {
|
||||
Button("Auto-detect Language") { retranscribe(note: note, localeID: nil) }
|
||||
Divider()
|
||||
ForEach(locales, id: \.identifier) { locale in
|
||||
Button(TranscriptionSettings.displayName(for: locale)) {
|
||||
retranscribe(note: note, localeID: TranscriptionSettings.canonicalID(locale))
|
||||
}
|
||||
}
|
||||
} label: {
|
||||
Label("Re-transcribe", systemImage: "arrow.clockwise")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func retranscribe(note: NoteEntity, localeID: String?) {
|
||||
guard let id = ULID(string: note.id) else { return }
|
||||
Task {
|
||||
try? await syncEngine.requestRetranscription(id: id, localeID: localeID)
|
||||
transcriptionService.enqueue(noteID: id, forcedLocaleID: localeID)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - State
|
||||
|
||||
private var isCreating: Bool {
|
||||
if case .create = mode { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
private var canSave: Bool {
|
||||
if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return true }
|
||||
if recordedAudio != nil { return true }
|
||||
if case .edit(let note) = mode, note.hasAudio { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
private var parsedTags: [String] {
|
||||
tagsText
|
||||
.split(separator: ",")
|
||||
@@ -96,18 +222,34 @@ struct NoteEditorView: View {
|
||||
.filter { !$0.isEmpty }
|
||||
}
|
||||
|
||||
// MARK: - Save
|
||||
|
||||
private func save() {
|
||||
guard canSave else { return }
|
||||
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
guard !trimmed.isEmpty else { return }
|
||||
Task {
|
||||
do {
|
||||
switch mode {
|
||||
case .create:
|
||||
try await syncEngine.createNote(
|
||||
text: trimmed,
|
||||
tags: parsedTags,
|
||||
context: contextService.current
|
||||
)
|
||||
if let recordedAudio {
|
||||
let note = try await syncEngine.createAudioNote(
|
||||
audioFileURL: recordedAudio.url,
|
||||
duration: recordedAudio.duration,
|
||||
text: trimmed,
|
||||
tags: parsedTags,
|
||||
context: contextService.current,
|
||||
transcriberDeviceID: TranscriptionSettings.installID
|
||||
)
|
||||
// Fast path: hand the recording's temp file straight to the
|
||||
// transcriber so it doesn't round-trip through the container.
|
||||
transcriptionService.enqueue(noteID: note.id, localAudioURL: recordedAudio.url)
|
||||
} else {
|
||||
try await syncEngine.createNote(
|
||||
text: trimmed,
|
||||
tags: parsedTags,
|
||||
context: contextService.current
|
||||
)
|
||||
}
|
||||
case .edit(let entity):
|
||||
guard let id = ULID(string: entity.id),
|
||||
var note = await syncEngine.loadNote(id: id) else { return }
|
||||
|
||||
@@ -5,11 +5,16 @@ struct NoteRowView: View {
|
||||
|
||||
var body: some View {
|
||||
VStack(alignment: .leading, spacing: 4) {
|
||||
Text(note.title.isEmpty ? "Untitled" : note.title)
|
||||
Text(displayTitle)
|
||||
.font(.body.weight(.medium))
|
||||
.lineLimit(2)
|
||||
|
||||
HStack(spacing: 4) {
|
||||
if note.hasAudio {
|
||||
Image(systemName: "waveform")
|
||||
Text(AudioPlayerView.timeString(note.audioDuration ?? 0))
|
||||
Text("·")
|
||||
}
|
||||
Text(note.createdAt, format: .dateTime.day().month().year())
|
||||
if let place = note.context.summary {
|
||||
Text("·")
|
||||
@@ -20,6 +25,8 @@ struct NoteRowView: View {
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
|
||||
transcriptionChip
|
||||
|
||||
if !note.tags.isEmpty {
|
||||
Text(note.tags.map { "#\($0)" }.joined(separator: " "))
|
||||
.font(.caption)
|
||||
@@ -29,4 +36,31 @@ struct NoteRowView: View {
|
||||
}
|
||||
.padding(.vertical, 2)
|
||||
}
|
||||
|
||||
/// Audio notes with no transcript yet still need a legible title.
|
||||
private var displayTitle: String {
|
||||
if !note.title.isEmpty { return note.title }
|
||||
return note.hasAudio ? "Audio note" : "Untitled"
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private var transcriptionChip: some View {
|
||||
switch note.audio?.transcriptionStatus {
|
||||
case .pending:
|
||||
capsule("Transcribing…", color: .secondary)
|
||||
case .failed:
|
||||
capsule("Transcription failed", color: .red)
|
||||
case .done, .none:
|
||||
EmptyView()
|
||||
}
|
||||
}
|
||||
|
||||
private func capsule(_ text: String, color: Color) -> some View {
|
||||
Text(text)
|
||||
.font(.caption2)
|
||||
.foregroundStyle(color)
|
||||
.padding(.horizontal, 8)
|
||||
.padding(.vertical, 2)
|
||||
.background(color.opacity(0.12), in: Capsule())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ struct NotesListView: View {
|
||||
|
||||
@State private var searchText = ""
|
||||
@State private var composing = false
|
||||
@State private var recording = false
|
||||
@State private var editingNote: NoteEntity?
|
||||
|
||||
var body: some View {
|
||||
@@ -50,6 +51,13 @@ struct NotesListView: View {
|
||||
}
|
||||
}
|
||||
.toolbar {
|
||||
ToolbarItem(placement: .primaryAction) {
|
||||
Button {
|
||||
recording = true
|
||||
} label: {
|
||||
Label("Record Note", systemImage: "mic")
|
||||
}
|
||||
}
|
||||
ToolbarItem(placement: .primaryAction) {
|
||||
Button {
|
||||
composing = true
|
||||
@@ -62,6 +70,9 @@ struct NotesListView: View {
|
||||
.sheet(isPresented: $composing) {
|
||||
NoteEditorView(mode: .create)
|
||||
}
|
||||
.sheet(isPresented: $recording) {
|
||||
NoteEditorView(mode: .create, startRecording: true)
|
||||
}
|
||||
.sheet(item: $editingNote) { note in
|
||||
NoteEditorView(mode: .edit(note))
|
||||
}
|
||||
|
||||
@@ -36,6 +36,14 @@ struct SettingsView: View {
|
||||
.disabled(syncEngine.iCloudStatus != .available || syncEngine.isSyncing)
|
||||
}
|
||||
|
||||
Section("Transcription") {
|
||||
NavigationLink {
|
||||
TranscriptionLanguagesView()
|
||||
} label: {
|
||||
Label("Languages", systemImage: "character.bubble")
|
||||
}
|
||||
}
|
||||
|
||||
Section("Context") {
|
||||
ContextSummaryRow(
|
||||
context: contextService.current,
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import SwiftUI
|
||||
|
||||
/// Settings screen for choosing which languages voice notes can be transcribed
|
||||
/// in. Enabling a language reserves it and downloads its on-device model; the
|
||||
/// system caps how many can be reserved at once, so the list disables further
|
||||
/// toggles at capacity and explains the limit in the footer.
|
||||
struct TranscriptionLanguagesView: View {
|
||||
@Environment(TranscriptionSettings.self) private var settings
|
||||
@State private var errorMessage: String?
|
||||
|
||||
var body: some View {
|
||||
List {
|
||||
if let errorMessage {
|
||||
Section {
|
||||
Label(errorMessage, systemImage: "exclamationmark.triangle")
|
||||
.font(.footnote)
|
||||
.foregroundStyle(.red)
|
||||
}
|
||||
}
|
||||
|
||||
if !enabledLocales.isEmpty {
|
||||
Section("Enabled") {
|
||||
ForEach(enabledLocales, id: \.identifier) { row($0) }
|
||||
}
|
||||
}
|
||||
|
||||
Section {
|
||||
ForEach(otherLocales, id: \.identifier) { row($0) }
|
||||
} header: {
|
||||
Text("All Languages")
|
||||
} footer: {
|
||||
Text(footerText)
|
||||
}
|
||||
}
|
||||
.navigationTitle("Languages")
|
||||
.overlay {
|
||||
if settings.supportedLocales.isEmpty {
|
||||
ProgressView()
|
||||
}
|
||||
}
|
||||
.task { await settings.prepare() }
|
||||
}
|
||||
|
||||
private var enabledLocales: [Locale] {
|
||||
settings.supportedLocales.filter { settings.isEnabled($0) }
|
||||
}
|
||||
|
||||
private var otherLocales: [Locale] {
|
||||
settings.supportedLocales.filter { !settings.isEnabled($0) }
|
||||
}
|
||||
|
||||
private var footerText: String {
|
||||
guard settings.maximumReserved > 0 else {
|
||||
return "Each new note is transcribed on device in your enabled languages."
|
||||
}
|
||||
return "Enable up to \(settings.maximumReserved) languages. Each new voice note is sampled in your enabled languages and transcribed in whichever fits best."
|
||||
}
|
||||
|
||||
private func row(_ locale: Locale) -> some View {
|
||||
let enabled = settings.isEnabled(locale)
|
||||
return HStack {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
Text(TranscriptionSettings.displayName(for: locale))
|
||||
stateSubtitle(for: locale)
|
||||
}
|
||||
Spacer()
|
||||
Toggle("", isOn: Binding(
|
||||
get: { settings.isEnabled(locale) },
|
||||
set: { toggle(locale, on: $0) }
|
||||
))
|
||||
.labelsHidden()
|
||||
.disabled(!enabled && settings.isAtCapacity)
|
||||
}
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private func stateSubtitle(for locale: Locale) -> some View {
|
||||
switch settings.assetStates[TranscriptionSettings.canonicalID(locale)] {
|
||||
case .installed:
|
||||
Label("Installed", systemImage: "checkmark.circle.fill")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.green)
|
||||
case .downloading(let fraction):
|
||||
HStack(spacing: 6) {
|
||||
ProgressView().controlSize(.small)
|
||||
if let fraction {
|
||||
Text("Downloading \(Int(fraction * 100))%")
|
||||
} else {
|
||||
Text("Downloading…")
|
||||
}
|
||||
}
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
case .notInstalled:
|
||||
Label("Available to download", systemImage: "icloud.and.arrow.down")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.secondary)
|
||||
case .unsupported, .none:
|
||||
EmptyView()
|
||||
}
|
||||
}
|
||||
|
||||
private func toggle(_ locale: Locale, on: Bool) {
|
||||
errorMessage = nil
|
||||
Task {
|
||||
if on {
|
||||
do {
|
||||
try await settings.enable(locale)
|
||||
} catch TranscriptionSettings.SettingsError.atCapacity {
|
||||
errorMessage = "You can enable up to \(settings.maximumReserved) languages. Turn one off first."
|
||||
} catch {
|
||||
errorMessage = "Couldn't enable \(TranscriptionSettings.displayName(for: locale)): \(error.localizedDescription)"
|
||||
}
|
||||
} else {
|
||||
await settings.disable(locale)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user