Add voice notes: on-device transcription, multi-language, watch capture

- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud
- AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline
  with enabled-language set and confidence-based auto-pick, re-transcribe menu
- Settings > Transcription > Languages with asset download/reserve handling
- watchOS app + accessory complication: one-button recorder, WCSession
  transferFile to phone, WatchInbox staging, direct-upsert ingestion
- Player UI, transcription status chips, quick-capture mic in notes list
- Version 0.2, changelog + README updated
This commit is contained in:
2026-07-15 13:30:43 -04:00
parent 6a93cedaa6
commit 86d74806c9
44 changed files with 3329 additions and 123 deletions
+6
View File
@@ -1,5 +1,11 @@
**July 2026**
Record voice notes that are transcribed to text right on your device, with the original audio kept alongside the note for playback
Choose the languages you speak in Settings and transcription picks the right one automatically, or re-transcribe any note in another language
A new Apple Watch app and watch-face complication let you capture voice notes from your wrist and send them to your iPhone automatically
First version: take quick text notes that remember where and when they were taken, see the most relevant ones resurface when you return to the same place, organize with tags, search everything, and sync across iPhone and Mac through iCloud
Back up all notes to a single file and restore from it, with automatic safety backups before every restore
@@ -0,0 +1,20 @@
{
"colors" : [
{
"color" : {
"color-space" : "srgb",
"components" : {
"alpha" : "1.000",
"blue" : "0.282",
"green" : "0.208",
"red" : "0.886"
}
},
"idiom" : "universal"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
@@ -0,0 +1,14 @@
{
"images" : [
{
"filename" : "icon-1024.png",
"idiom" : "universal",
"platform" : "watchos",
"size" : "1024x1024"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 529 KiB

@@ -0,0 +1,6 @@
{
"info" : {
"author" : "xcode",
"version" : 1
}
}
@@ -0,0 +1,155 @@
import Foundation
import IndieSync
import os
import WatchConnectivity
/// Watch side of the recording hand-off. The watch has no iCloud; a finished
/// recording becomes durable on the phone, which ingests and transcribes it.
///
/// Each queued recording is a `<ULID>.m4a` plus a `<ULID>.meta.plist` sidecar in
/// `Documents/Recordings/`. `transferFile` hands the audio to WatchConnectivity
/// with an `AudioNotePayload` as metadata. Local files are deleted **only** in
/// `didFinish` with a nil error so a delivery we never confirm (app killed
/// mid-flight, transfer failed) is re-enqueued by `reconcileQueue()` from the
/// persisted sidecar on the next activation. `pendingCount` (files still on disk)
/// drives the "N waiting for iPhone" footer.
///
/// Strict concurrency: the delegate callbacks are nonisolated and run on
/// WatchConnectivity's background queue they extract Sendable values (a file
/// name string) before hopping to the main actor. `WCSessionFile*` are not
/// Sendable and never cross the hop.
@Observable
@MainActor
final class WatchTransferService: NSObject {
private nonisolated static let log = Logger(
subsystem: "dev.rzen.indie.Notes.watchkitapp", category: "watch-transfer")
/// Recordings still on disk (queued or in-flight) the count the phone has
/// not yet confirmed receiving.
private(set) var pendingCount = 0
private var session: WCSession?
func activate() {
guard WCSession.isSupported() else { return }
let session = WCSession.default
session.delegate = self
session.activate()
self.session = session
refreshPendingCount()
}
/// Queue a finished recording for delivery. Writes the sidecar first (so a
/// relaunch before delivery can re-enqueue it), then hands the file to
/// `transferFile`.
func enqueue(_ recording: WatchAudioRecorder.Recording) {
let tz = TimeZone.current.identifier
writeSidecar(base: recording.id.stringValue,
recordedAt: recording.recordedAt, duration: recording.duration, timeZoneID: tz)
startTransfer(id: recording.id.stringValue, url: recording.url,
recordedAt: recording.recordedAt, duration: recording.duration, timeZoneID: tz)
refreshPendingCount()
}
/// Re-enqueue any recording still on disk that the session isn't already
/// transferring a file whose delivery we never confirmed. Runs when the
/// session activates (relaunch) and when the app returns to the foreground.
func reconcileQueue() {
guard let session else { refreshPendingCount(); return }
let inFlight = Set(session.outstandingFileTransfers.map { $0.file.fileURL.lastPathComponent })
let fm = FileManager.default
let dir = WatchAudioRecorder.recordingsDirectory
let entries = (try? fm.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)) ?? []
for m4a in entries where m4a.pathExtension == "m4a" {
if inFlight.contains(m4a.lastPathComponent) { continue }
let base = m4a.deletingPathExtension().lastPathComponent
guard ULID(string: base) != nil else { continue }
let meta = readSidecar(base: base)
startTransfer(id: base, url: m4a,
recordedAt: meta?.recordedAt ?? Date(),
duration: meta?.duration ?? 0,
timeZoneID: meta?.timeZoneID)
}
refreshPendingCount()
}
// MARK: - Transfer
private func startTransfer(id: String, url: URL, recordedAt: Date, duration: TimeInterval, timeZoneID: String?) {
guard let session, session.activationState == .activated,
FileManager.default.fileExists(atPath: url.path) else { return }
let payload = AudioNotePayload(
id: id, recordedAt: recordedAt, duration: duration, timeZoneID: timeZoneID)
session.transferFile(url, metadata: payload.metadata())
}
/// Delete a delivered recording and its sidecar (called only on a confirmed
/// `didFinish` with no error).
private func completeTransfer(fileName: String) {
let dir = WatchAudioRecorder.recordingsDirectory
let base = (fileName as NSString).deletingPathExtension
try? FileManager.default.removeItem(at: dir.appending(path: fileName))
try? FileManager.default.removeItem(at: sidecarURL(base: base))
refreshPendingCount()
}
// MARK: - Pending count
private func refreshPendingCount() {
let dir = WatchAudioRecorder.recordingsDirectory
let entries = (try? FileManager.default.contentsOfDirectory(
at: dir, includingPropertiesForKeys: nil)) ?? []
pendingCount = entries.filter { $0.pathExtension == "m4a" }.count
}
// MARK: - Sidecar (persisted metadata for re-enqueue after relaunch)
private func sidecarURL(base: String) -> URL {
WatchAudioRecorder.recordingsDirectory.appending(path: "\(base).meta.plist")
}
private func writeSidecar(base: String, recordedAt: Date, duration: TimeInterval, timeZoneID: String?) {
var dict: [String: Any] = ["recordedAt": recordedAt, "duration": duration]
if let timeZoneID { dict["timeZoneID"] = timeZoneID }
if let data = try? PropertyListSerialization.data(fromPropertyList: dict, format: .binary, options: 0) {
try? data.write(to: sidecarURL(base: base))
}
}
private func readSidecar(base: String) -> (recordedAt: Date, duration: TimeInterval, timeZoneID: String?)? {
guard let data = try? Data(contentsOf: sidecarURL(base: base)),
let dict = try? PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any],
let recordedAt = dict["recordedAt"] as? Date,
let duration = dict["duration"] as? Double else { return nil }
return (recordedAt, duration, dict["timeZoneID"] as? String)
}
}
// MARK: - WCSessionDelegate
extension WatchTransferService: WCSessionDelegate {
nonisolated func session(_ session: WCSession, activationDidCompleteWith activationState: WCSessionActivationState, error: (any Error)?) {
if let error { Self.log.error("watch WC activation failed: \(error, privacy: .public)") }
Task { @MainActor in self.reconcileQueue() }
}
nonisolated func sessionReachabilityDidChange(_ session: WCSession) {
// A queued file whose transfer never registered (session wasn't yet
// activated at enqueue) gets re-driven once the phone is reachable again.
if session.isReachable {
Task { @MainActor in self.reconcileQueue() }
}
}
nonisolated func session(_ session: WCSession, didFinish fileTransfer: WCSessionFileTransfer, error: (any Error)?) {
// Extract the Sendable file name before hopping WCSessionFileTransfer /
// WCSessionFile are not Sendable.
let fileName = fileTransfer.file.fileURL.lastPathComponent
if let error {
Self.log.warning("file transfer failed (\(fileName, privacy: .public)): \(error, privacy: .public)")
Task { @MainActor in self.refreshPendingCount() }
return
}
Task { @MainActor in self.completeTransfer(fileName: fileName) }
}
}
+21
View File
@@ -0,0 +1,21 @@
import SwiftUI
@main
struct NotesWatchApp: App {
@State private var services = WatchAppServices()
@Environment(\.scenePhase) private var scenePhase
var body: some Scene {
WindowGroup {
NavigationStack {
RecordView()
}
.environment(services)
.task { services.bootstrap() }
.onOpenURL { url in services.handleDeepLink(url) }
}
.onChange(of: scenePhase) { _, phase in
services.handleScenePhase(phase)
}
}
}
@@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleDisplayName</key>
<string>Contextful</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>NSMicrophoneUsageDescription</key>
<string>Contextful records voice notes on your watch and sends them to your iPhone to transcribe.</string>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
</array>
<key>WKApplication</key>
<true/>
<key>WKCompanionAppBundleIdentifier</key>
<string>dev.rzen.indie.Notes</string>
<key>CFBundleURLTypes</key>
<array>
<dict>
<key>CFBundleURLName</key>
<string>dev.rzen.indie.Notes.watchkitapp</string>
<key>CFBundleURLSchemes</key>
<array>
<string>contextful</string>
</array>
</dict>
</array>
</dict>
</plist>
@@ -0,0 +1,160 @@
import AVFoundation
import Foundation
import IndieSync
import WatchKit
/// Records a single voice note on the watch into a persisted
/// `Documents/Recordings/<ULID>.m4a` (AAC mono 22.05 kHz ~32 kbps). The ULID is
/// minted at record **start**, so the note's identity and its UTC time bucket
/// once ingested on the phone reflects when it was recorded, not when it later
/// reaches the phone.
///
/// The file lands in the app's Documents (not temp) so a queued recording
/// survives suspension until `WatchTransferService` confirms the phone received
/// it. `AVAudioRecorder` is not Sendable, so the recorder stays confined to the
/// main actor for its whole life.
@Observable
@MainActor
final class WatchAudioRecorder {
/// One finished recording: the file on disk, its length, its ULID, and the
/// instant it started. The caller (`WatchTransferService`) owns delivery.
struct Recording: Equatable, Sendable {
let id: ULID
let url: URL
let duration: TimeInterval
let recordedAt: Date
}
private(set) var isRecording = false
/// Seconds recorded so far, refreshed by the tick loop while recording.
private(set) var elapsed: TimeInterval = 0
/// Fired when a recording ends without the user tapping Stop the 10-minute
/// safety cap fires. The finished file is handed up to be queued rather than
/// lost. (Scene-deactivation stop-and-queue is driven by `WatchAppServices`,
/// which calls `stopRecording()` directly.)
var onInvoluntaryFinish: ((Recording) -> Void)?
/// Hard cap on a single take so a recording the user forgot about can't run
/// the battery down. The finished file is queued, never dropped.
private static let maxDuration: TimeInterval = 600
private var recorder: AVAudioRecorder?
private var current: (id: ULID, url: URL, startedAt: Date)?
private var tickTask: Task<Void, Never>?
/// Directory the finished recordings (and their `.meta.plist` sidecars) live
/// in until the phone confirms delivery. Persisted, so a queued file survives
/// app suspension and can be re-enqueued on the next launch.
static var recordingsDirectory: URL {
URL.documentsDirectory.appending(path: "Recordings", directoryHint: .isDirectory)
}
/// Ask for microphone access, returning whether it was granted. Safe to call
/// every time recording starts the system only prompts once.
func requestPermission() async -> Bool {
await AVAudioApplication.requestRecordPermission()
}
/// Whether the mic has been explicitly denied, so the UI can show a jump to
/// Settings instead of a dead record button.
var permissionDenied: Bool {
AVAudioApplication.shared.recordPermission == .denied
}
/// Begin recording into a fresh `<ULID>.m4a`. A no-op if already recording.
/// Throws if the audio session or recorder can't be configured.
func startRecording() throws {
guard !isRecording else { return }
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record, mode: .spokenAudio)
try session.setActive(true)
let dir = Self.recordingsDirectory
try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
let id = ULID()
let url = dir.appending(path: "\(id.stringValue).m4a")
let settings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 22_050.0,
AVNumberOfChannelsKey: 1,
AVEncoderBitRateKey: 32_000,
AVEncoderAudioQualityKey: AVAudioQuality.medium.rawValue,
]
let recorder = try AVAudioRecorder(url: url, settings: settings)
guard recorder.record() else { throw RecorderError.couldNotStart }
self.recorder = recorder
self.current = (id, url, Date())
isRecording = true
elapsed = 0
// Continuity through a wrist-down rests on the `audio` background mode
// (declared in Info-watchOS.plist) keeping the active audio session
// alive; the old `WKExtension.isFrontmostTimeoutExtended` knob is
// deprecated and unsupported since watchOS 7. The stop-and-queue on
// scene-background (see `WatchAppServices`) is the never-lose-audio net.
WKInterfaceDevice.current().play(.start)
startTicking()
}
/// Stop recording and return the finished file plus its duration, or nil if
/// nothing was recording.
@discardableResult
func stopRecording() -> Recording? {
guard let recorder, let current else { return nil }
let duration = recorder.currentTime
recorder.stop()
WKInterfaceDevice.current().play(.stop)
let recording = Recording(
id: current.id, url: current.url, duration: duration, recordedAt: current.startedAt)
teardown()
return recording
}
/// Abandon the in-progress recording and delete its file. (Not surfaced in the
/// single-button UI, but kept for symmetry / future use.)
func discard() {
recorder?.stop()
if let url = current?.url { try? FileManager.default.removeItem(at: url) }
teardown()
}
// MARK: - Metering / watchdog
/// ~5 Hz loop that refreshes `elapsed` and enforces the safety cap. Runs on
/// the main actor (a plain `Task` inherits it), so it can touch the
/// non-Sendable recorder safely.
private func startTicking() {
tickTask?.cancel()
tickTask = Task { [weak self] in
while !Task.isCancelled {
guard let self, let recorder = self.recorder, recorder.isRecording else { return }
self.elapsed = recorder.currentTime
if self.elapsed >= Self.maxDuration {
if let finished = self.stopRecording() {
self.onInvoluntaryFinish?(finished)
}
return
}
try? await Task.sleep(for: .milliseconds(200))
}
}
}
private func teardown() {
tickTask?.cancel()
tickTask = nil
recorder = nil
current = nil
isRecording = false
elapsed = 0
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
}
enum RecorderError: Error {
case couldNotStart
}
}
+84
View File
@@ -0,0 +1,84 @@
import SwiftUI
/// The watch app's single screen: a big record/stop button, an elapsed timer
/// while recording, and a footer counting recordings still waiting to reach the
/// iPhone. Shows a permission-denied state when the mic is refused.
struct RecordView: View {
@Environment(WatchAppServices.self) private var services
var body: some View {
let recorder = services.recorder
VStack(spacing: 10) {
if recorder.permissionDenied {
permissionDenied
} else {
recordButton(isRecording: recorder.isRecording)
if recorder.isRecording {
Text(Self.elapsedString(recorder.elapsed))
.font(.system(.title3, design: .rounded).monospacedDigit())
.foregroundStyle(.secondary)
} else {
Text("Tap to record")
.font(.footnote)
.foregroundStyle(.secondary)
}
pendingFooter
}
}
.padding()
.containerBackground(.red.gradient, for: .navigation)
.navigationTitle("Contextful")
}
private func recordButton(isRecording: Bool) -> some View {
Button {
services.toggleRecording()
} label: {
ZStack {
Circle()
.fill(.red)
.frame(width: 96, height: 96)
Image(systemName: isRecording ? "stop.fill" : "mic.fill")
.font(.system(size: 40))
.foregroundStyle(.white)
.contentTransition(.symbolEffect(.replace))
}
}
.buttonStyle(.plain)
.accessibilityLabel(isRecording ? "Stop recording" : "Start recording")
}
@ViewBuilder
private var pendingFooter: some View {
let count = services.transfer.pendingCount
if count > 0 {
Label(
count == 1 ? "1 waiting for iPhone" : "\(count) waiting for iPhone",
systemImage: "arrow.up.circle"
)
.font(.caption2)
.foregroundStyle(.secondary)
}
}
private var permissionDenied: some View {
VStack(spacing: 8) {
Image(systemName: "mic.slash.fill")
.font(.title)
.foregroundStyle(.secondary)
Text("Microphone access is off")
.font(.headline)
.multilineTextAlignment(.center)
Text("Enable it in the Watch app on iPhone under Contextful.")
.font(.caption2)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
}
}
/// mm:ss for the elapsed readout.
static func elapsedString(_ interval: TimeInterval) -> String {
let total = Int(interval)
return String(format: "%02d:%02d", total / 60, total % 60)
}
}
+73
View File
@@ -0,0 +1,73 @@
import Foundation
import Observation
import SwiftUI
/// Composition root for the watch app. Owns the recorder and the transfer
/// service and mediates between them: a finished recording is handed to the
/// transfer queue. The watch has no iCloud or local database its whole job is
/// to capture audio and get it to the phone.
@Observable
@MainActor
final class WatchAppServices {
let recorder = WatchAudioRecorder()
let transfer = WatchTransferService()
func bootstrap() {
transfer.activate()
transfer.reconcileQueue()
// The 10-minute safety cap ends a take without a Stop tap queue it
// rather than lose it.
recorder.onInvoluntaryFinish = { [weak self] recording in
self?.transfer.enqueue(recording)
}
}
/// Single-button behaviour: start if idle, stop-and-queue if recording.
func toggleRecording() {
if recorder.isRecording {
stopAndQueue()
} else {
Task { await startRecording() }
}
}
/// Request permission (once) then begin recording. A no-op if already
/// recording or permission is refused.
func startRecording() async {
guard !recorder.isRecording else { return }
guard await recorder.requestPermission() else { return }
try? recorder.startRecording()
}
private func stopAndQueue() {
if let recording = recorder.stopRecording() {
transfer.enqueue(recording)
}
}
/// Complication deep link (`contextful://record?autostart=1`) auto-start a
/// recording. Plain launches never carry the URL, so they never auto-record.
func handleDeepLink(_ url: URL) {
guard url.scheme == "contextful", url.host == "record" else { return }
let components = URLComponents(url: url, resolvingAgainstBaseURL: false)
let autostart = components?.queryItems?.first { $0.name == "autostart" }?.value == "1"
guard autostart else { return }
Task { await startRecording() }
}
func handleScenePhase(_ phase: ScenePhase) {
switch phase {
case .active:
// Catch up any file whose delivery we never confirmed.
transfer.reconcileQueue()
case .background:
// Never lose audio to the app being suspended: stop-and-queue
// whatever is recording rather than dropping the take. (A brief
// wrist-down is `.inactive` and is covered by the recorder's
// frontmost-timeout extension, so it keeps recording.)
stopAndQueue()
default:
break
}
}
}
@@ -0,0 +1,91 @@
import SwiftUI
import WidgetKit
// A launcher complication: a static button on the watch face that opens the
// Contextful watch app straight into recording. It carries no data, so the
// timeline is a single entry that never refreshes. The `widgetURL` deep link
// (`contextful://record?autostart=1`) is what tells the app to auto-start
// only a complication tap carries it, so a plain launch never auto-records.
private struct LauncherEntry: TimelineEntry {
let date: Date
}
private struct LauncherProvider: TimelineProvider {
func placeholder(in context: Context) -> LauncherEntry {
LauncherEntry(date: .now)
}
func getSnapshot(in context: Context, completion: @escaping (LauncherEntry) -> Void) {
completion(LauncherEntry(date: .now))
}
func getTimeline(in context: Context, completion: @escaping (Timeline<LauncherEntry>) -> Void) {
// Nothing ever changes one entry, never reload.
completion(Timeline(entries: [LauncherEntry(date: .now)], policy: .never))
}
}
private struct LauncherView: View {
@Environment(\.widgetFamily) private var family
private let glyph = "mic.fill"
// `.widgetAccentable()` puts the glyph in the accent group so the watch face
// tints it with its main color instead of leaving it plain white.
private var mic: some View {
Image(systemName: glyph).widgetAccentable()
}
var body: some View {
Group {
switch family {
case .accessoryInline:
// Inline templates render the symbol upright next to text.
Label("Record", systemImage: glyph)
case .accessoryRectangular:
HStack(spacing: 6) {
mic.font(.title3)
Text("Record a Note").font(.headline)
}
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .leading)
case .accessoryCorner:
mic
.font(.title2)
.widgetLabel("Record")
default: // .accessoryCircular and any future families
ZStack {
AccessoryWidgetBackground()
mic.font(.title3)
}
}
}
.widgetURL(URL(string: "contextful://record?autostart=1"))
}
}
struct ContextfulRecorderLauncher: Widget {
private let kind = "ContextfulRecorderLauncher"
var body: some WidgetConfiguration {
StaticConfiguration(kind: kind, provider: LauncherProvider()) { _ in
LauncherView()
.containerBackground(.clear, for: .widget)
}
.configurationDisplayName("Record a Note")
.description("Tap to start recording a voice note.")
.supportedFamilies([
.accessoryCircular,
.accessoryCorner,
.accessoryInline,
.accessoryRectangular,
])
}
}
@main
struct ContextfulWatchWidgetBundle: WidgetBundle {
var body: some Widget {
ContextfulRecorderLauncher()
}
}
@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleDisplayName</key>
<string>Contextful</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>NSExtension</key>
<dict>
<key>NSExtensionPointIdentifier</key>
<string>com.apple.widgetkit-extension</string>
</dict>
</dict>
</plist>
+26
View File
@@ -0,0 +1,26 @@
#if os(iOS)
import UIKit
/// iOS application delegate. Its sole job is to stand up the WatchConnectivity
/// receiver in `didFinishLaunching` which runs even on a background launch the
/// system makes purely to deliver a watch file transfer, unlike a SwiftUI
/// `.task` that may not run until the UI is built.
///
/// The receiver is created and its `WCSession` activated here immediately, so a
/// delivered recording is staged to the inbox even before `AppServices` exists.
/// `NotesApp` wires the receiver's `syncEngine` in once services are ready, so a
/// live delivery can drain right away; a delivery that arrives before then is
/// drained on the next foreground connect.
@MainActor
final class AppDelegate: NSObject, UIApplicationDelegate {
let watchReceiver = PhoneWatchReceiver()
func application(
_ application: UIApplication,
didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
) -> Bool {
watchReceiver.activate()
return true
}
}
#endif
+24
View File
@@ -11,6 +11,9 @@ final class AppServices {
let syncEngine: SyncEngine
let contextService: ContextService
let backupController: BackupController
let audioRecorder: AudioRecorderService
let transcriptionSettings: TranscriptionSettings
let transcriptionService: TranscriptionService
init() async {
let container = NotesModelContainer.make()
@@ -21,6 +24,20 @@ final class AppServices {
self.backupController = BackupController(
configuration: NotesBackupConfiguration(syncEngine: engine)
)
self.audioRecorder = AudioRecorderService()
let settings = TranscriptionSettings()
self.transcriptionSettings = settings
let transcription = TranscriptionService(
syncEngine: engine,
settings: settings,
modelContainer: container
)
self.transcriptionService = transcription
// A watch recording, once ingested, is this device's to transcribe.
engine.onAudioNoteIngested = { [weak transcription] id in
transcription?.enqueue(noteID: id)
}
}
/// The file extension this app owns for backup documents. Used to route
@@ -30,5 +47,12 @@ final class AppServices {
/// Slow startup work (iCloud container discovery) runs after the UI is up.
func startDeferredServices() async {
await syncEngine.connect()
// Drain any watch recordings delivered while the app was closed the
// container is connected now, so they can be written and ingested.
await syncEngine.ingestStagedWatchRecordings()
// Load the supported languages / reservation cap, then pick up any of
// this device's notes that were still pending when it last quit.
await transcriptionSettings.prepare()
transcriptionService.resumePendingWork()
}
}
+114
View File
@@ -0,0 +1,114 @@
#if os(iOS)
import Foundation
import os
import WatchConnectivity
/// Phone side of the watch recording hand-off. Receives a `WCSessionFile` the
/// watch sent via `transferFile`, stages it into the local `WatchInbox` (with the
/// `<ULID>.m4a` + `<ULID>.meta.plist` layout `SyncEngine.ingestStagedWatchRecordings`
/// expects), then asks the sync engine to drain the inbox.
///
/// The move happens **synchronously in the nonisolated delegate callback**: the
/// system deletes the received inbox file the instant `didReceive` returns, so
/// deferring the move to a `MainActor` hop would race the bytes away. Only after
/// staging do we hop to the main actor to trigger ingest.
///
/// Activated from the `UIApplicationDelegate` (not a SwiftUI `.task`) so a
/// background launch the system makes purely to deliver a transfer still stages
/// the file. `syncEngine` is wired in later, once `AppServices` exists; when it's
/// nil (a background launch with no UI yet) the staged file is drained on the
/// next foreground connect instead.
@MainActor
final class PhoneWatchReceiver: NSObject {
private nonisolated static let log = Logger(
subsystem: "dev.rzen.indie.Notes", category: "phone-watch-receiver")
/// The live sync engine, set once `AppServices` is built. Weak: `AppServices`
/// owns it for the app's lifetime; the receiver only borrows it to trigger a
/// drain when one is available.
weak var syncEngine: SyncEngine?
private var session: WCSession?
func activate() {
guard WCSession.isSupported() else { return }
let session = WCSession.default
session.delegate = self
session.activate()
self.session = session
}
}
// MARK: - WCSessionDelegate
extension PhoneWatchReceiver: WCSessionDelegate {
nonisolated func session(_ session: WCSession, activationDidCompleteWith activationState: WCSessionActivationState, error: (any Error)?) {
if let error { Self.log.error("phone WC activation failed: \(error, privacy: .public)") }
}
nonisolated func sessionDidBecomeInactive(_ session: WCSession) {}
nonisolated func sessionDidDeactivate(_ session: WCSession) {
// Reactivate for a switched watch.
session.activate()
}
nonisolated func session(_ session: WCSession, didReceive file: WCSessionFile) {
// Synchronous move the inbox file is deleted the moment this returns.
Self.stage(file: file)
// Drain if the engine is live; otherwise the next foreground connect drains.
Task { @MainActor in await self.syncEngine?.ingestStagedWatchRecordings() }
}
/// Move the received audio into `WatchInbox` and write the metadata sidecar
/// `SyncEngine`'s drain reads. On a decodable payload this stages a full
/// `<ULID>.m4a` + `<ULID>.meta.plist` pair. On an unknown/newer payload
/// version (the forward gate) the bytes are still quarantined in the inbox
/// but without a sidecar, so the drain skips them and a future build can pick
/// them up.
nonisolated private static func stage(file: WCSessionFile) {
let fm = FileManager.default
let inbox = SyncEngine.watchInboxURL
try? fm.createDirectory(at: inbox, withIntermediateDirectories: true)
let metadata = file.metadata ?? [:]
guard let payload = AudioNotePayload(metadata: metadata) else {
// Forward gate: keep the bytes but don't make them ingestible.
let base = (metadata["id"] as? String) ?? UUID().uuidString
let dest = inbox.appending(path: "\(base).m4a")
try? fm.removeItem(at: dest)
do {
try fm.moveItem(at: file.fileURL, to: dest)
log.error("staged watch recording \(base, privacy: .public) with unknown payload version — skipping ingest")
} catch {
log.error("failed to stage unknown-version recording: \(error, privacy: .public)")
}
return
}
let base = payload.id
let audioDest = inbox.appending(path: "\(base).m4a")
let metaDest = inbox.appending(path: "\(base).meta.plist")
// Idempotent for a redelivery of the same recording.
try? fm.removeItem(at: audioDest)
do {
try fm.moveItem(at: file.fileURL, to: audioDest)
} catch {
log.error("failed to stage recording \(base, privacy: .public): \(error, privacy: .public)")
return
}
var meta: [String: Any] = [
"id": payload.id,
"recordedAt": payload.recordedAt,
"duration": payload.duration,
]
if let tz = payload.timeZoneID { meta["timeZoneID"] = tz }
if let data = try? PropertyListSerialization.data(fromPropertyList: meta, format: .binary, options: 0) {
try? data.write(to: metaDest)
}
log.info("staged watch recording \(base, privacy: .public) for ingest")
}
}
#endif
+84 -5
View File
@@ -40,7 +40,10 @@ struct CaptureContext: Codable, Sendable, Equatable {
/// One note = one JSON file in the iCloud container, filed under
/// `Records/YYYY/MM/<ULID>.json` (UTC-bucketed by the ULID's timestamp).
struct Note: SyncDocument, VersionedDocument, Equatable {
static let currentSchemaVersion = 1
/// v2 added `Payload.audio` for voice notes. Because it is optional, a v1
/// file decodes unchanged (missing key nil) and `migrateIfNeeded` only
/// bumps the stamped version no field back-fill is needed.
static let currentSchemaVersion = 2
struct Meta: Codable, Sendable, Equatable {
var id: ULID
@@ -49,18 +52,43 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
var modifiedAt: Date
}
/// How the note's text came to be. `transcribed` is reserved for the
/// planned voice-capture path.
/// How the note's text came to be. `transcribed` marks a voice note whose
/// text is (or will be) produced by on-device transcription.
enum Source: String, Codable, Sendable {
case typed
case transcribed
}
/// Where a voice note is in its transcription lifecycle.
enum TranscriptionStatus: String, Codable, Sendable {
case pending
case done
case failed
}
/// The audio-note sidecar metadata. Present only on voice notes; the audio
/// bytes themselves live in a `.m4a` next to the note JSON (path derived,
/// never stored see `audioRelativePath`).
struct AudioInfo: Codable, Sendable, Equatable {
var duration: TimeInterval
/// Per-install UUID of the device that owns pending transcription. Only
/// the device whose `InstallIdentity.installID` matches this transcribes;
/// every other device renders status only. Re-transcribe reassigns it.
var transcriberDeviceID: String
var transcriptionStatus: TranscriptionStatus
/// BCP-47 locale the transcript was produced in; nil while pending.
var transcriptionLocaleID: String?
var transcribedAt: Date?
}
struct Payload: Codable, Sendable, Equatable {
var text: String
var source: Source
var tags: [String]
var context: CaptureContext
/// Voice-note sidecar metadata, nil for a plain typed note. Optional so
/// a v1 document (no `audio` key) decodes as a text note unchanged.
var audio: AudioInfo?
}
var meta: Meta
@@ -75,16 +103,67 @@ struct Note: SyncDocument, VersionedDocument, Equatable {
TimeBucketedLayout.relativePath(for: id)
}
/// The `.m4a` sidecar path for a note, derived by swapping `.json` `.m4a`
/// on the note's **actual** JSON path. The audio path is never stored it
/// is always this swap so it cannot drift from the note it belongs to, and
/// deleting a note whose file sits under a legacy (pre-fix) month bucket
/// still targets the sidecar that shares that bucket.
static func audioRelativePath(forJSONPath jsonPath: String) -> String {
guard jsonPath.hasSuffix(".json") else { return jsonPath + ".m4a" }
return String(jsonPath.dropLast(".json".count)) + ".m4a"
}
/// The `.m4a` sidecar path recomputed from a note id, sharing the note's
/// UTC-bucketed layout. Safe for load/warm paths because audio exists only
/// on v2+ records, which always file under the current UTC bucket the
/// legacy local-month concern applies solely to v1 text notes (no audio).
static func audioRelativePath(forID id: ULID) -> String {
audioRelativePath(forJSONPath: relativePath(forID: id))
}
static func create(
text: String,
source: Source = .typed,
tags: [String] = [],
context: CaptureContext
context: CaptureContext,
audio: AudioInfo? = nil
) -> Note {
let now = Date()
return Note(
meta: Meta(id: ULID(), schemaVersion: currentSchemaVersion, createdAt: now, modifiedAt: now),
payload: Payload(text: text, source: source, tags: tags, context: context)
payload: Payload(text: text, source: source, tags: tags, context: context, audio: audio)
)
}
/// Factory for a note ingested from another device's recording (the watch,
/// which has no iCloud of its own). The audio was captured elsewhere; this
/// device only stores and transcribes it. `createdAt` is the ULID's embedded
/// timestamp the record-start instant on the capturing device so the note
/// buckets under the month it was *recorded*, not the (possibly much later)
/// month it was ingested. Text is empty until transcription lands.
static func createIngested(
id: ULID,
duration: TimeInterval,
transcriberDeviceID: String,
tags: [String] = [],
context: CaptureContext
) -> Note {
let recordedAt = id.timestamp
return Note(
meta: Meta(id: id, schemaVersion: currentSchemaVersion, createdAt: recordedAt, modifiedAt: recordedAt),
payload: Payload(
text: "",
source: .transcribed,
tags: tags,
context: context,
audio: AudioInfo(
duration: duration,
transcriberDeviceID: transcriberDeviceID,
transcriptionStatus: .pending,
transcriptionLocaleID: nil,
transcribedAt: nil
)
)
)
}
+36
View File
@@ -30,6 +30,17 @@ final class NoteEntity {
var timeZoneID: String?
var device: String?
// Flattened AudioInfo nil columns mean a plain text note. `hasAudio` is
// stored rather than derived from `audioDuration` so a zero-length recording
// is still recognizably a voice note. `transcribedAt` is deliberately not
// cached (the file is authoritative for that timestamp), so a cache-only
// reconstruction of a completed note has a nil `transcribedAt`.
var hasAudio: Bool = false
var audioDuration: Double?
var transcriptionStatusRaw: String?
var transcriptionLocaleID: String?
var transcriberDeviceID: String?
init(id: String, jsonRelativePath: String) {
self.id = id
self.jsonRelativePath = jsonRelativePath
@@ -70,6 +81,31 @@ extension NoteEntity {
Note.Source(rawValue: sourceRaw) ?? .typed
}
/// Reconstructs `AudioInfo` from the flattened columns (nil for a text
/// note). `transcribedAt` is not cached, so it comes back nil even for a
/// completed transcription acceptable for the cache-only offline fallback,
/// where the file (which carries it) simply wasn't reachable.
var audio: Note.AudioInfo? {
get {
guard hasAudio else { return nil }
return Note.AudioInfo(
duration: audioDuration ?? 0,
transcriberDeviceID: transcriberDeviceID ?? "",
transcriptionStatus: transcriptionStatusRaw
.flatMap(Note.TranscriptionStatus.init(rawValue:)) ?? .pending,
transcriptionLocaleID: transcriptionLocaleID,
transcribedAt: nil
)
}
set {
hasAudio = newValue != nil
audioDuration = newValue?.duration
transcriberDeviceID = newValue?.transcriberDeviceID
transcriptionStatusRaw = newValue?.transcriptionStatus.rawValue
transcriptionLocaleID = newValue?.transcriptionLocaleID
}
}
/// First line of the text, used as the display title.
var title: String {
text.split(separator: "\n", omittingEmptySubsequences: true)
+3 -1
View File
@@ -35,6 +35,7 @@ enum NoteMapper {
entity.modifiedAt = note.meta.modifiedAt
entity.jsonRelativePath = relativePath
entity.context = note.payload.context
entity.audio = note.payload.audio
}
/// Near-lossless reconstruction from the cache, used only as the offline
@@ -52,7 +53,8 @@ enum NoteMapper {
text: entity.text,
source: entity.source,
tags: entity.tags,
context: entity.context
context: entity.context,
audio: entity.audio
)
)
}
+2 -1
View File
@@ -6,7 +6,8 @@ import SwiftData
/// from the files (an empty cache is the entire contract between the two).
enum NotesModelContainer {
/// Bump whenever the cache schema changes wipes and rebuilds from files.
static let cacheSchemaVersion = 1
/// v2 added the flattened audio columns on `NoteEntity`.
static let cacheSchemaVersion = 2
private static let cacheSchemaVersionKey = "Notes.cacheSchemaVersion"
private static let identityTokenKey = "Notes.iCloudIdentityToken"
+14
View File
@@ -4,6 +4,9 @@ import SwiftUI
@main
struct NotesApp: App {
#if os(iOS)
@UIApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate
#endif
@State private var appServices: AppServices?
@State private var pendingBackupURL: URL?
@Environment(\.scenePhase) private var scenePhase
@@ -16,6 +19,9 @@ struct NotesApp: App {
.modelContainer(appServices.modelContainer)
.environment(appServices.syncEngine)
.environment(appServices.contextService)
.environment(appServices.audioRecorder)
.environment(appServices.transcriptionSettings)
.environment(appServices.transcriptionService)
.environmentObject(appServices.backupController)
} else {
ProgressView()
@@ -25,6 +31,11 @@ struct NotesApp: App {
if appServices == nil {
let services = await AppServices()
appServices = services
#if os(iOS)
// Hand the (already-activated) watch receiver the live sync
// engine so a delivery arriving now drains immediately.
appDelegate.watchReceiver.syncEngine = services.syncEngine
#endif
await services.startDeferredServices()
if let pending = pendingBackupURL {
pendingBackupURL = nil
@@ -58,6 +69,9 @@ struct NotesApp: App {
// file swap would import a half-restored state.
if !appServices.backupController.isRestoring {
await syncEngine.reconcile()
// Ingest any watch recordings staged since we last
// looked (delivered while backgrounded).
await syncEngine.ingestStagedWatchRecordings()
}
case .checking:
break
+53 -49
View File
@@ -6,51 +6,6 @@
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
<string>Contextful</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>UILaunchScreen</key>
<dict/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>NSLocationWhenInUseUsageDescription</key>
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
<key>NSUbiquitousContainers</key>
<dict>
<key>iCloud.dev.rzen.indie.Notes</key>
<dict>
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
<true/>
<key>NSUbiquitousContainerSupportedFolderLevels</key>
<string>Any</string>
<key>NSUbiquitousContainerName</key>
<string>Notes</string>
</dict>
</dict>
<key>LSSupportsOpeningDocumentsInPlace</key>
<true/>
<key>CFBundleDocumentTypes</key>
<array>
<dict>
@@ -66,18 +21,67 @@
</array>
</dict>
</array>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>LSSupportsOpeningDocumentsInPlace</key>
<true/>
<key>NSLocationWhenInUseUsageDescription</key>
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
<key>NSMicrophoneUsageDescription</key>
<string>Notes records voice memos and transcribes them on-device, so you can capture a thought by speaking.</string>
<key>NSUbiquitousContainers</key>
<dict>
<key>iCloud.dev.rzen.indie.Notes</key>
<dict>
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
<true/>
<key>NSUbiquitousContainerName</key>
<string>Notes</string>
<key>NSUbiquitousContainerSupportedFolderLevels</key>
<string>Any</string>
</dict>
</dict>
<key>UILaunchScreen</key>
<dict/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UTExportedTypeDeclarations</key>
<array>
<dict>
<key>UTTypeIdentifier</key>
<string>dev.rzen.indie.Notes.backup</string>
<key>UTTypeDescription</key>
<string>Notes Backup</string>
<key>UTTypeConformsTo</key>
<array>
<string>public.data</string>
<string>public.archive</string>
</array>
<key>UTTypeDescription</key>
<string>Notes Backup</string>
<key>UTTypeIdentifier</key>
<string>dev.rzen.indie.Notes.backup</string>
<key>UTTypeTagSpecification</key>
<dict>
<key>public.filename-extension</key>
+42 -38
View File
@@ -6,40 +6,6 @@
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleDisplayName</key>
<string>Contextful</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>LSMinimumSystemVersion</key>
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
<key>LSApplicationCategoryType</key>
<string>public.app-category.productivity</string>
<key>NSLocationWhenInUseUsageDescription</key>
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
<key>NSLocationUsageDescription</key>
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
<key>NSUbiquitousContainers</key>
<dict>
<key>iCloud.dev.rzen.indie.Notes</key>
<dict>
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
<true/>
<key>NSUbiquitousContainerSupportedFolderLevels</key>
<string>Any</string>
<key>NSUbiquitousContainerName</key>
<string>Notes</string>
</dict>
</dict>
<key>CFBundleDocumentTypes</key>
<array>
<dict>
@@ -55,18 +21,56 @@
</array>
</dict>
</array>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>LSApplicationCategoryType</key>
<string>public.app-category.productivity</string>
<key>LSMinimumSystemVersion</key>
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
<key>NSLocationUsageDescription</key>
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
<key>NSLocationWhenInUseUsageDescription</key>
<string>Notes remembers where each note was taken, so the right notes can resurface when you return to the same place.</string>
<key>NSMicrophoneUsageDescription</key>
<string>Notes records voice memos and transcribes them on-device, so you can capture a thought by speaking.</string>
<key>NSUbiquitousContainers</key>
<dict>
<key>iCloud.dev.rzen.indie.Notes</key>
<dict>
<key>NSUbiquitousContainerIsDocumentScopePublic</key>
<true/>
<key>NSUbiquitousContainerName</key>
<string>Notes</string>
<key>NSUbiquitousContainerSupportedFolderLevels</key>
<string>Any</string>
</dict>
</dict>
<key>UTExportedTypeDeclarations</key>
<array>
<dict>
<key>UTTypeIdentifier</key>
<string>dev.rzen.indie.Notes.backup</string>
<key>UTTypeDescription</key>
<string>Notes Backup</string>
<key>UTTypeConformsTo</key>
<array>
<string>public.data</string>
<string>public.archive</string>
</array>
<key>UTTypeDescription</key>
<string>Notes Backup</string>
<key>UTTypeIdentifier</key>
<string>dev.rzen.indie.Notes.backup</string>
<key>UTTypeTagSpecification</key>
<dict>
<key>public.filename-extension</key>
+2
View File
@@ -4,6 +4,8 @@
<dict>
<key>com.apple.security.app-sandbox</key>
<true/>
<key>com.apple.security.device.audio-input</key>
<true/>
<key>com.apple.security.files.user-selected.read-write</key>
<true/>
<key>com.apple.security.network.client</key>
+190
View File
@@ -0,0 +1,190 @@
import AVFoundation
import Foundation
import IndieSync
/// Records a single voice note into a temporary `<ULID>.m4a` (AAC mono
/// 44.1 kHz ~48 kbps). Modeled on `ContextService`: `@Observable @MainActor`,
/// so the UI observes `isRecording` / `elapsed` / `level` directly.
///
/// Never records into the iCloud container the file is written to the temp
/// directory and handed up as a URL; `SyncEngine.createAudioNote` is what copies
/// the finished bytes into the container (audio before JSON). `AVAudioRecorder`
/// is not Sendable, so it stays confined to the main actor for its whole life.
@Observable
@MainActor
final class AudioRecorderService {
/// One finished recording: the temp file and how long it runs. The caller
/// owns the file from here on (moves its bytes into the container, then
/// discards the temp copy).
struct Recording: Equatable {
let url: URL
let duration: TimeInterval
}
private(set) var isRecording = false
/// Seconds recorded so far, updated by the metering loop while recording.
private(set) var elapsed: TimeInterval = 0
/// Normalized input level (01) for the level-bar UI; 0 when idle.
private(set) var level: Float = 0
/// Fired when a system interruption (phone call, Siri) forces recording to
/// stop. The partial file is finalized and handed up rather than lost, so the
/// UI can move straight to the "recorded" state. nil unless a recorder wires
/// it up for the duration of a take.
var onInterrupted: ((Recording) -> Void)?
private var recorder: AVAudioRecorder?
private var meterTask: Task<Void, Never>?
private var currentURL: URL?
#if os(iOS)
private var interruptionObserver: (any NSObjectProtocol)?
#endif
/// Ask for microphone access, returning whether it was granted. Safe to call
/// every time recording starts the system only prompts once.
func requestPermission() async -> Bool {
await AVAudioApplication.requestRecordPermission()
}
/// Whether the mic has been explicitly denied (so the UI can show a jump to
/// Settings instead of a dead record button).
var permissionDenied: Bool {
AVAudioApplication.shared.recordPermission == .denied
}
/// Begin recording into a fresh temp file. Throws if the audio session or
/// recorder can't be configured (surfaced by the UI). A no-op if already
/// recording.
func startRecording() throws {
guard !isRecording else { return }
#if os(iOS)
let session = AVAudioSession.sharedInstance()
try session.setCategory(.playAndRecord, mode: .spokenAudio, options: [.duckOthers])
try session.setActive(true)
observeInterruptions()
#endif
let url = FileManager.default.temporaryDirectory
.appending(path: "\(ULID().stringValue).m4a")
let settings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44_100.0,
AVNumberOfChannelsKey: 1,
AVEncoderBitRateKey: 48_000,
AVEncoderAudioQualityKey: AVAudioQuality.medium.rawValue,
]
let recorder = try AVAudioRecorder(url: url, settings: settings)
recorder.isMeteringEnabled = true
guard recorder.record() else {
throw RecorderError.couldNotStart
}
self.recorder = recorder
self.currentURL = url
isRecording = true
elapsed = 0
level = 0
startMetering()
}
/// Stop recording and return the finished file plus its duration, or nil if
/// nothing was recording. Deactivates the audio session on iOS.
@discardableResult
func stopRecording() -> Recording? {
guard let recorder, let url = currentURL else { return nil }
let duration = recorder.currentTime
recorder.stop()
teardown()
return Recording(url: url, duration: duration)
}
/// Abandon the in-progress (or just-finished) recording and delete its temp
/// file. Used by "Discard" and when the composer is dismissed unsaved.
func discard() {
recorder?.stop()
if let url = currentURL {
try? FileManager.default.removeItem(at: url)
}
teardown()
}
// MARK: - Metering
/// ~30 Hz loop that refreshes `elapsed` and `level` while recording. Runs on
/// the main actor (a plain `Task` inherits this actor), so it can touch the
/// non-Sendable recorder safely.
private func startMetering() {
meterTask?.cancel()
meterTask = Task { [weak self] in
while !Task.isCancelled {
guard let self, let recorder = self.recorder, recorder.isRecording else { return }
recorder.updateMeters()
self.elapsed = recorder.currentTime
self.level = Self.normalizedLevel(recorder.averagePower(forChannel: 0))
try? await Task.sleep(for: .milliseconds(33))
}
}
}
/// Map a dBFS reading (roughly -600) to a 01 bar height.
private static func normalizedLevel(_ power: Float) -> Float {
let floorDB: Float = -50
guard power > floorDB else { return 0 }
return min(1, (power - floorDB) / -floorDB)
}
private func teardown() {
meterTask?.cancel()
meterTask = nil
recorder = nil
currentURL = nil
isRecording = false
elapsed = 0
level = 0
#if os(iOS)
if let interruptionObserver {
NotificationCenter.default.removeObserver(interruptionObserver)
self.interruptionObserver = nil
}
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
#endif
}
// MARK: - Interruptions (iOS)
#if os(iOS)
private func observeInterruptions() {
guard interruptionObserver == nil else { return }
interruptionObserver = NotificationCenter.default.addObserver(
forName: AVAudioSession.interruptionNotification,
object: AVAudioSession.sharedInstance(),
queue: .main
) { [weak self] note in
// Extract the Sendable interruption type before hopping the
// Notification itself isn't Sendable. Delivered on the main queue, so
// assuming main-actor isolation is sound.
let began = (note.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt)
.flatMap(AVAudioSession.InterruptionType.init(rawValue:)) == .began
guard began else { return }
MainActor.assumeIsolated {
self?.handleInterruptionBegan()
}
}
}
private func handleInterruptionBegan() {
guard isRecording else { return }
// Finalize rather than drop the take the partial recording is still a
// valid note.
if let finished = stopRecording() {
onInterrupted?(finished)
}
}
#endif
enum RecorderError: Error {
case couldNotStart
}
}
+24
View File
@@ -0,0 +1,24 @@
import Foundation
/// Stable per-install identifier a UUID minted once on first access and kept
/// in UserDefaults for the life of the install. Distinct from the iCloud
/// account: two devices signed into the same account get different install IDs.
///
/// It answers "which device owns a pending transcription?" A voice note stamps
/// the recording device's `installID` into `AudioInfo.transcriberDeviceID`; only
/// the device whose `installID` matches transcribes it, while the others render
/// status only. Re-transcribing on any device reassigns the id to itself, which
/// is also the recovery path when the original recording device is gone.
enum InstallIdentity {
private static let key = "Notes.installID"
/// The per-install UUID, created lazily on first read and reused forever.
static var installID: String {
if let existing = UserDefaults.standard.string(forKey: key) {
return existing
}
let fresh = UUID().uuidString
UserDefaults.standard.set(fresh, forKey: key)
return fresh
}
}
+57
View File
@@ -0,0 +1,57 @@
import Foundation
import Speech
/// One language's showing in the sampling pass: how well `SpeechTranscriber`
/// transcribed the first few seconds of audio in that locale.
struct LanguageCandidate: Equatable {
/// BCP-47 identifier of the locale (e.g. `en-US`, `ru-RU`).
let localeID: String
/// Character-count-weighted mean of the per-run transcription confidence,
/// 01. A locale that produced no text scores 0.
let meanConfidence: Double
/// Number of characters the transcript came out to the tie-breaker and the
/// reason an empty transcript can't win.
let transcriptLength: Int
}
/// Pure, testable language selection for a voice note. No `SpeechAnalyzer` calls
/// live here: the service transcribes a ~15 s sample in each enabled language,
/// turns each into a `LanguageCandidate`, and this picks the winner. Keeping it
/// side-effect-free is what makes the (undocumented, cross-locale) confidence
/// calibration unit-testable.
enum LanguagePicker {
/// Character-count-weighted mean confidence over an attributed transcript.
/// Only runs that actually carry a `transcriptionConfidence` attribute
/// contribute, weighted by their character length; a transcript with no
/// confidence anywhere (or no characters) scores 0.
static func meanConfidence(of attributed: AttributedString) -> Double {
var weighted = 0.0
var totalCharacters = 0
for run in attributed.runs {
guard let confidence = run.transcriptionConfidence else { continue }
let length = attributed[run.range].characters.count
guard length > 0 else { continue }
weighted += confidence * Double(length)
totalCharacters += length
}
return totalCharacters > 0 ? weighted / Double(totalCharacters) : 0
}
/// The best candidate, or nil when given nothing. Argmax on `meanConfidence`;
/// a zero-length transcript can only win if every candidate is empty. Ties
/// break deterministically higher confidence, then longer transcript, then
/// lexicographically smaller `localeID` so the same samples always yield the
/// same pick.
static func pick(from candidates: [LanguageCandidate]) -> LanguageCandidate? {
candidates.max { lhs, rhs in
if lhs.meanConfidence != rhs.meanConfidence {
return lhs.meanConfidence < rhs.meanConfidence
}
if lhs.transcriptLength != rhs.transcriptLength {
return lhs.transcriptLength < rhs.transcriptLength
}
// `max` keeps the greater element; invert so the smaller localeID wins.
return lhs.localeID > rhs.localeID
}
}
}
+252
View File
@@ -0,0 +1,252 @@
import AVFoundation
import Foundation
import IndieSync
import Speech
import SwiftData
/// On-device, after-the-fact transcription of voice notes. `@Observable
/// @MainActor` with a FIFO one-at-a-time queue: a `SpeechAnalyzer` is memory- and
/// model-heavy, so notes are transcribed strictly one after another. The analyzer
/// work is awaited (the analyzer is an actor), so it doesn't block the main actor
/// despite this class being main-actor confined.
///
/// Only this device's own pending notes are ever enqueued ownership is decided
/// by `AudioInfo.transcriberDeviceID` upstream (`SyncEngine`), which this service
/// trusts. A failed job is terminal until the user asks to retry.
@Observable
@MainActor
final class TranscriptionService {
private let syncEngine: SyncEngine
private let settings: TranscriptionSettings
private let modelContainer: ModelContainer
/// Seconds of audio the sampling pass listens to when choosing a language.
private static let sampleSeconds: Double = 15
private struct Job {
let noteID: ULID
/// A fresh recording's temp file, passed on enqueue for the fast path; nil
/// for re-transcription, which pulls the audio back down from the container.
let localAudioURL: URL?
/// A forced language (re-transcribe menu), else auto-select.
let forcedLocaleID: String?
}
private var queue: [Job] = []
/// Ids currently queued or in flight, so overlapping triggers (launch resume,
/// watch ingest, a re-transcribe) can't enqueue the same note twice.
private var inFlightIDs: Set<String> = []
private var pumpTask: Task<Void, Never>?
init(syncEngine: SyncEngine, settings: TranscriptionSettings, modelContainer: ModelContainer) {
self.syncEngine = syncEngine
self.settings = settings
self.modelContainer = modelContainer
}
// MARK: - Queue
/// Queue a note for transcription. `localAudioURL` is the fresh-recording fast
/// path (skips the container round-trip); `forcedLocaleID` skips language
/// auto-selection. Duplicate enqueues of an in-flight note are ignored.
func enqueue(noteID: ULID, localAudioURL: URL? = nil, forcedLocaleID: String? = nil) {
let key = noteID.stringValue
guard !inFlightIDs.contains(key) else { return }
inFlightIDs.insert(key)
queue.append(Job(noteID: noteID, localAudioURL: localAudioURL, forcedLocaleID: forcedLocaleID))
pump()
}
/// On launch, re-enqueue every note this device owns that is still pending
/// recovering work that a crash or kill interrupted mid-transcription. Reads
/// the cache (already rebuilt by the sync engine's connect).
func resumePendingWork() {
let context = modelContainer.mainContext
let installID = TranscriptionSettings.installID
let pendingRaw = Note.TranscriptionStatus.pending.rawValue
let descriptor = FetchDescriptor<NoteEntity>(predicate: #Predicate {
$0.hasAudio
&& $0.transcriptionStatusRaw == pendingRaw
&& $0.transcriberDeviceID == installID
})
let rows = (try? context.fetch(descriptor)) ?? []
for row in rows {
if let id = ULID(string: row.id) { enqueue(noteID: id) }
}
}
private func pump() {
guard pumpTask == nil else { return }
pumpTask = Task { [weak self] in
while let self, let job = self.dequeue() {
await self.process(job)
self.inFlightIDs.remove(job.noteID.stringValue)
}
self?.pumpTask = nil
}
}
private func dequeue() -> Job? {
queue.isEmpty ? nil : queue.removeFirst()
}
// MARK: - Processing
/// Transcribe one job, retrying once before marking the note failed. A failed
/// note is left for the user to retry explicitly (re-transcribe / Retry).
private func process(_ job: Job) async {
do {
try await transcribe(job)
} catch {
do {
try await transcribe(job)
} catch {
print("[Transcription] \(job.noteID.stringValue) failed: \(error)")
try? await syncEngine.markTranscriptionFailed(id: job.noteID)
}
}
}
private func transcribe(_ job: Job) async throws {
let audio = try await resolveAudioFile(for: job)
defer { cleanUp(audio: audio, job: job) }
let locale = try await selectLocale(fileURL: audio.url, forcedLocaleID: job.forcedLocaleID)
let transcript = try await runTranscription(fileURL: audio.url, locale: locale)
let text = String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
try await syncEngine.applyTranscription(
id: job.noteID,
text: text,
localeID: TranscriptionSettings.canonicalID(locale)
)
}
// MARK: - Audio acquisition
private struct AudioFile {
let url: URL
/// Whether we wrote a temp copy that must be cleaned up after the job.
let isTemporary: Bool
}
/// The fresh recording if it's still on disk, else a temp copy pulled back
/// from the container (bounded wait for an evicted file).
private func resolveAudioFile(for job: Job) async throws -> AudioFile {
if let local = job.localAudioURL, FileManager.default.fileExists(atPath: local.path) {
return AudioFile(url: local, isTemporary: false)
}
let data = try await syncEngine.loadAudioDataWaiting(for: job.noteID)
let url = FileManager.default.temporaryDirectory
.appending(path: "transcribe-\(job.noteID.stringValue).m4a")
try data.write(to: url)
return AudioFile(url: url, isTemporary: true)
}
private func cleanUp(audio: AudioFile, job: Job) {
let fm = FileManager.default
if audio.isTemporary { try? fm.removeItem(at: audio.url) }
// A fresh recording's temp file is no longer needed once the container
// holds the bytes; drop it to keep the temp dir tidy.
if let local = job.localAudioURL,
local.path.hasPrefix(fm.temporaryDirectory.path) {
try? fm.removeItem(at: local)
}
}
// MARK: - Language selection
private func selectLocale(fileURL: URL, forcedLocaleID: String?) async throws -> Locale {
if let forcedLocaleID {
return Locale(identifier: forcedLocaleID)
}
let candidates = await settings.enabledInstalledLocales()
guard !candidates.isEmpty else { throw TranscriptionError.noInstalledLanguage }
if candidates.count == 1 { return candidates[0] }
// Sampling pass: transcribe a short clip in each language and pick the
// most confident. Run sequentially one analyzer at a time keeps the
// memory ceiling low.
let sample = try makeSampleClip(from: fileURL, seconds: Self.sampleSeconds)
defer { if sample.isTemporary { try? FileManager.default.removeItem(at: sample.url) } }
var scored: [LanguageCandidate] = []
for locale in candidates {
let transcript = (try? await runTranscription(fileURL: sample.url, locale: locale)) ?? AttributedString()
scored.append(LanguageCandidate(
localeID: TranscriptionSettings.canonicalID(locale),
meanConfidence: LanguagePicker.meanConfidence(of: transcript),
transcriptLength: transcript.characters.count
))
}
guard let winner = LanguagePicker.pick(from: scored) else { return candidates[0] }
return Locale(identifier: winner.localeID)
}
/// The first `seconds` of the recording as its own file, so the sampling pass
/// isn't run over a whole long note. Returns the original file (no copy) when
/// the recording is already shorter than the sample window.
private func makeSampleClip(from fileURL: URL, seconds: Double) throws -> AudioFile {
let source = try AVAudioFile(forReading: fileURL)
let format = source.processingFormat
let wantFrames = format.sampleRate * seconds
guard Double(source.length) > wantFrames else {
return AudioFile(url: fileURL, isTemporary: false)
}
let frameCount = AVAudioFrameCount(wantFrames)
guard frameCount > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) else {
throw TranscriptionError.audioUnreadable
}
try source.read(into: buffer, frameCount: frameCount)
let clipURL = FileManager.default.temporaryDirectory
.appending(path: "sample-\(UUID().uuidString).caf")
let clip = try AVAudioFile(forWriting: clipURL, settings: format.settings)
try clip.write(from: buffer)
return AudioFile(url: clipURL, isTemporary: true)
}
// MARK: - Analyzer
/// Transcribe a whole file in one locale and return the finalized attributed
/// transcript (carrying per-run confidence). Follows the documented pattern:
/// start a collector over `transcriber.results`, feed the file, finalize.
private func runTranscription(fileURL: URL, locale: Locale) async throws -> AttributedString {
let transcriber = SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: [.transcriptionConfidence]
)
let analyzer = SpeechAnalyzer(
modules: [transcriber],
options: SpeechAnalyzer.Options(priority: .utility, modelRetention: .whileInUse)
)
let file = try AVAudioFile(forReading: fileURL)
// Collect finalized results concurrently; `results` terminates when the
// analyzer finishes, ending the loop.
let collector = Task {
var full = AttributedString()
for try await result in transcriber.results where result.isFinal {
full.append(result.text)
}
return full
}
do {
if let lastSample = try await analyzer.analyzeSequence(from: file) {
try await analyzer.finalizeAndFinish(through: lastSample)
} else {
await analyzer.cancelAndFinishNow()
}
} catch {
collector.cancel()
throw error
}
return try await collector.value
}
enum TranscriptionError: Error {
case audioUnreadable
case noInstalledLanguage
}
}
+231
View File
@@ -0,0 +1,231 @@
import Foundation
import Speech
/// The user's transcription-language configuration and the Speech asset state
/// behind it. `@Observable @MainActor` so the Settings UI binds to it directly.
///
/// There is no automatic language detection in `SpeechAnalyzer`, so the user
/// enables a small set of languages; a new voice note is sampled in each and the
/// highest-confidence one wins (see `TranscriptionService`). Enabling a language
/// *reserves* it with `AssetInventory` which is both what keeps its model from
/// being purged and what the system caps at `AssetInventory.maximumReservedLocales`
/// so "enabled" and "reserved" are kept in lock-step and the UI enforces the cap.
@Observable
@MainActor
final class TranscriptionSettings {
private static let enabledKey = "Notes.enabledTranscriptionLocales"
private static let seededKey = "Notes.transcriptionLocalesSeeded"
/// Per-install id of the transcription owner, reused from `InstallIdentity`
/// a voice note stamps this into `AudioInfo.transcriberDeviceID` so only the
/// recording device transcribes it.
static var installID: String { InstallIdentity.installID }
/// Canonical BCP-47 ids of the enabled languages, persisted in UserDefaults.
private(set) var enabledLocaleIDs: [String]
/// Every locale `SpeechTranscriber` can transcribe (36+), loaded once by
/// `prepare()`. Empty until then.
private(set) var supportedLocales: [Locale] = []
/// Runtime cap on how many languages can be enabled at once read from the
/// system, not hard-coded, because it can change.
private(set) var maximumReserved = 0
/// Per-language asset state, keyed by canonical BCP-47 id, for the row UI.
private(set) var assetStates: [String: AssetState] = [:]
/// Where a language's on-device model sits.
enum AssetState: Equatable {
case unsupported
case notInstalled
/// Downloading; the fraction is nil until progress is known.
case downloading(Double?)
case installed
}
private let defaults: UserDefaults
init(defaults: UserDefaults = .standard) {
self.defaults = defaults
self.enabledLocaleIDs = defaults.stringArray(forKey: Self.enabledKey) ?? []
}
// MARK: - Canonical ids
/// The stable key for a locale everywhere in the app: its BCP-47 identifier
/// (`en-US`), so it round-trips through `Locale(identifier:)`.
static func canonicalID(_ locale: Locale) -> String { locale.identifier(.bcp47) }
/// The enabled languages as `Locale`s, in the order the user has them.
var enabledLocales: [Locale] { enabledLocaleIDs.map(Locale.init(identifier:)) }
/// Localized display name for a locale ("English (United States)"), falling
/// back to the raw identifier.
static func displayName(for locale: Locale) -> String {
Locale.current.localizedString(forIdentifier: locale.identifier(.bcp47))
?? locale.identifier(.bcp47)
}
// MARK: - Loading
/// One-time load of the supported-locale list and the reservation cap, plus a
/// first-run seed of the enabled set from the user's preferred languages. Idempotent
/// and cheap to call again it re-fetches the list but only seeds once.
func prepare() async {
maximumReserved = AssetInventory.maximumReservedLocales
supportedLocales = await SpeechTranscriber.supportedLocales
.sorted { Self.displayName(for: $0) < Self.displayName(for: $1) }
if !defaults.bool(forKey: Self.seededKey) {
seedDefaults()
defaults.set(true, forKey: Self.seededKey)
}
await refreshStates()
}
/// Seed the enabled set from `Locale.preferredLanguages supportedLocales`,
/// matched by language code and capped at the reservation limit. Falls back to
/// English (or the first supported locale) so a note recorded before the user
/// visits Settings still has something to transcribe with.
private func seedDefaults() {
guard enabledLocaleIDs.isEmpty else { return }
var seed: [String] = []
for preferred in Locale.preferredLanguages.map(Locale.init(identifier:)) {
guard let match = supportedLocales.first(where: {
$0.language.languageCode == preferred.language.languageCode
}) else { continue }
let id = Self.canonicalID(match)
if !seed.contains(id) { seed.append(id) }
}
if seed.isEmpty {
let fallback = supportedLocales.first { $0.language.languageCode?.identifier == "en" }
?? supportedLocales.first
if let fallback { seed = [Self.canonicalID(fallback)] }
}
enabledLocaleIDs = Array(seed.prefix(max(1, maximumReserved)))
persist()
// Reserve the seeded languages so their models are protected from purge.
Task { await self.reserveEnabled() }
}
private func reserveEnabled() async {
for locale in enabledLocales {
_ = try? await AssetInventory.reserve(locale: locale)
}
}
/// Refresh every supported locale's asset state (installed / downloadable /
/// unsupported). Runs sequentially `status(forModules:)` is cheap but there
/// are dozens of locales, and there's no need to fan out.
func refreshStates() async {
for locale in supportedLocales {
let id = Self.canonicalID(locale)
// Preserve an in-flight download's fraction rather than stomping it.
if case .downloading = assetStates[id] { continue }
assetStates[id] = await currentState(for: locale)
}
}
private func currentState(for locale: Locale) async -> AssetState {
switch await AssetInventory.status(forModules: [Self.transcriber(for: locale)]) {
case .unsupported: return .unsupported
case .supported: return .notInstalled
case .downloading: return .downloading(nil)
case .installed: return .installed
@unknown default: return .notInstalled
}
}
// MARK: - Enable / disable
var isAtCapacity: Bool { enabledLocaleIDs.count >= maximumReserved }
func isEnabled(_ locale: Locale) -> Bool {
enabledLocaleIDs.contains(Self.canonicalID(locale))
}
/// Enable a language: reserve it, mark it enabled, and download its model if
/// needed (surfacing progress through `assetStates`). Throws `atCapacity` when
/// already at the reservation cap.
func enable(_ locale: Locale) async throws {
let id = Self.canonicalID(locale)
guard !enabledLocaleIDs.contains(id) else { return }
guard !isAtCapacity else { throw SettingsError.atCapacity }
_ = try await AssetInventory.reserve(locale: locale)
enabledLocaleIDs.append(id)
persist()
try await installIfNeeded(locale)
}
/// Disable a language: drop it from the enabled set and release its
/// reservation (letting the system reclaim the model later). Best-effort.
func disable(_ locale: Locale) async {
let id = Self.canonicalID(locale)
enabledLocaleIDs.removeAll { $0 == id }
persist()
_ = await AssetInventory.release(reservedLocale: locale)
assetStates[id] = await currentState(for: locale)
}
private func installIfNeeded(_ locale: Locale) async throws {
let id = Self.canonicalID(locale)
let transcriber = Self.transcriber(for: locale)
guard await AssetInventory.status(forModules: [transcriber]) != .installed else {
assetStates[id] = .installed
return
}
guard let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) else {
assetStates[id] = await currentState(for: locale)
return
}
let progress = request.progress
assetStates[id] = .downloading(0)
// Poll the Progress so the @Observable dict updates the row; Progress KVO
// doesn't drive SwiftUI on its own.
let poll = Task { [weak self] in
while !Task.isCancelled {
try? await Task.sleep(for: .milliseconds(300))
guard let self else { return }
self.assetStates[id] = .downloading(progress.fractionCompleted)
if progress.isFinished { return }
}
}
defer { poll.cancel() }
try await request.downloadAndInstall()
assetStates[id] = await currentState(for: locale)
}
/// The subset of enabled languages whose models are installed the only ones
/// `TranscriptionService` can actually run.
func enabledInstalledLocales() async -> [Locale] {
var installed: [Locale] = []
for locale in enabledLocales
where await AssetInventory.status(forModules: [Self.transcriber(for: locale)]) == .installed {
installed.append(locale)
}
return installed
}
// MARK: - Helpers
/// A minimal transcriber for a locale, used purely for asset queries. The
/// attribute options don't affect asset status, so they're omitted here.
static func transcriber(for locale: Locale) -> SpeechTranscriber {
SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: []
)
}
private func persist() {
defaults.set(enabledLocaleIDs, forKey: Self.enabledKey)
}
enum SettingsError: Error {
case atCapacity
}
}
+289 -1
View File
@@ -37,6 +37,12 @@ final class SyncEngine {
/// the next successful write or rebuild.
private(set) var lastSyncError: String?
/// Fired after a watch recording is ingested, carrying the new note's id, so
/// the composition root can enqueue on-device transcription. Wired by
/// `AppServices`; nil until then, so ingestion is safe before the
/// transcription stack exists.
var onAudioNoteIngested: ((ULID) -> Void)?
static let recordsDirectory = "Records"
static let stubsDirectory = "Stubs"
@@ -54,6 +60,10 @@ final class SyncEngine {
private var dataMonitorTask: Task<Void, Never>?
private var stubMonitorTask: Task<Void, Never>?
/// Guards `ingestStagedWatchRecordings` against overlapping triggers (connect,
/// foreground reconcile, receiver callback) double-ingesting the same inbox.
private var isDrainingInbox = false
private var cacheContext: ModelContext { modelContainer.mainContext }
init(modelContainer: ModelContainer) {
@@ -142,6 +152,12 @@ final class SyncEngine {
}
let path = Note.relativePath(forID: id)
try await tombstones.softDelete(dataRelativePath: Self.dataPath(path), at: path)
// Drop the audio sidecar too. Best-effort and harmless when the note had
// no audio (nothing to remove); the sidecar never gets its own stub, so
// this is the only thing that reaps it.
if let store {
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
}
lastSyncError = nil
// Cache cleanup happens via the monitors (file removal + stub
// appearance both drop the row), but do it eagerly too so the UI
@@ -163,6 +179,273 @@ final class SyncEngine {
return NoteMapper.note(from: entity)
}
// MARK: - Audio notes
/// Create a voice note: write the `.m4a` sidecar **before** the JSON, so a
/// note that claims audio can never sync to another device ahead of its
/// bytes (which would surface a pending audio note whose `.m4a` doesn't yet
/// exist). The note starts with empty (or provisional) text and a `.pending`
/// transcription owned by `transcriberDeviceID`; the transcript arrives later
/// through `applyTranscription`. Like every other write, the cache catches up
/// via the monitor no direct upsert here.
@discardableResult
func createAudioNote(
audioFileURL: URL,
duration: TimeInterval,
text: String = "",
tags: [String] = [],
context: CaptureContext,
transcriberDeviceID: String
) async throws -> Note {
guard let store else {
report("Note not saved — iCloud not connected")
throw SyncError.iCloudUnavailable
}
let audio = Note.AudioInfo(
duration: duration,
transcriberDeviceID: transcriberDeviceID,
transcriptionStatus: .pending,
transcriptionLocaleID: nil,
transcribedAt: nil
)
let note = Note.create(text: text, source: .transcribed, tags: tags, context: context, audio: audio)
do {
let audioData = try Data(contentsOf: audioFileURL)
try await store.writeData(audioData, to: Self.dataPath(Note.audioRelativePath(forID: note.id)))
_ = try await store.write(note, to: Self.dataPath(note.relativePath))
} catch {
report("Failed to save audio note", error)
throw error
}
lastSyncError = nil
return note
}
/// Load a note's audio bytes for playback. Fail-fast (`.requestOnly`): an
/// evicted file kicks off a download and returns nil immediately rather than
/// stalling the player ~30s the UI shows a "downloading" retry state and
/// the bytes land on a later attempt (or via `requestAudioDownload`).
func loadAudioData(for id: ULID) async -> Data? {
guard let store else { return nil }
return try? await store.readData(from: audioPath(for: id), downloadPolicy: .requestOnly)
}
/// Load a note's audio with a bounded wait for an evicted file to download
/// for re-transcription, where blocking to fetch the source is acceptable
/// (unlike interactive playback, which uses `loadAudioData`). Throws on a
/// download timeout rather than yielding a dataless placeholder.
func loadAudioDataWaiting(for id: ULID) async throws -> Data {
guard let store else { throw SyncError.iCloudUnavailable }
return try await store.readData(from: audioPath(for: id), downloadPolicy: .waitBounded)
}
/// Warm an evicted audio file ahead of an anticipated playback, so a later
/// `loadAudioData` finds the bytes already local. Fire-and-forget.
func requestAudioDownload(for id: ULID) async {
guard let store else { return }
await store.requestDownload(at: audioPath(for: id))
}
/// Store-rooted `.m4a` sidecar path for a note. Audio exists only on v2+
/// records, which file under the current UTC bucket, so recomputing the path
/// from the id is always correct here.
private func audioPath(for id: ULID) -> String {
Self.dataPath(Note.audioRelativePath(forID: id))
}
// MARK: - Transcription
/// Land a completed transcription onto a note. Re-reads the file (the
/// authority) so a concurrent edit isn't clobbered, then **appends** rather
/// than replaces when the user typed while the note was pending non-empty
/// text whose `modifiedAt` moved past the note's `createdAt`. Otherwise the
/// transcript replaces the placeholder-empty text. The cache updates through
/// the monitor as usual (via `updateNote`).
func applyTranscription(id: ULID, text: String, localeID: String) async throws {
guard let store else { throw SyncError.iCloudUnavailable }
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
note.isReadable, var audio = note.payload.audio else { return }
let userEdited = !note.payload.text.isEmpty && note.meta.modifiedAt > note.meta.createdAt
note.payload.text = userEdited ? note.payload.text + "\n\n" + text : text
audio.transcriptionStatus = .done
audio.transcriptionLocaleID = localeID
audio.transcribedAt = Date()
note.payload.audio = audio
try await updateNote(note)
}
/// Flag a note's transcription as failed (retryable only by an explicit
/// re-transcribe). Re-reads first so it can't resurrect a since-deleted or
/// non-audio note.
func markTranscriptionFailed(id: ULID) async throws {
guard let store else { throw SyncError.iCloudUnavailable }
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
note.isReadable, var audio = note.payload.audio else { return }
audio.transcriptionStatus = .failed
note.payload.audio = audio
try await updateNote(note)
}
/// Reassign a note's transcription to *this* device and reset it to
/// `.pending`, optionally forcing a language (`localeID`, else auto-detect).
/// Claiming `transcriberDeviceID` is also the recovery path when the original
/// recording device is gone any device can take a stuck note over.
func requestRetranscription(id: ULID, localeID: String?) async throws {
guard let store else { throw SyncError.iCloudUnavailable }
guard var note = try? await store.read(Note.self, from: Self.dataPath(Note.relativePath(forID: id))),
note.isReadable, var audio = note.payload.audio else { return }
audio.transcriptionStatus = .pending
audio.transcriberDeviceID = InstallIdentity.installID
audio.transcriptionLocaleID = localeID
note.payload.audio = audio
try await updateNote(note)
}
// MARK: - Watch ingestion
/// Local directory where the phone receiver (`PhoneWatchReceiver`, a later
/// stage) stages recordings handed over from the watch before they are
/// ingested. It sits outside the iCloud container on purpose: the watch has
/// no iCloud, so a delivered recording becomes durable only once
/// `ingestWatchRecording` writes it into the container. Each staged
/// recording is a `<ULID>.m4a` plus a `<ULID>.meta.plist` sidecar carrying
/// `id` / `recordedAt` / `duration` / `timeZoneID`.
/// `nonisolated` so the phone receiver can stage a delivery synchronously in
/// its nonisolated `WCSessionDelegate` callback (the system reclaims the inbox
/// file the moment that returns). Pure no actor state so it is safe to
/// share this one source of truth for the inbox path with the drain.
nonisolated static var watchInboxURL: URL {
URL.applicationSupportDirectory.appending(path: "WatchInbox", directoryHint: .isDirectory)
}
/// Idempotent drain of the watch staging inbox. Called after `connect()`, on
/// foreground reconcile, and (a later stage) on the receiver callback, so a
/// recording delivered while the app was dead still lands. Serialized against
/// itself so overlapping triggers can't double-ingest.
func ingestStagedWatchRecordings() async {
guard store != nil, !isDrainingInbox else { return }
isDrainingInbox = true
defer { isDrainingInbox = false }
for staged in Self.stagedWatchRecordings() {
await ingestWatchRecording(
audioURL: staged.audioURL,
id: staged.id,
recordedAt: staged.recordedAt,
duration: staged.duration,
context: staged.context
)
}
}
/// Ingest one recording handed over from the watch the single sanctioned
/// direct-cache-upsert path (icloud-sync-engine §9). The phone is where the
/// note first becomes durable, so: veto if the note was already deleted
/// (tombstone) or already ingested (duplicate redelivery), then write audio
/// **before** JSON (same ordering invariant as `createAudioNote`), upsert the
/// cache directly (a WatchConnectivity delivery has no other live trigger to
/// refresh an open list), remove the staged files, and fire
/// `onAudioNoteIngested` so transcription can enqueue. Returns whether a new
/// note was created.
@discardableResult
func ingestWatchRecording(
audioURL: URL,
id: ULID,
recordedAt: Date,
duration: TimeInterval,
context: CaptureContext
) async -> Bool {
guard let store, let tombstones else { return false }
// Permanent skips clear the stale staged copy so it stops re-draining.
if await tombstones.stubExists(id: id.stringValue) {
Self.removeStagedFiles(audioURL: audioURL)
return false
}
let jsonPath = Note.relativePath(forID: id)
if await store.fileExists(Self.dataPath(jsonPath)) {
Self.removeStagedFiles(audioURL: audioURL)
return false
}
let note = Note.createIngested(
id: id,
duration: duration,
transcriberDeviceID: InstallIdentity.installID,
context: context
)
do {
let audioData = try Data(contentsOf: audioURL)
try await store.writeData(audioData, to: Self.dataPath(Note.audioRelativePath(forID: id)))
try await store.write(note, to: Self.dataPath(jsonPath))
} catch {
// Transient (iCloud write, or the staged file vanished mid-drain):
// leave the staged files for the next drain to retry.
report("Failed to ingest watch recording \(id.stringValue)", error)
return false
}
NoteMapper.upsert(note, relativePath: jsonPath, into: cacheContext)
try? cacheContext.save()
Self.removeStagedFiles(audioURL: audioURL)
lastSyncError = nil
print("[Sync] Ingested watch recording \(id.stringValue) recorded \(recordedAt)")
onAudioNoteIngested?(id)
return true
}
/// One recording staged in the watch inbox: the `.m4a` and the metadata the
/// receiver wrote in its `<ULID>.meta.plist` sidecar.
private struct StagedRecording {
var id: ULID
var recordedAt: Date
var duration: TimeInterval
var context: CaptureContext
var audioURL: URL
}
/// Enumerate the staging inbox, pairing each `<ULID>.meta.plist` with its
/// `.m4a`. A sidecar whose id doesn't parse, whose required fields are
/// missing, or whose audio is absent is skipped (a half-delivered pair can't
/// be ingested; the next full delivery re-stages it).
private static func stagedWatchRecordings() -> [StagedRecording] {
let fm = FileManager.default
guard let entries = try? fm.contentsOfDirectory(
at: watchInboxURL, includingPropertiesForKeys: nil) else { return [] }
var staged: [StagedRecording] = []
for metaURL in entries where metaURL.pathExtension == "plist" {
// `<ULID>.meta.plist` drop `.plist` then `.meta` to the ULID base.
let base = metaURL.deletingPathExtension().deletingPathExtension().lastPathComponent
guard let id = ULID(string: base),
let data = try? Data(contentsOf: metaURL),
let dict = try? PropertyListSerialization.propertyList(
from: data, options: [], format: nil) as? [String: Any],
let duration = dict["duration"] as? Double,
let recordedAt = dict["recordedAt"] as? Date else { continue }
let audioURL = watchInboxURL.appending(path: "\(base).m4a")
guard fm.fileExists(atPath: audioURL.path) else { continue }
var context = CaptureContext.empty
context.device = "Watch"
context.timeZoneID = dict["timeZoneID"] as? String
staged.append(StagedRecording(
id: id, recordedAt: recordedAt, duration: duration,
context: context, audioURL: audioURL))
}
return staged
}
/// Remove a staged recording's `.m4a` and its `<ULID>.meta.plist` sibling.
private static func removeStagedFiles(audioURL: URL) {
let fm = FileManager.default
try? fm.removeItem(at: audioURL)
try? fm.removeItem(at: audioURL.deletingPathExtension().appendingPathExtension("meta.plist"))
}
// MARK: - Persistence plumbing
/// Writes touch the file layer ONLY. iCloud JSON is the source of truth:
@@ -263,6 +546,7 @@ final class SyncEngine {
let id = DocumentFileStore.documentID(fromRelativePath: path)
if await tombstones.stubExists(id: id) {
try? await store.remove(at: Self.dataPath(path))
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
return
}
await importRecord(at: path, using: store)
@@ -275,6 +559,7 @@ final class SyncEngine {
private func handleStubAdded(relativePath: String) async {
if let store {
try? await store.remove(at: Self.dataPath(relativePath))
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: relativePath)))
}
removeFromCache(jsonRelativePath: relativePath)
}
@@ -344,9 +629,11 @@ final class SyncEngine {
isSyncing = true
defer { isSyncing = false }
// Clean up files a stub has tombstoned (mirrors the rebuild path).
// Clean up files a stub has tombstoned (mirrors the rebuild path),
// including each record's audio sidecar sibling.
for path in orphanedPaths {
try? await store.remove(at: Self.dataPath(path))
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
}
removeFromCache(ids: toRemove)
@@ -382,6 +669,7 @@ final class SyncEngine {
for path in await listRecordPaths(in: store) {
if stubIDs.contains(DocumentFileStore.documentID(fromRelativePath: path)) {
try? await store.remove(at: Self.dataPath(path))
try? await store.remove(at: Self.dataPath(Note.audioRelativePath(forJSONPath: path)))
continue
}
await importRecord(at: path, using: store)
+206
View File
@@ -0,0 +1,206 @@
import AVFoundation
import IndieSync
import SwiftUI
/// Plays back a voice note's audio. Two sources: a fresh recording still on disk
/// (`localURL`, used by the composer) or a saved note whose bytes come back
/// through `SyncEngine.loadAudioData` (fail-fast, so an evicted file shows a
/// "downloading" retry state instead of stalling). `AVAudioPlayer` is not
/// Sendable, so the playback model stays main-actor confined.
struct AudioPlayerView: View {
enum Source: Equatable {
case localURL(URL)
case note(id: ULID, duration: TimeInterval)
}
let source: Source
@Environment(SyncEngine.self) private var syncEngine
@State private var model = AudioPlaybackModel()
var body: some View {
Group {
switch model.loadState {
case .idle, .loading:
HStack {
ProgressView().controlSize(.small)
Text("Loading audio…").foregroundStyle(.secondary)
}
.font(.callout)
case .downloading:
VStack(alignment: .leading, spacing: 6) {
Label("Downloading from iCloud…", systemImage: "icloud.and.arrow.down")
.foregroundStyle(.secondary)
Button("Retry") { Task { await load() } }
.buttonStyle(.bordered)
.controlSize(.small)
}
.font(.callout)
case .failed:
Label("Couldn't play audio", systemImage: "exclamationmark.triangle")
.foregroundStyle(.secondary)
.font(.callout)
case .ready:
player
}
}
.task { await load() }
.onDisappear { model.stop() }
}
private var player: some View {
HStack(spacing: 12) {
Button {
model.togglePlay()
} label: {
Image(systemName: model.isPlaying ? "pause.circle.fill" : "play.circle.fill")
.font(.largeTitle)
.foregroundStyle(.tint)
}
.buttonStyle(.plain)
VStack(spacing: 2) {
Slider(
value: Binding(get: { model.progress }, set: { model.seek(toFraction: $0) }),
in: 0...1
)
HStack {
Text(Self.timeString(model.currentTime))
Spacer()
Text(Self.timeString(model.duration))
}
.font(.caption2.monospacedDigit())
.foregroundStyle(.secondary)
}
}
}
private func load() async {
switch source {
case .localURL(let url):
model.loadLocal(url)
case .note(let id, let duration):
await model.loadNote(id: id, fallbackDuration: duration, syncEngine: syncEngine)
}
}
static func timeString(_ interval: TimeInterval) -> String {
let total = Int(interval.rounded())
return String(format: "%d:%02d", total / 60, total % 60)
}
}
/// Owns the `AVAudioPlayer` and a lightweight progress ticker. Main-actor
/// confined because `AVAudioPlayer` isn't Sendable.
@Observable
@MainActor
final class AudioPlaybackModel {
enum LoadState: Equatable { case idle, loading, ready, downloading, failed }
private(set) var loadState: LoadState = .idle
private(set) var isPlaying = false
/// 01 position, bound to the scrubber.
private(set) var progress: Double = 0
private(set) var duration: TimeInterval = 0
var currentTime: TimeInterval { duration * progress }
private var player: AVAudioPlayer?
private var ticker: Task<Void, Never>?
func loadLocal(_ url: URL) {
loadState = .loading
do {
let player = try AVAudioPlayer(contentsOf: url)
player.prepareToPlay()
adopt(player)
} catch {
loadState = .failed
}
}
/// Fetch a saved note's audio (fail-fast). A miss means the file is evicted:
/// show the downloading state and warm it up for the retry.
func loadNote(id: ULID, fallbackDuration: TimeInterval, syncEngine: SyncEngine) async {
loadState = .loading
duration = fallbackDuration
guard let data = await syncEngine.loadAudioData(for: id) else {
loadState = .downloading
await syncEngine.requestAudioDownload(for: id)
return
}
do {
let player = try AVAudioPlayer(data: data)
player.prepareToPlay()
adopt(player)
} catch {
loadState = .failed
}
}
private func adopt(_ player: AVAudioPlayer) {
self.player = player
duration = player.duration
progress = 0
loadState = .ready
}
func togglePlay() {
guard let player else { return }
if player.isPlaying {
player.pause()
isPlaying = false
ticker?.cancel()
} else {
activatePlaybackSession()
player.play()
isPlaying = true
startTicker()
}
}
func seek(toFraction fraction: Double) {
guard let player else { return }
progress = min(max(fraction, 0), 1)
player.currentTime = progress * duration
}
func stop() {
player?.stop()
isPlaying = false
ticker?.cancel()
ticker = nil
#if os(iOS)
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
#endif
}
private func startTicker() {
ticker?.cancel()
ticker = Task { [weak self] in
while !Task.isCancelled {
guard let self, let player = self.player else { return }
if player.isPlaying {
self.progress = self.duration > 0 ? player.currentTime / self.duration : 0
} else {
// Reached the end (or was paused elsewhere): reset to the start.
self.isPlaying = false
self.progress = 0
return
}
try? await Task.sleep(for: .milliseconds(50))
}
}
}
private func activatePlaybackSession() {
#if os(iOS)
let session = AVAudioSession.sharedInstance()
try? session.setCategory(.playback, mode: .spokenAudio)
try? session.setActive(true)
#endif
}
}
@@ -0,0 +1,145 @@
import SwiftUI
/// The record surface in the composer: a mic button that, once tapped (or
/// auto-started), shows the elapsed time and a live level meter with Stop /
/// Discard. When recording ends it hands the finished `(url, duration)` up via
/// `onFinish`; the composer owns everything after that.
struct AudioRecordingControl: View {
@Environment(AudioRecorderService.self) private var recorder
/// Begin recording as soon as the view appears (the toolbar "record a note"
/// entry point), after the permission prompt resolves.
var autostart = false
let onFinish: (URL, TimeInterval) -> Void
@State private var permissionDenied = false
@State private var startError: String?
@State private var didAutostart = false
var body: some View {
VStack(spacing: 12) {
if permissionDenied {
deniedState
} else if recorder.isRecording {
recordingState
} else {
idleState
}
if let startError {
Text(startError)
.font(.footnote)
.foregroundStyle(.red)
}
}
.frame(maxWidth: .infinity)
.task {
recorder.onInterrupted = { recording in onFinish(recording.url, recording.duration) }
if autostart, !didAutostart {
didAutostart = true
await beginRecording()
}
}
.onDisappear {
recorder.onInterrupted = nil
if recorder.isRecording { recorder.discard() }
}
}
private var idleState: some View {
Button {
Task { await beginRecording() }
} label: {
VStack(spacing: 6) {
Image(systemName: "mic.circle.fill")
.font(.system(size: 52))
.foregroundStyle(.tint)
Text("Record")
.font(.callout)
}
}
.buttonStyle(.plain)
}
private var recordingState: some View {
VStack(spacing: 14) {
LevelMeter(level: recorder.level)
Text(AudioPlayerView.timeString(recorder.elapsed))
.font(.title2.monospacedDigit())
.foregroundStyle(.primary)
HStack(spacing: 24) {
Button(role: .destructive) {
recorder.discard()
} label: {
Label("Discard", systemImage: "trash")
}
.buttonStyle(.bordered)
Button {
if let finished = recorder.stopRecording() {
onFinish(finished.url, finished.duration)
}
} label: {
Label("Stop", systemImage: "stop.circle.fill")
}
.buttonStyle(.borderedProminent)
}
}
}
private var deniedState: some View {
VStack(spacing: 8) {
Image(systemName: "mic.slash.circle.fill")
.font(.system(size: 44))
.foregroundStyle(.secondary)
Text("Microphone access is off. Enable it for Contextful in Settings to record voice notes.")
.font(.footnote)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
}
}
private func beginRecording() async {
startError = nil
guard await recorder.requestPermission() else {
permissionDenied = true
return
}
permissionDenied = false
do {
try recorder.startRecording()
} catch {
startError = "Couldn't start recording: \(error.localizedDescription)"
}
}
}
/// A small row of bars whose height rides the current input level enough to
/// show the mic is live without pretending to be a real waveform.
private struct LevelMeter: View {
let level: Float
private let barCount = 9
/// Fixed per-bar weights so the meter has a stable, mic-like shape.
private let weights: [CGFloat] = [0.35, 0.55, 0.75, 0.9, 1.0, 0.9, 0.75, 0.55, 0.35]
var body: some View {
HStack(alignment: .center, spacing: 4) {
ForEach(0..<barCount, id: \.self) { index in
Capsule()
.fill(.tint)
.frame(width: 4, height: height(index))
}
}
.frame(height: 40)
.animation(.easeOut(duration: 0.08), value: level)
}
private func height(_ index: Int) -> CGFloat {
let base: CGFloat = 4
let peak: CGFloat = 36
return base + peak * CGFloat(level) * weights[index]
}
}
+145 -3
View File
@@ -4,6 +4,10 @@ import SwiftUI
/// Composer for a new note and editor for an existing one. On create it
/// snapshots the current context (refreshing location in the background);
/// on edit the original capture context is preserved untouched.
///
/// A note can carry audio: create mode can record one (optionally auto-starting
/// from the toolbar mic), and edit mode plays it back, shows transcription
/// status, and offers a per-note "Re-transcribe" override.
struct NoteEditorView: View {
enum Mode {
case create
@@ -11,25 +15,38 @@ struct NoteEditorView: View {
}
let mode: Mode
/// Begin recording the moment the composer appears (toolbar mic entry point).
var startRecording = false
@Environment(SyncEngine.self) private var syncEngine
@Environment(ContextService.self) private var contextService
@Environment(TranscriptionService.self) private var transcriptionService
@Environment(TranscriptionSettings.self) private var transcriptionSettings
@Environment(\.dismiss) private var dismiss
@State private var text = ""
@State private var tagsText = ""
@State private var saveError: String?
@State private var recordedAudio: AudioRecorderService.Recording?
@FocusState private var textFocused: Bool
var body: some View {
NavigationStack {
Form {
if isCreating {
audioComposeSection
}
Section {
TextEditor(text: $text)
.focused($textFocused)
.frame(minHeight: 160)
}
if case .edit(let note) = mode, note.hasAudio {
audioPlaybackSection(for: note)
}
Section("Tags") {
TextField("coffee, people, ideas…", text: $tagsText)
.autocorrectionDisabled()
@@ -65,14 +82,15 @@ struct NoteEditorView: View {
}
ToolbarItem(placement: .confirmationAction) {
Button("Save") { save() }
.disabled(text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
.disabled(!canSave)
}
}
}
.task {
switch mode {
case .create:
textFocused = true
// Don't steal focus from the mic entry point.
if !startRecording { textFocused = true }
await contextService.refresh()
case .edit(let note):
text = note.text
@@ -84,11 +102,119 @@ struct NoteEditorView: View {
#endif
}
// MARK: - Audio (create)
@ViewBuilder
private var audioComposeSection: some View {
Section("Voice") {
if let recordedAudio {
AudioPlayerView(source: .localURL(recordedAudio.url))
Button(role: .destructive) {
try? FileManager.default.removeItem(at: recordedAudio.url)
self.recordedAudio = nil
} label: {
Label("Discard Recording", systemImage: "trash")
}
} else {
AudioRecordingControl(autostart: startRecording) { url, duration in
recordedAudio = AudioRecorderService.Recording(url: url, duration: duration)
}
.padding(.vertical, 4)
}
}
}
// MARK: - Audio (edit)
@ViewBuilder
private func audioPlaybackSection(for note: NoteEntity) -> some View {
Section("Voice") {
if let id = ULID(string: note.id) {
AudioPlayerView(source: .note(id: id, duration: note.audioDuration ?? 0))
}
transcriptionStatusRow(for: note)
retranscribeMenu(for: note)
}
}
@ViewBuilder
private func transcriptionStatusRow(for note: NoteEntity) -> some View {
switch note.audio?.transcriptionStatus {
case .pending:
if note.transcriberDeviceID == TranscriptionSettings.installID {
Label {
Text("Transcribing…")
} icon: {
ProgressView().controlSize(.small)
}
.foregroundStyle(.secondary)
} else {
Label("Waiting to transcribe on another device", systemImage: "hourglass")
.foregroundStyle(.secondary)
}
case .failed:
HStack {
Label("Transcription failed", systemImage: "exclamationmark.triangle")
.foregroundStyle(.secondary)
Spacer()
Button("Retry") { retranscribe(note: note, localeID: nil) }
.buttonStyle(.bordered)
.controlSize(.small)
}
case .done:
if let localeID = note.audio?.transcriptionLocaleID {
Label(
"Transcribed · \(TranscriptionSettings.displayName(for: Locale(identifier: localeID)))",
systemImage: "checkmark.circle"
)
.foregroundStyle(.secondary)
.font(.footnote)
}
case .none:
EmptyView()
}
}
@ViewBuilder
private func retranscribeMenu(for note: NoteEntity) -> some View {
let locales = transcriptionSettings.enabledLocales
if !locales.isEmpty {
Menu {
Button("Auto-detect Language") { retranscribe(note: note, localeID: nil) }
Divider()
ForEach(locales, id: \.identifier) { locale in
Button(TranscriptionSettings.displayName(for: locale)) {
retranscribe(note: note, localeID: TranscriptionSettings.canonicalID(locale))
}
}
} label: {
Label("Re-transcribe", systemImage: "arrow.clockwise")
}
}
}
private func retranscribe(note: NoteEntity, localeID: String?) {
guard let id = ULID(string: note.id) else { return }
Task {
try? await syncEngine.requestRetranscription(id: id, localeID: localeID)
transcriptionService.enqueue(noteID: id, forcedLocaleID: localeID)
}
}
// MARK: - State
private var isCreating: Bool {
if case .create = mode { return true }
return false
}
private var canSave: Bool {
if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return true }
if recordedAudio != nil { return true }
if case .edit(let note) = mode, note.hasAudio { return true }
return false
}
private var parsedTags: [String] {
tagsText
.split(separator: ",")
@@ -96,18 +222,34 @@ struct NoteEditorView: View {
.filter { !$0.isEmpty }
}
// MARK: - Save
private func save() {
guard canSave else { return }
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
Task {
do {
switch mode {
case .create:
if let recordedAudio {
let note = try await syncEngine.createAudioNote(
audioFileURL: recordedAudio.url,
duration: recordedAudio.duration,
text: trimmed,
tags: parsedTags,
context: contextService.current,
transcriberDeviceID: TranscriptionSettings.installID
)
// Fast path: hand the recording's temp file straight to the
// transcriber so it doesn't round-trip through the container.
transcriptionService.enqueue(noteID: note.id, localAudioURL: recordedAudio.url)
} else {
try await syncEngine.createNote(
text: trimmed,
tags: parsedTags,
context: contextService.current
)
}
case .edit(let entity):
guard let id = ULID(string: entity.id),
var note = await syncEngine.loadNote(id: id) else { return }
+35 -1
View File
@@ -5,11 +5,16 @@ struct NoteRowView: View {
var body: some View {
VStack(alignment: .leading, spacing: 4) {
Text(note.title.isEmpty ? "Untitled" : note.title)
Text(displayTitle)
.font(.body.weight(.medium))
.lineLimit(2)
HStack(spacing: 4) {
if note.hasAudio {
Image(systemName: "waveform")
Text(AudioPlayerView.timeString(note.audioDuration ?? 0))
Text("·")
}
Text(note.createdAt, format: .dateTime.day().month().year())
if let place = note.context.summary {
Text("·")
@@ -20,6 +25,8 @@ struct NoteRowView: View {
.font(.caption)
.foregroundStyle(.secondary)
transcriptionChip
if !note.tags.isEmpty {
Text(note.tags.map { "#\($0)" }.joined(separator: " "))
.font(.caption)
@@ -29,4 +36,31 @@ struct NoteRowView: View {
}
.padding(.vertical, 2)
}
/// Audio notes with no transcript yet still need a legible title.
private var displayTitle: String {
if !note.title.isEmpty { return note.title }
return note.hasAudio ? "Audio note" : "Untitled"
}
@ViewBuilder
private var transcriptionChip: some View {
switch note.audio?.transcriptionStatus {
case .pending:
capsule("Transcribing…", color: .secondary)
case .failed:
capsule("Transcription failed", color: .red)
case .done, .none:
EmptyView()
}
}
private func capsule(_ text: String, color: Color) -> some View {
Text(text)
.font(.caption2)
.foregroundStyle(color)
.padding(.horizontal, 8)
.padding(.vertical, 2)
.background(color.opacity(0.12), in: Capsule())
}
}
+11
View File
@@ -11,6 +11,7 @@ struct NotesListView: View {
@State private var searchText = ""
@State private var composing = false
@State private var recording = false
@State private var editingNote: NoteEntity?
var body: some View {
@@ -50,6 +51,13 @@ struct NotesListView: View {
}
}
.toolbar {
ToolbarItem(placement: .primaryAction) {
Button {
recording = true
} label: {
Label("Record Note", systemImage: "mic")
}
}
ToolbarItem(placement: .primaryAction) {
Button {
composing = true
@@ -62,6 +70,9 @@ struct NotesListView: View {
.sheet(isPresented: $composing) {
NoteEditorView(mode: .create)
}
.sheet(isPresented: $recording) {
NoteEditorView(mode: .create, startRecording: true)
}
.sheet(item: $editingNote) { note in
NoteEditorView(mode: .edit(note))
}
+8
View File
@@ -36,6 +36,14 @@ struct SettingsView: View {
.disabled(syncEngine.iCloudStatus != .available || syncEngine.isSyncing)
}
Section("Transcription") {
NavigationLink {
TranscriptionLanguagesView()
} label: {
Label("Languages", systemImage: "character.bubble")
}
}
Section("Context") {
ContextSummaryRow(
context: contextService.current,
@@ -0,0 +1,119 @@
import SwiftUI
/// Settings screen for choosing which languages voice notes can be transcribed
/// in. Enabling a language reserves it and downloads its on-device model; the
/// system caps how many can be reserved at once, so the list disables further
/// toggles at capacity and explains the limit in the footer.
struct TranscriptionLanguagesView: View {
@Environment(TranscriptionSettings.self) private var settings
@State private var errorMessage: String?
var body: some View {
List {
if let errorMessage {
Section {
Label(errorMessage, systemImage: "exclamationmark.triangle")
.font(.footnote)
.foregroundStyle(.red)
}
}
if !enabledLocales.isEmpty {
Section("Enabled") {
ForEach(enabledLocales, id: \.identifier) { row($0) }
}
}
Section {
ForEach(otherLocales, id: \.identifier) { row($0) }
} header: {
Text("All Languages")
} footer: {
Text(footerText)
}
}
.navigationTitle("Languages")
.overlay {
if settings.supportedLocales.isEmpty {
ProgressView()
}
}
.task { await settings.prepare() }
}
private var enabledLocales: [Locale] {
settings.supportedLocales.filter { settings.isEnabled($0) }
}
private var otherLocales: [Locale] {
settings.supportedLocales.filter { !settings.isEnabled($0) }
}
private var footerText: String {
guard settings.maximumReserved > 0 else {
return "Each new note is transcribed on device in your enabled languages."
}
return "Enable up to \(settings.maximumReserved) languages. Each new voice note is sampled in your enabled languages and transcribed in whichever fits best."
}
private func row(_ locale: Locale) -> some View {
let enabled = settings.isEnabled(locale)
return HStack {
VStack(alignment: .leading, spacing: 2) {
Text(TranscriptionSettings.displayName(for: locale))
stateSubtitle(for: locale)
}
Spacer()
Toggle("", isOn: Binding(
get: { settings.isEnabled(locale) },
set: { toggle(locale, on: $0) }
))
.labelsHidden()
.disabled(!enabled && settings.isAtCapacity)
}
}
@ViewBuilder
private func stateSubtitle(for locale: Locale) -> some View {
switch settings.assetStates[TranscriptionSettings.canonicalID(locale)] {
case .installed:
Label("Installed", systemImage: "checkmark.circle.fill")
.font(.caption)
.foregroundStyle(.green)
case .downloading(let fraction):
HStack(spacing: 6) {
ProgressView().controlSize(.small)
if let fraction {
Text("Downloading \(Int(fraction * 100))%")
} else {
Text("Downloading…")
}
}
.font(.caption)
.foregroundStyle(.secondary)
case .notInstalled:
Label("Available to download", systemImage: "icloud.and.arrow.down")
.font(.caption)
.foregroundStyle(.secondary)
case .unsupported, .none:
EmptyView()
}
}
private func toggle(_ locale: Locale, on: Bool) {
errorMessage = nil
Task {
if on {
do {
try await settings.enable(locale)
} catch TranscriptionSettings.SettingsError.atCapacity {
errorMessage = "You can enable up to \(settings.maximumReserved) languages. Turn one off first."
} catch {
errorMessage = "Couldn't enable \(TranscriptionSettings.displayName(for: locale)): \(error.localizedDescription)"
}
} else {
await settings.disable(locale)
}
}
}
}
+83
View File
@@ -0,0 +1,83 @@
import Foundation
import Testing
@testable import Notes
struct AudioNotePayloadTests {
@Test func metadataRoundTrips() throws {
let payload = AudioNotePayload(
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
recordedAt: Date(timeIntervalSince1970: 1_700_000_000),
duration: 12.5,
timeZoneID: "Europe/Berlin"
)
let decoded = try #require(AudioNotePayload(metadata: payload.metadata()))
#expect(decoded == payload)
}
@Test func metadataRoundTripsWithoutTimeZone() throws {
let payload = AudioNotePayload(
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
recordedAt: Date(timeIntervalSince1970: 1_700_000_100),
duration: 4,
timeZoneID: nil
)
let dict = payload.metadata()
#expect(dict["tz"] == nil)
let decoded = try #require(AudioNotePayload(metadata: dict))
#expect(decoded == payload)
}
/// A survivable trip through actual plist serialization the metadata really
/// crosses WatchConnectivity as a plist dictionary.
@Test func metadataSurvivesPlistSerialization() throws {
let payload = AudioNotePayload(
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
recordedAt: Date(timeIntervalSince1970: 1_700_000_050),
duration: 7.25,
timeZoneID: "America/Los_Angeles"
)
let data = try PropertyListSerialization.data(
fromPropertyList: payload.metadata(), format: .binary, options: 0)
let dict = try #require(
try PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any])
let decoded = try #require(AudioNotePayload(metadata: dict))
#expect(decoded.id == payload.id)
#expect(decoded.duration == payload.duration)
#expect(decoded.timeZoneID == payload.timeZoneID)
#expect(abs(decoded.recordedAt.timeIntervalSince(payload.recordedAt)) < 0.001)
}
@Test func decodeRejectsUnknownNewerVersion() {
var dict = AudioNotePayload(
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
recordedAt: Date(),
duration: 3,
timeZoneID: nil
).metadata()
dict["v"] = AudioNotePayload.currentVersion + 1
#expect(AudioNotePayload(metadata: dict) == nil)
}
@Test func decodeRejectsMissingVersion() {
let dict: [String: Any] = [
"id": "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
"recordedAt": Date(),
"duration": 3.0,
]
#expect(AudioNotePayload(metadata: dict) == nil)
}
@Test func decodeRejectsMissingRequiredField() {
// Version present, but no duration.
let dict: [String: Any] = [
"v": AudioNotePayload.currentVersion,
"id": "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
"recordedAt": Date(),
]
#expect(AudioNotePayload(metadata: dict) == nil)
}
@Test func decodeRejectsEmptyMetadata() {
#expect(AudioNotePayload(metadata: [:]) == nil)
}
}
+94
View File
@@ -0,0 +1,94 @@
import Foundation
import Speech
import Testing
@testable import Notes
struct LanguagePickerTests {
/// Build an attributed transcript out of `(text, confidence?)` fragments, so a
/// test can hand-craft the confidence attributes the picker reads.
private func transcript(_ fragments: [(String, Double?)]) -> AttributedString {
var result = AttributedString()
for (text, confidence) in fragments {
var fragment = AttributedString(text)
if let confidence { fragment.transcriptionConfidence = confidence }
result.append(fragment)
}
return result
}
// MARK: - Confidence extraction
@Test func meanConfidenceIsCharacterWeighted() {
// "hello" (5 chars @ 0.9) + " world" (6 chars @ 0.4) 0.627
let attributed = transcript([("hello", 0.9), (" world", 0.4)])
let mean = LanguagePicker.meanConfidence(of: attributed)
#expect(abs(mean - ((0.9 * 5 + 0.4 * 6) / 11)) < 0.0001)
}
@Test func meanConfidenceIgnoresRunsWithoutConfidence() {
// Only the confident run counts toward the mean.
let attributed = transcript([("sure", 1.0), (" maybe", nil)])
#expect(abs(LanguagePicker.meanConfidence(of: attributed) - 1.0) < 0.0001)
}
@Test func meanConfidenceOfEmptyOrUnattributedIsZero() {
#expect(LanguagePicker.meanConfidence(of: AttributedString("")) == 0)
#expect(LanguagePicker.meanConfidence(of: transcript([("no attributes", nil)])) == 0)
}
// MARK: - Argmax
@Test func pickChoosesHighestConfidence() {
let winner = LanguagePicker.pick(from: [
LanguageCandidate(localeID: "en-US", meanConfidence: 0.42, transcriptLength: 20),
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.88, transcriptLength: 18),
LanguageCandidate(localeID: "fr-FR", meanConfidence: 0.31, transcriptLength: 25),
])
#expect(winner?.localeID == "ru-RU")
}
@Test func pickReturnsNilForNoCandidates() {
#expect(LanguagePicker.pick(from: []) == nil)
}
// MARK: - Empty-transcript zeroing
@Test func emptyTranscriptNeverBeatsRealText() {
// A zero-length, zero-confidence candidate must lose to any real result.
let winner = LanguagePicker.pick(from: [
LanguageCandidate(localeID: "en-US", meanConfidence: 0, transcriptLength: 0),
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.05, transcriptLength: 3),
])
#expect(winner?.localeID == "ru-RU")
}
// MARK: - Tie-break determinism
@Test func equalConfidencePrefersLongerTranscript() {
let winner = LanguagePicker.pick(from: [
LanguageCandidate(localeID: "en-US", meanConfidence: 0.7, transcriptLength: 10),
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.7, transcriptLength: 30),
])
#expect(winner?.localeID == "ru-RU")
}
@Test func fullTieBreaksByLocaleID() {
// Identical confidence and length smallest localeID wins, deterministically.
let candidates = [
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.5, transcriptLength: 10),
LanguageCandidate(localeID: "en-US", meanConfidence: 0.5, transcriptLength: 10),
LanguageCandidate(localeID: "fr-FR", meanConfidence: 0.5, transcriptLength: 10),
]
#expect(LanguagePicker.pick(from: candidates)?.localeID == "en-US")
// Order of input must not change the result.
#expect(LanguagePicker.pick(from: candidates.reversed())?.localeID == "en-US")
}
@Test func allEmptyCandidatesStillPickDeterministically() {
let candidates = [
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0, transcriptLength: 0),
LanguageCandidate(localeID: "en-US", meanConfidence: 0, transcriptLength: 0),
]
#expect(LanguagePicker.pick(from: candidates)?.localeID == "en-US")
}
}
+107
View File
@@ -44,4 +44,111 @@ struct NoteDocumentTests {
@Test func titleIsFirstLine() {
#expect(makeNote().title == "Barista's name is Sam")
}
// MARK: - Schema v2 (audio)
/// A v1 file predates the `audio` key. It must decode unchanged as a text
/// note with `audio == nil` and pass the forward gate (1 current).
@Test func v1DocumentDecodesAsTextNoteWithNoAudio() throws {
let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 1, randomLo: 2)
let json = """
{
"meta": {
"id": "\(id.stringValue)",
"schemaVersion": 1,
"createdAt": "2023-11-14T22:13:20Z",
"modifiedAt": "2023-11-14T22:13:20Z"
},
"payload": {
"text": "A v1 note",
"source": "typed",
"tags": ["legacy"],
"context": { "device": "iPhone", "timeZoneID": "UTC" }
}
}
"""
let note = try DocumentCoder.decode(Note.self, from: Data(json.utf8))
#expect(note.meta.schemaVersion == 1)
#expect(note.payload.audio == nil)
#expect(note.payload.text == "A v1 note")
#expect(note.isReadable)
}
/// A file stamped with a newer schema (and carrying a field this build
/// doesn't model) must be quarantined by the gate Codable silently drops
/// the unknown key, so partial-decoding then rewriting would downgrade it.
@Test func forwardGateQuarantinesV3Fixture() throws {
let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 3, randomLo: 4)
let json = """
{
"meta": {
"id": "\(id.stringValue)",
"schemaVersion": 3,
"createdAt": "2023-11-14T22:13:20Z",
"modifiedAt": "2023-11-14T22:13:20Z"
},
"payload": {
"text": "From the future",
"source": "typed",
"tags": [],
"context": {},
"audio": null,
"unknownFutureField": { "whatever": true }
}
}
"""
let note = try DocumentCoder.decode(Note.self, from: Data(json.utf8))
#expect(note.meta.schemaVersion == 3)
#expect(!note.isReadable)
}
@Test func audioNoteCodableRoundTrip() throws {
var context = CaptureContext.empty
context.device = "Watch"
context.timeZoneID = "Europe/Berlin"
let audio = Note.AudioInfo(
duration: 12.5,
transcriberDeviceID: "install-abc",
transcriptionStatus: .done,
transcriptionLocaleID: "ru_RU",
transcribedAt: Date(timeIntervalSince1970: 1_700_000_100)
)
let note = Note.create(text: "привет", source: .transcribed, tags: ["voice"], context: context, audio: audio)
let data = try DocumentCoder.encode(note)
let decoded = try DocumentCoder.decode(Note.self, from: data)
#expect(decoded.payload == note.payload)
#expect(decoded.payload.audio?.transcriptionStatus == .done)
#expect(decoded.payload.audio?.transcriptionLocaleID == "ru_RU")
}
@Test func audioPathDerivationSwapsExtension() {
let id = ULID()
let jsonPath = Note.relativePath(forID: id)
#expect(jsonPath.hasSuffix(".json"))
#expect(Note.audioRelativePath(forID: id) == jsonPath.replacingOccurrences(of: ".json", with: ".m4a"))
#expect(Note.audioRelativePath(forID: id).hasSuffix("\(id.stringValue).m4a"))
}
/// The swap must operate on the note's actual path, so a record filed under
/// a pre-fix local-time-zone month keeps its sidecar in the same bucket.
@Test func audioPathPreservesLegacyBucket() {
let id = ULID()
let legacyJSONPath = "1999/12/\(id.stringValue).json"
#expect(Note.audioRelativePath(forJSONPath: legacyJSONPath) == "1999/12/\(id.stringValue).m4a")
}
@Test func createIngestedDerivesCreatedAtFromULID() {
let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 5, randomLo: 6)
var context = CaptureContext.empty
context.device = "Watch"
let note = Note.createIngested(id: id, duration: 8, transcriberDeviceID: "install-x", context: context)
#expect(note.meta.id == id)
#expect(note.meta.createdAt == id.timestamp)
#expect(note.meta.modifiedAt == id.timestamp)
#expect(note.payload.text.isEmpty)
#expect(note.payload.source == .transcribed)
#expect(note.payload.audio?.transcriptionStatus == .pending)
#expect(note.payload.audio?.transcriberDeviceID == "install-x")
#expect(note.relativePath == Note.relativePath(forID: id))
}
}
+65
View File
@@ -54,4 +54,69 @@ struct NoteMapperTests {
#expect(restored.payload == note.payload)
#expect(restored.meta.id == note.meta.id)
}
// MARK: - Audio columns
@Test func pendingAudioColumnsRoundTrip() throws {
let (container, context) = try makeStore()
defer { withExtendedLifetime(container) {} }
var ctx = CaptureContext.empty
ctx.device = "Watch"
let audio = Note.AudioInfo(
duration: 30,
transcriberDeviceID: "install-42",
transcriptionStatus: .pending,
transcriptionLocaleID: nil,
transcribedAt: nil
)
let note = Note.create(text: "", source: .transcribed, context: ctx, audio: audio)
NoteMapper.upsert(note, relativePath: note.relativePath, into: context)
let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context))
#expect(entity.hasAudio)
#expect(entity.audioDuration == 30)
#expect(entity.transcriberDeviceID == "install-42")
#expect(entity.transcriptionStatusRaw == "pending")
#expect(entity.source == .transcribed)
// A pending audio note carries no `transcribedAt`, so it reconstructs
// losslessly (that field is the only one the cache intentionally drops).
let restored = try #require(NoteMapper.note(from: entity))
#expect(restored.payload == note.payload)
#expect(restored.payload.audio == audio)
}
@Test func doneTranscriptionCachesColumnsButNotTranscribedAt() throws {
let (container, context) = try makeStore()
defer { withExtendedLifetime(container) {} }
let audio = Note.AudioInfo(
duration: 5,
transcriberDeviceID: "install-7",
transcriptionStatus: .done,
transcriptionLocaleID: "en_US",
transcribedAt: Date()
)
let note = Note.create(text: "Hello there", source: .transcribed, context: .empty, audio: audio)
NoteMapper.upsert(note, relativePath: note.relativePath, into: context)
let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context))
#expect(entity.transcriptionStatusRaw == "done")
#expect(entity.transcriptionLocaleID == "en_US")
// The file is authoritative for the timestamp the cache doesn't store it.
#expect(entity.audio?.transcribedAt == nil)
#expect(entity.audio?.transcriptionStatus == .done)
}
@Test func textNoteHasNoAudioColumns() throws {
let (container, context) = try makeStore()
defer { withExtendedLifetime(container) {} }
let note = makeNote()
NoteMapper.upsert(note, relativePath: note.relativePath, into: context)
let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context))
#expect(!entity.hasAudio)
#expect(entity.audio == nil)
#expect(entity.audioDuration == nil)
#expect(entity.transcriptionStatusRaw == nil)
}
}
+4 -2
View File
@@ -10,16 +10,18 @@ the top of the list.
## Key features
- **Quick text notes** — jot something down in two taps; first line doubles as the title
- **Voice notes** — record a memo (iPhone, Mac, or Apple Watch); the audio is kept and transcribed on-device afterwards, with a per-note "re-transcribe in another language" override and a settings screen to enable transcription languages
- **Apple Watch capture** — a watch-face complication opens a one-button recorder; the recording transfers to your iPhone over WatchConnectivity, which ingests and transcribes it (no iCloud on the watch)
- **Automatic context capture** — location, place name (reverse-geocoded), time zone, and device are stamped onto every note, best-effort and privacy-friendly (all data stays in your iCloud)
- **"Right here, right now" recall** — a ranked shelf of notes whose capture context matches your current place and time, above the regular newest-first list
- **Tags** — manual organization alongside the automatic context; browse notes by tag
- **Search** — full-text over note text, tags, and captured place names
- **iCloud sync** — notes live as JSON files in your iCloud Drive (visible in Files.app) and sync across iPhone and Mac; the local database is just a rebuildable cache
- **iOS + macOS** — same app, same data, both platforms
- **iOS + macOS + watchOS** — same app and data across iPhone and Mac, with an Apple Watch companion for voice capture
- **Backups** — one-tap local backup and restore of all notes (IndieBackup); backup files can be shared and imported across devices
- **About screen** — version info, changelog, and license in Settings (iOS) and the About window (macOS) via IndieAbout
Planned: voice notes with transcription, richer context signals, smarter ranking.
Planned: richer context signals, smarter ranking.
## Architecture
@@ -0,0 +1,73 @@
import Foundation
/// Versioned metadata carried alongside a watch audio recording over
/// `WCSession.transferFile(_:metadata:)`. The watch encodes it into the
/// transfer's metadata dictionary; the phone decodes it on delivery.
///
/// Versioned so a newer watch build can extend the field set without an older
/// phone build silently misreading it: a missing, unknown, or newer version
/// decodes to `nil` (a forward gate). On that outcome the phone still stages the
/// audio bytes but skips ingest, so a future phone build can pick them up.
///
/// Lives in `Shared/`, compiled into the iOS app and the watch app (not the Mac
/// app). Foundation-only, so it is safe on both platforms.
struct AudioNotePayload: Equatable, Sendable {
/// The wire version this build writes. Bump only when the field set changes
/// in a way an older decoder can't interpret.
static let currentVersion = 1
/// ULID string minted on the watch at record start the note's identity and
/// its UTC time bucket. Also the staged filename base on the phone.
let id: String
/// Wall-clock instant recording started.
let recordedAt: Date
/// Recording length in seconds.
let duration: TimeInterval
/// IANA time-zone identifier the recording was made in (e.g. "Europe/Berlin"),
/// or nil if unknown.
let timeZoneID: String?
private enum Key {
static let version = "v"
static let id = "id"
static let recordedAt = "recordedAt"
static let duration = "duration"
static let timeZone = "tz"
}
init(id: String, recordedAt: Date, duration: TimeInterval, timeZoneID: String?) {
self.id = id
self.recordedAt = recordedAt
self.duration = duration
self.timeZoneID = timeZoneID
}
/// A plist-serializable dictionary for `WCSession.transferFile(_:metadata:)`.
/// Every value type (Int, String, Date, Double) survives WatchConnectivity's
/// metadata plist encoding.
func metadata() -> [String: Any] {
var dict: [String: Any] = [
Key.version: Self.currentVersion,
Key.id: id,
Key.recordedAt: recordedAt,
Key.duration: duration,
]
if let timeZoneID { dict[Key.timeZone] = timeZoneID }
return dict
}
/// Decode a received metadata dictionary. Returns `nil` when the version is
/// missing, unknown, or newer than this build understands (the forward gate),
/// or when a required field is absent or malformed.
init?(metadata: [String: Any]) {
guard let version = metadata[Key.version] as? Int,
version >= 1, version <= Self.currentVersion,
let id = metadata[Key.id] as? String,
let recordedAt = metadata[Key.recordedAt] as? Date,
let duration = metadata[Key.duration] as? Double else { return nil }
self.id = id
self.recordedAt = recordedAt
self.duration = duration
self.timeZoneID = metadata[Key.timeZone] as? String
}
}
+78 -17
View File
@@ -4,6 +4,7 @@ options:
deploymentTarget:
iOS: "26.0"
macOS: "26.0"
watchOS: "26.0"
xcodeVersion: "26.0"
defaultConfig: Debug
@@ -11,7 +12,7 @@ settings:
base:
SWIFT_VERSION: "6.0"
DEVELOPMENT_TEAM: ${APPLE_TEAM_ID}
MARKETING_VERSION: "0.1"
MARKETING_VERSION: "0.2"
CURRENT_PROJECT_VERSION: "1"
ENABLE_USER_SCRIPT_SANDBOXING: "NO"
@@ -26,6 +27,20 @@ packages:
url: https://git.rzen.dev/rzen/indie-backup.git
from: "2.0.0"
# Shared post-build phase — stamps CFBundleVersion (git commit count), BuildDate, and
# BuildHash into each target's (and dSYM's) Info.plist. Aliased into every code-bearing
# target below so the iOS app, Mac app, watch app, and watch widget stamp identical
# metadata (versions must match across bundles for App Store validation).
scriptAnchors:
updateBuildInfo: &updateBuildInfo
- script: '"${SRCROOT}/Scripts/update_build_number.sh"'
name: Update Build Info
shell: /bin/sh
basedOnDependencyAnalysis: false
inputFiles:
- $(TARGET_BUILD_DIR)/$(INFOPLIST_PATH)
- $(DWARF_DSYM_FOLDER_PATH)/$(DWARF_DSYM_FILE_NAME)/Contents/Info.plist
schemes:
Notes:
build:
@@ -40,6 +55,15 @@ schemes:
- NotesTests
archive:
config: Release
# Per-target scheme so the watch app can be built / run on its own destination.
Notes Watch App:
build:
targets:
Notes Watch App: all
run:
config: Debug
archive:
config: Release
targets:
NotesTests:
@@ -61,10 +85,14 @@ targets:
TEST_HOST: $(BUILT_PRODUCTS_DIR)/Contextful.app/Contextful
BUNDLE_LOADER: $(TEST_HOST)
# ---- iOS app (owns iCloud Drive sync; embeds the watch app) ----------------
Notes:
type: application
platform: iOS
sources:
# Cross-platform code shared with the watch app (WatchConnectivity payload
# codec). Deliberately NOT compiled into NotesMac.
- path: Shared
- path: Notes
excludes:
- "Resources/Info-*.plist"
@@ -82,6 +110,8 @@ targets:
- package: IndieSync
- package: IndieAbout
- package: IndieBackup
# XcodeGen auto-embeds the watchOS app into the iOS app's Watch/ folder.
- target: Notes Watch App
settings:
base:
PRODUCT_BUNDLE_IDENTIFIER: dev.rzen.indie.Notes
@@ -95,14 +125,7 @@ targets:
ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME: AccentColor
TARGETED_DEVICE_FAMILY: "1,2"
postBuildScripts:
- script: '"${SRCROOT}/Scripts/update_build_number.sh"'
name: Update Build Info
shell: /bin/sh
basedOnDependencyAnalysis: false
inputFiles:
- $(TARGET_BUILD_DIR)/$(INFOPLIST_PATH)
- $(DWARF_DSYM_FOLDER_PATH)/$(DWARF_DSYM_FILE_NAME)/Contents/Info.plist
postBuildScripts: *updateBuildInfo
NotesMac:
type: application
@@ -137,11 +160,49 @@ targets:
MACOSX_DEPLOYMENT_TARGET: "26.0"
ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME: AccentColor
postBuildScripts:
- script: '"${SRCROOT}/Scripts/update_build_number.sh"'
name: Update Build Info
shell: /bin/sh
basedOnDependencyAnalysis: false
inputFiles:
- $(TARGET_BUILD_DIR)/$(INFOPLIST_PATH)
- $(DWARF_DSYM_FOLDER_PATH)/$(DWARF_DSYM_FILE_NAME)/Contents/Info.plist
postBuildScripts: *updateBuildInfo
# ---- watchOS app (no iCloud; recordings transfer to the phone via WC) -------
Notes Watch App:
type: application
platform: watchOS
sources:
- path: Shared
- path: Notes Watch App
excludes:
- "Resources/Info-*.plist"
dependencies:
- package: IndieSync
- target: Notes Watch Widget
embed: true
postBuildScripts: *updateBuildInfo
settings:
base:
PRODUCT_BUNDLE_IDENTIFIER: dev.rzen.indie.Notes.watchkitapp
PRODUCT_NAME: Contextful
PRODUCT_MODULE_NAME: NotesWatchApp
INFOPLIST_FILE: "Notes Watch App/Resources/Info-watchOS.plist"
GENERATE_INFOPLIST_FILE: false
SWIFT_STRICT_CONCURRENCY: complete
WATCHOS_DEPLOYMENT_TARGET: "26.0"
ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME: AccentColor
TARGETED_DEVICE_FAMILY: "4"
# ---- watchOS widget extension (a launcher complication for the watch face) --
Notes Watch Widget:
type: app-extension
platform: watchOS
sources:
- path: Notes Watch Widget
excludes:
- "Resources/Info-*.plist"
postBuildScripts: *updateBuildInfo
settings:
base:
PRODUCT_BUNDLE_IDENTIFIER: dev.rzen.indie.Notes.watchkitapp.widget
INFOPLIST_FILE: "Notes Watch Widget/Resources/Info-WatchWidget.plist"
GENERATE_INFOPLIST_FILE: false
SWIFT_STRICT_CONCURRENCY: complete
WATCHOS_DEPLOYMENT_TARGET: "26.0"
TARGETED_DEVICE_FAMILY: "4"