Add voice notes: on-device transcription, multi-language, watch capture

- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud
- AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline
  with enabled-language set and confidence-based auto-pick, re-transcribe menu
- Settings > Transcription > Languages with asset download/reserve handling
- watchOS app + accessory complication: one-button recorder, WCSession
  transferFile to phone, WatchInbox staging, direct-upsert ingestion
- Player UI, transcription status chips, quick-capture mic in notes list
- Version 0.2, changelog + README updated
This commit is contained in:
2026-07-15 13:30:43 -04:00
parent 6a93cedaa6
commit 86d74806c9
44 changed files with 3329 additions and 123 deletions
@@ -0,0 +1,20 @@
{
"colors" : [
{
"color" : {
"color-space" : "srgb",
"components" : {
"alpha" : "1.000",
"blue" : "0.282",
"green" : "0.208",
"red" : "0.886"
}
},
"idiom" : "universal"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
@@ -0,0 +1,14 @@
{
"images" : [
{
"filename" : "icon-1024.png",
"idiom" : "universal",
"platform" : "watchos",
"size" : "1024x1024"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 529 KiB

@@ -0,0 +1,6 @@
{
"info" : {
"author" : "xcode",
"version" : 1
}
}
@@ -0,0 +1,155 @@
import Foundation
import IndieSync
import os
import WatchConnectivity
/// Watch side of the recording hand-off. The watch has no iCloud; a finished
/// recording becomes durable on the phone, which ingests and transcribes it.
///
/// Each queued recording is a `<ULID>.m4a` plus a `<ULID>.meta.plist` sidecar in
/// `Documents/Recordings/`. `transferFile` hands the audio to WatchConnectivity
/// with an `AudioNotePayload` as metadata. Local files are deleted **only** in
/// `didFinish` with a nil error so a delivery we never confirm (app killed
/// mid-flight, transfer failed) is re-enqueued by `reconcileQueue()` from the
/// persisted sidecar on the next activation. `pendingCount` (files still on disk)
/// drives the "N waiting for iPhone" footer.
///
/// Strict concurrency: the delegate callbacks are nonisolated and run on
/// WatchConnectivity's background queue they extract Sendable values (a file
/// name string) before hopping to the main actor. `WCSessionFile*` are not
/// Sendable and never cross the hop.
@Observable
@MainActor
final class WatchTransferService: NSObject {
private nonisolated static let log = Logger(
subsystem: "dev.rzen.indie.Notes.watchkitapp", category: "watch-transfer")
/// Recordings still on disk (queued or in-flight) the count the phone has
/// not yet confirmed receiving.
private(set) var pendingCount = 0
private var session: WCSession?
func activate() {
guard WCSession.isSupported() else { return }
let session = WCSession.default
session.delegate = self
session.activate()
self.session = session
refreshPendingCount()
}
/// Queue a finished recording for delivery. Writes the sidecar first (so a
/// relaunch before delivery can re-enqueue it), then hands the file to
/// `transferFile`.
func enqueue(_ recording: WatchAudioRecorder.Recording) {
let tz = TimeZone.current.identifier
writeSidecar(base: recording.id.stringValue,
recordedAt: recording.recordedAt, duration: recording.duration, timeZoneID: tz)
startTransfer(id: recording.id.stringValue, url: recording.url,
recordedAt: recording.recordedAt, duration: recording.duration, timeZoneID: tz)
refreshPendingCount()
}
/// Re-enqueue any recording still on disk that the session isn't already
/// transferring a file whose delivery we never confirmed. Runs when the
/// session activates (relaunch) and when the app returns to the foreground.
func reconcileQueue() {
guard let session else { refreshPendingCount(); return }
let inFlight = Set(session.outstandingFileTransfers.map { $0.file.fileURL.lastPathComponent })
let fm = FileManager.default
let dir = WatchAudioRecorder.recordingsDirectory
let entries = (try? fm.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)) ?? []
for m4a in entries where m4a.pathExtension == "m4a" {
if inFlight.contains(m4a.lastPathComponent) { continue }
let base = m4a.deletingPathExtension().lastPathComponent
guard ULID(string: base) != nil else { continue }
let meta = readSidecar(base: base)
startTransfer(id: base, url: m4a,
recordedAt: meta?.recordedAt ?? Date(),
duration: meta?.duration ?? 0,
timeZoneID: meta?.timeZoneID)
}
refreshPendingCount()
}
// MARK: - Transfer
private func startTransfer(id: String, url: URL, recordedAt: Date, duration: TimeInterval, timeZoneID: String?) {
guard let session, session.activationState == .activated,
FileManager.default.fileExists(atPath: url.path) else { return }
let payload = AudioNotePayload(
id: id, recordedAt: recordedAt, duration: duration, timeZoneID: timeZoneID)
session.transferFile(url, metadata: payload.metadata())
}
/// Delete a delivered recording and its sidecar (called only on a confirmed
/// `didFinish` with no error).
private func completeTransfer(fileName: String) {
let dir = WatchAudioRecorder.recordingsDirectory
let base = (fileName as NSString).deletingPathExtension
try? FileManager.default.removeItem(at: dir.appending(path: fileName))
try? FileManager.default.removeItem(at: sidecarURL(base: base))
refreshPendingCount()
}
// MARK: - Pending count
private func refreshPendingCount() {
let dir = WatchAudioRecorder.recordingsDirectory
let entries = (try? FileManager.default.contentsOfDirectory(
at: dir, includingPropertiesForKeys: nil)) ?? []
pendingCount = entries.filter { $0.pathExtension == "m4a" }.count
}
// MARK: - Sidecar (persisted metadata for re-enqueue after relaunch)
private func sidecarURL(base: String) -> URL {
WatchAudioRecorder.recordingsDirectory.appending(path: "\(base).meta.plist")
}
private func writeSidecar(base: String, recordedAt: Date, duration: TimeInterval, timeZoneID: String?) {
var dict: [String: Any] = ["recordedAt": recordedAt, "duration": duration]
if let timeZoneID { dict["timeZoneID"] = timeZoneID }
if let data = try? PropertyListSerialization.data(fromPropertyList: dict, format: .binary, options: 0) {
try? data.write(to: sidecarURL(base: base))
}
}
private func readSidecar(base: String) -> (recordedAt: Date, duration: TimeInterval, timeZoneID: String?)? {
guard let data = try? Data(contentsOf: sidecarURL(base: base)),
let dict = try? PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any],
let recordedAt = dict["recordedAt"] as? Date,
let duration = dict["duration"] as? Double else { return nil }
return (recordedAt, duration, dict["timeZoneID"] as? String)
}
}
// MARK: - WCSessionDelegate
extension WatchTransferService: WCSessionDelegate {
nonisolated func session(_ session: WCSession, activationDidCompleteWith activationState: WCSessionActivationState, error: (any Error)?) {
if let error { Self.log.error("watch WC activation failed: \(error, privacy: .public)") }
Task { @MainActor in self.reconcileQueue() }
}
nonisolated func sessionReachabilityDidChange(_ session: WCSession) {
// A queued file whose transfer never registered (session wasn't yet
// activated at enqueue) gets re-driven once the phone is reachable again.
if session.isReachable {
Task { @MainActor in self.reconcileQueue() }
}
}
nonisolated func session(_ session: WCSession, didFinish fileTransfer: WCSessionFileTransfer, error: (any Error)?) {
// Extract the Sendable file name before hopping WCSessionFileTransfer /
// WCSessionFile are not Sendable.
let fileName = fileTransfer.file.fileURL.lastPathComponent
if let error {
Self.log.warning("file transfer failed (\(fileName, privacy: .public)): \(error, privacy: .public)")
Task { @MainActor in self.refreshPendingCount() }
return
}
Task { @MainActor in self.completeTransfer(fileName: fileName) }
}
}
+21
View File
@@ -0,0 +1,21 @@
import SwiftUI
@main
struct NotesWatchApp: App {
@State private var services = WatchAppServices()
@Environment(\.scenePhase) private var scenePhase
var body: some Scene {
WindowGroup {
NavigationStack {
RecordView()
}
.environment(services)
.task { services.bootstrap() }
.onOpenURL { url in services.handleDeepLink(url) }
}
.onChange(of: scenePhase) { _, phase in
services.handleScenePhase(phase)
}
}
}
@@ -0,0 +1,47 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleDisplayName</key>
<string>Contextful</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>$(MARKETING_VERSION)</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<key>NSMicrophoneUsageDescription</key>
<string>Contextful records voice notes on your watch and sends them to your iPhone to transcribe.</string>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
</array>
<key>WKApplication</key>
<true/>
<key>WKCompanionAppBundleIdentifier</key>
<string>dev.rzen.indie.Notes</string>
<key>CFBundleURLTypes</key>
<array>
<dict>
<key>CFBundleURLName</key>
<string>dev.rzen.indie.Notes.watchkitapp</string>
<key>CFBundleURLSchemes</key>
<array>
<string>contextful</string>
</array>
</dict>
</array>
</dict>
</plist>
@@ -0,0 +1,160 @@
import AVFoundation
import Foundation
import IndieSync
import WatchKit
/// Records a single voice note on the watch into a persisted
/// `Documents/Recordings/<ULID>.m4a` (AAC mono 22.05 kHz ~32 kbps). The ULID is
/// minted at record **start**, so the note's identity and its UTC time bucket
/// once ingested on the phone reflects when it was recorded, not when it later
/// reaches the phone.
///
/// The file lands in the app's Documents (not temp) so a queued recording
/// survives suspension until `WatchTransferService` confirms the phone received
/// it. `AVAudioRecorder` is not Sendable, so the recorder stays confined to the
/// main actor for its whole life.
@Observable
@MainActor
final class WatchAudioRecorder {
/// One finished recording: the file on disk, its length, its ULID, and the
/// instant it started. The caller (`WatchTransferService`) owns delivery.
struct Recording: Equatable, Sendable {
let id: ULID
let url: URL
let duration: TimeInterval
let recordedAt: Date
}
private(set) var isRecording = false
/// Seconds recorded so far, refreshed by the tick loop while recording.
private(set) var elapsed: TimeInterval = 0
/// Fired when a recording ends without the user tapping Stop the 10-minute
/// safety cap fires. The finished file is handed up to be queued rather than
/// lost. (Scene-deactivation stop-and-queue is driven by `WatchAppServices`,
/// which calls `stopRecording()` directly.)
var onInvoluntaryFinish: ((Recording) -> Void)?
/// Hard cap on a single take so a recording the user forgot about can't run
/// the battery down. The finished file is queued, never dropped.
private static let maxDuration: TimeInterval = 600
private var recorder: AVAudioRecorder?
private var current: (id: ULID, url: URL, startedAt: Date)?
private var tickTask: Task<Void, Never>?
/// Directory the finished recordings (and their `.meta.plist` sidecars) live
/// in until the phone confirms delivery. Persisted, so a queued file survives
/// app suspension and can be re-enqueued on the next launch.
static var recordingsDirectory: URL {
URL.documentsDirectory.appending(path: "Recordings", directoryHint: .isDirectory)
}
/// Ask for microphone access, returning whether it was granted. Safe to call
/// every time recording starts the system only prompts once.
func requestPermission() async -> Bool {
await AVAudioApplication.requestRecordPermission()
}
/// Whether the mic has been explicitly denied, so the UI can show a jump to
/// Settings instead of a dead record button.
var permissionDenied: Bool {
AVAudioApplication.shared.recordPermission == .denied
}
/// Begin recording into a fresh `<ULID>.m4a`. A no-op if already recording.
/// Throws if the audio session or recorder can't be configured.
func startRecording() throws {
guard !isRecording else { return }
let session = AVAudioSession.sharedInstance()
try session.setCategory(.record, mode: .spokenAudio)
try session.setActive(true)
let dir = Self.recordingsDirectory
try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
let id = ULID()
let url = dir.appending(path: "\(id.stringValue).m4a")
let settings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 22_050.0,
AVNumberOfChannelsKey: 1,
AVEncoderBitRateKey: 32_000,
AVEncoderAudioQualityKey: AVAudioQuality.medium.rawValue,
]
let recorder = try AVAudioRecorder(url: url, settings: settings)
guard recorder.record() else { throw RecorderError.couldNotStart }
self.recorder = recorder
self.current = (id, url, Date())
isRecording = true
elapsed = 0
// Continuity through a wrist-down rests on the `audio` background mode
// (declared in Info-watchOS.plist) keeping the active audio session
// alive; the old `WKExtension.isFrontmostTimeoutExtended` knob is
// deprecated and unsupported since watchOS 7. The stop-and-queue on
// scene-background (see `WatchAppServices`) is the never-lose-audio net.
WKInterfaceDevice.current().play(.start)
startTicking()
}
/// Stop recording and return the finished file plus its duration, or nil if
/// nothing was recording.
@discardableResult
func stopRecording() -> Recording? {
guard let recorder, let current else { return nil }
let duration = recorder.currentTime
recorder.stop()
WKInterfaceDevice.current().play(.stop)
let recording = Recording(
id: current.id, url: current.url, duration: duration, recordedAt: current.startedAt)
teardown()
return recording
}
/// Abandon the in-progress recording and delete its file. (Not surfaced in the
/// single-button UI, but kept for symmetry / future use.)
func discard() {
recorder?.stop()
if let url = current?.url { try? FileManager.default.removeItem(at: url) }
teardown()
}
// MARK: - Metering / watchdog
/// ~5 Hz loop that refreshes `elapsed` and enforces the safety cap. Runs on
/// the main actor (a plain `Task` inherits it), so it can touch the
/// non-Sendable recorder safely.
private func startTicking() {
tickTask?.cancel()
tickTask = Task { [weak self] in
while !Task.isCancelled {
guard let self, let recorder = self.recorder, recorder.isRecording else { return }
self.elapsed = recorder.currentTime
if self.elapsed >= Self.maxDuration {
if let finished = self.stopRecording() {
self.onInvoluntaryFinish?(finished)
}
return
}
try? await Task.sleep(for: .milliseconds(200))
}
}
}
private func teardown() {
tickTask?.cancel()
tickTask = nil
recorder = nil
current = nil
isRecording = false
elapsed = 0
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
}
enum RecorderError: Error {
case couldNotStart
}
}
+84
View File
@@ -0,0 +1,84 @@
import SwiftUI
/// The watch app's single screen: a big record/stop button, an elapsed timer
/// while recording, and a footer counting recordings still waiting to reach the
/// iPhone. Shows a permission-denied state when the mic is refused.
struct RecordView: View {
@Environment(WatchAppServices.self) private var services
var body: some View {
let recorder = services.recorder
VStack(spacing: 10) {
if recorder.permissionDenied {
permissionDenied
} else {
recordButton(isRecording: recorder.isRecording)
if recorder.isRecording {
Text(Self.elapsedString(recorder.elapsed))
.font(.system(.title3, design: .rounded).monospacedDigit())
.foregroundStyle(.secondary)
} else {
Text("Tap to record")
.font(.footnote)
.foregroundStyle(.secondary)
}
pendingFooter
}
}
.padding()
.containerBackground(.red.gradient, for: .navigation)
.navigationTitle("Contextful")
}
private func recordButton(isRecording: Bool) -> some View {
Button {
services.toggleRecording()
} label: {
ZStack {
Circle()
.fill(.red)
.frame(width: 96, height: 96)
Image(systemName: isRecording ? "stop.fill" : "mic.fill")
.font(.system(size: 40))
.foregroundStyle(.white)
.contentTransition(.symbolEffect(.replace))
}
}
.buttonStyle(.plain)
.accessibilityLabel(isRecording ? "Stop recording" : "Start recording")
}
@ViewBuilder
private var pendingFooter: some View {
let count = services.transfer.pendingCount
if count > 0 {
Label(
count == 1 ? "1 waiting for iPhone" : "\(count) waiting for iPhone",
systemImage: "arrow.up.circle"
)
.font(.caption2)
.foregroundStyle(.secondary)
}
}
private var permissionDenied: some View {
VStack(spacing: 8) {
Image(systemName: "mic.slash.fill")
.font(.title)
.foregroundStyle(.secondary)
Text("Microphone access is off")
.font(.headline)
.multilineTextAlignment(.center)
Text("Enable it in the Watch app on iPhone under Contextful.")
.font(.caption2)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
}
}
/// mm:ss for the elapsed readout.
static func elapsedString(_ interval: TimeInterval) -> String {
let total = Int(interval)
return String(format: "%02d:%02d", total / 60, total % 60)
}
}
+73
View File
@@ -0,0 +1,73 @@
import Foundation
import Observation
import SwiftUI
/// Composition root for the watch app. Owns the recorder and the transfer
/// service and mediates between them: a finished recording is handed to the
/// transfer queue. The watch has no iCloud or local database its whole job is
/// to capture audio and get it to the phone.
@Observable
@MainActor
final class WatchAppServices {
let recorder = WatchAudioRecorder()
let transfer = WatchTransferService()
func bootstrap() {
transfer.activate()
transfer.reconcileQueue()
// The 10-minute safety cap ends a take without a Stop tap queue it
// rather than lose it.
recorder.onInvoluntaryFinish = { [weak self] recording in
self?.transfer.enqueue(recording)
}
}
/// Single-button behaviour: start if idle, stop-and-queue if recording.
func toggleRecording() {
if recorder.isRecording {
stopAndQueue()
} else {
Task { await startRecording() }
}
}
/// Request permission (once) then begin recording. A no-op if already
/// recording or permission is refused.
func startRecording() async {
guard !recorder.isRecording else { return }
guard await recorder.requestPermission() else { return }
try? recorder.startRecording()
}
private func stopAndQueue() {
if let recording = recorder.stopRecording() {
transfer.enqueue(recording)
}
}
/// Complication deep link (`contextful://record?autostart=1`) auto-start a
/// recording. Plain launches never carry the URL, so they never auto-record.
func handleDeepLink(_ url: URL) {
guard url.scheme == "contextful", url.host == "record" else { return }
let components = URLComponents(url: url, resolvingAgainstBaseURL: false)
let autostart = components?.queryItems?.first { $0.name == "autostart" }?.value == "1"
guard autostart else { return }
Task { await startRecording() }
}
func handleScenePhase(_ phase: ScenePhase) {
switch phase {
case .active:
// Catch up any file whose delivery we never confirmed.
transfer.reconcileQueue()
case .background:
// Never lose audio to the app being suspended: stop-and-queue
// whatever is recording rather than dropping the take. (A brief
// wrist-down is `.inactive` and is covered by the recorder's
// frontmost-timeout extension, so it keeps recording.)
stopAndQueue()
default:
break
}
}
}