Add voice notes: on-device transcription, multi-language, watch capture

- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud
- AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline
  with enabled-language set and confidence-based auto-pick, re-transcribe menu
- Settings > Transcription > Languages with asset download/reserve handling
- watchOS app + accessory complication: one-button recorder, WCSession
  transferFile to phone, WatchInbox staging, direct-upsert ingestion
- Player UI, transcription status chips, quick-capture mic in notes list
- Version 0.2, changelog + README updated
This commit is contained in:
2026-07-15 13:30:43 -04:00
parent 6a93cedaa6
commit 86d74806c9
44 changed files with 3329 additions and 123 deletions
+206
View File
@@ -0,0 +1,206 @@
import AVFoundation
import IndieSync
import SwiftUI
/// Plays back a voice note's audio. Two sources: a fresh recording still on disk
/// (`localURL`, used by the composer) or a saved note whose bytes come back
/// through `SyncEngine.loadAudioData` (fail-fast, so an evicted file shows a
/// "downloading" retry state instead of stalling). `AVAudioPlayer` is not
/// Sendable, so the playback model stays main-actor confined.
struct AudioPlayerView: View {
enum Source: Equatable {
case localURL(URL)
case note(id: ULID, duration: TimeInterval)
}
let source: Source
@Environment(SyncEngine.self) private var syncEngine
@State private var model = AudioPlaybackModel()
var body: some View {
Group {
switch model.loadState {
case .idle, .loading:
HStack {
ProgressView().controlSize(.small)
Text("Loading audio…").foregroundStyle(.secondary)
}
.font(.callout)
case .downloading:
VStack(alignment: .leading, spacing: 6) {
Label("Downloading from iCloud…", systemImage: "icloud.and.arrow.down")
.foregroundStyle(.secondary)
Button("Retry") { Task { await load() } }
.buttonStyle(.bordered)
.controlSize(.small)
}
.font(.callout)
case .failed:
Label("Couldn't play audio", systemImage: "exclamationmark.triangle")
.foregroundStyle(.secondary)
.font(.callout)
case .ready:
player
}
}
.task { await load() }
.onDisappear { model.stop() }
}
private var player: some View {
HStack(spacing: 12) {
Button {
model.togglePlay()
} label: {
Image(systemName: model.isPlaying ? "pause.circle.fill" : "play.circle.fill")
.font(.largeTitle)
.foregroundStyle(.tint)
}
.buttonStyle(.plain)
VStack(spacing: 2) {
Slider(
value: Binding(get: { model.progress }, set: { model.seek(toFraction: $0) }),
in: 0...1
)
HStack {
Text(Self.timeString(model.currentTime))
Spacer()
Text(Self.timeString(model.duration))
}
.font(.caption2.monospacedDigit())
.foregroundStyle(.secondary)
}
}
}
private func load() async {
switch source {
case .localURL(let url):
model.loadLocal(url)
case .note(let id, let duration):
await model.loadNote(id: id, fallbackDuration: duration, syncEngine: syncEngine)
}
}
static func timeString(_ interval: TimeInterval) -> String {
let total = Int(interval.rounded())
return String(format: "%d:%02d", total / 60, total % 60)
}
}
/// Owns the `AVAudioPlayer` and a lightweight progress ticker. Main-actor
/// confined because `AVAudioPlayer` isn't Sendable.
@Observable
@MainActor
final class AudioPlaybackModel {
enum LoadState: Equatable { case idle, loading, ready, downloading, failed }
private(set) var loadState: LoadState = .idle
private(set) var isPlaying = false
/// 01 position, bound to the scrubber.
private(set) var progress: Double = 0
private(set) var duration: TimeInterval = 0
var currentTime: TimeInterval { duration * progress }
private var player: AVAudioPlayer?
private var ticker: Task<Void, Never>?
func loadLocal(_ url: URL) {
loadState = .loading
do {
let player = try AVAudioPlayer(contentsOf: url)
player.prepareToPlay()
adopt(player)
} catch {
loadState = .failed
}
}
/// Fetch a saved note's audio (fail-fast). A miss means the file is evicted:
/// show the downloading state and warm it up for the retry.
func loadNote(id: ULID, fallbackDuration: TimeInterval, syncEngine: SyncEngine) async {
loadState = .loading
duration = fallbackDuration
guard let data = await syncEngine.loadAudioData(for: id) else {
loadState = .downloading
await syncEngine.requestAudioDownload(for: id)
return
}
do {
let player = try AVAudioPlayer(data: data)
player.prepareToPlay()
adopt(player)
} catch {
loadState = .failed
}
}
private func adopt(_ player: AVAudioPlayer) {
self.player = player
duration = player.duration
progress = 0
loadState = .ready
}
func togglePlay() {
guard let player else { return }
if player.isPlaying {
player.pause()
isPlaying = false
ticker?.cancel()
} else {
activatePlaybackSession()
player.play()
isPlaying = true
startTicker()
}
}
func seek(toFraction fraction: Double) {
guard let player else { return }
progress = min(max(fraction, 0), 1)
player.currentTime = progress * duration
}
func stop() {
player?.stop()
isPlaying = false
ticker?.cancel()
ticker = nil
#if os(iOS)
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
#endif
}
private func startTicker() {
ticker?.cancel()
ticker = Task { [weak self] in
while !Task.isCancelled {
guard let self, let player = self.player else { return }
if player.isPlaying {
self.progress = self.duration > 0 ? player.currentTime / self.duration : 0
} else {
// Reached the end (or was paused elsewhere): reset to the start.
self.isPlaying = false
self.progress = 0
return
}
try? await Task.sleep(for: .milliseconds(50))
}
}
}
private func activatePlaybackSession() {
#if os(iOS)
let session = AVAudioSession.sharedInstance()
try? session.setCategory(.playback, mode: .spokenAudio)
try? session.setActive(true)
#endif
}
}
@@ -0,0 +1,145 @@
import SwiftUI
/// The record surface in the composer: a mic button that, once tapped (or
/// auto-started), shows the elapsed time and a live level meter with Stop /
/// Discard. When recording ends it hands the finished `(url, duration)` up via
/// `onFinish`; the composer owns everything after that.
struct AudioRecordingControl: View {
@Environment(AudioRecorderService.self) private var recorder
/// Begin recording as soon as the view appears (the toolbar "record a note"
/// entry point), after the permission prompt resolves.
var autostart = false
let onFinish: (URL, TimeInterval) -> Void
@State private var permissionDenied = false
@State private var startError: String?
@State private var didAutostart = false
var body: some View {
VStack(spacing: 12) {
if permissionDenied {
deniedState
} else if recorder.isRecording {
recordingState
} else {
idleState
}
if let startError {
Text(startError)
.font(.footnote)
.foregroundStyle(.red)
}
}
.frame(maxWidth: .infinity)
.task {
recorder.onInterrupted = { recording in onFinish(recording.url, recording.duration) }
if autostart, !didAutostart {
didAutostart = true
await beginRecording()
}
}
.onDisappear {
recorder.onInterrupted = nil
if recorder.isRecording { recorder.discard() }
}
}
private var idleState: some View {
Button {
Task { await beginRecording() }
} label: {
VStack(spacing: 6) {
Image(systemName: "mic.circle.fill")
.font(.system(size: 52))
.foregroundStyle(.tint)
Text("Record")
.font(.callout)
}
}
.buttonStyle(.plain)
}
private var recordingState: some View {
VStack(spacing: 14) {
LevelMeter(level: recorder.level)
Text(AudioPlayerView.timeString(recorder.elapsed))
.font(.title2.monospacedDigit())
.foregroundStyle(.primary)
HStack(spacing: 24) {
Button(role: .destructive) {
recorder.discard()
} label: {
Label("Discard", systemImage: "trash")
}
.buttonStyle(.bordered)
Button {
if let finished = recorder.stopRecording() {
onFinish(finished.url, finished.duration)
}
} label: {
Label("Stop", systemImage: "stop.circle.fill")
}
.buttonStyle(.borderedProminent)
}
}
}
private var deniedState: some View {
VStack(spacing: 8) {
Image(systemName: "mic.slash.circle.fill")
.font(.system(size: 44))
.foregroundStyle(.secondary)
Text("Microphone access is off. Enable it for Contextful in Settings to record voice notes.")
.font(.footnote)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
}
}
private func beginRecording() async {
startError = nil
guard await recorder.requestPermission() else {
permissionDenied = true
return
}
permissionDenied = false
do {
try recorder.startRecording()
} catch {
startError = "Couldn't start recording: \(error.localizedDescription)"
}
}
}
/// A small row of bars whose height rides the current input level enough to
/// show the mic is live without pretending to be a real waveform.
private struct LevelMeter: View {
let level: Float
private let barCount = 9
/// Fixed per-bar weights so the meter has a stable, mic-like shape.
private let weights: [CGFloat] = [0.35, 0.55, 0.75, 0.9, 1.0, 0.9, 0.75, 0.55, 0.35]
var body: some View {
HStack(alignment: .center, spacing: 4) {
ForEach(0..<barCount, id: \.self) { index in
Capsule()
.fill(.tint)
.frame(width: 4, height: height(index))
}
}
.frame(height: 40)
.animation(.easeOut(duration: 0.08), value: level)
}
private func height(_ index: Int) -> CGFloat {
let base: CGFloat = 4
let peak: CGFloat = 36
return base + peak * CGFloat(level) * weights[index]
}
}
+150 -8
View File
@@ -4,6 +4,10 @@ import SwiftUI
/// Composer for a new note and editor for an existing one. On create it
/// snapshots the current context (refreshing location in the background);
/// on edit the original capture context is preserved untouched.
///
/// A note can carry audio: create mode can record one (optionally auto-starting
/// from the toolbar mic), and edit mode plays it back, shows transcription
/// status, and offers a per-note "Re-transcribe" override.
struct NoteEditorView: View {
enum Mode {
case create
@@ -11,25 +15,38 @@ struct NoteEditorView: View {
}
let mode: Mode
/// Begin recording the moment the composer appears (toolbar mic entry point).
var startRecording = false
@Environment(SyncEngine.self) private var syncEngine
@Environment(ContextService.self) private var contextService
@Environment(TranscriptionService.self) private var transcriptionService
@Environment(TranscriptionSettings.self) private var transcriptionSettings
@Environment(\.dismiss) private var dismiss
@State private var text = ""
@State private var tagsText = ""
@State private var saveError: String?
@State private var recordedAudio: AudioRecorderService.Recording?
@FocusState private var textFocused: Bool
var body: some View {
NavigationStack {
Form {
if isCreating {
audioComposeSection
}
Section {
TextEditor(text: $text)
.focused($textFocused)
.frame(minHeight: 160)
}
if case .edit(let note) = mode, note.hasAudio {
audioPlaybackSection(for: note)
}
Section("Tags") {
TextField("coffee, people, ideas…", text: $tagsText)
.autocorrectionDisabled()
@@ -65,14 +82,15 @@ struct NoteEditorView: View {
}
ToolbarItem(placement: .confirmationAction) {
Button("Save") { save() }
.disabled(text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
.disabled(!canSave)
}
}
}
.task {
switch mode {
case .create:
textFocused = true
// Don't steal focus from the mic entry point.
if !startRecording { textFocused = true }
await contextService.refresh()
case .edit(let note):
text = note.text
@@ -84,11 +102,119 @@ struct NoteEditorView: View {
#endif
}
// MARK: - Audio (create)
@ViewBuilder
private var audioComposeSection: some View {
Section("Voice") {
if let recordedAudio {
AudioPlayerView(source: .localURL(recordedAudio.url))
Button(role: .destructive) {
try? FileManager.default.removeItem(at: recordedAudio.url)
self.recordedAudio = nil
} label: {
Label("Discard Recording", systemImage: "trash")
}
} else {
AudioRecordingControl(autostart: startRecording) { url, duration in
recordedAudio = AudioRecorderService.Recording(url: url, duration: duration)
}
.padding(.vertical, 4)
}
}
}
// MARK: - Audio (edit)
@ViewBuilder
private func audioPlaybackSection(for note: NoteEntity) -> some View {
Section("Voice") {
if let id = ULID(string: note.id) {
AudioPlayerView(source: .note(id: id, duration: note.audioDuration ?? 0))
}
transcriptionStatusRow(for: note)
retranscribeMenu(for: note)
}
}
@ViewBuilder
private func transcriptionStatusRow(for note: NoteEntity) -> some View {
switch note.audio?.transcriptionStatus {
case .pending:
if note.transcriberDeviceID == TranscriptionSettings.installID {
Label {
Text("Transcribing…")
} icon: {
ProgressView().controlSize(.small)
}
.foregroundStyle(.secondary)
} else {
Label("Waiting to transcribe on another device", systemImage: "hourglass")
.foregroundStyle(.secondary)
}
case .failed:
HStack {
Label("Transcription failed", systemImage: "exclamationmark.triangle")
.foregroundStyle(.secondary)
Spacer()
Button("Retry") { retranscribe(note: note, localeID: nil) }
.buttonStyle(.bordered)
.controlSize(.small)
}
case .done:
if let localeID = note.audio?.transcriptionLocaleID {
Label(
"Transcribed · \(TranscriptionSettings.displayName(for: Locale(identifier: localeID)))",
systemImage: "checkmark.circle"
)
.foregroundStyle(.secondary)
.font(.footnote)
}
case .none:
EmptyView()
}
}
@ViewBuilder
private func retranscribeMenu(for note: NoteEntity) -> some View {
let locales = transcriptionSettings.enabledLocales
if !locales.isEmpty {
Menu {
Button("Auto-detect Language") { retranscribe(note: note, localeID: nil) }
Divider()
ForEach(locales, id: \.identifier) { locale in
Button(TranscriptionSettings.displayName(for: locale)) {
retranscribe(note: note, localeID: TranscriptionSettings.canonicalID(locale))
}
}
} label: {
Label("Re-transcribe", systemImage: "arrow.clockwise")
}
}
}
private func retranscribe(note: NoteEntity, localeID: String?) {
guard let id = ULID(string: note.id) else { return }
Task {
try? await syncEngine.requestRetranscription(id: id, localeID: localeID)
transcriptionService.enqueue(noteID: id, forcedLocaleID: localeID)
}
}
// MARK: - State
private var isCreating: Bool {
if case .create = mode { return true }
return false
}
private var canSave: Bool {
if !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { return true }
if recordedAudio != nil { return true }
if case .edit(let note) = mode, note.hasAudio { return true }
return false
}
private var parsedTags: [String] {
tagsText
.split(separator: ",")
@@ -96,18 +222,34 @@ struct NoteEditorView: View {
.filter { !$0.isEmpty }
}
// MARK: - Save
private func save() {
guard canSave else { return }
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
Task {
do {
switch mode {
case .create:
try await syncEngine.createNote(
text: trimmed,
tags: parsedTags,
context: contextService.current
)
if let recordedAudio {
let note = try await syncEngine.createAudioNote(
audioFileURL: recordedAudio.url,
duration: recordedAudio.duration,
text: trimmed,
tags: parsedTags,
context: contextService.current,
transcriberDeviceID: TranscriptionSettings.installID
)
// Fast path: hand the recording's temp file straight to the
// transcriber so it doesn't round-trip through the container.
transcriptionService.enqueue(noteID: note.id, localAudioURL: recordedAudio.url)
} else {
try await syncEngine.createNote(
text: trimmed,
tags: parsedTags,
context: contextService.current
)
}
case .edit(let entity):
guard let id = ULID(string: entity.id),
var note = await syncEngine.loadNote(id: id) else { return }
+35 -1
View File
@@ -5,11 +5,16 @@ struct NoteRowView: View {
var body: some View {
VStack(alignment: .leading, spacing: 4) {
Text(note.title.isEmpty ? "Untitled" : note.title)
Text(displayTitle)
.font(.body.weight(.medium))
.lineLimit(2)
HStack(spacing: 4) {
if note.hasAudio {
Image(systemName: "waveform")
Text(AudioPlayerView.timeString(note.audioDuration ?? 0))
Text("·")
}
Text(note.createdAt, format: .dateTime.day().month().year())
if let place = note.context.summary {
Text("·")
@@ -20,6 +25,8 @@ struct NoteRowView: View {
.font(.caption)
.foregroundStyle(.secondary)
transcriptionChip
if !note.tags.isEmpty {
Text(note.tags.map { "#\($0)" }.joined(separator: " "))
.font(.caption)
@@ -29,4 +36,31 @@ struct NoteRowView: View {
}
.padding(.vertical, 2)
}
/// Audio notes with no transcript yet still need a legible title.
private var displayTitle: String {
if !note.title.isEmpty { return note.title }
return note.hasAudio ? "Audio note" : "Untitled"
}
@ViewBuilder
private var transcriptionChip: some View {
switch note.audio?.transcriptionStatus {
case .pending:
capsule("Transcribing…", color: .secondary)
case .failed:
capsule("Transcription failed", color: .red)
case .done, .none:
EmptyView()
}
}
private func capsule(_ text: String, color: Color) -> some View {
Text(text)
.font(.caption2)
.foregroundStyle(color)
.padding(.horizontal, 8)
.padding(.vertical, 2)
.background(color.opacity(0.12), in: Capsule())
}
}
+11
View File
@@ -11,6 +11,7 @@ struct NotesListView: View {
@State private var searchText = ""
@State private var composing = false
@State private var recording = false
@State private var editingNote: NoteEntity?
var body: some View {
@@ -50,6 +51,13 @@ struct NotesListView: View {
}
}
.toolbar {
ToolbarItem(placement: .primaryAction) {
Button {
recording = true
} label: {
Label("Record Note", systemImage: "mic")
}
}
ToolbarItem(placement: .primaryAction) {
Button {
composing = true
@@ -62,6 +70,9 @@ struct NotesListView: View {
.sheet(isPresented: $composing) {
NoteEditorView(mode: .create)
}
.sheet(isPresented: $recording) {
NoteEditorView(mode: .create, startRecording: true)
}
.sheet(item: $editingNote) { note in
NoteEditorView(mode: .edit(note))
}
+8
View File
@@ -36,6 +36,14 @@ struct SettingsView: View {
.disabled(syncEngine.iCloudStatus != .available || syncEngine.isSyncing)
}
Section("Transcription") {
NavigationLink {
TranscriptionLanguagesView()
} label: {
Label("Languages", systemImage: "character.bubble")
}
}
Section("Context") {
ContextSummaryRow(
context: contextService.current,
@@ -0,0 +1,119 @@
import SwiftUI
/// Settings screen for choosing which languages voice notes can be transcribed
/// in. Enabling a language reserves it and downloads its on-device model; the
/// system caps how many can be reserved at once, so the list disables further
/// toggles at capacity and explains the limit in the footer.
struct TranscriptionLanguagesView: View {
@Environment(TranscriptionSettings.self) private var settings
@State private var errorMessage: String?
var body: some View {
List {
if let errorMessage {
Section {
Label(errorMessage, systemImage: "exclamationmark.triangle")
.font(.footnote)
.foregroundStyle(.red)
}
}
if !enabledLocales.isEmpty {
Section("Enabled") {
ForEach(enabledLocales, id: \.identifier) { row($0) }
}
}
Section {
ForEach(otherLocales, id: \.identifier) { row($0) }
} header: {
Text("All Languages")
} footer: {
Text(footerText)
}
}
.navigationTitle("Languages")
.overlay {
if settings.supportedLocales.isEmpty {
ProgressView()
}
}
.task { await settings.prepare() }
}
private var enabledLocales: [Locale] {
settings.supportedLocales.filter { settings.isEnabled($0) }
}
private var otherLocales: [Locale] {
settings.supportedLocales.filter { !settings.isEnabled($0) }
}
private var footerText: String {
guard settings.maximumReserved > 0 else {
return "Each new note is transcribed on device in your enabled languages."
}
return "Enable up to \(settings.maximumReserved) languages. Each new voice note is sampled in your enabled languages and transcribed in whichever fits best."
}
private func row(_ locale: Locale) -> some View {
let enabled = settings.isEnabled(locale)
return HStack {
VStack(alignment: .leading, spacing: 2) {
Text(TranscriptionSettings.displayName(for: locale))
stateSubtitle(for: locale)
}
Spacer()
Toggle("", isOn: Binding(
get: { settings.isEnabled(locale) },
set: { toggle(locale, on: $0) }
))
.labelsHidden()
.disabled(!enabled && settings.isAtCapacity)
}
}
@ViewBuilder
private func stateSubtitle(for locale: Locale) -> some View {
switch settings.assetStates[TranscriptionSettings.canonicalID(locale)] {
case .installed:
Label("Installed", systemImage: "checkmark.circle.fill")
.font(.caption)
.foregroundStyle(.green)
case .downloading(let fraction):
HStack(spacing: 6) {
ProgressView().controlSize(.small)
if let fraction {
Text("Downloading \(Int(fraction * 100))%")
} else {
Text("Downloading…")
}
}
.font(.caption)
.foregroundStyle(.secondary)
case .notInstalled:
Label("Available to download", systemImage: "icloud.and.arrow.down")
.font(.caption)
.foregroundStyle(.secondary)
case .unsupported, .none:
EmptyView()
}
}
private func toggle(_ locale: Locale, on: Bool) {
errorMessage = nil
Task {
if on {
do {
try await settings.enable(locale)
} catch TranscriptionSettings.SettingsError.atCapacity {
errorMessage = "You can enable up to \(settings.maximumReserved) languages. Turn one off first."
} catch {
errorMessage = "Couldn't enable \(TranscriptionSettings.displayName(for: locale)): \(error.localizedDescription)"
}
} else {
await settings.disable(locale)
}
}
}
}