Files
notes/Notes/Services/TranscriptionService.swift
T
rzen 86d74806c9 Add voice notes: on-device transcription, multi-language, watch capture
- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud
- AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline
  with enabled-language set and confidence-based auto-pick, re-transcribe menu
- Settings > Transcription > Languages with asset download/reserve handling
- watchOS app + accessory complication: one-button recorder, WCSession
  transferFile to phone, WatchInbox staging, direct-upsert ingestion
- Player UI, transcription status chips, quick-capture mic in notes list
- Version 0.2, changelog + README updated
2026-07-15 13:30:43 -04:00

253 lines
10 KiB
Swift

import AVFoundation
import Foundation
import IndieSync
import Speech
import SwiftData
/// On-device, after-the-fact transcription of voice notes. `@Observable
/// @MainActor` with a FIFO one-at-a-time queue: a `SpeechAnalyzer` is memory- and
/// model-heavy, so notes are transcribed strictly one after another. The analyzer
/// work is awaited (the analyzer is an actor), so it doesn't block the main actor
/// despite this class being main-actor confined.
///
/// Only this device's own pending notes are ever enqueued — ownership is decided
/// by `AudioInfo.transcriberDeviceID` upstream (`SyncEngine`), which this service
/// trusts. A failed job is terminal until the user asks to retry.
@Observable
@MainActor
final class TranscriptionService {
private let syncEngine: SyncEngine
private let settings: TranscriptionSettings
private let modelContainer: ModelContainer
/// Seconds of audio the sampling pass listens to when choosing a language.
private static let sampleSeconds: Double = 15
private struct Job {
let noteID: ULID
/// A fresh recording's temp file, passed on enqueue for the fast path; nil
/// for re-transcription, which pulls the audio back down from the container.
let localAudioURL: URL?
/// A forced language (re-transcribe menu), else auto-select.
let forcedLocaleID: String?
}
private var queue: [Job] = []
/// Ids currently queued or in flight, so overlapping triggers (launch resume,
/// watch ingest, a re-transcribe) can't enqueue the same note twice.
private var inFlightIDs: Set<String> = []
private var pumpTask: Task<Void, Never>?
init(syncEngine: SyncEngine, settings: TranscriptionSettings, modelContainer: ModelContainer) {
self.syncEngine = syncEngine
self.settings = settings
self.modelContainer = modelContainer
}
// MARK: - Queue
/// Queue a note for transcription. `localAudioURL` is the fresh-recording fast
/// path (skips the container round-trip); `forcedLocaleID` skips language
/// auto-selection. Duplicate enqueues of an in-flight note are ignored.
func enqueue(noteID: ULID, localAudioURL: URL? = nil, forcedLocaleID: String? = nil) {
let key = noteID.stringValue
guard !inFlightIDs.contains(key) else { return }
inFlightIDs.insert(key)
queue.append(Job(noteID: noteID, localAudioURL: localAudioURL, forcedLocaleID: forcedLocaleID))
pump()
}
/// On launch, re-enqueue every note this device owns that is still pending —
/// recovering work that a crash or kill interrupted mid-transcription. Reads
/// the cache (already rebuilt by the sync engine's connect).
func resumePendingWork() {
let context = modelContainer.mainContext
let installID = TranscriptionSettings.installID
let pendingRaw = Note.TranscriptionStatus.pending.rawValue
let descriptor = FetchDescriptor<NoteEntity>(predicate: #Predicate {
$0.hasAudio
&& $0.transcriptionStatusRaw == pendingRaw
&& $0.transcriberDeviceID == installID
})
let rows = (try? context.fetch(descriptor)) ?? []
for row in rows {
if let id = ULID(string: row.id) { enqueue(noteID: id) }
}
}
private func pump() {
guard pumpTask == nil else { return }
pumpTask = Task { [weak self] in
while let self, let job = self.dequeue() {
await self.process(job)
self.inFlightIDs.remove(job.noteID.stringValue)
}
self?.pumpTask = nil
}
}
private func dequeue() -> Job? {
queue.isEmpty ? nil : queue.removeFirst()
}
// MARK: - Processing
/// Transcribe one job, retrying once before marking the note failed. A failed
/// note is left for the user to retry explicitly (re-transcribe / Retry).
private func process(_ job: Job) async {
do {
try await transcribe(job)
} catch {
do {
try await transcribe(job)
} catch {
print("[Transcription] \(job.noteID.stringValue) failed: \(error)")
try? await syncEngine.markTranscriptionFailed(id: job.noteID)
}
}
}
private func transcribe(_ job: Job) async throws {
let audio = try await resolveAudioFile(for: job)
defer { cleanUp(audio: audio, job: job) }
let locale = try await selectLocale(fileURL: audio.url, forcedLocaleID: job.forcedLocaleID)
let transcript = try await runTranscription(fileURL: audio.url, locale: locale)
let text = String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
try await syncEngine.applyTranscription(
id: job.noteID,
text: text,
localeID: TranscriptionSettings.canonicalID(locale)
)
}
// MARK: - Audio acquisition
private struct AudioFile {
let url: URL
/// Whether we wrote a temp copy that must be cleaned up after the job.
let isTemporary: Bool
}
/// The fresh recording if it's still on disk, else a temp copy pulled back
/// from the container (bounded wait for an evicted file).
private func resolveAudioFile(for job: Job) async throws -> AudioFile {
if let local = job.localAudioURL, FileManager.default.fileExists(atPath: local.path) {
return AudioFile(url: local, isTemporary: false)
}
let data = try await syncEngine.loadAudioDataWaiting(for: job.noteID)
let url = FileManager.default.temporaryDirectory
.appending(path: "transcribe-\(job.noteID.stringValue).m4a")
try data.write(to: url)
return AudioFile(url: url, isTemporary: true)
}
private func cleanUp(audio: AudioFile, job: Job) {
let fm = FileManager.default
if audio.isTemporary { try? fm.removeItem(at: audio.url) }
// A fresh recording's temp file is no longer needed once the container
// holds the bytes; drop it to keep the temp dir tidy.
if let local = job.localAudioURL,
local.path.hasPrefix(fm.temporaryDirectory.path) {
try? fm.removeItem(at: local)
}
}
// MARK: - Language selection
private func selectLocale(fileURL: URL, forcedLocaleID: String?) async throws -> Locale {
if let forcedLocaleID {
return Locale(identifier: forcedLocaleID)
}
let candidates = await settings.enabledInstalledLocales()
guard !candidates.isEmpty else { throw TranscriptionError.noInstalledLanguage }
if candidates.count == 1 { return candidates[0] }
// Sampling pass: transcribe a short clip in each language and pick the
// most confident. Run sequentially — one analyzer at a time keeps the
// memory ceiling low.
let sample = try makeSampleClip(from: fileURL, seconds: Self.sampleSeconds)
defer { if sample.isTemporary { try? FileManager.default.removeItem(at: sample.url) } }
var scored: [LanguageCandidate] = []
for locale in candidates {
let transcript = (try? await runTranscription(fileURL: sample.url, locale: locale)) ?? AttributedString()
scored.append(LanguageCandidate(
localeID: TranscriptionSettings.canonicalID(locale),
meanConfidence: LanguagePicker.meanConfidence(of: transcript),
transcriptLength: transcript.characters.count
))
}
guard let winner = LanguagePicker.pick(from: scored) else { return candidates[0] }
return Locale(identifier: winner.localeID)
}
/// The first `seconds` of the recording as its own file, so the sampling pass
/// isn't run over a whole long note. Returns the original file (no copy) when
/// the recording is already shorter than the sample window.
private func makeSampleClip(from fileURL: URL, seconds: Double) throws -> AudioFile {
let source = try AVAudioFile(forReading: fileURL)
let format = source.processingFormat
let wantFrames = format.sampleRate * seconds
guard Double(source.length) > wantFrames else {
return AudioFile(url: fileURL, isTemporary: false)
}
let frameCount = AVAudioFrameCount(wantFrames)
guard frameCount > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) else {
throw TranscriptionError.audioUnreadable
}
try source.read(into: buffer, frameCount: frameCount)
let clipURL = FileManager.default.temporaryDirectory
.appending(path: "sample-\(UUID().uuidString).caf")
let clip = try AVAudioFile(forWriting: clipURL, settings: format.settings)
try clip.write(from: buffer)
return AudioFile(url: clipURL, isTemporary: true)
}
// MARK: - Analyzer
/// Transcribe a whole file in one locale and return the finalized attributed
/// transcript (carrying per-run confidence). Follows the documented pattern:
/// start a collector over `transcriber.results`, feed the file, finalize.
private func runTranscription(fileURL: URL, locale: Locale) async throws -> AttributedString {
let transcriber = SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: [.transcriptionConfidence]
)
let analyzer = SpeechAnalyzer(
modules: [transcriber],
options: SpeechAnalyzer.Options(priority: .utility, modelRetention: .whileInUse)
)
let file = try AVAudioFile(forReading: fileURL)
// Collect finalized results concurrently; `results` terminates when the
// analyzer finishes, ending the loop.
let collector = Task {
var full = AttributedString()
for try await result in transcriber.results where result.isFinal {
full.append(result.text)
}
return full
}
do {
if let lastSample = try await analyzer.analyzeSequence(from: file) {
try await analyzer.finalizeAndFinish(through: lastSample)
} else {
await analyzer.cancelAndFinishNow()
}
} catch {
collector.cancel()
throw error
}
return try await collector.value
}
enum TranscriptionError: Error {
case audioUnreadable
case noInstalledLanguage
}
}