Add voice notes: on-device transcription, multi-language, watch capture
- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud - AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline with enabled-language set and confidence-based auto-pick, re-transcribe menu - Settings > Transcription > Languages with asset download/reserve handling - watchOS app + accessory complication: one-button recorder, WCSession transferFile to phone, WatchInbox staging, direct-upsert ingestion - Player UI, transcription status chips, quick-capture mic in notes list - Version 0.2, changelog + README updated
This commit is contained in:
@@ -0,0 +1,252 @@
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
import IndieSync
|
||||
import Speech
|
||||
import SwiftData
|
||||
|
||||
/// On-device, after-the-fact transcription of voice notes. `@Observable
|
||||
/// @MainActor` with a FIFO one-at-a-time queue: a `SpeechAnalyzer` is memory- and
|
||||
/// model-heavy, so notes are transcribed strictly one after another. The analyzer
|
||||
/// work is awaited (the analyzer is an actor), so it doesn't block the main actor
|
||||
/// despite this class being main-actor confined.
|
||||
///
|
||||
/// Only this device's own pending notes are ever enqueued — ownership is decided
|
||||
/// by `AudioInfo.transcriberDeviceID` upstream (`SyncEngine`), which this service
|
||||
/// trusts. A failed job is terminal until the user asks to retry.
|
||||
@Observable
|
||||
@MainActor
|
||||
final class TranscriptionService {
|
||||
private let syncEngine: SyncEngine
|
||||
private let settings: TranscriptionSettings
|
||||
private let modelContainer: ModelContainer
|
||||
|
||||
/// Seconds of audio the sampling pass listens to when choosing a language.
|
||||
private static let sampleSeconds: Double = 15
|
||||
|
||||
private struct Job {
|
||||
let noteID: ULID
|
||||
/// A fresh recording's temp file, passed on enqueue for the fast path; nil
|
||||
/// for re-transcription, which pulls the audio back down from the container.
|
||||
let localAudioURL: URL?
|
||||
/// A forced language (re-transcribe menu), else auto-select.
|
||||
let forcedLocaleID: String?
|
||||
}
|
||||
|
||||
private var queue: [Job] = []
|
||||
/// Ids currently queued or in flight, so overlapping triggers (launch resume,
|
||||
/// watch ingest, a re-transcribe) can't enqueue the same note twice.
|
||||
private var inFlightIDs: Set<String> = []
|
||||
private var pumpTask: Task<Void, Never>?
|
||||
|
||||
init(syncEngine: SyncEngine, settings: TranscriptionSettings, modelContainer: ModelContainer) {
|
||||
self.syncEngine = syncEngine
|
||||
self.settings = settings
|
||||
self.modelContainer = modelContainer
|
||||
}
|
||||
|
||||
// MARK: - Queue
|
||||
|
||||
/// Queue a note for transcription. `localAudioURL` is the fresh-recording fast
|
||||
/// path (skips the container round-trip); `forcedLocaleID` skips language
|
||||
/// auto-selection. Duplicate enqueues of an in-flight note are ignored.
|
||||
func enqueue(noteID: ULID, localAudioURL: URL? = nil, forcedLocaleID: String? = nil) {
|
||||
let key = noteID.stringValue
|
||||
guard !inFlightIDs.contains(key) else { return }
|
||||
inFlightIDs.insert(key)
|
||||
queue.append(Job(noteID: noteID, localAudioURL: localAudioURL, forcedLocaleID: forcedLocaleID))
|
||||
pump()
|
||||
}
|
||||
|
||||
/// On launch, re-enqueue every note this device owns that is still pending —
|
||||
/// recovering work that a crash or kill interrupted mid-transcription. Reads
|
||||
/// the cache (already rebuilt by the sync engine's connect).
|
||||
func resumePendingWork() {
|
||||
let context = modelContainer.mainContext
|
||||
let installID = TranscriptionSettings.installID
|
||||
let pendingRaw = Note.TranscriptionStatus.pending.rawValue
|
||||
let descriptor = FetchDescriptor<NoteEntity>(predicate: #Predicate {
|
||||
$0.hasAudio
|
||||
&& $0.transcriptionStatusRaw == pendingRaw
|
||||
&& $0.transcriberDeviceID == installID
|
||||
})
|
||||
let rows = (try? context.fetch(descriptor)) ?? []
|
||||
for row in rows {
|
||||
if let id = ULID(string: row.id) { enqueue(noteID: id) }
|
||||
}
|
||||
}
|
||||
|
||||
private func pump() {
|
||||
guard pumpTask == nil else { return }
|
||||
pumpTask = Task { [weak self] in
|
||||
while let self, let job = self.dequeue() {
|
||||
await self.process(job)
|
||||
self.inFlightIDs.remove(job.noteID.stringValue)
|
||||
}
|
||||
self?.pumpTask = nil
|
||||
}
|
||||
}
|
||||
|
||||
private func dequeue() -> Job? {
|
||||
queue.isEmpty ? nil : queue.removeFirst()
|
||||
}
|
||||
|
||||
// MARK: - Processing
|
||||
|
||||
/// Transcribe one job, retrying once before marking the note failed. A failed
|
||||
/// note is left for the user to retry explicitly (re-transcribe / Retry).
|
||||
private func process(_ job: Job) async {
|
||||
do {
|
||||
try await transcribe(job)
|
||||
} catch {
|
||||
do {
|
||||
try await transcribe(job)
|
||||
} catch {
|
||||
print("[Transcription] \(job.noteID.stringValue) failed: \(error)")
|
||||
try? await syncEngine.markTranscriptionFailed(id: job.noteID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func transcribe(_ job: Job) async throws {
|
||||
let audio = try await resolveAudioFile(for: job)
|
||||
defer { cleanUp(audio: audio, job: job) }
|
||||
|
||||
let locale = try await selectLocale(fileURL: audio.url, forcedLocaleID: job.forcedLocaleID)
|
||||
let transcript = try await runTranscription(fileURL: audio.url, locale: locale)
|
||||
let text = String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
try await syncEngine.applyTranscription(
|
||||
id: job.noteID,
|
||||
text: text,
|
||||
localeID: TranscriptionSettings.canonicalID(locale)
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Audio acquisition
|
||||
|
||||
private struct AudioFile {
|
||||
let url: URL
|
||||
/// Whether we wrote a temp copy that must be cleaned up after the job.
|
||||
let isTemporary: Bool
|
||||
}
|
||||
|
||||
/// The fresh recording if it's still on disk, else a temp copy pulled back
|
||||
/// from the container (bounded wait for an evicted file).
|
||||
private func resolveAudioFile(for job: Job) async throws -> AudioFile {
|
||||
if let local = job.localAudioURL, FileManager.default.fileExists(atPath: local.path) {
|
||||
return AudioFile(url: local, isTemporary: false)
|
||||
}
|
||||
let data = try await syncEngine.loadAudioDataWaiting(for: job.noteID)
|
||||
let url = FileManager.default.temporaryDirectory
|
||||
.appending(path: "transcribe-\(job.noteID.stringValue).m4a")
|
||||
try data.write(to: url)
|
||||
return AudioFile(url: url, isTemporary: true)
|
||||
}
|
||||
|
||||
private func cleanUp(audio: AudioFile, job: Job) {
|
||||
let fm = FileManager.default
|
||||
if audio.isTemporary { try? fm.removeItem(at: audio.url) }
|
||||
// A fresh recording's temp file is no longer needed once the container
|
||||
// holds the bytes; drop it to keep the temp dir tidy.
|
||||
if let local = job.localAudioURL,
|
||||
local.path.hasPrefix(fm.temporaryDirectory.path) {
|
||||
try? fm.removeItem(at: local)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Language selection
|
||||
|
||||
private func selectLocale(fileURL: URL, forcedLocaleID: String?) async throws -> Locale {
|
||||
if let forcedLocaleID {
|
||||
return Locale(identifier: forcedLocaleID)
|
||||
}
|
||||
let candidates = await settings.enabledInstalledLocales()
|
||||
guard !candidates.isEmpty else { throw TranscriptionError.noInstalledLanguage }
|
||||
if candidates.count == 1 { return candidates[0] }
|
||||
|
||||
// Sampling pass: transcribe a short clip in each language and pick the
|
||||
// most confident. Run sequentially — one analyzer at a time keeps the
|
||||
// memory ceiling low.
|
||||
let sample = try makeSampleClip(from: fileURL, seconds: Self.sampleSeconds)
|
||||
defer { if sample.isTemporary { try? FileManager.default.removeItem(at: sample.url) } }
|
||||
|
||||
var scored: [LanguageCandidate] = []
|
||||
for locale in candidates {
|
||||
let transcript = (try? await runTranscription(fileURL: sample.url, locale: locale)) ?? AttributedString()
|
||||
scored.append(LanguageCandidate(
|
||||
localeID: TranscriptionSettings.canonicalID(locale),
|
||||
meanConfidence: LanguagePicker.meanConfidence(of: transcript),
|
||||
transcriptLength: transcript.characters.count
|
||||
))
|
||||
}
|
||||
guard let winner = LanguagePicker.pick(from: scored) else { return candidates[0] }
|
||||
return Locale(identifier: winner.localeID)
|
||||
}
|
||||
|
||||
/// The first `seconds` of the recording as its own file, so the sampling pass
|
||||
/// isn't run over a whole long note. Returns the original file (no copy) when
|
||||
/// the recording is already shorter than the sample window.
|
||||
private func makeSampleClip(from fileURL: URL, seconds: Double) throws -> AudioFile {
|
||||
let source = try AVAudioFile(forReading: fileURL)
|
||||
let format = source.processingFormat
|
||||
let wantFrames = format.sampleRate * seconds
|
||||
guard Double(source.length) > wantFrames else {
|
||||
return AudioFile(url: fileURL, isTemporary: false)
|
||||
}
|
||||
let frameCount = AVAudioFrameCount(wantFrames)
|
||||
guard frameCount > 0, let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frameCount) else {
|
||||
throw TranscriptionError.audioUnreadable
|
||||
}
|
||||
try source.read(into: buffer, frameCount: frameCount)
|
||||
let clipURL = FileManager.default.temporaryDirectory
|
||||
.appending(path: "sample-\(UUID().uuidString).caf")
|
||||
let clip = try AVAudioFile(forWriting: clipURL, settings: format.settings)
|
||||
try clip.write(from: buffer)
|
||||
return AudioFile(url: clipURL, isTemporary: true)
|
||||
}
|
||||
|
||||
// MARK: - Analyzer
|
||||
|
||||
/// Transcribe a whole file in one locale and return the finalized attributed
|
||||
/// transcript (carrying per-run confidence). Follows the documented pattern:
|
||||
/// start a collector over `transcriber.results`, feed the file, finalize.
|
||||
private func runTranscription(fileURL: URL, locale: Locale) async throws -> AttributedString {
|
||||
let transcriber = SpeechTranscriber(
|
||||
locale: locale,
|
||||
transcriptionOptions: [],
|
||||
reportingOptions: [],
|
||||
attributeOptions: [.transcriptionConfidence]
|
||||
)
|
||||
let analyzer = SpeechAnalyzer(
|
||||
modules: [transcriber],
|
||||
options: SpeechAnalyzer.Options(priority: .utility, modelRetention: .whileInUse)
|
||||
)
|
||||
let file = try AVAudioFile(forReading: fileURL)
|
||||
|
||||
// Collect finalized results concurrently; `results` terminates when the
|
||||
// analyzer finishes, ending the loop.
|
||||
let collector = Task {
|
||||
var full = AttributedString()
|
||||
for try await result in transcriber.results where result.isFinal {
|
||||
full.append(result.text)
|
||||
}
|
||||
return full
|
||||
}
|
||||
|
||||
do {
|
||||
if let lastSample = try await analyzer.analyzeSequence(from: file) {
|
||||
try await analyzer.finalizeAndFinish(through: lastSample)
|
||||
} else {
|
||||
await analyzer.cancelAndFinishNow()
|
||||
}
|
||||
} catch {
|
||||
collector.cancel()
|
||||
throw error
|
||||
}
|
||||
return try await collector.value
|
||||
}
|
||||
|
||||
enum TranscriptionError: Error {
|
||||
case audioUnreadable
|
||||
case noInstalledLanguage
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user