rzen/notes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dictation-locale transcription (incl. Russian), rework list capture UI

Browse Source 
- TranscriptionSettings/Service route per-locale to SpeechTranscriber or
  DictationTranscriber (union of both supported sets, speech preferred);
  neutral-confidence ranking fallback keeps mixed-engine auto-pick working
- NotesListView: text + voice capture buttons above the list (glassProminent),
  search moved to the iOS 26 toolbar search accessory (.minimize)
- Changelog entries for languages, capture buttons, search placement
This commit is contained in:
rzen
2026-07-15 15:31:03 -04:00
parent 3a9c36d843
commit 8bdda913a7
6 changed files with 313 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+6
View File
@@ -1,5 +1,11 @@
**July 2026**
Transcription now supports many more languages, including Russian, Ukrainian, and Polish
New notes start from two big buttons right above the list — one for text, one for voice
On iPhone, search now lives in the toolbar at the bottom of the screen
Record voice notes that are transcribed to text right on your device, with the original audio kept alongside the note for playback
Choose the languages you speak in Settings and transcription picks the right one automatically, or re-transcribe any note in another language
Notes/Services/LanguagePicker.swift
+25
View File
@@ -20,6 +20,15 @@ struct LanguageCandidate: Equatable {
/// side-effect-free is what makes the (undocumented, cross-locale) confidence
/// calibration unit-testable.
enum LanguagePicker {
/// The confidence assigned to a transcript that carried text but no
/// `transcriptionConfidence` attribute at all — the graceful-degradation case
/// for `DictationTranscriber`, which need not emit per-run confidence. A
/// mid-scale value so such a candidate is ranked by transcript length against
/// its peers rather than being zeroed out and always losing to a confidence-
/// reporting engine. Two confidence-less candidates then tie here and fall
/// through to the length tie-break in `pick`.
static let neutralConfidence = 0.5
/// Character-count-weighted mean confidence over an attributed transcript.
/// Only runs that actually carry a `transcriptionConfidence` attribute
/// contribute, weighted by their character length; a transcript with no
@@ -37,6 +46,22 @@ enum LanguagePicker {
return totalCharacters > 0 ? weighted / Double(totalCharacters) : 0
}
/// Whether any run in the transcript carries a `transcriptionConfidence`
/// attribute — i.e. whether `meanConfidence` measured anything real.
static func hasConfidence(of attributed: AttributedString) -> Bool {
attributed.runs.contains { $0.transcriptionConfidence != nil }
}
/// The confidence used to *rank* a candidate. Prefers the real weighted mean;
/// when the transcript has text but no confidence attribute anywhere (an engine
/// that doesn't emit it), falls back to `neutralConfidence` so the candidate
/// competes on length instead of being zeroed. An empty transcript scores 0 —
/// it can never win against real text.
static func rankingConfidence(of attributed: AttributedString) -> Double {
if hasConfidence(of: attributed) { return meanConfidence(of: attributed) }
return attributed.characters.isEmpty ? 0 : neutralConfidence
}
/// The best candidate, or nil when given nothing. Argmax on `meanConfidence`;
/// a zero-length transcript can only win if every candidate is empty. Ties
/// break deterministically — higher confidence, then longer transcript, then
Notes/Services/TranscriptionService.swift
+50 -12
View File
@@ -172,9 +172,13 @@ final class TranscriptionService {
var scored: [LanguageCandidate] = []
for locale in candidates {
let transcript = (try? await runTranscription(fileURL: sample.url, locale: locale)) ?? AttributedString()
// `rankingConfidence`, not `meanConfidence`: a `DictationTranscriber`
// locale may leave runs unattributed, so an unscored-but-non-empty
// transcript gets a neutral score and competes on length rather than
// losing outright to a confidence-reporting `SpeechTranscriber` locale.
scored.append(LanguageCandidate(
localeID: TranscriptionSettings.canonicalID(locale),
meanConfidence: LanguagePicker.meanConfidence(of: transcript),
meanConfidence: LanguagePicker.rankingConfidence(of: transcript),
transcriptLength: transcript.characters.count
))
}
@@ -207,17 +211,51 @@ final class TranscriptionService {
// MARK: - Analyzer
/// Transcribe a whole file in one locale and return the finalized attributed
/// transcript (carrying per-run confidence). Follows the documented pattern:
/// start a collector over `transcriber.results`, feed the file, finalize.
/// transcript (carrying per-run confidence where the engine emits it). Routes
/// to whichever engine supports the locale — `SpeechTranscriber` or, for its
/// dictation-only locales (ru-RU, uk-UA, pl-PL …), `DictationTranscriber` —
/// via `TranscriptionSettings.engine(for:)`. Both request confidence; the two
/// `Result` types share `isFinal`/`text` but only concretely, so the engine
/// branch supplies the module and its result stream to a shared `analyze`.
private func runTranscription(fileURL: URL, locale: Locale) async throws -> AttributedString {
let transcriber = SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: [.transcriptionConfidence]
)
switch settings.engine(for: locale) {
case .speech:
let transcriber = SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: [.transcriptionConfidence]
)
return try await analyze(module: transcriber, results: transcriber.results, fileURL: fileURL) { $0.text }
case .dictation:
let transcriber = DictationTranscriber(
locale: locale,
contentHints: [],
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: [.transcriptionConfidence]
)
return try await analyze(module: transcriber, results: transcriber.results, fileURL: fileURL) { $0.text }
}
}
/// Drive `SpeechAnalyzer` over the whole file with a single module and fold its
/// finalized results into one transcript. Generic over the module's result
/// stream so one pipeline serves both engines (their `Result`s share `isFinal`
/// via `SpeechModuleResult` but expose `text` only concretely — hence `textOf`).
/// Follows the documented pattern: start a collector over `results`, feed the
/// file, finalize.
private func analyze<Results>(
module: any SpeechModule,
results: Results,
fileURL: URL,
textOf: @escaping @Sendable (Results.Element) -> AttributedString
) async throws -> AttributedString
where Results: AsyncSequence & Sendable,
Results.Element: SpeechModuleResult & Sendable,
Results.Failure == any Error {
let analyzer = SpeechAnalyzer(
modules: [transcriber],
modules: [module],
options: SpeechAnalyzer.Options(priority: .utility, modelRetention: .whileInUse)
)
let file = try AVAudioFile(forReading: fileURL)
@@ -226,8 +264,8 @@ final class TranscriptionService {
// analyzer finishes, ending the loop.
let collector = Task {
var full = AttributedString()
for try await result in transcriber.results where result.isFinal {
full.append(result.text)
for try await result in results where result.isFinal {
full.append(textOf(result))
}
return full
}
Notes/Services/TranscriptionSettings.swift
+94 -16
View File
@@ -10,9 +10,24 @@ import Speech
/// *reserves* it with `AssetInventory` — which is both what keeps its model from
/// being purged and what the system caps at `AssetInventory.maximumReservedLocales`
/// — so "enabled" and "reserved" are kept in lock-step and the UI enforces the cap.
///
/// The supported set is the *union* of two on-device SpeechAnalyzer modules'
/// coverage: `SpeechTranscriber` (30 locales) and `DictationTranscriber` (54,
/// including ru-RU, uk-UA, pl-PL … that `SpeechTranscriber` lacks). Each locale
/// is routed to whichever engine supports it, preferring `SpeechTranscriber`
/// when both do (see `engine(for:)`). Reservation is locale-based and engine-
/// agnostic; only asset queries and analyzer construction pick a module type.
@Observable
@MainActor
final class TranscriptionSettings {
/// Which SpeechAnalyzer module transcribes a given locale. `SpeechTranscriber`
/// is preferred where available (it reports per-run confidence the sampling
/// pass relies on); `DictationTranscriber` covers the locales it doesn't.
enum EngineKind: Equatable {
case speech
case dictation
}
private static let enabledKey = "Notes.enabledTranscriptionLocales"
private static let seededKey = "Notes.transcriptionLocalesSeeded"
@@ -24,10 +39,20 @@ final class TranscriptionSettings {
/// Canonical BCP-47 ids of the enabled languages, persisted in UserDefaults.
private(set) var enabledLocaleIDs: [String]
/// Every locale `SpeechTranscriber` can transcribe (36+), loaded once by
/// `prepare()`. Empty until then.
/// Every locale either engine can transcribe — the union of `SpeechTranscriber`
/// and `DictationTranscriber` coverage, de-duplicated by canonical id and
/// loaded once by `prepare()`. Empty until then.
private(set) var supportedLocales: [Locale] = []
/// Canonical ids `SpeechTranscriber` supports, for engine resolution. A locale
/// in this set is routed to `.speech`; anything else supported falls to
/// `.dictation`. Populated by `prepare()`.
private var speechLocaleIDs: Set<String> = []
/// Canonical ids `DictationTranscriber` supports, for engine resolution and
/// for distinguishing a genuinely unsupported locale from a dictation-only one.
private var dictationLocaleIDs: Set<String> = []
/// Runtime cap on how many languages can be enabled at once — read from the
/// system, not hard-coded, because it can change.
private(set) var maximumReserved = 0
@@ -74,7 +99,22 @@ final class TranscriptionSettings {
/// and cheap to call again — it re-fetches the list but only seeds once.
func prepare() async {
maximumReserved = AssetInventory.maximumReservedLocales
supportedLocales = await SpeechTranscriber.supportedLocales
// Query both engines concurrently; the supported set is their union.
async let speech = SpeechTranscriber.supportedLocales
async let dictation = DictationTranscriber.supportedLocales
let speechList = await speech
let dictationList = await dictation
speechLocaleIDs = Set(speechList.map(Self.canonicalID))
dictationLocaleIDs = Set(dictationList.map(Self.canonicalID))
// De-duplicate by canonical id, keeping the `SpeechTranscriber` locale
// object where both engines cover it (harmless — same bcp47 either way).
var byID: [String: Locale] = [:]
for locale in dictationList { byID[Self.canonicalID(locale)] = locale }
for locale in speechList { byID[Self.canonicalID(locale)] = locale }
supportedLocales = byID.values
.sorted { Self.displayName(for: $0) < Self.displayName(for: $1) }
if !defaults.bool(forKey: Self.seededKey) {
@@ -128,7 +168,7 @@ final class TranscriptionSettings {
}
private func currentState(for locale: Locale) async -> AssetState {
switch await AssetInventory.status(forModules: [Self.transcriber(for: locale)]) {
switch await AssetInventory.status(forModules: [module(for: locale)]) {
case .unsupported: return .unsupported
case .supported: return .notInstalled
case .downloading: return .downloading(nil)
@@ -171,7 +211,7 @@ final class TranscriptionSettings {
private func installIfNeeded(_ locale: Locale) async throws {
let id = Self.canonicalID(locale)
let transcriber = Self.transcriber(for: locale)
let transcriber = module(for: locale)
guard await AssetInventory.status(forModules: [transcriber]) != .installed else {
assetStates[id] = .installed
return
@@ -202,23 +242,61 @@ final class TranscriptionSettings {
func enabledInstalledLocales() async -> [Locale] {
var installed: [Locale] = []
for locale in enabledLocales
where await AssetInventory.status(forModules: [Self.transcriber(for: locale)]) == .installed {
where await AssetInventory.status(forModules: [module(for: locale)]) == .installed {
installed.append(locale)
}
return installed
}
// MARK: - Helpers
// MARK: - Engine routing
/// A minimal transcriber for a locale, used purely for asset queries. The
/// attribute options don't affect asset status, so they're omitted here.
static func transcriber(for locale: Locale) -> SpeechTranscriber {
SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: []
)
/// Which module transcribes `locale`: `.dictation` only for a locale the
/// dictation engine covers and `SpeechTranscriber` does not; `.speech`
/// otherwise. This both prefers `SpeechTranscriber` when both cover a locale
/// and defaults to it before `prepare()` has populated the sets (empty sets ⇒
/// `.speech` everywhere, the historical behavior) — so a transcription that
/// races startup still routes sanely rather than sending English to dictation.
/// A truly unsupported locale reports `.unsupported` under either engine.
func engine(for locale: Locale) -> EngineKind {
let id = Self.canonicalID(locale)
if dictationLocaleIDs.contains(id), !speechLocaleIDs.contains(id) { return .dictation }
return .speech
}
/// The analyzer module for a locale, built with its resolved engine. Used for
/// asset queries (confidence omitted, which doesn't affect asset status) and,
/// with `requestingConfidence: true`, for the transcription passes.
func module(for locale: Locale, requestingConfidence: Bool = false) -> any SpeechModule {
Self.module(for: locale, engine: engine(for: locale), requestingConfidence: requestingConfidence)
}
/// A locale's analyzer module for a given engine. Pure, so `TranscriptionService`
/// can build the same module the settings screen queries. `requestingConfidence`
/// asks the engine for the `.transcriptionConfidence` attribute — honored where
/// the engine emits it (`SpeechTranscriber` always; `DictationTranscriber` may
/// leave runs unattributed, which `LanguagePicker` degrades gracefully).
static func module(
for locale: Locale,
engine: EngineKind,
requestingConfidence: Bool = false
) -> any SpeechModule {
switch engine {
case .speech:
return SpeechTranscriber(
locale: locale,
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: requestingConfidence ? [.transcriptionConfidence] : []
)
case .dictation:
return DictationTranscriber(
locale: locale,
contentHints: [],
transcriptionOptions: [],
reportingOptions: [],
attributeOptions: requestingConfidence ? [.transcriptionConfidence] : []
)
}
}
private func persist() {
Notes/Views/Notes/NotesListView.swift
+73 -48
View File
@@ -16,57 +16,20 @@ struct NotesListView: View {
var body: some View {
NavigationStack {
List {
if let error = syncEngine.lastSyncError {
Section {
Label(error, systemImage: "exclamationmark.icloud")
.foregroundStyle(.red)
.font(.footnote)
}
}
if searchText.isEmpty, !relevantNotes.isEmpty {
Section("Right here, right now") {
ForEach(relevantNotes) { note in
row(note)
}
}
}
Section(searchText.isEmpty ? "All notes" : "Results") {
ForEach(filteredNotes) { note in
row(note)
}
}
VStack(spacing: 0) {
captureButtons
list
}
.navigationTitle("Notes")
// iOS 26: attached at the NavigationStack root inside a TabView, this
// renders as the Liquid-Glass toolbar search accessory (bottom, in the
// tab bar). `.searchToolbarBehavior(.minimize)` collapses it to the
// magnifier capsule until tapped — a modifier that only exists on iOS,
// hence the guard; macOS keeps the standard toolbar search field.
.searchable(text: $searchText, prompt: "Search notes and tags")
.overlay {
if notes.isEmpty {
ContentUnavailableView(
"No notes yet",
systemImage: "square.and.pencil",
description: Text("Jot something down — Notes remembers where and when, so it can resurface at the right moment.")
)
}
}
.toolbar {
ToolbarItem(placement: .primaryAction) {
Button {
recording = true
} label: {
Label("Record Note", systemImage: "mic")
}
}
ToolbarItem(placement: .primaryAction) {
Button {
composing = true
} label: {
Label("New Note", systemImage: "square.and.pencil")
}
.keyboardShortcut("n", modifiers: .command)
}
}
#if os(iOS)
.searchToolbarBehavior(.minimize)
#endif
.sheet(isPresented: $composing) {
NoteEditorView(mode: .create)
}
@@ -79,6 +42,68 @@ struct NotesListView: View {
}
}
/// The two prominent capture actions, side by side above the list — the
/// primary way to start a note, promoted out of the toolbar. Glass-prominent
/// capsules inherit the app's red accent; each fills half the width.
private var captureButtons: some View {
HStack(spacing: 12) {
Button {
composing = true
} label: {
Label("Text", systemImage: "square.and.pencil")
.frame(maxWidth: .infinity)
}
.keyboardShortcut("n", modifiers: .command)
Button {
recording = true
} label: {
Label("Audio", systemImage: "mic.fill")
.frame(maxWidth: .infinity)
}
}
.buttonStyle(.glassProminent)
.controlSize(.large)
.padding(.horizontal)
.padding(.top, 4)
.padding(.bottom, 8)
}
private var list: some View {
List {
if let error = syncEngine.lastSyncError {
Section {
Label(error, systemImage: "exclamationmark.icloud")
.foregroundStyle(.red)
.font(.footnote)
}
}
if searchText.isEmpty, !relevantNotes.isEmpty {
Section("Right here, right now") {
ForEach(relevantNotes) { note in
row(note)
}
}
}
Section(searchText.isEmpty ? "All notes" : "Results") {
ForEach(filteredNotes) { note in
row(note)
}
}
}
.overlay {
if notes.isEmpty {
ContentUnavailableView(
"No notes yet",
systemImage: "square.and.pencil",
description: Text("Jot something down — Notes remembers where and when, so it can resurface at the right moment.")
)
}
}
}
private func row(_ note: NoteEntity) -> some View {
Button {
editingNote = note
NotesTests/LanguagePickerTests.swift
+65
View File
@@ -36,6 +36,71 @@ struct LanguagePickerTests {
#expect(LanguagePicker.meanConfidence(of: transcript([("no attributes", nil)])) == 0)
}
// MARK: - Ranking confidence (mixed-engine graceful degradation)
@Test func hasConfidenceDetectsAttributedRuns() {
#expect(LanguagePicker.hasConfidence(of: transcript([("sure", 0.9)])))
#expect(LanguagePicker.hasConfidence(of: transcript([("a", nil), ("b", 0.2)])))
#expect(!LanguagePicker.hasConfidence(of: transcript([("none", nil)])))
#expect(!LanguagePicker.hasConfidence(of: AttributedString("")))
}
@Test func rankingConfidenceUsesRealMeanWhenPresent() {
// With any confidence attribute, ranking == the character-weighted mean.
let attributed = transcript([("hello", 0.9), (" world", 0.4)])
#expect(abs(LanguagePicker.rankingConfidence(of: attributed)
- LanguagePicker.meanConfidence(of: attributed)) < 0.0001)
}
@Test func rankingConfidenceFallsBackToNeutralForUnattributedText() {
// A DictationTranscriber transcript with text but no confidence attribute
// scores the documented neutral value, not 0, so it can still compete.
let dictation = transcript([("привет мир", nil)])
#expect(LanguagePicker.rankingConfidence(of: dictation) == LanguagePicker.neutralConfidence)
}
@Test func rankingConfidenceOfEmptyIsZero() {
#expect(LanguagePicker.rankingConfidence(of: AttributedString("")) == 0)
}
@Test func neutralDictationBeatsLowConfidenceSpeech() {
// Audio is Russian: the dictation locale produces good (neutral-scored)
// text; the mismatched speech locale produces low-confidence garbage.
let winner = LanguagePicker.pick(from: [
LanguageCandidate(localeID: "en-US", meanConfidence: 0.2, transcriptLength: 12),
LanguageCandidate(localeID: "ru-RU",
meanConfidence: LanguagePicker.neutralConfidence,
transcriptLength: 18),
])
#expect(winner?.localeID == "ru-RU")
}
@Test func highConfidenceSpeechBeatsNeutralDictation() {
// Audio is English: the speech locale's real high confidence outranks the
// mismatched dictation locale's neutral score.
let winner = LanguagePicker.pick(from: [
LanguageCandidate(localeID: "en-US", meanConfidence: 0.85, transcriptLength: 20),
LanguageCandidate(localeID: "ru-RU",
meanConfidence: LanguagePicker.neutralConfidence,
transcriptLength: 22),
])
#expect(winner?.localeID == "en-US")
}
@Test func twoNeutralDictationCandidatesBreakByLength() {
// Both engines dictation-only (e.g. ru-RU vs uk-UA), both neutral-scored:
// the longer transcript wins the tie-break.
let winner = LanguagePicker.pick(from: [
LanguageCandidate(localeID: "uk-UA",
meanConfidence: LanguagePicker.neutralConfidence,
transcriptLength: 8),
LanguageCandidate(localeID: "ru-RU",
meanConfidence: LanguagePicker.neutralConfidence,
transcriptLength: 24),
])
#expect(winner?.localeID == "ru-RU")
}
// MARK: - Argmax
@Test func pickChoosesHighestConfidence() {