Add voice notes: on-device transcription, multi-language, watch capture
- Schema v2: Note.AudioInfo, .m4a sidecar next to the note JSON in iCloud - AudioRecorderService (iOS/macOS) + SpeechAnalyzer transcription pipeline with enabled-language set and confidence-based auto-pick, re-transcribe menu - Settings > Transcription > Languages with asset download/reserve handling - watchOS app + accessory complication: one-button recorder, WCSession transferFile to phone, WatchInbox staging, direct-upsert ingestion - Player UI, transcription status chips, quick-capture mic in notes list - Version 0.2, changelog + README updated
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
import Foundation
|
||||
import Testing
|
||||
@testable import Notes
|
||||
|
||||
struct AudioNotePayloadTests {
|
||||
@Test func metadataRoundTrips() throws {
|
||||
let payload = AudioNotePayload(
|
||||
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
|
||||
recordedAt: Date(timeIntervalSince1970: 1_700_000_000),
|
||||
duration: 12.5,
|
||||
timeZoneID: "Europe/Berlin"
|
||||
)
|
||||
let decoded = try #require(AudioNotePayload(metadata: payload.metadata()))
|
||||
#expect(decoded == payload)
|
||||
}
|
||||
|
||||
@Test func metadataRoundTripsWithoutTimeZone() throws {
|
||||
let payload = AudioNotePayload(
|
||||
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
|
||||
recordedAt: Date(timeIntervalSince1970: 1_700_000_100),
|
||||
duration: 4,
|
||||
timeZoneID: nil
|
||||
)
|
||||
let dict = payload.metadata()
|
||||
#expect(dict["tz"] == nil)
|
||||
let decoded = try #require(AudioNotePayload(metadata: dict))
|
||||
#expect(decoded == payload)
|
||||
}
|
||||
|
||||
/// A survivable trip through actual plist serialization — the metadata really
|
||||
/// crosses WatchConnectivity as a plist dictionary.
|
||||
@Test func metadataSurvivesPlistSerialization() throws {
|
||||
let payload = AudioNotePayload(
|
||||
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
|
||||
recordedAt: Date(timeIntervalSince1970: 1_700_000_050),
|
||||
duration: 7.25,
|
||||
timeZoneID: "America/Los_Angeles"
|
||||
)
|
||||
let data = try PropertyListSerialization.data(
|
||||
fromPropertyList: payload.metadata(), format: .binary, options: 0)
|
||||
let dict = try #require(
|
||||
try PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any])
|
||||
let decoded = try #require(AudioNotePayload(metadata: dict))
|
||||
#expect(decoded.id == payload.id)
|
||||
#expect(decoded.duration == payload.duration)
|
||||
#expect(decoded.timeZoneID == payload.timeZoneID)
|
||||
#expect(abs(decoded.recordedAt.timeIntervalSince(payload.recordedAt)) < 0.001)
|
||||
}
|
||||
|
||||
@Test func decodeRejectsUnknownNewerVersion() {
|
||||
var dict = AudioNotePayload(
|
||||
id: "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
|
||||
recordedAt: Date(),
|
||||
duration: 3,
|
||||
timeZoneID: nil
|
||||
).metadata()
|
||||
dict["v"] = AudioNotePayload.currentVersion + 1
|
||||
#expect(AudioNotePayload(metadata: dict) == nil)
|
||||
}
|
||||
|
||||
@Test func decodeRejectsMissingVersion() {
|
||||
let dict: [String: Any] = [
|
||||
"id": "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
|
||||
"recordedAt": Date(),
|
||||
"duration": 3.0,
|
||||
]
|
||||
#expect(AudioNotePayload(metadata: dict) == nil)
|
||||
}
|
||||
|
||||
@Test func decodeRejectsMissingRequiredField() {
|
||||
// Version present, but no duration.
|
||||
let dict: [String: Any] = [
|
||||
"v": AudioNotePayload.currentVersion,
|
||||
"id": "01J8Z3M9K7QW2R4T6Y8B0C1D2E",
|
||||
"recordedAt": Date(),
|
||||
]
|
||||
#expect(AudioNotePayload(metadata: dict) == nil)
|
||||
}
|
||||
|
||||
@Test func decodeRejectsEmptyMetadata() {
|
||||
#expect(AudioNotePayload(metadata: [:]) == nil)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import Foundation
|
||||
import Speech
|
||||
import Testing
|
||||
@testable import Notes
|
||||
|
||||
struct LanguagePickerTests {
|
||||
/// Build an attributed transcript out of `(text, confidence?)` fragments, so a
|
||||
/// test can hand-craft the confidence attributes the picker reads.
|
||||
private func transcript(_ fragments: [(String, Double?)]) -> AttributedString {
|
||||
var result = AttributedString()
|
||||
for (text, confidence) in fragments {
|
||||
var fragment = AttributedString(text)
|
||||
if let confidence { fragment.transcriptionConfidence = confidence }
|
||||
result.append(fragment)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// MARK: - Confidence extraction
|
||||
|
||||
@Test func meanConfidenceIsCharacterWeighted() {
|
||||
// "hello" (5 chars @ 0.9) + " world" (6 chars @ 0.4) → 0.627…
|
||||
let attributed = transcript([("hello", 0.9), (" world", 0.4)])
|
||||
let mean = LanguagePicker.meanConfidence(of: attributed)
|
||||
#expect(abs(mean - ((0.9 * 5 + 0.4 * 6) / 11)) < 0.0001)
|
||||
}
|
||||
|
||||
@Test func meanConfidenceIgnoresRunsWithoutConfidence() {
|
||||
// Only the confident run counts toward the mean.
|
||||
let attributed = transcript([("sure", 1.0), (" maybe", nil)])
|
||||
#expect(abs(LanguagePicker.meanConfidence(of: attributed) - 1.0) < 0.0001)
|
||||
}
|
||||
|
||||
@Test func meanConfidenceOfEmptyOrUnattributedIsZero() {
|
||||
#expect(LanguagePicker.meanConfidence(of: AttributedString("")) == 0)
|
||||
#expect(LanguagePicker.meanConfidence(of: transcript([("no attributes", nil)])) == 0)
|
||||
}
|
||||
|
||||
// MARK: - Argmax
|
||||
|
||||
@Test func pickChoosesHighestConfidence() {
|
||||
let winner = LanguagePicker.pick(from: [
|
||||
LanguageCandidate(localeID: "en-US", meanConfidence: 0.42, transcriptLength: 20),
|
||||
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.88, transcriptLength: 18),
|
||||
LanguageCandidate(localeID: "fr-FR", meanConfidence: 0.31, transcriptLength: 25),
|
||||
])
|
||||
#expect(winner?.localeID == "ru-RU")
|
||||
}
|
||||
|
||||
@Test func pickReturnsNilForNoCandidates() {
|
||||
#expect(LanguagePicker.pick(from: []) == nil)
|
||||
}
|
||||
|
||||
// MARK: - Empty-transcript zeroing
|
||||
|
||||
@Test func emptyTranscriptNeverBeatsRealText() {
|
||||
// A zero-length, zero-confidence candidate must lose to any real result.
|
||||
let winner = LanguagePicker.pick(from: [
|
||||
LanguageCandidate(localeID: "en-US", meanConfidence: 0, transcriptLength: 0),
|
||||
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.05, transcriptLength: 3),
|
||||
])
|
||||
#expect(winner?.localeID == "ru-RU")
|
||||
}
|
||||
|
||||
// MARK: - Tie-break determinism
|
||||
|
||||
@Test func equalConfidencePrefersLongerTranscript() {
|
||||
let winner = LanguagePicker.pick(from: [
|
||||
LanguageCandidate(localeID: "en-US", meanConfidence: 0.7, transcriptLength: 10),
|
||||
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.7, transcriptLength: 30),
|
||||
])
|
||||
#expect(winner?.localeID == "ru-RU")
|
||||
}
|
||||
|
||||
@Test func fullTieBreaksByLocaleID() {
|
||||
// Identical confidence and length → smallest localeID wins, deterministically.
|
||||
let candidates = [
|
||||
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0.5, transcriptLength: 10),
|
||||
LanguageCandidate(localeID: "en-US", meanConfidence: 0.5, transcriptLength: 10),
|
||||
LanguageCandidate(localeID: "fr-FR", meanConfidence: 0.5, transcriptLength: 10),
|
||||
]
|
||||
#expect(LanguagePicker.pick(from: candidates)?.localeID == "en-US")
|
||||
// Order of input must not change the result.
|
||||
#expect(LanguagePicker.pick(from: candidates.reversed())?.localeID == "en-US")
|
||||
}
|
||||
|
||||
@Test func allEmptyCandidatesStillPickDeterministically() {
|
||||
let candidates = [
|
||||
LanguageCandidate(localeID: "ru-RU", meanConfidence: 0, transcriptLength: 0),
|
||||
LanguageCandidate(localeID: "en-US", meanConfidence: 0, transcriptLength: 0),
|
||||
]
|
||||
#expect(LanguagePicker.pick(from: candidates)?.localeID == "en-US")
|
||||
}
|
||||
}
|
||||
@@ -44,4 +44,111 @@ struct NoteDocumentTests {
|
||||
@Test func titleIsFirstLine() {
|
||||
#expect(makeNote().title == "Barista's name is Sam")
|
||||
}
|
||||
|
||||
// MARK: - Schema v2 (audio)
|
||||
|
||||
/// A v1 file predates the `audio` key. It must decode unchanged — as a text
|
||||
/// note with `audio == nil` — and pass the forward gate (1 ≤ current).
|
||||
@Test func v1DocumentDecodesAsTextNoteWithNoAudio() throws {
|
||||
let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 1, randomLo: 2)
|
||||
let json = """
|
||||
{
|
||||
"meta": {
|
||||
"id": "\(id.stringValue)",
|
||||
"schemaVersion": 1,
|
||||
"createdAt": "2023-11-14T22:13:20Z",
|
||||
"modifiedAt": "2023-11-14T22:13:20Z"
|
||||
},
|
||||
"payload": {
|
||||
"text": "A v1 note",
|
||||
"source": "typed",
|
||||
"tags": ["legacy"],
|
||||
"context": { "device": "iPhone", "timeZoneID": "UTC" }
|
||||
}
|
||||
}
|
||||
"""
|
||||
let note = try DocumentCoder.decode(Note.self, from: Data(json.utf8))
|
||||
#expect(note.meta.schemaVersion == 1)
|
||||
#expect(note.payload.audio == nil)
|
||||
#expect(note.payload.text == "A v1 note")
|
||||
#expect(note.isReadable)
|
||||
}
|
||||
|
||||
/// A file stamped with a newer schema (and carrying a field this build
|
||||
/// doesn't model) must be quarantined by the gate — Codable silently drops
|
||||
/// the unknown key, so partial-decoding then rewriting would downgrade it.
|
||||
@Test func forwardGateQuarantinesV3Fixture() throws {
|
||||
let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 3, randomLo: 4)
|
||||
let json = """
|
||||
{
|
||||
"meta": {
|
||||
"id": "\(id.stringValue)",
|
||||
"schemaVersion": 3,
|
||||
"createdAt": "2023-11-14T22:13:20Z",
|
||||
"modifiedAt": "2023-11-14T22:13:20Z"
|
||||
},
|
||||
"payload": {
|
||||
"text": "From the future",
|
||||
"source": "typed",
|
||||
"tags": [],
|
||||
"context": {},
|
||||
"audio": null,
|
||||
"unknownFutureField": { "whatever": true }
|
||||
}
|
||||
}
|
||||
"""
|
||||
let note = try DocumentCoder.decode(Note.self, from: Data(json.utf8))
|
||||
#expect(note.meta.schemaVersion == 3)
|
||||
#expect(!note.isReadable)
|
||||
}
|
||||
|
||||
@Test func audioNoteCodableRoundTrip() throws {
|
||||
var context = CaptureContext.empty
|
||||
context.device = "Watch"
|
||||
context.timeZoneID = "Europe/Berlin"
|
||||
let audio = Note.AudioInfo(
|
||||
duration: 12.5,
|
||||
transcriberDeviceID: "install-abc",
|
||||
transcriptionStatus: .done,
|
||||
transcriptionLocaleID: "ru_RU",
|
||||
transcribedAt: Date(timeIntervalSince1970: 1_700_000_100)
|
||||
)
|
||||
let note = Note.create(text: "привет", source: .transcribed, tags: ["voice"], context: context, audio: audio)
|
||||
let data = try DocumentCoder.encode(note)
|
||||
let decoded = try DocumentCoder.decode(Note.self, from: data)
|
||||
#expect(decoded.payload == note.payload)
|
||||
#expect(decoded.payload.audio?.transcriptionStatus == .done)
|
||||
#expect(decoded.payload.audio?.transcriptionLocaleID == "ru_RU")
|
||||
}
|
||||
|
||||
@Test func audioPathDerivationSwapsExtension() {
|
||||
let id = ULID()
|
||||
let jsonPath = Note.relativePath(forID: id)
|
||||
#expect(jsonPath.hasSuffix(".json"))
|
||||
#expect(Note.audioRelativePath(forID: id) == jsonPath.replacingOccurrences(of: ".json", with: ".m4a"))
|
||||
#expect(Note.audioRelativePath(forID: id).hasSuffix("\(id.stringValue).m4a"))
|
||||
}
|
||||
|
||||
/// The swap must operate on the note's actual path, so a record filed under
|
||||
/// a pre-fix local-time-zone month keeps its sidecar in the same bucket.
|
||||
@Test func audioPathPreservesLegacyBucket() {
|
||||
let id = ULID()
|
||||
let legacyJSONPath = "1999/12/\(id.stringValue).json"
|
||||
#expect(Note.audioRelativePath(forJSONPath: legacyJSONPath) == "1999/12/\(id.stringValue).m4a")
|
||||
}
|
||||
|
||||
@Test func createIngestedDerivesCreatedAtFromULID() {
|
||||
let id = ULID(millisecondsSinceEpoch: 1_700_000_000_000, randomHi: 5, randomLo: 6)
|
||||
var context = CaptureContext.empty
|
||||
context.device = "Watch"
|
||||
let note = Note.createIngested(id: id, duration: 8, transcriberDeviceID: "install-x", context: context)
|
||||
#expect(note.meta.id == id)
|
||||
#expect(note.meta.createdAt == id.timestamp)
|
||||
#expect(note.meta.modifiedAt == id.timestamp)
|
||||
#expect(note.payload.text.isEmpty)
|
||||
#expect(note.payload.source == .transcribed)
|
||||
#expect(note.payload.audio?.transcriptionStatus == .pending)
|
||||
#expect(note.payload.audio?.transcriberDeviceID == "install-x")
|
||||
#expect(note.relativePath == Note.relativePath(forID: id))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,4 +54,69 @@ struct NoteMapperTests {
|
||||
#expect(restored.payload == note.payload)
|
||||
#expect(restored.meta.id == note.meta.id)
|
||||
}
|
||||
|
||||
// MARK: - Audio columns
|
||||
|
||||
@Test func pendingAudioColumnsRoundTrip() throws {
|
||||
let (container, context) = try makeStore()
|
||||
defer { withExtendedLifetime(container) {} }
|
||||
var ctx = CaptureContext.empty
|
||||
ctx.device = "Watch"
|
||||
let audio = Note.AudioInfo(
|
||||
duration: 30,
|
||||
transcriberDeviceID: "install-42",
|
||||
transcriptionStatus: .pending,
|
||||
transcriptionLocaleID: nil,
|
||||
transcribedAt: nil
|
||||
)
|
||||
let note = Note.create(text: "", source: .transcribed, context: ctx, audio: audio)
|
||||
NoteMapper.upsert(note, relativePath: note.relativePath, into: context)
|
||||
|
||||
let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context))
|
||||
#expect(entity.hasAudio)
|
||||
#expect(entity.audioDuration == 30)
|
||||
#expect(entity.transcriberDeviceID == "install-42")
|
||||
#expect(entity.transcriptionStatusRaw == "pending")
|
||||
#expect(entity.source == .transcribed)
|
||||
|
||||
// A pending audio note carries no `transcribedAt`, so it reconstructs
|
||||
// losslessly (that field is the only one the cache intentionally drops).
|
||||
let restored = try #require(NoteMapper.note(from: entity))
|
||||
#expect(restored.payload == note.payload)
|
||||
#expect(restored.payload.audio == audio)
|
||||
}
|
||||
|
||||
@Test func doneTranscriptionCachesColumnsButNotTranscribedAt() throws {
|
||||
let (container, context) = try makeStore()
|
||||
defer { withExtendedLifetime(container) {} }
|
||||
let audio = Note.AudioInfo(
|
||||
duration: 5,
|
||||
transcriberDeviceID: "install-7",
|
||||
transcriptionStatus: .done,
|
||||
transcriptionLocaleID: "en_US",
|
||||
transcribedAt: Date()
|
||||
)
|
||||
let note = Note.create(text: "Hello there", source: .transcribed, context: .empty, audio: audio)
|
||||
NoteMapper.upsert(note, relativePath: note.relativePath, into: context)
|
||||
|
||||
let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context))
|
||||
#expect(entity.transcriptionStatusRaw == "done")
|
||||
#expect(entity.transcriptionLocaleID == "en_US")
|
||||
// The file is authoritative for the timestamp — the cache doesn't store it.
|
||||
#expect(entity.audio?.transcribedAt == nil)
|
||||
#expect(entity.audio?.transcriptionStatus == .done)
|
||||
}
|
||||
|
||||
@Test func textNoteHasNoAudioColumns() throws {
|
||||
let (container, context) = try makeStore()
|
||||
defer { withExtendedLifetime(container) {} }
|
||||
let note = makeNote()
|
||||
NoteMapper.upsert(note, relativePath: note.relativePath, into: context)
|
||||
|
||||
let entity = try #require(NoteMapper.fetch(id: note.meta.id.stringValue, in: context))
|
||||
#expect(!entity.hasAudio)
|
||||
#expect(entity.audio == nil)
|
||||
#expect(entity.audioDuration == nil)
|
||||
#expect(entity.transcriptionStatusRaw == nil)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user