Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 31 additions & 11 deletions Sources/LumeEngineCore/Decode/DolbyVisionMapping.swift
Original file line number Diff line number Diff line change
Expand Up @@ -169,22 +169,41 @@ extension DolbyVisionMapping {

/// Row-major 3×3 helpers. Tiny and allocation-happy by design: they run once
/// per frame on nine numbers.
///
/// Written as plain loops with every type spelled out. The closure-and-literal
/// form (`reduce(0) { $0 + a[…] * b[…] }` inside a `map`) type-checks
/// instantly on Swift 6.4 but exceeds Swift 6.2's solver limit ("unable to
/// type-check this expression in reasonable time") — which is what CI's
/// Xcode 26.3 runs.
enum Matrix3 {
static func multiply(_ a: [Double], _ b: [Double]) -> [Double] {
(0..<9).map { index in
let row = index / 3, column = index % 3
return (0..<3).reduce(0) { $0 + a[row * 3 + $1] * b[$1 * 3 + column] }
var product = [Double](repeating: 0, count: 9)
for row in 0..<3 {
for column in 0..<3 {
var sum: Double = 0
for k in 0..<3 {
sum += a[row * 3 + k] * b[k * 3 + column]
}
product[row * 3 + column] = sum
}
}
return product
}

static func inverse(_ m: [Double]) -> [Double] {
let cofactors = [
m[4] * m[8] - m[5] * m[7], m[2] * m[7] - m[1] * m[8], m[1] * m[5] - m[2] * m[4],
m[5] * m[6] - m[3] * m[8], m[0] * m[8] - m[2] * m[6], m[2] * m[3] - m[0] * m[5],
m[3] * m[7] - m[4] * m[6], m[1] * m[6] - m[0] * m[7], m[0] * m[4] - m[1] * m[3],
// Cofactor (i, j) of the transpose, i.e. the adjugate, row-major.
func minor(_ a: Int, _ b: Int, _ c: Int, _ d: Int) -> Double {
let lhs: Double = m[a] * m[b]
let rhs: Double = m[c] * m[d]
return lhs - rhs
}
let adjugate: [Double] = [
minor(4, 8, 5, 7), minor(2, 7, 1, 8), minor(1, 5, 2, 4),
minor(5, 6, 3, 8), minor(0, 8, 2, 6), minor(2, 3, 0, 5),
minor(3, 7, 4, 6), minor(1, 6, 0, 7), minor(0, 4, 1, 3),
]
let determinant = m[0] * cofactors[0] + m[1] * cofactors[3] + m[2] * cofactors[6]
return cofactors.map { $0 / determinant }
let determinant: Double = m[0] * adjugate[0] + m[1] * adjugate[3] + m[2] * adjugate[6]
return adjugate.map { (value: Double) -> Double in value / determinant }
}

/// Linear LMS (Hunt-Pointer-Estevez, no crosstalk) to linear BT.2020 RGB.
Expand All @@ -194,8 +213,9 @@ enum Matrix3 {
/// `M = X(0.04) · HPE`, where `X(c)` has `1 − 2c` on the diagonal and `c`
/// elsewhere. So `HPE⁻¹ = M⁻¹ · X(0.04)`.
static let hpeLMSToBT2020: [Double] = {
let ictcp = [1688.0, 2146, 262, 683, 2951, 462, 99, 309, 3688].map { $0 / 4096 }
let crosstalk = [0.92, 0.04, 0.04, 0.04, 0.92, 0.04, 0.04, 0.04, 0.92]
let ictcpCodes: [Double] = [1688, 2146, 262, 683, 2951, 462, 99, 309, 3688]
let ictcp = ictcpCodes.map { (code: Double) -> Double in code / 4096 }
let crosstalk: [Double] = [0.92, 0.04, 0.04, 0.04, 0.92, 0.04, 0.04, 0.04, 0.92]
return multiply(inverse(ictcp), crosstalk)
}()
}
20 changes: 16 additions & 4 deletions Sources/LumeEngineCore/Subtitle/SubtitleCue.swift
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ public struct SubtitleCue: Sendable, Equatable, Identifiable {

/// Time-indexed cue container, safe for concurrent insert (decoder thread) and
/// query (UI tick). Cues are kept sorted by start time.
///
/// Cues outlive seeks on purpose — the store is a timeline, not a queue — so
/// the same packet reaches it again whenever the demuxer re-reads a range it
/// already delivered: a backward seek, or the seek that backfills a subtitle
/// track selected after the demuxer read past its packets. An identical cue
/// (same start, end, and text) is therefore dropped, or every re-read would
/// show its lines doubled.
public final class SubtitleStore: @unchecked Sendable {
private let lock = NSLock()
private var cues: [SubtitleCue] = []
Expand All @@ -32,12 +39,17 @@ public final class SubtitleStore: @unchecked Sendable {
guard !trimmed.isEmpty, end > start else { return }
lock.lock()
defer { lock.unlock() }
let cue = SubtitleCue(id: nextID, start: start, end: end, text: trimmed)
nextID += 1
// Insertion point by start time (cues arrive nearly sorted).
var index = cues.count
while index > 0 && cues[index - 1].start > cue.start { index -= 1 }
cues.insert(cue, at: index)
while index > 0 && cues[index - 1].start > start { index -= 1 }
// Cues with the same start sit directly before the insertion point.
var existing = index - 1
while existing >= 0 && cues[existing].start == start {
if cues[existing].end == end && cues[existing].text == trimmed { return }
existing -= 1
}
cues.insert(SubtitleCue(id: nextID, start: start, end: end, text: trimmed), at: index)
nextID += 1
}

/// All cues covering `time` (engine µs).
Expand Down
29 changes: 23 additions & 6 deletions Tests/LumeEngineCoreTests/DecoderTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,10 @@ struct DecoderTests {
}
var failure: EngineError?
var reachedEndOfStream = false
var downgraded = false
while let event = await decodeEvents.next() {
if case .failed(let error) = event { failure = error; break }
if case .downgradedToSoftware = event { downgraded = true }
if case .endOfStream = event { reachedEndOfStream = true; break }
}
let hardwareActive = decoder.isHardwareActive
Expand All @@ -153,17 +155,32 @@ struct DecoderTests {
#expect(failure == nil, "AV1 must decode on this host: \(failure.map(String.init(describing:)) ?? "")")
#expect(reachedEndOfStream)
let collected = await collector.value
#expect(collected.count >= 90, "4 s @ 24 fps ≈ 96 frames, got \(collected.count)")
#expect(collected.allSatisfy { $0.width == 320 && $0.height == 180 })
let ptsValues = collected.map(\.pts)
#expect(zip(ptsValues, ptsValues.dropFirst()).allSatisfy { $0 < $1 }, "output PTS must be monotonic")

// Hardware exactly where the chip has an AV1 decoder, software
// (dav1d) everywhere else — never the hardware-only native decoder
// on a chip that cannot run it.
let chipDecodesAV1 = VTIsHardwareDecodeSupported(kCMVideoCodecType_AV1)
#expect(hardwareActive == chipDecodesAV1)
#expect(Set(collected.map(\.hardware)) == [chipDecodesAV1])
if !chipDecodesAV1 {
// The case that played black: no AV1 hardware (Apple TV 4K, M1,
// …). dav1d from the first frame — no failed hardware attempt,
// nothing lost.
#expect(!downgraded, "without AV1 hardware the hardware path must not even be tried")
#expect(!hardwareActive)
#expect(collected.allSatisfy { !$0.hardware })
#expect(collected.count >= 90, "4 s @ 24 fps ≈ 96 frames, got \(collected.count)")
} else if downgraded {
// VideoToolbox claimed AV1 but failed — CI's virtualized macOS
// does exactly that. The engine's ordinary recovery: rebuild on
// dav1d, report it, resume at the next keyframe (one GOP, 24
// frames, of the fixture).
#expect(!hardwareActive)
let afterOneLostGOP: Int = 66 // 96 frames − one 24-frame GOP − slack
#expect(collected.count >= afterOneLostGOP, "at most one GOP lost to the recovery, got \(collected.count)")
} else {
#expect(hardwareActive)
#expect(collected.allSatisfy { $0.hardware })
#expect(collected.count >= 90, "4 s @ 24 fps ≈ 96 frames, got \(collected.count)")
}
// 10-bit stays 10-bit on either path.
#expect(Set(collected.map(\.format)).isSubset(of: [
kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange,
Expand Down
54 changes: 41 additions & 13 deletions Tests/LumeEngineCoreTests/DolbyVisionTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,23 @@ struct DolbyVisionTests {
#expect(parsed.baseLayerBitDepth == 10)
#expect(parsed.curves.count == 3)

#expect(parsed.curves[0].pivots == [0, 400.0 / 1023, 1])
#expect(parsed.curves[0].pieces == [
// Expected values are typed up front rather than written inline in
// `#expect`: comparing against an untyped literal of enum cases took
// Swift 6.4 over two seconds to type-check, beyond what Swift 6.2
// (CI's Xcode 26.3) accepts at all.
let expectedPivots: [Double] = [0, 400.0 / 1023, 1]
let expectedI: [DolbyVisionMapping.Piece] = [
.polynomial([-612_867.0 / 8_388_608, 9_803_467.0 / 8_388_608, 0]),
.polynomial([1, 0, -0.25]),
])
#expect(parsed.curves[1].pieces == [
.mmr(constant: -0.125, coefficients: [[1, 2, 3, 4, 5, 6, 7], [-1, -2, -3, -4, -5, -6, -7]]),
])
#expect(parsed.curves[2].pieces == [.polynomial([0, 1, 0])])
]
let mmrRows: [[Double]] = [[1, 2, 3, 4, 5, 6, 7], [-1, -2, -3, -4, -5, -6, -7]]
let expectedP: [DolbyVisionMapping.Piece] = [.mmr(constant: -0.125, coefficients: mmrRows)]
let expectedT: [DolbyVisionMapping.Piece] = [.polynomial([0, 1, 0])]

#expect(parsed.curves[0].pivots == expectedPivots)
#expect(parsed.curves[0].pieces == expectedI)
#expect(parsed.curves[1].pieces == expectedP)
#expect(parsed.curves[2].pieces == expectedT)

#expect(parsed.yccToRGB[0] == 1)
#expect(parsed.yccToRGB[1] == 799.0 / 8192)
Expand Down Expand Up @@ -446,11 +454,15 @@ private struct Reference {
out = c[0] + c[1] * s + c[2] * s * s
case .mmr(let constant, let rows):
let (y, u, v) = (sig[0], sig[1], sig[2])
let terms = [y, u, v, y * u, y * v, u * v, y * u * v]
out = rows.enumerated().reduce(constant) { total, row in
let order = Double(row.offset + 1)
return total + zip(row.element, terms).reduce(0) { $0 + $1.0 * pow($1.1, order) }
let terms: [Double] = [y, u, v, y * u, y * v, u * v, y * u * v]
var total: Double = constant
for (row, coefficients) in rows.enumerated() {
let order = Double(row + 1)
for (coefficient, term) in zip(coefficients, terms) {
total += coefficient * pow(term, order)
}
}
out = total
}
return min(max(out, 0), 1)
}
Expand All @@ -472,12 +484,28 @@ private struct Reference {
0.01736321, -0.04725154, 1.03004253,
]

// Loops with explicit types, not `map { reduce(0) { … } }`: the closure
// form is beyond Swift 6.2's type checker (CI's Xcode 26.3).
private static func apply(_ m: [Double], _ v: [Double]) -> [Double] {
(0..<3).map { row in (0..<3).reduce(0) { $0 + m[row * 3 + $1] * v[$1] } }
var result = [Double](repeating: 0, count: 3)
for row in 0..<3 {
var sum: Double = 0
for k in 0..<3 { sum += m[row * 3 + k] * v[k] }
result[row] = sum
}
return result
}

private static func multiply(_ a: [Double], _ b: [Double]) -> [Double] {
(0..<9).map { index in (0..<3).reduce(0) { $0 + a[index / 3 * 3 + $1] * b[$1 * 3 + index % 3] } }
var result = [Double](repeating: 0, count: 9)
for row in 0..<3 {
for column in 0..<3 {
var sum: Double = 0
for k in 0..<3 { sum += a[row * 3 + k] * b[k * 3 + column] }
result[row * 3 + column] = sum
}
}
return result
}

private static let m1 = 2610.0 / 16384, m2 = 2523.0 / 4096 * 128
Expand Down
28 changes: 20 additions & 8 deletions Tests/LumeEngineCoreTests/HEVCBaseLayerFilterTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -87,23 +87,35 @@ struct HEVCBaseLayerFilterTests {
@Test("hvcC: enhancement parameter sets and the arrays they empty are removed")
func hvcC() {
func array(_ type: UInt8, _ units: [[UInt8]]) -> [UInt8] {
[0x80 | type, UInt8(units.count >> 8), UInt8(units.count & 0xFF)]
+ units.flatMap { [UInt8($0.count >> 8), UInt8($0.count & 0xFF)] + $0 }
var bytes: [UInt8] = [0x80 | type, UInt8(units.count >> 8), UInt8(units.count & 0xFF)]
for unit in units {
bytes.append(UInt8(unit.count >> 8))
bytes.append(UInt8(unit.count & 0xFF))
bytes.append(contentsOf: unit)
}
return bytes
}
let header = [UInt8](repeating: 0xEE, count: 22)
let vps = Self.nal(type: 32, layer: 0), sps = Self.nal(type: 33, layer: 0)
let pps = Self.nal(type: 34, layer: 0), sei = Self.nal(type: 39, layer: 0)
let sps1 = Self.nal(type: 33, layer: 1), pps1 = Self.nal(type: 34, layer: 1)

let merged = header + [6]
+ array(32, [vps]) + array(33, [sps]) + array(34, [pps]) + array(39, [sei])
+ array(33, [sps1]) + array(34, [pps1])
let expected = header + [4] + array(32, [vps]) + array(33, [sps]) + array(34, [pps]) + array(39, [sei])
// Joined through a typed variadic, not `a + [6] + b + …`: a long `+`
// chain over untyped literals is beyond Swift 6.2's type checker.
func bytes(_ parts: [UInt8]...) -> [UInt8] { Array(parts.joined()) }

let merged = bytes(
header, [6],
array(32, [vps]), array(33, [sps]), array(34, [pps]), array(39, [sei]),
array(33, [sps1]), array(34, [pps1])
)
let expected = bytes(header, [4], array(32, [vps]), array(33, [sps]), array(34, [pps]), array(39, [sei]))
merged.withUnsafeBytes { #expect(HEVCBaseLayerFilter.baseLayerHVCC($0) == expected) }

// Mixed within one array: the array stays, with a corrected count.
let shared = header + [1] + array(33, [sps, sps1])
shared.withUnsafeBytes { #expect(HEVCBaseLayerFilter.baseLayerHVCC($0) == header + [1] + array(33, [sps])) }
let shared = bytes(header, [1], array(33, [sps, sps1]))
let sharedExpected = bytes(header, [1], array(33, [sps]))
shared.withUnsafeBytes { #expect(HEVCBaseLayerFilter.baseLayerHVCC($0) == sharedExpected) }

expected.withUnsafeBytes { #expect(HEVCBaseLayerFilter.baseLayerHVCC($0) == nil, "single-layer hvcC stays as is") }
Array(merged.dropLast()).withUnsafeBytes { #expect(HEVCBaseLayerFilter.baseLayerHVCC($0) == nil, "truncated: untouched") }
Expand Down
44 changes: 44 additions & 0 deletions Tests/LumeEngineCoreTests/SubtitleTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,25 @@ struct SubtitleParserTests {
#expect(Set(store.activeCues(at: 4_500_000).map(\.text)) == ["overlap", "two"])
#expect(store.activeCues(at: 6_500_000).isEmpty)
}

/// Re-reading a range — backward seek, or the backfill seek of a track
/// selected late — delivers the same packets again. Stored twice, every
/// line of that range showed doubled.
@Test("store drops a cue it already holds, and only that")
func storeDeduplicates() {
let store = SubtitleStore()
store.insert(start: 1_000_000, end: 3_000_000, text: "one")
store.insert(start: 4_000_000, end: 6_000_000, text: "two")
store.insert(start: 1_000_000, end: 3_000_000, text: "one")
store.insert(start: 1_000_000, end: 3_000_000, text: " one\n") // trims to the same cue
#expect(store.count == 2)
#expect(store.activeCues(at: 2_000_000).map(\.text) == ["one"])

// Same start, but a different cue: both stay.
store.insert(start: 1_000_000, end: 3_000_000, text: "other speaker")
store.insert(start: 1_000_000, end: 2_000_000, text: "one")
#expect(store.count == 4)
}
}

@Suite("Tracks & subtitles", .serialized)
Expand Down Expand Up @@ -89,6 +108,31 @@ struct TrackSubtitleTests {
await session.shutdown()
}

/// A backward seek re-reads subtitle packets the store already holds. Before
/// the store deduplicated, every re-read range showed its lines twice — and
/// `embeddedSubtitles` flaked whenever its backfill seek landed after live
/// delivery had already stored the cues.
@Test("seeking back does not duplicate cues", .timeLimit(.minutes(1)))
func seekBackKeepsCuesUnique() async throws {
let session = makeSession()
let info = try await session.open(url: try Fixtures.path("multitrack.mkv"))
let subtitleTrack = try #require(info.subtitleTracks.first)

await session.selectSubtitleTrack(subtitleTrack.index)
await session.play()
let sawCues = await eventually { session.subtitles.count >= 2 }
#expect(sawCues, "both cues should decode first, store has \(session.subtitles.count)")

await session.seek(to: 0)
// The 8 s fixture is re-read within milliseconds of the seek; give the
// subtitle lane ample time to decode it all a second time.
let grew = await eventually(timeout: 2) { session.subtitles.count > 2 }
#expect(!grew, "re-read cues were stored again: \(session.subtitles.count) cues")
#expect(session.subtitles.activeCues(at: info.startTime + 2_000_000).map(\.text) == ["Hello from LumeEngine"])

await session.shutdown()
}

@Test("subtitles selected before play backfill cues the demuxer already read", .timeLimit(.minutes(1)))
func subtitlesSelectedBeforePlayBackfill() async throws {
// multitrack.mkv is 8 s and the default read-ahead is 15 s, so open()
Expand Down
7 changes: 5 additions & 2 deletions Tests/LumeEngineCoreTests/Support/ChannelDrain.swift
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,12 @@ final class ChannelDrain<Output: Sendable>: @unchecked Sendable {
get async {
let result: Result = await withCheckedContinuation { continuation in
lock.lock()
if let result {
// `self.` is load-bearing: inside the initializer of the local
// `result`, Swift 6.2 (CI's Xcode 26.3) binds a bare `result`
// to that local, not to the stored property, and rejects it.
if let stored = self.result {
lock.unlock()
continuation.resume(returning: result)
continuation.resume(returning: stored)
} else {
waiter = continuation
lock.unlock()
Expand Down
Loading