diff --git a/AGENTS.md b/AGENTS.md index 44c312e..0fae7fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,6 +117,17 @@ checks the output against a context pixelforge created itself. It needs no ffmpeg. Anything that reaches a core 1.2 or 1.3 entry point panics there, so run it after touching submission or barrier code. +The RGB input path (`EncodeConfig::with_rgb_input`) needs +`VK_VALVE_video_encode_rgb_conversion`, which today only RADV has. +`tests/rgb_encode.rs` compares it against the colour converter, checks that +`supports_rgb_encode_input` agrees with encoder creation, and skips what the +device cannot do, so a green run elsewhere says little about it. Under +validation it draws two messages that are the layer's, not pixelforge's: the +layer does not model that extension, so it rejects the RGB profile struct in a +query pool's profile chain (the spec requires that chain to match the +session's) and checks the RGB input view against the plain format features +rather than the video profile's. + Make sure there are no Vulkan validation layer errors during execution. Enable them with `PIXELFORGE_VALIDATION=1`; the layer's messages are routed through `tracing`, so pair it with `RUST_LOG=warn` (or `debug` for the layer's own diff --git a/CHANGELOG.md b/CHANGELOG.md index 637102d..8f8166e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add RGB input converted by the encoder itself through + `VK_VALVE_video_encode_rgb_conversion`: `EncodeConfig::with_rgb_input`. +- Add `VideoContext::supports_rgb_encode_input` and + `ColorConverterConfig::rgb_encode_input`, which say when a conversion can be + left to the encoder. +- Add `VideoContext::has_video_encode_rgb_conversion` and + `DeviceFeatures::video_encode_rgb_conversion`. + ## [0.10.0] - 19-10-2026 ### Added diff --git a/README.md b/README.md index 448b44f..985d41a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ H.265 and AV1 encode, and H.264 decode. - **Asynchronous pipelines**: both directions submit without waiting. Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a [`DecodeSource`] as the GPU finishes with them. -- **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ). +- **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ), + or in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` is available. - **Shared devices**: encode and decode on a Vulkan device your application created, from Vulkan 1.1 up, on queues you choose, ordered against your own work with timeline semaphores. @@ -250,6 +251,23 @@ let mut encoder = Encoder::new(context, encode_config)?; // converter.convert(input_image, layout, encoder.input_image())?; ``` +Where the conversion is nothing but the YUV matrix, as for ordinary SDR +content, some encoders can do it themselves and the shader need not run at +all. [`ColorConverterConfig::rgb_encode_input`] says whether that works for +a given conversion and codec on this device. When it does, build the +encoder with [`EncodeConfig::with_rgb_input`] and hand it the RGB frames. + +```rust +let encode_config = EncodeConfig::h264(1920, 1080) + .with_color_description(config.color_description().expect("SDR is encodable")); +let encoder = match config.rgb_encode_input(&context, Codec::H264) { + // The encoder converts: give it the RGB images as they are. + Some(format) => Encoder::new(context, encode_config.with_rgb_input(format))?, + // The shader converts into the encoder's input image, as above. + None => Encoder::new(context, encode_config)?, +}; +``` + ### Encoding on your own device An application that already renders with Vulkan can give pixelforge its own diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 15eadaa..b543400 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -8,7 +8,7 @@ mod pipeline; -use crate::encoder::ColorDescription; +use crate::encoder::{Codec, ColorDescription}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::TimelineChain; @@ -325,6 +325,38 @@ impl ColorConverterConfig { ) } + /// The RGB input to give a `codec` encoder instead of running this + /// conversion, when the encoder can do it itself. + /// + /// The hardware applies only the YUV matrix, so this is `Some` only for a + /// conversion that is nothing more: source and target are the same space, + /// so no transfer function or gamut change is involved, and the output is + /// 4:2:0 at the input's bit depth, NV12 from 8-bit RGB or P010 from + /// `ABGR2101010`. The device is then asked, with + /// [`VideoContext::supports_rgb_encode_input`], whether it does that + /// matrix and range for `codec` and takes this format. + /// + /// When `Some`, build the encoder with + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// and this format, and with [`Self::color_description`], then hand it the + /// RGB images directly and skip the converter. + pub fn rgb_encode_input(&self, context: &VideoContext, codec: Codec) -> Option { + if self.source != self.target { + return None; + } + let shape_matches = match self.output_format { + OutputFormat::NV12 => matches!( + self.input_format, + InputFormat::BGRx | InputFormat::BGRA | InputFormat::RGBx | InputFormat::RGBA + ), + OutputFormat::P010 => self.input_format == InputFormat::ABGR2101010, + _ => false, + }; + let desc = self.color_description()?; + (shape_matches && context.supports_rgb_encode_input(codec, self.input_format, &desc)) + .then_some(self.input_format) + } + /// Whether the shader can get from this source to this target. /// /// Anything that would need tone mapping or a forward gamma encode is not diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index d988af5..adc0097 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -4,6 +4,7 @@ use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; use crate::encoder::profile::av1_profile; +use crate::encoder::rgb::{RgbConversionCaps, rgb_conversion_profile}; use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -44,13 +45,17 @@ impl Av1 { .tuning_mode(encoder_tuning_mode); let mut av1_profile_info = vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(profile); - let profile_info = vk::VideoProfileInfoKHR::default() + let mut rgb_profile = rgb_conversion_profile(); + let mut profile_info = vk::VideoProfileInfoKHR::default() .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_AV1) .chroma_subsampling(chroma_subsampling) .luma_bit_depth(bit_depth) .chroma_bit_depth(bit_depth) .push(&mut av1_profile_info) .push(&mut video_encode_usage_info); + if config.rgb_input.is_some() { + profile_info = profile_info.push(&mut rgb_profile); + } let bitstream_buffer_size = MIN_BITSTREAM_BUFFER_SIZE .max(config.dimensions.width as usize * config.dimensions.height as usize); @@ -62,7 +67,12 @@ impl Av1 { let mut capabilities = vk::VideoCapabilitiesKHR::default() .push(&mut encode_caps) .push(&mut av1_caps); + let mut rgb_caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); + if config.rgb_input.is_some() { + capabilities = capabilities.push(&mut rgb_caps); + } let caps = query_video_caps(&context, &profile_info, &mut capabilities)?; + let rgb_caps = config.rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); let init = build_encoder_common(&CommonInitRequest { context: &context, @@ -81,6 +91,7 @@ impl Av1 { // when the driver does not support separate reference images // (required for AMD RADV). allow_layered_dpb: true, + rgb_caps, })?; let active_reference_count = init.active_reference_count; let common = init.common; diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 32444f6..95065c0 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -13,6 +13,7 @@ //! scaffolding is here. use ash::vk; +use ash::vk::TaggedStructure; use crate::encoder::dpb::MAX_DPB_SLOTS; use crate::encoder::gop::{GopFrameType, GopPosition, GopStructure}; @@ -22,6 +23,7 @@ use crate::encoder::resources::{ create_dpb_images, destroy_encoder_resources, get_video_format, lcm, query_supported_video_formats, upload_image_to_input, }; +use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControlMode}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; @@ -115,6 +117,7 @@ impl EncoderCommon { height: self.config.dimensions.height, pixel_format: self.config.pixel_format, input_image_layout, + rgb: self.config.rgb_input.is_some(), upload_queue: self.context.transfer_queue(), waits: &waits, }; @@ -447,6 +450,10 @@ pub(crate) struct CommonInitRequest<'a> { /// Whether the codec can use a layered DPB image when the driver lacks /// `SEPARATE_REFERENCE_IMAGES` (H.264/H.265 yes; AV1 no). pub allow_layered_dpb: bool, + /// What the hardware RGB conversion supports for this profile, queried + /// alongside the codec capabilities when [`EncodeConfig::rgb_input`] is + /// set. + pub rgb_caps: Option, } /// Result of [`build_encoder_common`]: the assembled common state plus the @@ -574,6 +581,27 @@ pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result { + let caps = req.rgb_caps.ok_or_else(|| { + PixelForgeError::NoSuitableDevice( + "RGB input: the driver reported no conversion capabilities".to_string(), + ) + })?; + Some( + caps.session_info( + &config + .color_description + .unwrap_or_else(ColorDescription::bt709), + )?, + ) + } + None => None, + }; + // Pick input (SRC) and reference (DPB) formats. let preferred_src_format = get_video_format(config.pixel_format, config.bit_depth); let supported_src_formats = query_supported_video_formats( @@ -596,20 +624,22 @@ pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result Result Result Result, + /// Take RGB input and let the encoder convert it to YUV itself, through + /// `VK_VALVE_video_encode_rgb_conversion`. See [`Self::with_rgb_input`]. + pub rgb_input: Option, /// Usage hint for encoding. pub encode_usage_hint: EncodeUsageHint, /// Content hint for encoding. @@ -375,6 +379,7 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, + rgb_input: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -403,6 +408,7 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, + rgb_input: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -431,6 +437,7 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, + rgb_input: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -519,6 +526,27 @@ impl EncodeConfig { self } + /// Take `format` RGB images as input and convert them to YUV in the + /// encoder itself, with no colour conversion shader. + /// + /// Needs `VK_VALVE_video_encode_rgb_conversion` (see + /// [`VideoContext::has_video_encode_rgb_conversion`]), and a driver that + /// accepts `format` as encode input for this codec, profile and bit depth; + /// [`Encoder::new`] fails otherwise. The YUV matrix and range come from + /// [`Self::color_description`], so set that to what the stream should + /// carry. The hardware applies only the matrix: a source that needs a + /// transfer function or gamut change still has to go through + /// [`ColorConverter`](crate::ColorConverter), which + /// [`ColorConverterConfig::rgb_encode_input`](crate::ColorConverterConfig::rgb_encode_input) + /// decides for you. + /// + /// With this set, [`Encoder::input_image`] is an RGB image, and + /// [`Encoder::encode`] copies an RGB source into it. + pub fn with_rgb_input(mut self, format: crate::converter::InputFormat) -> Self { + self.rgb_input = Some(format); + self + } + /// Set the usage hint for encoding. pub fn with_encode_usage_hint(mut self, hint: EncodeUsageHint) -> Self { self.encode_usage_hint = hint; @@ -620,6 +648,9 @@ pub struct Encoder(Box); impl Encoder { /// Create a new encoder for the codec named in `config`. pub fn new(context: VideoContext, config: EncodeConfig) -> Result { + if let Some(format) = config.rgb_input { + rgb::check_rgb_input(&context, &config, format)?; + } let inner: Box = match config.codec { Codec::H264 => Box::new(self::h264::H264::create(context, config)?), Codec::H265 => Box::new(self::h265::H265::create(context, config)?), diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index d46ef58..4d6ea16 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -165,6 +165,8 @@ pub(crate) struct PipelineConfig<'a> { pub aligned_width: u32, pub aligned_height: u32, pub picture_format: vk::Format, + /// Whether the input image is RGB, for the encoder to convert. + pub rgb_input: bool, pub pixel_format: PixelFormat, pub bit_depth: BitDepth, pub bitstream_buffer_size: usize, @@ -256,6 +258,7 @@ impl EncodePipeline { height: config.aligned_height, pixel_format: config.pixel_format, bit_depth: config.bit_depth, + rgb: config.rgb_input, }, )?; diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index ccf17d5..e5fa4fd 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -205,9 +205,17 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam // YUV 4:4:4 (e.g., NV24): UV plane is full width, full height, 2 components per pixel. PixelFormat::Yuv444 => params.width * params.height * 2 * bytes_per_component, }; - let total_size = (plane0_size + plane1_size) as vk::DeviceSize; + // Every RGB input format is 4 bytes a pixel in one plane. + let rgb_size = params.width * params.height * 4; + let total_size = if params.rgb { + rgb_size + } else { + plane0_size + plane1_size + } as vk::DeviceSize; - // Create a staging buffer filled with zeros. + // Create a staging buffer filled with zeros. A copy rather than a clear + // command, since this runs on the transfer queue and a clear needs a + // graphics or compute one. let buffer_create_info = vk::BufferCreateInfo::default() .size(total_size) .usage(vk::BufferUsageFlags::TRANSFER_SRC) @@ -327,13 +335,25 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam }, ]; + let rgb_region = vk::BufferImageCopy { + image_subresource: vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + ..copy_regions[0].image_subresource + }, + ..copy_regions[0] + }; + let regions: &[vk::BufferImageCopy] = if params.rgb { + std::slice::from_ref(&rgb_region) + } else { + ©_regions + }; unsafe { device.cmd_copy_buffer_to_image( params.command_buffer, staging_buffer, params.image, vk::ImageLayout::TRANSFER_DST_OPTIMAL, - ©_regions, + regions, ); } @@ -408,6 +428,8 @@ pub(crate) struct UploadParams<'a> { pub pixel_format: PixelFormat, /// The current layout of the input image. pub input_image_layout: vk::ImageLayout, + /// Whether both images are single-plane RGB rather than YUV. + pub rgb: bool, /// The queue to submit transfer operations to. pub upload_queue: vk::Queue, /// Caller work the copy must wait for. @@ -544,6 +566,28 @@ pub(crate) fn upload_image_to_input( }, }; + // An RGB input image has one colour plane, copied whole. + let rgb_copy_region = vk::ImageCopy { + src_subresource: vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }, + dst_subresource: vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }, + ..y_copy_region + }; + let regions: &[vk::ImageCopy] = if params.rgb { + &[rgb_copy_region] + } else { + &[y_copy_region, uv_copy_region] + }; + unsafe { device.cmd_copy_image( params.upload_command_buffer, @@ -551,7 +595,7 @@ pub(crate) fn upload_image_to_input( vk::ImageLayout::TRANSFER_SRC_OPTIMAL, params.dst_image, vk::ImageLayout::TRANSFER_DST_OPTIMAL, - &[y_copy_region, uv_copy_region], + regions, ); } @@ -1141,6 +1185,9 @@ pub(crate) struct ClearImageParams { pub height: u32, pub pixel_format: PixelFormat, pub bit_depth: BitDepth, + /// Whether the image is a single-plane, 4-bytes-per-pixel RGB input + /// image rather than a YUV one. + pub rgb: bool, } #[cfg(test)] diff --git a/src/encoder/rgb.rs b/src/encoder/rgb.rs new file mode 100644 index 0000000..fe1910b --- /dev/null +++ b/src/encoder/rgb.rs @@ -0,0 +1,223 @@ +//! RGB input converted to YUV by the encoder itself, through +//! `VK_VALVE_video_encode_rgb_conversion`. +//! +//! The extension adds one struct to the video profile, which then means "this +//! encoder takes RGB pictures". Everything that names the profile has to name +//! that version of it: the capability and format queries, the input and DPB +//! images, the query pools and the session. The session also says which +//! matrix, range and chroma siting to convert with. + +use super::support::report; +use super::{BitDepth, Codec, ColorDescription, EncodeConfig, PixelFormat}; +use crate::converter::InputFormat; +use crate::error::{PixelForgeError, Result}; +use crate::video::query_supported_video_formats; +use crate::vulkan::VideoContext; +use ash::vk; + +/// The profile struct that turns on RGB input. +pub(crate) fn rgb_conversion_profile() -> vk::VideoEncodeProfileRgbConversionInfoVALVE<'static> { + vk::VideoEncodeProfileRgbConversionInfoVALVE::default().perform_encode_rgb_conversion(true) +} + +/// The YUV bit depth an RGB input format converts to, or `None` for a format +/// the hardware conversion does not take. +/// +/// 8-bit RGB becomes 8-bit YUV and 10-bit RGB 10-bit YUV. Half floats carry +/// values outside the 0 to 1 range that need tone mapping first, which only +/// the colour converter does. +pub(crate) fn rgb_input_bit_depth(format: InputFormat) -> Option { + match format { + InputFormat::BGRx | InputFormat::BGRA | InputFormat::RGBx | InputFormat::RGBA => { + Some(BitDepth::Eight) + } + InputFormat::ABGR2101010 => Some(BitDepth::Ten), + InputFormat::RGBA16F => None, + } +} + +/// Refuse an RGB input configuration that cannot work, before any driver is +/// asked about it. +/// +/// The device must have the extension, and the encoder must be set up for +/// the YUV the conversion produces: 4:2:0, at 8 bits from 8-bit RGB and 10 +/// bits from 10-bit RGB. +pub(crate) fn check_rgb_input( + context: &VideoContext, + config: &EncodeConfig, + format: InputFormat, +) -> Result<()> { + if !context.has_video_encode_rgb_conversion() { + return Err(PixelForgeError::NoSuitableDevice( + "RGB input needs VK_VALVE_video_encode_rgb_conversion, which this device does \ + not have enabled" + .to_string(), + )); + } + let Some(bit_depth) = rgb_input_bit_depth(format) else { + return Err(PixelForgeError::InvalidInput(format!( + "RGB input: {format:?} needs tone mapping first, which only the colour \ + converter does" + ))); + }; + if config.pixel_format != PixelFormat::Yuv420 || config.bit_depth != bit_depth { + return Err(PixelForgeError::InvalidInput(format!( + "RGB input: {format:?} converts to {bit_depth:?} 4:2:0, but the encoder is set \ + up for {:?} {:?}", + config.bit_depth, config.pixel_format + ))); + } + Ok(()) +} + +/// What the driver's RGB conversion can do for one profile: +/// `VkVideoEncodeRgbConversionCapabilitiesVALVE` without its `pNext`. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RgbConversionCaps { + models: vk::VideoEncodeRgbModelConversionFlagsVALVE, + ranges: vk::VideoEncodeRgbRangeCompressionFlagsVALVE, + x_chroma_offsets: vk::VideoEncodeRgbChromaOffsetFlagsVALVE, + y_chroma_offsets: vk::VideoEncodeRgbChromaOffsetFlagsVALVE, +} + +impl From<&vk::VideoEncodeRgbConversionCapabilitiesVALVE<'_>> for RgbConversionCaps { + fn from(caps: &vk::VideoEncodeRgbConversionCapabilitiesVALVE<'_>) -> Self { + Self { + models: caps.rgb_models, + ranges: caps.rgb_ranges, + x_chroma_offsets: caps.x_chroma_offsets, + y_chroma_offsets: caps.y_chroma_offsets, + } + } +} + +impl RgbConversionCaps { + /// The session settings that make the hardware produce what `desc` + /// describes, or why it cannot. + pub(crate) fn session_info( + &self, + desc: &ColorDescription, + ) -> Result> { + use vk::VideoEncodeRgbChromaOffsetFlagsVALVE as Offset; + use vk::VideoEncodeRgbModelConversionFlagsVALVE as Model; + use vk::VideoEncodeRgbRangeCompressionFlagsVALVE as Range; + + // The matrix coefficients code the stream carries (H.273 numbering). + let model = match desc.matrix_coefficients { + ColorDescription::MATRIX_BT709 => Model::YCBCR_709, + 5 | 6 => Model::YCBCR_601, + ColorDescription::MATRIX_BT2020_NCL => Model::YCBCR_2020, + other => { + return Err(PixelForgeError::InvalidInput(format!( + "RGB input: no hardware conversion for matrix coefficients {other}" + ))); + } + }; + let range = if desc.full_range { + Range::FULL_RANGE + } else { + Range::NARROW_RANGE + }; + if !self.models.contains(model) || !self.ranges.contains(range) { + return Err(PixelForgeError::NoSuitableDevice(format!( + "RGB input: the driver converts {:?} in {:?}, not {:?} in {:?}", + self.models, self.ranges, model, range + ))); + } + + // Where chroma samples sit relative to luma. The stream does not say, + // and a player that is not told assumes chroma sits on the left luma + // sample of each pair, and midway between the two rows (H.264 and + // H.265 chroma location type 0, which most AV1 players follow as well). + // Asking the hardware for exactly that keeps colour edges where the + // player will put them. A driver that cannot gets the other option. + let pick = |supported: Offset, preferred: Offset| { + if supported.contains(preferred) { + preferred + } else if supported.contains(Offset::COSITED_EVEN) { + Offset::COSITED_EVEN + } else { + Offset::MIDPOINT + } + }; + Ok( + vk::VideoEncodeSessionRgbConversionCreateInfoVALVE::default() + .rgb_model(model) + .rgb_range(range) + .x_chroma_offset(pick(self.x_chroma_offsets, Offset::COSITED_EVEN)) + .y_chroma_offset(pick(self.y_chroma_offsets, Offset::MIDPOINT)), + ) + } +} + +impl VideoContext { + /// Whether an [`Encoder`](crate::Encoder) for `codec` can take `format` + /// RGB input and convert it to the YUV that `desc` describes itself, + /// through `VK_VALVE_video_encode_rgb_conversion`. + /// + /// Asks the driver everything [`Encoder::new`](crate::Encoder::new) will + /// need: that the device has the extension, that the codec's RGB input + /// profile exists, that its conversion does `desc`'s matrix and range, + /// and that it accepts `format` as an input image. The output is always + /// 4:2:0, at 8 bits for 8-bit RGB and 10 bits for + /// [`InputFormat::ABGR2101010`]. Resolution and rate control are not + /// considered. + /// + /// When this is true, build the encoder with + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// and [`EncodeConfig::with_color_description`](crate::EncodeConfig::with_color_description) + /// set to `desc`. [`ColorConverterConfig::rgb_encode_input`](crate::ColorConverterConfig::rgb_encode_input) + /// asks this for a conversion it would otherwise run. + pub fn supports_rgb_encode_input( + &self, + codec: Codec, + format: InputFormat, + desc: &ColorDescription, + ) -> bool { + if !self.has_video_encode_rgb_conversion() || !self.supports_encode(codec) { + return false; + } + let Some(bit_depth) = rgb_input_bit_depth(format) else { + return false; + }; + + let mut caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); + let what = format!("{codec:?} encode from {format:?} RGB"); + let formats = self.query_encode_profile( + codec, + PixelFormat::Yuv420, + bit_depth, + Some(&mut caps), + |profile| { + query_supported_video_formats( + self, + profile, + vk::ImageUsageFlags::VIDEO_ENCODE_SRC_KHR, + ) + }, + ); + let formats = match formats { + Ok(Ok(formats)) => Ok(formats), + Ok(Err(e)) => { + tracing::warn!("Failed to query {what} input formats: {e}"); + return false; + } + Err(e) => Err(e), + }; + let Some(formats) = report(&what, formats) else { + return false; + }; + if !formats.iter().any(|f| f.format == format.vk_format()) { + tracing::debug!( + "{what}: {:?} is not an accepted input format", + format.vk_format() + ); + return false; + } + if let Err(e) = RgbConversionCaps::from(&caps).session_info(desc) { + tracing::debug!("{what}: {e}"); + return false; + } + true + } +} diff --git a/src/encoder/support.rs b/src/encoder/support.rs index d88c0f4..cd5de01 100644 --- a/src/encoder/support.rs +++ b/src/encoder/support.rs @@ -2,12 +2,21 @@ use super::codec::query_video_capabilities; use super::profile::{av1_profile, h264_profile_idc, h265_profile_idc}; +use super::rgb::rgb_conversion_profile; use super::{BitDepth, Codec, PixelFormat}; use crate::vulkan::VideoContext; use ash::vk; use ash::vk::TaggedStructure; use tracing::{debug, warn}; +/// Why [`VideoContext::query_encode_profile`] has no answer to give. +pub(crate) enum ProfileQueryError { + /// The encoder has no profile for this combination (H.265 4:2:2). + NoProfile, + /// The driver rejected the profile's capability query. + Query(vk::Result), +} + impl VideoContext { /// Check whether the device can encode `codec` in the given pixel format and /// bit depth. @@ -26,91 +35,115 @@ impl VideoContext { if !self.supports_encode(codec) { return false; } + let result = self.query_encode_profile(codec, pixel_format, bit_depth, None, |_| ()); + report( + &format!("{codec:?} encode {pixel_format:?} {bit_depth:?}"), + result, + ) + .is_some() + } + /// Build the profile an encoder would request for `codec`, `pixel_format` + /// and `bit_depth`, ask the driver for its capabilities, and on success + /// hand the profile to `then` for any further query about it. + /// + /// With `rgb_caps`, the profile is the RGB input one an encoder built with + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// requests, and the driver fills in what its conversion supports. + /// + /// The encoders build their profiles themselves, since theirs also carry + /// usage hints, but from the same profile numbers in + /// [`super::profile`], so this answers for the profile they use. + pub(crate) fn query_encode_profile( + &self, + codec: Codec, + pixel_format: PixelFormat, + bit_depth: BitDepth, + mut rgb_caps: Option<&mut vk::VideoEncodeRgbConversionCapabilitiesVALVE<'static>>, + then: impl FnOnce(&vk::VideoProfileInfoKHR<'_>) -> R, + ) -> Result { let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = pixel_format.into(); let component_bit_depth: vk::VideoComponentBitDepthFlagsKHR = bit_depth.into(); let mut encode_caps = vk::VideoEncodeCapabilitiesKHR::default(); + let mut rgb_profile = rgb_conversion_profile(); + + let base = vk::VideoProfileInfoKHR::default() + .chroma_subsampling(chroma_subsampling) + .luma_bit_depth(component_bit_depth) + .chroma_bit_depth(component_bit_depth); - // Each arm keeps its profile and capability structs alive across the call. - let result = match codec { - Codec::H264 => { - let mut profile = vk::VideoEncodeH264ProfileInfoKHR::default() - .std_profile_idc(h264_profile_idc(pixel_format)); - let profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_H264) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(component_bit_depth) - .chroma_bit_depth(component_bit_depth) - .push(&mut profile); - let mut codec_caps = vk::VideoEncodeH264CapabilitiesKHR::default(); + // Each arm keeps its profile and capability structs alive across the + // calls that use them. + macro_rules! query { + ($operation:expr, $profile:expr, $codec_caps:expr) => {{ + let mut profile = $profile; + let mut profile_info = base.video_codec_operation($operation).push(&mut profile); + if rgb_caps.is_some() { + profile_info = profile_info.push(&mut rgb_profile); + } + let mut codec_caps = $codec_caps; let mut caps = vk::VideoCapabilitiesKHR::default() .push(&mut encode_caps) .push(&mut codec_caps); - query_video_capabilities(self, &profile_info, &mut caps) - } + if let Some(rgb_caps) = rgb_caps.as_deref_mut() { + caps = caps.push(rgb_caps); + } + match query_video_capabilities(self, &profile_info, &mut caps) { + vk::Result::SUCCESS => Ok(then(&profile_info)), + err => Err(ProfileQueryError::Query(err)), + } + }}; + } + + match codec { + Codec::H264 => query!( + vk::VideoCodecOperationFlagsKHR::ENCODE_H264, + vk::VideoEncodeH264ProfileInfoKHR::default() + .std_profile_idc(h264_profile_idc(pixel_format)), + vk::VideoEncodeH264CapabilitiesKHR::default() + ), Codec::H265 => { - let Some(profile_idc) = h265_profile_idc(pixel_format, bit_depth) else { - debug!( - "{:?} encode {:?} {:?}: no profile", - codec, pixel_format, bit_depth - ); - return false; - }; - let mut profile = - vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(profile_idc); - let profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_H265) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(component_bit_depth) - .chroma_bit_depth(component_bit_depth) - .push(&mut profile); - let mut codec_caps = vk::VideoEncodeH265CapabilitiesKHR::default(); - let mut caps = vk::VideoCapabilitiesKHR::default() - .push(&mut encode_caps) - .push(&mut codec_caps); - query_video_capabilities(self, &profile_info, &mut caps) + let profile_idc = h265_profile_idc(pixel_format, bit_depth) + .ok_or(ProfileQueryError::NoProfile)?; + query!( + vk::VideoCodecOperationFlagsKHR::ENCODE_H265, + vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(profile_idc), + vk::VideoEncodeH265CapabilitiesKHR::default() + ) } - Codec::AV1 => { - let mut profile = vk::VideoEncodeAV1ProfileInfoKHR::default() - .std_profile(av1_profile(pixel_format)); - let profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_AV1) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(component_bit_depth) - .chroma_bit_depth(component_bit_depth) - .push(&mut profile); - let mut codec_caps = vk::VideoEncodeAV1CapabilitiesKHR::default(); - let mut caps = vk::VideoCapabilitiesKHR::default() - .push(&mut encode_caps) - .push(&mut codec_caps); - query_video_capabilities(self, &profile_info, &mut caps) - } - }; + Codec::AV1 => query!( + vk::VideoCodecOperationFlagsKHR::ENCODE_AV1, + vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(av1_profile(pixel_format)), + vk::VideoEncodeAV1CapabilitiesKHR::default() + ), + } + } +} - match result { - vk::Result::SUCCESS => { - debug!( - "{:?} encode {:?} {:?}: supported", - codec, pixel_format, bit_depth - ); - true - } - vk::Result::ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR +/// Log the outcome of a profile query for `what`, and turn it into an +/// `Option`. A profile the driver does not support is ordinary and logged at +/// debug; any other failure is unexpected and logged at warn. +pub(crate) fn report(what: &str, result: Result) -> Option { + match result { + Ok(value) => { + debug!("{what}: supported"); + Some(value) + } + Err(ProfileQueryError::NoProfile) => { + debug!("{what}: no profile"); + None + } + Err(ProfileQueryError::Query( + err @ (vk::Result::ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR | vk::Result::ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR - | vk::Result::ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR => { - debug!( - "{:?} encode {:?} {:?}: not supported ({:?})", - codec, pixel_format, bit_depth, result - ); - false - } - err => { - warn!( - "Failed to query {:?} encode {:?} {:?} capabilities: {:?}", - codec, pixel_format, bit_depth, err - ); - false - } + | vk::Result::ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR), + )) => { + debug!("{what}: not supported ({err:?})"); + None + } + Err(ProfileQueryError::Query(err)) => { + warn!("Failed to query {what} capabilities: {err:?}"); + None } } } diff --git a/src/lib.rs b/src/lib.rs index 5eb9f1c..7294313 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,8 @@ //! - **Asynchronous pipelines**: both directions submit without waiting. //! Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a //! [`DecodeSource`] as the GPU finishes with them. -//! - **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ). +//! - **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ), +//! or in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` is available. //! - **Shared devices**: encode and decode on a Vulkan device your application //! created, from Vulkan 1.1 up, on queues you choose, ordered against your own //! work with timeline semaphores. @@ -247,6 +248,30 @@ //! # } //! ``` //! +//! Where the conversion is nothing but the YUV matrix, as for ordinary SDR +//! content, some encoders can do it themselves and the shader need not run at +//! all. [`ColorConverterConfig::rgb_encode_input`] says whether that works for +//! a given conversion and codec on this device. When it does, build the +//! encoder with [`EncodeConfig::with_rgb_input`] and hand it the RGB frames. +//! +//! ```rust,no_run +//! # use pixelforge::{Codec, ColorConverterConfig, EncodeConfig, Encoder, VideoContext}; +//! # fn pick( +//! # context: VideoContext, +//! # config: ColorConverterConfig, +//! # ) -> Result<(), pixelforge::PixelForgeError> { +//! let encode_config = EncodeConfig::h264(1920, 1080) +//! .with_color_description(config.color_description().expect("SDR is encodable")); +//! let encoder = match config.rgb_encode_input(&context, Codec::H264) { +//! // The encoder converts: give it the RGB images as they are. +//! Some(format) => Encoder::new(context, encode_config.with_rgb_input(format))?, +//! // The shader converts into the encoder's input image, as above. +//! None => Encoder::new(context, encode_config)?, +//! }; +//! # Ok(()) +//! # } +//! ``` +//! //! ## Encoding on your own device //! //! An application that already renders with Vulkan can give pixelforge its own diff --git a/src/vulkan.rs b/src/vulkan.rs index 9d5928f..5a77765 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -356,6 +356,9 @@ pub struct DeviceFeatures { pub ycbcr_2plane_444_formats: bool, /// `videoEncodeAV1`, from `VkPhysicalDeviceVideoEncodeAV1FeaturesKHR`. pub video_encode_av1: bool, + /// `videoEncodeRgbConversion`, from + /// `VkPhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE`. + pub video_encode_rgb_conversion: bool, } /// The queue families pixelforge selects for decoding on a given device. @@ -555,6 +558,15 @@ fn supports_internally_synchronized_queues( features.internally_synchronized_queues != 0 } +/// Whether `physical_device` has the `videoEncodeRgbConversion` feature. The +/// caller checks for the extension first. +fn supports_rgb_conversion(instance: &ash::Instance, physical_device: vk::PhysicalDevice) -> bool { + let mut features = vk::PhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE::default(); + let mut query = vk::PhysicalDeviceFeatures2::default().push(&mut features); + unsafe { instance.get_physical_device_features2(physical_device, &mut query) }; + features.video_encode_rgb_conversion != 0 +} + /// Device extensions required to decode the given codecs. fn decode_extension_names(decode_codecs: &[Codec]) -> Vec<&'static std::ffi::CStr> { // Synchronization2 and timeline semaphores are core in 1.3 and 1.2, but @@ -614,6 +626,7 @@ struct VideoContextInner { supported_encode_codecs: Vec, supported_decode_codecs: Vec, has_push_descriptor: bool, + has_video_encode_rgb_conversion: bool, has_unified_image_layouts: bool, /// Whether this context created (and therefore must destroy) the device and /// instance. A context adopted from a caller's device borrows them and @@ -761,6 +774,14 @@ impl VideoContext { self.inner.has_push_descriptor } + /// Whether `VK_VALVE_video_encode_rgb_conversion` is enabled, extension + /// and feature both: encoders can then take RGB input and convert it + /// themselves, see + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input). + pub fn has_video_encode_rgb_conversion(&self) -> bool { + self.inner.has_video_encode_rgb_conversion + } + /// Whether `VK_KHR_unified_image_layouts` is enabled, with its video bit. /// /// When it is, `VK_IMAGE_LAYOUT_GENERAL` is valid for every use including @@ -1187,6 +1208,23 @@ impl VideoContext { warn!("VK_KHR_push_descriptor not available; the colour converter will be unavailable"); } + // Lets an encoder take RGB input and convert it to YUV itself, so a + // plain matrix conversion needs no shader at all. Extension and feature + // both, or the profile struct is parsed and ignored. + let has_rgb_conversion_ext = video_encode_queue_family.is_some() + && selected_device_exts.as_ref().is_some_and(|exts| { + has_extension(exts, ash::valve::video_encode_rgb_conversion::NAME) + }); + let has_rgb_conversion = + has_rgb_conversion_ext && supports_rgb_conversion(&instance, physical_device); + let mut rgb_conversion_features = + vk::PhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE::default() + .video_encode_rgb_conversion(true); + if has_rgb_conversion { + push_ext(ash::valve::video_encode_rgb_conversion::NAME.as_ptr()); + debug!("Video encode RGB conversion available"); + } + // Enable synchronization2 feature. let mut sync2_features = vk::PhysicalDeviceSynchronization2Features::default().synchronization2(true); @@ -1303,6 +1341,9 @@ impl VideoContext { if has_ycbcr_conversion { device_create_info = device_create_info.push(&mut ycbcr_features); } + if has_rgb_conversion { + device_create_info = device_create_info.push(&mut rgb_conversion_features); + } // Enable YCbCr 2-plane 444 formats feature (required for YUV444 encoding with NVIDIA). // Its extension is enabled above whenever present, so the feature has // to follow it unconditionally: an extension without its feature bit @@ -1353,6 +1394,7 @@ impl VideoContext { supported_encode_codecs, supported_decode_codecs, has_push_descriptor, + has_video_encode_rgb_conversion: has_rgb_conversion, has_unified_image_layouts: has_unified_layouts, owns_device: true, retired_semaphores: Default::default(), @@ -1457,6 +1499,7 @@ impl VideoContext { supported_encode_codecs: Vec::new(), supported_decode_codecs, has_push_descriptor: false, + has_video_encode_rgb_conversion: false, has_unified_image_layouts: declared_unified_image_layouts, owns_device: false, retired_semaphores: Default::default(), diff --git a/src/vulkan/encode_device.rs b/src/vulkan/encode_device.rs index 3534820..c6b462b 100644 --- a/src/vulkan/encode_device.rs +++ b/src/vulkan/encode_device.rs @@ -41,6 +41,7 @@ struct EncodePlan { ycbcr_2plane_444: bool, sampler_ycbcr_conversion: bool, internally_synchronized_queues: bool, + rgb_conversion: bool, } impl EncodePlan { @@ -80,7 +81,7 @@ impl EncodePlan { let extensions = DeviceExtensions::of(instance, physical_device); let has = |name: &std::ffi::CStr| extensions.has(name); - // The feature bits behind two of the extensions. A driver may offer an + // The feature bits behind three of the extensions. A driver may offer an // extension and still report its feature off, and a caller told to // enable a feature the device lacks gets VK_ERROR_FEATURE_NOT_PRESENT // from vkCreateDevice. A struct is chained only when its extension is @@ -88,6 +89,7 @@ impl EncodePlan { let mut ycbcr = vk::PhysicalDeviceSamplerYcbcrConversionFeatures::default(); let mut ycbcr_444 = vk::PhysicalDeviceYcbcr2Plane444FormatsFeaturesEXT::default(); let mut av1 = vk::PhysicalDeviceVideoEncodeAV1FeaturesKHR::default(); + let mut rgb = vk::PhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE::default(); let mut query = vk::PhysicalDeviceFeatures2::default().push(&mut ycbcr); if has(ash::ext::ycbcr_2plane_444_formats::NAME) { query = query.push(&mut ycbcr_444); @@ -95,6 +97,9 @@ impl EncodePlan { if has(ash::khr::video_encode_av1::NAME) { query = query.push(&mut av1); } + if has(ash::valve::video_encode_rgb_conversion::NAME) { + query = query.push(&mut rgb); + } unsafe { instance.get_physical_device_features2(physical_device, &mut query) }; let family = encode.family; @@ -151,6 +156,8 @@ impl EncodePlan { physical_device, &extensions, ), + rgb_conversion: has(ash::valve::video_encode_rgb_conversion::NAME) + && rgb.video_encode_rgb_conversion != 0, }) } @@ -177,6 +184,9 @@ impl EncodePlan { if self.ycbcr_2plane_444 { names.push(ash::ext::ycbcr_2plane_444_formats::NAME); } + if self.rgb_conversion { + names.push(ash::valve::video_encode_rgb_conversion::NAME); + } names } @@ -187,6 +197,7 @@ impl EncodePlan { sampler_ycbcr_conversion: self.sampler_ycbcr_conversion, ycbcr_2plane_444_formats: self.ycbcr_2plane_444, video_encode_av1: self.codecs.contains(&Codec::AV1), + video_encode_rgb_conversion: self.rgb_conversion, } } @@ -213,6 +224,8 @@ impl VideoContextBuilder { /// The colour converter needs `VK_KHR_push_descriptor`, which is included /// whenever the device has it. On a device without it, encoding from YUV /// input still works and only the converter is unavailable. + /// `VK_VALVE_video_encode_rgb_conversion` is included the same way, for + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input). pub fn encode_device_requirements( &self, entry: &ash::Entry, @@ -340,6 +353,7 @@ impl VideoContext { supported_encode_codecs: plan.codecs.clone(), supported_decode_codecs: Vec::new(), has_push_descriptor: plan.push_descriptor, + has_video_encode_rgb_conversion: plan.rgb_conversion, has_unified_image_layouts: false, owns_device: false, retired_semaphores: Default::default(), diff --git a/tests/common/source.rs b/tests/common/source.rs index 3c3292d..ecfa0c2 100644 --- a/tests/common/source.rs +++ b/tests/common/source.rs @@ -92,8 +92,9 @@ pub unsafe fn host_buffer( /// /// `GENERAL` because that is where the colour converter leaves its source after /// every conversion, so it is the layout the image is in for every conversion -/// after the first. Starting it anywhere else means a second conversion told -/// the wrong layout. +/// after the first, and the one an encoder taking RGB input expects to copy +/// from. Starting it anywhere else means a second conversion told the wrong +/// layout. pub unsafe fn create_src_image( context: &VideoContext, width: u32, @@ -113,7 +114,13 @@ pub unsafe fn create_src_image( .array_layers(1) .samples(vk::SampleCountFlags::TYPE_1) .tiling(vk::ImageTiling::OPTIMAL) - .usage(vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST) + // Sampled by the converter, and copied from by an encoder taking RGB + // input, which reads its source with a transfer. + .usage( + vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::TRANSFER_DST + | vk::ImageUsageFlags::TRANSFER_SRC, + ) .sharing_mode(vk::SharingMode::EXCLUSIVE) .initial_layout(vk::ImageLayout::UNDEFINED); let image = unsafe { device.create_image(&info, None) }?; @@ -198,7 +205,7 @@ pub unsafe fn create_src_image( .image(image) .subresource_range(range) .src_access_mask(vk::AccessFlags::TRANSFER_WRITE) - .dst_access_mask(vk::AccessFlags::SHADER_READ); + .dst_access_mask(vk::AccessFlags::SHADER_READ | vk::AccessFlags::TRANSFER_READ); device.cmd_pipeline_barrier( cb, vk::PipelineStageFlags::TRANSFER, diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs new file mode 100644 index 0000000..e2d2568 --- /dev/null +++ b/tests/rgb_encode.rs @@ -0,0 +1,255 @@ +//! Encoding RGB input with the encoder's own colour conversion, through +//! `VK_VALVE_video_encode_rgb_conversion`, against the colour converter. +//! +//! When the conversion is nothing but the YUV matrix, the encoder can take the +//! RGB frame as it is and the shader does not run at all. These tests encode +//! the same frames both ways and check the hardware path lands where the +//! shader does. +//! +//! Skipped on devices without the extension, which today is everything but +//! AMD under RADV. + +#[allow(dead_code)] +mod common; +use common::source::{SrcImage, create_src_image}; +use common::{Readback, decode_stream}; + +use ash::vk; +use pixelforge::decoder::{DecodeConfig, Decoder}; +use pixelforge::{ + Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorRange, ColorSpec, + EncodeBitDepth, EncodeConfig, Encoder, InputFormat, OutputFormat, RateControlMode, + VideoContext, VideoContextBuilder, +}; + +const WIDTH: u32 = 320; +const HEIGHT: u32 = 240; +const FRAMES: u32 = 8; + +fn context(codec: Codec) -> Result> { + common::init_logging(); + let mut builder = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(codec); + if codec == Codec::H264 { + builder = builder.require_decode(Codec::H264); + } + Ok(builder.build()?) +} + +fn converter_config() -> ColorConverterConfig { + ColorConverterConfig::new( + WIDTH, + HEIGHT, + InputFormat::BGRA, + OutputFormat::NV12, + ColorSpec::Srgb, + ColorSpec::Srgb, + ColorRange::Limited, + ) +} + +/// A BGRA frame that moves with `n`. +fn moving_frame(n: u32) -> Vec { + let mut data = Vec::with_capacity((WIDTH * HEIGHT * 4) as usize); + for y in 0..HEIGHT { + for x in 0..WIDTH { + let (sx, sy) = (x + n * 3, y + n); + data.extend_from_slice(&[(sy * 3) as u8, (sx * 2 + sy) as u8, (sx ^ sy) as u8, 255]); + } + } + data +} + +/// Upload frame `n`. It is left in `GENERAL`, which is where [`Encoder::encode`] +/// expects a source it copies from. +fn source(context: &VideoContext, n: u32) -> Result> { + Ok(unsafe { create_src_image(context, WIDTH, HEIGHT, &moving_frame(n))? }) +} + +fn encode_config(codec: Codec) -> EncodeConfig { + let config = match codec { + Codec::H264 => EncodeConfig::h264(WIDTH, HEIGHT), + Codec::H265 => EncodeConfig::h265(WIDTH, HEIGHT), + Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), + }; + config + .with_rate_control(RateControlMode::Cqp) + .with_quality_level(22) + .with_frame_rate(30, 1) + .with_b_frames(0) + .with_color_description(converter_config().color_description().unwrap()) +} + +/// Encode the frames with the colour converter doing the conversion. +fn encode_with_shader( + context: &VideoContext, + codec: Codec, +) -> Result, Box> { + let mut encoder = Encoder::new(context.clone(), encode_config(codec))?; + let mut converter = ColorConverter::new(context.clone(), converter_config())?; + let mut stream = Vec::new(); + for n in 0..FRAMES { + let src = source(context, n)?; + converter.convert(src.image, vk::ImageLayout::GENERAL, encoder.input_image())?; + let packet = pollster::block_on(encoder.encode(encoder.input_image())?)?; + stream.extend_from_slice(&packet.data); + } + Ok(stream) +} + +/// Encode the frames with the encoder doing the conversion. +fn encode_rgb( + context: &VideoContext, + codec: Codec, + format: InputFormat, +) -> Result, Box> { + let mut encoder = Encoder::new(context.clone(), encode_config(codec).with_rgb_input(format))?; + let mut stream = Vec::new(); + for n in 0..FRAMES { + let src = source(context, n)?; + let packet = pollster::block_on(encoder.encode(src.image)?)?; + stream.extend_from_slice(&packet.data); + } + Ok(stream) +} + +fn decode_all( + context: &VideoContext, + stream: &[u8], +) -> Result>, Box> { + let mut readback = Readback::new(context)?; + let mut decoder = Decoder::new(context.clone(), DecodeConfig::h264().with_byte_stream())?; + let mut frames = Vec::new(); + decode_stream(&mut decoder, stream, |frame| { + frames.push(readback.read(&frame)?.y); + Ok(()) + })?; + Ok(frames) +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse = a + .iter() + .zip(b) + .map(|(&x, &y)| (x as f64 - y as f64).powi(2)) + .sum::() + / a.len() as f64; + if mse == 0.0 { + f64::INFINITY + } else { + 10.0 * (255.0f64 * 255.0 / mse).log10() + } +} + +/// The format to hand a `codec` encoder, or `None` to skip it on this device. +fn rgb_format(context: &VideoContext, codec: Codec, test: &str) -> Option { + let format = converter_config().rgb_encode_input(context, codec); + if format.is_none() { + eprintln!("skipping {test} for {codec:?}: this device has no RGB input for it"); + } + format +} + +#[test] +#[ignore = "requires a Vulkan Video device with VK_VALVE_video_encode_rgb_conversion"] +fn rgb_input_matches_the_converter() -> Result<(), Box> { + let context = context(Codec::H264)?; + let Some(format) = rgb_format(&context, Codec::H264, "rgb_input_matches_the_converter") else { + return Ok(()); + }; + let shader = decode_all(&context, &encode_with_shader(&context, Codec::H264)?)?; + let hardware = decode_all(&context, &encode_rgb(&context, Codec::H264, format)?)?; + assert_eq!(hardware.len(), FRAMES as usize, "frame count"); + let psnrs: Vec = hardware + .iter() + .zip(&shader) + .map(|(h, s)| psnr(h, s)) + .collect(); + println!("hardware vs shader conversion, luma PSNR per frame: {psnrs:?}"); + // Both apply the same BT.709 matrix to the same frames, then the same + // encoder settings; what is left is rounding in the two conversions and + // what the encoder makes of it. + for (i, p) in psnrs.iter().enumerate() { + assert!(*p >= 40.0, "frame {i}: {p:.2} dB against the shader path"); + } + Ok(()) +} + +#[test] +#[ignore = "requires a Vulkan Video device with VK_VALVE_video_encode_rgb_conversion"] +fn rgb_input_encodes_every_codec() -> Result<(), Box> { + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + let context = match context(codec) { + Ok(context) => context, + Err(e) => { + eprintln!("{codec:?}: no encoder on this device ({e})"); + continue; + } + }; + let Some(format) = rgb_format(&context, codec, "rgb_input_encodes_every_codec") else { + continue; + }; + let stream = encode_rgb(&context, codec, format)?; + assert!(!stream.is_empty(), "{codec:?}: empty stream"); + if let Ok(dir) = std::env::var("PIXELFORGE_RGB_STREAM_DIR") { + std::fs::write(format!("{dir}/rgb.{codec:?}").to_lowercase(), &stream)?; + } + } + Ok(()) +} + +/// `supports_rgb_encode_input` must agree with what an encoder can actually +/// be created for, on every device: one that answers yes for something the +/// encoder then refuses sends a caller down a path that fails, and one that +/// answers no for something that works leaves the shader running for nothing. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn rgb_query_matches_encoder_creation() -> Result<(), Box> { + common::init_logging(); + let context = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .build()?; + let descriptions = [ + ("BT.709 limited", ColorDescription::bt709()), + ( + "BT.709 full", + ColorDescription::bt709().with_full_range(true), + ), + ("BT.2020", ColorDescription::bt2020_pq()), + ]; + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + if !context.supports_encode(codec) { + continue; + } + for format in [ + InputFormat::BGRA, + InputFormat::RGBx, + InputFormat::ABGR2101010, + InputFormat::RGBA16F, + ] { + let bit_depth = if format == InputFormat::ABGR2101010 { + EncodeBitDepth::Ten + } else { + EncodeBitDepth::Eight + }; + for (name, desc) in &descriptions { + let supported = context.supports_rgb_encode_input(codec, format, desc); + let created = Encoder::new( + context.clone(), + encode_config(codec) + .with_bit_depth(bit_depth) + .with_color_description(*desc) + .with_rgb_input(format), + ) + .is_ok(); + eprintln!("{codec:?} {format:?} {name}: query={supported} encoder={created}"); + assert_eq!( + supported, created, + "{codec:?} {format:?} {name}: query and encoder creation disagree" + ); + } + } + } + Ok(()) +}