From 2ca8062e6c795249bdfc499d1153566a9a6f8694 Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Thu, 22 Oct 2026 13:53:59 +0300 Subject: [PATCH 1/3] docs(encoder): the default colour description is limited range EncodeConfig::color_description said a missing description defaults to BT.709 full range. Every encoder falls back to ColorDescription::bt709(), which is limited range, and that is what the stream then declares. Co-Authored-By: Claude Opus 5.5 --- src/encoder/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 1d287a5..091f72c 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -342,7 +342,7 @@ pub struct EncodeConfig { /// IDR frames maximum headroom. pub initial_virtual_buffer_size_ms: u32, /// Color description for VUI signaling. - /// Defaults to BT.709 (full-range) when `None`. + /// Defaults to BT.709, limited range, when `None`. pub color_description: Option, /// Usage hint for encoding. pub encode_usage_hint: EncodeUsageHint, From e290c6e2a85eba46b21c70b976a5702ed3ae6b6a Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Thu, 22 Oct 2026 13:53:59 +0300 Subject: [PATCH 2/3] feat(encoder): convert RGB input in the encoder where the device can VK_VALVE_video_encode_rgb_conversion lets an encoder take an RGB image and apply the YUV matrix itself. Where the colour conversion is nothing but that matrix, which covers ordinary SDR capture, the converter's compute dispatch and its buffer-to-image copy disappear from the frame entirely. EncodeConfig::with_rgb_input(format) turns it on. The extension adds one struct to the video profile, and everything that names the profile has to name that version of it: the capability and format queries, the input and DPB images, the query pools and the session. Each codec's one profile gains it. The input image becomes a single-plane RGB image while the DPB stays YUV, and Encoder::encode copies an RGB source into it. The matrix and range come from the configured colour description, checked against what the driver reports it can do. Encoder::new refuses a configuration that cannot work before asking the driver anything: a device without the extension, half-float input, or an encoder not set up for the 4:2:0 at the input's bit depth that the conversion produces. VideoContext::supports_rgb_encode_input answers whether that will work for a codec, input format and colour description, without creating an encoder: the extension, the codec's RGB profile, the conversion's matrix and range, and the input format. It shares one profile builder with supports_encode_format, which now takes the RGB struct as an option rather than a fourth copy of the per-codec profiles existing. ColorConverterConfig::rgb_encode_input asks it for a conversion that is only the matrix, so it no longer says yes for something Encoder::new then refuses. On current RADV that matters at once: the conversion reports limited range only, so full-range RGB input is refused up front rather than producing a stream whose samples and label disagree. Chroma siting asks for what a player assumes when the stream does not say: co-sited with the left luma sample horizontally, midway between rows vertically (H.264 and H.265 chroma location type 0). A driver that cannot gets its other option. The RGB input image is zeroed by the same buffer copy as a YUV one, since that runs on the transfer queue, where a clear command is not allowed. On an RX 9060 XT, H.264 lands at 46 dB in luma against the shader path on the same frames, and H.265 and AV1 at 46 and 48 dB against the H.264 stream. supports_rgb_encode_input agrees with encoder creation for every codec, input format and colour description tested, here and on an Arc A310, which has no extension and answers no throughout. Based on the approach in hgaiser/pixelforge#14, reworked for the current encoder structure. Co-authored-by: Ryan Bateman Co-Authored-By: Claude Opus 5.5 --- src/converter/mod.rs | 34 ++++- src/encoder/av1/init.rs | 13 +- src/encoder/codec.rs | 47 ++++++- src/encoder/h264/init.rs | 13 +- src/encoder/h265/init.rs | 13 +- src/encoder/mod.rs | 31 +++++ src/encoder/pipeline.rs | 3 + src/encoder/resources.rs | 55 +++++++- src/encoder/rgb.rs | 223 +++++++++++++++++++++++++++++++ src/encoder/support.rs | 179 ++++++++++++++----------- src/lib.rs | 27 +++- src/vulkan.rs | 43 ++++++ src/vulkan/encode_device.rs | 16 ++- tests/common/source.rs | 15 ++- tests/rgb_encode.rs | 255 ++++++++++++++++++++++++++++++++++++ 15 files changed, 874 insertions(+), 93 deletions(-) create mode 100644 src/encoder/rgb.rs create mode 100644 tests/rgb_encode.rs diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 15eadaa..b543400 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -8,7 +8,7 @@ mod pipeline; -use crate::encoder::ColorDescription; +use crate::encoder::{Codec, ColorDescription}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; use crate::video::TimelineChain; @@ -325,6 +325,38 @@ impl ColorConverterConfig { ) } + /// The RGB input to give a `codec` encoder instead of running this + /// conversion, when the encoder can do it itself. + /// + /// The hardware applies only the YUV matrix, so this is `Some` only for a + /// conversion that is nothing more: source and target are the same space, + /// so no transfer function or gamut change is involved, and the output is + /// 4:2:0 at the input's bit depth, NV12 from 8-bit RGB or P010 from + /// `ABGR2101010`. The device is then asked, with + /// [`VideoContext::supports_rgb_encode_input`], whether it does that + /// matrix and range for `codec` and takes this format. + /// + /// When `Some`, build the encoder with + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// and this format, and with [`Self::color_description`], then hand it the + /// RGB images directly and skip the converter. + pub fn rgb_encode_input(&self, context: &VideoContext, codec: Codec) -> Option { + if self.source != self.target { + return None; + } + let shape_matches = match self.output_format { + OutputFormat::NV12 => matches!( + self.input_format, + InputFormat::BGRx | InputFormat::BGRA | InputFormat::RGBx | InputFormat::RGBA + ), + OutputFormat::P010 => self.input_format == InputFormat::ABGR2101010, + _ => false, + }; + let desc = self.color_description()?; + (shape_matches && context.supports_rgb_encode_input(codec, self.input_format, &desc)) + .then_some(self.input_format) + } + /// Whether the shader can get from this source to this target. /// /// Anything that would need tone mapping or a forward gamma encode is not diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index d988af5..adc0097 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -4,6 +4,7 @@ use crate::encoder::codec::{ CodecEncoder, CommonInitRequest, build_encoder_common, query_video_caps, }; use crate::encoder::profile::av1_profile; +use crate::encoder::rgb::{RgbConversionCaps, rgb_conversion_profile}; use crate::encoder::{ColorDescription, EncodeConfig}; use crate::error::{PixelForgeError, Result}; use crate::vulkan::VideoContext; @@ -44,13 +45,17 @@ impl Av1 { .tuning_mode(encoder_tuning_mode); let mut av1_profile_info = vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(profile); - let profile_info = vk::VideoProfileInfoKHR::default() + let mut rgb_profile = rgb_conversion_profile(); + let mut profile_info = vk::VideoProfileInfoKHR::default() .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_AV1) .chroma_subsampling(chroma_subsampling) .luma_bit_depth(bit_depth) .chroma_bit_depth(bit_depth) .push(&mut av1_profile_info) .push(&mut video_encode_usage_info); + if config.rgb_input.is_some() { + profile_info = profile_info.push(&mut rgb_profile); + } let bitstream_buffer_size = MIN_BITSTREAM_BUFFER_SIZE .max(config.dimensions.width as usize * config.dimensions.height as usize); @@ -62,7 +67,12 @@ impl Av1 { let mut capabilities = vk::VideoCapabilitiesKHR::default() .push(&mut encode_caps) .push(&mut av1_caps); + let mut rgb_caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); + if config.rgb_input.is_some() { + capabilities = capabilities.push(&mut rgb_caps); + } let caps = query_video_caps(&context, &profile_info, &mut capabilities)?; + let rgb_caps = config.rgb_input.map(|_| RgbConversionCaps::from(&rgb_caps)); let init = build_encoder_common(&CommonInitRequest { context: &context, @@ -81,6 +91,7 @@ impl Av1 { // when the driver does not support separate reference images // (required for AMD RADV). allow_layered_dpb: true, + rgb_caps, })?; let active_reference_count = init.active_reference_count; let common = init.common; diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 32444f6..95065c0 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -13,6 +13,7 @@ //! scaffolding is here. use ash::vk; +use ash::vk::TaggedStructure; use crate::encoder::dpb::MAX_DPB_SLOTS; use crate::encoder::gop::{GopFrameType, GopPosition, GopStructure}; @@ -22,6 +23,7 @@ use crate::encoder::resources::{ create_dpb_images, destroy_encoder_resources, get_video_format, lcm, query_supported_video_formats, upload_image_to_input, }; +use crate::encoder::rgb::RgbConversionCaps; use crate::encoder::{ColorDescription, EncodeConfig, FrameType, RateControlMode}; use crate::error::{PixelForgeError, Result}; use crate::sync::TimelinePoint; @@ -115,6 +117,7 @@ impl EncoderCommon { height: self.config.dimensions.height, pixel_format: self.config.pixel_format, input_image_layout, + rgb: self.config.rgb_input.is_some(), upload_queue: self.context.transfer_queue(), waits: &waits, }; @@ -447,6 +450,10 @@ pub(crate) struct CommonInitRequest<'a> { /// Whether the codec can use a layered DPB image when the driver lacks /// `SEPARATE_REFERENCE_IMAGES` (H.264/H.265 yes; AV1 no). pub allow_layered_dpb: bool, + /// What the hardware RGB conversion supports for this profile, queried + /// alongside the codec capabilities when [`EncodeConfig::rgb_input`] is + /// set. + pub rgb_caps: Option, } /// Result of [`build_encoder_common`]: the assembled common state plus the @@ -574,6 +581,27 @@ pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result { + let caps = req.rgb_caps.ok_or_else(|| { + PixelForgeError::NoSuitableDevice( + "RGB input: the driver reported no conversion capabilities".to_string(), + ) + })?; + Some( + caps.session_info( + &config + .color_description + .unwrap_or_else(ColorDescription::bt709), + )?, + ) + } + None => None, + }; + // Pick input (SRC) and reference (DPB) formats. let preferred_src_format = get_video_format(config.pixel_format, config.bit_depth); let supported_src_formats = query_supported_video_formats( @@ -596,20 +624,22 @@ pub(crate) fn build_encoder_common(req: &CommonInitRequest) -> Result Result Result Result, + /// Take RGB input and let the encoder convert it to YUV itself, through + /// `VK_VALVE_video_encode_rgb_conversion`. See [`Self::with_rgb_input`]. + pub rgb_input: Option, /// Usage hint for encoding. pub encode_usage_hint: EncodeUsageHint, /// Content hint for encoding. @@ -375,6 +379,7 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, + rgb_input: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -403,6 +408,7 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, + rgb_input: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -431,6 +437,7 @@ impl EncodeConfig { virtual_buffer_size_ms: 1000, initial_virtual_buffer_size_ms: 1000, color_description: None, + rgb_input: None, encode_usage_hint: EncodeUsageHint::Default, encode_content_hint: EncodeContentHint::Default, encoder_tuning_mode: EncoderTuningMode::Default, @@ -519,6 +526,27 @@ impl EncodeConfig { self } + /// Take `format` RGB images as input and convert them to YUV in the + /// encoder itself, with no colour conversion shader. + /// + /// Needs `VK_VALVE_video_encode_rgb_conversion` (see + /// [`VideoContext::has_video_encode_rgb_conversion`]), and a driver that + /// accepts `format` as encode input for this codec, profile and bit depth; + /// [`Encoder::new`] fails otherwise. The YUV matrix and range come from + /// [`Self::color_description`], so set that to what the stream should + /// carry. The hardware applies only the matrix: a source that needs a + /// transfer function or gamut change still has to go through + /// [`ColorConverter`](crate::ColorConverter), which + /// [`ColorConverterConfig::rgb_encode_input`](crate::ColorConverterConfig::rgb_encode_input) + /// decides for you. + /// + /// With this set, [`Encoder::input_image`] is an RGB image, and + /// [`Encoder::encode`] copies an RGB source into it. + pub fn with_rgb_input(mut self, format: crate::converter::InputFormat) -> Self { + self.rgb_input = Some(format); + self + } + /// Set the usage hint for encoding. pub fn with_encode_usage_hint(mut self, hint: EncodeUsageHint) -> Self { self.encode_usage_hint = hint; @@ -620,6 +648,9 @@ pub struct Encoder(Box); impl Encoder { /// Create a new encoder for the codec named in `config`. pub fn new(context: VideoContext, config: EncodeConfig) -> Result { + if let Some(format) = config.rgb_input { + rgb::check_rgb_input(&context, &config, format)?; + } let inner: Box = match config.codec { Codec::H264 => Box::new(self::h264::H264::create(context, config)?), Codec::H265 => Box::new(self::h265::H265::create(context, config)?), diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index d46ef58..4d6ea16 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -165,6 +165,8 @@ pub(crate) struct PipelineConfig<'a> { pub aligned_width: u32, pub aligned_height: u32, pub picture_format: vk::Format, + /// Whether the input image is RGB, for the encoder to convert. + pub rgb_input: bool, pub pixel_format: PixelFormat, pub bit_depth: BitDepth, pub bitstream_buffer_size: usize, @@ -256,6 +258,7 @@ impl EncodePipeline { height: config.aligned_height, pixel_format: config.pixel_format, bit_depth: config.bit_depth, + rgb: config.rgb_input, }, )?; diff --git a/src/encoder/resources.rs b/src/encoder/resources.rs index ccf17d5..e5fa4fd 100644 --- a/src/encoder/resources.rs +++ b/src/encoder/resources.rs @@ -205,9 +205,17 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam // YUV 4:4:4 (e.g., NV24): UV plane is full width, full height, 2 components per pixel. PixelFormat::Yuv444 => params.width * params.height * 2 * bytes_per_component, }; - let total_size = (plane0_size + plane1_size) as vk::DeviceSize; + // Every RGB input format is 4 bytes a pixel in one plane. + let rgb_size = params.width * params.height * 4; + let total_size = if params.rgb { + rgb_size + } else { + plane0_size + plane1_size + } as vk::DeviceSize; - // Create a staging buffer filled with zeros. + // Create a staging buffer filled with zeros. A copy rather than a clear + // command, since this runs on the transfer queue and a clear needs a + // graphics or compute one. let buffer_create_info = vk::BufferCreateInfo::default() .size(total_size) .usage(vk::BufferUsageFlags::TRANSFER_SRC) @@ -327,13 +335,25 @@ pub(crate) fn clear_input_image(context: &VideoContext, params: &ClearImageParam }, ]; + let rgb_region = vk::BufferImageCopy { + image_subresource: vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + ..copy_regions[0].image_subresource + }, + ..copy_regions[0] + }; + let regions: &[vk::BufferImageCopy] = if params.rgb { + std::slice::from_ref(&rgb_region) + } else { + ©_regions + }; unsafe { device.cmd_copy_buffer_to_image( params.command_buffer, staging_buffer, params.image, vk::ImageLayout::TRANSFER_DST_OPTIMAL, - ©_regions, + regions, ); } @@ -408,6 +428,8 @@ pub(crate) struct UploadParams<'a> { pub pixel_format: PixelFormat, /// The current layout of the input image. pub input_image_layout: vk::ImageLayout, + /// Whether both images are single-plane RGB rather than YUV. + pub rgb: bool, /// The queue to submit transfer operations to. pub upload_queue: vk::Queue, /// Caller work the copy must wait for. @@ -544,6 +566,28 @@ pub(crate) fn upload_image_to_input( }, }; + // An RGB input image has one colour plane, copied whole. + let rgb_copy_region = vk::ImageCopy { + src_subresource: vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }, + dst_subresource: vk::ImageSubresourceLayers { + aspect_mask: vk::ImageAspectFlags::COLOR, + mip_level: 0, + base_array_layer: 0, + layer_count: 1, + }, + ..y_copy_region + }; + let regions: &[vk::ImageCopy] = if params.rgb { + &[rgb_copy_region] + } else { + &[y_copy_region, uv_copy_region] + }; + unsafe { device.cmd_copy_image( params.upload_command_buffer, @@ -551,7 +595,7 @@ pub(crate) fn upload_image_to_input( vk::ImageLayout::TRANSFER_SRC_OPTIMAL, params.dst_image, vk::ImageLayout::TRANSFER_DST_OPTIMAL, - &[y_copy_region, uv_copy_region], + regions, ); } @@ -1141,6 +1185,9 @@ pub(crate) struct ClearImageParams { pub height: u32, pub pixel_format: PixelFormat, pub bit_depth: BitDepth, + /// Whether the image is a single-plane, 4-bytes-per-pixel RGB input + /// image rather than a YUV one. + pub rgb: bool, } #[cfg(test)] diff --git a/src/encoder/rgb.rs b/src/encoder/rgb.rs new file mode 100644 index 0000000..fe1910b --- /dev/null +++ b/src/encoder/rgb.rs @@ -0,0 +1,223 @@ +//! RGB input converted to YUV by the encoder itself, through +//! `VK_VALVE_video_encode_rgb_conversion`. +//! +//! The extension adds one struct to the video profile, which then means "this +//! encoder takes RGB pictures". Everything that names the profile has to name +//! that version of it: the capability and format queries, the input and DPB +//! images, the query pools and the session. The session also says which +//! matrix, range and chroma siting to convert with. + +use super::support::report; +use super::{BitDepth, Codec, ColorDescription, EncodeConfig, PixelFormat}; +use crate::converter::InputFormat; +use crate::error::{PixelForgeError, Result}; +use crate::video::query_supported_video_formats; +use crate::vulkan::VideoContext; +use ash::vk; + +/// The profile struct that turns on RGB input. +pub(crate) fn rgb_conversion_profile() -> vk::VideoEncodeProfileRgbConversionInfoVALVE<'static> { + vk::VideoEncodeProfileRgbConversionInfoVALVE::default().perform_encode_rgb_conversion(true) +} + +/// The YUV bit depth an RGB input format converts to, or `None` for a format +/// the hardware conversion does not take. +/// +/// 8-bit RGB becomes 8-bit YUV and 10-bit RGB 10-bit YUV. Half floats carry +/// values outside the 0 to 1 range that need tone mapping first, which only +/// the colour converter does. +pub(crate) fn rgb_input_bit_depth(format: InputFormat) -> Option { + match format { + InputFormat::BGRx | InputFormat::BGRA | InputFormat::RGBx | InputFormat::RGBA => { + Some(BitDepth::Eight) + } + InputFormat::ABGR2101010 => Some(BitDepth::Ten), + InputFormat::RGBA16F => None, + } +} + +/// Refuse an RGB input configuration that cannot work, before any driver is +/// asked about it. +/// +/// The device must have the extension, and the encoder must be set up for +/// the YUV the conversion produces: 4:2:0, at 8 bits from 8-bit RGB and 10 +/// bits from 10-bit RGB. +pub(crate) fn check_rgb_input( + context: &VideoContext, + config: &EncodeConfig, + format: InputFormat, +) -> Result<()> { + if !context.has_video_encode_rgb_conversion() { + return Err(PixelForgeError::NoSuitableDevice( + "RGB input needs VK_VALVE_video_encode_rgb_conversion, which this device does \ + not have enabled" + .to_string(), + )); + } + let Some(bit_depth) = rgb_input_bit_depth(format) else { + return Err(PixelForgeError::InvalidInput(format!( + "RGB input: {format:?} needs tone mapping first, which only the colour \ + converter does" + ))); + }; + if config.pixel_format != PixelFormat::Yuv420 || config.bit_depth != bit_depth { + return Err(PixelForgeError::InvalidInput(format!( + "RGB input: {format:?} converts to {bit_depth:?} 4:2:0, but the encoder is set \ + up for {:?} {:?}", + config.bit_depth, config.pixel_format + ))); + } + Ok(()) +} + +/// What the driver's RGB conversion can do for one profile: +/// `VkVideoEncodeRgbConversionCapabilitiesVALVE` without its `pNext`. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RgbConversionCaps { + models: vk::VideoEncodeRgbModelConversionFlagsVALVE, + ranges: vk::VideoEncodeRgbRangeCompressionFlagsVALVE, + x_chroma_offsets: vk::VideoEncodeRgbChromaOffsetFlagsVALVE, + y_chroma_offsets: vk::VideoEncodeRgbChromaOffsetFlagsVALVE, +} + +impl From<&vk::VideoEncodeRgbConversionCapabilitiesVALVE<'_>> for RgbConversionCaps { + fn from(caps: &vk::VideoEncodeRgbConversionCapabilitiesVALVE<'_>) -> Self { + Self { + models: caps.rgb_models, + ranges: caps.rgb_ranges, + x_chroma_offsets: caps.x_chroma_offsets, + y_chroma_offsets: caps.y_chroma_offsets, + } + } +} + +impl RgbConversionCaps { + /// The session settings that make the hardware produce what `desc` + /// describes, or why it cannot. + pub(crate) fn session_info( + &self, + desc: &ColorDescription, + ) -> Result> { + use vk::VideoEncodeRgbChromaOffsetFlagsVALVE as Offset; + use vk::VideoEncodeRgbModelConversionFlagsVALVE as Model; + use vk::VideoEncodeRgbRangeCompressionFlagsVALVE as Range; + + // The matrix coefficients code the stream carries (H.273 numbering). + let model = match desc.matrix_coefficients { + ColorDescription::MATRIX_BT709 => Model::YCBCR_709, + 5 | 6 => Model::YCBCR_601, + ColorDescription::MATRIX_BT2020_NCL => Model::YCBCR_2020, + other => { + return Err(PixelForgeError::InvalidInput(format!( + "RGB input: no hardware conversion for matrix coefficients {other}" + ))); + } + }; + let range = if desc.full_range { + Range::FULL_RANGE + } else { + Range::NARROW_RANGE + }; + if !self.models.contains(model) || !self.ranges.contains(range) { + return Err(PixelForgeError::NoSuitableDevice(format!( + "RGB input: the driver converts {:?} in {:?}, not {:?} in {:?}", + self.models, self.ranges, model, range + ))); + } + + // Where chroma samples sit relative to luma. The stream does not say, + // and a player that is not told assumes chroma sits on the left luma + // sample of each pair, and midway between the two rows (H.264 and + // H.265 chroma location type 0, which most AV1 players follow as well). + // Asking the hardware for exactly that keeps colour edges where the + // player will put them. A driver that cannot gets the other option. + let pick = |supported: Offset, preferred: Offset| { + if supported.contains(preferred) { + preferred + } else if supported.contains(Offset::COSITED_EVEN) { + Offset::COSITED_EVEN + } else { + Offset::MIDPOINT + } + }; + Ok( + vk::VideoEncodeSessionRgbConversionCreateInfoVALVE::default() + .rgb_model(model) + .rgb_range(range) + .x_chroma_offset(pick(self.x_chroma_offsets, Offset::COSITED_EVEN)) + .y_chroma_offset(pick(self.y_chroma_offsets, Offset::MIDPOINT)), + ) + } +} + +impl VideoContext { + /// Whether an [`Encoder`](crate::Encoder) for `codec` can take `format` + /// RGB input and convert it to the YUV that `desc` describes itself, + /// through `VK_VALVE_video_encode_rgb_conversion`. + /// + /// Asks the driver everything [`Encoder::new`](crate::Encoder::new) will + /// need: that the device has the extension, that the codec's RGB input + /// profile exists, that its conversion does `desc`'s matrix and range, + /// and that it accepts `format` as an input image. The output is always + /// 4:2:0, at 8 bits for 8-bit RGB and 10 bits for + /// [`InputFormat::ABGR2101010`]. Resolution and rate control are not + /// considered. + /// + /// When this is true, build the encoder with + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// and [`EncodeConfig::with_color_description`](crate::EncodeConfig::with_color_description) + /// set to `desc`. [`ColorConverterConfig::rgb_encode_input`](crate::ColorConverterConfig::rgb_encode_input) + /// asks this for a conversion it would otherwise run. + pub fn supports_rgb_encode_input( + &self, + codec: Codec, + format: InputFormat, + desc: &ColorDescription, + ) -> bool { + if !self.has_video_encode_rgb_conversion() || !self.supports_encode(codec) { + return false; + } + let Some(bit_depth) = rgb_input_bit_depth(format) else { + return false; + }; + + let mut caps = vk::VideoEncodeRgbConversionCapabilitiesVALVE::default(); + let what = format!("{codec:?} encode from {format:?} RGB"); + let formats = self.query_encode_profile( + codec, + PixelFormat::Yuv420, + bit_depth, + Some(&mut caps), + |profile| { + query_supported_video_formats( + self, + profile, + vk::ImageUsageFlags::VIDEO_ENCODE_SRC_KHR, + ) + }, + ); + let formats = match formats { + Ok(Ok(formats)) => Ok(formats), + Ok(Err(e)) => { + tracing::warn!("Failed to query {what} input formats: {e}"); + return false; + } + Err(e) => Err(e), + }; + let Some(formats) = report(&what, formats) else { + return false; + }; + if !formats.iter().any(|f| f.format == format.vk_format()) { + tracing::debug!( + "{what}: {:?} is not an accepted input format", + format.vk_format() + ); + return false; + } + if let Err(e) = RgbConversionCaps::from(&caps).session_info(desc) { + tracing::debug!("{what}: {e}"); + return false; + } + true + } +} diff --git a/src/encoder/support.rs b/src/encoder/support.rs index d88c0f4..cd5de01 100644 --- a/src/encoder/support.rs +++ b/src/encoder/support.rs @@ -2,12 +2,21 @@ use super::codec::query_video_capabilities; use super::profile::{av1_profile, h264_profile_idc, h265_profile_idc}; +use super::rgb::rgb_conversion_profile; use super::{BitDepth, Codec, PixelFormat}; use crate::vulkan::VideoContext; use ash::vk; use ash::vk::TaggedStructure; use tracing::{debug, warn}; +/// Why [`VideoContext::query_encode_profile`] has no answer to give. +pub(crate) enum ProfileQueryError { + /// The encoder has no profile for this combination (H.265 4:2:2). + NoProfile, + /// The driver rejected the profile's capability query. + Query(vk::Result), +} + impl VideoContext { /// Check whether the device can encode `codec` in the given pixel format and /// bit depth. @@ -26,91 +35,115 @@ impl VideoContext { if !self.supports_encode(codec) { return false; } + let result = self.query_encode_profile(codec, pixel_format, bit_depth, None, |_| ()); + report( + &format!("{codec:?} encode {pixel_format:?} {bit_depth:?}"), + result, + ) + .is_some() + } + /// Build the profile an encoder would request for `codec`, `pixel_format` + /// and `bit_depth`, ask the driver for its capabilities, and on success + /// hand the profile to `then` for any further query about it. + /// + /// With `rgb_caps`, the profile is the RGB input one an encoder built with + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input) + /// requests, and the driver fills in what its conversion supports. + /// + /// The encoders build their profiles themselves, since theirs also carry + /// usage hints, but from the same profile numbers in + /// [`super::profile`], so this answers for the profile they use. + pub(crate) fn query_encode_profile( + &self, + codec: Codec, + pixel_format: PixelFormat, + bit_depth: BitDepth, + mut rgb_caps: Option<&mut vk::VideoEncodeRgbConversionCapabilitiesVALVE<'static>>, + then: impl FnOnce(&vk::VideoProfileInfoKHR<'_>) -> R, + ) -> Result { let chroma_subsampling: vk::VideoChromaSubsamplingFlagsKHR = pixel_format.into(); let component_bit_depth: vk::VideoComponentBitDepthFlagsKHR = bit_depth.into(); let mut encode_caps = vk::VideoEncodeCapabilitiesKHR::default(); + let mut rgb_profile = rgb_conversion_profile(); + + let base = vk::VideoProfileInfoKHR::default() + .chroma_subsampling(chroma_subsampling) + .luma_bit_depth(component_bit_depth) + .chroma_bit_depth(component_bit_depth); - // Each arm keeps its profile and capability structs alive across the call. - let result = match codec { - Codec::H264 => { - let mut profile = vk::VideoEncodeH264ProfileInfoKHR::default() - .std_profile_idc(h264_profile_idc(pixel_format)); - let profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_H264) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(component_bit_depth) - .chroma_bit_depth(component_bit_depth) - .push(&mut profile); - let mut codec_caps = vk::VideoEncodeH264CapabilitiesKHR::default(); + // Each arm keeps its profile and capability structs alive across the + // calls that use them. + macro_rules! query { + ($operation:expr, $profile:expr, $codec_caps:expr) => {{ + let mut profile = $profile; + let mut profile_info = base.video_codec_operation($operation).push(&mut profile); + if rgb_caps.is_some() { + profile_info = profile_info.push(&mut rgb_profile); + } + let mut codec_caps = $codec_caps; let mut caps = vk::VideoCapabilitiesKHR::default() .push(&mut encode_caps) .push(&mut codec_caps); - query_video_capabilities(self, &profile_info, &mut caps) - } + if let Some(rgb_caps) = rgb_caps.as_deref_mut() { + caps = caps.push(rgb_caps); + } + match query_video_capabilities(self, &profile_info, &mut caps) { + vk::Result::SUCCESS => Ok(then(&profile_info)), + err => Err(ProfileQueryError::Query(err)), + } + }}; + } + + match codec { + Codec::H264 => query!( + vk::VideoCodecOperationFlagsKHR::ENCODE_H264, + vk::VideoEncodeH264ProfileInfoKHR::default() + .std_profile_idc(h264_profile_idc(pixel_format)), + vk::VideoEncodeH264CapabilitiesKHR::default() + ), Codec::H265 => { - let Some(profile_idc) = h265_profile_idc(pixel_format, bit_depth) else { - debug!( - "{:?} encode {:?} {:?}: no profile", - codec, pixel_format, bit_depth - ); - return false; - }; - let mut profile = - vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(profile_idc); - let profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_H265) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(component_bit_depth) - .chroma_bit_depth(component_bit_depth) - .push(&mut profile); - let mut codec_caps = vk::VideoEncodeH265CapabilitiesKHR::default(); - let mut caps = vk::VideoCapabilitiesKHR::default() - .push(&mut encode_caps) - .push(&mut codec_caps); - query_video_capabilities(self, &profile_info, &mut caps) + let profile_idc = h265_profile_idc(pixel_format, bit_depth) + .ok_or(ProfileQueryError::NoProfile)?; + query!( + vk::VideoCodecOperationFlagsKHR::ENCODE_H265, + vk::VideoEncodeH265ProfileInfoKHR::default().std_profile_idc(profile_idc), + vk::VideoEncodeH265CapabilitiesKHR::default() + ) } - Codec::AV1 => { - let mut profile = vk::VideoEncodeAV1ProfileInfoKHR::default() - .std_profile(av1_profile(pixel_format)); - let profile_info = vk::VideoProfileInfoKHR::default() - .video_codec_operation(vk::VideoCodecOperationFlagsKHR::ENCODE_AV1) - .chroma_subsampling(chroma_subsampling) - .luma_bit_depth(component_bit_depth) - .chroma_bit_depth(component_bit_depth) - .push(&mut profile); - let mut codec_caps = vk::VideoEncodeAV1CapabilitiesKHR::default(); - let mut caps = vk::VideoCapabilitiesKHR::default() - .push(&mut encode_caps) - .push(&mut codec_caps); - query_video_capabilities(self, &profile_info, &mut caps) - } - }; + Codec::AV1 => query!( + vk::VideoCodecOperationFlagsKHR::ENCODE_AV1, + vk::VideoEncodeAV1ProfileInfoKHR::default().std_profile(av1_profile(pixel_format)), + vk::VideoEncodeAV1CapabilitiesKHR::default() + ), + } + } +} - match result { - vk::Result::SUCCESS => { - debug!( - "{:?} encode {:?} {:?}: supported", - codec, pixel_format, bit_depth - ); - true - } - vk::Result::ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR +/// Log the outcome of a profile query for `what`, and turn it into an +/// `Option`. A profile the driver does not support is ordinary and logged at +/// debug; any other failure is unexpected and logged at warn. +pub(crate) fn report(what: &str, result: Result) -> Option { + match result { + Ok(value) => { + debug!("{what}: supported"); + Some(value) + } + Err(ProfileQueryError::NoProfile) => { + debug!("{what}: no profile"); + None + } + Err(ProfileQueryError::Query( + err @ (vk::Result::ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR | vk::Result::ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR - | vk::Result::ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR => { - debug!( - "{:?} encode {:?} {:?}: not supported ({:?})", - codec, pixel_format, bit_depth, result - ); - false - } - err => { - warn!( - "Failed to query {:?} encode {:?} {:?} capabilities: {:?}", - codec, pixel_format, bit_depth, err - ); - false - } + | vk::Result::ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR), + )) => { + debug!("{what}: not supported ({err:?})"); + None + } + Err(ProfileQueryError::Query(err)) => { + warn!("Failed to query {what} capabilities: {err:?}"); + None } } } diff --git a/src/lib.rs b/src/lib.rs index 5eb9f1c..7294313 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,8 @@ //! - **Asynchronous pipelines**: both directions submit without waiting. //! Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a //! [`DecodeSource`] as the GPU finishes with them. -//! - **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ). +//! - **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ), +//! or in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` is available. //! - **Shared devices**: encode and decode on a Vulkan device your application //! created, from Vulkan 1.1 up, on queues you choose, ordered against your own //! work with timeline semaphores. @@ -247,6 +248,30 @@ //! # } //! ``` //! +//! Where the conversion is nothing but the YUV matrix, as for ordinary SDR +//! content, some encoders can do it themselves and the shader need not run at +//! all. [`ColorConverterConfig::rgb_encode_input`] says whether that works for +//! a given conversion and codec on this device. When it does, build the +//! encoder with [`EncodeConfig::with_rgb_input`] and hand it the RGB frames. +//! +//! ```rust,no_run +//! # use pixelforge::{Codec, ColorConverterConfig, EncodeConfig, Encoder, VideoContext}; +//! # fn pick( +//! # context: VideoContext, +//! # config: ColorConverterConfig, +//! # ) -> Result<(), pixelforge::PixelForgeError> { +//! let encode_config = EncodeConfig::h264(1920, 1080) +//! .with_color_description(config.color_description().expect("SDR is encodable")); +//! let encoder = match config.rgb_encode_input(&context, Codec::H264) { +//! // The encoder converts: give it the RGB images as they are. +//! Some(format) => Encoder::new(context, encode_config.with_rgb_input(format))?, +//! // The shader converts into the encoder's input image, as above. +//! None => Encoder::new(context, encode_config)?, +//! }; +//! # Ok(()) +//! # } +//! ``` +//! //! ## Encoding on your own device //! //! An application that already renders with Vulkan can give pixelforge its own diff --git a/src/vulkan.rs b/src/vulkan.rs index 9d5928f..5a77765 100644 --- a/src/vulkan.rs +++ b/src/vulkan.rs @@ -356,6 +356,9 @@ pub struct DeviceFeatures { pub ycbcr_2plane_444_formats: bool, /// `videoEncodeAV1`, from `VkPhysicalDeviceVideoEncodeAV1FeaturesKHR`. pub video_encode_av1: bool, + /// `videoEncodeRgbConversion`, from + /// `VkPhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE`. + pub video_encode_rgb_conversion: bool, } /// The queue families pixelforge selects for decoding on a given device. @@ -555,6 +558,15 @@ fn supports_internally_synchronized_queues( features.internally_synchronized_queues != 0 } +/// Whether `physical_device` has the `videoEncodeRgbConversion` feature. The +/// caller checks for the extension first. +fn supports_rgb_conversion(instance: &ash::Instance, physical_device: vk::PhysicalDevice) -> bool { + let mut features = vk::PhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE::default(); + let mut query = vk::PhysicalDeviceFeatures2::default().push(&mut features); + unsafe { instance.get_physical_device_features2(physical_device, &mut query) }; + features.video_encode_rgb_conversion != 0 +} + /// Device extensions required to decode the given codecs. fn decode_extension_names(decode_codecs: &[Codec]) -> Vec<&'static std::ffi::CStr> { // Synchronization2 and timeline semaphores are core in 1.3 and 1.2, but @@ -614,6 +626,7 @@ struct VideoContextInner { supported_encode_codecs: Vec, supported_decode_codecs: Vec, has_push_descriptor: bool, + has_video_encode_rgb_conversion: bool, has_unified_image_layouts: bool, /// Whether this context created (and therefore must destroy) the device and /// instance. A context adopted from a caller's device borrows them and @@ -761,6 +774,14 @@ impl VideoContext { self.inner.has_push_descriptor } + /// Whether `VK_VALVE_video_encode_rgb_conversion` is enabled, extension + /// and feature both: encoders can then take RGB input and convert it + /// themselves, see + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input). + pub fn has_video_encode_rgb_conversion(&self) -> bool { + self.inner.has_video_encode_rgb_conversion + } + /// Whether `VK_KHR_unified_image_layouts` is enabled, with its video bit. /// /// When it is, `VK_IMAGE_LAYOUT_GENERAL` is valid for every use including @@ -1187,6 +1208,23 @@ impl VideoContext { warn!("VK_KHR_push_descriptor not available; the colour converter will be unavailable"); } + // Lets an encoder take RGB input and convert it to YUV itself, so a + // plain matrix conversion needs no shader at all. Extension and feature + // both, or the profile struct is parsed and ignored. + let has_rgb_conversion_ext = video_encode_queue_family.is_some() + && selected_device_exts.as_ref().is_some_and(|exts| { + has_extension(exts, ash::valve::video_encode_rgb_conversion::NAME) + }); + let has_rgb_conversion = + has_rgb_conversion_ext && supports_rgb_conversion(&instance, physical_device); + let mut rgb_conversion_features = + vk::PhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE::default() + .video_encode_rgb_conversion(true); + if has_rgb_conversion { + push_ext(ash::valve::video_encode_rgb_conversion::NAME.as_ptr()); + debug!("Video encode RGB conversion available"); + } + // Enable synchronization2 feature. let mut sync2_features = vk::PhysicalDeviceSynchronization2Features::default().synchronization2(true); @@ -1303,6 +1341,9 @@ impl VideoContext { if has_ycbcr_conversion { device_create_info = device_create_info.push(&mut ycbcr_features); } + if has_rgb_conversion { + device_create_info = device_create_info.push(&mut rgb_conversion_features); + } // Enable YCbCr 2-plane 444 formats feature (required for YUV444 encoding with NVIDIA). // Its extension is enabled above whenever present, so the feature has // to follow it unconditionally: an extension without its feature bit @@ -1353,6 +1394,7 @@ impl VideoContext { supported_encode_codecs, supported_decode_codecs, has_push_descriptor, + has_video_encode_rgb_conversion: has_rgb_conversion, has_unified_image_layouts: has_unified_layouts, owns_device: true, retired_semaphores: Default::default(), @@ -1457,6 +1499,7 @@ impl VideoContext { supported_encode_codecs: Vec::new(), supported_decode_codecs, has_push_descriptor: false, + has_video_encode_rgb_conversion: false, has_unified_image_layouts: declared_unified_image_layouts, owns_device: false, retired_semaphores: Default::default(), diff --git a/src/vulkan/encode_device.rs b/src/vulkan/encode_device.rs index 3534820..c6b462b 100644 --- a/src/vulkan/encode_device.rs +++ b/src/vulkan/encode_device.rs @@ -41,6 +41,7 @@ struct EncodePlan { ycbcr_2plane_444: bool, sampler_ycbcr_conversion: bool, internally_synchronized_queues: bool, + rgb_conversion: bool, } impl EncodePlan { @@ -80,7 +81,7 @@ impl EncodePlan { let extensions = DeviceExtensions::of(instance, physical_device); let has = |name: &std::ffi::CStr| extensions.has(name); - // The feature bits behind two of the extensions. A driver may offer an + // The feature bits behind three of the extensions. A driver may offer an // extension and still report its feature off, and a caller told to // enable a feature the device lacks gets VK_ERROR_FEATURE_NOT_PRESENT // from vkCreateDevice. A struct is chained only when its extension is @@ -88,6 +89,7 @@ impl EncodePlan { let mut ycbcr = vk::PhysicalDeviceSamplerYcbcrConversionFeatures::default(); let mut ycbcr_444 = vk::PhysicalDeviceYcbcr2Plane444FormatsFeaturesEXT::default(); let mut av1 = vk::PhysicalDeviceVideoEncodeAV1FeaturesKHR::default(); + let mut rgb = vk::PhysicalDeviceVideoEncodeRgbConversionFeaturesVALVE::default(); let mut query = vk::PhysicalDeviceFeatures2::default().push(&mut ycbcr); if has(ash::ext::ycbcr_2plane_444_formats::NAME) { query = query.push(&mut ycbcr_444); @@ -95,6 +97,9 @@ impl EncodePlan { if has(ash::khr::video_encode_av1::NAME) { query = query.push(&mut av1); } + if has(ash::valve::video_encode_rgb_conversion::NAME) { + query = query.push(&mut rgb); + } unsafe { instance.get_physical_device_features2(physical_device, &mut query) }; let family = encode.family; @@ -151,6 +156,8 @@ impl EncodePlan { physical_device, &extensions, ), + rgb_conversion: has(ash::valve::video_encode_rgb_conversion::NAME) + && rgb.video_encode_rgb_conversion != 0, }) } @@ -177,6 +184,9 @@ impl EncodePlan { if self.ycbcr_2plane_444 { names.push(ash::ext::ycbcr_2plane_444_formats::NAME); } + if self.rgb_conversion { + names.push(ash::valve::video_encode_rgb_conversion::NAME); + } names } @@ -187,6 +197,7 @@ impl EncodePlan { sampler_ycbcr_conversion: self.sampler_ycbcr_conversion, ycbcr_2plane_444_formats: self.ycbcr_2plane_444, video_encode_av1: self.codecs.contains(&Codec::AV1), + video_encode_rgb_conversion: self.rgb_conversion, } } @@ -213,6 +224,8 @@ impl VideoContextBuilder { /// The colour converter needs `VK_KHR_push_descriptor`, which is included /// whenever the device has it. On a device without it, encoding from YUV /// input still works and only the converter is unavailable. + /// `VK_VALVE_video_encode_rgb_conversion` is included the same way, for + /// [`EncodeConfig::with_rgb_input`](crate::EncodeConfig::with_rgb_input). pub fn encode_device_requirements( &self, entry: &ash::Entry, @@ -340,6 +353,7 @@ impl VideoContext { supported_encode_codecs: plan.codecs.clone(), supported_decode_codecs: Vec::new(), has_push_descriptor: plan.push_descriptor, + has_video_encode_rgb_conversion: plan.rgb_conversion, has_unified_image_layouts: false, owns_device: false, retired_semaphores: Default::default(), diff --git a/tests/common/source.rs b/tests/common/source.rs index 3c3292d..ecfa0c2 100644 --- a/tests/common/source.rs +++ b/tests/common/source.rs @@ -92,8 +92,9 @@ pub unsafe fn host_buffer( /// /// `GENERAL` because that is where the colour converter leaves its source after /// every conversion, so it is the layout the image is in for every conversion -/// after the first. Starting it anywhere else means a second conversion told -/// the wrong layout. +/// after the first, and the one an encoder taking RGB input expects to copy +/// from. Starting it anywhere else means a second conversion told the wrong +/// layout. pub unsafe fn create_src_image( context: &VideoContext, width: u32, @@ -113,7 +114,13 @@ pub unsafe fn create_src_image( .array_layers(1) .samples(vk::SampleCountFlags::TYPE_1) .tiling(vk::ImageTiling::OPTIMAL) - .usage(vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST) + // Sampled by the converter, and copied from by an encoder taking RGB + // input, which reads its source with a transfer. + .usage( + vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::TRANSFER_DST + | vk::ImageUsageFlags::TRANSFER_SRC, + ) .sharing_mode(vk::SharingMode::EXCLUSIVE) .initial_layout(vk::ImageLayout::UNDEFINED); let image = unsafe { device.create_image(&info, None) }?; @@ -198,7 +205,7 @@ pub unsafe fn create_src_image( .image(image) .subresource_range(range) .src_access_mask(vk::AccessFlags::TRANSFER_WRITE) - .dst_access_mask(vk::AccessFlags::SHADER_READ); + .dst_access_mask(vk::AccessFlags::SHADER_READ | vk::AccessFlags::TRANSFER_READ); device.cmd_pipeline_barrier( cb, vk::PipelineStageFlags::TRANSFER, diff --git a/tests/rgb_encode.rs b/tests/rgb_encode.rs new file mode 100644 index 0000000..e2d2568 --- /dev/null +++ b/tests/rgb_encode.rs @@ -0,0 +1,255 @@ +//! Encoding RGB input with the encoder's own colour conversion, through +//! `VK_VALVE_video_encode_rgb_conversion`, against the colour converter. +//! +//! When the conversion is nothing but the YUV matrix, the encoder can take the +//! RGB frame as it is and the shader does not run at all. These tests encode +//! the same frames both ways and check the hardware path lands where the +//! shader does. +//! +//! Skipped on devices without the extension, which today is everything but +//! AMD under RADV. + +#[allow(dead_code)] +mod common; +use common::source::{SrcImage, create_src_image}; +use common::{Readback, decode_stream}; + +use ash::vk; +use pixelforge::decoder::{DecodeConfig, Decoder}; +use pixelforge::{ + Codec, ColorConverter, ColorConverterConfig, ColorDescription, ColorRange, ColorSpec, + EncodeBitDepth, EncodeConfig, Encoder, InputFormat, OutputFormat, RateControlMode, + VideoContext, VideoContextBuilder, +}; + +const WIDTH: u32 = 320; +const HEIGHT: u32 = 240; +const FRAMES: u32 = 8; + +fn context(codec: Codec) -> Result> { + common::init_logging(); + let mut builder = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .require_encode(codec); + if codec == Codec::H264 { + builder = builder.require_decode(Codec::H264); + } + Ok(builder.build()?) +} + +fn converter_config() -> ColorConverterConfig { + ColorConverterConfig::new( + WIDTH, + HEIGHT, + InputFormat::BGRA, + OutputFormat::NV12, + ColorSpec::Srgb, + ColorSpec::Srgb, + ColorRange::Limited, + ) +} + +/// A BGRA frame that moves with `n`. +fn moving_frame(n: u32) -> Vec { + let mut data = Vec::with_capacity((WIDTH * HEIGHT * 4) as usize); + for y in 0..HEIGHT { + for x in 0..WIDTH { + let (sx, sy) = (x + n * 3, y + n); + data.extend_from_slice(&[(sy * 3) as u8, (sx * 2 + sy) as u8, (sx ^ sy) as u8, 255]); + } + } + data +} + +/// Upload frame `n`. It is left in `GENERAL`, which is where [`Encoder::encode`] +/// expects a source it copies from. +fn source(context: &VideoContext, n: u32) -> Result> { + Ok(unsafe { create_src_image(context, WIDTH, HEIGHT, &moving_frame(n))? }) +} + +fn encode_config(codec: Codec) -> EncodeConfig { + let config = match codec { + Codec::H264 => EncodeConfig::h264(WIDTH, HEIGHT), + Codec::H265 => EncodeConfig::h265(WIDTH, HEIGHT), + Codec::AV1 => EncodeConfig::av1(WIDTH, HEIGHT), + }; + config + .with_rate_control(RateControlMode::Cqp) + .with_quality_level(22) + .with_frame_rate(30, 1) + .with_b_frames(0) + .with_color_description(converter_config().color_description().unwrap()) +} + +/// Encode the frames with the colour converter doing the conversion. +fn encode_with_shader( + context: &VideoContext, + codec: Codec, +) -> Result, Box> { + let mut encoder = Encoder::new(context.clone(), encode_config(codec))?; + let mut converter = ColorConverter::new(context.clone(), converter_config())?; + let mut stream = Vec::new(); + for n in 0..FRAMES { + let src = source(context, n)?; + converter.convert(src.image, vk::ImageLayout::GENERAL, encoder.input_image())?; + let packet = pollster::block_on(encoder.encode(encoder.input_image())?)?; + stream.extend_from_slice(&packet.data); + } + Ok(stream) +} + +/// Encode the frames with the encoder doing the conversion. +fn encode_rgb( + context: &VideoContext, + codec: Codec, + format: InputFormat, +) -> Result, Box> { + let mut encoder = Encoder::new(context.clone(), encode_config(codec).with_rgb_input(format))?; + let mut stream = Vec::new(); + for n in 0..FRAMES { + let src = source(context, n)?; + let packet = pollster::block_on(encoder.encode(src.image)?)?; + stream.extend_from_slice(&packet.data); + } + Ok(stream) +} + +fn decode_all( + context: &VideoContext, + stream: &[u8], +) -> Result>, Box> { + let mut readback = Readback::new(context)?; + let mut decoder = Decoder::new(context.clone(), DecodeConfig::h264().with_byte_stream())?; + let mut frames = Vec::new(); + decode_stream(&mut decoder, stream, |frame| { + frames.push(readback.read(&frame)?.y); + Ok(()) + })?; + Ok(frames) +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse = a + .iter() + .zip(b) + .map(|(&x, &y)| (x as f64 - y as f64).powi(2)) + .sum::() + / a.len() as f64; + if mse == 0.0 { + f64::INFINITY + } else { + 10.0 * (255.0f64 * 255.0 / mse).log10() + } +} + +/// The format to hand a `codec` encoder, or `None` to skip it on this device. +fn rgb_format(context: &VideoContext, codec: Codec, test: &str) -> Option { + let format = converter_config().rgb_encode_input(context, codec); + if format.is_none() { + eprintln!("skipping {test} for {codec:?}: this device has no RGB input for it"); + } + format +} + +#[test] +#[ignore = "requires a Vulkan Video device with VK_VALVE_video_encode_rgb_conversion"] +fn rgb_input_matches_the_converter() -> Result<(), Box> { + let context = context(Codec::H264)?; + let Some(format) = rgb_format(&context, Codec::H264, "rgb_input_matches_the_converter") else { + return Ok(()); + }; + let shader = decode_all(&context, &encode_with_shader(&context, Codec::H264)?)?; + let hardware = decode_all(&context, &encode_rgb(&context, Codec::H264, format)?)?; + assert_eq!(hardware.len(), FRAMES as usize, "frame count"); + let psnrs: Vec = hardware + .iter() + .zip(&shader) + .map(|(h, s)| psnr(h, s)) + .collect(); + println!("hardware vs shader conversion, luma PSNR per frame: {psnrs:?}"); + // Both apply the same BT.709 matrix to the same frames, then the same + // encoder settings; what is left is rounding in the two conversions and + // what the encoder makes of it. + for (i, p) in psnrs.iter().enumerate() { + assert!(*p >= 40.0, "frame {i}: {p:.2} dB against the shader path"); + } + Ok(()) +} + +#[test] +#[ignore = "requires a Vulkan Video device with VK_VALVE_video_encode_rgb_conversion"] +fn rgb_input_encodes_every_codec() -> Result<(), Box> { + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + let context = match context(codec) { + Ok(context) => context, + Err(e) => { + eprintln!("{codec:?}: no encoder on this device ({e})"); + continue; + } + }; + let Some(format) = rgb_format(&context, codec, "rgb_input_encodes_every_codec") else { + continue; + }; + let stream = encode_rgb(&context, codec, format)?; + assert!(!stream.is_empty(), "{codec:?}: empty stream"); + if let Ok(dir) = std::env::var("PIXELFORGE_RGB_STREAM_DIR") { + std::fs::write(format!("{dir}/rgb.{codec:?}").to_lowercase(), &stream)?; + } + } + Ok(()) +} + +/// `supports_rgb_encode_input` must agree with what an encoder can actually +/// be created for, on every device: one that answers yes for something the +/// encoder then refuses sends a caller down a path that fails, and one that +/// answers no for something that works leaves the shader running for nothing. +#[test] +#[ignore = "requires a Vulkan Video device"] +fn rgb_query_matches_encoder_creation() -> Result<(), Box> { + common::init_logging(); + let context = VideoContextBuilder::new() + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .build()?; + let descriptions = [ + ("BT.709 limited", ColorDescription::bt709()), + ( + "BT.709 full", + ColorDescription::bt709().with_full_range(true), + ), + ("BT.2020", ColorDescription::bt2020_pq()), + ]; + for codec in [Codec::H264, Codec::H265, Codec::AV1] { + if !context.supports_encode(codec) { + continue; + } + for format in [ + InputFormat::BGRA, + InputFormat::RGBx, + InputFormat::ABGR2101010, + InputFormat::RGBA16F, + ] { + let bit_depth = if format == InputFormat::ABGR2101010 { + EncodeBitDepth::Ten + } else { + EncodeBitDepth::Eight + }; + for (name, desc) in &descriptions { + let supported = context.supports_rgb_encode_input(codec, format, desc); + let created = Encoder::new( + context.clone(), + encode_config(codec) + .with_bit_depth(bit_depth) + .with_color_description(*desc) + .with_rgb_input(format), + ) + .is_ok(); + eprintln!("{codec:?} {format:?} {name}: query={supported} encoder={created}"); + assert_eq!( + supported, created, + "{codec:?} {format:?} {name}: query and encoder creation disagree" + ); + } + } + } + Ok(()) +} From fba3d0672b29d85515ce7ee6cde69a5ebc405e3a Mon Sep 17 00:00:00 2001 From: DatCaptainHorse Date: Thu, 22 Oct 2026 13:53:59 +0300 Subject: [PATCH 3/3] docs: document RGB input converted by the encoder The crate docs point at the hardware RGB path from the colour conversion section, with an example of picking it or the shader per codec. README.md is regenerated from them. CHANGELOG lists the additions. AGENTS.md says which device the RGB tests mean anything on, and which two validation messages that path draws because the layer does not model the extension. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 11 +++++++++++ CHANGELOG.md | 10 ++++++++++ README.md | 20 +++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 44c312e..0fae7fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,6 +117,17 @@ checks the output against a context pixelforge created itself. It needs no ffmpeg. Anything that reaches a core 1.2 or 1.3 entry point panics there, so run it after touching submission or barrier code. +The RGB input path (`EncodeConfig::with_rgb_input`) needs +`VK_VALVE_video_encode_rgb_conversion`, which today only RADV has. +`tests/rgb_encode.rs` compares it against the colour converter, checks that +`supports_rgb_encode_input` agrees with encoder creation, and skips what the +device cannot do, so a green run elsewhere says little about it. Under +validation it draws two messages that are the layer's, not pixelforge's: the +layer does not model that extension, so it rejects the RGB profile struct in a +query pool's profile chain (the spec requires that chain to match the +session's) and checks the RGB input view against the plain format features +rather than the video profile's. + Make sure there are no Vulkan validation layer errors during execution. Enable them with `PIXELFORGE_VALIDATION=1`; the layer's messages are routed through `tracing`, so pair it with `RUST_LOG=warn` (or `debug` for the layer's own diff --git a/CHANGELOG.md b/CHANGELOG.md index 637102d..8f8166e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add RGB input converted by the encoder itself through + `VK_VALVE_video_encode_rgb_conversion`: `EncodeConfig::with_rgb_input`. +- Add `VideoContext::supports_rgb_encode_input` and + `ColorConverterConfig::rgb_encode_input`, which say when a conversion can be + left to the encoder. +- Add `VideoContext::has_video_encode_rgb_conversion` and + `DeviceFeatures::video_encode_rgb_conversion`. + ## [0.10.0] - 19-10-2026 ### Added diff --git a/README.md b/README.md index 448b44f..985d41a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ H.265 and AV1 encode, and H.264 decode. - **Asynchronous pipelines**: both directions submit without waiting. Encoding hands back an [`EncodeFuture`]; decoding delivers frames through a [`DecodeSource`] as the GPU finishes with them. -- **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ). +- **GPU color conversion**: RGB/BGR → YUV via Vulkan compute shaders (BT.709, BT.2020, sRGB→BT.2020+PQ, scRGB-linear→BT.2020+PQ), + or in the encoder itself where `VK_VALVE_video_encode_rgb_conversion` is available. - **Shared devices**: encode and decode on a Vulkan device your application created, from Vulkan 1.1 up, on queues you choose, ordered against your own work with timeline semaphores. @@ -250,6 +251,23 @@ let mut encoder = Encoder::new(context, encode_config)?; // converter.convert(input_image, layout, encoder.input_image())?; ``` +Where the conversion is nothing but the YUV matrix, as for ordinary SDR +content, some encoders can do it themselves and the shader need not run at +all. [`ColorConverterConfig::rgb_encode_input`] says whether that works for +a given conversion and codec on this device. When it does, build the +encoder with [`EncodeConfig::with_rgb_input`] and hand it the RGB frames. + +```rust +let encode_config = EncodeConfig::h264(1920, 1080) + .with_color_description(config.color_description().expect("SDR is encodable")); +let encoder = match config.rgb_encode_input(&context, Codec::H264) { + // The encoder converts: give it the RGB images as they are. + Some(format) => Encoder::new(context, encode_config.with_rgb_input(format))?, + // The shader converts into the encoder's input image, as above. + None => Encoder::new(context, encode_config)?, +}; +``` + ### Encoding on your own device An application that already renders with Vulkan can give pixelforge its own