From 37ab751e94df9a988ef9995bb6b621ddb0113ec2 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Wed, 26 Nov 2025 16:04:37 +0530 Subject: [PATCH 01/16] encode: make H.264 level limits table readable This makes it easier to read and compare these limits against the data from table A-1 of the H.264 specification (from where these limits have been copied), and to add new entries if required in the future. Signed-off-by: Srinath Kumarapuram --- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index 6d8865a5..77ee1bd1 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -90,23 +90,23 @@ struct EncoderConfigH264 : public EncoderConfig { { // Level limits (Table A-1) static const LevelLimits levelLimitsTbl[] = { - // level_idc, maxMBPS, maxFS, maxDPB, maxBR, maxCPB, maxVmvR, prog, level - {10, 1485, 99, 148.5, 64, 175, 64, 1, STD_VIDEO_H264_LEVEL_IDC_1_0}, - {11, 3000, 396, 337.5, 192, 500, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_1}, - {12, 6000, 396, 891.0, 384, 1000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_2}, - {13, 11880, 396, 891.0, 768, 2000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_3}, - {20, 11880, 396, 891.0, 2000, 2000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_2_0}, - {21, 19800, 792, 1782.0, 4000, 4000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_2_1}, - {22, 20250, 1620, 3037.5, 4000, 4000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_2_2}, - {30, 40500, 1620, 3037.5, 10000, 10000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_3_0}, - {31, 108000, 3600, 6750.0, 14000, 14000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_3_1}, - {32, 216000, 5120, 7680.0, 20000, 20000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_3_2}, - {40, 245760, 8192, 12288.0, 20000, 25000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_0}, - {41, 245760, 8192, 12288.0, 50000, 62500, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_1}, - {42, 522240, 8704, 13056.0, 50000, 62500, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_2}, - {50, 589824, 22080, 41400.0, 135000, 135000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_0}, - {51, 983040, 36864, 69120.0, 240000, 240000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_1}, - {52, 2073600, 36864, 69120.0, 240000, 240000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_2}, + // level_idc, maxMBPS, maxFS, maxDPB, maxBR, maxCPB, maxVmvR, prog, level + {10, 1485, 99, 148.5, 64, 175, 64, 1, STD_VIDEO_H264_LEVEL_IDC_1_0}, + {11, 3000, 396, 337.5, 192, 500, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_1}, + {12, 6000, 396, 891.0, 384, 1000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_2}, + {13, 11880, 396, 891.0, 768, 2000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_3}, + {20, 11880, 396, 891.0, 2000, 2000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_2_0}, + {21, 19800, 792, 1782.0, 4000, 4000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_2_1}, + {22, 20250, 1620, 3037.5, 4000, 4000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_2_2}, + {30, 40500, 1620, 3037.5, 10000, 10000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_3_0}, + {31, 108000, 3600, 6750.0, 14000, 14000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_3_1}, + {32, 216000, 5120, 7680.0, 20000, 20000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_3_2}, + {40, 245760, 8192, 12288.0, 20000, 25000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_0}, + {41, 245760, 8192, 12288.0, 50000, 62500, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_1}, + {42, 522240, 8704, 13056.0, 50000, 62500, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_2}, + {50, 589824, 22080, 41400.0, 135000, 135000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_0}, + {51, 983040, 36864, 69120.0, 240000, 240000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_1}, + {52, 2073600, 36864, 69120.0, 240000, 240000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_2}, }; levelLimits = levelLimitsTbl; From 6e2dab3fdd0a20d04b1d1f09c9da4c037d59f792 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Wed, 26 Nov 2025 16:37:37 +0530 Subject: [PATCH 02/16] encode: remove "prog" column from H.264 level limits table The values in this column have been taken from table A-4 of the H.264 specification and specify the value to be used for the SPS frame_mbs_only_flag syntax element for a given level. These values are not used anywhere in the code as interlaced encoding is not supported (and therefore frame_mbs_only_flag = 1 always). Signed-off-by: Srinath Kumarapuram --- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index 77ee1bd1..e0e027cd 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -54,7 +54,6 @@ struct EncoderConfigH264 : public EncoderConfig { uint32_t maxBR; // 1200 bits/s uint32_t maxCPB; // 1200 bits uint32_t maxVmvR; // [-MaxVmvR..+MaxVmvR-0.25] - uint32_t prog; // frame_mbs_only_flag = 1 StdVideoH264LevelIdc level; }; @@ -90,23 +89,23 @@ struct EncoderConfigH264 : public EncoderConfig { { // Level limits (Table A-1) static const LevelLimits levelLimitsTbl[] = { - // level_idc, maxMBPS, maxFS, maxDPB, maxBR, maxCPB, maxVmvR, prog, level - {10, 1485, 99, 148.5, 64, 175, 64, 1, STD_VIDEO_H264_LEVEL_IDC_1_0}, - {11, 3000, 396, 337.5, 192, 500, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_1}, - {12, 6000, 396, 891.0, 384, 1000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_2}, - {13, 11880, 396, 891.0, 768, 2000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_1_3}, - {20, 11880, 396, 891.0, 2000, 2000, 128, 1, STD_VIDEO_H264_LEVEL_IDC_2_0}, - {21, 19800, 792, 1782.0, 4000, 4000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_2_1}, - {22, 20250, 1620, 3037.5, 4000, 4000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_2_2}, - {30, 40500, 1620, 3037.5, 10000, 10000, 256, 0, STD_VIDEO_H264_LEVEL_IDC_3_0}, - {31, 108000, 3600, 6750.0, 14000, 14000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_3_1}, - {32, 216000, 5120, 7680.0, 20000, 20000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_3_2}, - {40, 245760, 8192, 12288.0, 20000, 25000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_0}, - {41, 245760, 8192, 12288.0, 50000, 62500, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_1}, - {42, 522240, 8704, 13056.0, 50000, 62500, 512, 0, STD_VIDEO_H264_LEVEL_IDC_4_2}, - {50, 589824, 22080, 41400.0, 135000, 135000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_0}, - {51, 983040, 36864, 69120.0, 240000, 240000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_1}, - {52, 2073600, 36864, 69120.0, 240000, 240000, 512, 0, STD_VIDEO_H264_LEVEL_IDC_5_2}, + // level_idc, maxMBPS, maxFS, maxDPB, maxBR, maxCPB, maxVmvR, level + {10, 1485, 99, 148.5, 64, 175, 64, STD_VIDEO_H264_LEVEL_IDC_1_0}, + {11, 3000, 396, 337.5, 192, 500, 128, STD_VIDEO_H264_LEVEL_IDC_1_1}, + {12, 6000, 396, 891.0, 384, 1000, 128, STD_VIDEO_H264_LEVEL_IDC_1_2}, + {13, 11880, 396, 891.0, 768, 2000, 128, STD_VIDEO_H264_LEVEL_IDC_1_3}, + {20, 11880, 396, 891.0, 2000, 2000, 128, STD_VIDEO_H264_LEVEL_IDC_2_0}, + {21, 19800, 792, 1782.0, 4000, 4000, 256, STD_VIDEO_H264_LEVEL_IDC_2_1}, + {22, 20250, 1620, 3037.5, 4000, 4000, 256, STD_VIDEO_H264_LEVEL_IDC_2_2}, + {30, 40500, 1620, 3037.5, 10000, 10000, 256, STD_VIDEO_H264_LEVEL_IDC_3_0}, + {31, 108000, 3600, 6750.0, 14000, 14000, 512, STD_VIDEO_H264_LEVEL_IDC_3_1}, + {32, 216000, 5120, 7680.0, 20000, 20000, 512, STD_VIDEO_H264_LEVEL_IDC_3_2}, + {40, 245760, 8192, 12288.0, 20000, 25000, 512, STD_VIDEO_H264_LEVEL_IDC_4_0}, + {41, 245760, 8192, 12288.0, 50000, 62500, 512, STD_VIDEO_H264_LEVEL_IDC_4_1}, + {42, 522240, 8704, 13056.0, 50000, 62500, 512, STD_VIDEO_H264_LEVEL_IDC_4_2}, + {50, 589824, 22080, 41400.0, 135000, 135000, 512, STD_VIDEO_H264_LEVEL_IDC_5_0}, + {51, 983040, 36864, 69120.0, 240000, 240000, 512, STD_VIDEO_H264_LEVEL_IDC_5_1}, + {52, 2073600, 36864, 69120.0, 240000, 240000, 512, STD_VIDEO_H264_LEVEL_IDC_5_2}, }; levelLimits = levelLimitsTbl; From fe29120c296f426a2e14590655b7c6570e5ab607 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Wed, 26 Nov 2025 16:40:36 +0530 Subject: [PATCH 03/16] encode: add H.264 level limits for levels 6.0, 6.1, 6.2 These are defined in table A-1 of the H.264 specification but were missing from the level limits table defined in our code. These new entries also allow removing some workarounds where the level was previously artifically limited to level 5.2 as the maximum supported level. Signed-off-by: Srinath Kumarapuram --- .../libs/VkVideoEncoder/VkEncoderConfigH264.cpp | 16 +++------------- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 3 +++ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index e9c94bed..062998b6 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -505,28 +505,18 @@ int8_t EncoderConfigH264::InitDpbCount() assert(pic_width_in_mbs > 0); assert(pic_height_in_map_units > 0); - uint32_t frameSizeInMbs = pic_width_in_mbs * pic_height_in_map_units; double frameRate = ((frameRateNumerator > 0) && (frameRateDenominator > 0)) ? (double)frameRateNumerator / frameRateDenominator : (double)FRAME_RATE_NUM_DEFAULT / (double)FRAME_RATE_DEN_DEFAULT; - // WAR for super HD resolution (bypass H264 level check for frame mode and use level 5.2) - if ((frameSizeInMbs > ((uint32_t)levelLimits[levelLimitsSize - 1].maxFS) || - ((frameSizeInMbs * frameRate) > ((uint32_t)levelLimits[levelLimitsSize - 1].maxMBPS)))) { - levelIdc = STD_VIDEO_H264_LEVEL_IDC_5_2; - } else { - // find lowest possible level - levelIdc = DetermineLevel(dpbCount, levelBitRate, vbvBufferSize, frameRate); - } + // find lowest possible level + levelIdc = DetermineLevel(dpbCount, levelBitRate, vbvBufferSize, frameRate); uint8_t levelDpbSize = (uint8_t)(((1024 * levelLimits[levelIdc].maxDPB)) / ((pic_width_in_mbs * pic_height_in_map_units) * 384)); - // XXX: If the level is 5.2, it is highly likely that we forced it to that - // value as a WAR for super HD. In that case, force the DPB size to - // DEFAULT_MAX_NUM_REF_FRAMES. Otherwise, clamp the computed DPB size to DEFAULT_MAX_NUM_REF_FRAMES. - levelDpbSize = (levelIdc == STD_VIDEO_H264_LEVEL_IDC_5_2) ? (uint8_t)DEFAULT_MAX_NUM_REF_FRAMES : std::min(uint8_t(DEFAULT_MAX_NUM_REF_FRAMES), levelDpbSize); + levelDpbSize = std::min(uint8_t(DEFAULT_MAX_NUM_REF_FRAMES), levelDpbSize); uint8_t dpbSize = (uint8_t)((dpbCount < 1) ? levelDpbSize : (uint8_t)(std::min((uint8_t)dpbCount, levelDpbSize))) + 1; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index e0e027cd..f1e9d061 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -106,6 +106,9 @@ struct EncoderConfigH264 : public EncoderConfig { {50, 589824, 22080, 41400.0, 135000, 135000, 512, STD_VIDEO_H264_LEVEL_IDC_5_0}, {51, 983040, 36864, 69120.0, 240000, 240000, 512, STD_VIDEO_H264_LEVEL_IDC_5_1}, {52, 2073600, 36864, 69120.0, 240000, 240000, 512, STD_VIDEO_H264_LEVEL_IDC_5_2}, + {60, 4177920, 139264, 261120.0, 240000, 240000, 8192, STD_VIDEO_H264_LEVEL_IDC_6_0}, + {61, 8355840, 139264, 261120.0, 480000, 480000, 8192, STD_VIDEO_H264_LEVEL_IDC_6_1}, + {62, 16711680, 139264, 261120.0, 800000, 800000, 8192, STD_VIDEO_H264_LEVEL_IDC_6_2}, }; levelLimits = levelLimitsTbl; From a8bb6ac6034c0add43f93eb31ac2b60be779bf03 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 16 Feb 2026 16:36:53 +0530 Subject: [PATCH 04/16] EncoderConfigH264: Add InitProfileLevel for common profile and level init The codec profile and level are currently set in the following places: - level in EncoderConfigH264::InitDpbCount() - profile in EncoderConfigH264::InitSpsPpsParameters() The profile and level are fairly basic encoding parameters and it is desirable to set these as early as possible. This commit adds a method to the EncoderConfigH264 class which sets the profile and level at an early stage in the encoder configuration. This also provides the opportunity to use the codec profile in the construction of the video profile which will be used in physical device queries (currently, this video profile uses a default codec profile). This method gets called from EncoderConfigH264::InitializeParameters(), immediately after the command line parameters have been parsed and is part of the overall encoder config structure initialization process. The profile is more important than the level, so it is essential to set these values in this order, as done within this change. Also, certain level limits are profile-specific, and this ordering makes it possible for a subsequent change to use the profile in the level determination code. This commit also removes a check on the rate control mode when lossless encoding is requested, as the rate control mode may not have been specified by the user (and it will be determined later when querying the quality level properties). This change works as lossless encoding inherently requires the HIGH_444_PREDICTIVE profile regardless of rate control settings. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigH264.cpp | 83 ++++++++++--------- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 4 + 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 062998b6..35b408fa 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -310,11 +310,6 @@ bool EncoderConfigH264::InitSpsPpsParameters(StdVideoH264SequenceParameterSet *s sps->pic_order_cnt_type = STD_VIDEO_H264_POC_TYPE_2; } - // FIXME: Check if the HW supports transform_8x8_mode_is_supported - // based on capabilities or profiles supported - const bool transform_8x8_mode_is_supported = true; - const bool bIsFastestPreset = false; - if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_ENABLE) { pps->flags.transform_8x8_mode_flag = true; if ((profileIdc == STD_VIDEO_H264_PROFILE_IDC_BASELINE) || @@ -326,12 +321,9 @@ bool EncoderConfigH264::InitSpsPpsParameters(StdVideoH264SequenceParameterSet *s pps->flags.transform_8x8_mode_flag = false; } else { // Autoselect - if (!bIsFastestPreset || transform_8x8_mode_is_supported) { - if ((profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) || - (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH)) { - // Unconditionally enable 8x8 transform - pps->flags.transform_8x8_mode_flag = true; - } + if (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH) { + // Unconditionally enable 8x8 transform + pps->flags.transform_8x8_mode_flag = true; } } @@ -348,28 +340,6 @@ bool EncoderConfigH264::InitSpsPpsParameters(StdVideoH264SequenceParameterSet *s pps->flags.constrained_intra_pred_flag = true; } - // If the profileIdc hasn't been specified, force set it now. - if (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_BASELINE; - - if (entropyCodingMode == ENTROPY_CODING_MODE_CABAC) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_MAIN; - } - - if ((gopStructure.GetConsecutiveBFrameCount() > 0) || pps->flags.entropy_coding_mode_flag || !sps->flags.frame_mbs_only_flag) - profileIdc = STD_VIDEO_H264_PROFILE_IDC_MAIN; - - if (pps->flags.transform_8x8_mode_flag) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH; - } - - if ((sps->flags.qpprime_y_zero_transform_bypass_flag && - (rateControlMode == VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR)) || - (sps->chroma_format_idc == STD_VIDEO_H264_CHROMA_FORMAT_IDC_444)) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE; - } - } - sps->profile_idc = profileIdc; sps->level_idc = levelIdc; @@ -491,14 +461,44 @@ VkResult EncoderConfigH264::InitDeviceCapabilities(const VulkanDeviceContext* vk return VK_SUCCESS; } -int8_t EncoderConfigH264::InitDpbCount() +void EncoderConfigH264::InitProfileLevel() { - dpbCount = 0; // TODO: What is the need for this? + // FIXME: Check if the HW supports transform_8x8_mode_is_supported + // based on capabilities or profiles supported + bool use8x8Transform = true; + + if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_ENABLE) { + use8x8Transform = true; + } else if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_DISABLE) { + use8x8Transform = false; + } else { + // Autoselect + if ((profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) || + (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH)) { + // Unconditionally enable 8x8 transform + use8x8Transform = true; + } + } + + // If the profileIdc hasn't been specified, force set it now. + if (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) { + profileIdc = STD_VIDEO_H264_PROFILE_IDC_BASELINE; + + // Upgrade to MAIN profile if using B-frames or CABAC entropy coding + if ((gopStructure.GetConsecutiveBFrameCount() > 0) || (entropyCodingMode == ENTROPY_CODING_MODE_CABAC)) + profileIdc = STD_VIDEO_H264_PROFILE_IDC_MAIN; + + if (use8x8Transform) { + profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH; + } + + // Upgrade to HIGH_444_PREDICTIVE for lossless encoding or 4:4:4 chroma + if ((tuningMode == VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR) || + (input.chromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR)) { + profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE; + } + } - // spsInfo->level represents the smallest level that we require for the - // given stream. This level constrains the maximum size (in terms of - // number of frames) that the DPB can have. levelDpbSize is this maximum - // value. uint32_t levelBitRate = ((rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) && hrdBitrate == 0) ? averageBitrate // constrained by avg bitrate : hrdBitrate; // constrained by max bitrate @@ -512,6 +512,11 @@ int8_t EncoderConfigH264::InitDpbCount() // find lowest possible level levelIdc = DetermineLevel(dpbCount, levelBitRate, vbvBufferSize, frameRate); +} + +int8_t EncoderConfigH264::InitDpbCount() +{ + dpbCount = 0; // TODO: What is the need for this? uint8_t levelDpbSize = (uint8_t)(((1024 * levelLimits[levelIdc].maxDPB)) / ((pic_width_in_mbs * pic_height_in_map_units) * 384)); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index f1e9d061..4e692c21 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -180,6 +180,8 @@ struct EncoderConfigH264 : public EncoderConfig { pic_height_in_map_units = DivUp(encodeHeight, 16); if ((pic_width_in_mbs > 0) && (pic_height_in_map_units > 0)) { + // Initialize codec profile and level based on encoder configuration + InitProfileLevel(); return VK_SUCCESS; } @@ -187,6 +189,8 @@ struct EncoderConfigH264 : public EncoderConfig { return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; } + void InitProfileLevel(); + virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_H264_PROFILE_IDC_HIGH; }; From f341c6be4b270edc11ff8a5a61e2906977702fa2 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Tue, 25 Nov 2025 14:57:04 +0530 Subject: [PATCH 05/16] encode: update H.264 level limits based on the profile The level determination code in EncoderConfigH264::DetermineLevel() uses a multipler of 1200 for the MaxBR and MaxCPB entries in the level limits table, taken from table A-1 of the H.264 specification. However, this multipler is to be used only for the Baseline, Constrained Baseline, Main and Extended profiles. Profiles greater than these use a different (larger) multipler, as specified in table A-2 of the H.264 specification. Due to the use of a smaller multipler, this code selects a greater level than is strictly necessary in some situations, based on the input settings. This commit updates this code to use a multipler dependent on the codec profile. Signed-off-by: Srinath Kumarapuram --- .../libs/VkVideoEncoder/VkEncoderConfigH264.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 35b408fa..07a623e7 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -63,6 +63,13 @@ StdVideoH264LevelIdc EncoderConfigH264::DetermineLevel(uint8_t dpbSize, uint32_t _vbvBufferSize, double frameRate) { + uint32_t cpbBrNalFactor = 1200; + + // Adjust cpbBrNalFactor depending on whether the High (or greater) profile + // is being used + if (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH) { + cpbBrNalFactor = (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE) ? 4800 : 1500; + } uint32_t frameSizeInMbs = pic_width_in_mbs * pic_height_in_map_units; for (uint32_t idx = 0; idx < levelLimitsSize; idx++) { @@ -71,8 +78,8 @@ StdVideoH264LevelIdc EncoderConfigH264::DetermineLevel(uint8_t dpbSize, if ((frameSizeInMbs) > ((uint32_t)levelLimits[idx].maxFS)) continue; if ((frameSizeInMbs * numRefFrames * 384) > levelLimits[idx].maxDPB * 1024) continue; - if ((bitrate != 0) && (bitrate > ((uint32_t)levelLimits[idx].maxBR * 1200))) continue; - if ((_vbvBufferSize != 0) && (_vbvBufferSize > ((uint32_t)levelLimits[idx].maxCPB * 1200))) continue; + if ((bitrate != 0) && (bitrate > ((uint32_t)levelLimits[idx].maxBR * cpbBrNalFactor))) continue; + if ((_vbvBufferSize != 0) && (_vbvBufferSize > ((uint32_t)levelLimits[idx].maxCPB * cpbBrNalFactor))) continue; return levelLimits[idx].level; } From b02b0772043fa9db52b06024184f21323c28b175 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 16 Feb 2026 17:28:35 +0530 Subject: [PATCH 06/16] EncoderConfigH265: Add InitProfileLevel for common profile, level, tier init This commit adds an InitProfileLevel() method similar to what EncoderConfigH264 contains, for common initialization of the codec profile, level and tier. This makes use of the preexisting profile and levelIdc members defined in EncoderConfigH265 but not used otherwise. With this addition, the InitParamameters() and InitRateControl() methods have been changed to remove the redundant and repetitive initialization of the profile and level. The codec profile and level determined earlier in the encoder config initialization are now used directly instead. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigH265.cpp | 56 ++++++++++--------- .../libs/VkVideoEncoder/VkEncoderConfigH265.h | 10 +++- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp index 33bcc53e..89cfe7e6 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp @@ -513,15 +513,34 @@ StdVideoH265ProfileTierLevel EncoderConfigH265::GetLevelTier() return profileTierLevel; } -bool EncoderConfigH265::InitRateControl() +void EncoderConfigH265::InitProfileLevel() { - StdVideoH265ProfileTierLevel profileTierLevel = GetLevelTier(); - // Assigning default main profile if invalid. - if (profileTierLevel.general_profile_idc == STD_VIDEO_H265_PROFILE_IDC_INVALID) { - profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_MAIN; + // If profile hasn't been specified, determine it based on bit depth and chroma + if (profile == STD_VIDEO_H265_PROFILE_IDC_INVALID) { + if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR) { + if (input.bpp == 8) { + profile = STD_VIDEO_H265_PROFILE_IDC_MAIN; + } else if (input.bpp <= 10) { + profile = STD_VIDEO_H265_PROFILE_IDC_MAIN_10; + } else { + profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; + } + } else { + // 4:2:2 or 4:4:4 requires Range Extensions profile + profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; + } } - uint32_t level = profileTierLevel.general_level_idc; - if (level >= levelLimitsTblSize) { + + // Determine level and tier based on resolution, bitrate, etc. + StdVideoH265ProfileTierLevel profileTierLevel = GetLevelTier(); + levelIdc = profileTierLevel.general_level_idc; + general_tier_flag = profileTierLevel.flags.general_tier_flag; +} + +bool EncoderConfigH265::InitRateControl() +{ + // Level is already initialized by InitProfileLevel() + if (levelIdc >= levelLimitsTblSize) { assert(!"The h.265 level index is invalid"); return false; } @@ -529,7 +548,7 @@ bool EncoderConfigH265::InitRateControl() // Safe default maximum bitrate uint32_t levelBitRate = std::max(averageBitrate, hrdBitrate); - levelBitRate = std::max(levelBitRate, std::min(levelLimits[level].maxBitRateMainTier * 800u, 120000000u)); + levelBitRate = std::max(levelBitRate, std::min(levelLimits[levelIdc].maxBitRateMainTier * 800u, 120000000u)); // If no bitrate is specified, use the level limit if (averageBitrate == 0) { @@ -553,7 +572,7 @@ bool EncoderConfigH265::InitRateControl() // Use the main tier level limit for the max VBV buffer size, and no more than 8 seconds at peak rate if (vbvBufferSize == 0) { - vbvBufferSize = std::min(levelLimits[level].maxCPBSizeMainTier * cpbVclFactor, 100000000u); + vbvBufferSize = std::min(levelLimits[levelIdc].maxCPBSizeMainTier * cpbVclFactor, 100000000u); if (rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) { if ((vbvBufferSize >> 3) > hrdBitrate) { vbvBufferSize = hrdBitrate << 3; @@ -626,7 +645,9 @@ bool EncoderConfigH265::InitParamameters(VpsH265 *vpsInfo, SpsH265 *spsInfo, spsInfo->decPicBufMgr.max_num_reorder_pics[i] = gopStructure.GetConsecutiveBFrameCount() ? 1 : 0; } - spsInfo->profileTierLevel = GetLevelTier(); + // Use pre-initialized profile, level, and tier from InitProfileLevel() + spsInfo->profileTierLevel.general_profile_idc = profile; + spsInfo->profileTierLevel.general_level_idc = levelIdc; spsInfo->profileTierLevel.flags.general_tier_flag = general_tier_flag; // Always insert profile tier flags assuming frame mode @@ -636,21 +657,6 @@ bool EncoderConfigH265::InitParamameters(VpsH265 *vpsInfo, SpsH265 *spsInfo, spsInfo->profileTierLevel.flags.general_non_packed_constraint_flag = 0; spsInfo->profileTierLevel.flags.general_frame_only_constraint_flag = 1; - // Assigning default main profile if invalid. - if (spsInfo->profileTierLevel.general_profile_idc == STD_VIDEO_H265_PROFILE_IDC_INVALID) { - if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR) { - if (input.bpp == 8) { - spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_MAIN; - } else if (input.bpp == 10) { - spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_MAIN_10; - } else { - spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; - } - } else { - spsInfo->profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; - } - } - const uint32_t ctbLog2SizeY = cuSize + 3; const uint32_t minCbLog2SizeY = cuMinSize + 3; const uint32_t log2MinTransformBlockSize = std::min(minCbLog2SizeY, minTransformUnitSize + 2U); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index 774bf1a9..2149d88d 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -83,8 +83,8 @@ struct EncoderConfigH265 : public EncoderConfig { size_t levelLimitsTblSize; EncoderConfigH265() - : profile(STD_VIDEO_H265_PROFILE_IDC_MAIN) - , levelIdc(STD_VIDEO_H265_LEVEL_IDC_5_2) + : profile(STD_VIDEO_H265_PROFILE_IDC_INVALID) + , levelIdc(STD_VIDEO_H265_LEVEL_IDC_INVALID) , h265EncodeCapabilities() , general_tier_flag(false) , numRefL0(1) @@ -147,10 +147,16 @@ struct EncoderConfigH265 : public EncoderConfig { if (result != VK_SUCCESS) { return result; } + + // Initialize profile, level, and tier based on encoder configuration + InitProfileLevel(); + // TODO: more h.265 parameters init ... return VK_SUCCESS; } + void InitProfileLevel(); + virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_H265_PROFILE_IDC_MAIN; }; From b8e01efffd6319dfffeffc35cb25e05e24dfb51f Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Fri, 6 Feb 2026 12:12:16 +0530 Subject: [PATCH 07/16] EncoderConfigH265: rename GetLevelTier to DetermineLevelTier This change is not a plain renaming, as it also modifies the method to directly set the level and tier members in the class instead of initializing and returning a StdVideoH265ProfileTierLevel structure. The method name is also now consistent with that for the other codecs. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigH265.cpp | 21 +++++++------------ .../libs/VkVideoEncoder/VkEncoderConfigH265.h | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp index 89cfe7e6..8b041b9d 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp @@ -483,25 +483,24 @@ void EncoderConfigH265::InitializeSpsRefPicSet(SpsH265 *pSps) memset(&pSps->longTermRefPicsSps.lt_ref_pic_poc_lsb_sps, 0, sizeof(pSps->longTermRefPicsSps.lt_ref_pic_poc_lsb_sps)); } -StdVideoH265ProfileTierLevel EncoderConfigH265::GetLevelTier() +void EncoderConfigH265::DetermineLevelTier() { - StdVideoH265ProfileTierLevel profileTierLevel{}; - profileTierLevel.general_profile_idc = STD_VIDEO_H265_PROFILE_IDC_INVALID; - profileTierLevel.general_level_idc = STD_VIDEO_H265_LEVEL_IDC_INVALID; + levelIdc = STD_VIDEO_H265_LEVEL_IDC_INVALID; + general_tier_flag = 0; uint32_t levelIdx = 0; for (; levelIdx < levelLimitsTblSize; levelIdx++) { if (IsSuitableLevel(levelIdx, 0)) { - profileTierLevel.general_level_idc = levelLimits[levelIdx].stdLevel; - profileTierLevel.flags.general_tier_flag = 0; // Main tier + levelIdc = levelLimits[levelIdx].stdLevel; + general_tier_flag = 0; // Main tier break; } if ((levelLimits[levelIdx].levelIdc >= 120) && // level 4.0 and above IsSuitableLevel(levelIdx, 1)) { - profileTierLevel.general_level_idc = levelLimits[levelIdx].stdLevel; - profileTierLevel.flags.general_tier_flag = 1; // Main tier + levelIdc = levelLimits[levelIdx].stdLevel; + general_tier_flag = 1; // High tier break; } } @@ -509,8 +508,6 @@ StdVideoH265ProfileTierLevel EncoderConfigH265::GetLevelTier() if (levelIdx >= levelLimitsTblSize) { assert(!"No suitable level selected"); } - - return profileTierLevel; } void EncoderConfigH265::InitProfileLevel() @@ -532,9 +529,7 @@ void EncoderConfigH265::InitProfileLevel() } // Determine level and tier based on resolution, bitrate, etc. - StdVideoH265ProfileTierLevel profileTierLevel = GetLevelTier(); - levelIdc = profileTierLevel.general_level_idc; - general_tier_flag = profileTierLevel.flags.general_tier_flag; + DetermineLevelTier(); } bool EncoderConfigH265::InitRateControl() diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index 2149d88d..4eae0069 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -197,7 +197,7 @@ struct EncoderConfigH265 : public EncoderConfig { StdVideoH265SequenceParameterSetVui* vui = nullptr); bool IsSuitableLevel(uint32_t levelIdx, bool highTier); - StdVideoH265ProfileTierLevel GetLevelTier(); + void DetermineLevelTier(); void InitializeSpsRefPicSet(SpsH265 *pSps); }; From deaad4d4e6f78901e59796c245f6e2ad3414813c Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 16 Feb 2026 17:34:38 +0530 Subject: [PATCH 08/16] EncoderConfigAV1: Add InitProfileLevel for common profile, level, tier init This commit adds an InitProfileLevel() method similar to what EncoderConfigH264 and EncoderConfigH265 contain, for common initialization of the codec profile, level and tier. With this addition, the InitRateControl() method have been changed to remove the redundant and repetitive initialization of the level and tier. The codec profile and level determined earlier in the encoder config initialization are now used directly instead. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigAV1.cpp | 26 +++++++++++++++++-- .../libs/VkVideoEncoder/VkEncoderConfigAV1.h | 8 ++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp index c3ba67c1..24cee2ef 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp @@ -285,6 +285,29 @@ VkResult EncoderConfigAV1::InitDeviceCapabilities(const VulkanDeviceContext* vkD return VK_SUCCESS; } +void EncoderConfigAV1::InitProfileLevel() +{ + // If profile hasn't been specified, determine it based on bit depth and chroma + if (profile == STD_VIDEO_AV1_PROFILE_INVALID) { + // PROFESSIONAL is required for 12-bit or 422 + if ((input.bpp > 10) || + (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_422_BIT_KHR)) { + profile = STD_VIDEO_AV1_PROFILE_PROFESSIONAL; + } + // HIGH is required for 444 chroma + else if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) { + profile = STD_VIDEO_AV1_PROFILE_HIGH; + } + // MAIN supports 8-bit and 10-bit with 420 + else { + profile = STD_VIDEO_AV1_PROFILE_MAIN; + } + } + + // Determine level and tier based on encoder configuration + DetermineLevelTier(); +} + int8_t EncoderConfigAV1::InitDpbCount() { dpbCount = STD_VIDEO_AV1_NUM_REF_FRAMES + 1; // BUFFER_POOL_MAX_SIZE = Number of frames in buffer pool = 10 @@ -351,8 +374,7 @@ bool EncoderConfigAV1::DetermineLevelTier() bool EncoderConfigAV1::InitRateControl() { - DetermineLevelTier(); - + // Level and tier are already initialized by InitProfileLevel() // use level max values for now. Limit it to 120Mbits/sec uint32_t levelBitrate = std::min(GetLevelBitrate(level, tier), 120000000u); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h index 622977d6..54c9ce8e 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h @@ -102,6 +102,8 @@ struct EncoderConfigAV1 : public EncoderConfig { pic_height_in_sbs = DivUp(encodeHeight, 16); if ((pic_width_in_sbs > 0) && (pic_height_in_sbs > 0)) { + // Initialize profile, level, and tier based on encoder configuration + InitProfileLevel(); return VK_SUCCESS; } @@ -111,6 +113,8 @@ struct EncoderConfigAV1 : public EncoderConfig { virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; + void InitProfileLevel(); + virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_AV1_PROFILE_MAIN; } virtual int8_t InitDpbCount() override; @@ -185,8 +189,8 @@ struct EncoderConfigAV1 : public EncoderConfig { return ((encodeWidth * encodeHeight * picSizeProfileFactor) >> 3); } - StdVideoAV1Profile profile{ STD_VIDEO_AV1_PROFILE_MAIN }; - StdVideoAV1Level level{ STD_VIDEO_AV1_LEVEL_5_0 }; + StdVideoAV1Profile profile{ STD_VIDEO_AV1_PROFILE_INVALID }; + StdVideoAV1Level level{ STD_VIDEO_AV1_LEVEL_INVALID }; uint8_t tier{}; VkVideoEncodeAV1CapabilitiesKHR av1EncodeCapabilities{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_CAPABILITIES_KHR }; VkVideoEncodeAV1QualityLevelPropertiesKHR av1QualityLevelProperties{ VK_STRUCTURE_TYPE_VIDEO_ENCODE_AV1_QUALITY_LEVEL_PROPERTIES_KHR }; From 4fba129e2ced9bd4645c65e94ca508d9abacbd84 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 16 Feb 2026 17:43:42 +0530 Subject: [PATCH 09/16] encode: remove videoProfileIdc and GetDefaultVideoProfileIdc() Currently, the video profile constructed for querying capabilities or quality level properties uses the videoProfileIdc member from the encoder config for the codec profile, or calls GetDefaultVideoProfileIdc() if that member is not initialized. No code initializes videoProfileIdc to a proper value, so GetDefaultVideoProfileIdc() gets called always and returns a fixed value for each codec. However, this fixed codec profile may not support all of the coding tools requested by the user when encoding, or may even be incompatible with the codec profile actually needed for encoding. We already have code in the InitProfileLevel() method to determine the codec profile based on various encoding features (B-frames, CABAC, bit depth, chroma format, etc.). This change makes the video profile construction code use this codec profile, thereby eliminating redundant profile storage and computation. The main changes in this refactoring are: - Remove videoProfileIdc member variable from EncoderConfig base class - Replace GetDefaultVideoProfileIdc() virtual method with GetCodecProfile() - Add GetCodecProfile() implementations in H.264/H.265/AV1 that return the already-initialized codec-specific profile members - Update InitVideoProfile() to call GetCodecProfile() instead of using videoProfileIdc - Add an assertion in each GetCodecProfile() implementation to ensure that the profile is initialized With this change, the codec profile is determined once and used everywhere else in the codebase. Signed-off-by: Srinath Kumarapuram --- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp | 6 ++++-- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h | 5 ++--- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h | 5 ++++- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h | 5 ++++- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h | 5 ++++- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp index 9918d573..56d4d804 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp @@ -823,13 +823,15 @@ void EncoderConfig::InitVideoProfile() encodeBitDepthChroma = encodeBitDepthLuma; } + // Get the codec-specific profile (already set by InitProfileLevel) + uint32_t codecProfile = GetCodecProfile(); + // update the video profile videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( codec, encodeChromaSubsampling, GetComponentBitDepthFlagBits(encodeBitDepthLuma), GetComponentBitDepthFlagBits(encodeBitDepthChroma), - (videoProfileIdc != (uint32_t)-1) ? videoProfileIdc : - GetDefaultVideoProfileIdc(), + codecProfile, tuningMode); } diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h index 8ad4b161..35e0727f 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h @@ -685,7 +685,6 @@ struct EncoderConfig : public VkVideoRefCountBase { bool noDeviceFallback; VkVideoCodecOperationFlagBitsKHR codec; bool useDpbArray; - uint32_t videoProfileIdc; uint32_t numInputImages; EncoderInputImageParameters input; uint8_t encodeBitDepthLuma; @@ -793,7 +792,6 @@ struct EncoderConfig : public VkVideoRefCountBase { , noDeviceFallback(false) , codec(VK_VIDEO_CODEC_OPERATION_NONE_KHR) , useDpbArray(false) - , videoProfileIdc((uint32_t)-1) , numInputImages(DEFAULT_NUM_INPUT_IMAGES) , input() , encodeBitDepthLuma(0) @@ -961,7 +959,8 @@ struct EncoderConfig : public VkVideoRefCountBase { // These functions should be overwritten from the codec-specific classes virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; }; - virtual uint32_t GetDefaultVideoProfileIdc() { return 0; }; + // Returns the codec-specific profile identifier (must be set by InitProfileLevel first) + virtual uint32_t GetCodecProfile() = 0; virtual int8_t InitDpbCount() { return 16; }; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h index 54c9ce8e..58168822 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h @@ -115,7 +115,10 @@ struct EncoderConfigAV1 : public EncoderConfig { void InitProfileLevel(); - virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_AV1_PROFILE_MAIN; } + virtual uint32_t GetCodecProfile() override { + assert(profile != STD_VIDEO_AV1_PROFILE_INVALID); + return static_cast(profile); + } virtual int8_t InitDpbCount() override; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index 4e692c21..32546503 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -193,7 +193,10 @@ struct EncoderConfigH264 : public EncoderConfig { virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; - virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_H264_PROFILE_IDC_HIGH; }; + virtual uint32_t GetCodecProfile() override { + assert(profileIdc != STD_VIDEO_H264_PROFILE_IDC_INVALID); + return static_cast(profileIdc); + } // 1. First h.264 determine the number of the Dpb buffers required virtual int8_t InitDpbCount() override; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index 4eae0069..39fcf019 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -159,7 +159,10 @@ struct EncoderConfigH265 : public EncoderConfig { virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; - virtual uint32_t GetDefaultVideoProfileIdc() override { return STD_VIDEO_H265_PROFILE_IDC_MAIN; }; + virtual uint32_t GetCodecProfile() override { + assert(profile != STD_VIDEO_H265_PROFILE_IDC_INVALID); + return static_cast(profile); + } // 1. First h.265 determine the number of the Dpb buffers required virtual int8_t InitDpbCount() override; From 4ea5eca6f6256d5511ee25696ff6f1ff9579fefa Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Mon, 6 Jul 2026 13:40:31 +0000 Subject: [PATCH 10/16] VkEncoderConfig: defer level selection to after device capability query The level selection / determination code depends on certain parameters which are decided based on capabilities queried from the device, but is currently executed at a time when the capabilities have not yet been queried, so there is a potential for an incorrect level to be used when encoding. This change introduces a DetermineLevelTier() virtual method on the base EncoderConfig class, which is overridden by the existing codec-specific DetermineLevelTier() methods. InitProfileLevel() has been renamed to InitCodecProfile() to reflect that it now only selects the codec profile (H.264 baseline/main/high, H.265 main/main10/ range-extensions, AV1 main/high/professional). The new virtual method is called from VkVideoEncoder::InitEncoder() after InitDeviceCapabilities() succeeds. Signed-off-by: Srinath Kumarapuram --- .../libs/VkVideoEncoder/VkEncoderConfig.cpp | 2 +- .../libs/VkVideoEncoder/VkEncoderConfig.h | 4 +++- .../libs/VkVideoEncoder/VkEncoderConfigAV1.cpp | 11 +++-------- .../libs/VkVideoEncoder/VkEncoderConfigAV1.h | 8 ++++---- .../libs/VkVideoEncoder/VkEncoderConfigH264.cpp | 5 ++++- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 8 +++++--- .../libs/VkVideoEncoder/VkEncoderConfigH265.cpp | 9 +++------ .../libs/VkVideoEncoder/VkEncoderConfigH265.h | 8 ++++---- .../libs/VkVideoEncoder/VkVideoEncoder.cpp | 2 ++ 9 files changed, 29 insertions(+), 28 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp index 56d4d804..0efd8613 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp @@ -823,7 +823,7 @@ void EncoderConfig::InitVideoProfile() encodeBitDepthChroma = encodeBitDepthLuma; } - // Get the codec-specific profile (already set by InitProfileLevel) + // Get the codec-specific profile (already set by InitCodecProfile) uint32_t codecProfile = GetCodecProfile(); // update the video profile diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h index 35e0727f..1563c949 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h @@ -959,9 +959,11 @@ struct EncoderConfig : public VkVideoRefCountBase { // These functions should be overwritten from the codec-specific classes virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; }; - // Returns the codec-specific profile identifier (must be set by InitProfileLevel first) + // Returns the codec-specific profile identifier (must be set by InitCodecProfile first) virtual uint32_t GetCodecProfile() = 0; + virtual void DetermineLevelTier() = 0; + virtual int8_t InitDpbCount() { return 16; }; virtual bool InitRateControl(); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp index 24cee2ef..b4c9bc3d 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp @@ -285,7 +285,7 @@ VkResult EncoderConfigAV1::InitDeviceCapabilities(const VulkanDeviceContext* vkD return VK_SUCCESS; } -void EncoderConfigAV1::InitProfileLevel() +void EncoderConfigAV1::InitCodecProfile() { // If profile hasn't been specified, determine it based on bit depth and chroma if (profile == STD_VIDEO_AV1_PROFILE_INVALID) { @@ -303,9 +303,6 @@ void EncoderConfigAV1::InitProfileLevel() profile = STD_VIDEO_AV1_PROFILE_MAIN; } } - - // Determine level and tier based on encoder configuration - DetermineLevelTier(); } int8_t EncoderConfigAV1::InitDpbCount() @@ -347,7 +344,7 @@ bool EncoderConfigAV1::ValidateLevel(uint32_t lvl, uint32_t lvlTier) return true; } -bool EncoderConfigAV1::DetermineLevelTier() +void EncoderConfigAV1::DetermineLevelTier() { uint32_t lvl = STD_VIDEO_AV1_LEVEL_2_0; @@ -368,13 +365,11 @@ bool EncoderConfigAV1::DetermineLevelTier() level = STD_VIDEO_AV1_LEVEL_7_3; tier = 0; } - - return true; } bool EncoderConfigAV1::InitRateControl() { - // Level and tier are already initialized by InitProfileLevel() + // Level and tier are already initialized by DetermineLevelTier() // use level max values for now. Limit it to 120Mbits/sec uint32_t levelBitrate = std::min(GetLevelBitrate(level, tier), 120000000u); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h index 58168822..4dc52968 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h @@ -102,8 +102,8 @@ struct EncoderConfigAV1 : public EncoderConfig { pic_height_in_sbs = DivUp(encodeHeight, 16); if ((pic_width_in_sbs > 0) && (pic_height_in_sbs > 0)) { - // Initialize profile, level, and tier based on encoder configuration - InitProfileLevel(); + // Initialize profile based on encoder configuration + InitCodecProfile(); return VK_SUCCESS; } @@ -113,7 +113,7 @@ struct EncoderConfigAV1 : public EncoderConfig { virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; - void InitProfileLevel(); + void InitCodecProfile(); virtual uint32_t GetCodecProfile() override { assert(profile != STD_VIDEO_AV1_PROFILE_INVALID); @@ -145,7 +145,7 @@ struct EncoderConfigAV1 : public EncoderConfig { bool ValidateLevel(uint32_t lLevel, uint32_t lTier); - bool DetermineLevelTier(); + virtual void DetermineLevelTier() override; uint32_t GetLevelMaxBitrate(uint32_t lLevel, uint32_t lTier) { if (lLevel < STD_VIDEO_AV1_LEVEL_4_0) { diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 07a623e7..6ab4987a 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -468,7 +468,7 @@ VkResult EncoderConfigH264::InitDeviceCapabilities(const VulkanDeviceContext* vk return VK_SUCCESS; } -void EncoderConfigH264::InitProfileLevel() +void EncoderConfigH264::InitCodecProfile() { // FIXME: Check if the HW supports transform_8x8_mode_is_supported // based on capabilities or profiles supported @@ -505,7 +505,10 @@ void EncoderConfigH264::InitProfileLevel() profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE; } } +} +void EncoderConfigH264::DetermineLevelTier() +{ uint32_t levelBitRate = ((rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) && hrdBitrate == 0) ? averageBitrate // constrained by avg bitrate : hrdBitrate; // constrained by max bitrate diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index 32546503..a3207a1f 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -180,8 +180,8 @@ struct EncoderConfigH264 : public EncoderConfig { pic_height_in_map_units = DivUp(encodeHeight, 16); if ((pic_width_in_mbs > 0) && (pic_height_in_map_units > 0)) { - // Initialize codec profile and level based on encoder configuration - InitProfileLevel(); + // Initialize codec profile based on encoder configuration + InitCodecProfile(); return VK_SUCCESS; } @@ -189,7 +189,9 @@ struct EncoderConfigH264 : public EncoderConfig { return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; } - void InitProfileLevel(); + void InitCodecProfile(); + + virtual void DetermineLevelTier() override; virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp index 8b041b9d..ec47bc8e 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp @@ -510,7 +510,7 @@ void EncoderConfigH265::DetermineLevelTier() } } -void EncoderConfigH265::InitProfileLevel() +void EncoderConfigH265::InitCodecProfile() { // If profile hasn't been specified, determine it based on bit depth and chroma if (profile == STD_VIDEO_H265_PROFILE_IDC_INVALID) { @@ -527,14 +527,11 @@ void EncoderConfigH265::InitProfileLevel() profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; } } - - // Determine level and tier based on resolution, bitrate, etc. - DetermineLevelTier(); } bool EncoderConfigH265::InitRateControl() { - // Level is already initialized by InitProfileLevel() + // Level is already initialized by DetermineLevelTier() if (levelIdc >= levelLimitsTblSize) { assert(!"The h.265 level index is invalid"); return false; @@ -640,7 +637,7 @@ bool EncoderConfigH265::InitParamameters(VpsH265 *vpsInfo, SpsH265 *spsInfo, spsInfo->decPicBufMgr.max_num_reorder_pics[i] = gopStructure.GetConsecutiveBFrameCount() ? 1 : 0; } - // Use pre-initialized profile, level, and tier from InitProfileLevel() + // Use pre-initialized profile, level, and tier from InitCodecProfile() / DetermineLevelTier() spsInfo->profileTierLevel.general_profile_idc = profile; spsInfo->profileTierLevel.general_level_idc = levelIdc; spsInfo->profileTierLevel.flags.general_tier_flag = general_tier_flag; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index 39fcf019..f12274bb 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -148,14 +148,14 @@ struct EncoderConfigH265 : public EncoderConfig { return result; } - // Initialize profile, level, and tier based on encoder configuration - InitProfileLevel(); + // Initialize profile based on encoder configuration + InitCodecProfile(); // TODO: more h.265 parameters init ... return VK_SUCCESS; } - void InitProfileLevel(); + void InitCodecProfile(); virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; @@ -200,7 +200,7 @@ struct EncoderConfigH265 : public EncoderConfig { StdVideoH265SequenceParameterSetVui* vui = nullptr); bool IsSuitableLevel(uint32_t levelIdx, bool highTier); - void DetermineLevelTier(); + virtual void DetermineLevelTier() override; void InitializeSpsRefPicSet(SpsH265 *pSps); }; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp index 93209af9..417a100c 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp @@ -788,6 +788,8 @@ VkResult VkVideoEncoder::InitEncoder(VkSharedBaseObj& encoderConf return result; } + encoderConfig->DetermineLevelTier(); + if (encoderConfig->verbose) { const VkVideoCapabilitiesKHR& vidCaps = encoderConfig->videoCapabilities; const VkVideoEncodeCapabilitiesKHR& encCaps = encoderConfig->videoEncodeCapabilities; From affa0b3ef2d5492bb064e00b1e65ba708b01613f Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Tue, 7 Jul 2026 06:38:48 +0000 Subject: [PATCH 11/16] VkEncoderConfig: introduce InitVideoProfileCapabilities We currently have two separate initialization steps against the encoder config, one in InitVideoProfile() followed by the other in InitDeviceCapabilities(). InitVideoProfile() sets up the video profile for use in device capability queries and other places, and uses the codec profile already determined by InitCodecProfile(). To allow the codec profile selection / determination to also use device capabilities, this commit replaces InitVideoProfile() and InitDeviceCapabilities() by a single virtual InitVideoProfileCapabilities() which is overridden by each codec. The code for bit depth resolution and VkVideoCoreProfile construction present in InitVideoProfile() has now been inlined into the per-codec overrides for InitVideoProfileCapabilities(). This makes video profile construction more flexible and lays the groundwork for H.264-specific changes for building and querying against multiple video profiles. Signed-off-by: Srinath Kumarapuram --- .../libs/VkVideoEncoder/VkEncoderConfig.cpp | 22 ------------------- .../libs/VkVideoEncoder/VkEncoderConfig.h | 4 +--- .../VkVideoEncoder/VkEncoderConfigAV1.cpp | 13 ++++++++++- .../libs/VkVideoEncoder/VkEncoderConfigAV1.h | 2 +- .../VkVideoEncoder/VkEncoderConfigH264.cpp | 13 ++++++++++- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 2 +- .../VkVideoEncoder/VkEncoderConfigH265.cpp | 13 ++++++++++- .../libs/VkVideoEncoder/VkEncoderConfigH265.h | 2 +- .../libs/VkVideoEncoder/VkVideoEncoder.cpp | 9 +++----- 9 files changed, 43 insertions(+), 37 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp index 0efd8613..97feee85 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp @@ -813,28 +813,6 @@ VkResult EncoderConfig::CreateCodecConfig(int argc, const char *argv[], return VK_ERROR_INITIALIZATION_FAILED; } -void EncoderConfig::InitVideoProfile() -{ - if (encodeBitDepthLuma == 0) { - encodeBitDepthLuma = input.bpp; - } - - if (encodeBitDepthChroma == 0) { - encodeBitDepthChroma = encodeBitDepthLuma; - } - - // Get the codec-specific profile (already set by InitCodecProfile) - uint32_t codecProfile = GetCodecProfile(); - - // update the video profile - videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( - codec, encodeChromaSubsampling, - GetComponentBitDepthFlagBits(encodeBitDepthLuma), - GetComponentBitDepthFlagBits(encodeBitDepthChroma), - codecProfile, - tuningMode); -} - bool EncoderConfig::InitRateControl() { uint32_t levelBitRate = ((rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) && hrdBitrate == 0) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h index 1563c949..786f4c27 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h @@ -913,8 +913,6 @@ struct EncoderConfig : public VkVideoRefCountBase { // Factory Function static VkResult CreateCodecConfig(int argc, const char *argv[], VkSharedBaseObj& encoderConfig); - void InitVideoProfile(); - int ParseArguments(int argc, const char *argv[]); virtual int DoParseArguments(int argc, const char *argv[]) { @@ -957,7 +955,7 @@ struct EncoderConfig : public VkVideoRefCountBase { } // These functions should be overwritten from the codec-specific classes - virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; }; + virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) { return VK_ERROR_INITIALIZATION_FAILED; }; // Returns the codec-specific profile identifier (must be set by InitCodecProfile first) virtual uint32_t GetCodecProfile() = 0; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp index b4c9bc3d..fb04cfe9 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp @@ -187,8 +187,19 @@ bool EncoderConfigAV1::InitSequenceHeader(StdVideoAV1SequenceHeader *seqHdr, return true; } -VkResult EncoderConfigAV1::InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) +VkResult EncoderConfigAV1::InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) { + if (encodeBitDepthLuma == 0) + encodeBitDepthLuma = input.bpp; + if (encodeBitDepthChroma == 0) + encodeBitDepthChroma = encodeBitDepthLuma; + + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( + codec, encodeChromaSubsampling, + GetComponentBitDepthFlagBits(encodeBitDepthLuma), + GetComponentBitDepthFlagBits(encodeBitDepthChroma), + GetCodecProfile(), tuningMode); + VkResult result = VulkanVideoCapabilities::GetVideoEncodeCapabilities (vkDevCtx, videoCoreProfile, diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h index 4dc52968..7960ff62 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h @@ -111,7 +111,7 @@ struct EncoderConfigAV1 : public EncoderConfig { return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; } - virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; + virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; void InitCodecProfile(); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 6ab4987a..65ef633b 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -367,8 +367,19 @@ bool EncoderConfigH264::InitSpsPpsParameters(StdVideoH264SequenceParameterSet *s return true; } -VkResult EncoderConfigH264::InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) +VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) { + if (encodeBitDepthLuma == 0) + encodeBitDepthLuma = input.bpp; + if (encodeBitDepthChroma == 0) + encodeBitDepthChroma = encodeBitDepthLuma; + + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( + codec, encodeChromaSubsampling, + GetComponentBitDepthFlagBits(encodeBitDepthLuma), + GetComponentBitDepthFlagBits(encodeBitDepthChroma), + GetCodecProfile(), tuningMode); + VkResult result = VulkanVideoCapabilities::GetVideoEncodeCapabilities (vkDevCtx, videoCoreProfile, diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index a3207a1f..47e28755 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -193,7 +193,7 @@ struct EncoderConfigH264 : public EncoderConfig { virtual void DetermineLevelTier() override; - virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; + virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; virtual uint32_t GetCodecProfile() override { assert(profileIdc != STD_VIDEO_H264_PROFILE_IDC_INVALID); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp index ec47bc8e..af87485d 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp @@ -111,8 +111,19 @@ int EncoderConfigH265::DoParseArguments(int argc, const char* argv[]) return 0; } -VkResult EncoderConfigH265::InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) +VkResult EncoderConfigH265::InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) { + if (encodeBitDepthLuma == 0) + encodeBitDepthLuma = input.bpp; + if (encodeBitDepthChroma == 0) + encodeBitDepthChroma = encodeBitDepthLuma; + + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( + codec, encodeChromaSubsampling, + GetComponentBitDepthFlagBits(encodeBitDepthLuma), + GetComponentBitDepthFlagBits(encodeBitDepthChroma), + GetCodecProfile(), tuningMode); + VkResult result = VulkanVideoCapabilities::GetVideoEncodeCapabilities (vkDevCtx, videoCoreProfile, diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index f12274bb..52605550 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -157,7 +157,7 @@ struct EncoderConfigH265 : public EncoderConfig { void InitCodecProfile(); - virtual VkResult InitDeviceCapabilities(const VulkanDeviceContext* vkDevCtx) override; + virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; virtual uint32_t GetCodecProfile() override { assert(profile != STD_VIDEO_H265_PROFILE_IDC_INVALID); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp index 417a100c..6b0e8d92 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp @@ -779,12 +779,9 @@ VkResult VkVideoEncoder::InitEncoder(VkSharedBaseObj& encoderConf m_encoderConfig = encoderConfig; - // Update the video profile - encoderConfig->InitVideoProfile(); - - result = encoderConfig->InitDeviceCapabilities(m_vkDevCtx); + result = encoderConfig->InitVideoProfileCapabilities(m_vkDevCtx); if (result != VK_SUCCESS) { - std::cerr << "InitDeviceCapabilties failed" << std::endl; + std::cerr << "InitVideoProfileCapabilities failed" << std::endl; return result; } @@ -891,7 +888,7 @@ VkResult VkVideoEncoder::InitEncoder(VkSharedBaseObj& encoderConf // Reconfigure the gopStructure structure because the device may not support // specific GOP structure. For example it may not support B-frames. - // gopStructure.Init() should be called after encoderConfig->InitDeviceCapabilities(). + // gopStructure.Init() should be called after encoderConfig->InitVideoProfileCapabilities(). m_encoderConfig->gopStructure.Init(m_encoderConfig->numFrames); if (encoderConfig->GetMaxBFrameCount() < m_encoderConfig->gopStructure.GetConsecutiveBFrameCount()) { if (m_encoderConfig->verbose) { From 5daa818945dfa8efe8d286f3a9dc9b74b82ce686 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Tue, 7 Jul 2026 07:20:53 +0000 Subject: [PATCH 12/16] VkEncoderConfig: inline InitCodecProfile into InitVideoProfileCapabilities This commit removes the separate InitCodecProfile() helper from all three codecs by inlining its profile auto-selection logic directly into InitVideoProfileCapabilities(), keeping all profile/capability init in one place per codec. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigAV1.cpp | 37 +++++------ .../libs/VkVideoEncoder/VkEncoderConfigAV1.h | 12 ++-- .../VkVideoEncoder/VkEncoderConfigH264.cpp | 63 +++++++------------ .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 12 ++-- .../VkVideoEncoder/VkEncoderConfigH265.cpp | 34 +++++----- .../libs/VkVideoEncoder/VkEncoderConfigH265.h | 5 -- 6 files changed, 65 insertions(+), 98 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp index fb04cfe9..1a0799b8 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp @@ -194,6 +194,23 @@ VkResult EncoderConfigAV1::InitVideoProfileCapabilities(const VulkanDeviceContex if (encodeBitDepthChroma == 0) encodeBitDepthChroma = encodeBitDepthLuma; + // If profile hasn't been specified, determine it based on bit depth and chroma subsampling + if (profile == STD_VIDEO_AV1_PROFILE_INVALID) { + // PROFESSIONAL is required for 12-bit or 422 + if ((input.bpp > 10) || + (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_422_BIT_KHR)) { + profile = STD_VIDEO_AV1_PROFILE_PROFESSIONAL; + } + // HIGH is required for 444 chroma + else if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) { + profile = STD_VIDEO_AV1_PROFILE_HIGH; + } + // MAIN supports 8-bit and 10-bit with 420 + else { + profile = STD_VIDEO_AV1_PROFILE_MAIN; + } + } + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( codec, encodeChromaSubsampling, GetComponentBitDepthFlagBits(encodeBitDepthLuma), @@ -296,26 +313,6 @@ VkResult EncoderConfigAV1::InitVideoProfileCapabilities(const VulkanDeviceContex return VK_SUCCESS; } -void EncoderConfigAV1::InitCodecProfile() -{ - // If profile hasn't been specified, determine it based on bit depth and chroma - if (profile == STD_VIDEO_AV1_PROFILE_INVALID) { - // PROFESSIONAL is required for 12-bit or 422 - if ((input.bpp > 10) || - (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_422_BIT_KHR)) { - profile = STD_VIDEO_AV1_PROFILE_PROFESSIONAL; - } - // HIGH is required for 444 chroma - else if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR) { - profile = STD_VIDEO_AV1_PROFILE_HIGH; - } - // MAIN supports 8-bit and 10-bit with 420 - else { - profile = STD_VIDEO_AV1_PROFILE_MAIN; - } - } -} - int8_t EncoderConfigAV1::InitDpbCount() { dpbCount = STD_VIDEO_AV1_NUM_REF_FRAMES + 1; // BUFFER_POOL_MAX_SIZE = Number of frames in buffer pool = 10 diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h index 7960ff62..e98ea6c4 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h @@ -101,20 +101,16 @@ struct EncoderConfigAV1 : public EncoderConfig { pic_width_in_sbs = DivUp(encodeWidth, 64); pic_height_in_sbs = DivUp(encodeHeight, 16); - if ((pic_width_in_sbs > 0) && (pic_height_in_sbs > 0)) { - // Initialize profile based on encoder configuration - InitCodecProfile(); - return VK_SUCCESS; + if (pic_width_in_sbs == 0 || pic_height_in_sbs == 0) { + assert(!"Invalid pic_width_in_sbs and pic_height_in_sbs"); + return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; } - assert(!"Invalid pic_width_in_sbs and pic_height_in_sbs"); - return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; + return VK_SUCCESS; } virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; - void InitCodecProfile(); - virtual uint32_t GetCodecProfile() override { assert(profile != STD_VIDEO_AV1_PROFILE_INVALID); return static_cast(profile); diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 65ef633b..19bba859 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -374,6 +374,30 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte if (encodeBitDepthChroma == 0) encodeBitDepthChroma = encodeBitDepthLuma; + // If the profile was not specified by the user, select the highest profile + // required by the requested features. + if (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) { + // FIXME: Check if the HW supports transform_8x8_mode_is_supported + // based on capabilities or profiles supported + bool use8x8Transform = (adaptiveTransformMode != ADAPTIVE_TRANSFORM_DISABLE); + + profileIdc = STD_VIDEO_H264_PROFILE_IDC_BASELINE; + + // Upgrade to MAIN profile if using B-frames or CABAC entropy coding + if ((gopStructure.GetConsecutiveBFrameCount() > 0) || (entropyCodingMode == ENTROPY_CODING_MODE_CABAC)) + profileIdc = STD_VIDEO_H264_PROFILE_IDC_MAIN; + + if (use8x8Transform) { + profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH; + } + + // Upgrade to HIGH_444_PREDICTIVE for lossless encoding or 4:4:4 chroma + if ((tuningMode == VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR) || + (input.chromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR)) { + profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE; + } + } + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( codec, encodeChromaSubsampling, GetComponentBitDepthFlagBits(encodeBitDepthLuma), @@ -479,45 +503,6 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte return VK_SUCCESS; } -void EncoderConfigH264::InitCodecProfile() -{ - // FIXME: Check if the HW supports transform_8x8_mode_is_supported - // based on capabilities or profiles supported - bool use8x8Transform = true; - - if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_ENABLE) { - use8x8Transform = true; - } else if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_DISABLE) { - use8x8Transform = false; - } else { - // Autoselect - if ((profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) || - (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH)) { - // Unconditionally enable 8x8 transform - use8x8Transform = true; - } - } - - // If the profileIdc hasn't been specified, force set it now. - if (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_BASELINE; - - // Upgrade to MAIN profile if using B-frames or CABAC entropy coding - if ((gopStructure.GetConsecutiveBFrameCount() > 0) || (entropyCodingMode == ENTROPY_CODING_MODE_CABAC)) - profileIdc = STD_VIDEO_H264_PROFILE_IDC_MAIN; - - if (use8x8Transform) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH; - } - - // Upgrade to HIGH_444_PREDICTIVE for lossless encoding or 4:4:4 chroma - if ((tuningMode == VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR) || - (input.chromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR)) { - profileIdc = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE; - } - } -} - void EncoderConfigH264::DetermineLevelTier() { uint32_t levelBitRate = ((rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) && hrdBitrate == 0) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index 47e28755..fa8f96f3 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -179,18 +179,14 @@ struct EncoderConfigH264 : public EncoderConfig { pic_width_in_mbs = DivUp(encodeWidth, 16); pic_height_in_map_units = DivUp(encodeHeight, 16); - if ((pic_width_in_mbs > 0) && (pic_height_in_map_units > 0)) { - // Initialize codec profile based on encoder configuration - InitCodecProfile(); - return VK_SUCCESS; + if (pic_width_in_mbs == 0 || pic_height_in_map_units == 0) { + assert(!"Invalid pic_width_in_mbs and pic_height_in_map_units"); + return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; } - assert(!"Invalid pic_width_in_mbs and pic_height_in_map_units"); - return VK_ERROR_INVALID_VIDEO_STD_PARAMETERS_KHR; + return VK_SUCCESS; } - void InitCodecProfile(); - virtual void DetermineLevelTier() override; virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp index af87485d..1bdd3383 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp @@ -118,6 +118,22 @@ VkResult EncoderConfigH265::InitVideoProfileCapabilities(const VulkanDeviceConte if (encodeBitDepthChroma == 0) encodeBitDepthChroma = encodeBitDepthLuma; + // If profile hasn't been specified, determine it based on bit depth and chroma subsampling + if (profile == STD_VIDEO_H265_PROFILE_IDC_INVALID) { + if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR) { + if (input.bpp == 8) { + profile = STD_VIDEO_H265_PROFILE_IDC_MAIN; + } else if (input.bpp <= 10) { + profile = STD_VIDEO_H265_PROFILE_IDC_MAIN_10; + } else { + profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; + } + } else { + // 4:2:2 or 4:4:4 requires Range Extensions profile + profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; + } + } + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( codec, encodeChromaSubsampling, GetComponentBitDepthFlagBits(encodeBitDepthLuma), @@ -521,24 +537,6 @@ void EncoderConfigH265::DetermineLevelTier() } } -void EncoderConfigH265::InitCodecProfile() -{ - // If profile hasn't been specified, determine it based on bit depth and chroma - if (profile == STD_VIDEO_H265_PROFILE_IDC_INVALID) { - if (encodeChromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_420_BIT_KHR) { - if (input.bpp == 8) { - profile = STD_VIDEO_H265_PROFILE_IDC_MAIN; - } else if (input.bpp <= 10) { - profile = STD_VIDEO_H265_PROFILE_IDC_MAIN_10; - } else { - profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; - } - } else { - // 4:2:2 or 4:4:4 requires Range Extensions profile - profile = STD_VIDEO_H265_PROFILE_IDC_FORMAT_RANGE_EXTENSIONS; - } - } -} bool EncoderConfigH265::InitRateControl() { diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index 52605550..5159a2e5 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -148,15 +148,10 @@ struct EncoderConfigH265 : public EncoderConfig { return result; } - // Initialize profile based on encoder configuration - InitCodecProfile(); - // TODO: more h.265 parameters init ... return VK_SUCCESS; } - void InitCodecProfile(); - virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; virtual uint32_t GetCodecProfile() override { From 8292e27324ae8ad45397b68e08167b93738f2d5c Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Tue, 7 Jul 2026 16:54:28 +0530 Subject: [PATCH 13/16] EncoderConfigH264: add a profile-probing loop in InitVideoProfileCapabilities This helps the automatic profile selection logic select an appropriate profile based both on the requested features and the codec profiles supported by the device. The prior logic would prefer only the requested features, potentially configuring a codec profile which would not be supported by the device. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigH264.cpp | 87 ++++++++++++++----- .../libs/VkVideoEncoder/VkEncoderConfigH264.h | 3 + 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 19bba859..75b2b2d4 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -367,6 +367,16 @@ bool EncoderConfigH264::InitSpsPpsParameters(StdVideoH264SequenceParameterSet *s return true; } +StdVideoH264ProfileIdc EncoderConfigH264::GetNextLowerProfile(StdVideoH264ProfileIdc profile) +{ + switch (profile) { + case STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE: return STD_VIDEO_H264_PROFILE_IDC_HIGH; + case STD_VIDEO_H264_PROFILE_IDC_HIGH: return STD_VIDEO_H264_PROFILE_IDC_MAIN; + case STD_VIDEO_H264_PROFILE_IDC_MAIN: return STD_VIDEO_H264_PROFILE_IDC_BASELINE; + default: return STD_VIDEO_H264_PROFILE_IDC_INVALID; + } +} + VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) { if (encodeBitDepthLuma == 0) @@ -374,11 +384,13 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte if (encodeBitDepthChroma == 0) encodeBitDepthChroma = encodeBitDepthLuma; - // If the profile was not specified by the user, select the highest profile - // required by the requested features. - if (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID) { - // FIXME: Check if the HW supports transform_8x8_mode_is_supported - // based on capabilities or profiles supported + // Track whether the profile was user-specified or auto-selected. + const bool autoSelected = (profileIdc == STD_VIDEO_H264_PROFILE_IDC_INVALID); + + if (autoSelected) { + // Select the highest profile required for the requested features. + // 8x8 transform support is hardware-dependent; it is verified after + // the capability query and the mode is adjusted accordingly. bool use8x8Transform = (adaptiveTransformMode != ADAPTIVE_TRANSFORM_DISABLE); profileIdc = STD_VIDEO_H264_PROFILE_IDC_BASELINE; @@ -398,14 +410,29 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte } } - videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( - codec, encodeChromaSubsampling, - GetComponentBitDepthFlagBits(encodeBitDepthLuma), - GetComponentBitDepthFlagBits(encodeBitDepthChroma), - GetCodecProfile(), tuningMode); + // Determine the minimum profile required by the content format. + StdVideoH264ProfileIdc minProfile = STD_VIDEO_H264_PROFILE_IDC_BASELINE; + if ((tuningMode == VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR) || + (input.chromaSubsampling == VK_VIDEO_CHROMA_SUBSAMPLING_444_BIT_KHR)) { + minProfile = STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE; + } else if (gopStructure.GetConsecutiveBFrameCount() > 0) { + minProfile = STD_VIDEO_H264_PROFILE_IDC_MAIN; + } + + // Probe from the desired profile down to minProfile, stopping at the first + // profile the hardware accepts. If the profile was user-specified, a single + // probe is made and failure is reported without fallback. + VkResult result = VK_SUCCESS; + + for (;;) { + videoCoreProfile = VkVideoCoreProfile::CreateEncodeProfile( + codec, encodeChromaSubsampling, + GetComponentBitDepthFlagBits(encodeBitDepthLuma), + GetComponentBitDepthFlagBits(encodeBitDepthChroma), + GetCodecProfile(), tuningMode); - VkResult result = VulkanVideoCapabilities::GetVideoEncodeCapabilities + result = VulkanVideoCapabilities::GetVideoEncodeCapabilities (vkDevCtx, videoCoreProfile, videoCapabilities, videoEncodeCapabilities, @@ -413,17 +440,35 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte quantizationMapCapabilities, h264QuantizationMapCapabilities, intraRefreshCapabilities); - if (result != VK_SUCCESS) { - // Distinguish between "not supported" and "actual error" - if (IsVideoUnsupportedResult(result)) { - // Not supported by hardware/driver - return VK_ERROR_INCOMPATIBLE_DRIVER - std::cerr << "*** Video encode capabilities not supported by hardware/driver (" - << VKVS_STRINGIFY(result) << ") ***" << std::endl; + if (result == VK_SUCCESS) { + break; + } + + if (!IsVideoUnsupportedResult(result)) { + std::cerr << "*** Error getting video encode capabilities: " << VKVS_STRINGIFY(result) << " ***" << std::endl; + return result; + } + + if (!autoSelected || (profileIdc == minProfile)) { + std::cerr << "*** H.264 profile " << profileIdc << " is not supported by the hardware/driver ***" << std::endl; return VK_ERROR_INCOMPATIBLE_DRIVER; } - // Actual error (e.g., out of memory) - std::cerr << "*** Error getting video capabilities: " << VKVS_STRINGIFY(result) << " ***" << std::endl; - return result; + + StdVideoH264ProfileIdc nextProfile = GetNextLowerProfile(profileIdc); + if (verbose) { + std::cout << "H.264 profile " << profileIdc << " not supported, retrying with profile " << nextProfile << std::endl; + } + profileIdc = nextProfile; + } + + // For auto-selected profiles, refine feature flags based on actual hardware + // capabilities reported for the chosen profile. + if (autoSelected && (adaptiveTransformMode == ADAPTIVE_TRANSFORM_AUTOSELECT)) { + bool hwSupports8x8 = (h264EncodeCapabilities.stdSyntaxFlags & + VK_VIDEO_ENCODE_H264_STD_TRANSFORM_8X8_MODE_FLAG_SET_BIT_KHR) != 0; + if (!hwSupports8x8) { + adaptiveTransformMode = ADAPTIVE_TRANSFORM_DISABLE; + } } if (verboseMsg) { diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index fa8f96f3..a0884f70 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -165,6 +165,9 @@ struct EncoderConfigH264 : public EncoderConfig { uint32_t vbvBufferSize, double frameRate); + // Returns the next lower profile in the H.264 hierarchy, or INVALID at the bottom. + static StdVideoH264ProfileIdc GetNextLowerProfile(StdVideoH264ProfileIdc profile); + static void SetAspectRatio(StdVideoH264SequenceParameterSetVui *vui, int32_t width, int32_t height, int32_t darWidth, int32_t darHeight); From d8fcc9c3fdd1ae83b92c5611fcd56392584f4041 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Wed, 8 Jul 2026 14:33:50 +0530 Subject: [PATCH 14/16] EncoderConfigH264: improve cpbBrNalFactor comment in DetermineLevel Signed-off-by: Srinath Kumarapuram --- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 75b2b2d4..045e8e56 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -65,8 +65,8 @@ StdVideoH264LevelIdc EncoderConfigH264::DetermineLevel(uint8_t dpbSize, { uint32_t cpbBrNalFactor = 1200; - // Adjust cpbBrNalFactor depending on whether the High (or greater) profile - // is being used + // Set cpbBrNalFactor depending on whether the High (or greater) profile is + // being used. See table A-2 in the H.264 specification. if (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH) { cpbBrNalFactor = (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH_444_PREDICTIVE) ? 4800 : 1500; } From 0fe29b346c1b1ff7bb854a58fc64703519a585e0 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Wed, 8 Jul 2026 14:33:44 +0530 Subject: [PATCH 15/16] encode: change DetermineLevelTier to return a boolean DetermineLevelTier() previously returned void with no way to signal failure (i.e. inability to select an appropriate level) to the caller. It has now been changed it to return false when no suitable level is found for the given encoding parameters, and true otherwise. Correspondingly, the caller of this method now checks the return value; in case of failure, an error message is printed and the failure is propagated up. In addition, the H.264- and H.265-specific implementations of DetermineLevelTier() validate the selected level against the maxLevelIdc value reported in the encoder capabilities. Signed-off-by: Srinath Kumarapuram --- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h | 2 +- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp | 6 +++--- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h | 2 +- .../libs/VkVideoEncoder/VkEncoderConfigH264.cpp | 6 +++++- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h | 2 +- .../libs/VkVideoEncoder/VkEncoderConfigH265.cpp | 5 +++-- vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h | 2 +- vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp | 5 ++++- 8 files changed, 19 insertions(+), 11 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h index 786f4c27..a4939ecc 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h @@ -960,7 +960,7 @@ struct EncoderConfig : public VkVideoRefCountBase { // Returns the codec-specific profile identifier (must be set by InitCodecProfile first) virtual uint32_t GetCodecProfile() = 0; - virtual void DetermineLevelTier() = 0; + virtual bool DetermineLevelTier() = 0; virtual int8_t InitDpbCount() { return 16; }; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp index 1a0799b8..fade4076 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.cpp @@ -352,7 +352,7 @@ bool EncoderConfigAV1::ValidateLevel(uint32_t lvl, uint32_t lvlTier) return true; } -void EncoderConfigAV1::DetermineLevelTier() +bool EncoderConfigAV1::DetermineLevelTier() { uint32_t lvl = STD_VIDEO_AV1_LEVEL_2_0; @@ -370,9 +370,9 @@ void EncoderConfigAV1::DetermineLevelTier() } } if (lvl > STD_VIDEO_AV1_LEVEL_7_3) { - level = STD_VIDEO_AV1_LEVEL_7_3; - tier = 0; + return false; } + return true; } bool EncoderConfigAV1::InitRateControl() diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h index e98ea6c4..01f8b383 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigAV1.h @@ -141,7 +141,7 @@ struct EncoderConfigAV1 : public EncoderConfig { bool ValidateLevel(uint32_t lLevel, uint32_t lTier); - virtual void DetermineLevelTier() override; + virtual bool DetermineLevelTier() override; uint32_t GetLevelMaxBitrate(uint32_t lLevel, uint32_t lTier) { if (lLevel < STD_VIDEO_AV1_LEVEL_4_0) { diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 045e8e56..8ee36df7 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -548,7 +548,7 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte return VK_SUCCESS; } -void EncoderConfigH264::DetermineLevelTier() +bool EncoderConfigH264::DetermineLevelTier() { uint32_t levelBitRate = ((rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) && hrdBitrate == 0) ? averageBitrate // constrained by avg bitrate @@ -563,6 +563,10 @@ void EncoderConfigH264::DetermineLevelTier() // find lowest possible level levelIdc = DetermineLevel(dpbCount, levelBitRate, vbvBufferSize, frameRate); + if (levelIdc == STD_VIDEO_H264_LEVEL_IDC_INVALID) { + return false; + } + return (levelIdc <= h264EncodeCapabilities.maxLevelIdc); } int8_t EncoderConfigH264::InitDpbCount() diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h index a0884f70..ed41c527 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.h @@ -190,7 +190,7 @@ struct EncoderConfigH264 : public EncoderConfig { return VK_SUCCESS; } - virtual void DetermineLevelTier() override; + virtual bool DetermineLevelTier() override; virtual VkResult InitVideoProfileCapabilities(const VulkanDeviceContext* vkDevCtx) override; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp index 1bdd3383..38e688f6 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.cpp @@ -510,7 +510,7 @@ void EncoderConfigH265::InitializeSpsRefPicSet(SpsH265 *pSps) memset(&pSps->longTermRefPicsSps.lt_ref_pic_poc_lsb_sps, 0, sizeof(pSps->longTermRefPicsSps.lt_ref_pic_poc_lsb_sps)); } -void EncoderConfigH265::DetermineLevelTier() +bool EncoderConfigH265::DetermineLevelTier() { levelIdc = STD_VIDEO_H265_LEVEL_IDC_INVALID; general_tier_flag = 0; @@ -533,8 +533,9 @@ void EncoderConfigH265::DetermineLevelTier() } if (levelIdx >= levelLimitsTblSize) { - assert(!"No suitable level selected"); + return false; } + return (levelIdc <= h265EncodeCapabilities.maxLevelIdc); } diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h index 5159a2e5..8d6a3c7d 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH265.h @@ -195,7 +195,7 @@ struct EncoderConfigH265 : public EncoderConfig { StdVideoH265SequenceParameterSetVui* vui = nullptr); bool IsSuitableLevel(uint32_t levelIdx, bool highTier); - virtual void DetermineLevelTier() override; + virtual bool DetermineLevelTier() override; void InitializeSpsRefPicSet(SpsH265 *pSps); }; diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp index 6b0e8d92..cfeabc4c 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp @@ -785,7 +785,10 @@ VkResult VkVideoEncoder::InitEncoder(VkSharedBaseObj& encoderConf return result; } - encoderConfig->DetermineLevelTier(); + if (!encoderConfig->DetermineLevelTier()) { + std::cerr << "Failed to determine a suitable level for the given encoding parameters" << std::endl; + return VK_ERROR_INITIALIZATION_FAILED; + } if (encoderConfig->verbose) { const VkVideoCapabilitiesKHR& vidCaps = encoderConfig->videoCapabilities; From e9599e95ef9a2d4a7b667074f4bc680772e1e477 Mon Sep 17 00:00:00 2001 From: Srinath Kumarapuram Date: Wed, 8 Jul 2026 15:34:21 +0530 Subject: [PATCH 16/16] EncoderConfigH264: simplify transform_8x8_mode_flag handling With this change, the adaptiveTransformMode parameter is either ENABLE or DISABLE at the end of InitVideoProfileCapabilities(), based on hardware support for transform_8x8_mode and the final determined profile. This removes the need for a three-way branch in InitSpsPpsParameters() and simplified code for setting this flag. Signed-off-by: Srinath Kumarapuram --- .../VkVideoEncoder/VkEncoderConfigH264.cpp | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 8ee36df7..ad463762 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -318,20 +318,11 @@ bool EncoderConfigH264::InitSpsPpsParameters(StdVideoH264SequenceParameterSet *s } if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_ENABLE) { - pps->flags.transform_8x8_mode_flag = true; - if ((profileIdc == STD_VIDEO_H264_PROFILE_IDC_BASELINE) || - (profileIdc == STD_VIDEO_H264_PROFILE_IDC_MAIN)) { - fprintf(stderr, "The profile selected does not support transform_8x8_mode_flag, disabling it.\n"); - pps->flags.transform_8x8_mode_flag = false; - } - } else if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_DISABLE) { - pps->flags.transform_8x8_mode_flag = false; + pps->flags.transform_8x8_mode_flag = (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH); + if (!pps->flags.transform_8x8_mode_flag) + fprintf(stderr, "The transform_8x8_mode_flag has been disabled because the selected profile selected does not support it.\n"); } else { - // Autoselect - if (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH) { - // Unconditionally enable 8x8 transform - pps->flags.transform_8x8_mode_flag = true; - } + pps->flags.transform_8x8_mode_flag = false; } if (entropyCodingMode == ENTROPY_CODING_MODE_CABAC) { @@ -461,14 +452,15 @@ VkResult EncoderConfigH264::InitVideoProfileCapabilities(const VulkanDeviceConte profileIdc = nextProfile; } - // For auto-selected profiles, refine feature flags based on actual hardware - // capabilities reported for the chosen profile. - if (autoSelected && (adaptiveTransformMode == ADAPTIVE_TRANSFORM_AUTOSELECT)) { + // If adaptiveTransformMode is AUTOSELECT, set it to either ENABLE or DISABLE + // based on hardware capabilities and the chosen profile, so that subsequent + // code using this parameter can take a simple binary decision. + if (adaptiveTransformMode == ADAPTIVE_TRANSFORM_AUTOSELECT) { bool hwSupports8x8 = (h264EncodeCapabilities.stdSyntaxFlags & VK_VIDEO_ENCODE_H264_STD_TRANSFORM_8X8_MODE_FLAG_SET_BIT_KHR) != 0; - if (!hwSupports8x8) { - adaptiveTransformMode = ADAPTIVE_TRANSFORM_DISABLE; - } + adaptiveTransformMode = (hwSupports8x8 && (profileIdc >= STD_VIDEO_H264_PROFILE_IDC_HIGH)) + ? ADAPTIVE_TRANSFORM_ENABLE + : ADAPTIVE_TRANSFORM_DISABLE; } if (verboseMsg) {