From 444564d51992db65280f10dfa16265af363bf6a9 Mon Sep 17 00:00:00 2001 From: Lynne Date: Tue, 22 Sep 2026 01:19:31 +0900 Subject: [PATCH] vulkan/prores_ks_trellis_node: size the per-subgroup totals for the smallest subgroups The driver may run the shader with subgroups smaller than the maximum, and the extra subgroups then wrote past the array. --- libavcodec/proresenc_kostya_vulkan.c | 3 ++- libavcodec/vulkan/prores_ks_trellis_node.comp.glsl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libavcodec/proresenc_kostya_vulkan.c b/libavcodec/proresenc_kostya_vulkan.c index 1f95bc90e2..2668d040f2 100644 --- a/libavcodec/proresenc_kostya_vulkan.c +++ b/libavcodec/proresenc_kostya_vulkan.c @@ -293,7 +293,8 @@ static int init_trellis_node_pipeline(ProresVulkanContext *pv, FFVulkanShader* s int err = 0; FFVulkanContext *vkctx = &pv->vkctx; FFVulkanDescriptorSetBinding *desc; - int subgroup_size = vkctx->subgroup_props.maxSubgroupSize; + /* The driver picks the subgroup size; size for the most it may use */ + int subgroup_size = vkctx->subgroup_props.minSubgroupSize; int num_subgroups = FFALIGN(pv->ctx.mb_height, subgroup_size) / subgroup_size; SPEC_LIST_CREATE(sl, 8, 8 * sizeof(uint32_t)) diff --git a/libavcodec/vulkan/prores_ks_trellis_node.comp.glsl b/libavcodec/vulkan/prores_ks_trellis_node.comp.glsl index 9d8cd1819c..a7c6fb9bcd 100644 --- a/libavcodec/vulkan/prores_ks_trellis_node.comp.glsl +++ b/libavcodec/vulkan/prores_ks_trellis_node.comp.glsl @@ -172,7 +172,8 @@ void main() [[maximally_reconverges]] slice_row_size = force_slice_row_quants(); int subgroup_sum = subgroupAdd(slice_row_size); - subgroup_sizes[gl_SubgroupID] = subgroup_sum; + if (subgroupElect()) + subgroup_sizes[gl_SubgroupID] = subgroup_sum; barrier(); int buf_start = subgroupExclusiveAdd(slice_row_size);