shithub: libvpx

Info  •  Files  •  Log  •  Branches

Add validation of UV partition size

For color sampling format other than 420, valid partion size in Y may
not work for UV plane. This commit adds validation of UV partition
size before select the partition choice.

This fixes a crash for real time encoding of 422 input.

Change-Id: I1fe3282accfd58625e8b5e6a4c8d2c84199751b6

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -432,11 +432,12 @@

     // Check vertical split.

     if (mi_row + block_height / 2 < cm->mi_rows) {

+      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_VERT);

       get_variance(&vt.part_variances->vert[0]);

       get_variance(&vt.part_variances->vert[1]);

       if (vt.part_variances->vert[0].variance < threshold &&

-          vt.part_variances->vert[1].variance < threshold) {

-        BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_VERT);

+          vt.part_variances->vert[1].variance < threshold &&

+          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {

         set_block_size(cpi, xd, mi_row, mi_col, subsize);

         set_block_size(cpi, xd, mi_row, mi_col + block_width / 2, subsize);

         return 1;

@@ -444,11 +445,12 @@

     }

     // Check horizontal split.

     if (mi_col + block_width / 2 < cm->mi_cols) {

+      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_HORZ);

       get_variance(&vt.part_variances->horz[0]);

       get_variance(&vt.part_variances->horz[1]);

       if (vt.part_variances->horz[0].variance < threshold &&

-          vt.part_variances->horz[1].variance < threshold) {

-        BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_HORZ);

+          vt.part_variances->horz[1].variance < threshold &&

+          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {

         set_block_size(cpi, xd, mi_row, mi_col, subsize);

         set_block_size(cpi, xd, mi_row + block_height / 2, mi_col, subsize);

         return 1;