shithub: libvpx

Info  •  Files  •  Log  •  Branches

Prevent integer overflow in choose_partitioning

Re-arrange the multiplication and right shift operations to avoid
integer overflow in choose_partitioning.

Change-Id: Ib4005cafb410a67c1960486471d75b6ebe38c4e0

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -611,12 +611,12 @@

       const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);

       if (bs == BLOCK_INVALID)

-        uv_sad = INT_MAX;

+        uv_sad = UINT_MAX;

       else

         uv_sad = cpi->fn_ptr[bs].sdf(p->src.buf, p->src.stride,

                                      pd->dst.buf, pd->dst.stride);

-      x->color_sensitivity[i - 1] = uv_sad * 4 > y_sad;

+      x->color_sensitivity[i - 1] = uv_sad > (y_sad >> 2);

     }

     d = xd->plane[0].dst.buf;