shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix intermediate height in convolve

intermediate_height for horizontal filtering must be at least 8
pixels to be able to do vertical filtering correctly. Currently
it can be less for small block and y_step_q4 sizes.

Change-Id: I2ee28b0591b2041c2fa9844d0ae2ff8a1a59cc21

--- a/vp9/common/vp9_convolve.c

+++ b/vp9/common/vp9_convolve.c

@@ -217,12 +217,13 @@

    * h == 64, taps == 8.

    */

   uint8_t temp[64 * 135];

-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;

+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;

   assert(w <= 64);

   assert(h <= 64);

   assert(taps <= 8);

   assert(y_step_q4 <= 32);

+  assert(x_step_q4 <= 32);

   if (intermediate_height < h)

     intermediate_height = h;

@@ -246,12 +247,13 @@

    * h == 64, taps == 8.

    */

   uint8_t temp[64 * 135];

-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;

+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;

   assert(w <= 64);

   assert(h <= 64);

   assert(taps <= 8);

   assert(y_step_q4 <= 32);

+  assert(x_step_q4 <= 32);

   if (intermediate_height < h)

     intermediate_height = h;