shithub: libvpx

Info  •  Files  •  Log  •  Branches

Move highbd txfm input range check from 2d iht transform to 1d idct/iadst

This change will make the highbd txfm input range check more comprehensive

The 25-bit highbd input range is composed by
12 signal input bits + 7 bits for 2D forward transform amplification + 5 bits for
1D inverse transform amplification + 1 bit for contingency in rounding and quantizing

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1286
BUG=https://bugs.chromium.org/p/chromium/issues/detail?id=651625

Change-Id: I04c0796edd7653f8d463fba5dc418132986131e7

--- a/vp9/common/vp9_idct.c

+++ b/vp9/common/vp9_idct.c

@@ -203,17 +203,6 @@

 #if CONFIG_VP9_HIGHBITDEPTH

-// 12 signal input bits + 7 forward transform amplify bits + 1 bit

-// for contingency in rounding and quantizing

-#define VALID_IHT_MAGNITUDE_RANGE (1 << 20)

-

-static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) {

-  int i;

-  for (i = 0; i < size; ++i)

-    if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1;

-  return 0;

-}

-

 void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,

                                 int stride, int tx_type, int bd) {

   const highbd_transform_2d IHT_4[] = {

@@ -229,13 +218,6 @@

   tran_low_t *outptr = out;

   tran_low_t temp_in[4], temp_out[4];

-  if (detect_invalid_iht_input(input, 16)) {

-#if CONFIG_COEFFICIENT_RANGE_CHECKING

-    assert(0 && "invalid highbd iht input");

-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

-    return;

-  }

-

   // Inverse transform row vectors.

   for (i = 0; i < 4; ++i) {

     IHT_4[tx_type].rows(input, outptr, bd);

@@ -270,13 +252,6 @@

   const highbd_transform_2d ht = HIGH_IHT_8[tx_type];

   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);

-  if (detect_invalid_iht_input(input, 64)) {

-#if CONFIG_COEFFICIENT_RANGE_CHECKING

-    assert(0 && "invalid highbd iht input");

-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

-    return;

-  }

-

   // Inverse transform row vectors.

   for (i = 0; i < 8; ++i) {

     ht.rows(input, outptr, bd);

@@ -310,13 +285,6 @@

   tran_low_t temp_in[16], temp_out[16];

   const highbd_transform_2d ht = HIGH_IHT_16[tx_type];

   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);

-

-  if (detect_invalid_iht_input(input, 256)) {

-#if CONFIG_COEFFICIENT_RANGE_CHECKING

-    assert(0 && "invalid highbd iht input");

-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

-    return;

-  }

   // Rows

   for (i = 0; i < 16; ++i) {

--- a/vpx_dsp/inv_txfm.c

+++ b/vpx_dsp/inv_txfm.c

@@ -9,6 +9,7 @@

  */

 #include <math.h>

+#include <stdlib.h>

 #include <string.h>

 #include "./vpx_dsp_rtcd.h"

@@ -1252,6 +1253,19 @@

 }

 #if CONFIG_VP9_HIGHBITDEPTH

+

+// 12 signal input bits + 7 2D forward transform amplify bits + 5 1D inverse

+// transform amplify bits + 1 bit for contingency in rounding and quantizing

+#define HIGHBD_VALID_TXFM_MAGNITUDE_RANGE (1 << 25)

+

+static INLINE int detect_invalid_highbd_input(const tran_low_t *input,

+                                              int size) {

+  int i;

+  for (i = 0; i < size; ++i)

+    if (abs(input[i]) >= HIGHBD_VALID_TXFM_MAGNITUDE_RANGE) return 1;

+  return 0;

+}

+

 void vpx_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,

                                  int stride, int bd) {

   /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds,

@@ -1347,6 +1361,15 @@

   tran_low_t step[4];

   tran_high_t temp1, temp2;

   (void)bd;

+

+  if (detect_invalid_highbd_input(input, 4)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 4);

+    return;

+  }

+

   // stage 1

   temp1 = (input[0] + input[2]) * cospi_16_64;

   temp2 = (input[0] - input[2]) * cospi_16_64;

@@ -1413,6 +1436,15 @@

 void vpx_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd) {

   tran_low_t step1[8], step2[8];

   tran_high_t temp1, temp2;

+

+  if (detect_invalid_highbd_input(input, 8)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 8);

+    return;

+  }

+

   // stage 1

   step1[0] = input[0];

   step1[2] = input[4];

@@ -1498,7 +1530,6 @@

 void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) {

   tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;

-

   tran_low_t x0 = input[0];

   tran_low_t x1 = input[1];

   tran_low_t x2 = input[2];

@@ -1505,6 +1536,14 @@

   tran_low_t x3 = input[3];

   (void)bd;

+  if (detect_invalid_highbd_input(input, 4)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 4);

+    return;

+  }

+

   if (!(x0 | x1 | x2 | x3)) {

     memset(output, 0, 4 * sizeof(*output));

     return;

@@ -1536,7 +1575,6 @@

 void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) {

   tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;

-

   tran_low_t x0 = input[7];

   tran_low_t x1 = input[0];

   tran_low_t x2 = input[5];

@@ -1547,6 +1585,14 @@

   tran_low_t x7 = input[6];

   (void)bd;

+  if (detect_invalid_highbd_input(input, 8)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 8);

+    return;

+  }

+

   if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) {

     memset(output, 0, 8 * sizeof(*output));

     return;

@@ -1642,6 +1688,14 @@

   tran_high_t temp1, temp2;

   (void)bd;

+  if (detect_invalid_highbd_input(input, 16)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 16);

+    return;

+  }

+

   // stage 1

   step1[0] = input[0 / 2];

   step1[1] = input[16 / 2];

@@ -1832,7 +1886,6 @@

 void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) {

   tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8;

   tran_high_t s9, s10, s11, s12, s13, s14, s15;

-

   tran_low_t x0 = input[15];

   tran_low_t x1 = input[0];

   tran_low_t x2 = input[13];

@@ -1851,6 +1904,14 @@

   tran_low_t x15 = input[14];

   (void)bd;

+  if (detect_invalid_highbd_input(input, 16)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 16);

+    return;

+  }

+

   if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 |

         x13 | x14 | x15)) {

     memset(output, 0, 16 * sizeof(*output));

@@ -2047,6 +2108,14 @@

   tran_low_t step1[32], step2[32];

   tran_high_t temp1, temp2;

   (void)bd;

+

+  if (detect_invalid_highbd_input(input, 32)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd txfm input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    memset(output, 0, sizeof(*output) * 32);

+    return;

+  }

   // stage 1

   step1[0] = input[0];