ref: 80338b91d352d3a961d9cee44be29eae9f425804
parent: 0695843a216dd63e8ac087e13c44de81e8a50afe
author: Angie Chiang <angiebird@google.com>
date: Tue Sep 20 07:59:23 EDT 2016
Detect invalid highbd iht input Do nothing in vp9_highbd_iht#x#_##_add_c when input magnitude is beyond 20 bits. Note that, sign bit is not included here. In the 20 bits, we use 12 bits for input signal, 7 bits for forward transform amplification, and 1 bit for contingency in rounding and quantizing BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1286 Change-Id: I332c6f68df4614fc2e7d2dc4c5bb0d0cff8a245c
--- a/vp9/common/vp9_idct.c
+++ b/vp9/common/vp9_idct.c
@@ -204,6 +204,18 @@
}
#if CONFIG_VP9_HIGHBITDEPTH
+
+// 12 signal input bits + 7 forward transform amplify bits + 1 bit
+// for contingency in rounding and quantizing
+#define VALID_IHT_MAGNITUDE_RANGE (1 << 20)
+
+static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) {
+ int i;
+ for (i = 0; i < size; ++i)
+ if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1;
+ return 0;
+}
+
void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
int stride, int tx_type, int bd) {
const highbd_transform_2d IHT_4[] = {
@@ -219,6 +231,13 @@
tran_low_t *outptr = out;
tran_low_t temp_in[4], temp_out[4];
+ if (detect_invalid_iht_input(input, 16)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+ assert(0 && "invalid highbd iht input");
+#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
+ return;
+ }
+
// Inverse transform row vectors.
for (i = 0; i < 4; ++i) {
IHT_4[tx_type].rows(input, outptr, bd);
@@ -253,6 +272,13 @@
const highbd_transform_2d ht = HIGH_IHT_8[tx_type];
uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
+ if (detect_invalid_iht_input(input, 64)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+ assert(0 && "invalid highbd iht input");
+#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
+ return;
+ }
+
// Inverse transform row vectors.
for (i = 0; i < 8; ++i) {
ht.rows(input, outptr, bd);
@@ -286,6 +312,13 @@
tran_low_t temp_in[16], temp_out[16];
const highbd_transform_2d ht = HIGH_IHT_16[tx_type];
uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
+
+ if (detect_invalid_iht_input(input, 256)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+ assert(0 && "invalid highbd iht input");
+#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
+ return;
+ }
// Rows
for (i = 0; i < 16; ++i) {