ref: 37102e55ec3614b3a2509ac57d9faf8f14e17300
parent: 4e2cfb63de974939b72d990005b36d8c2fac8352
author: Hien Ho <hienho@google.com>
date: Tue Oct 1 11:50:29 EDT 2019
vpx_dsp/quantize: fix int sanitizer warnings From unit test: AVX/VP9QuantizeTest; SSSE3/VP9QuantizeTest ... implicit conversion from type 'int' of value -139812 (32-bit, signed) to type 'tran_low_t' (aka 'short') changed the value to -8740 (16-bit, signed) BUG=webm:1615 Change-Id: I730946ac6c7a250dcbcfd8a2712c0f1150ddb4fd
--- a/vpx_dsp/quantize.c
+++ b/vpx_dsp/quantize.c
@@ -156,7 +156,7 @@
quant_shift_ptr[rc != 0]) >>
16; // quantization
qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
- dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
+ dqcoeff_ptr[rc] = (tran_low_t)(qcoeff_ptr[rc] * dequant_ptr[rc != 0]);
if (tmp) eob = i;
}