shithub: libvpx

Info  •  Files  •  Log  •  Branches

partial_idct_test,InitInput: fix rollover in mult

promote coeff to signed 64-bit to avoid exceeding integer bounds when
squaring the value

Change-Id: If77bef6bc0a6a4c39ca3013e5e2ddb426a1c6e1f

--- a/test/partial_idct_test.cc

+++ b/test/partial_idct_test.cc

@@ -133,7 +133,7 @@

     for (int j = 0; j < last_nonzero_; ++j) {

       tran_low_t coeff = static_cast<tran_low_t>(

           sqrt(1.0 * max_energy_leftover) * (rnd_.Rand16() - 32768) / 65536);

-      max_energy_leftover -= coeff * coeff;

+      max_energy_leftover -= static_cast<int64_t>(coeff) * coeff;

       if (max_energy_leftover < 0) {

         max_energy_leftover = 0;

         coeff = 0;