shithub: libvpx

Info  •  Files  •  Log

Avoid overflow in calc_iframe_target_size

The changed product was observed to attempt to multiply 1800 by 2500000,
which overflows unsigned 32 bits. Converting to unsigned 64 bits first
and testing whether the final result fits in 32 bits solves the problem.

BUG=b:179686142

Change-Id: I5d27317bf14b0311b739144c451d8e172db01945

--- a/vp8/encoder/ratectrl.c

+++ b/vp8/encoder/ratectrl.c

@@ -349,8 +349,12 @@

   }

   if (cpi->oxcf.rc_max_intra_bitrate_pct) {

-    unsigned int max_rate =

-        cpi->per_frame_bandwidth * cpi->oxcf.rc_max_intra_bitrate_pct / 100;

+    unsigned int max_rate;

+    // This product may overflow unsigned int

+    uint64_t product = cpi->per_frame_bandwidth;

+    product *= cpi->oxcf.rc_max_intra_bitrate_pct;

+    product /= 100;

+    max_rate = (unsigned int)VPXMIN(INT_MAX, product);

     if (target > max_rate) target = max_rate;

   }