shithub: libvpx

Info  •  Files  •  Log  •  Branches

vp8: clamp probability values

fixes out of bounds reads on vp8_prob_cost table.
present since:
217591f Added rate-targeted temporal scalability
v0.9.7-p1-71-g217591f

Change-Id: I9194b773098a381f1e3f67bd3307f05df27c24fd

--- a/vp8/encoder/bitstream.c

+++ b/vp8/encoder/bitstream.c

@@ -980,6 +980,12 @@

                               int prob_garf

                              )

 {

+    assert(prob_intra >= 0);

+    assert(prob_intra <= 255);

+    assert(prob_last >= 0);

+    assert(prob_last <= 255);

+    assert(prob_garf >= 0);

+    assert(prob_garf <= 255);

     ref_frame_cost[INTRA_FRAME]   = vp8_cost_zero(prob_intra);

     ref_frame_cost[LAST_FRAME]    = vp8_cost_one(prob_intra)

                                     + vp8_cost_zero(prob_last);

--- a/vp8/encoder/onyx_if.c

+++ b/vp8/encoder/onyx_if.c

@@ -2816,6 +2816,8 @@

         if (cpi->common.refresh_alt_ref_frame)

         {

             cpi->prob_intra_coded += 40;

+            if (cpi->prob_intra_coded > 255)

+                cpi->prob_intra_coded = 255;

             cpi->prob_last_coded = 200;

             cpi->prob_gf_coded = 1;

         }

-- 

⑨