shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix potential ioc issue in vp9_get_prob for 4K above sizes

This commit turns on the existing vp9_get_prob function using
64 bit in the intermediate step. It fixes the ioc issue for 4K
above frame sizes (issue 828).

Change-Id: I9f627f3beca2c522f73b38fd2a3e7eefdff01a7c

--- a/test/frame_size_tests.cc

+++ b/test/frame_size_tests.cc

@@ -89,8 +89,7 @@

   // one for each lag in frames (for 2 pass), and then one for each possible

   // reference buffer (8) - we can end up with up to 30 buffers of roughly this

   // size or almost 1 gig of memory.

-  // TODO(jzern): restore this to at least 4096x4096 after issue #828 is fixed.

-  video.SetSize(4096, 2160);

+  video.SetSize(4096, 4096);

   video.set_limit(2);

   expected_res_ = VPX_CODEC_OK;

   ASSERT_NO_FATAL_FAILURE(RunLoop(&video));

--- a/vp9/common/vp9_prob.h

+++ b/vp9/common/vp9_prob.h

@@ -47,18 +47,9 @@

   return (p > 255) ? 255u : (p < 1) ? 1u : p;

 }

-// int64 is not needed for normal frame level calculations.

-// However when outputting entropy stats accumulated over many frames

-// or even clips we can overflow int math.

-#ifdef ENTROPY_STATS

 static INLINE vp9_prob get_prob(int num, int den) {

   return (den == 0) ? 128u : clip_prob(((int64_t)num * 256 + (den >> 1)) / den);

 }

-#else

-static INLINE vp9_prob get_prob(int num, int den) {

-  return (den == 0) ? 128u : clip_prob((num * 256 + (den >> 1)) / den);

-}

-#endif

 static INLINE vp9_prob get_binary_prob(int n0, int n1) {

   return get_prob(n0, n0 + n1);