shithub: libvpx

Info  •  Files  •  Log  •  Branches

vp8: remove assertion in tree coder.

Cast the counter to uint64_t in case it overflows.

The assert was to prevent c[0] * Pfac being overflow beyong unsigned int
since Pfac could be 2^8. Thus c[0] needs to be smaller than 2^24.

In VP9, the assert was removed and c[0] was casted to uint64_t.

Bug: 805277
Change-Id: Ic46a3c5b4af2f267de4e32c1518b64e8d6e9d856

--- a/vp8/common/treecoder.c

+++ b/vp8/common/treecoder.c

@@ -12,6 +12,7 @@

 #include <stdio.h>

 #include "vp8/common/treecoder.h"

+#include "vpx/vpx_integer.h"

 static void tree2tok(struct vp8_token_struct *const p, vp8_tree t, int i, int v,

                      int L) {

@@ -89,10 +90,9 @@

     const unsigned int *const c = branch_ct[t];

     const unsigned int tot = c[0] + c[1];

-    assert(tot < (1 << 24)); /* no overflow below */

-

     if (tot) {

-      const unsigned int p = ((c[0] * Pfac) + (rd ? tot >> 1 : 0)) / tot;

+      const unsigned int p =

+          (unsigned int)(((uint64_t)c[0] * Pfac) + (rd ? tot >> 1 : 0)) / tot;

       probs[t] = p < 256 ? (p ? p : 1) : 255; /* agree w/old version for now */

     } else {

       probs[t] = vp8_prob_half;