shithub: libvpx

Info  •  Files  •  Log  •  Branches

Bug fix in probability update savings computation

Found this bug while tracking down some anomalies in my experiments.
Since vp8_cost_one and vp8_cost_zero return unsigned int, the
bit shift by 8 will be incorrect if the value is negative.
I am cautiously optimistic that this fix will make the prob
updates more correct and somewhat improve results across the board.
But the update probabilities will need to be retuned I think.

Patch 2: Adding more of the same fixes using a macro.

Change-Id: I1a168f040e74e8c67e7225103b1c2af9a611da49

--- a/vp8/encoder/bitstream.c

+++ b/vp8/encoder/bitstream.c

@@ -44,6 +44,8 @@

 int count_mb_seg[4] = { 0, 0, 0, 0 };

 #endif

+#define vp8_cost_upd  ((int)(vp8_cost_one(upd) - vp8_cost_zero(upd)) >> 8)

+

 static void update_mode(

     vp8_writer *const w,

     int n,

@@ -157,8 +159,7 @@

 {

     const int old_b = vp8_cost_branch(ct, oldp);

     const int new_b = vp8_cost_branch(ct, newp);

-    const int update_b = 8 +

-                         ((vp8_cost_one(upd) - vp8_cost_zero(upd)) >> 8);

+    const int update_b = 8 + vp8_cost_upd;

     return old_b - new_b - update_b;

 }

@@ -1330,8 +1331,7 @@

                         const int old_b = vp8_cost_branch(ct, old);

                         const int new_b = vp8_cost_branch(ct, newp);

-                        const int update_b = 8 +

-                            ((vp8_cost_one(upd) - vp8_cost_zero(upd)) >> 8);

+                        const int update_b = 8 + vp8_cost_upd;

                         const int s = old_b - new_b - update_b;

@@ -1519,8 +1519,7 @@

                         const vp8_prob upd = vp8_coef_update_probs_8x8 [i][j][k][t];

                         const int old_b = vp8_cost_branch(ct, old);

                         const int new_b = vp8_cost_branch(ct, newp);

-                        const int update_b = 8 +

-                            ((vp8_cost_one(upd) - vp8_cost_zero(upd)) >> 8);

+                        const int update_b = 8 + vp8_cost_upd;

                         const int s = old_b - new_b - update_b;

                         const int u = s > 0 ? 1 : 0;

@@ -1586,8 +1585,7 @@

                             const vp8_prob upd = vp8_coef_update_probs_8x8 [i][j][k][t];

                             const int old_b = vp8_cost_branch(ct, old);

                             const int new_b = vp8_cost_branch(ct, newp);

-                            const int update_b = 8 +

-                                ((vp8_cost_one(upd) - vp8_cost_zero(upd)) >> 8);

+                            const int update_b = 8 + vp8_cost_upd;

                             const int s = old_b - new_b - update_b;

                             const int u = s > 0 ? 1 : 0;

                             vp8_write(w, u, upd);