shithub: libvpx

Download patch

ref: b7480454706a6b15bf091e659cd6227ab373c1a6
parent: 034cea5e726b851294baad4d77e81bb6ce45703c
author: Yunqing Wang <yunqingwang@google.com>
date: Thu Jun 30 07:20:13 EDT 2011 

Bug fix in motion vector limit calculation

Motion vector limits are calculated using right shifts, which
could give wrong results for negative numbers. James Berry's
test on one clip showed encoder produced some artifacts. This
change fixed that.

Change-Id: I035fc02280b10455b7f6eb388f7c2e33b796b018


--- a/vp8/encoder/mcomp.c
+++ b/vp8/encoder/mcomp.c
@@ -286,8 +286,8 @@
     bestmv->as_mv.row = br << 1;
     bestmv->as_mv.col = bc << 1;
 
-    if ((abs(bestmv->as_mv.col - ref_mv->as_mv.col) > MAX_FULL_PEL_VAL) ||
-        (abs(bestmv->as_mv.row - ref_mv->as_mv.row) > MAX_FULL_PEL_VAL))
+    if ((abs(bestmv->as_mv.col - ref_mv->as_mv.col) > (MAX_FULL_PEL_VAL<<3)) ||
+        (abs(bestmv->as_mv.row - ref_mv->as_mv.row) > (MAX_FULL_PEL_VAL<<3)))
         return INT_MAX;
 
     return besterr;
--- a/vp8/encoder/mcomp.h
+++ b/vp8/encoder/mcomp.h
@@ -22,7 +22,7 @@
 
 
 #define MAX_MVSEARCH_STEPS 8                                    // The maximum number of steps in a step search given the largest allowed initial step
-#define MAX_FULL_PEL_VAL ((1 << (MAX_MVSEARCH_STEPS+3)) - 8)    // Max full pel mv specified in 1/8 pel units
+#define MAX_FULL_PEL_VAL ((1 << (MAX_MVSEARCH_STEPS)) - 1)      // Max full pel mv specified in 1 pel units
 #define MAX_FIRST_STEP (1 << (MAX_MVSEARCH_STEPS-1))            // Maximum size of the first step in full pel units
 
 extern void print_mode_context(void);
--- a/vp8/encoder/pickinter.c
+++ b/vp8/encoder/pickinter.c
@@ -588,10 +588,10 @@
 
             /* adjust mvp to make sure it is within MV range */
             vp8_clamp_mv(&mvp,
-                         best_ref_mv.as_mv.col - MAX_FULL_PEL_VAL,
-                         best_ref_mv.as_mv.col + MAX_FULL_PEL_VAL,
-                         best_ref_mv.as_mv.row - MAX_FULL_PEL_VAL,
-                         best_ref_mv.as_mv.row + MAX_FULL_PEL_VAL);
+                         best_ref_mv.as_mv.col - (MAX_FULL_PEL_VAL<<3),
+                         best_ref_mv.as_mv.col + (MAX_FULL_PEL_VAL<<3),
+                         best_ref_mv.as_mv.row - (MAX_FULL_PEL_VAL<<3),
+                         best_ref_mv.as_mv.row + (MAX_FULL_PEL_VAL<<3));
         }
 
         switch (this_mode)
@@ -681,10 +681,14 @@
                 mvp.as_int = best_ref_mv.as_int;
             }
 
-            col_min = (best_ref_mv.as_mv.col - MAX_FULL_PEL_VAL) >>3;
-            col_max = (best_ref_mv.as_mv.col + MAX_FULL_PEL_VAL) >>3;
-            row_min = (best_ref_mv.as_mv.row - MAX_FULL_PEL_VAL) >>3;
-            row_max = (best_ref_mv.as_mv.row + MAX_FULL_PEL_VAL) >>3;
+            col_min = (best_ref_mv.as_mv.col < 0)?(-((abs(best_ref_mv.as_mv.col))>>3) - MAX_FULL_PEL_VAL)
+                                                 :((best_ref_mv.as_mv.col>>3) - MAX_FULL_PEL_VAL);
+            col_max = (best_ref_mv.as_mv.col < 0)?(-((abs(best_ref_mv.as_mv.col))>>3) + MAX_FULL_PEL_VAL)
+                                                 :((best_ref_mv.as_mv.col>>3) + MAX_FULL_PEL_VAL);
+            row_min = (best_ref_mv.as_mv.row < 0)?(-((abs(best_ref_mv.as_mv.row))>>3) - MAX_FULL_PEL_VAL)
+                                                 :((best_ref_mv.as_mv.row>>3) - MAX_FULL_PEL_VAL);
+            row_max = (best_ref_mv.as_mv.row < 0)?(-((abs(best_ref_mv.as_mv.row))>>3) + MAX_FULL_PEL_VAL)
+                                                 :((best_ref_mv.as_mv.row>>3) + MAX_FULL_PEL_VAL);
 
             // Get intersection of UMV window and valid MV window to reduce # of checks in diamond search.
             if (x->mv_col_min < col_min )
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -1330,10 +1330,14 @@
 
         if (bsi.segment_rd < best_rd)
         {
-            int col_min = (best_ref_mv->as_mv.col - MAX_FULL_PEL_VAL) >>3;
-            int col_max = (best_ref_mv->as_mv.col + MAX_FULL_PEL_VAL) >>3;
-            int row_min = (best_ref_mv->as_mv.row - MAX_FULL_PEL_VAL) >>3;
-            int row_max = (best_ref_mv->as_mv.row + MAX_FULL_PEL_VAL) >>3;
+            int col_min = (best_ref_mv->as_mv.col < 0)?(-((abs(best_ref_mv->as_mv.col))>>3) - MAX_FULL_PEL_VAL)
+                                                      :((best_ref_mv->as_mv.col>>3) - MAX_FULL_PEL_VAL);
+            int col_max = (best_ref_mv->as_mv.col < 0)?(-((abs(best_ref_mv->as_mv.col))>>3) + MAX_FULL_PEL_VAL)
+                                                      :((best_ref_mv->as_mv.col>>3) + MAX_FULL_PEL_VAL);
+            int row_min = (best_ref_mv->as_mv.row < 0)?(-((abs(best_ref_mv->as_mv.row))>>3) - MAX_FULL_PEL_VAL)
+                                                      :((best_ref_mv->as_mv.row>>3) - MAX_FULL_PEL_VAL);
+            int row_max = (best_ref_mv->as_mv.row < 0)?(-((abs(best_ref_mv->as_mv.row))>>3) + MAX_FULL_PEL_VAL)
+                                                      :((best_ref_mv->as_mv.row>>3) + MAX_FULL_PEL_VAL);
 
             int tmp_col_min = x->mv_col_min;
             int tmp_col_max = x->mv_col_max;
@@ -1876,10 +1880,10 @@
 
             /* adjust mvp to make sure it is within MV range */
             vp8_clamp_mv(&mvp,
-                         best_ref_mv.as_mv.col - MAX_FULL_PEL_VAL,
-                         best_ref_mv.as_mv.col + MAX_FULL_PEL_VAL,
-                         best_ref_mv.as_mv.row - MAX_FULL_PEL_VAL,
-                         best_ref_mv.as_mv.row + MAX_FULL_PEL_VAL);
+                         best_ref_mv.as_mv.col - (MAX_FULL_PEL_VAL<<3),
+                         best_ref_mv.as_mv.col + (MAX_FULL_PEL_VAL<<3),
+                         best_ref_mv.as_mv.row - (MAX_FULL_PEL_VAL<<3),
+                         best_ref_mv.as_mv.row + (MAX_FULL_PEL_VAL<<3));
         }
 
         // Check to see if the testing frequency for this mode is at its max
@@ -2011,10 +2015,14 @@
 
             int sadpb = x->sadperbit16;
 
-            int col_min = (best_ref_mv.as_mv.col - MAX_FULL_PEL_VAL) >>3;
-            int col_max = (best_ref_mv.as_mv.col + MAX_FULL_PEL_VAL) >>3;
-            int row_min = (best_ref_mv.as_mv.row - MAX_FULL_PEL_VAL) >>3;
-            int row_max = (best_ref_mv.as_mv.row + MAX_FULL_PEL_VAL) >>3;
+            int col_min = (best_ref_mv.as_mv.col < 0)?(-((abs(best_ref_mv.as_mv.col))>>3) - MAX_FULL_PEL_VAL)
+                                                     :((best_ref_mv.as_mv.col>>3) - MAX_FULL_PEL_VAL);
+            int col_max = (best_ref_mv.as_mv.col < 0)?(-((abs(best_ref_mv.as_mv.col))>>3) + MAX_FULL_PEL_VAL)
+                                                     :((best_ref_mv.as_mv.col>>3) + MAX_FULL_PEL_VAL);
+            int row_min = (best_ref_mv.as_mv.row < 0)?(-((abs(best_ref_mv.as_mv.row))>>3) - MAX_FULL_PEL_VAL)
+                                                     :((best_ref_mv.as_mv.row>>3) - MAX_FULL_PEL_VAL);
+            int row_max = (best_ref_mv.as_mv.row < 0)?(-((abs(best_ref_mv.as_mv.row))>>3) + MAX_FULL_PEL_VAL)
+                                                     :((best_ref_mv.as_mv.row>>3) + MAX_FULL_PEL_VAL);
 
             int tmp_col_min = x->mv_col_min;
             int tmp_col_max = x->mv_col_max;
-- 
⑨