shithub: libvpx

Download patch

ref: 1d12559b0978e26c53053bb6576abb3f00515b03
parent: 7cf13826b7234e933d9aa4dac8ebddeab89ab4ac
author: Yunqing Wang <yunqingwang@google.com>
date: Tue Dec 27 06:52:39 EST 2016 

Make sub-pixel mv search's return value consistent with the return type

For out-of-range cases, returned UINT_MAX instead of INT_MAX in the
sub-pixel mv search to be consistent with the "uint32_t" return type.

Change-Id: I8e206d771228c13d89bafbbe9f14722c8ecc6a7a


--- a/vp9/encoder/vp9_mcomp.c
+++ b/vp9/encoder/vp9_mcomp.c
@@ -277,7 +277,7 @@
   const uint8_t *const z = x->plane[0].src.buf;                            \
   const int src_stride = x->plane[0].src.stride;                           \
   const MACROBLOCKD *xd = &x->e_mbd;                                       \
-  unsigned int besterr = INT_MAX;                                          \
+  unsigned int besterr = UINT_MAX;                                         \
   unsigned int sse;                                                        \
   unsigned int whichdir;                                                   \
   int thismse;                                                             \
@@ -472,7 +472,7 @@
 
   if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||
       (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))
-    return INT_MAX;
+    return UINT_MAX;
 
   return besterr;
 }
@@ -622,7 +622,7 @@
 
   if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||
       (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))
-    return INT_MAX;
+    return UINT_MAX;
 
   return besterr;
 }
@@ -646,7 +646,7 @@
   const uint8_t *const src_address = z;
   const int src_stride = x->plane[0].src.stride;
   const MACROBLOCKD *xd = &x->e_mbd;
-  unsigned int besterr = INT_MAX;
+  unsigned int besterr = UINT_MAX;
   unsigned int sse;
   int thismse;
   const int y_stride = xd->plane[0].pre[0].stride;
@@ -708,7 +708,7 @@
           *sse1 = sse;
         }
       } else {
-        cost_array[idx] = INT_MAX;
+        cost_array[idx] = UINT_MAX;
       }
     }
 
@@ -737,7 +737,7 @@
         *sse1 = sse;
       }
     } else {
-      cost_array[idx] = INT_MAX;
+      cost_array[idx] = UINT_MAX;
     }
 
     if (best_idx < 4 && best_idx >= 0) {
@@ -771,7 +771,7 @@
 
   if ((abs(bestmv->col - ref_mv->col) > (MAX_FULL_PEL_VAL << 3)) ||
       (abs(bestmv->row - ref_mv->row) > (MAX_FULL_PEL_VAL << 3)))
-    return INT_MAX;
+    return UINT_MAX;
 
   return besterr;
 }