shithub: libvpx

Info  •  Files  •  Log  •  Branches

Bug fix to avoid random crashes during ARNR filtering

The function 'vp9_find_best_sub_pixel_tree_pruned_more' is modified
to return INT_MAX for handling invalid MV cases from UINT32_MAX.

yunqingwang:
patch 3: rebased on top of the tree.
patch 4: The return type of vp9_find_best_sub_pixel_tree* was changed
to uint32_t to fix ubsan warnings. Changing UINT_MAX back to INT_MAX
was not quite right. Patch 4 modified vp9_temporal_filter.c to accept
uint32_t.
(Note: Inconsistency exists in vp9_find_best_sub_pixel_tree*, which
will be fixed in a separate CL.)

Change-Id: Ib1a79dc2aa41ea6335c21669c76883cdbb7e0535

--- a/vp9/encoder/vp9_temporal_filter.c

+++ b/vp9/encoder/vp9_temporal_filter.c

@@ -208,10 +208,10 @@

 }

 #endif  // CONFIG_VP9_HIGHBITDEPTH

-static int temporal_filter_find_matching_mb_c(VP9_COMP *cpi,

-                                              uint8_t *arf_frame_buf,

-                                              uint8_t *frame_ptr_buf,

-                                              int stride) {

+static uint32_t temporal_filter_find_matching_mb_c(VP9_COMP *cpi,

+                                                   uint8_t *arf_frame_buf,

+                                                   uint8_t *frame_ptr_buf,

+                                                   int stride) {

   MACROBLOCK *const x = &cpi->td.mb;

   MACROBLOCKD *const xd = &x->e_mbd;

   MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv;

@@ -218,7 +218,7 @@

   const SEARCH_METHODS old_search_method = mv_sf->search_method;

   int step_param;

   int sadpb = x->sadperbit16;

-  int bestsme = INT_MAX;

+  uint32_t bestsme = UINT_MAX;

   uint32_t distortion;

   uint32_t sse;

   int cost_list[5];

@@ -334,8 +334,8 @@

           ((mb_cols - 1 - mb_col) * 16) + (17 - 2 * VP9_INTERP_EXTEND);

       for (frame = 0; frame < frame_count; frame++) {

-        const int thresh_low = 10000;

-        const int thresh_high = 20000;

+        const uint32_t thresh_low = 10000;

+        const uint32_t thresh_high = 20000;

         if (frames[frame] == NULL) continue;

@@ -346,7 +346,7 @@

           filter_weight = 2;

         } else {

           // Find best match in this frame by MC

-          int err = temporal_filter_find_matching_mb_c(

+          uint32_t err = temporal_filter_find_matching_mb_c(

               cpi, frames[alt_ref_index]->y_buffer + mb_y_offset,

               frames[frame]->y_buffer + mb_y_offset, frames[frame]->y_stride);