ref: 0930dde2496e911d24c930751b08c71ae722bd89
parent: 46f9ad2ca5f161ccbdfc6787da1333427f246263
author: Ronald S. Bultje <rbultje@google.com>
date: Wed Feb 15 03:30:36 EST 2012
Fix overflows in dual prediction mode selection. Change-Id: I265ad46e01a307bca21e6223725e4055f5e08648
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1348,6 +1348,7 @@
{int frame_type, pred_type;
int redo = 0;
+ int single_diff, dual_diff, hybrid_diff;
/*
* This code does a single RD pass over the whole frame assuming
@@ -1384,20 +1385,20 @@
cpi->common.dual_pred_mode = pred_type;
encode_frame_internal(cpi);
- cpi->rd_single_diff /= cpi->common.MBs;
- cpi->rd_prediction_type_threshes[frame_type][0] += cpi->rd_single_diff;
+ single_diff = cpi->rd_single_diff / cpi->common.MBs;
+ cpi->rd_prediction_type_threshes[frame_type][0] += single_diff;
cpi->rd_prediction_type_threshes[frame_type][0] >>= 1;
- cpi->rd_dual_diff /= cpi->common.MBs;
- cpi->rd_prediction_type_threshes[frame_type][1] += cpi->rd_dual_diff;
+ dual_diff = cpi->rd_dual_diff / cpi->common.MBs;
+ cpi->rd_prediction_type_threshes[frame_type][1] += dual_diff;
cpi->rd_prediction_type_threshes[frame_type][1] >>= 1;
- cpi->rd_hybrid_diff /= cpi->common.MBs;
- cpi->rd_prediction_type_threshes[frame_type][2] += cpi->rd_hybrid_diff;
+ hybrid_diff = cpi->rd_hybrid_diff / cpi->common.MBs;
+ cpi->rd_prediction_type_threshes[frame_type][2] += hybrid_diff;
cpi->rd_prediction_type_threshes[frame_type][2] >>= 1;
/* FIXME make "100" (the threshold at which to re-encode the
* current frame) a commandline option. */
if (cpi->common.dual_pred_mode == SINGLE_PREDICTION_ONLY &&
- (cpi->rd_dual_diff >= 100 || cpi->rd_hybrid_diff >= 100))
+ (dual_diff >= 100 || hybrid_diff >= 100))
{redo = 1;
cpi->common.dual_pred_mode = cpi->rd_dual_diff > cpi->rd_hybrid_diff ?
@@ -1404,7 +1405,7 @@
DUAL_PREDICTION_ONLY : HYBRID_PREDICTION;
}
else if (cpi->common.dual_pred_mode == DUAL_PREDICTION_ONLY &&
- (cpi->rd_single_diff >= 100 || cpi->rd_hybrid_diff >= 100))
+ (single_diff >= 100 || hybrid_diff >= 100))
{redo = 1;
cpi->common.dual_pred_mode = cpi->rd_single_diff > cpi->rd_hybrid_diff ?
@@ -1430,7 +1431,7 @@
{cpi->common.dual_pred_mode = DUAL_PREDICTION_ONLY;
}
- else if (cpi->rd_single_diff >= 100 || cpi->rd_dual_diff >= 100)
+ else if (single_diff >= 100 || dual_diff >= 100)
{redo = 1;
cpi->common.dual_pred_mode = cpi->rd_single_diff > cpi->rd_dual_diff ?
--- a/vp8/encoder/onyx_int.h
+++ b/vp8/encoder/onyx_int.h
@@ -353,7 +353,7 @@
int rd_thresh_mult[MAX_MODES];
int rd_baseline_thresh[MAX_MODES];
int rd_threshes[MAX_MODES];
- int rd_single_diff, rd_dual_diff, rd_hybrid_diff;
+ int64_t rd_single_diff, rd_dual_diff, rd_hybrid_diff;
int rd_prediction_type_threshes[4][NB_PREDICTION_TYPES];
int dual_pred_count[DUAL_PRED_CONTEXTS];
int single_pred_count[DUAL_PRED_CONTEXTS];
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -3142,9 +3142,18 @@
rd_update_mvcount(cpi, x, &frame_best_ref_mv[xd->mode_info_context->mbmi.ref_frame]);
- *best_single_rd_diff = best_rd - best_single_rd;
- *best_dual_rd_diff = best_rd - best_dual_rd;
- *best_hybrid_rd_diff = best_rd - best_hybrid_rd;
+ if (best_single_rd == INT_MAX)
+ *best_single_rd_diff = INT_MIN;
+ else
+ *best_single_rd_diff = best_rd - best_single_rd;
+ if (best_dual_rd == INT_MAX)
+ *best_dual_rd_diff = INT_MIN;
+ else
+ *best_dual_rd_diff = best_rd - best_dual_rd;
+ if (best_hybrid_rd == INT_MAX)
+ *best_hybrid_rd_diff = INT_MIN;
+ else
+ *best_hybrid_rd_diff = best_rd - best_hybrid_rd;
}
void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_)
--
⑨