ref: b47cef056e51751d5d7b1cb50f91a71110294a9c
parent: 850a919640030a7a9813163491cf7c5bde77ce65
author: Yaowu Xu <yaowu@google.com>
date: Thu Oct 10 07:00:57 EDT 2013
change to avoid out-of-range computation Change-Id: Id5e31833a0ef40de9f64c2f5674af7083233bf14
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -3642,10 +3642,17 @@
// values, which actually are bigger than this_rd itself. This can
// cause negative best_filter_rd[] values, which is obviously silly.
// Therefore, if filter_cache < ref, we do an adjusted calculation.
- if (cpi->rd_filter_cache[i] >= ref)
+ if (cpi->rd_filter_cache[i] >= ref) {adj_rd = this_rd + cpi->rd_filter_cache[i] - ref;
- else // FIXME(rbultje) do this for comppred also
- adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref;
+ } else {+ // FIXME(rbultje) do this for comppsred also
+ //
+ // To prevent out-of-range computation in
+ // adj_rd = cpi->rd_filter_cache[i] * this_rd / ref
+ // cpi->rd_filter_cache[i] / ref is converted to a 256 based ratio.
+ int tmp = cpi->rd_filter_cache[i] * 256 / ref;
+ adj_rd = (this_rd * tmp) >> 8;
+ }
best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd);
}
}