ref: d8a18186ba4482ff121aa526fbf6df31f9d90d35
parent: bd2e7fa1c8b9923c741d338f33b63c1a92649af0
author: Alex Converse <aconverse@google.com>
date: Mon May 9 07:21:20 EDT 2016
pickmode: Fix a pair of unsigned overflows. block_variance: This operates on 8x8s and would be safe with a int32 * int32 to uint32 multiply, but this is potentially unsafe for 12-bit input. Unfortunately the code already segfaults on 12-bit input: https://bugs.chromium.org/p/webm/issues/detail?id=1223 calculate_variance: This operates on up to a 32x32 of 8x8s and can overflow even with 8-bit input (log2((256*32*32)**2) == 36). BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1220 Change-Id: I1ca4ff6092db9a7580da371ee9a21f403fdadc40
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -244,7 +244,7 @@
&sse8x8[k], &sum8x8[k]);
*sse += sse8x8[k];
*sum += sum8x8[k];
- var8x8[k] = sse8x8[k] - (((unsigned int)sum8x8[k] * sum8x8[k]) >> 6);
+ var8x8[k] = sse8x8[k] - (uint32_t)(((int64_t)sum8x8[k] * sum8x8[k]) >> 6);
k++;
}
}
@@ -265,7 +265,7 @@
sse_i[(i + 1) * nw + j] + sse_i[(i + 1) * nw + j + 1];
sum_o[k] = sum_i[i * nw + j] + sum_i[i * nw + j + 1] +
sum_i[(i + 1) * nw + j] + sum_i[(i + 1) * nw + j + 1];
- var_o[k] = sse_o[k] - (((unsigned int)sum_o[k] * sum_o[k]) >>
+ var_o[k] = sse_o[k] - (uint32_t)(((int64_t)sum_o[k] * sum_o[k]) >>
(b_width_log2_lookup[unit_size] +
b_height_log2_lookup[unit_size] + 6));
k++;