ref: 6becad426cddf8d98c35172eeac8bcc3282e27ec
parent: 5a955973d9949e57a419c6b420b83702d9108640
author: John Koleszar <jkoleszar@google.com>
date: Wed Nov 14 04:51:23 EST 2012
detokenize: use SEG_LVL_EOB feature consistently Update decode_coefs() to break when c >= eob, since it's possible that c starts the loop from 1 and eob is 0. The loop won't terminate in that case. Add new get_eob() function to consistently clamp the eob based on the segment level EOB and the block size. It's possible to code a segment level EOB that's greater than the block size, and that leads to an out of bounds access. Change-Id: I859563b30414615cf1b30dcc2aef8a1de358c42d
--- a/vp9/common/seg_common.c
+++ b/vp9/common/seg_common.c
@@ -12,7 +12,7 @@
static const int segfeaturedata_signed[SEG_LVL_MAX] = { 1, 1, 0, 0, 0, 0 };
static const int seg_feature_data_bits[SEG_LVL_MAX] =
- { QINDEX_BITS, 6, 4, 5, 6, 2 };
+ { QINDEX_BITS, 6, 4, 5, 8, 2 };
// These functions provide access to new segment level features.
// Eventually these function may be "optimized out" but for the moment,
--- a/vp9/decoder/detokenize.c
+++ b/vp9/decoder/detokenize.c
@@ -324,12 +324,12 @@
while (1) {
int val;
const uint8_t *cat6 = cat6_prob;
- if (c == seg_eob) break;
+ if (c >= seg_eob) break;
prob += coef_bands[c];
if (!vp9_read(br, prob[EOB_CONTEXT_NODE]))
break;
SKIP_START:
- if (c == seg_eob) break;
+ if (c >= seg_eob) break;
if (!vp9_read(br, prob[ZERO_CONTEXT_NODE])) {
++c;
prob = coef_probs + coef_bands[c];
@@ -414,6 +414,17 @@
return c;
}
+
+int get_eob(MACROBLOCKD* const xd, int segment_id, int eob_max) {
+ int active = vp9_segfeature_active(xd, segment_id, SEG_LVL_EOB);
+ int eob = vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
+
+ if (!active || eob > eob_max)
+ eob = eob_max;
+ return eob;
+}
+
+
int vp9_decode_mb_tokens_16x16(VP9D_COMP* const pbi,
MACROBLOCKD* const xd,
BOOL_DECODER* const bc) {
@@ -424,17 +435,12 @@
PLANE_TYPE type;
int c, i, eobtotal = 0, seg_eob;
const int segment_id = xd->mode_info_context->mbmi.segment_id;
- const int seg_active = vp9_segfeature_active(xd, segment_id, SEG_LVL_EOB);
INT16 *qcoeff_ptr = &xd->qcoeff[0];
TX_TYPE tx_type = get_tx_type(xd, &xd->block[0]);
type = PLANE_TYPE_Y_WITH_DC;
+ seg_eob = get_eob(xd, segment_id, 256);
- if (seg_active)
- seg_eob = vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
- else
- seg_eob = 256;
-
// Luma block
{
const int* const scan = vp9_default_zig_zag1d_16x16;
@@ -453,10 +459,7 @@
qcoeff_ptr += 256;
type = PLANE_TYPE_UV;
tx_type = DCT_DCT;
- if (seg_active)
- seg_eob = vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
- else
- seg_eob = 64;
+ seg_eob = get_eob(xd, segment_id, 64);
for (i = 16; i < 24; i += 4) {
ENTROPY_CONTEXT* const a = A + vp9_block2above_8x8[i];
ENTROPY_CONTEXT* const l = L + vp9_block2left_8x8[i];
@@ -488,7 +491,6 @@
PLANE_TYPE type;
int c, i, eobtotal = 0, seg_eob;
const int segment_id = xd->mode_info_context->mbmi.segment_id;
- const int seg_active = vp9_segfeature_active(xd, segment_id, SEG_LVL_EOB);
INT16 *qcoeff_ptr = &xd->qcoeff[0];
TX_TYPE tx_type = DCT_DCT;
@@ -502,10 +504,7 @@
const int *const scan = vp9_default_zig_zag1d;
type = PLANE_TYPE_Y2;
- if (seg_active)
- seg_eob = vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
- else
- seg_eob = 4;
+ seg_eob = get_eob(xd, segment_id, 4);
c = decode_coefs(pbi, xd, bc, a, l, type,
tx_type,
seg_eob, qcoeff_ptr + 24 * 16,
@@ -518,10 +517,7 @@
} else
type = PLANE_TYPE_Y_WITH_DC;
- if (seg_active)
- seg_eob = vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
- else
- seg_eob = 64;
+ seg_eob = get_eob(xd, segment_id, 64);
for (i = 0; i < bufthred ; i += 4) {
ENTROPY_CONTEXT *const a = A + vp9_block2above_8x8[i];
@@ -550,7 +546,7 @@
if (bufthred == 16) {
type = PLANE_TYPE_UV;
tx_type = DCT_DCT;
- seg_eob = 16;
+ seg_eob = get_eob(xd, segment_id, 16);
// use 4x4 transform for U, V components in I8X8 prediction mode
for (i = 16; i < 24; i++) {
@@ -581,11 +577,11 @@
INT16 *qcoeff_ptr = &xd->qcoeff[0];
const int *scan = vp9_default_zig_zag1d;
unsigned short *const eobs = xd->eobs;
- int c, seg_eob = 16;
+ int c, seg_eob;
TX_TYPE tx_type = DCT_DCT;
int segment_id = xd->mode_info_context->mbmi.segment_id;
- if (vp9_segfeature_active(xd, segment_id, SEG_LVL_EOB))
- seg_eob = vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
+
+ seg_eob = get_eob(xd, segment_id, 16);
if (i == 24)
type = PLANE_TYPE_Y2;