shithub: libvpx

Info  •  Files  •  Log  •  Branches

Change to prevent invalid memory access

After change of MI context storage , mi_8x8[]  pointer may be null for
a block outside of image border. The commit changes to access the data
only after validation of mi_row and mi_col.

Change-Id: I039c4eb486a228ea9d8e5f35ab9ae6717d718bf3

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -2348,18 +2348,19 @@

                                    int mis, TX_SIZE max_tx_size, int bw, int bh,

                                    int mi_row, int mi_col, BLOCK_SIZE bsize) {

   VP9_COMMON * const cm = &cpi->common;

-  MB_MODE_INFO * const mbmi = &mi_8x8[0]->mbmi;

-  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols)

+  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) {

     return;

+  } else {

+    MB_MODE_INFO * const mbmi = &mi_8x8[0]->mbmi;

+    if (mbmi->tx_size > max_tx_size) {

+      const int ymbs = MIN(bh, cm->mi_rows - mi_row);

+      const int xmbs = MIN(bw, cm->mi_cols - mi_col);

-  if (mbmi->tx_size > max_tx_size) {

-    const int ymbs = MIN(bh, cm->mi_rows - mi_row);

-    const int xmbs = MIN(bw, cm->mi_cols - mi_col);

-

-    assert(vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) ||

-           get_skip_flag(mi_8x8, mis, ymbs, xmbs));

-    set_txfm_flag(mi_8x8, mis, ymbs, xmbs, max_tx_size);

+      assert(vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) ||

+             get_skip_flag(mi_8x8, mis, ymbs, xmbs));

+      set_txfm_flag(mi_8x8, mis, ymbs, xmbs, max_tx_size);

+    }

   }

 }