shithub: libvpx

Info  •  Files  •  Log  •  Branches

vp9: Reduce stack usage of choose_partitioning.

Move vt2 to heap.
Reduce the stack usage from ~87K to ~44K.

BUG=b/68362457

Change-Id: I8f5f93712934d59a8cc4564378172d409a736a2e

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -1044,7 +1044,7 @@

   MACROBLOCKD *xd = &x->e_mbd;

   int i, j, k, m;

   v64x64 vt;

-  v16x16 vt2[16];

+  v16x16 *vt2 = NULL;

   int force_split[21];

   int avg_32x32;

   int max_var_32x32 = 0;

@@ -1113,6 +1113,8 @@

   // For non keyframes, disable 4x4 average for low resolution when speed = 8

   threshold_4x4avg = (cpi->oxcf.speed < 8) ? thresholds[1] << 1 : INT64_MAX;

+  if (low_res && threshold_4x4avg < INT64_MAX)

+    CHECK_MEM_ERROR(cm, vt2, vpx_calloc(16, sizeof(*vt2)));

   memset(x->variance_low, 0, sizeof(x->variance_low));

   if (xd->mb_to_right_edge < 0) pixels_wide += (xd->mb_to_right_edge >> 3);

@@ -1222,6 +1224,7 @@

         set_block_size(cpi, x, xd, mi_row, mi_col, BLOCK_64X64);

         x->variance_low[0] = 1;

         chroma_check(cpi, x, bsize, y_sad, is_key_frame);

+        if (vt2) vpx_free(vt2);

         return 0;

       }

     }

@@ -1233,6 +1236,7 @@

     if (cpi->sf.copy_partition_flag && y_sad_last < cpi->vbp_threshold_copy &&

         copy_partitioning(cpi, x, xd, mi_row, mi_col, segment_id, sb_offset)) {

       chroma_check(cpi, x, bsize, y_sad, is_key_frame);

+      if (vt2) vpx_free(vt2);

       return 0;

     }

   } else {

@@ -1459,6 +1463,7 @@

   }

   chroma_check(cpi, x, bsize, y_sad, is_key_frame);

+  if (vt2) vpx_free(vt2);

   return 0;

 }