shithub: libvpx

Info  •  Files  •  Log  •  Branches

vp9: Reduce stack usage of choose_partioning.

Change type of sum_square_error from int64_t to uint32_t.
Change type of sum_error from int64_t to int32_t.

This reduces the stack usage from ~131K to ~87K.

BUG=b/68362457

Change-Id: I147d7c7b226bceb4f0817bb86848e1fa9d9ac149

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -287,8 +287,12 @@

 }

 typedef struct {

-  int64_t sum_square_error;

-  int64_t sum_error;

+  // This struct is used for computing variance in choose_partitioning(), where

+  // the max number of samples within a superblock is 16x16 (with 4x4 avg). Even

+  // in high bitdepth, uint32_t is enough for sum_square_error (2^12 * 2^12 * 16

+  // * 16 = 2^32).

+  uint32_t sum_square_error;

+  int32_t sum_error;

   int log2_count;

   int variance;

 } var;

@@ -381,7 +385,7 @@

 }

 // Set variance values given sum square error, sum error, count.

-static void fill_variance(int64_t s2, int64_t s, int c, var *v) {

+static void fill_variance(uint32_t s2, int32_t s, int c, var *v) {

   v->sum_square_error = s2;

   v->sum_error = s;

   v->log2_count = c;