shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix bug in intra mode rd penalty.

The intra mode rd penalty was implemented as a rate penalty.
Code was added to scale the penalty according to block size but
this was not done correctly for the SB level or sub 8x8.

The code did a weird double scaling in regard to bit depth that
has been removed. Given that it is a rate penalty the bit depth
should not matter.

This bug fix improves average metrics  on our standard test
sets by about 0.1%

Change-Id: I7cf81b66aad0cda389fe234f47beba01c7493b1e

--- a/vp9/encoder/vp9_pickmode.c

+++ b/vp9/encoder/vp9_pickmode.c

@@ -1182,19 +1182,6 @@

   { LAST_FRAME, NEWMV },       { GOLDEN_FRAME, NEWMV }

 };

-static int set_intra_cost_penalty(const VP9_COMP *const cpi, BLOCK_SIZE bsize) {

-  const VP9_COMMON *const cm = &cpi->common;

-  // Reduce the intra cost penalty for small blocks (<=16x16).

-  int reduction_fac =

-      (bsize <= BLOCK_16X16) ? ((bsize <= BLOCK_8X8) ? 4 : 2) : 0;

-  if (cpi->noise_estimate.enabled && cpi->noise_estimate.level == kHigh)

-    // Don't reduce intra cost penalty if estimated noise level is high.

-    reduction_fac = 0;

-  return vp9_get_intra_cost_penalty(cm->base_qindex, cm->y_dc_delta_q,

-                                    cm->bit_depth) >>

-         reduction_fac;

-}

-

 static INLINE void find_predictors(

     VP9_COMP *cpi, MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame,

     int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES],

@@ -1450,7 +1437,8 @@

   // var_y and sse_y are saved to be used in skipping checking

   unsigned int var_y = UINT_MAX;

   unsigned int sse_y = UINT_MAX;

-  const int intra_cost_penalty = set_intra_cost_penalty(cpi, bsize);

+  const int intra_cost_penalty =

+      vp9_get_intra_cost_penalty(cpi, bsize, cm->base_qindex, cm->y_dc_delta_q);

   int64_t inter_mode_thresh =

       RDCOST(x->rdmult, x->rddiv, intra_cost_penalty, 0);

   const int *const rd_threshes = cpi->rd.threshes[mi->segment_id][bsize];

--- a/vp9/encoder/vp9_rd.c

+++ b/vp9/encoder/vp9_rd.c

@@ -670,19 +670,21 @@

   }

 }

-int vp9_get_intra_cost_penalty(int qindex, int qdelta,

-                               vpx_bit_depth_t bit_depth) {

-  const int q = vp9_dc_quant(qindex, qdelta, bit_depth);

-#if CONFIG_VP9_HIGHBITDEPTH

-  switch (bit_depth) {

-    case VPX_BITS_8: return 20 * q;

-    case VPX_BITS_10: return 5 * q;

-    case VPX_BITS_12: return ROUND_POWER_OF_TWO(5 * q, 2);

-    default:

-      assert(0 && "bit_depth should be VPX_BITS_8, VPX_BITS_10 or VPX_BITS_12");

-      return -1;

-  }

-#else

-  return 20 * q;

-#endif  // CONFIG_VP9_HIGHBITDEPTH

+int vp9_get_intra_cost_penalty(const VP9_COMP *const cpi, BLOCK_SIZE bsize,

+                               int qindex, int qdelta) {

+  // Reduce the intra cost penalty for small blocks (<=16x16).

+  int reduction_fac =

+      (bsize <= BLOCK_16X16) ? ((bsize <= BLOCK_8X8) ? 4 : 2) : 0;

+

+  if (cpi->noise_estimate.enabled && cpi->noise_estimate.level == kHigh)

+    // Don't reduce intra cost penalty if estimated noise level is high.

+    reduction_fac = 0;

+

+  // Always use VPX_BITS_8 as input here because the penalty is applied

+  // to rate not distortion so we want a consistent penalty for all bit

+  // depths. If the actual bit depth were passed in here then the value

+  // retured by vp9_dc_quant() would scale with the bit depth and we would

+  // then need to apply inverse scaling to correct back to a bit depth

+  // independent rate penalty.

+  return (20 * vp9_dc_quant(qindex, qdelta, VPX_BITS_8)) >> reduction_fac;

 }

--- a/vp9/encoder/vp9_rd.h

+++ b/vp9/encoder/vp9_rd.h

@@ -191,8 +191,8 @@

                           const struct scale_factors *scale,

                           const struct scale_factors *scale_uv);

-int vp9_get_intra_cost_penalty(int qindex, int qdelta,

-                               vpx_bit_depth_t bit_depth);

+int vp9_get_intra_cost_penalty(const struct VP9_COMP *const cpi,

+                               BLOCK_SIZE bsize, int qindex, int qdelta);

 unsigned int vp9_get_sby_perpixel_variance(struct VP9_COMP *cpi,

                                            const struct buf_2d *ref,

--- a/vp9/encoder/vp9_rdopt.c

+++ b/vp9/encoder/vp9_rdopt.c

@@ -3038,8 +3038,8 @@

   int64_t dist_uv[TX_SIZES];

   int skip_uv[TX_SIZES];

   PREDICTION_MODE mode_uv[TX_SIZES];

-  const int intra_cost_penalty = vp9_get_intra_cost_penalty(

-      cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth);

+  const int intra_cost_penalty =

+      vp9_get_intra_cost_penalty(cpi, bsize, cm->base_qindex, cm->y_dc_delta_q);

   int best_skip2 = 0;

   uint8_t ref_frame_skip_mask[2] = { 0 };

   uint16_t mode_skip_mask[MAX_REF_FRAMES] = { 0 };

@@ -3802,8 +3802,8 @@

   int64_t dist_uv;

   int skip_uv;

   PREDICTION_MODE mode_uv = DC_PRED;

-  const int intra_cost_penalty = vp9_get_intra_cost_penalty(

-      cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth);

+  const int intra_cost_penalty =

+      vp9_get_intra_cost_penalty(cpi, bsize, cm->base_qindex, cm->y_dc_delta_q);

   int_mv seg_mvs[4][MAX_REF_FRAMES];

   b_mode_info best_bmodes[4];

   int best_skip2 = 0;