ref: a73cee2870d96f45960b546da931fe78e45a5407
parent: 55fc4d95af8ef330b1156f513aa8797aa65fbc5f
author: paulwilkins <paulwilkins@google.com>
date: Wed Sep 27 14:17:18 EDT 2017
New content type to improve grain retention. For new VP9 only content type adjust the rate distortion and ARF filter based on the relative spatial variance of the source and reconstruction. In regards to the RD loop the method favors modes where the reconstruction variance is similar to the source variance. However it is currently only applied to regions where the source variance is quite low. For very low variance blocks it applies a further bias against intra coding and large prediction block sizes (the later in particular limit the usefulness of the loop filter). The final part of this change is to lower the strength of the ARF filter for blocks where the source has very low spatial variance, to encourage some low amplitude texture or noise to pass through the filter. This change improves the retention of film grain and fine noise / texture in spatially flat regions, but as expected causes a significant drop in PSNR on many clips. This is to be expected because similar but misaligned noise or texture will give a lower PSNR than a flat noise free reconstruction. However, it is worth noting that most clips show a strong gain in FAST SSIM. The features are enabled on the vpxenc command line by setting --tune-content=film. VPX_ENCODER_ABI_VERSION bumped for this change and cvbr. Change-Id: I26a4e4edfa3dc5cacead82fa701fe7a9118ccd0a
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -125,19 +125,17 @@
};
#endif // CONFIG_VP9_HIGHBITDEPTH
-unsigned int vp9_get_sby_perpixel_variance(VP9_COMP *cpi,
- const struct buf_2d *ref,
- BLOCK_SIZE bs) {+unsigned int vp9_get_sby_variance(VP9_COMP *cpi, const struct buf_2d *ref,
+ BLOCK_SIZE bs) {unsigned int sse;
const unsigned int var =
cpi->fn_ptr[bs].vf(ref->buf, ref->stride, VP9_VAR_OFFS, 0, &sse);
- return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
+ return var;
}
#if CONFIG_VP9_HIGHBITDEPTH
-unsigned int vp9_high_get_sby_perpixel_variance(VP9_COMP *cpi,
- const struct buf_2d *ref,
- BLOCK_SIZE bs, int bd) {+unsigned int vp9_high_get_sby_variance(VP9_COMP *cpi, const struct buf_2d *ref,
+ BLOCK_SIZE bs, int bd) {unsigned int var, sse;
switch (bd) {case 10:
@@ -157,8 +155,24 @@
CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_8), 0, &sse);
break;
}
- return (unsigned int)ROUND64_POWER_OF_TWO((int64_t)var,
- num_pels_log2_lookup[bs]);
+ return var;
+}
+#endif // CONFIG_VP9_HIGHBITDEPTH
+
+unsigned int vp9_get_sby_perpixel_variance(VP9_COMP *cpi,
+ const struct buf_2d *ref,
+ BLOCK_SIZE bs) {+ return ROUND_POWER_OF_TWO(vp9_get_sby_variance(cpi, ref, bs),
+ num_pels_log2_lookup[bs]);
+}
+
+#if CONFIG_VP9_HIGHBITDEPTH
+unsigned int vp9_high_get_sby_perpixel_variance(VP9_COMP *cpi,
+ const struct buf_2d *ref,
+ BLOCK_SIZE bs, int bd) {+ return (unsigned int)ROUND64_POWER_OF_TWO(
+ (int64_t)vp9_high_get_sby_variance(cpi, ref, bs, bd),
+ num_pels_log2_lookup[bs]);
}
#endif // CONFIG_VP9_HIGHBITDEPTH
--- a/vp9/encoder/vp9_rd.h
+++ b/vp9/encoder/vp9_rd.h
@@ -194,10 +194,15 @@
int vp9_get_intra_cost_penalty(const struct VP9_COMP *const cpi,
BLOCK_SIZE bsize, int qindex, int qdelta);
+unsigned int vp9_get_sby_variance(struct VP9_COMP *cpi,
+ const struct buf_2d *ref, BLOCK_SIZE bs);
unsigned int vp9_get_sby_perpixel_variance(struct VP9_COMP *cpi,
const struct buf_2d *ref,
BLOCK_SIZE bs);
#if CONFIG_VP9_HIGHBITDEPTH
+unsigned int vp9_high_get_sby_variance(struct VP9_COMP *cpi,
+ const struct buf_2d *ref, BLOCK_SIZE bs,
+ int bd);
unsigned int vp9_high_get_sby_perpixel_variance(struct VP9_COMP *cpi,
const struct buf_2d *ref,
BLOCK_SIZE bs, int bd);
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2876,57 +2876,82 @@
// This function is designed to apply a bias or adjustment to an rd value based
// on the relative variance of the source and reconstruction.
-#define LOW_VAR_THRESH 16
-#define VLOW_ADJ_MAX 25
-#define VHIGH_ADJ_MAX 8
+#define VERY_LOW_VAR_THRESH 2
+#define LOW_VAR_THRESH 5
+#define VAR_MULT 100
+static unsigned int max_var_adjust[VP9E_CONTENT_INVALID] = { 16, 16, 100 };+
static void rd_variance_adjustment(VP9_COMP *cpi, MACROBLOCK *x,
BLOCK_SIZE bsize, int64_t *this_rd,
MV_REFERENCE_FRAME ref_frame,
unsigned int source_variance) {MACROBLOCKD *const xd = &x->e_mbd;
- unsigned int recon_variance;
+ unsigned int rec_variance;
+ unsigned int src_variance;
+ unsigned int src_rec_min;
unsigned int absvar_diff = 0;
- int64_t var_error = 0;
- int64_t var_factor = 0;
+ unsigned int var_factor = 0;
+ unsigned int adj_max;
+ vp9e_tune_content content_type = cpi->oxcf.content;
if (*this_rd == INT64_MAX) return;
#if CONFIG_VP9_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {- recon_variance = vp9_high_get_sby_perpixel_variance(cpi, &xd->plane[0].dst,
+ if (source_variance > 0) {+ rec_variance = vp9_high_get_sby_perpixel_variance(cpi, &xd->plane[0].dst,
bsize, xd->bd);
+ src_variance = source_variance;
+ } else {+ rec_variance =
+ vp9_high_get_sby_variance(cpi, &xd->plane[0].dst, bsize, xd->bd);
+ src_variance =
+ vp9_high_get_sby_variance(cpi, &x->plane[0].src, bsize, xd->bd);
+ }
} else {- recon_variance =
- vp9_get_sby_perpixel_variance(cpi, &xd->plane[0].dst, bsize);
+ if (source_variance > 0) {+ rec_variance =
+ vp9_get_sby_perpixel_variance(cpi, &xd->plane[0].dst, bsize);
+ src_variance = source_variance;
+ } else {+ rec_variance = vp9_get_sby_variance(cpi, &xd->plane[0].dst, bsize);
+ src_variance = vp9_get_sby_variance(cpi, &x->plane[0].src, bsize);
+ }
}
#else
- recon_variance = vp9_get_sby_perpixel_variance(cpi, &xd->plane[0].dst, bsize);
+ if (source_variance > 0) {+ rec_variance = vp9_get_sby_perpixel_variance(cpi, &xd->plane[0].dst, bsize);
+ src_variance = source_variance;
+ } else {+ rec_variance = vp9_get_sby_variance(cpi, &xd->plane[0].dst, bsize);
+ src_variance = vp9_get_sby_variance(cpi, &x->plane[0].src, bsize);
+ }
#endif // CONFIG_VP9_HIGHBITDEPTH
- if ((source_variance + recon_variance) > LOW_VAR_THRESH) {- absvar_diff = (source_variance > recon_variance)
- ? (source_variance - recon_variance)
- : (recon_variance - source_variance);
+ // Lower of source (raw per pixel value) and recon variance. Note that
+ // if the source per pixel is 0 then the recon value here will not be per
+ // pixel (see above) so will likely be much larger.
+ src_rec_min = VPXMIN(source_variance, rec_variance);
- var_error = ((int64_t)200 * source_variance * recon_variance) /
- (((int64_t)source_variance * source_variance) +
- ((int64_t)recon_variance * recon_variance));
- var_error = 100 - var_error;
- }
+ if (src_rec_min > LOW_VAR_THRESH) return;
- // Source variance above a threshold and ref frame is intra.
- // This case is targeted mainly at discouraging intra modes that give rise
- // to a predictor with a low spatial complexity compared to the source.
- if ((source_variance > LOW_VAR_THRESH) && (ref_frame == INTRA_FRAME) &&
- (source_variance > recon_variance)) {- var_factor = VPXMIN(absvar_diff, VPXMIN(VLOW_ADJ_MAX, var_error));
- // A second possible case of interest is where the source variance
- // is very low and we wish to discourage false texture or motion trails.
- } else if ((source_variance < (LOW_VAR_THRESH >> 1)) &&
- (recon_variance > source_variance)) {- var_factor = VPXMIN(absvar_diff, VPXMIN(VHIGH_ADJ_MAX, var_error));
- }
+ absvar_diff = (src_variance > rec_variance) ? (src_variance - rec_variance)
+ : (rec_variance - src_variance);
+
+ adj_max = max_var_adjust[content_type];
+
+ var_factor =
+ (unsigned int)((int64_t)VAR_MULT * absvar_diff) / VPXMAX(1, src_variance);
+ var_factor = VPXMIN(adj_max, var_factor);
+
*this_rd += (*this_rd * var_factor) / 100;
+
+ if (content_type == VP9E_CONTENT_FILM) {+ if (src_rec_min <= VERY_LOW_VAR_THRESH) {+ if (ref_frame == INTRA_FRAME) *this_rd *= 2;
+ if (bsize > 6) *this_rd *= 2;
+ }
+ }
}
// Do we have an internal image edge (e.g. formatting bars).
--- a/vp9/encoder/vp9_temporal_filter.c
+++ b/vp9/encoder/vp9_temporal_filter.c
@@ -350,6 +350,27 @@
td->mb.mv_limits.col_max =
((mb_cols - 1 - mb_col) * 16) + (17 - 2 * VP9_INTERP_EXTEND);
+ if (cpi->oxcf.content == VP9E_CONTENT_FILM) {+ unsigned int src_variance;
+ struct buf_2d src;
+
+ src.buf = f->y_buffer + mb_y_offset;
+ src.stride = f->y_stride;
+
+#if CONFIG_VP9_HIGHBITDEPTH
+ if (mbd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {+ src_variance =
+ vp9_high_get_sby_perpixel_variance(cpi, &src, BLOCK_16X16, mbd->bd);
+ } else {+ src_variance = vp9_get_sby_perpixel_variance(cpi, &src, BLOCK_16X16);
+ }
+#else
+ src_variance = vp9_get_sby_perpixel_variance(cpi, &src, BLOCK_16X16);
+#endif // CONFIG_VP9_HIGHBITDEPTH
+
+ if (src_variance <= 2) strength = VPXMAX(0, (int)strength - 2);
+ }
+
for (frame = 0; frame < frame_count; frame++) {const uint32_t thresh_low = 10000;
const uint32_t thresh_high = 20000;
--- a/vpx/vp8cx.h
+++ b/vpx/vp8cx.h
@@ -444,6 +444,7 @@
* \note Valid parameter range:
* VP9E_CONTENT_DEFAULT = Regular video content (Default)
* VP9E_CONTENT_SCREEN = Screen capture content
+ * VP9E_CONTENT_FILM = Film content: improves grain retention
*
* Supported in codecs: VP9
*/
@@ -696,6 +697,7 @@
typedef enum {VP9E_CONTENT_DEFAULT,
VP9E_CONTENT_SCREEN,
+ VP9E_CONTENT_FILM,
VP9E_CONTENT_INVALID
} vp9e_tune_content;
--- a/vpxenc.c
+++ b/vpxenc.c
@@ -463,6 +463,7 @@
static const struct arg_enum_list tune_content_enum[] = { { "default", VP9E_CONTENT_DEFAULT }, { "screen", VP9E_CONTENT_SCREEN },+ { "film", VP9E_CONTENT_FILM }, { NULL, 0 }};