ref: 0f8ebddec40c65a8aa0c60b35ab44822745d7027
parent: 847394fe77f57a0752c02834cf545db1cdea9132
author: Jerome Jiang <jianj@google.com>
date: Mon Aug 7 12:32:26 EDT 2017
vp9 SVC: Fix the denoiser frame buffer management. Change the denoiser frame buffer management for SVC to more generally handle the layer patterns in SVC (where last is not always refreshed). This change is only for SVC with denoising and is bitexact. Change-Id: Ic2b146a924cdf6e7114609158afa3d4880fe3fae
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -190,11 +190,12 @@
VP9_COMMON *const cm, VP9_DENOISER *denoiser, MACROBLOCK *mb, BLOCK_SIZE bs,
int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx,
int motion_magnitude, int is_skin, int *zeromv_filter, int consec_zeromv,
- int num_spatial_layers, int width) {+ int num_spatial_layers, int width, int lst_fb_idx, int gld_fb_idx,
+ int use_svc) {const int sse_diff = (ctx->newmv_sse == UINT_MAX)
? 0
: ((int)ctx->zeromv_sse - (int)ctx->newmv_sse);
- MV_REFERENCE_FRAME frame;
+ int frame;
MACROBLOCKD *filter_mbd = &mb->e_mbd;
MODE_INFO *mi = filter_mbd->mi[0];
MODE_INFO saved_mi;
@@ -202,8 +203,10 @@
struct buf_2d saved_dst[MAX_MB_PLANE];
struct buf_2d saved_pre[MAX_MB_PLANE];
RefBuffer *saved_block_refs[2];
+ MV_REFERENCE_FRAME saved_frame;
frame = ctx->best_reference_frame;
+
saved_mi = *mi;
if (is_skin && (motion_magnitude > 0 || consec_zeromv < 4)) return COPY_BLOCK;
@@ -246,6 +249,15 @@
}
}
+ saved_frame = frame;
+ // When using SVC, we need to map REF_FRAME to the frame buffer index.
+ if (use_svc) {+ if (frame == LAST_FRAME)
+ frame = lst_fb_idx + 1;
+ else if (frame == GOLDEN_FRAME)
+ frame = gld_fb_idx + 1;
+ }
+
if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {// Restore everything to its original state
*mi = saved_mi;
@@ -292,7 +304,7 @@
denoiser->mc_running_avg_y.uv_stride, mi_row, mi_col);
filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
- set_ref_ptrs(cm, filter_mbd, frame, NONE);
+ set_ref_ptrs(cm, filter_mbd, saved_frame, NONE);
vp9_build_inter_predictors_sby(filter_mbd, mi_row, mi_col, bs);
// Restore everything to its original state
@@ -370,7 +382,8 @@
decision = perform_motion_compensation(
&cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx,
motion_magnitude, is_skin, &zeromv_filter, consec_zeromv,
- cpi->svc.number_spatial_layers, cpi->Source->y_width);
+ cpi->svc.number_spatial_layers, cpi->Source->y_width, cpi->lst_fb_idx,
+ cpi->gld_fb_idx, cpi->use_svc);
if (decision == FILTER_BLOCK) {decision = vp9_denoiser_filter(src.buf, src.stride, mc_avg_start,
@@ -417,23 +430,11 @@
src->y_buffer = tmp_buf;
}
-void vp9_denoise_init_svc(VP9_COMP *cpi) {- // For fixed pattern SVC, on base temporal layer. Note we only denoise
- // higher spatial layer for SVC.
- if (cpi->svc.temporal_layering_mode != VP9E_TEMPORAL_LAYERING_MODE_BYPASS &&
- cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
- cpi->svc.temporal_layer_id == 0) {- VP9_DENOISER *denoiser = &cpi->denoiser;
- copy_frame(&denoiser->running_avg_y[LAST_FRAME],
- &denoiser->running_avg_y[GOLDEN_FRAME]);
- }
-}
-
void vp9_denoiser_update_frame_info(
VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
- int resized, int svc_base_is_key, int svc_fixed_pattern,
- int temporal_layer_id) {+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
+ int svc_base_is_key) {// Copy source into denoised reference buffers on KEY_FRAME or
// if the just encoded frame was resized. For SVC, copy source if the base
// spatial layer was key frame.
@@ -441,48 +442,42 @@
svc_base_is_key) {int i;
// Start at 1 so as not to overwrite the INTRA_FRAME
- for (i = 1; i < denoiser->num_ref_frames; ++i)
- copy_frame(&denoiser->running_avg_y[i], &src);
+ for (i = 1; i < denoiser->num_ref_frames; ++i) {+ if (denoiser->running_avg_y[i].buffer_alloc != NULL)
+ copy_frame(&denoiser->running_avg_y[i], &src);
+ }
denoiser->reset = 0;
return;
}
// If more than one refresh occurs, must copy frame buffer.
- if (refresh_golden_frame + refresh_last_frame + refresh_alt_ref_frame > 1) {+ if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) > 1) {+ if (refresh_alt_ref_frame) {+ copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1],
+ &denoiser->running_avg_y[INTRA_FRAME]);
+ }
if (refresh_golden_frame) {- copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
+ copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
- // For fixed pattern SVC: update denoised last_frame if alt_ref is
- // refreshed, only for non-zero temporal layer.
- if (refresh_last_frame ||
- (refresh_alt_ref_frame && svc_fixed_pattern && temporal_layer_id > 0)) {- copy_frame(&denoiser->running_avg_y[LAST_FRAME],
+ if (refresh_last_frame) {+ copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
} else {+ if (refresh_alt_ref_frame) {+ swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1],
+ &denoiser->running_avg_y[INTRA_FRAME]);
+ }
if (refresh_golden_frame) {- swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
+ swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
- // For fixed pattern SVC: update denoised last_frame if alt_ref is
- // refreshed, only for non-zero temporal layer.
- if (refresh_last_frame ||
- (refresh_alt_ref_frame && svc_fixed_pattern && temporal_layer_id > 0)) {- swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
+ if (refresh_last_frame) {+ swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
}
- // For fixed pattern SVC we need to keep track of denoised last_frame for base
- // temporal layer (since alt_ref refresh may update denoised last_frame on
- // the upper/middle temporal layers).We do this by copying the current
- // denoised last into the denoised golden_frame, for temporal_layer_id = 0.
- // For the fixed pattern SVC golden is always spatial reference and is never
- // used for denoising, so we can use it to keep track of denoised last_frame.
- if (svc_fixed_pattern && temporal_layer_id == 0) {- copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
- &denoiser->running_avg_y[LAST_FRAME]);
- }
}
void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {@@ -509,6 +504,46 @@
}
}
+static int vp9_denoiser_realloc_svc_helper(VP9_COMMON *cm,
+ VP9_DENOISER *denoiser, int fb_idx) {+ int fail = 0;
+ if (denoiser->running_avg_y[fb_idx].buffer_alloc == NULL) {+ fail =
+ vpx_alloc_frame_buffer(&denoiser->running_avg_y[fb_idx], cm->width,
+ cm->height, cm->subsampling_x, cm->subsampling_y,
+#if CONFIG_VP9_HIGHBITDEPTH
+ cm->use_highbitdepth,
+#endif
+ VP9_ENC_BORDER_IN_PIXELS, 0);
+ if (fail) {+ vp9_denoiser_free(denoiser);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+int vp9_denoiser_realloc_svc(VP9_COMMON *cm, VP9_DENOISER *denoiser,
+ int refresh_alt, int refresh_gld, int refresh_lst,
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) {+ int fail = 0;
+ if (refresh_alt) {+ // Increase the frame buffer index by 1 to map it to the buffer index in the
+ // denoiser.
+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, alt_fb_idx + 1);
+ if (fail) return 1;
+ }
+ if (refresh_gld) {+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, gld_fb_idx + 1);
+ if (fail) return 1;
+ }
+ if (refresh_lst) {+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, lst_fb_idx + 1);
+ if (fail) return 1;
+ }
+ return 0;
+}
+
int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
int width, int height, int ssx, int ssy,
#if CONFIG_VP9_HIGHBITDEPTH
@@ -515,15 +550,16 @@
int use_highbitdepth,
#endif
int border) {- int i, fail;
+ int i, fail, init_num_ref_frames;
const int legacy_byte_alignment = 0;
assert(denoiser != NULL);
- denoiser->num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
+ denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES;
+ init_num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
CHECK_MEM_ERROR(
cm, denoiser->running_avg_y,
vpx_calloc(denoiser->num_ref_frames, sizeof(denoiser->running_avg_y[0])));
- for (i = 0; i < denoiser->num_ref_frames; ++i) {+ for (i = 0; i < init_num_ref_frames; ++i) {fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
ssx, ssy,
#if CONFIG_VP9_HIGHBITDEPTH
@@ -594,7 +630,8 @@
denoiser->prev_denoising_level = denoiser->denoising_level;
}
-// Scale/increase the partition threshold for denoiser speed-up.
+// Scale/increase the partition threshold
+// for denoiser speed-up.
int64_t vp9_scale_part_thresh(int64_t threshold, VP9_DENOISER_LEVEL noise_level,
int content_state, int temporal_layer_id) {if ((content_state == kLowSadLowSumdiff) ||
@@ -609,7 +646,8 @@
}
}
-// Scale/increase the ac skip threshold for denoiser speed-up.
+// Scale/increase the ac skip threshold for
+// denoiser speed-up.
int64_t vp9_scale_acskip_thresh(int64_t threshold,
VP9_DENOISER_LEVEL noise_level, int abs_sumdiff,
int temporal_layer_id) {--- a/vp9/encoder/vp9_denoiser.h
+++ b/vp9/encoder/vp9_denoiser.h
@@ -25,6 +25,10 @@
// need to allocate for it, and hence we need MAX_REF_FRAME - 1
#define NONSVC_REF_FRAMES MAX_REF_FRAMES - 1
+// Number of frame buffers when SVC is used. [0] for current denoised buffer and
+// [1..8] for REF_FRAMES
+#define SVC_REF_FRAMES 9
+
typedef enum vp9_denoiser_decision {COPY_BLOCK,
FILTER_BLOCK,
@@ -63,13 +67,11 @@
struct VP9_COMP;
-void vp9_denoise_init_svc(struct VP9_COMP *cpi);
-
void vp9_denoiser_update_frame_info(
VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
- int resized, int svc_base_is_key, int svc_fixed_pattern,
- int temporal_layer_id);
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
+ int svc_base_is_key);
void vp9_denoiser_denoise(struct VP9_COMP *cpi, MACROBLOCK *mb, int mi_row,
int mi_col, BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
@@ -80,6 +82,10 @@
void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse,
PREDICTION_MODE mode,
PICK_MODE_CONTEXT *ctx);
+
+int vp9_denoiser_realloc_svc(VP9_COMMON *cm, VP9_DENOISER *denoiser,
+ int refresh_alt, int refresh_gld, int refresh_lst,
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx);
int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
int width, int height, int ssx, int ssy,
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -2833,21 +2833,29 @@
if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
cpi->denoiser.denoising_level > kDenLowLow) {int svc_base_is_key = 0;
- int svc_fixed_pattern = 0;
if (cpi->use_svc) {+ int realloc_fail = 0;
int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
cpi->svc.temporal_layer_id,
cpi->svc.number_temporal_layers);
LAYER_CONTEXT *lc = &cpi->svc.layer_context[layer];
svc_base_is_key = lc->is_key_frame;
- svc_fixed_pattern = (cpi->svc.temporal_layering_mode !=
- VP9E_TEMPORAL_LAYERING_MODE_BYPASS);
+
+ // Check if we need to allocate extra buffers in the denoiser for
+ // refreshed frames.
+ realloc_fail = vp9_denoiser_realloc_svc(
+ cm, &cpi->denoiser, cpi->refresh_alt_ref_frame,
+ cpi->refresh_golden_frame, cpi->refresh_last_frame, cpi->alt_fb_idx,
+ cpi->gld_fb_idx, cpi->lst_fb_idx);
+ if (realloc_fail)
+ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+ "Failed to re-allocate denoiser for SVC");
}
vp9_denoiser_update_frame_info(
&cpi->denoiser, *cpi->Source, cpi->common.frame_type,
cpi->refresh_alt_ref_frame, cpi->refresh_golden_frame,
- cpi->refresh_last_frame, cpi->resize_pending, svc_base_is_key,
- svc_fixed_pattern, cpi->svc.temporal_layer_id);
+ cpi->refresh_last_frame, cpi->alt_fb_idx, cpi->gld_fb_idx,
+ cpi->lst_fb_idx, cpi->resize_pending, svc_base_is_key);
}
#endif
if (is_one_pass_cbr_svc(cpi)) {@@ -3480,12 +3488,6 @@
}
vp9_update_noise_estimate(cpi);
-
-#if CONFIG_VP9_TEMPORAL_DENOISING
- if (cpi->oxcf.noise_sensitivity > 0 && cpi->use_svc &&
- cpi->denoiser.denoising_level > kDenLowLow)
- vp9_denoise_init_svc(cpi);
-#endif
// Scene detection is always used for VBR mode or screen-content case.
// For other cases (e.g., CBR mode) use it for 5 <= speed < 8 for now