ref: d48be6ad734890e0093d25df61b2943d1f402c0d
parent: 15193ce51f8a18ec0c58a94980657015d50c28fd
parent: 0f8ebddec40c65a8aa0c60b35ab44822745d7027
author: Jerome Jiang <jianj@google.com>
date: Thu Aug 10 20:54:35 EDT 2017
Merge "vp9 SVC: Fix the denoiser frame buffer management."
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -190,11 +190,12 @@
VP9_COMMON *const cm, VP9_DENOISER *denoiser, MACROBLOCK *mb, BLOCK_SIZE bs,
int increase_denoising, int mi_row, int mi_col, PICK_MODE_CONTEXT *ctx,
int motion_magnitude, int is_skin, int *zeromv_filter, int consec_zeromv,
- int num_spatial_layers, int width) {+ int num_spatial_layers, int width, int lst_fb_idx, int gld_fb_idx,
+ int use_svc) {const int sse_diff = (ctx->newmv_sse == UINT_MAX)
? 0
: ((int)ctx->zeromv_sse - (int)ctx->newmv_sse);
- MV_REFERENCE_FRAME frame;
+ int frame;
MACROBLOCKD *filter_mbd = &mb->e_mbd;
MODE_INFO *mi = filter_mbd->mi[0];
MODE_INFO saved_mi;
@@ -202,8 +203,10 @@
struct buf_2d saved_dst[MAX_MB_PLANE];
struct buf_2d saved_pre[MAX_MB_PLANE];
RefBuffer *saved_block_refs[2];
+ MV_REFERENCE_FRAME saved_frame;
frame = ctx->best_reference_frame;
+
saved_mi = *mi;
if (is_skin && (motion_magnitude > 0 || consec_zeromv < 4)) return COPY_BLOCK;
@@ -246,6 +249,15 @@
}
}
+ saved_frame = frame;
+ // When using SVC, we need to map REF_FRAME to the frame buffer index.
+ if (use_svc) {+ if (frame == LAST_FRAME)
+ frame = lst_fb_idx + 1;
+ else if (frame == GOLDEN_FRAME)
+ frame = gld_fb_idx + 1;
+ }
+
if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {// Restore everything to its original state
*mi = saved_mi;
@@ -292,7 +304,7 @@
denoiser->mc_running_avg_y.uv_stride, mi_row, mi_col);
filter_mbd->plane[2].dst.stride = denoiser->mc_running_avg_y.uv_stride;
- set_ref_ptrs(cm, filter_mbd, frame, NONE);
+ set_ref_ptrs(cm, filter_mbd, saved_frame, NONE);
vp9_build_inter_predictors_sby(filter_mbd, mi_row, mi_col, bs);
// Restore everything to its original state
@@ -370,7 +382,8 @@
decision = perform_motion_compensation(
&cpi->common, denoiser, mb, bs, increase_denoising, mi_row, mi_col, ctx,
motion_magnitude, is_skin, &zeromv_filter, consec_zeromv,
- cpi->svc.number_spatial_layers, cpi->Source->y_width);
+ cpi->svc.number_spatial_layers, cpi->Source->y_width, cpi->lst_fb_idx,
+ cpi->gld_fb_idx, cpi->use_svc);
if (decision == FILTER_BLOCK) {decision = vp9_denoiser_filter(src.buf, src.stride, mc_avg_start,
@@ -417,23 +430,11 @@
src->y_buffer = tmp_buf;
}
-void vp9_denoise_init_svc(VP9_COMP *cpi) {- // For fixed pattern SVC, on base temporal layer. Note we only denoise
- // higher spatial layer for SVC.
- if (cpi->svc.temporal_layering_mode != VP9E_TEMPORAL_LAYERING_MODE_BYPASS &&
- cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
- cpi->svc.temporal_layer_id == 0) {- VP9_DENOISER *denoiser = &cpi->denoiser;
- copy_frame(&denoiser->running_avg_y[LAST_FRAME],
- &denoiser->running_avg_y[GOLDEN_FRAME]);
- }
-}
-
void vp9_denoiser_update_frame_info(
VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
- int resized, int svc_base_is_key, int svc_fixed_pattern,
- int temporal_layer_id) {+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
+ int svc_base_is_key) {// Copy source into denoised reference buffers on KEY_FRAME or
// if the just encoded frame was resized. For SVC, copy source if the base
// spatial layer was key frame.
@@ -441,48 +442,42 @@
svc_base_is_key) {int i;
// Start at 1 so as not to overwrite the INTRA_FRAME
- for (i = 1; i < denoiser->num_ref_frames; ++i)
- copy_frame(&denoiser->running_avg_y[i], &src);
+ for (i = 1; i < denoiser->num_ref_frames; ++i) {+ if (denoiser->running_avg_y[i].buffer_alloc != NULL)
+ copy_frame(&denoiser->running_avg_y[i], &src);
+ }
denoiser->reset = 0;
return;
}
// If more than one refresh occurs, must copy frame buffer.
- if (refresh_golden_frame + refresh_last_frame + refresh_alt_ref_frame > 1) {+ if ((refresh_alt_ref_frame + refresh_golden_frame + refresh_last_frame) > 1) {+ if (refresh_alt_ref_frame) {+ copy_frame(&denoiser->running_avg_y[alt_fb_idx + 1],
+ &denoiser->running_avg_y[INTRA_FRAME]);
+ }
if (refresh_golden_frame) {- copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
+ copy_frame(&denoiser->running_avg_y[gld_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
- // For fixed pattern SVC: update denoised last_frame if alt_ref is
- // refreshed, only for non-zero temporal layer.
- if (refresh_last_frame ||
- (refresh_alt_ref_frame && svc_fixed_pattern && temporal_layer_id > 0)) {- copy_frame(&denoiser->running_avg_y[LAST_FRAME],
+ if (refresh_last_frame) {+ copy_frame(&denoiser->running_avg_y[lst_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
} else {+ if (refresh_alt_ref_frame) {+ swap_frame_buffer(&denoiser->running_avg_y[alt_fb_idx + 1],
+ &denoiser->running_avg_y[INTRA_FRAME]);
+ }
if (refresh_golden_frame) {- swap_frame_buffer(&denoiser->running_avg_y[GOLDEN_FRAME],
+ swap_frame_buffer(&denoiser->running_avg_y[gld_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
- // For fixed pattern SVC: update denoised last_frame if alt_ref is
- // refreshed, only for non-zero temporal layer.
- if (refresh_last_frame ||
- (refresh_alt_ref_frame && svc_fixed_pattern && temporal_layer_id > 0)) {- swap_frame_buffer(&denoiser->running_avg_y[LAST_FRAME],
+ if (refresh_last_frame) {+ swap_frame_buffer(&denoiser->running_avg_y[lst_fb_idx + 1],
&denoiser->running_avg_y[INTRA_FRAME]);
}
}
- // For fixed pattern SVC we need to keep track of denoised last_frame for base
- // temporal layer (since alt_ref refresh may update denoised last_frame on
- // the upper/middle temporal layers).We do this by copying the current
- // denoised last into the denoised golden_frame, for temporal_layer_id = 0.
- // For the fixed pattern SVC golden is always spatial reference and is never
- // used for denoising, so we can use it to keep track of denoised last_frame.
- if (svc_fixed_pattern && temporal_layer_id == 0) {- copy_frame(&denoiser->running_avg_y[GOLDEN_FRAME],
- &denoiser->running_avg_y[LAST_FRAME]);
- }
}
void vp9_denoiser_reset_frame_stats(PICK_MODE_CONTEXT *ctx) {@@ -509,6 +504,46 @@
}
}
+static int vp9_denoiser_realloc_svc_helper(VP9_COMMON *cm,
+ VP9_DENOISER *denoiser, int fb_idx) {+ int fail = 0;
+ if (denoiser->running_avg_y[fb_idx].buffer_alloc == NULL) {+ fail =
+ vpx_alloc_frame_buffer(&denoiser->running_avg_y[fb_idx], cm->width,
+ cm->height, cm->subsampling_x, cm->subsampling_y,
+#if CONFIG_VP9_HIGHBITDEPTH
+ cm->use_highbitdepth,
+#endif
+ VP9_ENC_BORDER_IN_PIXELS, 0);
+ if (fail) {+ vp9_denoiser_free(denoiser);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+int vp9_denoiser_realloc_svc(VP9_COMMON *cm, VP9_DENOISER *denoiser,
+ int refresh_alt, int refresh_gld, int refresh_lst,
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx) {+ int fail = 0;
+ if (refresh_alt) {+ // Increase the frame buffer index by 1 to map it to the buffer index in the
+ // denoiser.
+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, alt_fb_idx + 1);
+ if (fail) return 1;
+ }
+ if (refresh_gld) {+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, gld_fb_idx + 1);
+ if (fail) return 1;
+ }
+ if (refresh_lst) {+ fail = vp9_denoiser_realloc_svc_helper(cm, denoiser, lst_fb_idx + 1);
+ if (fail) return 1;
+ }
+ return 0;
+}
+
int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
int width, int height, int ssx, int ssy,
#if CONFIG_VP9_HIGHBITDEPTH
@@ -515,15 +550,16 @@
int use_highbitdepth,
#endif
int border) {- int i, fail;
+ int i, fail, init_num_ref_frames;
const int legacy_byte_alignment = 0;
assert(denoiser != NULL);
- denoiser->num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
+ denoiser->num_ref_frames = use_svc ? SVC_REF_FRAMES : NONSVC_REF_FRAMES;
+ init_num_ref_frames = use_svc ? MAX_REF_FRAMES : NONSVC_REF_FRAMES;
CHECK_MEM_ERROR(
cm, denoiser->running_avg_y,
vpx_calloc(denoiser->num_ref_frames, sizeof(denoiser->running_avg_y[0])));
- for (i = 0; i < denoiser->num_ref_frames; ++i) {+ for (i = 0; i < init_num_ref_frames; ++i) {fail = vpx_alloc_frame_buffer(&denoiser->running_avg_y[i], width, height,
ssx, ssy,
#if CONFIG_VP9_HIGHBITDEPTH
@@ -594,7 +630,8 @@
denoiser->prev_denoising_level = denoiser->denoising_level;
}
-// Scale/increase the partition threshold for denoiser speed-up.
+// Scale/increase the partition threshold
+// for denoiser speed-up.
int64_t vp9_scale_part_thresh(int64_t threshold, VP9_DENOISER_LEVEL noise_level,
int content_state, int temporal_layer_id) {if ((content_state == kLowSadLowSumdiff) ||
@@ -609,7 +646,8 @@
}
}
-// Scale/increase the ac skip threshold for denoiser speed-up.
+// Scale/increase the ac skip threshold for
+// denoiser speed-up.
int64_t vp9_scale_acskip_thresh(int64_t threshold,
VP9_DENOISER_LEVEL noise_level, int abs_sumdiff,
int temporal_layer_id) {--- a/vp9/encoder/vp9_denoiser.h
+++ b/vp9/encoder/vp9_denoiser.h
@@ -25,6 +25,10 @@
// need to allocate for it, and hence we need MAX_REF_FRAME - 1
#define NONSVC_REF_FRAMES MAX_REF_FRAMES - 1
+// Number of frame buffers when SVC is used. [0] for current denoised buffer and
+// [1..8] for REF_FRAMES
+#define SVC_REF_FRAMES 9
+
typedef enum vp9_denoiser_decision {COPY_BLOCK,
FILTER_BLOCK,
@@ -63,13 +67,11 @@
struct VP9_COMP;
-void vp9_denoise_init_svc(struct VP9_COMP *cpi);
-
void vp9_denoiser_update_frame_info(
VP9_DENOISER *denoiser, YV12_BUFFER_CONFIG src, FRAME_TYPE frame_type,
int refresh_alt_ref_frame, int refresh_golden_frame, int refresh_last_frame,
- int resized, int svc_base_is_key, int svc_fixed_pattern,
- int temporal_layer_id);
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx, int resized,
+ int svc_base_is_key);
void vp9_denoiser_denoise(struct VP9_COMP *cpi, MACROBLOCK *mb, int mi_row,
int mi_col, BLOCK_SIZE bs, PICK_MODE_CONTEXT *ctx,
@@ -80,6 +82,10 @@
void vp9_denoiser_update_frame_stats(MODE_INFO *mi, unsigned int sse,
PREDICTION_MODE mode,
PICK_MODE_CONTEXT *ctx);
+
+int vp9_denoiser_realloc_svc(VP9_COMMON *cm, VP9_DENOISER *denoiser,
+ int refresh_alt, int refresh_gld, int refresh_lst,
+ int alt_fb_idx, int gld_fb_idx, int lst_fb_idx);
int vp9_denoiser_alloc(VP9_COMMON *cm, int use_svc, VP9_DENOISER *denoiser,
int width, int height, int ssx, int ssy,
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -2833,21 +2833,29 @@
if (cpi->oxcf.noise_sensitivity > 0 && denoise_svc(cpi) &&
cpi->denoiser.denoising_level > kDenLowLow) {int svc_base_is_key = 0;
- int svc_fixed_pattern = 0;
if (cpi->use_svc) {+ int realloc_fail = 0;
int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
cpi->svc.temporal_layer_id,
cpi->svc.number_temporal_layers);
LAYER_CONTEXT *lc = &cpi->svc.layer_context[layer];
svc_base_is_key = lc->is_key_frame;
- svc_fixed_pattern = (cpi->svc.temporal_layering_mode !=
- VP9E_TEMPORAL_LAYERING_MODE_BYPASS);
+
+ // Check if we need to allocate extra buffers in the denoiser for
+ // refreshed frames.
+ realloc_fail = vp9_denoiser_realloc_svc(
+ cm, &cpi->denoiser, cpi->refresh_alt_ref_frame,
+ cpi->refresh_golden_frame, cpi->refresh_last_frame, cpi->alt_fb_idx,
+ cpi->gld_fb_idx, cpi->lst_fb_idx);
+ if (realloc_fail)
+ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
+ "Failed to re-allocate denoiser for SVC");
}
vp9_denoiser_update_frame_info(
&cpi->denoiser, *cpi->Source, cpi->common.frame_type,
cpi->refresh_alt_ref_frame, cpi->refresh_golden_frame,
- cpi->refresh_last_frame, cpi->resize_pending, svc_base_is_key,
- svc_fixed_pattern, cpi->svc.temporal_layer_id);
+ cpi->refresh_last_frame, cpi->alt_fb_idx, cpi->gld_fb_idx,
+ cpi->lst_fb_idx, cpi->resize_pending, svc_base_is_key);
}
#endif
if (is_one_pass_cbr_svc(cpi)) {@@ -3480,12 +3488,6 @@
}
vp9_update_noise_estimate(cpi);
-
-#if CONFIG_VP9_TEMPORAL_DENOISING
- if (cpi->oxcf.noise_sensitivity > 0 && cpi->use_svc &&
- cpi->denoiser.denoising_level > kDenLowLow)
- vp9_denoise_init_svc(cpi);
-#endif
// Scene detection is always used for VBR mode or screen-content case.
// For other cases (e.g., CBR mode) use it for 5 <= speed < 8 for now