ref: 4066c8b20566183f995a981173ae803870e05a8a
parent: f31c40086ef4e4c3aa0f399831a6c2388474bf32
author: Yunqing Wang <yunqingwang@google.com>
date: Fri Jun 8 07:17:50 EDT 2012
multi-res: add drop_frame support Added drop_frame support in multi-resolution encoder. If one frame is dropped at a lower-resolution level, the next upper-resolution level encoder needs to encode that frame independently without any lower-resolution level motion information. Another issue is that if one frame is dropped at some but not all resolution levels, a frame after that one may use different set of reference frames at different resolution levels. This reference frame asynchronization could degrade motion search precision in upper-resolution level encoding, which uses lower-resolution level motion result. This change compares the lower-resolution and upper- resolution level's reference frames. If they are not the same, the upper-resolution level encoder can not use lower-resolution level motion result. Change-Id: I61afa4f313630e75b7cbdd5742e230e8724a988a
--- a/vp8/common/blockd.h
+++ b/vp8/common/blockd.h
@@ -182,6 +182,9 @@
typedef struct
{FRAME_TYPE frame_type;
+ int is_frame_dropped;
+ /* The frame number of each reference frames */
+ unsigned int low_res_ref_frames[MAX_REF_FRAMES];
LOWER_RES_MB_INFO *mb_info;
} LOWER_RES_FRAME_INFO;
#endif
--- a/vp8/encoder/mr_dissim.c
+++ b/vp8/encoder/mr_dissim.c
@@ -53,6 +53,7 @@
void vp8_cal_dissimilarity(VP8_COMP *cpi)
{VP8_COMMON *cm = &cpi->common;
+ int i;
/* Note: The first row & first column in mip are outside the frame, which
* were initialized to all 0.(ref_frame, mode, mv...)
@@ -72,6 +73,13 @@
if(cm->frame_type != KEY_FRAME)
{+ store_info->is_frame_dropped = 0;
+ for (i = 1; i < MAX_REF_FRAMES; i++)
+ store_info->low_res_ref_frames[i] = cpi->current_ref_frames[i];
+ }
+
+ if(cm->frame_type != KEY_FRAME)
+ {int mb_row;
int mb_col;
/* Point to beginning of allocated MODE_INFO arrays. */
@@ -201,5 +209,28 @@
}
}
}
+ }
+}
+
+/* This function is called only when this frame is dropped at current
+ resolution level. */
+void vp8_store_drop_frame_info(VP8_COMP *cpi)
+{+ /* If the frame is dropped in lower-resolution encoding, this information
+ is passed to higher resolution level so that the encoder knows there
+ is no mode & motion info available.
+ */
+ if (cpi->oxcf.mr_total_resolutions >1
+ && cpi->oxcf.mr_encoder_id < (cpi->oxcf.mr_total_resolutions - 1))
+ {+ /* Store info for show/no-show frames for supporting alt_ref.
+ * If parent frame is alt_ref, child has one too.
+ */
+ LOWER_RES_FRAME_INFO* store_info
+ = (LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info;
+
+ /* Set frame_type to be INTER_FRAME since we won't drop key frame. */
+ store_info->frame_type = INTER_FRAME;
+ store_info->is_frame_dropped = 1;
}
}
--- a/vp8/encoder/mr_dissim.h
+++ b/vp8/encoder/mr_dissim.h
@@ -15,5 +15,6 @@
extern void vp8_cal_low_res_mb_cols(VP8_COMP *cpi);
extern void vp8_cal_dissimilarity(VP8_COMP *cpi);
+extern void vp8_store_drop_frame_info(VP8_COMP *cpi);
#endif
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -3014,8 +3014,9 @@
return force_recode;
}
-static void update_reference_frames(VP8_COMMON *cm)
+static void update_reference_frames(VP8_COMP *cpi)
{+ VP8_COMMON *cm = &cpi->common;
YV12_BUFFER_CONFIG *yv12_fb = cm->yv12_fb;
/* At this point the new frame has been encoded.
@@ -3030,6 +3031,11 @@
yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
cm->alt_fb_idx = cm->gld_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[GOLDEN_FRAME] = cm->current_video_frame;
+ cpi->current_ref_frames[ALTREF_FRAME] = cm->current_video_frame;
+#endif
}
else /* For non key frames */
{@@ -3040,6 +3046,10 @@
cm->yv12_fb[cm->new_fb_idx].flags |= VP8_ALTR_FRAME;
cm->yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
cm->alt_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[ALTREF_FRAME] = cm->current_video_frame;
+#endif
}
else if (cm->copy_buffer_to_arf)
{@@ -3052,6 +3062,11 @@
yv12_fb[cm->lst_fb_idx].flags |= VP8_ALTR_FRAME;
yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
cm->alt_fb_idx = cm->lst_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[ALTREF_FRAME] =
+ cpi->current_ref_frames[LAST_FRAME];
+#endif
}
}
else /* if (cm->copy_buffer_to_arf == 2) */
@@ -3061,6 +3076,11 @@
yv12_fb[cm->gld_fb_idx].flags |= VP8_ALTR_FRAME;
yv12_fb[cm->alt_fb_idx].flags &= ~VP8_ALTR_FRAME;
cm->alt_fb_idx = cm->gld_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[ALTREF_FRAME] =
+ cpi->current_ref_frames[GOLDEN_FRAME];
+#endif
}
}
}
@@ -3072,6 +3092,10 @@
cm->yv12_fb[cm->new_fb_idx].flags |= VP8_GOLD_FRAME;
cm->yv12_fb[cm->gld_fb_idx].flags &= ~VP8_GOLD_FRAME;
cm->gld_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[GOLDEN_FRAME] = cm->current_video_frame;
+#endif
}
else if (cm->copy_buffer_to_gf)
{@@ -3084,6 +3108,11 @@
yv12_fb[cm->lst_fb_idx].flags |= VP8_GOLD_FRAME;
yv12_fb[cm->gld_fb_idx].flags &= ~VP8_GOLD_FRAME;
cm->gld_fb_idx = cm->lst_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[GOLDEN_FRAME] =
+ cpi->current_ref_frames[LAST_FRAME];
+#endif
}
}
else /* if (cm->copy_buffer_to_gf == 2) */
@@ -3093,6 +3122,11 @@
yv12_fb[cm->alt_fb_idx].flags |= VP8_GOLD_FRAME;
yv12_fb[cm->gld_fb_idx].flags &= ~VP8_GOLD_FRAME;
cm->gld_fb_idx = cm->alt_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[GOLDEN_FRAME] =
+ cpi->current_ref_frames[ALTREF_FRAME];
+#endif
}
}
}
@@ -3103,6 +3137,10 @@
cm->yv12_fb[cm->new_fb_idx].flags |= VP8_LAST_FRAME;
cm->yv12_fb[cm->lst_fb_idx].flags &= ~VP8_LAST_FRAME;
cm->lst_fb_idx = cm->new_fb_idx;
+
+#if CONFIG_MULTI_RES_ENCODING
+ cpi->current_ref_frames[LAST_FRAME] = cm->current_video_frame;
+#endif
}
}
@@ -3313,8 +3351,28 @@
*/
if (cpi->oxcf.mr_encoder_id)
{- cm->frame_type =
- ((LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info)->frame_type;
+ LOWER_RES_FRAME_INFO* low_res_frame_info
+ = (LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info;
+
+ cm->frame_type = low_res_frame_info->frame_type;
+
+ if(cm->frame_type != KEY_FRAME)
+ {+ cpi->mr_low_res_mv_avail = 1;
+ cpi->mr_low_res_mv_avail &= !(low_res_frame_info->is_frame_dropped);
+
+ if (cpi->ref_frame_flags & VP8_LAST_FRAME)
+ cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[LAST_FRAME]
+ == low_res_frame_info->low_res_ref_frames[LAST_FRAME]);
+
+ if (cpi->ref_frame_flags & VP8_GOLD_FRAME)
+ cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[GOLDEN_FRAME]
+ == low_res_frame_info->low_res_ref_frames[GOLDEN_FRAME]);
+
+ if (cpi->ref_frame_flags & VP8_ALTR_FRAME)
+ cpi->mr_low_res_mv_avail &= (cpi->current_ref_frames[ALTREF_FRAME]
+ == low_res_frame_info->low_res_ref_frames[ALTREF_FRAME]);
+ }
}
#endif
@@ -3422,6 +3480,10 @@
if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size)
cpi->bits_off_target = cpi->oxcf.maximum_buffer_size;
+#if CONFIG_MULTI_RES_ENCODING
+ vp8_store_drop_frame_info(cpi);
+#endif
+
cm->current_video_frame++;
cpi->frames_since_key++;
@@ -3459,6 +3521,10 @@
/* Decide how big to make the frame */
if (!vp8_pick_frame_size(cpi))
{+ /*TODO: 2 drop_frame and return code could be put together. */
+#if CONFIG_MULTI_RES_ENCODING
+ vp8_store_drop_frame_info(cpi);
+#endif
cm->current_video_frame++;
cpi->frames_since_key++;
return;
@@ -4231,7 +4297,7 @@
vp8_loopfilter_frame(cpi, cm);
}
- update_reference_frames(cm);
+ update_reference_frames(cpi);
#if !(CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING)
if (cpi->oxcf.error_resilient_mode)
--- a/vp8/encoder/onyx_int.h
+++ b/vp8/encoder/onyx_int.h
@@ -701,6 +701,10 @@
#if CONFIG_MULTI_RES_ENCODING
/* Number of MBs per row at lower-resolution level */
int mr_low_res_mb_cols;
+ /* Indicate if lower-res mv info is available */
+ unsigned char mr_low_res_mv_avail;
+ /* The frame number of each reference frames */
+ unsigned int current_ref_frames[MAX_REF_FRAMES];
#endif
struct rd_costs_struct
--- a/vp8/encoder/pickinter.c
+++ b/vp8/encoder/pickinter.c
@@ -542,7 +542,7 @@
int_mv parent_ref_mv;
MB_PREDICTION_MODE parent_mode = 0;
- if (cpi->oxcf.mr_encoder_id)
+ if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
get_lower_res_motion_info(cpi, xd, &dissim, &parent_ref_frame,
&parent_mode, &parent_ref_mv, mb_row, mb_col);
#endif
@@ -600,11 +600,14 @@
x->e_mbd.mode_info_context->mbmi.ref_frame = this_ref_frame;
#if CONFIG_MULTI_RES_ENCODING
- if (cpi->oxcf.mr_encoder_id)
+ if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
{- /* If parent MB is intra, child MB is intra. */
- if (!parent_ref_frame && this_ref_frame)
- continue;
+ /* TODO: If parent MB is intra, child MB is intra. This is removed
+ * now since it cause noticeable quality loss for some test clip.
+ * Will come back to evaluate more.
+ * if (!parent_ref_frame && this_ref_frame)
+ * continue;
+ */
/* If parent MB is inter, and it is unlikely there are multiple
* objects in parent MB, we use parent ref frame as child MB's
@@ -630,7 +633,7 @@
}
#if CONFIG_MULTI_RES_ENCODING
- if (cpi->oxcf.mr_encoder_id)
+ if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
{if (vp8_mode_order[mode_index] == NEARESTMV &&
mode_mv[NEARESTMV].as_int ==0)
@@ -783,7 +786,14 @@
step_param = cpi->sf.first_step + speed_adjust;
#if CONFIG_MULTI_RES_ENCODING
- if (cpi->oxcf.mr_encoder_id)
+ /* If lower-res drops this frame, then higher-res encoder does
+ motion search without any previous knowledge. Also, since
+ last frame motion info is not stored, then we can not
+ use improved_mv_pred. */
+ if (cpi->oxcf.mr_encoder_id && !cpi->mr_low_res_mv_avail)
+ cpi->sf.improved_mv_pred = 0;
+
+ if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail)
{/* Use parent MV as predictor. Adjust search range
* accordingly.
@@ -827,7 +837,8 @@
}
#if CONFIG_MULTI_RES_ENCODING
- if (cpi->oxcf.mr_encoder_id && dissim <= 2 &&
+ if (cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail &&
+ dissim <= 2 &&
MAX(abs(best_ref_mv.as_mv.row - parent_ref_mv.as_mv.row),
abs(best_ref_mv.as_mv.col - parent_ref_mv.as_mv.col)) <= 4)
{@@ -864,7 +875,10 @@
* change the behavior in lowest-resolution encoder.
* Will improve it later.
*/
- if (!cpi->oxcf.mr_encoder_id)
+ /* Set step_param to 0 to ensure large-range motion search
+ when encoder drops this frame at lower-resolution.
+ */
+ if (!cpi->oxcf.mr_encoder_id || !cpi->mr_low_res_mv_avail)
step_param = 0;
#endif
bestsme = vp8_hex_search(x, b, d, &mvp_full, &d->bmi.mv,
--- a/vp8/vp8_cx_iface.c
+++ b/vp8/vp8_cx_iface.c
@@ -163,14 +163,11 @@
* multi-res-encoder.*/
#if CONFIG_MULTI_RES_ENCODING
if (ctx->base.enc.total_encoders > 1)
- {RANGE_CHECK_HI(cfg, rc_resize_allowed, 0);
- RANGE_CHECK_HI(cfg, rc_dropframe_thresh, 0);
- }
#else
RANGE_CHECK_BOOL(cfg, rc_resize_allowed);
- RANGE_CHECK_HI(cfg, rc_dropframe_thresh, 100);
#endif
+ RANGE_CHECK_HI(cfg, rc_dropframe_thresh, 100);
RANGE_CHECK_HI(cfg, rc_resize_up_thresh, 100);
RANGE_CHECK_HI(cfg, rc_resize_down_thresh, 100);
--- a/vp8_multi_resolution_encoder.c
+++ b/vp8_multi_resolution_encoder.c
@@ -273,7 +273,7 @@
cfg[0].g_w = width;
cfg[0].g_h = height;
cfg[0].g_threads = 1; /* number of threads used */
- cfg[0].rc_dropframe_thresh = 0;
+ cfg[0].rc_dropframe_thresh = 30;
cfg[0].rc_end_usage = VPX_CBR;
cfg[0].rc_resize_allowed = 0;
cfg[0].rc_min_quantizer = 4;
@@ -283,7 +283,6 @@
cfg[0].rc_buf_initial_sz = 500;
cfg[0].rc_buf_optimal_sz = 600;
cfg[0].rc_buf_sz = 1000;
- //cfg[0].rc_dropframe_thresh = 10;
cfg[0].g_error_resilient = 1; /* Enable error resilient mode */
cfg[0].g_lag_in_frames = 0;
@@ -293,8 +292,8 @@
*/
//cfg[0].kf_mode = VPX_KF_DISABLED;
cfg[0].kf_mode = VPX_KF_AUTO;
- cfg[0].kf_min_dist = 0;
- cfg[0].kf_max_dist = 150;
+ cfg[0].kf_min_dist = 3000;
+ cfg[0].kf_max_dist = 3000;
cfg[0].rc_target_bitrate = target_bitrate[0]; /* Set target bitrate */
cfg[0].g_timebase.num = 1; /* Set fps */
@@ -365,6 +364,12 @@
unsigned int static_thresh = 0;
if(vpx_codec_control(&codec[i], VP8E_SET_STATIC_THRESHOLD, static_thresh))
die_codec(&codec[i], "Failed to set static threshold");
+ }
+ /* Set NOISE_SENSITIVITY to do TEMPORAL_DENOISING */
+ for ( i=0; i< NUM_ENCODERS; i++)
+ {+ if(vpx_codec_control(&codec[i], VP8E_SET_NOISE_SENSITIVITY, 0))
+ die_codec(&codec[i], "Failed to set noise_sensitivity");
}
frame_avail = 1;