ref: 292d221fed21a84b3a9902bcaecda00cc08e6029
parent: c0180325793e7f1dcd50bf8a30dfe2ce41e6ae75
author: Yury Gitman <yuryg@google.com>
date: Mon Jul 18 11:44:40 EDT 2016
Create interface for the ALT_REF_AQ class Current commit is just an API template for the rest of the code, and I will add inner logic later. Altref frames generate a lot of bitrate and at the same time other frames refer to them a lot, so it makes sense to apply special compensation-based adaptive quantization scheme for altref frames. E.g., for blocks that are good predictors for the future apply rate-control chosen quantizer while for bad predictors apply worse one. Change-Id: Iba3f8ec349470673b7249f6a125f6859336a47c8
--- /dev/null
+++ b/vp9/encoder/vp9_alt_ref_aq.c
@@ -1,0 +1,61 @@
+/*
+ * Copyright (c) 2016 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file in the root of the source tree. An additional
+ * intellectual property rights grant can be found in the file PATENTS.
+ * All contributing project authors may be found in the AUTHORS file in
+ * the root of the source tree.
+ */
+
+#include "vp9/encoder/vp9_encoder.h"
+
+#include "vp9/encoder/vp9_alt_ref_aq_private.h"
+#include "vp9/encoder/vp9_alt_ref_aq.h"
+
+struct ALT_REF_AQ *vp9_alt_ref_aq_create() {+ return (struct ALT_REF_AQ *)vpx_malloc(sizeof(struct ALT_REF_AQ));
+}
+
+void vp9_alt_ref_aq_destroy(struct ALT_REF_AQ *const self) { vpx_free(self); }+
+void vp9_alt_ref_aq_upload_map(struct ALT_REF_AQ *const self,
+ const struct MATX_8U *segmentation_map) {+ (void)self;
+ (void)segmentation_map;
+}
+
+void vp9_alt_ref_aq_set_nsegments(struct ALT_REF_AQ *const self,
+ int nsegments) {+ (void)self;
+ (void)nsegments;
+}
+
+void vp9_alt_ref_aq_setup_mode(struct ALT_REF_AQ *const self,
+ struct VP9_COMP *const cpi) {+ (void)cpi;
+ (void)self;
+}
+
+// set basic segmentation to the altref's one
+void vp9_alt_ref_aq_setup_map(struct ALT_REF_AQ *const self,
+ struct VP9_COMP *const cpi) {+ (void)cpi;
+ (void)self;
+}
+
+// restore cpi->aq_mode
+void vp9_alt_ref_aq_unset_all(struct ALT_REF_AQ *const self,
+ struct VP9_COMP *const cpi) {+ (void)cpi;
+ (void)self;
+}
+
+int vp9_alt_ref_aq_disable_if(const struct ALT_REF_AQ *self,
+ int segmentation_overhead, int bandwidth) {+ (void)bandwidth;
+ (void)self;
+ (void)segmentation_overhead;
+
+ return 0;
+}
--- /dev/null
+++ b/vp9/encoder/vp9_alt_ref_aq.h
@@ -1,0 +1,124 @@
+/*
+ * Copyright (c) 2016 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file in the root of the source tree. An additional
+ * intellectual property rights grant can be found in the file PATENTS.
+ * All contributing project authors may be found in the AUTHORS file in
+ * the root of the source tree.
+ */
+
+/*
+ * \file vp9_alt_ref_aq.h
+ *
+ * This file contains public interface for setting up adaptive segmentation
+ * for altref frames. Go to alt_ref_aq_private.h for implmentation details.
+ */
+
+#ifndef VP9_ENCODER_VP9_ALT_REF_AQ_H_
+#define VP9_ENCODER_VP9_ALT_REF_AQ_H_
+
+#include "vpx/vpx_integer.h"
+
+// Where to disable segmentation
+#define ALT_REF_AQ_LOW_BITRATE_BOUNDARY 150
+
+// Last frame always has overall quality = 0,
+// so it is questionable if I can process it
+#define ALT_REF_AQ_APPLY_TO_LAST_FRAME 1
+
+// If I should try to compare gain
+// against segmentation overhead
+#define ALT_REF_AQ_PROTECT_GAIN 0
+
+// Threshold to disable segmentation
+#define ALT_REF_AQ_PROTECT_GAIN_THRESH 0.5
+
+#ifdef __cplusplus
+extern "C" {+#endif
+
+// Simple structure for storing images
+struct MATX_8U {+ int rows;
+ int cols;
+ int stride;
+
+ uint8_t *data;
+};
+
+struct VP9_COMP;
+struct ALT_REF_AQ;
+
+/*!\brief Constructor
+ *
+ * \return Instance of the class
+ */
+struct ALT_REF_AQ *vp9_alt_ref_aq_create();
+
+/*!\brief Upload segmentation_map to self object
+ *
+ * \param self Instance of the class
+ * \param segmentation_map Segmentation map to upload
+ */
+void vp9_alt_ref_aq_upload_map(struct ALT_REF_AQ *const self,
+ const struct MATX_8U *segmentation_map);
+
+/*!\brief Return pointer to the altref segmentation map
+ *
+ * \param self Instance of the class
+ * \param segmentation_overhead Segmentation overhead in bytes
+ * \param bandwidth Current frame bandwidth in bytes
+ *
+ * \return Boolean value to disable segmentation
+ */
+int vp9_alt_ref_aq_disable_if(const struct ALT_REF_AQ *self,
+ int segmentation_overhead, int bandwidth);
+
+/*!\brief Set number of segments
+ *
+ * It is used for delta quantizer computations
+ * and thus it can be larger than
+ * maximum value of the segmentation map
+ *
+ * \param self Instance of the class
+ * \param nsegments Maximum number of segments
+ */
+void vp9_alt_ref_aq_set_nsegments(struct ALT_REF_AQ *self, int nsegments);
+
+/*!\brief Set up LOOKAHEAD_AQ segmentation mode
+ *
+ * Set up segmentation mode to LOOKAHEAD_AQ
+ * (expected future frames prediction
+ * quality refering to the current frame).
+ *
+ * \param self Instance of the class
+ * \param cpi Encoder context
+ */
+void vp9_alt_ref_aq_setup_mode(struct ALT_REF_AQ *self, struct VP9_COMP *cpi);
+
+/*!\brief Set up LOOKAHEAD_AQ segmentation map and delta quantizers
+ *
+ * \param self Instance of the class
+ * \param cpi Encoder context
+ */
+void vp9_alt_ref_aq_setup_map(struct ALT_REF_AQ *self, struct VP9_COMP *cpi);
+
+/*!\brief Restore main segmentation map mode and reset the class variables
+ *
+ * \param self Instance of the class
+ * \param cpi Encoder context
+ */
+void vp9_alt_ref_aq_unset_all(struct ALT_REF_AQ *self, struct VP9_COMP *cpi);
+
+/*!\brief Destructor
+ *
+ * \param self Instance of the class
+ */
+void vp9_alt_ref_aq_destroy(struct ALT_REF_AQ *self);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // VP9_ENCODER_VP9_ALT_REF_AQ_H_
--- /dev/null
+++ b/vp9/encoder/vp9_alt_ref_aq_private.h
@@ -1,0 +1,35 @@
+/*
+ * Copyright (c) 2016 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file in the root of the source tree. An additional
+ * intellectual property rights grant can be found in the file PATENTS.
+ * All contributing project authors may be found in the AUTHORS file in
+ * the root of the source tree.
+ */
+
+/*
+ * \file vp9_alt_ref_aq_private.h
+ *
+ * This file describes class used for setting up adaptive segmentation
+ * for altref frames. It is private file and most likely you need
+ * alt_ref_aq.h instead.
+ */
+
+#ifndef VP9_ENCODER_VP9_ALT_REF_AQ_PRIVATE_H_
+#define VP9_ENCODER_VP9_ALT_REF_AQ_PRIVATE_H_
+
+#include "vp9/encoder/vp9_alt_ref_aq.h"
+
+#ifdef __cplusplus
+
+extern "C" {+#endif
+
+struct ALT_REF_AQ {};+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // VP9_ENCODER_VP9_ALT_REF_AQ_PRIVATE_H_
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -216,7 +216,7 @@
// Setup segment ID.
if (seg->enabled) {- if (cpi->oxcf.aq_mode != VARIANCE_AQ &&
+ if (cpi->oxcf.aq_mode != VARIANCE_AQ && cpi->oxcf.aq_mode != LOOKAHEAD_AQ &&
cpi->oxcf.aq_mode != EQUATOR360_AQ) {const uint8_t *const map =
seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
@@ -1354,6 +1354,11 @@
mi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
}
x->rdmult = set_segment_rdmult(cpi, x, mi->segment_id);
+ } else if (aq_mode == LOOKAHEAD_AQ) {+ const uint8_t *const map = cpi->segmentation_map;
+
+ // I do not change rdmult here consciously.
+ mi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
} else if (aq_mode == EQUATOR360_AQ) { if (cm->frame_type == KEY_FRAME || cpi->force_update_segmentation) {mi->segment_id = vp9_360aq_segment_id(mi_row, cm->mi_rows);
@@ -2501,7 +2506,8 @@
set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize);
- if (bsize == BLOCK_16X16 && cpi->oxcf.aq_mode != NO_AQ)
+ if (bsize == BLOCK_16X16 && cpi->oxcf.aq_mode != NO_AQ &&
+ cpi->oxcf.aq_mode != LOOKAHEAD_AQ)
x->mb_energy = vp9_block_energy(cpi, x, bsize);
if (cpi->sf.cb_partition_search && bsize == BLOCK_16X16) {--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -36,6 +36,7 @@
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_tile_common.h"
+#include "vp9/encoder/vp9_alt_ref_aq.h"
#include "vp9/encoder/vp9_aq_360.h"
#include "vp9/encoder/vp9_aq_complexity.h"
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
@@ -1618,6 +1619,8 @@
cpi->use_skin_detection = 0;
cpi->common.buffer_pool = pool;
+ cpi->force_update_segmentation = 0;
+
init_config(cpi, oxcf);
vp9_rc_init(&cpi->oxcf, oxcf->pass, &cpi->rc);
@@ -1627,6 +1630,8 @@
realloc_segmentation_maps(cpi);
+ CHECK_MEM_ERROR(cm, cpi->alt_ref_aq, vp9_alt_ref_aq_create());
+
CHECK_MEM_ERROR(
cm, cpi->consec_zero_mv,
vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv)));
@@ -2024,6 +2029,8 @@
if (cpi->num_workers > 1) vp9_loop_filter_dealloc(&cpi->lf_row_sync);
+ vp9_alt_ref_aq_destroy(cpi->alt_ref_aq);
+
dealloc_compressor_data(cpi);
for (i = 0; i < sizeof(cpi->mbgraph_stats) / sizeof(cpi->mbgraph_stats[0]);
@@ -3138,6 +3145,7 @@
setup_frame(cpi);
suppress_active_map(cpi);
+
// Variance adaptive and in frame q adjustment experiments are mutually
// exclusive.
if (cpi->oxcf.aq_mode == VARIANCE_AQ) {@@ -3148,7 +3156,12 @@
vp9_setup_in_frame_q_adj(cpi);
} else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) {vp9_cyclic_refresh_setup(cpi);
+ } else if (cpi->oxcf.aq_mode == LOOKAHEAD_AQ) {+ // it may be pretty bad for rate-control,
+ // and I should handle it somehow
+ vp9_alt_ref_aq_setup_map(cpi->alt_ref_aq, cpi);
}
+
apply_active_map(cpi);
vp9_encode_frame(cpi);
@@ -3286,6 +3299,8 @@
vp9_360aq_frame_setup(cpi);
} else if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) {vp9_setup_in_frame_q_adj(cpi);
+ } else if (cpi->oxcf.aq_mode == LOOKAHEAD_AQ) {+ vp9_alt_ref_aq_setup_map(cpi->alt_ref_aq, cpi);
}
vp9_encode_frame(cpi);
@@ -3748,6 +3763,30 @@
}
#endif // ENABLE_KF_DENOISE
+static void vp9_try_disable_lookahead_aq(VP9_COMP *cpi, size_t *size,
+ uint8_t *dest) {+ if (cpi->common.seg.enabled)
+ if (ALT_REF_AQ_PROTECT_GAIN) {+ size_t nsize = *size;
+ int overhead;
+
+ // TODO(yuryg): optimize this, as
+ // we don't really need to repack
+
+ save_coding_context(cpi);
+ vp9_disable_segmentation(&cpi->common.seg);
+ vp9_pack_bitstream(cpi, dest, &nsize);
+ restore_coding_context(cpi);
+
+ overhead = (int)*size - (int)nsize;
+
+ if (vp9_alt_ref_aq_disable_if(cpi->alt_ref_aq, overhead, (int)*size))
+ vp9_encode_frame(cpi);
+ else
+ vp9_enable_segmentation(&cpi->common.seg);
+ }
+}
+
static void encode_frame_to_data_rate(VP9_COMP *cpi, size_t *size,
uint8_t *dest,
unsigned int *frame_flags) {@@ -3873,6 +3912,10 @@
encode_with_recode_loop(cpi, size, dest);
}
+ // Disable segmentation if it decrease rate/distortion ratio
+ if (cpi->oxcf.aq_mode == LOOKAHEAD_AQ)
+ vp9_try_disable_lookahead_aq(cpi, size, dest);
+
#if CONFIG_VP9_TEMPORAL_DENOISING
#ifdef OUTPUT_YUV_DENOISED
if (oxcf->noise_sensitivity > 0) {@@ -3996,6 +4039,11 @@
cpi->svc.number_temporal_layers +
cpi->svc.temporal_layer_id]
.last_frame_type = cm->frame_type;
+
+ cpi->force_update_segmentation = 0;
+
+ if (cpi->oxcf.aq_mode == LOOKAHEAD_AQ)
+ vp9_alt_ref_aq_unset_all(cpi->alt_ref_aq, cpi);
}
static void SvcEncode(VP9_COMP *cpi, size_t *size, uint8_t *dest,
@@ -4424,9 +4472,21 @@
#endif
if ((oxcf->arnr_max_frames > 0) && (oxcf->arnr_strength > 0)) {+ int bitrate = cpi->rc.avg_frame_bandwidth / 40;
+ int not_low_bitrate = bitrate > ALT_REF_AQ_LOW_BITRATE_BOUNDARY;
+
+ int not_last_frame = (cpi->lookahead->sz - arf_src_index > 1);
+ not_last_frame |= ALT_REF_AQ_APPLY_TO_LAST_FRAME;
+
// Produce the filtered ARF frame.
vp9_temporal_filter(cpi, arf_src_index);
vpx_extend_frame_borders(&cpi->alt_ref_buffer);
+
+ // for small bitrates segmentation overhead usually
+ // eats all bitrate gain from enabling delta quantizers
+ if (cpi->oxcf.alt_ref_aq != 0 && not_low_bitrate && not_last_frame)
+ vp9_alt_ref_aq_setup_mode(cpi->alt_ref_aq, cpi);
+
force_src_buffer = &cpi->alt_ref_buffer;
}
@@ -4788,6 +4848,7 @@
cpi->svc.spatial_layer_to_encode = 0;
}
}
+
vpx_clear_system_state();
return 0;
}
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -29,6 +29,7 @@
#include "vp9/common/vp9_thread_common.h"
#include "vp9/common/vp9_onyxc_int.h"
+#include "vp9/encoder/vp9_alt_ref_aq.h"
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_context_tree.h"
#include "vp9/encoder/vp9_encodemb.h"
@@ -487,6 +488,10 @@
uint8_t force_update_segmentation;
YV12_BUFFER_CONFIG alt_ref_buffer;
+
+ // class responsible for adaptive
+ // quantization of altref frames
+ struct ALT_REF_AQ *alt_ref_aq;
#if CONFIG_INTERNAL_STATS
unsigned int mode_chosen_counts[MAX_MODES];
--- a/vp9/vp9cx.mk
+++ b/vp9/vp9cx.mk
@@ -81,6 +81,9 @@
VP9_CX_SRCS-yes += encoder/vp9_aq_cyclicrefresh.h
VP9_CX_SRCS-yes += encoder/vp9_aq_complexity.c
VP9_CX_SRCS-yes += encoder/vp9_aq_complexity.h
+VP9_CX_SRCS-yes += encoder/vp9_alt_ref_aq_private.h
+VP9_CX_SRCS-yes += encoder/vp9_alt_ref_aq.h
+VP9_CX_SRCS-yes += encoder/vp9_alt_ref_aq.c
VP9_CX_SRCS-yes += encoder/vp9_skin_detection.c
VP9_CX_SRCS-yes += encoder/vp9_skin_detection.h
VP9_CX_SRCS-yes += encoder/vp9_noise_estimate.c