Separate superres functions from encoder.c
Created superres_scale.c and superres_scale.h
to improve modularity of encoder.c
superres_scale.c : To keep super resolution
related functions
superres_scale.h : To keep super resolution
related data structures,
defs and enum.
Change-Id: Ia23befe931e821c1e546af286315d36cb86b6f21
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 857bcc8..b95018d 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -212,6 +212,8 @@
"${AOM_ROOT}/av1/encoder/segmentation.h"
"${AOM_ROOT}/av1/encoder/speed_features.c"
"${AOM_ROOT}/av1/encoder/speed_features.h"
+ "${AOM_ROOT}/av1/encoder/superres_scale.c"
+ "${AOM_ROOT}/av1/encoder/superres_scale.h"
"${AOM_ROOT}/av1/encoder/svc_layercontext.c"
"${AOM_ROOT}/av1/encoder/svc_layercontext.h"
"${AOM_ROOT}/av1/encoder/temporal_filter.c"
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 3badbcc..151fd98 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -71,6 +71,7 @@
#include "av1/encoder/rdopt.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/speed_features.h"
+#include "av1/encoder/superres_scale.h"
#include "av1/encoder/tpl_model.h"
#include "av1/encoder/reconinter_enc.h"
#include "av1/encoder/var_based_part.h"
@@ -512,66 +513,6 @@
}
}
-// Compute the horizontal frequency components' energy in a frame
-// by calculuating the 16x4 Horizontal DCT. This is to be used to
-// decide the superresolution parameters.
-static void analyze_hor_freq(const AV1_COMP *cpi, double *energy) {
- uint64_t freq_energy[16] = { 0 };
- const YV12_BUFFER_CONFIG *buf = cpi->source;
- const int bd = cpi->td.mb.e_mbd.bd;
- const int width = buf->y_crop_width;
- const int height = buf->y_crop_height;
- DECLARE_ALIGNED(16, int32_t, coeff[16 * 4]);
- int n = 0;
- memset(freq_energy, 0, sizeof(freq_energy));
- if (buf->flags & YV12_FLAG_HIGHBITDEPTH) {
- const int16_t *src16 = (const int16_t *)CONVERT_TO_SHORTPTR(buf->y_buffer);
- for (int i = 0; i < height - 4; i += 4) {
- for (int j = 0; j < width - 16; j += 16) {
- av1_fwd_txfm2d_16x4(src16 + i * buf->y_stride + j, coeff, buf->y_stride,
- H_DCT, bd);
- for (int k = 1; k < 16; ++k) {
- const uint64_t this_energy =
- ((int64_t)coeff[k] * coeff[k]) +
- ((int64_t)coeff[k + 16] * coeff[k + 16]) +
- ((int64_t)coeff[k + 32] * coeff[k + 32]) +
- ((int64_t)coeff[k + 48] * coeff[k + 48]);
- freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2 + 2 * (bd - 8));
- }
- n++;
- }
- }
- } else {
- assert(bd == 8);
- DECLARE_ALIGNED(16, int16_t, src16[16 * 4]);
- for (int i = 0; i < height - 4; i += 4) {
- for (int j = 0; j < width - 16; j += 16) {
- for (int ii = 0; ii < 4; ++ii)
- for (int jj = 0; jj < 16; ++jj)
- src16[ii * 16 + jj] =
- buf->y_buffer[(i + ii) * buf->y_stride + (j + jj)];
- av1_fwd_txfm2d_16x4(src16, coeff, 16, H_DCT, bd);
- for (int k = 1; k < 16; ++k) {
- const uint64_t this_energy =
- ((int64_t)coeff[k] * coeff[k]) +
- ((int64_t)coeff[k + 16] * coeff[k + 16]) +
- ((int64_t)coeff[k + 32] * coeff[k + 32]) +
- ((int64_t)coeff[k + 48] * coeff[k + 48]);
- freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2);
- }
- n++;
- }
- }
- }
- if (n) {
- for (int k = 1; k < 16; ++k) energy[k] = (double)freq_energy[k] / n;
- // Convert to cumulative energy
- for (int k = 14; k > 0; --k) energy[k] += energy[k + 1];
- } else {
- for (int k = 1; k < 16; ++k) energy[k] = 1e+20;
- }
-}
-
static BLOCK_SIZE select_sb_size(const AV1_COMP *const cpi) {
const AV1_COMMON *const cm = &cpi->common;
const AV1EncoderConfig *const oxcf = &cpi->oxcf;
@@ -4480,370 +4421,6 @@
set_ref_ptrs(cm, xd, LAST_FRAME, LAST_FRAME);
}
-static uint8_t calculate_next_resize_scale(const AV1_COMP *cpi) {
- // Choose an arbitrary random number
- static unsigned int seed = 56789;
- const ResizeCfg *resize_cfg = &cpi->oxcf.resize_cfg;
- if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR;
- uint8_t new_denom = SCALE_NUMERATOR;
-
- if (cpi->common.seq_params.reduced_still_picture_hdr) return SCALE_NUMERATOR;
- switch (resize_cfg->resize_mode) {
- case RESIZE_NONE: new_denom = SCALE_NUMERATOR; break;
- case RESIZE_FIXED:
- if (cpi->common.current_frame.frame_type == KEY_FRAME)
- new_denom = resize_cfg->resize_kf_scale_denominator;
- else
- new_denom = resize_cfg->resize_scale_denominator;
- break;
- case RESIZE_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break;
- default: assert(0);
- }
- return new_denom;
-}
-
-#if CONFIG_SUPERRES_IN_RECODE
-static int superres_in_recode_allowed(const AV1_COMP *const cpi) {
- const AV1EncoderConfig *const oxcf = &cpi->oxcf;
- // Empirically found to not be beneficial for AOM_Q mode and images coding.
- return oxcf->superres_cfg.superres_mode == AOM_SUPERRES_AUTO &&
- (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ) &&
- cpi->rc.frames_to_key > 1;
-}
-#endif // CONFIG_SUPERRES_IN_RECODE
-
-#define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO 0.012
-#define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME 0.008
-#define SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME 0.008
-#define SUPERRES_ENERGY_BY_AC_THRESH 0.2
-
-static double get_energy_by_q2_thresh(const GF_GROUP *gf_group,
- const RATE_CONTROL *rc) {
- // TODO(now): Return keyframe thresh * factor based on frame type / pyramid
- // level.
- if (gf_group->update_type[gf_group->index] == ARF_UPDATE) {
- return SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME;
- } else if (gf_group->update_type[gf_group->index] == KF_UPDATE) {
- if (rc->frames_to_key <= 1)
- return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO;
- else
- return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME;
- } else {
- assert(0);
- }
- return 0;
-}
-
-static uint8_t get_superres_denom_from_qindex_energy(int qindex, double *energy,
- double threshq,
- double threshp) {
- const double q = av1_convert_qindex_to_q(qindex, AOM_BITS_8);
- const double tq = threshq * q * q;
- const double tp = threshp * energy[1];
- const double thresh = AOMMIN(tq, tp);
- int k;
- for (k = SCALE_NUMERATOR * 2; k > SCALE_NUMERATOR; --k) {
- if (energy[k - 1] > thresh) break;
- }
- return 3 * SCALE_NUMERATOR - k;
-}
-
-static uint8_t get_superres_denom_for_qindex(const AV1_COMP *cpi, int qindex,
- int sr_kf, int sr_arf) {
- // Use superres for Key-frames and Alt-ref frames only.
- const GF_GROUP *gf_group = &cpi->gf_group;
- if (gf_group->update_type[gf_group->index] != KF_UPDATE &&
- gf_group->update_type[gf_group->index] != ARF_UPDATE) {
- return SCALE_NUMERATOR;
- }
- if (gf_group->update_type[gf_group->index] == KF_UPDATE && !sr_kf) {
- return SCALE_NUMERATOR;
- }
- if (gf_group->update_type[gf_group->index] == ARF_UPDATE && !sr_arf) {
- return SCALE_NUMERATOR;
- }
-
- double energy[16];
- analyze_hor_freq(cpi, energy);
-
- const double energy_by_q2_thresh =
- get_energy_by_q2_thresh(gf_group, &cpi->rc);
- int denom = get_superres_denom_from_qindex_energy(
- qindex, energy, energy_by_q2_thresh, SUPERRES_ENERGY_BY_AC_THRESH);
- /*
- printf("\nenergy = [");
- for (int k = 1; k < 16; ++k) printf("%f, ", energy[k]);
- printf("]\n");
- printf("boost = %d\n",
- (gf_group->update_type[gf_group->index] == KF_UPDATE)
- ? cpi->rc.kf_boost
- : cpi->rc.gfu_boost);
- printf("denom = %d\n", denom);
- */
-#if CONFIG_SUPERRES_IN_RECODE
- if (superres_in_recode_allowed(cpi)) {
- assert(cpi->superres_mode != AOM_SUPERRES_NONE);
- // Force superres to be tried in the recode loop, as full-res is also going
- // to be tried anyway.
- denom = AOMMAX(denom, SCALE_NUMERATOR + 1);
- }
-#endif // CONFIG_SUPERRES_IN_RECODE
- return denom;
-}
-
-// If true, AOM_SUPERRES_AUTO mode will exhaustively search over all superres
-// denominators for all frames (except overlay and internal overlay frames).
-#define SUPERRES_RECODE_ALL_RATIOS 0
-
-static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) {
- // Choose an arbitrary random number
- static unsigned int seed = 34567;
- const AV1EncoderConfig *oxcf = &cpi->oxcf;
- const SuperResCfg *const superres_cfg = &oxcf->superres_cfg;
-
- if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR;
- uint8_t new_denom = SCALE_NUMERATOR;
-
- // Make sure that superres mode of the frame is consistent with the
- // sequence-level flag.
- assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_NONE,
- cpi->common.seq_params.enable_superres));
- assert(IMPLIES(!cpi->common.seq_params.enable_superres,
- superres_cfg->superres_mode == AOM_SUPERRES_NONE));
- // Make sure that superres mode for current encoding is consistent with user
- // provided superres mode.
- assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_AUTO,
- cpi->superres_mode == superres_cfg->superres_mode));
-
- // Note: we must look at the current superres_mode to be tried in 'cpi' here,
- // not the user given mode in 'oxcf'.
- switch (cpi->superres_mode) {
- case AOM_SUPERRES_NONE: new_denom = SCALE_NUMERATOR; break;
- case AOM_SUPERRES_FIXED:
- if (cpi->common.current_frame.frame_type == KEY_FRAME)
- new_denom = superres_cfg->superres_kf_scale_denominator;
- else
- new_denom = superres_cfg->superres_scale_denominator;
- break;
- case AOM_SUPERRES_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break;
- case AOM_SUPERRES_QTHRESH: {
- // Do not use superres when screen content tools are used.
- if (cpi->common.features.allow_screen_content_tools) break;
- if (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ)
- av1_set_target_rate(cpi, cpi->oxcf.width, cpi->oxcf.height);
-
- // Now decide the use of superres based on 'q'.
- int bottom_index, top_index;
- const int q = av1_rc_pick_q_and_bounds(
- cpi, &cpi->rc, cpi->oxcf.width, cpi->oxcf.height, cpi->gf_group.index,
- &bottom_index, &top_index);
-
- const int qthresh = (frame_is_intra_only(&cpi->common))
- ? superres_cfg->superres_kf_qthresh
- : superres_cfg->superres_qthresh;
- if (q <= qthresh) {
- new_denom = SCALE_NUMERATOR;
- } else {
- new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1);
- }
- break;
- }
- case AOM_SUPERRES_AUTO: {
- // Do not use superres when screen content tools are used.
- if (cpi->common.features.allow_screen_content_tools) break;
- if (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ)
- av1_set_target_rate(cpi, cpi->oxcf.width, cpi->oxcf.height);
-
- // Now decide the use of superres based on 'q'.
- int bottom_index, top_index;
- const int q = av1_rc_pick_q_and_bounds(
- cpi, &cpi->rc, cpi->oxcf.width, cpi->oxcf.height, cpi->gf_group.index,
- &bottom_index, &top_index);
-
- const int qthresh = 128;
- if (q <= qthresh) {
- new_denom = SCALE_NUMERATOR;
- } else {
-#if SUPERRES_RECODE_ALL_RATIOS
- if (cpi->common.current_frame.frame_type == KEY_FRAME)
- new_denom = superres_cfg->superres_kf_scale_denominator;
- else
- new_denom = superres_cfg->superres_scale_denominator;
-#else
- new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1);
-#endif // SUPERRES_RECODE_ALL_RATIOS
- }
- break;
- }
- default: assert(0);
- }
- return new_denom;
-}
-
-static int dimension_is_ok(int orig_dim, int resized_dim, int denom) {
- return (resized_dim * SCALE_NUMERATOR >= orig_dim * denom / 2);
-}
-
-static int dimensions_are_ok(int owidth, int oheight, size_params_type *rsz) {
- // Only need to check the width, as scaling is horizontal only.
- (void)oheight;
- return dimension_is_ok(owidth, rsz->resize_width, rsz->superres_denom);
-}
-
-static int validate_size_scales(RESIZE_MODE resize_mode,
- aom_superres_mode superres_mode, int owidth,
- int oheight, size_params_type *rsz) {
- if (dimensions_are_ok(owidth, oheight, rsz)) { // Nothing to do.
- return 1;
- }
-
- // Calculate current resize scale.
- int resize_denom =
- AOMMAX(DIVIDE_AND_ROUND(owidth * SCALE_NUMERATOR, rsz->resize_width),
- DIVIDE_AND_ROUND(oheight * SCALE_NUMERATOR, rsz->resize_height));
-
- if (resize_mode != RESIZE_RANDOM && superres_mode == AOM_SUPERRES_RANDOM) {
- // Alter superres scale as needed to enforce conformity.
- rsz->superres_denom =
- (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / resize_denom;
- if (!dimensions_are_ok(owidth, oheight, rsz)) {
- if (rsz->superres_denom > SCALE_NUMERATOR) --rsz->superres_denom;
- }
- } else if (resize_mode == RESIZE_RANDOM &&
- superres_mode != AOM_SUPERRES_RANDOM) {
- // Alter resize scale as needed to enforce conformity.
- resize_denom =
- (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / rsz->superres_denom;
- rsz->resize_width = owidth;
- rsz->resize_height = oheight;
- av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height,
- resize_denom);
- if (!dimensions_are_ok(owidth, oheight, rsz)) {
- if (resize_denom > SCALE_NUMERATOR) {
- --resize_denom;
- rsz->resize_width = owidth;
- rsz->resize_height = oheight;
- av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height,
- resize_denom);
- }
- }
- } else if (resize_mode == RESIZE_RANDOM &&
- superres_mode == AOM_SUPERRES_RANDOM) {
- // Alter both resize and superres scales as needed to enforce conformity.
- do {
- if (resize_denom > rsz->superres_denom)
- --resize_denom;
- else
- --rsz->superres_denom;
- rsz->resize_width = owidth;
- rsz->resize_height = oheight;
- av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height,
- resize_denom);
- } while (!dimensions_are_ok(owidth, oheight, rsz) &&
- (resize_denom > SCALE_NUMERATOR ||
- rsz->superres_denom > SCALE_NUMERATOR));
- } else { // We are allowed to alter neither resize scale nor superres
- // scale.
- return 0;
- }
- return dimensions_are_ok(owidth, oheight, rsz);
-}
-
-// Calculates resize and superres params for next frame
-static size_params_type calculate_next_size_params(AV1_COMP *cpi) {
- const AV1EncoderConfig *oxcf = &cpi->oxcf;
- ResizePendingParams *resize_pending_params = &cpi->resize_pending_params;
- size_params_type rsz = { oxcf->width, oxcf->height, SCALE_NUMERATOR };
- int resize_denom = SCALE_NUMERATOR;
- if (has_no_stats_stage(cpi) && cpi->use_svc &&
- cpi->svc.spatial_layer_id < cpi->svc.number_spatial_layers - 1) {
- rsz.resize_width = cpi->common.width;
- rsz.resize_height = cpi->common.height;
- return rsz;
- }
- if (is_stat_generation_stage(cpi)) return rsz;
- if (resize_pending_params->width && resize_pending_params->height) {
- rsz.resize_width = resize_pending_params->width;
- rsz.resize_height = resize_pending_params->height;
- resize_pending_params->width = resize_pending_params->height = 0;
- } else {
- resize_denom = calculate_next_resize_scale(cpi);
- rsz.resize_width = oxcf->width;
- rsz.resize_height = oxcf->height;
- av1_calculate_scaled_size(&rsz.resize_width, &rsz.resize_height,
- resize_denom);
- }
- rsz.superres_denom = calculate_next_superres_scale(cpi);
- if (!validate_size_scales(oxcf->resize_cfg.resize_mode, cpi->superres_mode,
- oxcf->width, oxcf->height, &rsz))
- assert(0 && "Invalid scale parameters");
- return rsz;
-}
-
-static void setup_frame_size_from_params(AV1_COMP *cpi,
- const size_params_type *rsz) {
- int encode_width = rsz->resize_width;
- int encode_height = rsz->resize_height;
-
- AV1_COMMON *cm = &cpi->common;
- cm->superres_upscaled_width = encode_width;
- cm->superres_upscaled_height = encode_height;
- cm->superres_scale_denominator = rsz->superres_denom;
- av1_calculate_scaled_superres_size(&encode_width, &encode_height,
- rsz->superres_denom);
- av1_set_frame_size(cpi, encode_width, encode_height);
-}
-
-void av1_setup_frame_size(AV1_COMP *cpi) {
- AV1_COMMON *cm = &cpi->common;
- // Reset superres params from previous frame.
- cm->superres_scale_denominator = SCALE_NUMERATOR;
- const size_params_type rsz = calculate_next_size_params(cpi);
- setup_frame_size_from_params(cpi, &rsz);
-
- assert(av1_is_min_tile_width_satisfied(cm));
-}
-
-static void superres_post_encode(AV1_COMP *cpi) {
- AV1_COMMON *cm = &cpi->common;
- const int num_planes = av1_num_planes(cm);
-
- if (!av1_superres_scaled(cm)) return;
-
- assert(cpi->oxcf.superres_cfg.enable_superres);
- assert(!is_lossless_requested(&cpi->oxcf));
- assert(!cm->features.all_lossless);
-
- av1_superres_upscale(cm, NULL);
-
- // If regular resizing is occurring the source will need to be downscaled to
- // match the upscaled superres resolution. Otherwise the original source is
- // used.
- if (!av1_resize_scaled(cm)) {
- cpi->source = cpi->unscaled_source;
- if (cpi->last_source != NULL) cpi->last_source = cpi->unscaled_last_source;
- } else {
- assert(cpi->unscaled_source->y_crop_width != cm->superres_upscaled_width);
- assert(cpi->unscaled_source->y_crop_height != cm->superres_upscaled_height);
- // Do downscale. cm->(width|height) has been updated by
- // av1_superres_upscale
- if (aom_realloc_frame_buffer(
- &cpi->scaled_source, cm->superres_upscaled_width,
- cm->superres_upscaled_height, cm->seq_params.subsampling_x,
- cm->seq_params.subsampling_y, cm->seq_params.use_highbitdepth,
- AOM_BORDER_IN_PIXELS, cm->features.byte_alignment, NULL, NULL,
- NULL))
- aom_internal_error(
- &cm->error, AOM_CODEC_MEM_ERROR,
- "Failed to reallocate scaled source buffer for superres");
- assert(cpi->scaled_source.y_crop_width == cm->superres_upscaled_width);
- assert(cpi->scaled_source.y_crop_height == cm->superres_upscaled_height);
- av1_resize_and_extend_frame(cpi->unscaled_source, &cpi->scaled_source,
- (int)cm->seq_params.bit_depth, num_planes);
- cpi->source = &cpi->scaled_source;
- }
-}
-
static void cdef_restoration_frame(AV1_COMP *cpi, AV1_COMMON *cm,
MACROBLOCKD *xd, int use_restoration,
int use_cdef) {
@@ -4872,7 +4449,7 @@
cm->cdef_info.cdef_uv_strengths[0] = 0;
}
- superres_post_encode(cpi);
+ av1_superres_post_encode(cpi);
#if CONFIG_COLLECT_COMPONENT_TIMING
start_timing(cpi, loop_restoration_time);
@@ -5594,7 +5171,7 @@
av1_setup_frame_size(cpi);
#if CONFIG_SUPERRES_IN_RECODE
- if (superres_in_recode_allowed(cpi) &&
+ if (av1_superres_in_recode_allowed(cpi) &&
cpi->superres_mode != AOM_SUPERRES_NONE &&
cm->superres_scale_denominator == SCALE_NUMERATOR) {
// Superres mode is currently enabled, but the denominator selected will
@@ -5985,7 +5562,7 @@
int *largest_tile_id) {
const AV1_COMMON *const cm = &cpi->common;
assert(cm->seq_params.enable_superres);
- assert(superres_in_recode_allowed(cpi));
+ assert(av1_superres_in_recode_allowed(cpi));
aom_codec_err_t err = AOM_CODEC_OK;
save_all_coding_context(cpi);
@@ -6600,7 +6177,7 @@
int largest_tile_id = 0;
#if CONFIG_SUPERRES_IN_RECODE
- if (superres_in_recode_allowed(cpi)) {
+ if (av1_superres_in_recode_allowed(cpi)) {
if (encode_with_and_without_superres(cpi, size, dest, &largest_tile_id) !=
AOM_CODEC_OK) {
return AOM_CODEC_ERROR;
diff --git a/av1/encoder/superres_scale.c b/av1/encoder/superres_scale.c
new file mode 100644
index 0000000..d677683
--- /dev/null
+++ b/av1/encoder/superres_scale.c
@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include "av1/encoder/superres_scale.h"
+#include "av1/encoder/random.h"
+
+// Compute the horizontal frequency components' energy in a frame
+// by calculuating the 16x4 Horizontal DCT. This is to be used to
+// decide the superresolution parameters.
+static void analyze_hor_freq(const AV1_COMP *cpi, double *energy) {
+ uint64_t freq_energy[16] = { 0 };
+ const YV12_BUFFER_CONFIG *buf = cpi->source;
+ const int bd = cpi->td.mb.e_mbd.bd;
+ const int width = buf->y_crop_width;
+ const int height = buf->y_crop_height;
+ DECLARE_ALIGNED(16, int32_t, coeff[16 * 4]);
+ int n = 0;
+ memset(freq_energy, 0, sizeof(freq_energy));
+ if (buf->flags & YV12_FLAG_HIGHBITDEPTH) {
+ const int16_t *src16 = (const int16_t *)CONVERT_TO_SHORTPTR(buf->y_buffer);
+ for (int i = 0; i < height - 4; i += 4) {
+ for (int j = 0; j < width - 16; j += 16) {
+ av1_fwd_txfm2d_16x4(src16 + i * buf->y_stride + j, coeff, buf->y_stride,
+ H_DCT, bd);
+ for (int k = 1; k < 16; ++k) {
+ const uint64_t this_energy =
+ ((int64_t)coeff[k] * coeff[k]) +
+ ((int64_t)coeff[k + 16] * coeff[k + 16]) +
+ ((int64_t)coeff[k + 32] * coeff[k + 32]) +
+ ((int64_t)coeff[k + 48] * coeff[k + 48]);
+ freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2 + 2 * (bd - 8));
+ }
+ n++;
+ }
+ }
+ } else {
+ assert(bd == 8);
+ DECLARE_ALIGNED(16, int16_t, src16[16 * 4]);
+ for (int i = 0; i < height - 4; i += 4) {
+ for (int j = 0; j < width - 16; j += 16) {
+ for (int ii = 0; ii < 4; ++ii)
+ for (int jj = 0; jj < 16; ++jj)
+ src16[ii * 16 + jj] =
+ buf->y_buffer[(i + ii) * buf->y_stride + (j + jj)];
+ av1_fwd_txfm2d_16x4(src16, coeff, 16, H_DCT, bd);
+ for (int k = 1; k < 16; ++k) {
+ const uint64_t this_energy =
+ ((int64_t)coeff[k] * coeff[k]) +
+ ((int64_t)coeff[k + 16] * coeff[k + 16]) +
+ ((int64_t)coeff[k + 32] * coeff[k + 32]) +
+ ((int64_t)coeff[k + 48] * coeff[k + 48]);
+ freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2);
+ }
+ n++;
+ }
+ }
+ }
+ if (n) {
+ for (int k = 1; k < 16; ++k) energy[k] = (double)freq_energy[k] / n;
+ // Convert to cumulative energy
+ for (int k = 14; k > 0; --k) energy[k] += energy[k + 1];
+ } else {
+ for (int k = 1; k < 16; ++k) energy[k] = 1e+20;
+ }
+}
+
+static uint8_t calculate_next_resize_scale(const AV1_COMP *cpi) {
+ // Choose an arbitrary random number
+ static unsigned int seed = 56789;
+ const ResizeCfg *resize_cfg = &cpi->oxcf.resize_cfg;
+ if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR;
+ uint8_t new_denom = SCALE_NUMERATOR;
+
+ if (cpi->common.seq_params.reduced_still_picture_hdr) return SCALE_NUMERATOR;
+ switch (resize_cfg->resize_mode) {
+ case RESIZE_NONE: new_denom = SCALE_NUMERATOR; break;
+ case RESIZE_FIXED:
+ if (cpi->common.current_frame.frame_type == KEY_FRAME)
+ new_denom = resize_cfg->resize_kf_scale_denominator;
+ else
+ new_denom = resize_cfg->resize_scale_denominator;
+ break;
+ case RESIZE_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break;
+ default: assert(0);
+ }
+ return new_denom;
+}
+
+#if CONFIG_SUPERRES_IN_RECODE
+int av1_superres_in_recode_allowed(const AV1_COMP *const cpi) {
+ const AV1EncoderConfig *const oxcf = &cpi->oxcf;
+ // Empirically found to not be beneficial for AOM_Q mode and images coding.
+ return oxcf->superres_cfg.superres_mode == AOM_SUPERRES_AUTO &&
+ (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ) &&
+ cpi->rc.frames_to_key > 1;
+}
+#endif // CONFIG_SUPERRES_IN_RECODE
+
+#define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO 0.012
+#define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME 0.008
+#define SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME 0.008
+#define SUPERRES_ENERGY_BY_AC_THRESH 0.2
+
+static double get_energy_by_q2_thresh(const GF_GROUP *gf_group,
+ const RATE_CONTROL *rc) {
+ // TODO(now): Return keyframe thresh * factor based on frame type / pyramid
+ // level.
+ if (gf_group->update_type[gf_group->index] == ARF_UPDATE) {
+ return SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME;
+ } else if (gf_group->update_type[gf_group->index] == KF_UPDATE) {
+ if (rc->frames_to_key <= 1)
+ return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO;
+ else
+ return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME;
+ } else {
+ assert(0);
+ }
+ return 0;
+}
+
+static uint8_t get_superres_denom_from_qindex_energy(int qindex, double *energy,
+ double threshq,
+ double threshp) {
+ const double q = av1_convert_qindex_to_q(qindex, AOM_BITS_8);
+ const double tq = threshq * q * q;
+ const double tp = threshp * energy[1];
+ const double thresh = AOMMIN(tq, tp);
+ int k;
+ for (k = SCALE_NUMERATOR * 2; k > SCALE_NUMERATOR; --k) {
+ if (energy[k - 1] > thresh) break;
+ }
+ return 3 * SCALE_NUMERATOR - k;
+}
+
+static uint8_t get_superres_denom_for_qindex(const AV1_COMP *cpi, int qindex,
+ int sr_kf, int sr_arf) {
+ // Use superres for Key-frames and Alt-ref frames only.
+ const GF_GROUP *gf_group = &cpi->gf_group;
+ if (gf_group->update_type[gf_group->index] != KF_UPDATE &&
+ gf_group->update_type[gf_group->index] != ARF_UPDATE) {
+ return SCALE_NUMERATOR;
+ }
+ if (gf_group->update_type[gf_group->index] == KF_UPDATE && !sr_kf) {
+ return SCALE_NUMERATOR;
+ }
+ if (gf_group->update_type[gf_group->index] == ARF_UPDATE && !sr_arf) {
+ return SCALE_NUMERATOR;
+ }
+
+ double energy[16];
+ analyze_hor_freq(cpi, energy);
+
+ const double energy_by_q2_thresh =
+ get_energy_by_q2_thresh(gf_group, &cpi->rc);
+ int denom = get_superres_denom_from_qindex_energy(
+ qindex, energy, energy_by_q2_thresh, SUPERRES_ENERGY_BY_AC_THRESH);
+ /*
+ printf("\nenergy = [");
+ for (int k = 1; k < 16; ++k) printf("%f, ", energy[k]);
+ printf("]\n");
+ printf("boost = %d\n",
+ (gf_group->update_type[gf_group->index] == KF_UPDATE)
+ ? cpi->rc.kf_boost
+ : cpi->rc.gfu_boost);
+ printf("denom = %d\n", denom);
+ */
+#if CONFIG_SUPERRES_IN_RECODE
+ if (av1_superres_in_recode_allowed(cpi)) {
+ assert(cpi->superres_mode != AOM_SUPERRES_NONE);
+ // Force superres to be tried in the recode loop, as full-res is also going
+ // to be tried anyway.
+ denom = AOMMAX(denom, SCALE_NUMERATOR + 1);
+ }
+#endif // CONFIG_SUPERRES_IN_RECODE
+ return denom;
+}
+
+static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) {
+ // Choose an arbitrary random number
+ static unsigned int seed = 34567;
+ const AV1EncoderConfig *oxcf = &cpi->oxcf;
+ const SuperResCfg *const superres_cfg = &oxcf->superres_cfg;
+
+ if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR;
+ uint8_t new_denom = SCALE_NUMERATOR;
+
+ // Make sure that superres mode of the frame is consistent with the
+ // sequence-level flag.
+ assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_NONE,
+ cpi->common.seq_params.enable_superres));
+ assert(IMPLIES(!cpi->common.seq_params.enable_superres,
+ superres_cfg->superres_mode == AOM_SUPERRES_NONE));
+ // Make sure that superres mode for current encoding is consistent with user
+ // provided superres mode.
+ assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_AUTO,
+ cpi->superres_mode == superres_cfg->superres_mode));
+
+ // Note: we must look at the current superres_mode to be tried in 'cpi' here,
+ // not the user given mode in 'oxcf'.
+ switch (cpi->superres_mode) {
+ case AOM_SUPERRES_NONE: new_denom = SCALE_NUMERATOR; break;
+ case AOM_SUPERRES_FIXED:
+ if (cpi->common.current_frame.frame_type == KEY_FRAME)
+ new_denom = superres_cfg->superres_kf_scale_denominator;
+ else
+ new_denom = superres_cfg->superres_scale_denominator;
+ break;
+ case AOM_SUPERRES_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break;
+ case AOM_SUPERRES_QTHRESH: {
+ // Do not use superres when screen content tools are used.
+ if (cpi->common.features.allow_screen_content_tools) break;
+ if (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ)
+ av1_set_target_rate(cpi, cpi->oxcf.width, cpi->oxcf.height);
+
+ // Now decide the use of superres based on 'q'.
+ int bottom_index, top_index;
+ const int q = av1_rc_pick_q_and_bounds(
+ cpi, &cpi->rc, cpi->oxcf.width, cpi->oxcf.height, cpi->gf_group.index,
+ &bottom_index, &top_index);
+
+ const int qthresh = (frame_is_intra_only(&cpi->common))
+ ? superres_cfg->superres_kf_qthresh
+ : superres_cfg->superres_qthresh;
+ if (q <= qthresh) {
+ new_denom = SCALE_NUMERATOR;
+ } else {
+ new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1);
+ }
+ break;
+ }
+ case AOM_SUPERRES_AUTO: {
+ // Do not use superres when screen content tools are used.
+ if (cpi->common.features.allow_screen_content_tools) break;
+ if (oxcf->rc_mode == AOM_VBR || oxcf->rc_mode == AOM_CQ)
+ av1_set_target_rate(cpi, cpi->oxcf.width, cpi->oxcf.height);
+
+ // Now decide the use of superres based on 'q'.
+ int bottom_index, top_index;
+ const int q = av1_rc_pick_q_and_bounds(
+ cpi, &cpi->rc, cpi->oxcf.width, cpi->oxcf.height, cpi->gf_group.index,
+ &bottom_index, &top_index);
+
+ const int qthresh = 128;
+ if (q <= qthresh) {
+ new_denom = SCALE_NUMERATOR;
+ } else {
+#if SUPERRES_RECODE_ALL_RATIOS
+ if (cpi->common.current_frame.frame_type == KEY_FRAME)
+ new_denom = superres_cfg->superres_kf_scale_denominator;
+ else
+ new_denom = superres_cfg->superres_scale_denominator;
+#else
+ new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1);
+#endif // SUPERRES_RECODE_ALL_RATIOS
+ }
+ break;
+ }
+ default: assert(0);
+ }
+ return new_denom;
+}
+
+static int dimension_is_ok(int orig_dim, int resized_dim, int denom) {
+ return (resized_dim * SCALE_NUMERATOR >= orig_dim * denom / 2);
+}
+
+static int dimensions_are_ok(int owidth, int oheight, size_params_type *rsz) {
+ // Only need to check the width, as scaling is horizontal only.
+ (void)oheight;
+ return dimension_is_ok(owidth, rsz->resize_width, rsz->superres_denom);
+}
+
+static int validate_size_scales(RESIZE_MODE resize_mode,
+ aom_superres_mode superres_mode, int owidth,
+ int oheight, size_params_type *rsz) {
+ if (dimensions_are_ok(owidth, oheight, rsz)) { // Nothing to do.
+ return 1;
+ }
+
+ // Calculate current resize scale.
+ int resize_denom =
+ AOMMAX(DIVIDE_AND_ROUND(owidth * SCALE_NUMERATOR, rsz->resize_width),
+ DIVIDE_AND_ROUND(oheight * SCALE_NUMERATOR, rsz->resize_height));
+
+ if (resize_mode != RESIZE_RANDOM && superres_mode == AOM_SUPERRES_RANDOM) {
+ // Alter superres scale as needed to enforce conformity.
+ rsz->superres_denom =
+ (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / resize_denom;
+ if (!dimensions_are_ok(owidth, oheight, rsz)) {
+ if (rsz->superres_denom > SCALE_NUMERATOR) --rsz->superres_denom;
+ }
+ } else if (resize_mode == RESIZE_RANDOM &&
+ superres_mode != AOM_SUPERRES_RANDOM) {
+ // Alter resize scale as needed to enforce conformity.
+ resize_denom =
+ (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / rsz->superres_denom;
+ rsz->resize_width = owidth;
+ rsz->resize_height = oheight;
+ av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height,
+ resize_denom);
+ if (!dimensions_are_ok(owidth, oheight, rsz)) {
+ if (resize_denom > SCALE_NUMERATOR) {
+ --resize_denom;
+ rsz->resize_width = owidth;
+ rsz->resize_height = oheight;
+ av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height,
+ resize_denom);
+ }
+ }
+ } else if (resize_mode == RESIZE_RANDOM &&
+ superres_mode == AOM_SUPERRES_RANDOM) {
+ // Alter both resize and superres scales as needed to enforce conformity.
+ do {
+ if (resize_denom > rsz->superres_denom)
+ --resize_denom;
+ else
+ --rsz->superres_denom;
+ rsz->resize_width = owidth;
+ rsz->resize_height = oheight;
+ av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height,
+ resize_denom);
+ } while (!dimensions_are_ok(owidth, oheight, rsz) &&
+ (resize_denom > SCALE_NUMERATOR ||
+ rsz->superres_denom > SCALE_NUMERATOR));
+ } else { // We are allowed to alter neither resize scale nor superres
+ // scale.
+ return 0;
+ }
+ return dimensions_are_ok(owidth, oheight, rsz);
+}
+
+// Calculates resize and superres params for next frame
+static size_params_type calculate_next_size_params(AV1_COMP *cpi) {
+ const AV1EncoderConfig *oxcf = &cpi->oxcf;
+ ResizePendingParams *resize_pending_params = &cpi->resize_pending_params;
+ size_params_type rsz = { oxcf->width, oxcf->height, SCALE_NUMERATOR };
+ int resize_denom = SCALE_NUMERATOR;
+ if (has_no_stats_stage(cpi) && cpi->use_svc &&
+ cpi->svc.spatial_layer_id < cpi->svc.number_spatial_layers - 1) {
+ rsz.resize_width = cpi->common.width;
+ rsz.resize_height = cpi->common.height;
+ return rsz;
+ }
+ if (is_stat_generation_stage(cpi)) return rsz;
+ if (resize_pending_params->width && resize_pending_params->height) {
+ rsz.resize_width = resize_pending_params->width;
+ rsz.resize_height = resize_pending_params->height;
+ resize_pending_params->width = resize_pending_params->height = 0;
+ } else {
+ resize_denom = calculate_next_resize_scale(cpi);
+ rsz.resize_width = oxcf->width;
+ rsz.resize_height = oxcf->height;
+ av1_calculate_scaled_size(&rsz.resize_width, &rsz.resize_height,
+ resize_denom);
+ }
+ rsz.superres_denom = calculate_next_superres_scale(cpi);
+ if (!validate_size_scales(oxcf->resize_cfg.resize_mode, cpi->superres_mode,
+ oxcf->width, oxcf->height, &rsz))
+ assert(0 && "Invalid scale parameters");
+ return rsz;
+}
+
+static void setup_frame_size_from_params(AV1_COMP *cpi,
+ const size_params_type *rsz) {
+ int encode_width = rsz->resize_width;
+ int encode_height = rsz->resize_height;
+
+ AV1_COMMON *cm = &cpi->common;
+ cm->superres_upscaled_width = encode_width;
+ cm->superres_upscaled_height = encode_height;
+ cm->superres_scale_denominator = rsz->superres_denom;
+ av1_calculate_scaled_superres_size(&encode_width, &encode_height,
+ rsz->superres_denom);
+ av1_set_frame_size(cpi, encode_width, encode_height);
+}
+
+void av1_setup_frame_size(AV1_COMP *cpi) {
+ AV1_COMMON *cm = &cpi->common;
+ // Reset superres params from previous frame.
+ cm->superres_scale_denominator = SCALE_NUMERATOR;
+ const size_params_type rsz = calculate_next_size_params(cpi);
+ setup_frame_size_from_params(cpi, &rsz);
+
+ assert(av1_is_min_tile_width_satisfied(cm));
+}
+
+void av1_superres_post_encode(AV1_COMP *cpi) {
+ AV1_COMMON *cm = &cpi->common;
+ const int num_planes = av1_num_planes(cm);
+
+ if (!av1_superres_scaled(cm)) return;
+
+ assert(cpi->oxcf.superres_cfg.enable_superres);
+ assert(!is_lossless_requested(&cpi->oxcf));
+ assert(!cm->features.all_lossless);
+
+ av1_superres_upscale(cm, NULL);
+
+ // If regular resizing is occurring the source will need to be downscaled to
+ // match the upscaled superres resolution. Otherwise the original source is
+ // used.
+ if (!av1_resize_scaled(cm)) {
+ cpi->source = cpi->unscaled_source;
+ if (cpi->last_source != NULL) cpi->last_source = cpi->unscaled_last_source;
+ } else {
+ assert(cpi->unscaled_source->y_crop_width != cm->superres_upscaled_width);
+ assert(cpi->unscaled_source->y_crop_height != cm->superres_upscaled_height);
+ // Do downscale. cm->(width|height) has been updated by
+ // av1_superres_upscale
+ if (aom_realloc_frame_buffer(
+ &cpi->scaled_source, cm->superres_upscaled_width,
+ cm->superres_upscaled_height, cm->seq_params.subsampling_x,
+ cm->seq_params.subsampling_y, cm->seq_params.use_highbitdepth,
+ AOM_BORDER_IN_PIXELS, cm->features.byte_alignment, NULL, NULL,
+ NULL))
+ aom_internal_error(
+ &cm->error, AOM_CODEC_MEM_ERROR,
+ "Failed to reallocate scaled source buffer for superres");
+ assert(cpi->scaled_source.y_crop_width == cm->superres_upscaled_width);
+ assert(cpi->scaled_source.y_crop_height == cm->superres_upscaled_height);
+ av1_resize_and_extend_frame(cpi->unscaled_source, &cpi->scaled_source,
+ (int)cm->seq_params.bit_depth, num_planes);
+ cpi->source = &cpi->scaled_source;
+ }
+}
diff --git a/av1/encoder/superres_scale.h b/av1/encoder/superres_scale.h
new file mode 100644
index 0000000..691329c
--- /dev/null
+++ b/av1/encoder/superres_scale.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_SUPERRES_SCALE_H_
+#define AOM_AV1_ENCODER_SUPERRES_SCALE_H_
+
+#include "av1/encoder/encoder.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// If true, AOM_SUPERRES_AUTO mode will exhaustively search over all superres
+// denominators for all frames (except overlay and internal overlay frames).
+#define SUPERRES_RECODE_ALL_RATIOS 0
+
+int av1_superres_in_recode_allowed(const AV1_COMP *const cpi);
+void av1_superres_post_encode(AV1_COMP *cpi);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // AOM_AV1_ENCODER_SUPERRES_SCALE_H_
