frame-superres: Pre-encode downscaling
Adds handling needed for the pre-encode downscaling for superres
encoding. Makes use of the existing resizing functionality, just
adjusting the target resolution after the resize logic and before the
resize has actually occurred.
Since the upscaling portion of superres is not yet implemented, superres
scaling is not yet allowed to trigger. The numerator is set to be equal
to the denominator and the superres_pending flag is not set.
Change-Id: Ia5dac752d653c3bda319f0c8196e95bac2c8c554
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index d4832a1..4e13409 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -483,6 +483,16 @@
oxcf->resize_mode = RESIZE_NONE;
}
+ // Initialize to input resolution if not specified.
+ if (oxcf->resize_mode != RESIZE_FIXED) {
+ oxcf->scaled_frame_width = oxcf->width;
+ oxcf->scaled_frame_height = oxcf->height;
+ }
+
+#if CONFIG_FRAME_SUPERRES
+ oxcf->superres_enabled = 1; // TODO(afergs): Check the config
+#endif // CONFIG_FRAME_SUPERRES
+
oxcf->maximum_buffer_size_ms = is_vbr ? 240000 : cfg->rc_buf_sz;
oxcf->starting_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_initial_sz;
oxcf->optimal_buffer_level_ms = is_vbr ? 60000 : cfg->rc_buf_optimal_sz;
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 5673fb8..03aaad9 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -2781,10 +2781,6 @@
static void setup_superres_size(AV1_COMMON *const cm,
struct aom_read_bit_buffer *rb, int *width,
int *height) {
- // TODO(afergs): Test this behaviour
- // Frame superres is probably in compatible with this render resolution
- assert(cm->width == cm->render_width && cm->height == cm->render_height);
-
cm->superres_width = cm->width;
cm->superres_height = cm->height;
if (aom_rb_read_bit(rb)) {
@@ -2797,9 +2793,9 @@
// by default (ie. when it isn't sent)...
// resize_context_buffers() will change cm->width to equal cm->render_width,
// then they'll be the same again
- *width = cm->render_width =
+ *width =
cm->width * cm->superres_scale_numerator / SUPERRES_SCALE_DENOMINATOR;
- *height = cm->render_height =
+ *height =
cm->height * cm->superres_scale_numerator / SUPERRES_SCALE_DENOMINATOR;
} else {
// 1:1 scaling - ie. no scaling, scale not provided
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 4be8557..b02e7de 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -4207,9 +4207,6 @@
#if CONFIG_FRAME_SUPERRES
static void write_superres_scale(const AV1_COMMON *const cm,
struct aom_write_bit_buffer *wb) {
- // This scaling and frame superres are probably incompatible
- assert(cm->width == cm->render_width && cm->height == cm->render_height);
-
// First bit is whether to to scale or not
if (cm->superres_scale_numerator == SUPERRES_SCALE_DENOMINATOR) {
aom_wb_write_bit(wb, 0); // no scaling
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index b1ac28f..84e4e93 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -3819,30 +3819,58 @@
AV1EncoderConfig *const oxcf = &cpi->oxcf;
MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
+ int scaled_size_set = 0;
+
+ const int one_pass_cbr_dynamic = oxcf->pass == 0 &&
+ oxcf->rc_mode == AOM_CBR &&
+ oxcf->resize_mode == RESIZE_DYNAMIC;
+
+ // TODO(afergs): Replace with call to av1_resize_pending? Could replace
+ // scaled_size_set as well.
+ // TODO(afergs): Realistically, if resize_pending is true, then the other
+ // conditions must already be satisfied.
+ // Try this first:
+ // av1_resize_pending &&
+ // (DYNAMIC && (1 Pass CBR || 2 Pass VBR)
+ // STATIC && FIRST_FRAME)
+ // Really, av1_resize_pending should just reflect the above.
// TODO(afergs): Allow fixed resizing in AOM_CBR mode?
-
- if (oxcf->pass == 2 && oxcf->rc_mode == AOM_VBR &&
- ((oxcf->resize_mode == RESIZE_FIXED && cm->current_video_frame == 0) ||
- (oxcf->resize_mode == RESIZE_DYNAMIC && av1_resize_pending(cpi)))) {
- av1_calculate_next_coded_size(cpi, &oxcf->scaled_frame_width,
- &oxcf->scaled_frame_height);
-
- // There has been a change in frame size.
- av1_set_size_literal(cpi, oxcf->scaled_frame_width,
- oxcf->scaled_frame_height);
+ // 2 Pass VBR: Resize if fixed resize and first frame, or dynamic resize and
+ // a resize is pending.
+ // 1 Pass CBR: Resize if dynamic resize and resize pending.
+ if ((oxcf->pass == 2 && oxcf->rc_mode == AOM_VBR &&
+ ((oxcf->resize_mode == RESIZE_FIXED && cm->current_video_frame == 0) ||
+ (oxcf->resize_mode == RESIZE_DYNAMIC && av1_resize_pending(cpi)))) ||
+ (one_pass_cbr_dynamic && av1_resize_pending(cpi))) {
+ av1_calculate_next_scaled_size(cpi, &oxcf->scaled_frame_width,
+ &oxcf->scaled_frame_height);
+ scaled_size_set = 1;
}
- if (oxcf->pass == 0 && oxcf->rc_mode == AOM_CBR &&
- oxcf->resize_mode == RESIZE_DYNAMIC && av1_resize_pending(cpi)) {
- av1_calculate_next_coded_size(cpi, &oxcf->scaled_frame_width,
- &oxcf->scaled_frame_height);
+#if CONFIG_FRAME_SUPERRES
+ // TODO(afergs): Make superres_pending a function like av1_resize_pending.
+ // TODO(afergs): Use av1_resize_pending? It's not a guarantee that a resize
+ // occurred yet, so wait for the above logic to be simplified.
+ if (scaled_size_set || cpi->superres_pending) {
+ // Recalculate superres resolution if resize or superres scales changed.
+ int encode_width, encode_height;
+ av1_calculate_superres_size(cpi, &encode_width, &encode_height);
+ // There has been a change in the encoded frame size
+ av1_set_size_literal(cpi, encode_width, encode_height);
+#else
+ if (scaled_size_set) {
// There has been a change in frame size.
av1_set_size_literal(cpi, oxcf->scaled_frame_width,
oxcf->scaled_frame_height);
+#endif // CONFIG_FRAME_SUPERRES
// TODO(agrange) Scale cpi->max_mv_magnitude if frame-size has changed.
- set_mv_search_params(cpi);
+ // TODO(afergs): Make condition just (pass == 0) or (rc_mode == CBR) -
+ // UNLESS CBR starts allowing FIXED resizing. Then the resize
+ // mode will need to get checked too.
+ if (one_pass_cbr_dynamic)
+ set_mv_search_params(cpi); // TODO(afergs): Needed? Caller calls after...
}
#if !CONFIG_XIPHRC
@@ -3939,6 +3967,13 @@
aom_clear_system_state();
+#if CONFIG_FRAME_SUPERRES
+ // TODO(afergs): Figure out when is actually a good time to do superres
+ cm->superres_scale_numerator = SUPERRES_SCALE_DENOMINATOR;
+ // (uint8_t)(rand() % 9 + SUPERRES_SCALE_NUMERATOR_MIN);
+ cpi->superres_pending = cpi->oxcf.superres_enabled && 0;
+#endif // CONFIG_FRAME_SUPERRES
+
set_frame_size(cpi);
av1_resize_step(cpi);
@@ -3995,6 +4030,11 @@
// transform / motion compensation build reconstruction frame
av1_encode_frame(cpi);
+#if CONFIG_FRAME_SUPERRES
+ // TODO(afergs): Upscale the frame to show
+ cpi->superres_pending = 0;
+#endif // CONFIG_FRAME_SUPERRES
+
// Update some stats from cyclic refresh, and check if we should not update
// golden reference, for 1 pass CBR.
if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->frame_type != KEY_FRAME &&
@@ -4031,7 +4071,11 @@
set_frame_size(cpi);
+#if CONFIG_FRAME_SUPERRES
+ if (loop_count == 0 || av1_resize_pending(cpi) || cpi->superres_pending) {
+#else
if (loop_count == 0 || av1_resize_pending(cpi)) {
+#endif // CONFIG_FRAME_SUPERRES
set_size_dependent_vars(cpi, &q, &bottom_index, &top_index);
// cpi->sf.use_upsampled_references can be different from frame to frame.
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index aec0047..164a6d9 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -208,6 +208,11 @@
int scaled_frame_width;
int scaled_frame_height;
+#if CONFIG_FRAME_SUPERRES
+ // Frame Super-Resolution size scaling
+ int superres_enabled;
+#endif // CONFIG_FRAME_SUPERRES
+
// Enable feature to reduce the frame quantization every x frames.
int frame_periodic_boost;
@@ -624,6 +629,10 @@
int resize_buffer_underflow;
int resize_count;
+#if CONFIG_FRAME_SUPERRES
+ int superres_pending;
+#endif // CONFIG_FRAME_SUPERRES
+
// VAR_BASED_PARTITION thresholds
// 0 - threshold_128x128;
// 1 - threshold_64x64;
diff --git a/av1/encoder/firstpass.c b/av1/encoder/firstpass.c
index 7797420..c5a7bbf 100644
--- a/av1/encoder/firstpass.c
+++ b/av1/encoder/firstpass.c
@@ -1235,14 +1235,27 @@
}
}
-void av1_calculate_next_coded_size(const AV1_COMP *cpi, int *scaled_frame_width,
- int *scaled_frame_height) {
+void av1_calculate_next_scaled_size(const AV1_COMP *cpi,
+ int *scaled_frame_width,
+ int *scaled_frame_height) {
*scaled_frame_width =
cpi->oxcf.width * cpi->resize_next_scale_num / cpi->resize_next_scale_den;
*scaled_frame_height = cpi->oxcf.height * cpi->resize_next_scale_num /
cpi->resize_next_scale_den;
}
+#if CONFIG_FRAME_SUPERRES
+void av1_calculate_superres_size(const AV1_COMP *cpi, int *encoded_width,
+ int *encoded_height) {
+ *encoded_width = cpi->oxcf.scaled_frame_width *
+ cpi->common.superres_scale_numerator /
+ SUPERRES_SCALE_DENOMINATOR;
+ *encoded_height = cpi->oxcf.scaled_frame_height *
+ cpi->common.superres_scale_numerator /
+ SUPERRES_SCALE_DENOMINATOR;
+}
+#endif // CONFIG_FRAME_SUPERRES
+
void av1_init_second_pass(AV1_COMP *cpi) {
const AV1EncoderConfig *const oxcf = &cpi->oxcf;
TWO_PASS *const twopass = &cpi->twopass;
diff --git a/av1/encoder/firstpass.h b/av1/encoder/firstpass.h
index 830f4e1..4310445 100644
--- a/av1/encoder/firstpass.h
+++ b/av1/encoder/firstpass.h
@@ -177,9 +177,17 @@
// Post encode update of the rate control parameters for 2-pass
void av1_twopass_postencode_update(struct AV1_COMP *cpi);
-void av1_calculate_next_coded_size(const struct AV1_COMP *cpi,
- int *scaled_frame_width,
- int *scaled_frame_height);
+void av1_calculate_next_scaled_size(const struct AV1_COMP *cpi,
+ int *scaled_frame_width,
+ int *scaled_frame_height);
+
+#if CONFIG_FRAME_SUPERRES
+// This is the size after superress scaling, which could be 1:1.
+// Superres scaling happens after regular downscaling.
+// TODO(afergs): Limit overall reduction to 1/2 of the original size
+void av1_calculate_superres_size(const struct AV1_COMP *cpi, int *encoded_width,
+ int *encoded_height);
+#endif // CONFIG_FRAME_SUPERRES
#if CONFIG_EXT_REFS
static INLINE int get_number_of_extra_arfs(int interval, int arf_pending) {
