Delta-code against scaled reference model
diff --git a/aom_dsp/bitwriter_buffer.c b/aom_dsp/bitwriter_buffer.c index 7e41949..ac0710f 100644 --- a/aom_dsp/bitwriter_buffer.c +++ b/aom_dsp/bitwriter_buffer.c
@@ -92,6 +92,7 @@ void aom_wb_write_primitive_quniform(struct aom_write_bit_buffer *wb, uint16_t n, uint16_t v) { if (n <= 1) return; + assert(v < n); const int l = get_msb(n) + 1; const int m = (1 << l) - n; if (v < m) {
diff --git a/av1/common/av1_common_int.h b/av1/common/av1_common_int.h index eee6f58..a1dac85 100644 --- a/av1/common/av1_common_int.h +++ b/av1/common/av1_common_int.h
@@ -260,6 +260,12 @@ // Frame's level within the hierarchical structure unsigned int pyramid_level; +#if CONFIG_IMPROVED_GLOBAL_MOTION + // How many ref frames did this frame use? + // This is set to 0 for intra frames + int num_ref_frames; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + MV_REF *mvs; uint8_t *seg_map; struct segmentation seg; @@ -1567,6 +1573,18 @@ */ DeltaQInfo delta_q_info; +#if CONFIG_IMPROVED_GLOBAL_MOTION + /*! + * Base model used for delta-coding global motion parameters + */ + WarpedMotionParams base_global_motion_model; + + /*! + * Temporal length of `base_global_motion_model` + */ + int base_global_motion_distance; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + /*! * Global motion parameters for each reference frame. */
diff --git a/av1/common/warped_motion.h b/av1/common/warped_motion.h index d7f2bca..e10a279 100644 --- a/av1/common/warped_motion.h +++ b/av1/common/warped_motion.h
@@ -341,4 +341,54 @@ return motion_mode == WARPED_CAUSAL; #endif // CONFIG_INTERINTRA_WARP } + +#if CONFIG_IMPROVED_GLOBAL_MOTION +static INLINE void av1_scale_warp_model(const WarpedMotionParams *in_params, + int in_distance, + WarpedMotionParams *out_params, + int out_distance) { + static int param_shift[MAX_PARAMDIM - 1] = { + GM_TRANS_PREC_DIFF, GM_TRANS_PREC_DIFF, GM_ALPHA_PREC_DIFF, + GM_ALPHA_PREC_DIFF, GM_ALPHA_PREC_DIFF, GM_ALPHA_PREC_DIFF, + GM_ROW3HOMO_PREC_DIFF, GM_ROW3HOMO_PREC_DIFF + }; + + static int param_min[MAX_PARAMDIM - 1] = { GM_TRANS_MIN, GM_TRANS_MIN, + GM_ALPHA_MIN, GM_ALPHA_MIN, + GM_ALPHA_MIN, GM_ALPHA_MIN, + GM_ROW3HOMO_MIN, GM_ROW3HOMO_MIN }; + + static int param_max[MAX_PARAMDIM - 1] = { GM_TRANS_MAX, GM_TRANS_MAX, + GM_ALPHA_MAX, GM_ALPHA_MAX, + GM_ALPHA_MAX, GM_ALPHA_MAX, + GM_ROW3HOMO_MAX, GM_ROW3HOMO_MAX }; + + assert(in_distance != 0); + assert(out_distance != 0); + + // Flip signs so that in_distance is positive. + // We do this because + // scaled_value = (... + divisor/2) / divisor + // is the simplest way to implement division with round-to-nearest in C, + // but it only works correctly if the divisor is positive + if (in_distance < 0) { + in_distance = -in_distance; + out_distance = -out_distance; + } + + out_params->wmtype = in_params->wmtype; + for (int param = 0; param < MAX_PARAMDIM - 1; param++) { + int center = default_warp_params.wmmat[param]; + + int input = in_params->wmmat[param] - center; + int divisor = in_distance << param_shift[param]; + int output = (int)(((int64_t)input * out_distance + divisor / 2) / divisor); + output = clamp(output, param_min[param], param_max[param]) + << param_shift[param]; + + out_params->wmmat[param] = center + output; + } +} +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + #endif // AOM_AV1_COMMON_WARPED_MOTION_H_
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 8651d3c..6836712 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -6203,8 +6203,10 @@ static AOM_INLINE void read_global_motion(AV1_COMMON *cm, struct aom_read_bit_buffer *rb) { #if CONFIG_IMPROVED_GLOBAL_MOTION + const SequenceHeader *const seq_params = &cm->seq_params; + int num_total_refs = cm->ref_frames_info.num_total_refs; bool use_global_motion = false; - if (cm->seq_params.enable_global_motion) { + if (seq_params->enable_global_motion) { use_global_motion = aom_rb_read_bit(rb); } if (!use_global_motion) { @@ -6214,12 +6216,59 @@ } return; } + + int our_ref = aom_rb_read_primitive_quniform(rb, num_total_refs + 1); + if (our_ref == num_total_refs) { + // Special case: Use IDENTITY model + cm->base_global_motion_model = default_warp_params; + cm->base_global_motion_distance = 1; + } else { + RefCntBuffer *buf = get_ref_frame_buf(cm, our_ref); + assert(buf); + int their_num_refs = buf->num_ref_frames; + if (their_num_refs == 0) { + // Special case: if an intra/key frame is used as a ref, use an + // IDENTITY model + cm->base_global_motion_model = default_warp_params; + cm->base_global_motion_distance = 1; + } else { + int their_ref = aom_rb_read_primitive_quniform(rb, their_num_refs); + cm->base_global_motion_model = buf->global_motion[their_ref]; + cm->base_global_motion_distance = + get_relative_dist(&seq_params->order_hint_info, buf->order_hint, + buf->ref_order_hints[their_ref]); + } + } #endif // CONFIG_IMPROVED_GLOBAL_MOTION for (int frame = 0; frame < cm->ref_frames_info.num_total_refs; ++frame) { +#if CONFIG_IMPROVED_GLOBAL_MOTION + int temporal_distance; + if (seq_params->order_hint_info.enable_order_hint) { + const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, frame); + temporal_distance = get_relative_dist(&seq_params->order_hint_info, + (int)cm->cur_frame->order_hint, + (int)ref_buf->order_hint); + } else { + temporal_distance = 1; + } + + if (temporal_distance == 0) { + // Don't code global motion for frames at the same temporal instant + cm->global_motion[frame] = default_warp_params; + continue; + } + + WarpedMotionParams ref_params_; + av1_scale_warp_model(&cm->base_global_motion_model, + cm->base_global_motion_distance, &ref_params_, + temporal_distance); + WarpedMotionParams *ref_params = &ref_params_; +#else const WarpedMotionParams *ref_params = cm->prev_frame ? &cm->prev_frame->global_motion[frame] : &default_warp_params; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION int good_params = #if !CONFIG_FLEX_MVRES read_global_motion_params(&cm->global_motion[frame], ref_params, rb, @@ -6782,6 +6831,10 @@ features->allow_ref_frame_mvs = 0; cm->prev_frame = NULL; + +#if CONFIG_IMPROVED_GLOBAL_MOTION + cm->cur_frame->num_ref_frames = 0; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION } else { features->allow_ref_frame_mvs = 0; #if CONFIG_TIP @@ -6807,6 +6860,10 @@ } #endif // CONFIG_IBC_SR_EXT +#if CONFIG_IMPROVED_GLOBAL_MOTION + cm->cur_frame->num_ref_frames = 0; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + } else if (pbi->need_resync != 1) { /* Skip if need resync */ // Implicitly derive the reference mapping RefFrameMapPair ref_frame_map_pairs[REF_FRAMES]; @@ -6887,6 +6944,9 @@ } av1_get_past_future_cur_ref_lists(cm, scores); } +#if CONFIG_IMPROVED_GLOBAL_MOTION + cm->cur_frame->num_ref_frames = cm->ref_frames_info.num_total_refs; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION if (!features->error_resilient_mode && frame_size_override_flag) { setup_frame_size_with_refs(cm, rb);
diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 2c94af4..afc3c26 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c
@@ -4582,15 +4582,20 @@ static AOM_INLINE void write_global_motion(AV1_COMP *cpi, struct aom_write_bit_buffer *wb) { AV1_COMMON *const cm = &cpi->common; + int num_total_refs = cm->ref_frames_info.num_total_refs; +#if CONFIG_IMPROVED_GLOBAL_MOTION + assert(cm->cur_frame->num_ref_frames == num_total_refs); +#endif // CONFIG_IMPROVED_GLOBAL_MOTION int frame; #if CONFIG_IMPROVED_GLOBAL_MOTION - if (!cm->seq_params.enable_global_motion) { + const SequenceHeader *const seq_params = &cm->seq_params; + if (!seq_params->enable_global_motion) { return; } bool use_global_motion = false; - for (frame = 0; frame < cm->ref_frames_info.num_total_refs; ++frame) { + for (frame = 0; frame < num_total_refs; ++frame) { if (cm->global_motion[frame].wmtype != IDENTITY) { use_global_motion = true; break; @@ -4601,12 +4606,58 @@ if (!use_global_motion) { return; } + + int our_ref = cpi->gm_info.base_model_our_ref; + int their_ref = cpi->gm_info.base_model_their_ref; + aom_wb_write_primitive_quniform(wb, num_total_refs + 1, our_ref); + if (our_ref >= num_total_refs) { + // Special case: Use IDENTITY model + // Nothing more to code + assert(their_ref == -1); + } else { + RefCntBuffer *buf = get_ref_frame_buf(cm, our_ref); + assert(buf); + int their_num_refs = buf->num_ref_frames; + if (their_num_refs == 0) { + // Special case: if an intra/key frame is used as a ref, use an + // IDENTITY model + // Nothing more to code + assert(their_ref == -1); + } else { + aom_wb_write_primitive_quniform(wb, their_num_refs, their_ref); + } + } #endif // CONFIG_IMPROVED_GLOBAL_MOTION - for (frame = 0; frame < cm->ref_frames_info.num_total_refs; ++frame) { + for (frame = 0; frame < num_total_refs; ++frame) { +#if CONFIG_IMPROVED_GLOBAL_MOTION + int temporal_distance; + if (seq_params->order_hint_info.enable_order_hint) { + const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, frame); + temporal_distance = get_relative_dist(&seq_params->order_hint_info, + (int)cm->cur_frame->order_hint, + (int)ref_buf->order_hint); + } else { + temporal_distance = 1; + } + + if (temporal_distance == 0) { + // Don't code global motion for frames at the same temporal instant + assert(cm->global_motion[frame].wmtype == IDENTITY); + continue; + } + + WarpedMotionParams ref_params_; + av1_scale_warp_model(&cm->base_global_motion_model, + cm->base_global_motion_distance, &ref_params_, + temporal_distance); + WarpedMotionParams *ref_params = &ref_params_; +#else const WarpedMotionParams *ref_params = cm->prev_frame ? &cm->prev_frame->global_motion[frame] : &default_warp_params; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + write_global_motion_params(&cm->global_motion[frame], ref_params, wb, #if !CONFIG_FLEX_MVRES cm->features.allow_high_precision_mv);
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c index a931f84..2683de9 100644 --- a/av1/encoder/encode_strategy.c +++ b/av1/encoder/encode_strategy.c
@@ -1105,6 +1105,9 @@ AOMMIN(cm->seq_params.num_same_ref_compound, cm->ref_frames_info.num_total_refs); #endif // CONFIG_ALLOW_SAME_REF_COMPOUND +#if CONFIG_IMPROVED_GLOBAL_MOTION + cm->cur_frame->num_ref_frames = cm->ref_frames_info.num_total_refs; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION // ref_frame_flags is defined based on the external flag // max-reference-frames.
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index 4b68d58..c272ed9 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -1978,6 +1978,25 @@ int segment_map_w; /*!< segment map width */ int segment_map_h; /*!< segment map height */ /**@}*/ + +#if CONFIG_IMPROVED_GLOBAL_MOTION + /*! + * \brief Error ratio for each selected global motion model + * + * This is used to help decide which models will actually be used, + * because that decision has to be deferred until we actually select a + * base model to use + */ + double erroradvantage[INTER_REFS_PER_FRAME]; + + /** + * \name Reference path for selected base model + */ + /**@{*/ + int base_model_our_ref; /*!< which of our ref frames to copy from */ + int base_model_their_ref; /*!< which model to copy from that frame */ + /**@}*/ +#endif // CONFIG_IMPROVED_GLOBAL_MOTION } GlobalMotionInfo; /*!
diff --git a/av1/encoder/global_motion_facade.c b/av1/encoder/global_motion_facade.c index 69b725c..648ca60 100644 --- a/av1/encoder/global_motion_facade.c +++ b/av1/encoder/global_motion_facade.c
@@ -106,14 +106,25 @@ // For the given reference frame, computes the global motion parameters for // different motion models and finds the best. +#if CONFIG_IMPROVED_GLOBAL_MOTION +static AOM_INLINE void compute_global_motion_for_ref_frame( + AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[INTER_REFS_PER_FRAME], int frame, + MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w, + const int segment_map_h) { +#else static AOM_INLINE void compute_global_motion_for_ref_frame( AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[INTER_REFS_PER_FRAME], int frame, MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w, const int segment_map_h, const WarpedMotionParams *ref_params) { +#endif // CONFIG_IMPROVED_GLOBAL_MOTION ThreadData *const td = &cpi->td; MACROBLOCK *const x = &td->mb; AV1_COMMON *const cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; +#if CONFIG_IMPROVED_GLOBAL_MOTION + GlobalMotionInfo *const gm_info = &cpi->gm_info; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + int i; int src_width = cpi->source->y_crop_width; int src_height = cpi->source->y_crop_height; @@ -204,6 +215,12 @@ if (tmp_wm_params.wmtype == IDENTITY) continue; #endif // CONFIG_IMPROVED_GLOBAL_MOTION +#if CONFIG_IMPROVED_GLOBAL_MOTION + // Apply initial quality filter, which depends only on the error metrics + // and not the model cost + if (warp_error >= ref_frame_error * erroradv_tr) continue; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + if (warp_error < best_warp_error) { best_ref_frame_error = ref_frame_error; best_warp_error = warp_error; @@ -246,6 +263,12 @@ // ref_frame_error == 0 assert(best_ref_frame_error > 0); +#if CONFIG_IMPROVED_GLOBAL_MOTION + gm_info->erroradvantage[frame] = + (double)best_warp_error / best_ref_frame_error; + + break; +#else // If the best error advantage found doesn't meet the threshold for // this motion type, revert to IDENTITY. if (!av1_is_enough_erroradvantage( @@ -260,6 +283,7 @@ } if (cm->global_motion[frame].wmtype != IDENTITY) break; +#endif // CONFIG_IMPROVED_GLOBAL_MOTION } aom_free_flow_data(flow_data); @@ -271,6 +295,11 @@ AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[INTER_REFS_PER_FRAME], int frame, MotionModel *motion_models, uint8_t *segment_map, int segment_map_w, int segment_map_h) { +#if CONFIG_IMPROVED_GLOBAL_MOTION + compute_global_motion_for_ref_frame(cpi, ref_buf, frame, motion_models, + segment_map, segment_map_w, + segment_map_h); +#else AV1_COMMON *const cm = &cpi->common; const WarpedMotionParams *ref_params = cm->prev_frame ? &cm->prev_frame->global_motion[frame] @@ -279,6 +308,7 @@ compute_global_motion_for_ref_frame(cpi, ref_buf, frame, motion_models, segment_map, segment_map_w, segment_map_h, ref_params); +#endif // CONFIG_IMPROVED_GLOBAL_MOTION } // Loops over valid reference frames and computes global motion estimation. @@ -445,6 +475,150 @@ return true; } +#if CONFIG_IMPROVED_GLOBAL_MOTION +// Select which global motion model to use as a base +static AOM_INLINE void pick_base_gm_params(AV1_COMP *cpi) { + AV1_COMMON *const cm = &cpi->common; + const SequenceHeader *const seq_params = &cm->seq_params; + GlobalMotionInfo *const gm_info = &cpi->gm_info; + int num_total_refs = cm->ref_frames_info.num_total_refs; + + int best_our_ref; + int best_their_ref; + const WarpedMotionParams *best_base_model; + int best_temporal_distance; + int best_num_models; + int best_cost; + + // Bitmask of which models we will actually use if we accept the current + // best base model + uint8_t best_enable_models; + + // First, evaluate the identity model as a base + { + int this_num_models = 0; + int this_cost = + aom_count_primitive_quniform(num_total_refs + 1, num_total_refs) + << AV1_PROB_COST_SHIFT; + uint8_t this_enable_models = 0; + + for (int frame = 0; frame < num_total_refs; frame++) { + const WarpedMotionParams *model = &cm->global_motion[frame]; + if (model->wmtype == IDENTITY) continue; + + int model_cost = gm_get_params_cost(model, &default_warp_params, + cm->features.fr_mv_precision); + bool use_model = av1_is_enough_erroradvantage( + gm_info->erroradvantage[frame], model_cost); + + if (use_model) { + this_num_models += 1; + this_cost += model_cost; + this_enable_models |= (1 << frame); + } + } + + // Set initial values + best_our_ref = cm->ref_frames_info.num_total_refs; + best_their_ref = -1; + best_base_model = &default_warp_params; + best_temporal_distance = 1; + best_num_models = this_num_models; + best_cost = this_cost; + best_enable_models = this_enable_models; + } + + // Then try each available reference model in turn + for (int our_ref = 0; our_ref < num_total_refs; ++our_ref) { + const int ref_disabled = !(cm->ref_frame_flags & (1 << our_ref)); + RefCntBuffer *buf = get_ref_frame_buf(cm, our_ref); + // Skip looking at invalid ref frames + if (buf == NULL || + (ref_disabled && cpi->sf.hl_sf.recode_loop != DISALLOW_RECODE)) { + continue; + } + + int their_num_refs = buf->num_ref_frames; + for (int their_ref = 0; their_ref < their_num_refs; ++their_ref) { + const WarpedMotionParams *base_model = &buf->global_motion[their_ref]; + if (base_model->wmtype == IDENTITY) { + continue; + } + + int base_temporal_distance = + get_relative_dist(&seq_params->order_hint_info, buf->order_hint, + buf->ref_order_hints[their_ref]); + + int this_num_models = 0; + int this_cost = + (aom_count_primitive_quniform(num_total_refs + 1, our_ref) + + aom_count_primitive_quniform(their_num_refs, their_ref)) + << AV1_PROB_COST_SHIFT; + uint8_t this_enable_models = 0; + + for (int frame = 0; frame < num_total_refs; frame++) { + const WarpedMotionParams *model = &cm->global_motion[frame]; + if (model->wmtype == IDENTITY) continue; + + int temporal_distance; + if (seq_params->order_hint_info.enable_order_hint) { + const RefCntBuffer *const ref_buf = get_ref_frame_buf(cm, frame); + temporal_distance = get_relative_dist(&seq_params->order_hint_info, + (int)cm->cur_frame->order_hint, + (int)ref_buf->order_hint); + } else { + temporal_distance = 1; + } + + if (temporal_distance == 0) { + // Don't code global motion for frames at the same temporal instant + assert(model->wmtype == IDENTITY); + continue; + } + + WarpedMotionParams ref_params; + av1_scale_warp_model(base_model, base_temporal_distance, &ref_params, + temporal_distance); + + int model_cost = gm_get_params_cost(model, &ref_params, + cm->features.fr_mv_precision); + bool use_model = av1_is_enough_erroradvantage( + gm_info->erroradvantage[frame], model_cost); + + if (use_model) { + this_num_models += 1; + this_cost += model_cost; + this_enable_models |= (1 << frame); + } + } + + if (this_num_models > best_num_models || + (this_num_models == best_num_models && this_cost < best_cost)) { + best_our_ref = our_ref; + best_their_ref = their_ref; + best_base_model = base_model; + best_temporal_distance = base_temporal_distance; + best_num_models = this_num_models; + best_cost = this_cost; + best_enable_models = this_enable_models; + } + } + } + + gm_info->base_model_our_ref = best_our_ref; + gm_info->base_model_their_ref = best_their_ref; + cm->base_global_motion_model = *best_base_model; + cm->base_global_motion_distance = best_temporal_distance; + + for (int frame = 0; frame < num_total_refs; frame++) { + if ((best_enable_models & (1 << frame)) == 0) { + // Disable this model + cm->global_motion[frame] = default_warp_params; + } + } +} +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + // Initializes parameters used for computing global motion. static AOM_INLINE void setup_global_motion_info_params(AV1_COMP *cpi) { GlobalMotionInfo *const gm_info = &cpi->gm_info; @@ -544,6 +718,12 @@ else global_motion_estimation(cpi); +#if CONFIG_IMPROVED_GLOBAL_MOTION + // Once we have determined the best motion model for each ref frame, + // choose the base parameters to minimize the total encoding cost + pick_base_gm_params(cpi); +#endif // CONFIG_IMPROVED_GLOBAL_MOTION + // Check if the current frame has any valid global motion model across its // reference frames if (cpi->sf.gm_sf.disable_gm_search_based_on_stats) {