Fix high target data rate overflow.
These change fixes issues that can occur if the user specifies a very
high target data rate or rate per frame.
Fixes some issue with overflow of int variables used to hold bitrate
values (rate per second, rate per frame etc).
This patch also imposes a new maximum for the passed in target bitrate.
This value is passed in in kbits (so multiplied by 1000
internally).
Change-Id: Ia3b60bf1110d85cdce161492561bdda1cff61c63
(cherry picked from commit 699cefee092b8e2f35f580f44c5be709c569722f)
diff --git a/aom/aom_encoder.h b/aom/aom_encoder.h
index 9bdadd6..15cf21b 100644
--- a/aom/aom_encoder.h
+++ b/aom/aom_encoder.h
@@ -637,6 +637,7 @@
/*!\brief Target data rate
*
* Target bitrate to use for this stream, in kilobits per second.
+ * Max allowed value is 2000000
*/
unsigned int rc_target_bitrate;
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 652c6ea..633b957 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -673,6 +673,7 @@
RANGE_CHECK(cfg, g_timebase.num, 1, cfg->g_timebase.den);
RANGE_CHECK_HI(cfg, g_profile, MAX_PROFILES - 1);
+ RANGE_CHECK_HI(cfg, rc_target_bitrate, 2000000);
RANGE_CHECK_HI(cfg, rc_max_quantizer, 63);
RANGE_CHECK_HI(cfg, rc_min_quantizer, cfg->rc_max_quantizer);
RANGE_CHECK_BOOL(extra_cfg, lossless);
@@ -3347,7 +3348,7 @@
if (ppi->cpi->oxcf.pass != 1) {
ppi->total_time_compress_data += cpi->time_compress_data;
ppi->total_recode_hits += cpi->frame_recode_hits;
- ppi->total_bytes += cpi->bytes;
+ ppi->total_bytes += (uint64_t)cpi->bytes;
for (int i = 0; i < MAX_MODES; i++) {
ppi->total_mode_chosen_counts[i] += cpi->mode_chosen_counts[i];
}
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 4de5d42..3188a36 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -2793,7 +2793,7 @@
double total_blockiness;
double worst_blockiness;
- int total_bytes;
+ uint64_t total_bytes;
double summed_quality;
double summed_weights;
double summed_quality_hbd;
diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index bd8620c..8618212 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -268,9 +268,12 @@
// Similar to find_qindex_by_rate() function in ratectrl.c, but includes
// calculation of a correction_factor.
-static int find_qindex_by_rate_with_correction(
- int desired_bits_per_mb, aom_bit_depth_t bit_depth, double error_per_mb,
- double group_weight_factor, int best_qindex, int worst_qindex) {
+static int find_qindex_by_rate_with_correction(uint64_t desired_bits_per_mb,
+ aom_bit_depth_t bit_depth,
+ double error_per_mb,
+ double group_weight_factor,
+ int best_qindex,
+ int worst_qindex) {
assert(best_qindex <= worst_qindex);
int low = best_qindex;
int high = worst_qindex;
@@ -279,7 +282,8 @@
const int mid = (low + high) >> 1;
const double q_factor = calc_correction_factor(error_per_mb, mid);
const double q = av1_convert_qindex_to_q(mid, bit_depth);
- const int mid_bits_per_mb = (int)((q_factor * group_weight_factor) / q);
+ const uint64_t mid_bits_per_mb =
+ (uint64_t)((q_factor * group_weight_factor) / q);
if (mid_bits_per_mb > desired_bits_per_mb) {
low = mid + 1;
@@ -328,8 +332,8 @@
: cpi->common.mi_params.MBs;
const int active_mbs = AOMMAX(1, num_mbs - (int)(num_mbs * inactive_zone));
const double av_err_per_mb = av_frame_err / (1.0 - inactive_zone);
- const int target_norm_bits_per_mb =
- (int)((uint64_t)av_target_bandwidth << BPER_MB_NORMBITS) / active_mbs;
+ const uint64_t target_norm_bits_per_mb =
+ ((uint64_t)av_target_bandwidth << BPER_MB_NORMBITS) / active_mbs;
int rate_err_tol = AOMMIN(rc_cfg->under_shoot_pct, rc_cfg->over_shoot_pct);
// Update bpm correction factor based on previous GOP rate error.
diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 7639484..62a4149 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2550,7 +2550,7 @@
void av1_rc_update_framerate(AV1_COMP *cpi, int width, int height) {
const AV1EncoderConfig *const oxcf = &cpi->oxcf;
RATE_CONTROL *const rc = &cpi->rc;
- int vbr_max_bits;
+ int64_t vbr_max_bits;
const int MBs = av1_get_MBs(width, height);
rc->avg_frame_bandwidth =
@@ -2569,10 +2569,11 @@
// be acheived because of a user specificed max q (e.g. when the user
// specifies lossless encode.
vbr_max_bits =
- (int)(((int64_t)rc->avg_frame_bandwidth * oxcf->rc_cfg.vbrmax_section) /
- 100);
+ ((int64_t)rc->avg_frame_bandwidth * oxcf->rc_cfg.vbrmax_section) / 100;
+ vbr_max_bits = (vbr_max_bits < INT_MAX) ? vbr_max_bits : INT_MAX;
+
rc->max_frame_bandwidth =
- AOMMAX(AOMMAX((MBs * MAX_MB_RATE), MAXRATE_1080P), vbr_max_bits);
+ AOMMAX(AOMMAX((MBs * MAX_MB_RATE), MAXRATE_1080P), (int)vbr_max_bits);
av1_rc_set_gf_interval_range(cpi, rc);
}
