Fix an integer overflow in target bits calculation

BUG=aomedia:503197490

Change-Id: I65eaba5e6d48da47ca0e67854ac0cff843e4ddea

av1/encoder/ratectrl.c

diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index dd2bd1b..f68c78f 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -505,10 +505,11 @@
   p_rc->rate_correction_factors[KF_STD] = 1.0;
   p_rc->bits_off_target = p_rc->starting_buffer_level;
 
-  p_rc->rolling_target_bits = AOMMAX(
-      1, (int)(oxcf->rc_cfg.target_bandwidth / oxcf->input_cfg.init_framerate));
-  p_rc->rolling_actual_bits = AOMMAX(
-      1, (int)(oxcf->rc_cfg.target_bandwidth / oxcf->input_cfg.init_framerate));
+  const double bits_per_frame =
+      oxcf->rc_cfg.target_bandwidth / oxcf->input_cfg.init_framerate;
+  p_rc->rolling_target_bits =
+      AOMMAX(1, bits_per_frame > INT_MAX ? INT_MAX : (int)bits_per_frame);
+  p_rc->rolling_actual_bits = p_rc->rolling_target_bits;
 }
 
 void av1_rc_init(const AV1EncoderConfig *oxcf, RATE_CONTROL *rc) {

test/encode_api_test.cc

diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 259fb34..239d29a 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -2408,4 +2408,22 @@
   ASSERT_EQ(aom_codec_destroy(&codec), AOM_CODEC_OK);
 }
 
+TEST(EncodeAPI, Buganizer503197490) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+            AOM_CODEC_OK);
+
+  cfg.g_w = 640;
+  cfg.g_h = 360;
+  cfg.g_timebase.num = 570;
+  cfg.g_timebase.den = 30;
+  cfg.rc_end_usage = AOM_CBR;
+  cfg.rc_target_bitrate = 1728882;  // 1728882000 bits/s
+
+  aom_codec_ctx_t codec;
+  ASSERT_EQ(aom_codec_enc_init(&codec, iface, &cfg, 0), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_destroy(&codec), AOM_CODEC_OK);
+}
+
 }  // namespace