Impose a maximum frame area of 2^30

Revert commit e645ba3c6e601fd86ff9d050d47e8c82c0ea3361, which imposed a
smaller maximum frame dimension of 32768 (= 2^15). Just impose a maximum
on the frame area (width * height). This allows one dimension (width or
height) to attain the 65536 (= 2^16) maximum allowed by the AV1
specification.

Bug: aomedia:3508
Change-Id: I2609df4b658ec161fba4bcb41634bed9ee4d6303

av1/av1_cx_iface.c

diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 719859d..588b0a4 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -636,20 +636,28 @@
 static aom_codec_err_t validate_config(aom_codec_alg_priv_t *ctx,
                                        const aom_codec_enc_cfg_t *cfg,
                                        const struct av1_extracfg *extra_cfg) {
-  // The AV1 specification allows a maximum frame dimension (width or height)
-  // of 65536 (= 2^16). To avoid integer overflows when multiplying width by
-  // height (or values derived from width and height) using the int type,
-  // impose a smaller maximum frame dimension of 32768 (= 2^15).
-  RANGE_CHECK(cfg, g_w, 1, 32768);
-  RANGE_CHECK(cfg, g_h, 1, 32768);
-  RANGE_CHECK_HI(cfg, g_forced_max_frame_width, 32768);
-  RANGE_CHECK_HI(cfg, g_forced_max_frame_height, 32768);
+  RANGE_CHECK(cfg, g_w, 1, 65536);                        // 16 bits available
+  RANGE_CHECK(cfg, g_h, 1, 65536);                        // 16 bits available
+  RANGE_CHECK_HI(cfg, g_forced_max_frame_width, 65536);   // 16 bits available
+  RANGE_CHECK_HI(cfg, g_forced_max_frame_height, 65536);  // 16 bits available
   if (cfg->g_forced_max_frame_width) {
     RANGE_CHECK_HI(cfg, g_w, cfg->g_forced_max_frame_width);
   }
   if (cfg->g_forced_max_frame_height) {
     RANGE_CHECK_HI(cfg, g_h, cfg->g_forced_max_frame_height);
   }
+  // To avoid integer overflows when multiplying width by height (or values
+  // derived from width and height) using the int type, impose a maximum frame
+  // area (width * height) of 2^30.
+  const unsigned int max_frame_width =
+      cfg->g_forced_max_frame_width ? cfg->g_forced_max_frame_width : cfg->g_w;
+  const unsigned int max_frame_height = cfg->g_forced_max_frame_height
+                                            ? cfg->g_forced_max_frame_height
+                                            : cfg->g_h;
+  const int64_t max_frame_area = (int64_t)max_frame_width * max_frame_height;
+  if (max_frame_area > (1 << 30)) {
+    ERROR("max_frame_area out of range [..2^30]");
+  }
   RANGE_CHECK(cfg, g_timebase.den, 1, 1000000000);
   RANGE_CHECK(cfg, g_timebase.num, 1, cfg->g_timebase.den);
   RANGE_CHECK_HI(cfg, g_profile, MAX_PROFILES - 1);

test/encode_api_test.cc

diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index eecc723..e6ef4c2 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -66,16 +66,20 @@
   EXPECT_EQ(AOM_CODEC_INVALID_PARAM,
             aom_codec_enc_config_default(iface, &cfg, 3));
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
-  cfg.g_w = (1 << 15) + 1;
+  cfg.g_w = 1 << 16;
+  cfg.g_h = (1 << 14) + 1;
   EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
-  cfg.g_h = (1 << 15) + 1;
+  cfg.g_w = (1 << 14) + 1;
+  cfg.g_h = 1 << 16;
   EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
-  cfg.g_forced_max_frame_width = (1 << 15) + 1;
+  cfg.g_forced_max_frame_width = 1 << 16;
+  cfg.g_forced_max_frame_height = (1 << 14) + 1;
   EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
-  cfg.g_forced_max_frame_height = (1 << 15) + 1;
+  cfg.g_forced_max_frame_width = (1 << 14) + 1;
+  cfg.g_forced_max_frame_height = 1 << 16;
   EXPECT_EQ(AOM_CODEC_INVALID_PARAM, aom_codec_enc_init(&enc, iface, &cfg, 0));
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, kUsage));
   EXPECT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));