commit 680b9b17be62512819b64d7a94212b92509d43cf
author Sarah Parker <sarahparker@google.com> Wed Aug 16 18:55:34 2017 -0700
committer Sebastien Alaiwan <sebastien.alaiwan@allegrodvt.com> Sat Aug 19 07:33:52 2017 +0000
tree 66f29e207d425ed50412e542c4f1a0ef3890c819
parent be984ebb0d7c69b79b9da1369770aa41418ceffd

Prevent bitstream from signaling illegal compound types

Currently nothing forbids wedge from being signalled when
the block is > 32X32, even though there is no corresponding wedge
mask for that block size.

BUG=aomedia:640
BUG=aomedia:636

Change-Id: I538be0229a12b5ef01b2e5a950c9f16ef9a5c51e

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 4aec878..2f413f7 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9253,11 +9253,19 @@
       tmp_rate_mv = rate_mv;
       best_rd_cur = INT64_MAX;
       mbmi->interinter_compound_type = cur_type;
+      int masked_type_cost = 0;
+      if (masked_compound_used) {
+#if CONFIG_WEDGE && CONFIG_COMPOUND_SEGMENT
+        if (!is_interinter_compound_used(COMPOUND_WEDGE, bsize))
+          masked_type_cost += av1_cost_literal(1);
+        else
+#endif  // CONFIG_WEDGE && CONFIG_COMPOUND_SEGMENT
+          masked_type_cost +=
+              compound_type_cost[mbmi->interinter_compound_type];
+      }
       rs2 = av1_cost_literal(get_interinter_compound_type_bits(
                 bsize, mbmi->interinter_compound_type)) +
-            (masked_compound_used
-                 ? compound_type_cost[mbmi->interinter_compound_type]
-                 : 0);
+            masked_type_cost;
 
       switch (cur_type) {
         case COMPOUND_AVERAGE: