commit 680b9b17be62512819b64d7a94212b92509d43cf
author Sarah Parker <sarahparker@google.com> Wed Aug 16 18:55:34 2017 -0700
committer Sebastien Alaiwan <sebastien.alaiwan@allegrodvt.com> Sat Aug 19 07:33:52 2017 +0000
tree 66f29e207d425ed50412e542c4f1a0ef3890c819
parent be984ebb0d7c69b79b9da1369770aa41418ceffd

Prevent bitstream from signaling illegal compound types

Currently nothing forbids wedge from being signalled when
the block is > 32X32, even though there is no corresponding wedge
mask for that block size.

BUG=aomedia:640
BUG=aomedia:636

Change-Id: I538be0229a12b5ef01b2e5a950c9f16ef9a5c51e

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 21d3c9b..4773be3 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -2110,10 +2110,16 @@
         is_any_masked_compound_used(bsize)) {
 #if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
       if (cm->allow_masked_compound) {
-        aom_write_symbol(w, mbmi->interinter_compound_type,
-                         ec_ctx->compound_type_cdf[bsize], COMPOUND_TYPES);
+#if CONFIG_WEDGE && CONFIG_COMPOUND_SEGMENT
+        if (!is_interinter_compound_used(COMPOUND_WEDGE, bsize))
+          aom_write_bit(w, mbmi->interinter_compound_type == COMPOUND_AVERAGE);
+        else
+#endif  // CONFIG_WEDGE && CONFIG_COMPOUND_SEGMENT
+          aom_write_symbol(w, mbmi->interinter_compound_type,
+                           ec_ctx->compound_type_cdf[bsize], COMPOUND_TYPES);
 #if CONFIG_WEDGE
-        if (mbmi->interinter_compound_type == COMPOUND_WEDGE) {
+        if (is_interinter_compound_used(COMPOUND_WEDGE, bsize) &&
+            mbmi->interinter_compound_type == COMPOUND_WEDGE) {
           aom_write_literal(w, mbmi->wedge_index, get_wedge_bits_lookup(bsize));
           aom_write_bit(w, mbmi->wedge_sign);
         }