commit 323d535d6a45e874d5d642dd6442e61e3f13b75e
author Sarah Parker <sarahparker@google.com> Thu Aug 17 14:58:03 2017 -0700
committer Fred BARBIER <frederic.barbier@allegrodvt.com> Wed Aug 23 15:18:55 2017 +0000
tree c9dbf0059bdd2f0621a7dc4742f9b7eb621ffeca
parent 1cc50bc2ffd1bfff4a2b68a5f8d63c55fe6e0a21

Prevent illegal signaling of gm types above AFFINE

BUG=aomedia:693

Change-Id: I09b34abc41ee6f85619f5a05f668e06491e542f0

av1/encoder/rd.c

diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 419a47f..8a2a63c 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -569,8 +569,17 @@
 #if CONFIG_GLOBAL_MOTION
   if (cpi->oxcf.pass != 1) {
     for (int i = 0; i < TRANS_TYPES; ++i)
+#if GLOBAL_TRANS_TYPES > 4
       cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                             << AV1_PROB_COST_SHIFT;
+#else
+      // IDENTITY: 1 bit
+      // TRANSLATION: 3 bits
+      // ROTZOOM: 2 bits
+      // AFFINE: 3 bits
+      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
+                            << AV1_PROB_COST_SHIFT;
+#endif  // GLOBAL_TRANS_TYPES > 4
   }
 #endif  // CONFIG_GLOBAL_MOTION
 }