Change/refactor compound mode handling for sub8x8

Turn off compound modes as long as one of the dimensions
is less than 8.

Imapct on AWCY (0.05% increase in BDRATE)
https://arewecompressedyet.com/?job=debargha-nocdef-sub8c8nc-0907%402017-09-07T20%3A28%3A38.251Z&job=debargha-nocdef-0907%402017-09-07T14%3A42%3A17.170Z

Change-Id: I4a70890c04149246a50e60990dede21cb8052fad

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 35d9f81..68e59dc 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -76,6 +76,15 @@
   FRAME_TYPES,
 } FRAME_TYPE;
 
+static INLINE int is_comp_ref_allowed(BLOCK_SIZE bsize) {
+  (void)bsize;
+#if SUB8X8_COMP_REF
+  return 1;
+#else
+  return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
+#endif  // SUB8X8_COMP_REF
+}
+
 static INLINE int is_inter_mode(PREDICTION_MODE mode) {
 #if CONFIG_EXT_INTER
   return mode >= NEARESTMV && mode <= NEW_NEWMV;

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index ad34ee7..6706abd 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1335,9 +1335,7 @@
 static REFERENCE_MODE read_block_reference_mode(AV1_COMMON *cm,
                                                 const MACROBLOCKD *xd,
                                                 aom_reader *r) {
-#if !SUB8X8_COMP_REF
-  if (xd->mi[0]->mbmi.sb_type == BLOCK_4X4) return SINGLE_REFERENCE;
-#endif
+  if (!is_comp_ref_allowed(xd->mi[0]->mbmi.sb_type)) return SINGLE_REFERENCE;
   if (cm->reference_mode == REFERENCE_MODE_SELECT) {
     const int ctx = av1_get_reference_mode_context(cm, xd);
 #if CONFIG_NEW_MULTISYMBOL

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 222f1f4..5ae0d39 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -1029,14 +1029,12 @@
     // does the feature use compound prediction or not
     // (if not specified at the frame/segment level)
     if (cm->reference_mode == REFERENCE_MODE_SELECT) {
-#if !SUB8X8_COMP_REF
-      if (mbmi->sb_type != BLOCK_4X4)
-#endif
+      if (is_comp_ref_allowed(mbmi->sb_type))
 #if CONFIG_NEW_MULTISYMBOL
         aom_write_symbol(w, is_compound, av1_get_reference_mode_cdf(cm, xd), 2);
 #else
-      aom_write(w, is_compound, av1_get_reference_mode_prob(cm, xd));
-#endif
+        aom_write(w, is_compound, av1_get_reference_mode_prob(cm, xd));
+#endif  // CONFIG_NEW_MULTISYMBOL
     } else {
       assert((!is_compound) == (cm->reference_mode == SINGLE_REFERENCE));
     }

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 23b0b0e..ba32645 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1672,23 +1672,14 @@
           else
             // This flag is also updated for 4x4 blocks
             rdc->single_ref_used_flag = 1;
-#if !SUB8X8_COMP_REF
-          if (mbmi->sb_type != BLOCK_4X4) {
+          if (is_comp_ref_allowed(mbmi->sb_type)) {
             counts->comp_inter[av1_get_reference_mode_context(cm, xd)]
                               [has_second_ref(mbmi)]++;
 #if CONFIG_NEW_MULTISYMBOL
             update_cdf(av1_get_reference_mode_cdf(cm, xd), has_second_ref(mbmi),
                        2);
-#endif
+#endif  // CONFIG_NEW_MULTISYMBOL
           }
-#else
-          counts->comp_inter[av1_get_reference_mode_context(cm, xd)]
-                            [has_second_ref(mbmi)]++;
-#if CONFIG_NEW_MULTISYMBOL
-          update_cdf(av1_get_reference_mode_cdf(cm, xd), has_second_ref(mbmi),
-                     2);
-#endif
-#endif
         }
 
         if (has_second_ref(mbmi)) {

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index a1b06a2..278a0e0 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -11022,9 +11022,8 @@
     } else {
       int_mv backup_ref_mv[2];
 
-#if !SUB8X8_COMP_REF
-      if (bsize == BLOCK_4X4 && mbmi->ref_frame[1] > INTRA_FRAME) continue;
-#endif  // !SUB8X8_COMP_REF
+      if (!is_comp_ref_allowed(bsize) && mbmi->ref_frame[1] > INTRA_FRAME)
+        continue;
 
       backup_ref_mv[0] = mbmi_ext->ref_mvs[ref_frame][0];
       if (comp_pred) backup_ref_mv[1] = mbmi_ext->ref_mvs[second_ref_frame][0];
@@ -11453,12 +11452,8 @@
 
       if (this_rd == INT64_MAX) continue;
 
-#if SUB8X8_COMP_REF
-      compmode_cost = av1_cost_bit(comp_mode_p, comp_pred);
-#else
-      if (mbmi->sb_type != BLOCK_4X4)
+      if (is_comp_ref_allowed(mbmi->sb_type))
         compmode_cost = av1_cost_bit(comp_mode_p, comp_pred);
-#endif  // SUB8X8_COMP_REF
 
       if (cm->reference_mode == REFERENCE_MODE_SELECT) rate2 += compmode_cost;
     }