commit 8ee5ab9f1348bf66912b8c46f1e76a501ee08f7b
author Debargha Mukherjee <debargha@google.com> Mon Aug 29 11:04:17 2016 -0700
committer Debargha Mukherjee <debargha@google.com> Mon Aug 29 11:16:17 2016 -0700
tree 58c34429bcc96fe6dd282337eb8c771cf5dd078b
parent 04ed7ad57b59abc0c1f9481f64f26996fd01a2f9

Fix for supertx with rect-tx

Change-Id: I0cc3523a8992f889f8dd203449ceb55f2a422324

vp10/common/blockd.h

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 98fbcf6..3c0fdef 100644
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -418,8 +418,9 @@
 
 #if CONFIG_SUPERTX
 static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
-  return (int)mbmi->tx_size > VPXMIN(b_width_log2_lookup[mbmi->sb_type],
-                                     b_height_log2_lookup[mbmi->sb_type]);
+  return (int)txsize_sqr_map[mbmi->tx_size] >
+         VPXMIN(b_width_log2_lookup[mbmi->sb_type],
+                b_height_log2_lookup[mbmi->sb_type]);
 }
 #endif  // CONFIG_SUPERTX
 
@@ -733,8 +734,8 @@
                                      const struct macroblockd_plane *pd) {
 #if CONFIG_SUPERTX
   if (supertx_enabled(mbmi))
-    return uvsupertx_size_lookup[mbmi->tx_size][pd->subsampling_x]
-                                [pd->subsampling_y];
+    return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
+                                [pd->subsampling_x][pd->subsampling_y];
 #endif  // CONFIG_SUPERTX
   return get_uv_tx_size_impl(mbmi->tx_size, mbmi->sb_type, pd->subsampling_x,
                              pd->subsampling_y);

vp10/encoder/rd.c

diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 0075d21..77c939f 100644
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -593,7 +593,7 @@
       for (i = 0; i < num_4x4_h; i += 8)
         t_left[i] = !!*(const uint64_t *)&left[i];
       break;
-#if CONFIG_EXT_TX
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
     case TX_4X8:
       memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
       for (i = 0; i < num_4x4_h; i += 2)
@@ -628,7 +628,7 @@
       for (i = 0; i < num_4x4_h; i += 4)
         t_left[i] = !!*(const uint32_t *)&left[i];
       break;
-#endif  // CONFIG_EXT_TX
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     default: assert(0 && "Invalid transform size."); break;
   }
 }