commit 49587a77f11f13ba186a3640cdae50f1aefec213
author Yue Chen <yuec@google.com> Wed Sep 28 17:09:47 2016 -0700
committer Debargha Mukherjee <debargha@google.com> Thu Sep 29 09:20:52 2016 -0700
tree 6b0320d273c99aa6367999d078b034917ce9f844
parent 6c6ddac3a494b0ed7325dc85416013a7373963f7

Fix unit test failure for RECT_TX + VAR_TX

Disable rect_tx because we only support 4x4 Walsh-Hadamard transform
in lossless mode.

Fixes failure in ./test_libaom --gtest_filter=*Large*ScreencastQ0/1
Configuration: --enable-experimental --enable-var-tx --enable-rect-tx
 --enable-ref-mv --enable-ext_intra --enable-ext_tx --enable-debug
 --disable-optimizations

Change-Id: Ib6b3494c7dcf7182f1cab9b138388d054851a23d

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 94eb089..266f8fe 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -516,8 +516,10 @@
   return LUT[bsize];
 }
 
-static INLINE int is_rect_tx_allowed(const MB_MODE_INFO *mbmi) {
-  return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type);
+static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
+                                     const MB_MODE_INFO *mbmi) {
+  return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) &&
+         !xd->lossless[mbmi->segment_id];
 }
 
 static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }