commit 92245c8738f3a8c40fa2078a563c8bd947c70c36
author Yaowu Xu <yaowu.google.com> Sat Jan 27 16:12:51 2018 -0800
committer Yaowu Xu <yaowu.google.com> Sat Jan 27 19:16:35 2018 -0800
tree cc51fca599a3ab2f1706d8095e8d91053c1a5b9c
parent a7d6f44297786468b65ce3ea81574a891e232bfa

Avoid out of bound array access

Change-Id: I4066561b769cf2bd4af515c9d351f609c08e3076

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 30d069c..c0bcf17 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4898,7 +4898,11 @@
       if (this_rd_stats.rate != INT_MAX) {
         const int ext_tx_set = get_ext_tx_set(
             mbmi->min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
-        const TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
+        TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
+#if CONFIG_TX64X64
+        // For TX_64X64, use TX_32X32 to lookup cost
+        if (square_tx_size == TX_64X64) square_tx_size = TX_32X32;
+#endif  // CONFIG_TX64X64
         this_rd_stats.rate +=
             x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
         this_rd_stats.rate -=