Merge "vp9 com/dec: avoid reading unavailable above/left"
diff --git a/test/dct16x16_test.cc b/test/dct16x16_test.cc
index 3d61d40..b990bf8 100644
--- a/test/dct16x16_test.cc
+++ b/test/dct16x16_test.cc
@@ -395,8 +395,7 @@
       for (int j = 0; j < kNumCoeffs; ++j)
         coeff[j] = round(out_r[j]);
 
-      const int pitch = 32;
-      REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, pitch));
+      REGISTER_STATE_CHECK(RunInvTxfm(coeff, dst, 16));
 
       for (int j = 0; j < kNumCoeffs; ++j) {
         const uint32_t diff = dst[j] - src[j];
@@ -421,7 +420,7 @@
     fwd_txfm_ = GET_PARAM(0);
     inv_txfm_ = GET_PARAM(1);
     tx_type_  = GET_PARAM(2);
-    pitch_    = 32;
+    pitch_    = 16;
     fwd_txfm_ref = fdct16x16_ref;
   }
   virtual void TearDown() { libvpx_test::ClearSystemState(); }
@@ -431,7 +430,7 @@
     fwd_txfm_(in, out, stride);
   }
   void RunInvTxfm(int16_t *out, uint8_t *dst, int stride) {
-    inv_txfm_(out, dst, stride >> 1);
+    inv_txfm_(out, dst, stride);
   }
 
   fdct_t fwd_txfm_;
diff --git a/vp9/common/vp9_blockd.h b/vp9/common/vp9_blockd.h
index 0a441bd..36e7e83 100644
--- a/vp9/common/vp9_blockd.h
+++ b/vp9/common/vp9_blockd.h
@@ -234,31 +234,6 @@
   int q_index;
 } MACROBLOCKD;
 
-static INLINE uint8_t *get_sb_index(MACROBLOCKD *xd, BLOCK_SIZE subsize) {
-  switch (subsize) {
-    case BLOCK_64X64:
-    case BLOCK_64X32:
-    case BLOCK_32X64:
-    case BLOCK_32X32:
-      return &xd->sb_index;
-    case BLOCK_32X16:
-    case BLOCK_16X32:
-    case BLOCK_16X16:
-      return &xd->mb_index;
-    case BLOCK_16X8:
-    case BLOCK_8X16:
-    case BLOCK_8X8:
-      return &xd->b_index;
-    case BLOCK_8X4:
-    case BLOCK_4X8:
-    case BLOCK_4X4:
-      return &xd->ab_index;
-    default:
-      assert(0);
-      return NULL;
-  }
-}
-
 static INLINE void update_partition_context(MACROBLOCKD *xd, BLOCK_SIZE sb_type,
                                             BLOCK_SIZE sb_size) {
   const int bsl = b_width_log2(sb_size), bs = (1 << bsl) / 2;
diff --git a/vp9/common/vp9_common_data.c b/vp9/common/vp9_common_data.c
index dc41efd..f858900 100644
--- a/vp9/common/vp9_common_data.c
+++ b/vp9/common/vp9_common_data.c
@@ -115,6 +115,16 @@
   TX_16X16, TX_16X16, TX_16X16, TX_32X32
 };
 
+const TX_SIZE tx_mode_to_biggest_tx_size[TX_MODES] = {
+  TX_4X4,  // ONLY_4X4
+  TX_8X8,  // ALLOW_8X8
+  TX_16X16,  // ALLOW_16X16
+  TX_32X32,  // ALLOW_32X32
+  TX_32X32,  // TX_MODE_SELECT
+};
+
+
+
 const BLOCK_SIZE ss_size_lookup[BLOCK_SIZES][2][2] = {
 //  ss_x == 0    ss_x == 0        ss_x == 1      ss_x == 1
 //  ss_y == 0    ss_y == 1        ss_y == 0      ss_y == 1
@@ -133,3 +143,4 @@
   {{BLOCK_64X64, BLOCK_64X32},   {BLOCK_32X64,   BLOCK_32X32}},
 };
 
+
diff --git a/vp9/common/vp9_common_data.h b/vp9/common/vp9_common_data.h
index 2945cd2..c1f6405 100644
--- a/vp9/common/vp9_common_data.h
+++ b/vp9/common/vp9_common_data.h
@@ -27,6 +27,7 @@
 extern const BLOCK_SIZE subsize_lookup[PARTITION_TYPES][BLOCK_SIZES];
 extern const TX_SIZE max_txsize_lookup[BLOCK_SIZES];
 extern const TX_SIZE max_uv_txsize_lookup[BLOCK_SIZES];
+extern const TX_SIZE tx_mode_to_biggest_tx_size[TX_MODES];
 extern const BLOCK_SIZE ss_size_lookup[BLOCK_SIZES][2][2];
 
 #endif  // VP9_COMMON_VP9_COMMON_DATA_H
diff --git a/vp9/common/vp9_rtcd_defs.sh b/vp9/common/vp9_rtcd_defs.sh
index af96bb3..6fa9e22 100644
--- a/vp9/common/vp9_rtcd_defs.sh
+++ b/vp9/common/vp9_rtcd_defs.sh
@@ -707,7 +707,7 @@
 prototype void vp9_short_fdct32x32_rd "int16_t *InputData, int16_t *OutputData, int stride"
 specialize vp9_short_fdct32x32_rd sse2
 
-prototype void vp9_short_fdct16x16 "int16_t *InputData, int16_t *OutputData, int pitch"
+prototype void vp9_short_fdct16x16 "int16_t *InputData, int16_t *OutputData, int stride"
 specialize vp9_short_fdct16x16 sse2
 
 prototype void vp9_short_walsh4x4 "int16_t *InputData, int16_t *OutputData, int pitch"
diff --git a/vp9/decoder/vp9_decodemv.c b/vp9/decoder/vp9_decodemv.c
index 8a8f7d5..fcd5f9e 100644
--- a/vp9/decoder/vp9_decodemv.c
+++ b/vp9/decoder/vp9_decodemv.c
@@ -78,17 +78,13 @@
                             vp9_reader *r) {
   VP9_COMMON *const cm = &pbi->common;
   MACROBLOCKD *const xd = &pbi->mb;
-
-  if (allow_select && tx_mode == TX_MODE_SELECT && bsize >= BLOCK_8X8)
+  if (allow_select && tx_mode == TX_MODE_SELECT && bsize >= BLOCK_8X8) {
     return read_selected_tx_size(cm, xd, bsize, r);
-  else if (tx_mode >= ALLOW_32X32 && bsize >= BLOCK_32X32)
-    return TX_32X32;
-  else if (tx_mode >= ALLOW_16X16 && bsize >= BLOCK_16X16)
-    return TX_16X16;
-  else if (tx_mode >= ALLOW_8X8 && bsize >= BLOCK_8X8)
-    return TX_8X8;
-  else
-    return TX_4X4;
+  } else {
+    const TX_SIZE max_tx_size_block = max_txsize_lookup[bsize];
+    const TX_SIZE max_tx_size_txmode = tx_mode_to_biggest_tx_size[tx_mode];
+    return MIN(max_tx_size_block, max_tx_size_txmode);
+  }
 }
 
 static void set_segment_id(VP9_COMMON *cm, BLOCK_SIZE bsize,
@@ -426,11 +422,12 @@
   mbmi->uv_mode = read_intra_mode_uv(cm, r, mbmi->mode);
 }
 
-static INLINE void assign_mv(VP9_COMMON *cm, MB_PREDICTION_MODE mode,
+static INLINE int assign_mv(VP9_COMMON *cm, MB_PREDICTION_MODE mode,
                              int_mv mv[2], int_mv best_mv[2],
                              int_mv nearest_mv[2], int_mv near_mv[2],
                              int is_compound, int allow_hp, vp9_reader *r) {
   int i;
+  int ret = 1;
 
   switch (mode) {
     case NEWMV:
@@ -439,6 +436,10 @@
        if (is_compound)
          read_mv(r, &mv[1].as_mv, &best_mv[1].as_mv,
                  &cm->fc.nmvc, &cm->counts.mv, allow_hp);
+       for (i = 0; i < 1 + is_compound; ++i) {
+         ret = ret && mv[i].as_mv.row < MV_UPP && mv[i].as_mv.row > MV_LOW;
+         ret = ret && mv[i].as_mv.col < MV_UPP && mv[i].as_mv.col > MV_LOW;
+       }
        break;
     case NEARESTMV:
       mv[0].as_int = nearest_mv[0].as_int;
@@ -456,13 +457,9 @@
         mv[1].as_int = 0;
       break;
     default:
-      assert(!"Invalid inter mode value.");
+      return 0;
   }
-
-  for (i = 0; i < 1 + is_compound; ++i) {
-    assert(mv[i].as_mv.row < MV_UPP && mv[i].as_mv.row > MV_LOW);
-    assert(mv[i].as_mv.col < MV_UPP && mv[i].as_mv.col > MV_LOW);
-  }
+  return ret;
 }
 
 static int read_is_inter_block(VP9D_COMP *pbi, int segment_id, vp9_reader *r) {
@@ -558,8 +555,12 @@
                                           mi_row, mi_col);
         }
 
-        assign_mv(cm, b_mode, block, best, nearest, nearmv,
-                  is_compound, allow_hp, r);
+        if (!assign_mv(cm, b_mode, block, best, nearest, nearmv,
+                       is_compound, allow_hp, r)) {
+          xd->corrupted |= 1;
+          break;
+        };
+
 
         mi->bmi[j].as_mv[0].as_int = block[0].as_int;
         if (is_compound)
@@ -577,8 +578,9 @@
     mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int;
     mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int;
   } else {
-    assign_mv(cm, mbmi->mode, mbmi->mv, best, nearest, nearmv,
-              is_compound, allow_hp, r);
+    xd->corrupted |= !assign_mv(cm, mbmi->mode, mbmi->mv,
+                                best, nearest, nearmv,
+                                is_compound, allow_hp, r);
   }
 }
 
diff --git a/vp9/decoder/vp9_decodframe.c b/vp9/decoder/vp9_decodframe.c
index b914de7..ec310f4 100644
--- a/vp9/decoder/vp9_decodframe.c
+++ b/vp9/decoder/vp9_decodframe.c
@@ -224,14 +224,14 @@
 
 static void decode_modes_b(VP9D_COMP *pbi, int tile_col,
                            int mi_row, int mi_col,
-                           vp9_reader *r, BLOCK_SIZE bsize) {
+                           vp9_reader *r, BLOCK_SIZE bsize, int index) {
   MACROBLOCKD *const xd = &pbi->mb;
   const int less8x8 = bsize < BLOCK_8X8;
   MB_MODE_INFO *mbmi;
   int eobtotal;
 
   if (less8x8)
-    if (xd->ab_index > 0)
+    if (index > 0)
       return;
 
   set_offsets(pbi, bsize, tile_col, mi_row, mi_col);
@@ -271,9 +271,10 @@
   xd->corrupted |= vp9_reader_has_error(r);
 }
 
+
 static void decode_modes_sb(VP9D_COMP *pbi, int tile_col,
                             int mi_row, int mi_col,
-                            vp9_reader* r, BLOCK_SIZE bsize) {
+                            vp9_reader* r, BLOCK_SIZE bsize, int index) {
   VP9_COMMON *const cm = &pbi->common;
   MACROBLOCKD *const xd = &pbi->mb;
   const int hbs = num_8x8_blocks_wide_lookup[bsize] / 2;
@@ -284,7 +285,7 @@
     return;
 
   if (bsize < BLOCK_8X8) {
-    if (xd->ab_index != 0)
+    if (index > 0)
       return;
   } else {
     int pl;
@@ -306,31 +307,27 @@
   }
 
   subsize = get_subsize(bsize, partition);
-  *get_sb_index(xd, subsize) = 0;
 
   switch (partition) {
     case PARTITION_NONE:
-      decode_modes_b(pbi, tile_col, mi_row, mi_col, r, subsize);
+      decode_modes_b(pbi, tile_col, mi_row, mi_col, r, subsize, 0);
       break;
     case PARTITION_HORZ:
-      decode_modes_b(pbi, tile_col, mi_row, mi_col, r, subsize);
-      *get_sb_index(xd, subsize) = 1;
+      decode_modes_b(pbi, tile_col, mi_row, mi_col, r, subsize, 0);
       if (mi_row + hbs < cm->mi_rows)
-        decode_modes_b(pbi, tile_col, mi_row + hbs, mi_col, r, subsize);
+        decode_modes_b(pbi, tile_col, mi_row + hbs, mi_col, r, subsize, 1);
       break;
     case PARTITION_VERT:
-      decode_modes_b(pbi, tile_col, mi_row, mi_col, r, subsize);
-      *get_sb_index(xd, subsize) = 1;
+      decode_modes_b(pbi, tile_col, mi_row, mi_col, r, subsize, 0);
       if (mi_col + hbs < cm->mi_cols)
-        decode_modes_b(pbi, tile_col, mi_row, mi_col + hbs, r, subsize);
+        decode_modes_b(pbi, tile_col, mi_row, mi_col + hbs, r, subsize, 1);
       break;
     case PARTITION_SPLIT: {
       int n;
       for (n = 0; n < 4; n++) {
         const int j = n >> 1, i = n & 1;
-        *get_sb_index(xd, subsize) = n;
         decode_modes_sb(pbi, tile_col, mi_row + j * hbs, mi_col + i * hbs,
-                        r, subsize);
+                        r, subsize, n);
       }
     } break;
     default:
@@ -611,7 +608,7 @@
     vp9_zero(cm->left_seg_context);
     for (mi_col = cm->cur_tile_mi_col_start; mi_col < cm->cur_tile_mi_col_end;
          mi_col += MI_BLOCK_SIZE)
-      decode_modes_sb(pbi, tile_col, mi_row, mi_col, r, BLOCK_64X64);
+      decode_modes_sb(pbi, tile_col, mi_row, mi_col, r, BLOCK_64X64, 0);
 
     if (pbi->do_loopfilter_inline) {
       // delay the loopfilter by 1 macroblock row.
@@ -802,6 +799,7 @@
                                        struct vp9_read_bit_buffer *rb) {
   VP9_COMMON *const cm = &pbi->common;
   MACROBLOCKD *const xd = &pbi->mb;
+  size_t sz;
   int i;
 
   cm->last_frame_type = cm->frame_type;
@@ -909,8 +907,9 @@
   setup_segmentation(&cm->seg, rb);
 
   setup_tile_info(cm, rb);
+  sz = vp9_rb_read_literal(rb, 16);
 
-  return vp9_rb_read_literal(rb, 16);
+  return sz > 0 ? sz : -1;
 }
 
 static int read_compressed_header(VP9D_COMP *pbi, const uint8_t *data,
diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index bfac5a7..ed795f0 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -565,13 +565,13 @@
 
 static void write_modes_b(VP9_COMP *cpi, MODE_INFO **mi_8x8, vp9_writer *bc,
                           TOKENEXTRA **tok, TOKENEXTRA *tok_end,
-                          int mi_row, int mi_col) {
+                          int mi_row, int mi_col, int index) {
   VP9_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->mb.e_mbd;
   MODE_INFO *m = mi_8x8[0];
 
   if (m->mbmi.sb_type < BLOCK_8X8)
-    if (xd->ab_index > 0)
+    if (index > 0)
       return;
 
   xd->mi_8x8 = mi_8x8;
@@ -597,7 +597,8 @@
 
 static void write_modes_sb(VP9_COMP *cpi, MODE_INFO **mi_8x8, vp9_writer *bc,
                            TOKENEXTRA **tok, TOKENEXTRA *tok_end,
-                           int mi_row, int mi_col, BLOCK_SIZE bsize) {
+                           int mi_row, int mi_col, BLOCK_SIZE bsize,
+                           int index) {
   VP9_COMMON *const cm = &cpi->common;
   MACROBLOCKD *xd = &cpi->mb.e_mbd;
   const int mis = cm->mode_info_stride;
@@ -613,11 +614,10 @@
 
   partition = partition_lookup[bsl][m->mbmi.sb_type];
 
-  if (bsize < BLOCK_8X8)
-    if (xd->ab_index > 0)
+  if (bsize < BLOCK_8X8) {
+    if (index > 0)
       return;
-
-  if (bsize >= BLOCK_8X8) {
+  } else {
     int pl;
     const int idx = check_bsize_coverage(bs, cm->mi_rows, cm->mi_cols,
                                          mi_row, mi_col);
@@ -634,31 +634,28 @@
   }
 
   subsize = get_subsize(bsize, partition);
-  *(get_sb_index(xd, subsize)) = 0;
 
   switch (partition) {
     case PARTITION_NONE:
-      write_modes_b(cpi, mi_8x8, bc, tok, tok_end, mi_row, mi_col);
+      write_modes_b(cpi, mi_8x8, bc, tok, tok_end, mi_row, mi_col, 0);
       break;
     case PARTITION_HORZ:
-      write_modes_b(cpi, mi_8x8, bc, tok, tok_end, mi_row, mi_col);
-      *(get_sb_index(xd, subsize)) = 1;
+      write_modes_b(cpi, mi_8x8, bc, tok, tok_end, mi_row, mi_col, 0);
       if ((mi_row + bs) < cm->mi_rows)
         write_modes_b(cpi, mi_8x8 + bs * mis, bc, tok, tok_end, mi_row + bs,
-                      mi_col);
+                      mi_col, 1);
       break;
     case PARTITION_VERT:
-      write_modes_b(cpi, mi_8x8, bc, tok, tok_end, mi_row, mi_col);
-      *(get_sb_index(xd, subsize)) = 1;
+      write_modes_b(cpi, mi_8x8, bc, tok, tok_end, mi_row, mi_col, 0);
       if ((mi_col + bs) < cm->mi_cols)
-        write_modes_b(cpi, mi_8x8 + bs, bc, tok, tok_end, mi_row, mi_col + bs);
+        write_modes_b(cpi, mi_8x8 + bs, bc, tok, tok_end, mi_row, mi_col + bs,
+                      1);
       break;
     case PARTITION_SPLIT:
       for (n = 0; n < 4; n++) {
-        int j = n >> 1, i = n & 0x01;
-        *(get_sb_index(xd, subsize)) = n;
+        const int j = n >> 1, i = n & 1;
         write_modes_sb(cpi, mi_8x8 + j * bs * mis + i * bs, bc, tok, tok_end,
-                       mi_row + j * bs, mi_col + i * bs, subsize);
+                       mi_row + j * bs, mi_col + i * bs, subsize, n);
       }
       break;
     default:
@@ -690,7 +687,7 @@
     for (mi_col = cm->cur_tile_mi_col_start; mi_col < cm->cur_tile_mi_col_end;
          mi_col += MI_BLOCK_SIZE, m_8x8 += MI_BLOCK_SIZE) {
       write_modes_sb(cpi, m_8x8, bc, tok, tok_end, mi_row, mi_col,
-                     BLOCK_64X64);
+                     BLOCK_64X64, 0);
     }
   }
 }
diff --git a/vp9/encoder/vp9_dct.c b/vp9/encoder/vp9_dct.c
index 00a2903..23c652d 100644
--- a/vp9/encoder/vp9_dct.c
+++ b/vp9/encoder/vp9_dct.c
@@ -302,14 +302,13 @@
   }
 }
 
-void vp9_short_fdct16x16_c(int16_t *input, int16_t *output, int pitch) {
+void vp9_short_fdct16x16_c(int16_t *input, int16_t *output, int stride) {
   // The 2D transform is done with two passes which are actually pretty
   // similar. In the first one, we transform the columns and transpose
   // the results. In the second one, we transform the rows. To achieve that,
   // as the first pass results are transposed, we tranpose the columns (that
   // is the transposed rows) and transpose the results (so that it goes back
   // in normal/row positions).
-  const int stride = pitch >> 1;
   int pass;
   // We need an intermediate buffer between passes.
   int16_t intermediate[256];
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 50ada52..6e8e1d1 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -30,7 +30,6 @@
 #include "vp9/common/vp9_reconinter.h"
 #include "vp9/common/vp9_seg_common.h"
 #include "vp9/common/vp9_tile_common.h"
-
 #include "vp9/encoder/vp9_encodeframe.h"
 #include "vp9/encoder/vp9_encodeintra.h"
 #include "vp9/encoder/vp9_encodemb.h"
@@ -46,19 +45,36 @@
 #define DBG_PRNT_SEGMAP 0
 
 
-static const TX_SIZE tx_mode_to_biggest_tx_size[TX_MODES] = {
-  TX_4X4,  // ONLY_4X4
-  TX_8X8,  // ONLY_8X8
-  TX_16X16,  // ONLY_16X16
-  TX_32X32,  // ONLY_32X32
-  TX_32X32,  // TX_MODE_SELECT
-};
-
 // #define ENC_DEBUG
 #ifdef ENC_DEBUG
 int enc_debug = 0;
 #endif
 
+static INLINE uint8_t *get_sb_index(MACROBLOCKD *xd, BLOCK_SIZE subsize) {
+  switch (subsize) {
+    case BLOCK_64X64:
+    case BLOCK_64X32:
+    case BLOCK_32X64:
+    case BLOCK_32X32:
+      return &xd->sb_index;
+    case BLOCK_32X16:
+    case BLOCK_16X32:
+    case BLOCK_16X16:
+      return &xd->mb_index;
+    case BLOCK_16X8:
+    case BLOCK_8X16:
+    case BLOCK_8X8:
+      return &xd->b_index;
+    case BLOCK_8X4:
+    case BLOCK_4X8:
+    case BLOCK_4X4:
+      return &xd->ab_index;
+    default:
+      assert(0);
+      return NULL;
+  }
+}
+
 static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t, int output_enabled,
                               int mi_row, int mi_col, BLOCK_SIZE bsize);
 
diff --git a/vp9/encoder/vp9_encodemb.c b/vp9/encoder/vp9_encodemb.c
index 2b5451b..13d8aa8 100644
--- a/vp9/encoder/vp9_encodemb.c
+++ b/vp9/encoder/vp9_encodemb.c
@@ -379,7 +379,7 @@
       xoff = 16 * (block & twmask);
       yoff = 16 * (block >> twl);
       src_diff = p->src_diff + 4 * bw * yoff + xoff;
-      vp9_short_fdct16x16(src_diff, coeff, bw * 8);
+      vp9_short_fdct16x16(src_diff, coeff, bw * 4);
       vp9_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round,
                      p->quant, p->quant_shift, qcoeff, dqcoeff,
                      pd->dequant, p->zbin_extra, eob, scan, iscan);
@@ -559,7 +559,7 @@
       if (tx_type != DCT_DCT)
         vp9_short_fht16x16(src_diff, coeff, bw * 4, tx_type);
       else
-        vp9_short_fdct16x16(src_diff, coeff, bw * 8);
+        vp9_short_fdct16x16(src_diff, coeff, bw * 4);
       vp9_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round,
                      p->quant, p->quant_shift, qcoeff, dqcoeff,
                      pd->dequant, p->zbin_extra, eob, scan, iscan);
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 7add494..30cdb3f 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -715,22 +715,12 @@
                                      BLOCK_SIZE bs) {
   const TX_SIZE max_tx_size = max_txsize_lookup[bs];
   VP9_COMMON *const cm = &cpi->common;
+  const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = &xd->mi_8x8[0]->mbmi;
-  if (max_tx_size == TX_32X32 &&
-      (cm->tx_mode == ALLOW_32X32 ||
-       cm->tx_mode == TX_MODE_SELECT)) {
-    mbmi->tx_size = TX_32X32;
-  } else if (max_tx_size >= TX_16X16 &&
-             (cm->tx_mode == ALLOW_16X16 ||
-              cm->tx_mode == ALLOW_32X32 ||
-              cm->tx_mode == TX_MODE_SELECT)) {
-    mbmi->tx_size = TX_16X16;
-  } else if (cm->tx_mode != ONLY_4X4) {
-    mbmi->tx_size = TX_8X8;
-  } else {
-    mbmi->tx_size = TX_4X4;
-  }
+
+  mbmi->tx_size = MIN(max_tx_size, largest_tx_size);
+
   txfm_rd_in_plane(x, &cpi->rdcost_stack, rate, distortion, skip,
                    &sse[mbmi->tx_size], ref_best_rd, 0, bs,
                    mbmi->tx_size);
diff --git a/vp9/encoder/x86/vp9_dct_sse2.c b/vp9/encoder/x86/vp9_dct_sse2.c
index 5e1e5ed..457883f 100644
--- a/vp9/encoder/x86/vp9_dct_sse2.c
+++ b/vp9/encoder/x86/vp9_dct_sse2.c
@@ -1056,14 +1056,13 @@
   write_buffer_8x8(output, in, 8);
 }
 
-void vp9_short_fdct16x16_sse2(int16_t *input, int16_t *output, int pitch) {
+void vp9_short_fdct16x16_sse2(int16_t *input, int16_t *output, int stride) {
   // The 2D transform is done with two passes which are actually pretty
   // similar. In the first one, we transform the columns and transpose
   // the results. In the second one, we transform the rows. To achieve that,
   // as the first pass results are transposed, we tranpose the columns (that
   // is the transposed rows) and transpose the results (so that it goes back
   // in normal/row positions).
-  const int stride = pitch >> 1;
   int pass;
   // We need an intermediate buffer between passes.
   DECLARE_ALIGNED_ARRAY(16, int16_t, intermediate, 256);