Merge "Add new_quant quantization in rdopt for 4x4 blocks and intra" into nextgenv2
diff --git a/test/obmc_variance_test.cc b/test/obmc_variance_test.cc
index 40295f2..b7de9ed 100644
--- a/test/obmc_variance_test.cc
+++ b/test/obmc_variance_test.cc
@@ -122,7 +122,7 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #if CONFIG_VP9_HIGHBITDEPTH
-class ObmcVarianceHBDTest : public FunctionEquivalenceTest<ObmcVarF> {}
+class ObmcVarianceHBDTest : public FunctionEquivalenceTest<ObmcVarF> {};
 
 TEST_P(ObmcVarianceHBDTest, RandomValues) {
   DECLARE_ALIGNED(32, uint16_t, pre[MAX_SB_SQUARE]);
@@ -133,19 +133,20 @@
     const int pre_stride = this->rng_(MAX_SB_SIZE + 1);
 
     for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
-      pre[i] = this->rng_(1 << this->bit_depth);
-      wsrc[i] = this->rng_(1 << this->bit_depth) *
+      pre[i] = this->rng_(1 << params_.bit_depth);
+      wsrc[i] = this->rng_(1 << params_.bit_depth) *
                 this->rng_(kMaskMax * kMaskMax + 1);
       mask[i] = this->rng_(kMaskMax * kMaskMax + 1);
     }
 
     unsigned int ref_sse, tst_sse;
-    const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(pre), pre_stride,
-                                           wsrc, mask, &ref_sse);
+    const unsigned int ref_res = params_.ref_func(CONVERT_TO_BYTEPTR(pre),
+                                                  pre_stride,
+                                                  wsrc, mask, &ref_sse);
     unsigned int tst_res;
     ASM_REGISTER_STATE_CHECK(
-        tst_res = tst_func_(CONVERT_TO_BYTEPTR(pre),
-                            pre_stride, wsrc, mask, &tst_sse));
+        tst_res = params_.tst_func(CONVERT_TO_BYTEPTR(pre),
+                                   pre_stride, wsrc, mask, &tst_sse));
 
     ASSERT_EQ(ref_res, tst_res);
     ASSERT_EQ(ref_sse, tst_sse);
@@ -161,18 +162,19 @@
     const int pre_stride = iter;
 
     for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
-      pre[i] = (1 << this->bit_depth) - 1;
-      wsrc[i] = ((1 << this->bit_depth) - 1) * kMaskMax * kMaskMax;
+      pre[i] = (1 << params_.bit_depth) - 1;
+      wsrc[i] = ((1 << params_.bit_depth) - 1) * kMaskMax * kMaskMax;
       mask[i] = kMaskMax * kMaskMax;
     }
 
     unsigned int ref_sse, tst_sse;
-    const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(pre), pre_stride,
-                                           wsrc, mask, &ref_sse);
+    const unsigned int ref_res = params_.ref_func(CONVERT_TO_BYTEPTR(pre),
+                                                  pre_stride,
+                                                  wsrc, mask, &ref_sse);
     unsigned int tst_res;
     ASM_REGISTER_STATE_CHECK(
-        tst_res = tst_func_(CONVERT_TO_BYTEPTR(pre), pre_stride,
-                            wsrc, mask, &tst_sse));
+        tst_res = params_.tst_func(CONVERT_TO_BYTEPTR(pre), pre_stride,
+                                   wsrc, mask, &tst_sse));
 
     ASSERT_EQ(ref_res, tst_res);
     ASSERT_EQ(ref_sse, tst_sse);
diff --git a/vp10/encoder/block.h b/vp10/encoder/block.h
index 4c9f8a5..c39aa21 100644
--- a/vp10/encoder/block.h
+++ b/vp10/encoder/block.h
@@ -140,9 +140,6 @@
   int mv_row_min;
   int mv_row_max;
 
-  // Notes transform blocks where no coefficents are coded.
-  // Set during mode selection. Read during block encoding.
-  uint8_t zcoeff_blk[TX_SIZES][MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
 #if CONFIG_VAR_TX
   uint8_t blk_skip[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
 #if CONFIG_REF_MV
diff --git a/vp10/encoder/context_tree.c b/vp10/encoder/context_tree.c
index 41155c9..8555cdb 100644
--- a/vp10/encoder/context_tree.c
+++ b/vp10/encoder/context_tree.c
@@ -34,8 +34,6 @@
   ctx->partition = partition;
 #endif
 
-  CHECK_MEM_ERROR(cm, ctx->zcoeff_blk,
-                  vpx_calloc(num_blk, sizeof(uint8_t)));
   for (i = 0; i < MAX_MB_PLANE; ++i) {
 #if CONFIG_VAR_TX
     CHECK_MEM_ERROR(cm, ctx->blk_skip[i],
@@ -68,8 +66,6 @@
 
 static void free_mode_context(PICK_MODE_CONTEXT *ctx) {
   int i, k;
-  vpx_free(ctx->zcoeff_blk);
-  ctx->zcoeff_blk = 0;
   for (i = 0; i < MAX_MB_PLANE; ++i) {
 #if CONFIG_VAR_TX
     vpx_free(ctx->blk_skip[i]);
diff --git a/vp10/encoder/context_tree.h b/vp10/encoder/context_tree.h
index 7b49354..a788788 100644
--- a/vp10/encoder/context_tree.h
+++ b/vp10/encoder/context_tree.h
@@ -26,7 +26,6 @@
 typedef struct {
   MODE_INFO mic;
   MB_MODE_INFO_EXT mbmi_ext;
-  uint8_t *zcoeff_blk;
   uint8_t *color_index_map[2];
 #if CONFIG_VAR_TX
   uint8_t *blk_skip[MAX_MB_PLANE];
diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index ff1ee6b..287fb17 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -1204,8 +1204,6 @@
     memcpy(x->blk_skip[i], ctx->blk_skip[i],
            sizeof(uint8_t) * ctx->num_4x4_blk);
 #endif
-  memcpy(x->zcoeff_blk[mbmi->tx_size], ctx->zcoeff_blk,
-         sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
 
   if (!output_enabled)
     return;
@@ -1362,8 +1360,6 @@
     memcpy(x->blk_skip[i], ctx->blk_skip[i],
            sizeof(uint8_t) * ctx->num_4x4_blk);
 #endif  // CONFIG_VAR_TX
-  memcpy(x->zcoeff_blk[mbmi->tx_size], ctx->zcoeff_blk,
-         sizeof(uint8_t) * ctx->num_4x4_blk);
 
 #if CONFIG_VAR_TX
   {
@@ -1569,8 +1565,6 @@
     memcpy(ctx->blk_skip[i], x->blk_skip[i],
            sizeof(uint8_t) * ctx->num_4x4_blk);
 #endif  // CONFIG_VAR_TX
-  memcpy(ctx->zcoeff_blk, x->zcoeff_blk[supertx_size],
-         sizeof(uint8_t) * ctx->num_4x4_blk);
   ctx->mic.mbmi.tx_size = supertx_size;
   ctx->skip = x->skip;
   ctx->mic.mbmi.tx_type = best_tx;
@@ -6083,7 +6077,6 @@
 #endif  // CONFIG_EXT_TX
   int tmp_rate_tx = 0, skip_tx = 0;
   int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX;
-  uint8_t tmp_zcoeff_blk = 0;
 
   set_skip_context(xd, mi_row, mi_col);
   set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
@@ -6230,10 +6223,8 @@
       tmp_rate_tx = *tmp_rate;
       tmp_dist_tx = *tmp_dist;
       skip_tx = x->skip;
-      tmp_zcoeff_blk = x->zcoeff_blk[tx_size][0];
     }
   }
-  x->zcoeff_blk[tx_size][0] = tmp_zcoeff_blk;
   *tmp_rate = tmp_rate_tx;
   *tmp_dist = tmp_dist_tx;
   x->skip = skip_tx;
diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c
index 3810be5..34428a7 100644
--- a/vp10/encoder/encodemb.c
+++ b/vp10/encoder/encodemb.c
@@ -982,16 +982,6 @@
   ctx = combine_entropy_contexts(*a, *l);
 #endif
 
-  // TODO(jingning): per transformed block zero forcing only enabled for
-  // luma component. will integrate chroma components as well.
-  // Turn this back on when the rate-distortion loop is synchronized with
-  // the recursive transform block coding.
-//  if (x->zcoeff_blk[tx_size][block] && plane == 0) {
-//    p->eobs[block] = 0;
-//    *a = *l = 0;
-//    return;
-//  }
-
 #if CONFIG_VAR_TX
   // Assert not magic number (uninitialised).
   assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 6c407d8..52dcf35 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1242,9 +1242,6 @@
 
   // TODO(jingning): temporarily enabled only for luma component
   rd = VPXMIN(rd1, rd2);
-  if (plane == 0)
-    x->zcoeff_blk[tx_size][block] = !x->plane[plane].eobs[block] ||
-        (rd1 > rd2 && !xd->lossless[mbmi->segment_id]);
 
   args->this_rate += rate;
   args->this_dist += dist;
@@ -1444,7 +1441,6 @@
   int64_t best_rd = INT64_MAX, last_rd = INT64_MAX;
   const TX_SIZE max_tx_size = max_txsize_lookup[bs];
   TX_SIZE best_tx = max_tx_size;
-  uint8_t zcoeff_blk[TX_SIZES][MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
   const int tx_select = cm->tx_mode == TX_MODE_SELECT;
   const int is_inter = is_inter_block(mbmi);
 #if CONFIG_EXT_TX
@@ -1521,17 +1517,10 @@
       *rate       = r;
       *skip       = s;
       *psse       = sse;
-      memcpy(zcoeff_blk[mbmi->tx_size], x->zcoeff_blk[mbmi->tx_size],
-             sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-             MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
     }
   }
   mbmi->tx_size = best_tx;
 
-  memcpy(x->zcoeff_blk[mbmi->tx_size], zcoeff_blk[mbmi->tx_size],
-         sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-         MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
-
   return best_rd;
 }
 
@@ -1721,7 +1710,6 @@
                                         BLOCK_SIZE bs) {
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
-  uint8_t zcoeff_blk[TX_SIZES][MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
   int r, s;
   int64_t d, sse;
   int64_t rd = INT64_MAX;
@@ -1756,9 +1744,6 @@
       *psse       = sse;
       best_tx_type = tx_type;
       best_tx = mbmi->tx_size;
-      memcpy(zcoeff_blk[mbmi->tx_size], x->zcoeff_blk[mbmi->tx_size],
-             sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-             MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
     }
   }
 
@@ -1769,10 +1754,6 @@
   if (mbmi->tx_size >= TX_32X32)
     assert(mbmi->tx_type == DCT_DCT);
 #endif
-
-  memcpy(x->zcoeff_blk[mbmi->tx_size], zcoeff_blk[mbmi->tx_size],
-         sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-         MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
 }
 
 static void super_block_yrd(VP10_COMP *cpi, MACROBLOCK *x, int *rate,
@@ -2455,9 +2436,7 @@
                                          TX_TYPE *best_tx_type,
                                          INTRA_FILTER *best_filter,
                                          BLOCK_SIZE bsize, int rate_overhead,
-                                         int64_t *best_rd,
-                                         uint8_t zcoeff_blk[][MAX_MIB_SIZE *
-                                                            MAX_MIB_SIZE * 4]) {
+                                         int64_t *best_rd) {
   int this_rate, this_rate_tokenonly, s;
   int64_t this_distortion, this_rd;
   MB_MODE_INFO *mbmi = &x->e_mbd.mi[0]->mbmi;
@@ -2479,9 +2458,6 @@
     *rate_tokenonly     = this_rate_tokenonly;
     *distortion         = this_distortion;
     *skippable          = s;
-    memcpy(zcoeff_blk[mbmi->tx_size], x->zcoeff_blk[mbmi->tx_size],
-           sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-           MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
   }
 }
 
@@ -2501,7 +2477,6 @@
   int64_t this_distortion, this_rd;
   TX_SIZE best_tx_size = mic->mbmi.tx_size;
   TX_TYPE best_tx_type = mbmi->tx_type;
-  uint8_t zcoeff_blk[TX_SIZES][MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
 
   if (ANGLE_FAST_SEARCH) {
     int deltas_level1[3] = {0, -2, 2};
@@ -2548,9 +2523,6 @@
           *rate_tokenonly     = this_rate_tokenonly;
           *distortion         = this_distortion;
           *skippable          = s;
-          memcpy(zcoeff_blk[mbmi->tx_size], x->zcoeff_blk[mbmi->tx_size],
-                 sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-                 MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
         }
       }
     }
@@ -2572,7 +2544,7 @@
                                        rate_overhead +
                                        cpi->intra_filter_cost
                                        [intra_filter_ctx][filter],
-                                       &best_rd, zcoeff_blk);
+                                       &best_rd);
         }
       }
     }
@@ -2594,7 +2566,7 @@
                                      rate_overhead +
                                      cpi->intra_filter_cost
                                      [intra_filter_ctx][filter],
-                                     &best_rd, zcoeff_blk);
+                                     &best_rd);
       }
     }
   }
@@ -2611,8 +2583,7 @@
                                      &best_angle_delta, &best_tx_size,
                                      &best_tx_type, &best_filter, bsize,
                                      rate_overhead + cpi->intra_filter_cost
-                                     [intra_filter_ctx][filter], &best_rd,
-                                     zcoeff_blk);
+                                     [intra_filter_ctx][filter], &best_rd);
       }
     }
   }
@@ -2621,11 +2592,6 @@
   mbmi->angle_delta[0] = best_angle_delta;
   mic->mbmi.intra_filter = best_filter;
   mbmi->tx_type = best_tx_type;
-  if (*rate_tokenonly < INT_MAX)
-    memcpy(x->zcoeff_blk[mbmi->tx_size], zcoeff_blk[mbmi->tx_size],
-           sizeof(zcoeff_blk[mbmi->tx_size][0]) *
-           MAX_MIB_SIZE * MAX_MIB_SIZE * 4);
-
   return best_rd;
 }
 
@@ -8400,8 +8366,6 @@
     *best_mbmode = *mbmi;
     *best_skip2 = 0;
     *best_mode_skippable = skippable;
-    memcpy(ctx->zcoeff_blk, x->zcoeff_blk[mbmi->tx_size],
-           sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
   }
 }
 #endif  // CONFIG_EXT_INTRA
@@ -9508,9 +9472,6 @@
         for (i = 0; i < MAX_MB_PLANE; ++i)
           memcpy(ctx->blk_skip[i], x->blk_skip[i],
                  sizeof(uint8_t) * ctx->num_4x4_blk);
-#else
-        memcpy(ctx->zcoeff_blk, x->zcoeff_blk[mbmi->tx_size],
-               sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
 #endif
 
         // TODO(debargha): enhance this test with a better distortion prediction
@@ -9772,8 +9733,6 @@
       best_mbmode = *mbmi;
       best_skip2 = 0;
       best_mode_skippable = skippable;
-      memcpy(ctx->zcoeff_blk, x->zcoeff_blk[mbmi->tx_size],
-             sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
     }
   }
   PALETTE_EXIT:
@@ -10236,7 +10195,6 @@
   best_rd = best_rd_so_far;
   best_yrd = best_rd_so_far;
 #endif  // CONFIG_SUPERTX
-  memset(x->zcoeff_blk[TX_4X4], 0, 4);
   vp10_zero(best_mbmode);
 
 #if CONFIG_EXT_INTRA
@@ -10664,7 +10622,6 @@
               tmp_best_mbmode = *mbmi;
               for (i = 0; i < 4; i++) {
                 tmp_best_bmodes[i] = xd->mi[0]->bmi[i];
-                x->zcoeff_blk[TX_4X4][i] = !x->plane[0].eobs[i];
               }
               pred_exists = 1;
             }
@@ -10861,9 +10818,6 @@
 #if CONFIG_VAR_TX
         for (i = 0; i < MAX_MB_PLANE; ++i)
           memset(ctx->blk_skip[i], 0, sizeof(uint8_t) * ctx->num_4x4_blk);
-#else
-        memcpy(ctx->zcoeff_blk, x->zcoeff_blk[TX_4X4],
-               sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
 #endif
 
         for (i = 0; i < 4; i++)