commit f70134f729dfe7429fb1e4e3fd68a381bdb47f7a
author Jingning Han <jingning@google.com> Thu Feb 11 09:41:06 2016 -0800
committer Jingning Han <jingning@google.com> Thu Feb 11 11:07:09 2016 -0800
tree 328f1c112933e8a163dcd558dfcdc48767e15163
parent 5c772f38faafb05459457117132a5fd9ad73fc72

Align rate-distortion cost metric for chroma compoments

This commit aligns the rate-distortion metrics for both luma and
chroma components in super transform rate-distortion optimization.
It improves the coding gains due to var-tx and supertx experiments
by 0.2% for high resolution test sets.

Change-Id: Ib89d99e29cb5ee27b1f867e301954d4164d8b364

vp10/encoder/encodeframe.c

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 41a697f..898b18f 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -5187,16 +5187,36 @@
   dist_uv = 0;
   sse_uv = 0;
   for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
+#if CONFIG_VAR_TX
+    ENTROPY_CONTEXT ctxa[16], ctxl[16];
+    const struct macroblockd_plane *const pd = &xd->plane[plane];
+    int coeff_ctx = 1;
+
+    this_rate = 0;
+    this_dist = 0;
+    pnsse = 0;
+    pnskip = 1;
+
+    tx_size = max_txsize_lookup[bsize];
+    tx_size = get_uv_tx_size_impl(tx_size, bsize,
+                                  cm->subsampling_x, cm->subsampling_y);
+    vp10_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl);
+    coeff_ctx = combine_entropy_contexts(ctxa[0], ctxl[0]);
+
+    vp10_subtract_plane(x, bsize, plane);
+    vp10_tx_block_rd_b(cpi, x, tx_size,
+                       0, 0, plane, 0,
+                       get_plane_block_size(bsize, pd), coeff_ctx,
+                       &this_rate, &this_dist, &pnsse, &pnskip);
+#else
     tx_size = max_txsize_lookup[bsize];
     tx_size = get_uv_tx_size_impl(tx_size, bsize,
                                   cm->subsampling_x, cm->subsampling_y);
     vp10_subtract_plane(x, bsize, plane);
-    vp10_txfm_rd_in_plane_supertx(x,
-#if CONFIG_VAR_TX
-                                  cpi,
-#endif
-                                  &this_rate, &this_dist, &pnskip, &pnsse,
+    vp10_txfm_rd_in_plane_supertx(x, &this_rate, &this_dist, &pnskip, &pnsse,
                                   INT64_MAX, plane, bsize, tx_size, 0);
+#endif  // CONFIG_VAR_TX
+
     rate_uv += this_rate;
     dist_uv += this_dist;
     sse_uv += pnsse;