Improve skip_txfm thresholds in the non-rd mode selection

Modified the thresholds of deciding whether or not to skip
the transforms in model_rd_for_sb_y(). Used zbin[] instead
of dequant[] to be more precise. Also, modified the checking
coditions.

Rtc set borg test results (at speed 6) showed:
average PSNR gain: 0.138%, overall PSNR gain: 0.158%,
and SSIM gain: 0.177%.

The data rate test was modified slightly as suggested by
Marco.

Change-Id: Ieaf633ab77f4838cb3c45cf69065b29d55f8ae6c

test/datarate_test.cc

diff --git a/test/datarate_test.cc b/test/datarate_test.cc
index e529347..94efeae 100644
--- a/test/datarate_test.cc
+++ b/test/datarate_test.cc
@@ -540,7 +540,7 @@
         << " The first dropped frame for drop_thresh " << i
         << " > first dropped frame for drop_thresh "
         << i - kDropFrameThreshTestStep;
-    ASSERT_GE(num_drops_, last_num_drops)
+    ASSERT_GE(num_drops_, last_num_drops * 0.90)
         << " The number of dropped frames for drop_thresh " << i
         << " < number of dropped frames for drop_thresh "
         << i - kDropFrameThreshTestStep;

vp9/encoder/vp9_pickmode.c

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 46a3547..71cea0e 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -216,6 +216,8 @@
   int64_t dist;
   struct macroblock_plane *const p = &x->plane[0];
   struct macroblockd_plane *const pd = &xd->plane[0];
+  const int64_t dc_thr = p->quant_thred[0] >> 6;
+  const int64_t ac_thr = p->quant_thred[1] >> 6;
   const uint32_t dc_quant = pd->dequant[0];
   const uint32_t ac_quant = pd->dequant[1];
   unsigned int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
@@ -223,12 +225,14 @@
   *var_y = var;
   *sse_y = sse;
 
-  if (sse < dc_quant * dc_quant >> 6)
-    x->skip_txfm[0] = 1;
-  else if (var < ac_quant * ac_quant >> 6)
+  x->skip_txfm[0] = 0;
+  // Check if all ac coefficients can be quantized to zero.
+  if (var < ac_thr || var == 0) {
     x->skip_txfm[0] = 2;
-  else
-    x->skip_txfm[0] = 0;
+    // Check if dc coefficient can be quantized to zero.
+    if (sse - var < dc_thr || sse == var)
+      x->skip_txfm[0] = 1;
+  }
 
   if (cpi->common.tx_mode == TX_MODE_SELECT) {
     if (sse > (var << 2))