commit 26b8a99eea1c0282ed4b48d6ebce98402cce3bf4
author Monty Montgomery <cmontgomery@mozilla.com> Fri Nov 10 22:45:23 2017 -0500
committer Christopher Montgomery <cmontgomery@mozilla.com> Sat Nov 11 21:02:28 2017 +0000
tree ba05efc43f05426da42f3fbcd8dab04b074b1ddd
parent 0492b99afa36f8a18aa4ffc0e9df03454c7f8caf

Add is_hbd field to TxfmParam

In preparation for Daala unified LBD/HBD TX, add (and use) is_hbd
field in TxfmPama structure.  This field indicates whether or not
pixel data is using 8 or 16 bit reference buffers (currently ambiguous
in the case of 8 bit input).

Change-Id: I28bca792a48ffa00e208617adb072b08ff816e3c

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index fcc56f9..1c9f3f4 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -557,17 +557,17 @@
 #endif  // CONFIG_MRC_TX
 
   txfm_param.bd = xd->bd;
-  const int is_hbd = get_bitdepth_data_path_index(xd);
+  txfm_param.is_hbd = get_bitdepth_data_path_index(xd);
 
 #if CONFIG_TXMG
   av1_highbd_fwd_txfm(src_diff, coeff, diff_stride, &txfm_param);
 #else   // CONFIG_TXMG
-  fwd_txfm_func[is_hbd](src_diff, coeff, diff_stride, &txfm_param);
+  fwd_txfm_func[txfm_param.is_hbd](src_diff, coeff, diff_stride, &txfm_param);
 #endif  // CONFIG_TXMG
 
   if (xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) {
     if (LIKELY(!x->skip_block)) {
-      quant_func_list[xform_quant_idx][is_hbd](
+      quant_func_list[xform_quant_idx][txfm_param.is_hbd](
           coeff, tx2d_size, p, qcoeff, dqcoeff, eob, scan_order, &qparam);
     } else {
       av1_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob);
@@ -724,6 +724,7 @@
 
   if (p->eobs[block] > 0) {
     txfm_param.bd = xd->bd;
+    txfm_param.is_hbd = get_bitdepth_data_path_index(xd);
     txfm_param.tx_type = DCT_DCT;
     txfm_param.tx_size = tx_size;
     txfm_param.eob = p->eobs[block];
@@ -732,7 +733,7 @@
         txfm_param.tx_size, plane_bsize, is_inter_block(&xd->mi[0]->mbmi),
         cm->reduced_tx_set_used);
 #if CONFIG_HIGHBITDEPTH
-    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
+    if (txfm_param.is_hbd) {
       av1_highbd_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride, &txfm_param);
       return;
     }

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 1ed95cb..d3090c4 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4700,6 +4700,7 @@
   param.tx_type = DCT_DCT;
   param.tx_size = max_txsize_rect_lookup[bsize];
   param.bd = xd->bd;
+  param.is_hbd = get_bitdepth_data_path_index(xd);
   param.lossless = 0;
   const struct macroblockd_plane *const pd = &xd->plane[0];
   const BLOCK_SIZE plane_bsize =
@@ -4713,7 +4714,7 @@
 #if CONFIG_TXMG
   av1_highbd_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);
 #else   // CONFIG_TXMG
-  if (get_bitdepth_data_path_index(xd))
+  if (param.is_hbd)
     av1_highbd_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);
   else
     av1_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);