commit 26b8a99eea1c0282ed4b48d6ebce98402cce3bf4
author Monty Montgomery <cmontgomery@mozilla.com> Fri Nov 10 22:45:23 2017 -0500
committer Christopher Montgomery <cmontgomery@mozilla.com> Sat Nov 11 21:02:28 2017 +0000
tree ba05efc43f05426da42f3fbcd8dab04b074b1ddd
parent 0492b99afa36f8a18aa4ffc0e9df03454c7f8caf

Add is_hbd field to TxfmParam

In preparation for Daala unified LBD/HBD TX, add (and use) is_hbd
field in TxfmPama structure.  This field indicates whether or not
pixel data is using 8 or 16 bit reference buffers (currently ambiguous
in the case of 8 bit input).

Change-Id: I28bca792a48ffa00e208617adb072b08ff816e3c

aom_dsp/txfm_common.h

diff --git a/aom_dsp/txfm_common.h b/aom_dsp/txfm_common.h
index 253aefc..4129a7b 100644
--- a/aom_dsp/txfm_common.h
+++ b/aom_dsp/txfm_common.h
@@ -28,6 +28,9 @@
   TX_SIZE tx_size;
   int lossless;
   int bd;
+  // are the pixel buffers octets or shorts?  This should collapse to
+  // bd==8 implies !is_hbd, but that's not certain right now.
+  int is_hbd;
   TxSetType tx_set_type;
 #if CONFIG_MRC_TX
   int is_inter;

av1/common/idct.c

diff --git a/av1/common/idct.c b/av1/common/idct.c
index 51c35cd..7a96237 100644
--- a/av1/common/idct.c
+++ b/av1/common/idct.c
@@ -2419,6 +2419,7 @@
   txfm_param->eob = eob;
   txfm_param->lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
   txfm_param->bd = xd->bd;
+  txfm_param->is_hbd = get_bitdepth_data_path_index(xd);
   const struct macroblockd_plane *const pd = &xd->plane[plane];
   const BLOCK_SIZE plane_bsize =
       get_plane_block_size(xd->mi[0]->mbmi.sb_type, pd);
@@ -2467,9 +2468,7 @@
   txfm_param.stride = stride;
 #endif  // CONFIG_MRC_TX
   assert(av1_ext_tx_used[txfm_param.tx_set_type][txfm_param.tx_type]);
-
-  const int is_hbd = get_bitdepth_data_path_index(xd);
-  inv_txfm_func[is_hbd](dqcoeff, dst, stride, &txfm_param);
+  inv_txfm_func[txfm_param.is_hbd](dqcoeff, dst, stride, &txfm_param);
 }
 
 void av1_inverse_transform_block_facade(MACROBLOCKD *xd, int plane, int block,

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index fcc56f9..1c9f3f4 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -557,17 +557,17 @@
 #endif  // CONFIG_MRC_TX
 
   txfm_param.bd = xd->bd;
-  const int is_hbd = get_bitdepth_data_path_index(xd);
+  txfm_param.is_hbd = get_bitdepth_data_path_index(xd);
 
 #if CONFIG_TXMG
   av1_highbd_fwd_txfm(src_diff, coeff, diff_stride, &txfm_param);
 #else   // CONFIG_TXMG
-  fwd_txfm_func[is_hbd](src_diff, coeff, diff_stride, &txfm_param);
+  fwd_txfm_func[txfm_param.is_hbd](src_diff, coeff, diff_stride, &txfm_param);
 #endif  // CONFIG_TXMG
 
   if (xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) {
     if (LIKELY(!x->skip_block)) {
-      quant_func_list[xform_quant_idx][is_hbd](
+      quant_func_list[xform_quant_idx][txfm_param.is_hbd](
           coeff, tx2d_size, p, qcoeff, dqcoeff, eob, scan_order, &qparam);
     } else {
       av1_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob);
@@ -724,6 +724,7 @@
 
   if (p->eobs[block] > 0) {
     txfm_param.bd = xd->bd;
+    txfm_param.is_hbd = get_bitdepth_data_path_index(xd);
     txfm_param.tx_type = DCT_DCT;
     txfm_param.tx_size = tx_size;
     txfm_param.eob = p->eobs[block];
@@ -732,7 +733,7 @@
         txfm_param.tx_size, plane_bsize, is_inter_block(&xd->mi[0]->mbmi),
         cm->reduced_tx_set_used);
 #if CONFIG_HIGHBITDEPTH
-    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
+    if (txfm_param.is_hbd) {
       av1_highbd_inv_txfm_add_4x4(dqcoeff, dst, pd->dst.stride, &txfm_param);
       return;
     }

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 1ed95cb7..d3090c4 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4700,6 +4700,7 @@
   param.tx_type = DCT_DCT;
   param.tx_size = max_txsize_rect_lookup[bsize];
   param.bd = xd->bd;
+  param.is_hbd = get_bitdepth_data_path_index(xd);
   param.lossless = 0;
   const struct macroblockd_plane *const pd = &xd->plane[0];
   const BLOCK_SIZE plane_bsize =
@@ -4713,7 +4714,7 @@
 #if CONFIG_TXMG
   av1_highbd_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);
 #else   // CONFIG_TXMG
-  if (get_bitdepth_data_path_index(xd))
+  if (param.is_hbd)
     av1_highbd_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);
   else
     av1_fwd_txfm(p->src_diff, DCT_coefs, bw, &param);