Fit dct's stage range into 32-bit when bitdepth is 12
Change-Id: I807e60c6dcacc50c087adcbdb1df022f8541efc5
diff --git a/vp10/common/vp10_fwd_txfm2d_cfg.h b/vp10/common/vp10_fwd_txfm2d_cfg.h
index 3c0a906..2f60baa 100644
--- a/vp10/common/vp10_fwd_txfm2d_cfg.h
+++ b/vp10/common/vp10_fwd_txfm2d_cfg.h
@@ -12,16 +12,17 @@
#define VP10_FWD_TXFM2D_CFG_H_
#include "vp10/common/vp10_fwd_txfm1d.h"
// ---------------- config fwd_dct_dct_4 ----------------
-static const int8_t fwd_shift_dct_dct_4[3] = {4, 0, -2};
+static const int8_t fwd_shift_dct_dct_4[3] = {2, 0, 0};
static const int8_t fwd_stage_range_col_dct_dct_4[4] = {15, 16, 17, 17};
static const int8_t fwd_stage_range_row_dct_dct_4[4] = {17, 18, 18, 18};
-static const int8_t fwd_cos_bit_col_dct_dct_4[4] = {15, 15, 15, 15};
-static const int8_t fwd_cos_bit_row_dct_dct_4[4] = {15, 14, 14, 14};
+static const int8_t fwd_cos_bit_col_dct_dct_4[4] = {13, 13, 13, 13};
+static const int8_t fwd_cos_bit_row_dct_dct_4[4] = {13, 13, 13, 13};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_4 = {
- 4, // .txfm_size
- 4, // .stage_num_col
- 4, // .stage_num_row
+ 4, // .txfm_size
+ 4, // .stage_num_col
+ 4, // .stage_num_row
+ // 0, // .log_scale
fwd_shift_dct_dct_4, // .shift
fwd_stage_range_col_dct_dct_4, // .stage_range_col
fwd_stage_range_row_dct_dct_4, // .stage_range_row
@@ -31,16 +32,17 @@
TXFM_TYPE_DCT4}; // .txfm_type_row
// ---------------- config fwd_dct_dct_8 ----------------
-static const int8_t fwd_shift_dct_dct_8[3] = {5, -3, -1};
+static const int8_t fwd_shift_dct_dct_8[3] = {3, -1, -1};
static const int8_t fwd_stage_range_col_dct_dct_8[6] = {16, 17, 18, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_8[6] = {16, 17, 18, 18, 18, 18};
-static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {15, 15, 14, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {15, 15, 14, 14, 14, 14};
+static const int8_t fwd_stage_range_row_dct_dct_8[6] = {18, 19, 20, 20, 20, 20};
+static const int8_t fwd_cos_bit_col_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
+static const int8_t fwd_cos_bit_row_dct_dct_8[6] = {13, 13, 12, 12, 12, 12};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_8 = {
- 8, // .txfm_size
- 6, // .stage_num_col
- 6, // .stage_num_row
+ 8, // .txfm_size
+ 6, // .stage_num_col
+ 6, // .stage_num_row
+ // 0, // .log_scale
fwd_shift_dct_dct_8, // .shift
fwd_stage_range_col_dct_dct_8, // .stage_range_col
fwd_stage_range_row_dct_dct_8, // .stage_range_row
@@ -50,20 +52,21 @@
TXFM_TYPE_DCT8}; // .txfm_type_row
// ---------------- config fwd_dct_dct_16 ----------------
-static const int8_t fwd_shift_dct_dct_16[3] = {4, -3, -1};
+static const int8_t fwd_shift_dct_dct_16[3] = {2, -2, 0};
static const int8_t fwd_stage_range_col_dct_dct_16[8] = {15, 16, 17, 18,
19, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_16[8] = {16, 17, 18, 19,
- 19, 19, 19, 19};
-static const int8_t fwd_cos_bit_col_dct_dct_16[8] = {15, 15, 15, 14,
+static const int8_t fwd_stage_range_row_dct_dct_16[8] = {17, 18, 19, 20,
+ 20, 20, 20, 20};
+static const int8_t fwd_cos_bit_col_dct_dct_16[8] = {13, 13, 13, 13,
13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_16[8] = {15, 15, 14, 13,
- 13, 13, 13, 13};
+static const int8_t fwd_cos_bit_row_dct_dct_16[8] = {13, 13, 13, 12,
+ 12, 12, 12, 12};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = {
- 16, // .txfm_size
- 8, // .stage_num_col
- 8, // .stage_num_row
+ 16, // .txfm_size
+ 8, // .stage_num_col
+ 8, // .stage_num_row
+ // 0, // .log_scale
fwd_shift_dct_dct_16, // .shift
fwd_stage_range_col_dct_dct_16, // .stage_range_col
fwd_stage_range_row_dct_dct_16, // .stage_range_row
@@ -73,20 +76,21 @@
TXFM_TYPE_DCT16}; // .txfm_type_row
// ---------------- config fwd_dct_dct_32 ----------------
-static const int8_t fwd_shift_dct_dct_32[3] = {3, -3, -1};
-static const int8_t fwd_stage_range_col_dct_dct_32[10] = {14, 15, 16, 17, 18,
- 19, 19, 19, 19, 19};
-static const int8_t fwd_stage_range_row_dct_dct_32[10] = {16, 17, 18, 19, 20,
+static const int8_t fwd_shift_dct_dct_32[3] = {2, -3, 0};
+static const int8_t fwd_stage_range_col_dct_dct_32[10] = {15, 16, 17, 18, 19,
20, 20, 20, 20, 20};
-static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {15, 15, 15, 15, 14,
- 13, 13, 13, 13, 13};
-static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {15, 15, 14, 13, 12,
+static const int8_t fwd_stage_range_row_dct_dct_32[10] = {17, 18, 19, 20, 21,
+ 21, 21, 21, 21, 21};
+static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13,
12, 12, 12, 12, 12};
+static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 12, 11,
+ 11, 11, 11, 11, 11};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = {
- 32, // .txfm_size
- 10, // .stage_num_col
- 10, // .stage_num_row
+ 32, // .txfm_size
+ 10, // .stage_num_col
+ 10, // .stage_num_row
+ // 0, // .log_scale
fwd_shift_dct_dct_32, // .shift
fwd_stage_range_col_dct_dct_32, // .stage_range_col
fwd_stage_range_row_dct_dct_32, // .stage_range_row
diff --git a/vp10/common/vp10_inv_txfm2d_cfg.h b/vp10/common/vp10_inv_txfm2d_cfg.h
index ee965ba..897e21c 100644
--- a/vp10/common/vp10_inv_txfm2d_cfg.h
+++ b/vp10/common/vp10_inv_txfm2d_cfg.h
@@ -12,16 +12,17 @@
#define VP10_INV_TXFM2D_CFG_H_
#include "vp10/common/vp10_inv_txfm1d.h"
// ---------------- config inv_dct_dct_4 ----------------
-static const int8_t inv_shift_dct_dct_4[2] = {1, -5};
-static const int8_t inv_stage_range_col_dct_dct_4[4] = {17, 17, 16, 16};
-static const int8_t inv_stage_range_row_dct_dct_4[4] = {16, 16, 16, 16};
-static const int8_t inv_cos_bit_col_dct_dct_4[4] = {15, 15, 15, 15};
-static const int8_t inv_cos_bit_row_dct_dct_4[4] = {15, 15, 15, 15};
+static const int8_t inv_shift_dct_dct_4[2] = {0, -4};
+static const int8_t inv_stage_range_col_dct_dct_4[4] = {18, 18, 17, 17};
+static const int8_t inv_stage_range_row_dct_dct_4[4] = {18, 18, 18, 18};
+static const int8_t inv_cos_bit_col_dct_dct_4[4] = {13, 13, 13, 13};
+static const int8_t inv_cos_bit_row_dct_dct_4[4] = {13, 13, 13, 13};
static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_4 = {
- 4, // .txfm_size
- 4, // .stage_num_col
- 4, // .stage_num_row
+ 4, // .txfm_size
+ 4, // .stage_num_col
+ 4, // .stage_num_row
+ // 0, // .log_scale
inv_shift_dct_dct_4, // .shift
inv_stage_range_col_dct_dct_4, // .stage_range_col
inv_stage_range_row_dct_dct_4, // .stage_range_row
@@ -31,16 +32,17 @@
TXFM_TYPE_DCT4}; // .txfm_type_row
// ---------------- config inv_dct_dct_8 ----------------
-static const int8_t inv_shift_dct_dct_8[2] = {0, -5};
-static const int8_t inv_stage_range_col_dct_dct_8[6] = {17, 17, 17, 17, 16, 16};
-static const int8_t inv_stage_range_row_dct_dct_8[6] = {17, 17, 17, 17, 17, 17};
-static const int8_t inv_cos_bit_col_dct_dct_8[6] = {15, 15, 15, 15, 15, 15};
-static const int8_t inv_cos_bit_row_dct_dct_8[6] = {15, 15, 15, 15, 15, 15};
+static const int8_t inv_shift_dct_dct_8[2] = {1, -6};
+static const int8_t inv_stage_range_col_dct_dct_8[6] = {20, 20, 20, 20, 19, 19};
+static const int8_t inv_stage_range_row_dct_dct_8[6] = {19, 19, 19, 19, 19, 19};
+static const int8_t inv_cos_bit_col_dct_dct_8[6] = {12, 12, 12, 12, 12, 13};
+static const int8_t inv_cos_bit_row_dct_dct_8[6] = {13, 13, 13, 13, 13, 13};
static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_8 = {
- 8, // .txfm_size
- 6, // .stage_num_col
- 6, // .stage_num_row
+ 8, // .txfm_size
+ 6, // .stage_num_col
+ 6, // .stage_num_row
+ // 0, // .log_scale
inv_shift_dct_dct_8, // .shift
inv_stage_range_col_dct_dct_8, // .stage_range_col
inv_stage_range_row_dct_dct_8, // .stage_range_row
@@ -50,20 +52,21 @@
TXFM_TYPE_DCT8}; // .txfm_type_row
// ---------------- config inv_dct_dct_16 ----------------
-static const int8_t inv_shift_dct_dct_16[2] = {0, -6};
-static const int8_t inv_stage_range_col_dct_dct_16[8] = {18, 18, 18, 18,
- 18, 18, 17, 17};
-static const int8_t inv_stage_range_row_dct_dct_16[8] = {18, 18, 18, 18,
- 18, 18, 18, 18};
-static const int8_t inv_cos_bit_col_dct_dct_16[8] = {14, 14, 14, 14,
- 14, 14, 14, 15};
-static const int8_t inv_cos_bit_row_dct_dct_16[8] = {14, 14, 14, 14,
- 14, 14, 14, 14};
+static const int8_t inv_shift_dct_dct_16[2] = {-1, -5};
+static const int8_t inv_stage_range_col_dct_dct_16[8] = {19, 19, 19, 19,
+ 19, 19, 18, 18};
+static const int8_t inv_stage_range_row_dct_dct_16[8] = {20, 20, 20, 20,
+ 20, 20, 20, 20};
+static const int8_t inv_cos_bit_col_dct_dct_16[8] = {13, 13, 13, 13,
+ 13, 13, 13, 13};
+static const int8_t inv_cos_bit_row_dct_dct_16[8] = {12, 12, 12, 12,
+ 12, 12, 12, 12};
static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = {
- 16, // .txfm_size
- 8, // .stage_num_col
- 8, // .stage_num_row
+ 16, // .txfm_size
+ 8, // .stage_num_col
+ 8, // .stage_num_row
+ // 0, // .log_scale
inv_shift_dct_dct_16, // .shift
inv_stage_range_col_dct_dct_16, // .stage_range_col
inv_stage_range_row_dct_dct_16, // .stage_range_row
@@ -74,19 +77,20 @@
// ---------------- config inv_dct_dct_32 ----------------
static const int8_t inv_shift_dct_dct_32[2] = {-1, -6};
-static const int8_t inv_stage_range_col_dct_dct_32[10] = {18, 18, 18, 18, 18,
- 18, 18, 18, 17, 17};
-static const int8_t inv_stage_range_row_dct_dct_32[10] = {19, 19, 19, 19, 19,
- 19, 19, 19, 19, 19};
-static const int8_t inv_cos_bit_col_dct_dct_32[10] = {14, 14, 14, 14, 14,
- 14, 14, 14, 14, 15};
-static const int8_t inv_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 13, 13,
- 13, 13, 13, 13, 13};
+static const int8_t inv_stage_range_col_dct_dct_32[10] = {20, 20, 20, 20, 20,
+ 20, 20, 20, 19, 19};
+static const int8_t inv_stage_range_row_dct_dct_32[10] = {21, 21, 21, 21, 21,
+ 21, 21, 21, 21, 21};
+static const int8_t inv_cos_bit_col_dct_dct_32[10] = {12, 12, 12, 12, 12,
+ 12, 12, 12, 12, 13};
+static const int8_t inv_cos_bit_row_dct_dct_32[10] = {11, 11, 11, 11, 11,
+ 11, 11, 11, 11, 11};
static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = {
- 32, // .txfm_size
- 10, // .stage_num_col
- 10, // .stage_num_row
+ 32, // .txfm_size
+ 10, // .stage_num_col
+ 10, // .stage_num_row
+ // 0, // .log_scale
inv_shift_dct_dct_32, // .shift
inv_stage_range_col_dct_dct_32, // .stage_range_col
inv_stage_range_row_dct_dct_32, // .stage_range_row
