Google Git
Sign in
aomedia / avm / d34877222e44a75c613347feb527f3ff0061c5af / . / aom_dsp / txfm_common.h
blob: ef9e9bc98d41061ed3b051feddb2d178f41bb02d [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]1/*
Yaowu Xu9c01aa12016-09-01 14:32:49 -0700[diff] [blame]2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]3 *
Yaowu Xu9c01aa12016-09-01 14:32:49 -0700[diff] [blame]4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]10 */
11
Yaowu Xuf883b422016-08-30 14:01:10 -0700[diff] [blame]12#ifndef AOM_DSP_TXFM_COMMON_H_
13#define AOM_DSP_TXFM_COMMON_H_
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]14
Yaowu Xuf883b422016-08-30 14:01:10 -0700[diff] [blame]15#include "aom_dsp/aom_dsp_common.h"
Urvang Joshi2283d372017-10-02 17:16:45 -0700[diff] [blame]16#include "av1/common/enums.h"
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]17
18// Constants and Macros used by all idct/dct functions
19#define DCT_CONST_BITS 14
20#define DCT_CONST_ROUNDING (1 << (DCT_CONST_BITS - 1))
21
22#define UNIT_QUANT_SHIFT 2
23#define UNIT_QUANT_FACTOR (1 << UNIT_QUANT_SHIFT)
24
Lester Lu27319b62017-07-10 16:57:15 -0700[diff] [blame]25typedef struct txfm_param {
26 // for both forward and inverse transforms
Urvang Joshi2283d372017-10-02 17:16:45 -0700[diff] [blame]27 TX_TYPE tx_type;
28 TX_SIZE tx_size;
Lester Lu27319b62017-07-10 16:57:15 -0700[diff] [blame]29 int lossless;
30 int bd;
Sarah Parker5b8e6d22017-07-24 15:30:53 -0700[diff] [blame]31#if CONFIG_MRC_TX || CONFIG_LGT
Sarah Parker2e08d962017-08-01 19:51:20 -0700[diff] [blame]32 int is_inter;
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]33#endif // CONFIG_MRC_TX || CONFIG_LGT
34#if CONFIG_MRC_TX || CONFIG_LGT_FROM_PRED
Sarah Parker5b8e6d22017-07-24 15:30:53 -0700[diff] [blame]35 int stride;
36 uint8_t *dst;
Sarah Parkerc5ccd4c2017-08-03 11:27:50 -0700[diff] [blame]37#if CONFIG_MRC_TX
38 int *valid_mask;
Sarah Parker99e7daa2017-08-29 10:30:13 -0700[diff] [blame]39 uint8_t *mask;
Sarah Parkerc5ccd4c2017-08-03 11:27:50 -0700[diff] [blame]40#endif // CONFIG_MRC_TX
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]41#if CONFIG_LGT_FROM_PRED
42 int mode;
43 int use_lgt;
44#endif // CONFIG_LGT_FROM_PRED
45#endif // CONFIG_MRC_TX || CONFIG_LGT_FROM_PRED
Lester Lu27319b62017-07-10 16:57:15 -0700[diff] [blame]46// for inverse transforms only
47#if CONFIG_ADAPT_SCAN
48 const int16_t *eob_threshold;
49#endif
50 int eob;
51} TxfmParam;
52
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]53// Constants:
54// for (int i = 1; i< 32; ++i)
55// printf("static const int cospi_%d_64 = %.0f;\n", i,
56// round(16384 * cos(i*M_PI/64)));
57// Note: sin(k*Pi/64) = cos((32-k)*Pi/64)
58static const tran_high_t cospi_1_64 = 16364;
59static const tran_high_t cospi_2_64 = 16305;
60static const tran_high_t cospi_3_64 = 16207;
61static const tran_high_t cospi_4_64 = 16069;
62static const tran_high_t cospi_5_64 = 15893;
63static const tran_high_t cospi_6_64 = 15679;
64static const tran_high_t cospi_7_64 = 15426;
65static const tran_high_t cospi_8_64 = 15137;
66static const tran_high_t cospi_9_64 = 14811;
67static const tran_high_t cospi_10_64 = 14449;
68static const tran_high_t cospi_11_64 = 14053;
69static const tran_high_t cospi_12_64 = 13623;
70static const tran_high_t cospi_13_64 = 13160;
71static const tran_high_t cospi_14_64 = 12665;
72static const tran_high_t cospi_15_64 = 12140;
73static const tran_high_t cospi_16_64 = 11585;
74static const tran_high_t cospi_17_64 = 11003;
75static const tran_high_t cospi_18_64 = 10394;
76static const tran_high_t cospi_19_64 = 9760;
77static const tran_high_t cospi_20_64 = 9102;
78static const tran_high_t cospi_21_64 = 8423;
79static const tran_high_t cospi_22_64 = 7723;
80static const tran_high_t cospi_23_64 = 7005;
81static const tran_high_t cospi_24_64 = 6270;
82static const tran_high_t cospi_25_64 = 5520;
83static const tran_high_t cospi_26_64 = 4756;
84static const tran_high_t cospi_27_64 = 3981;
85static const tran_high_t cospi_28_64 = 3196;
86static const tran_high_t cospi_29_64 = 2404;
87static const tran_high_t cospi_30_64 = 1606;
88static const tran_high_t cospi_31_64 = 804;
89
Yaowu Xu9c01aa12016-09-01 14:32:49 -0700[diff] [blame]90// 16384 * sqrt(2) * sin(kPi/9) * 2 / 3
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]91static const tran_high_t sinpi_1_9 = 5283;
92static const tran_high_t sinpi_2_9 = 9929;
93static const tran_high_t sinpi_3_9 = 13377;
94static const tran_high_t sinpi_4_9 = 15212;
95
96// 16384 * sqrt(2)
97static const tran_high_t Sqrt2 = 23170;
Debargha Mukherjee570423c2017-10-01 00:35:20 -0700[diff] [blame]98static const tran_high_t InvSqrt2 = 11585;
Yaowu Xuc27fc142016-08-22 16:08:15 -0700[diff] [blame]99
Sarah Parker31c66502017-05-19 16:51:07 -0700[diff] [blame]100static INLINE tran_high_t fdct_round_shift(tran_high_t input) {
101 tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
102 return rv;
103}
Lester Luad8290b2017-06-12 18:26:18 -0700[diff] [blame]104
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]105#if CONFIG_LGT_FROM_PRED
106// Use negative numbers so they do not coincide with lgt*[0][0], which are
107// always nonnegative.
108typedef enum {
109 DCT4 = -1,
110 ADST4 = -2,
111 DCT8 = -3,
112 ADST8 = -4,
113 DCT16 = -5,
114 ADST16 = -6,
115 DCT32 = -7,
116 ADST32 = -8,
117} ButterflyLgt;
Lester Lu708c1ec2017-06-14 14:54:49 -0700[diff] [blame]118
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]119/* These are some LGTs already implementated in the codec. When any of them
120 * is chosen, the flgt or ilgt function will call the existing fast
121 * transform instead of the matrix product implementation. Thus, we
122 * do not need the actual basis functions here */
123static const tran_high_t lgt4_000[1][1] = { { (tran_high_t)DCT4 } };
124static const tran_high_t lgt4_100[1][1] = { { (tran_high_t)ADST4 } };
125static const tran_high_t lgt8_000[1][1] = { { (tran_high_t)DCT8 } };
126static const tran_high_t lgt8_200[1][1] = { { (tran_high_t)ADST8 } };
127static const tran_high_t lgt16_000[1][1] = { { (tran_high_t)DCT16 } };
128static const tran_high_t lgt16_200[1][1] = { { (tran_high_t)ADST16 } };
129static const tran_high_t lgt32_000[1][1] = { { (tran_high_t)DCT32 } };
130static const tran_high_t lgt32_200[1][1] = { { (tran_high_t)ADST32 } };
131
132/* The Line Graph Transforms (LGTs) matrices are written as follows.
133 Each 2D array is sqrt(2)*16384 times an LGT matrix, which is the
134 matrix of eigenvectors of the graph Laplacian matrix of the associated
135 line graph. Some of those transforms have fast algorithms but not
136 implemented yet for now. */
137
138// LGT4 name: lgt4_150_000w3
139// Self loops: 1.500, 0.000, 0.000, 0.000
140// Edges: 1.000, 1.000, 0.000
141static const tran_high_t lgt4_150_000w3[4][4] = {
142 { 0, 0, 0, 23170 },
143 { 5991, 13537, 17825, 0 },
144 { 15515, 10788, -13408, 0 },
145 { 16133, -15403, 6275, 0 },
Lester Luad8290b2017-06-12 18:26:18 -0700[diff] [blame]146};
147
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]148// LGT4 name: lgt4_100_000w3
149// Self loops: 1.000, 0.000, 0.000, 0.000
150// Edges: 1.000, 1.000, 0.000
151static const tran_high_t lgt4_100_000w3[4][4] = {
152 { 0, 0, 0, 23170 },
153 { 7600, 13694, 17076, 0 },
154 { 17076, 7600, -13694, 0 },
155 { 13694, -17076, 7600, 0 },
156};
157
158// LGT4 name: lgt4_060_000w3
159// Self loops: 0.600, 0.000, 0.000, 0.000
160// Edges: 1.000, 1.000, 0.000
161static const tran_high_t lgt4_060_000w3[4][4] = {
162 { 0, 0, 0, 23170 },
163 { 9449, 13755, 16075, 0 },
164 { 17547, 4740, -14370, 0 },
165 { 11819, -18034, 8483, 0 },
166};
167
168// LGT4 name: lgt4_000w3
169// Self loops: 0.000, 0.000, 0.000, 0.000
170// Edges: 1.000, 1.000, 0.000
171static const tran_high_t lgt4_000w3[4][4] = {
172 { 0, 0, 0, 23170 },
173 { 13377, 13377, 13377, 0 },
174 { 16384, 0, -16384, 0 },
175 { 9459, -18919, 9459, 0 },
176};
177
178// LGT4 name: lgt4_150_000w2
179// Self loops: 1.500, 0.000, 0.000, 0.000
180// Edges: 1.000, 0.000, 1.000
181static const tran_high_t lgt4_150_000w2[4][4] = {
182 { 10362, 20724, 0, 0 },
183 { 20724, -10362, 0, 0 },
184 { 0, 0, 16384, 16384 },
185 { 0, 0, 16384, -16384 },
186};
187
188// LGT4 name: lgt4_100_000w2
189// Self loops: 1.000, 0.000, 0.000, 0.000
190// Edges: 1.000, 0.000, 1.000
191static const tran_high_t lgt4_100_000w2[4][4] = {
192 { 12181, 19710, 0, 0 },
193 { 19710, -12181, 0, 0 },
194 { 0, 0, 16384, 16384 },
195 { 0, 0, 16384, -16384 },
196};
197
198// LGT4 name: lgt4_060_000w2
199// Self loops: 0.600, 0.000, 0.000, 0.000
200// Edges: 1.000, 0.000, 1.000
201static const tran_high_t lgt4_060_000w2[4][4] = {
202 { 13831, 18590, 0, 0 },
203 { 18590, -13831, 0, 0 },
204 { 0, 0, 16384, 16384 },
205 { 0, 0, 16384, -16384 },
206};
207
208// LGT4 name: lgt4_000w2
209// Self loops: 0.000, 0.000, 0.000, 0.000
210// Edges: 1.000, 0.000, 1.000
211static const tran_high_t lgt4_000w2[4][4] = {
212 { 16384, 16384, 0, 0 },
213 { 16384, -16384, 0, 0 },
214 { 0, 0, 16384, 16384 },
215 { 0, 0, 16384, -16384 },
216};
217
218// LGT4 name: lgt4_150_000w1
219// Self loops: 1.500, 0.000, 0.000, 0.000
220// Edges: 0.000, 1.000, 1.000
221static const tran_high_t lgt4_150_000w1[4][4] = {
222 { 23170, 0, 0, 0 },
223 { 0, 13377, 13377, 13377 },
224 { 0, 16384, 0, -16384 },
225 { 0, 9459, -18919, 9459 },
226};
227
228// LGT4 name: lgt4_100_000w1
229// Self loops: 1.000, 0.000, 0.000, 0.000
230// Edges: 0.000, 1.000, 1.000
231static const tran_high_t lgt4_100_000w1[4][4] = {
232 { 23170, 0, 0, 0 },
233 { 0, 13377, 13377, 13377 },
234 { 0, 16384, 0, -16384 },
235 { 0, 9459, -18919, 9459 },
236};
237
238// LGT4 name: lgt4_060_000w1
239// Self loops: 0.600, 0.000, 0.000, 0.000
240// Edges: 0.000, 1.000, 1.000
241static const tran_high_t lgt4_060_000w1[4][4] = {
242 { 23170, 0, 0, 0 },
243 { 0, 13377, 13377, 13377 },
244 { 0, 16384, 0, -16384 },
245 { 0, 9459, -18919, 9459 },
246};
247
248// LGT4 name: lgt4_000w1
249// Self loops: 0.000, 0.000, 0.000, 0.000
250// Edges: 0.000, 1.000, 1.000
251static const tran_high_t lgt4_000w1[4][4] = {
252 { 23170, 0, 0, 0 },
253 { 0, 13377, 13377, 13377 },
254 { 0, 16384, 0, -16384 },
255 { 0, 9459, -18919, 9459 },
256};
257
258// LGT4 name: lgt4_060
259// Self loops: 0.600, 0.000, 0.000, 0.000
260// Edges: 1.000, 1.000, 1.000
261static const tran_high_t lgt4_060[4][4] = {
262 { 6971, 10504, 13060, 14400 },
263 { 14939, 11211, -2040, -13559 },
264 { 14096, -8258, -12561, 10593 },
265 { 8150, -15253, 14295, -5784 },
266};
267
268// LGT4 name: lgt4_150
269// Self loops: 1.500, 0.000, 0.000, 0.000
270// Edges: 1.000, 1.000, 1.000
271static const tran_high_t lgt4_150[4][4] = {
272 { 3998, 9435, 13547, 15759 },
273 { 11106, 15105, 1886, -13483 },
274 { 15260, -1032, -14674, 9361 },
275 { 12833, -14786, 11596, -4372 },
276};
277
278// LGT8 name: lgt8_150_000w7
279// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
280// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
281static const tran_high_t lgt8_150_000w7[8][8] = {
282 { 0, 0, 0, 0, 0, 0, 0, 32768 },
283 { 2522, 6185, 9551, 12461, 14775, 16381, 17204, 0 },
284 { 7390, 15399, 16995, 11515, 1240, -9551, -16365, 0 },
285 { 11716, 16625, 3560, -13353, -15831, -1194, 14733, 0 },
286 { 15073, 8866, -14291, -10126, 13398, 11308, -12401, 0 },
287 { 16848, -4177, -13724, 14441, 2923, -16628, 9513, 0 },
288 { 15942, -14888, 5405, 7137, -15640, 15288, -6281, 0 },
289 { 10501, -14293, 16099, -15670, 13063, -8642, 3021, 0 },
290};
291
292// LGT8 name: lgt8_100_000w7
293// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
294// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
295static const tran_high_t lgt8_100_000w7[8][8] = {
296 { 0, 0, 0, 0, 0, 0, 0, 32768 },
297 { 3518, 6883, 9946, 12575, 14654, 16093, 16829, 0 },
298 { 9946, 16093, 16093, 9946, 0, -9946, -16093, 0 },
299 { 14654, 14654, 0, -14654, -14654, 0, 14654, 0 },
300 { 16829, 3518, -16093, -6883, 14654, 9946, -12575, 0 },
301 { 16093, -9946, -9946, 16093, 0, -16093, 9946, 0 },
302 { 12575, -16829, 9946, 3518, -14654, 16093, -6883, 0 },
303 { 6883, -12575, 16093, -16829, 14654, -9946, 3518, 0 },
304};
305
306// LGT8 name: lgt8_060_000w7
307// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
308// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
309static const tran_high_t lgt8_060_000w7[8][8] = {
310 { 0, 0, 0, 0, 0, 0, 0, 32768 },
311 { 5087, 7951, 10521, 12701, 14411, 15587, 16186, 0 },
312 { 13015, 16486, 14464, 7621, -1762, -10557, -15834, 0 },
313 { 16581, 11475, -4050, -15898, -13311, 1362, 14798, 0 },
314 { 16536, -1414, -16981, -3927, 15746, 8879, -12953, 0 },
315 { 14104, -13151, -7102, 16932, -1912, -15914, 10385, 0 },
316 { 10156, -17168, 11996, 1688, -14174, 16602, -7249, 0 },
317 { 5295, -11721, 15961, -17224, 15274, -10476, 3723, 0 },
318};
319
320// LGT8 name: lgt8_000w7
321// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
322// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
323static const tran_high_t lgt8_000w7[8][8] = {
324 { 0, 0, 0, 0, 0, 0, 0, 32768 },
325 { 12385, 12385, 12385, 12385, 12385, 12385, 12385, 0 },
326 { 17076, 13694, 7600, 0, -7600, -13694, -17076, 0 },
327 { 15781, 3898, -10921, -17515, -10921, 3898, 15781, 0 },
328 { 13694, -7600, -17076, 0, 17076, 7600, -13694, 0 },
329 { 10921, -15781, -3898, 17515, -3898, -15781, 10921, 0 },
330 { 7600, -17076, 13694, 0, -13694, 17076, -7600, 0 },
331 { 3898, -10921, 15781, -17515, 15781, -10921, 3898, 0 },
332};
333
334// LGT8 name: lgt8_150_000w6
335// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
336// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
337static const tran_high_t lgt8_150_000w6[8][8] = {
338 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
339 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
340 { 3157, 7688, 11723, 15002, 17312, 18506, 0, 0 },
341 { 9167, 17832, 16604, 6164, -7696, -17286, 0, 0 },
342 { 14236, 15584, -4969, -18539, -6055, 14938, 0, 0 },
343 { 17558, 1891, -18300, 5288, 16225, -11653, 0, 0 },
344 { 17776, -13562, -647, 14380, -17514, 7739, 0, 0 },
345 { 12362, -16318, 17339, -15240, 10399, -3688, 0, 0 },
346};
347
348// LGT8 name: lgt8_100_000w6
349// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
350// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
351static const tran_high_t lgt8_100_000w6[8][8] = {
352 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
353 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
354 { 4350, 8447, 12053, 14959, 16995, 18044, 0, 0 },
355 { 12053, 18044, 14959, 4350, -8447, -16995, 0, 0 },
356 { 16995, 12053, -8447, -18044, -4350, 14959, 0, 0 },
357 { 18044, -4350, -16995, 8447, 14959, -12053, 0, 0 },
358 { 14959, -16995, 4350, 12053, -18044, 8447, 0, 0 },
359 { 8447, -14959, 18044, -16995, 12053, -4350, 0, 0 },
360};
361
362// LGT8 name: lgt8_060_000w6
363// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
364// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
365static const tran_high_t lgt8_060_000w6[8][8] = {
366 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
367 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
368 { 6154, 9551, 12487, 14823, 16446, 17277, 0, 0 },
369 { 15149, 17660, 12503, 1917, -9502, -16795, 0, 0 },
370 { 18166, 7740, -11772, -17465, -2656, 15271, 0, 0 },
371 { 16682, -8797, -15561, 10779, 14189, -12586, 0, 0 },
372 { 12436, -18234, 7007, 10763, -18483, 8945, 0, 0 },
373 { 6591, -14172, 18211, -17700, 12766, -4642, 0, 0 },
374};
375
376// LGT8 name: lgt8_000w6
377// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
378// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
379static const tran_high_t lgt8_000w6[8][8] = {
380 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
381 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
382 { 13377, 13377, 13377, 13377, 13377, 13377, 0, 0 },
383 { 18274, 13377, 4896, -4896, -13377, -18274, 0, 0 },
384 { 16384, 0, -16384, -16384, 0, 16384, 0, 0 },
385 { 13377, -13377, -13377, 13377, 13377, -13377, 0, 0 },
386 { 9459, -18919, 9459, 9459, -18919, 9459, 0, 0 },
387 { 4896, -13377, 18274, -18274, 13377, -4896, 0, 0 },
388};
389
390// LGT8 name: lgt8_150_000w5
391// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
392// Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
393static const tran_high_t lgt8_150_000w5[8][8] = {
394 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
395 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
396 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
397 { 4109, 9895, 14774, 18299, 20146, 0, 0, 0 },
398 { 11753, 20300, 13161, -4148, -18252, 0, 0, 0 },
399 { 17573, 10921, -16246, -12895, 14679, 0, 0, 0 },
400 { 19760, -9880, -9880, 19760, -9880, 0, 0, 0 },
401 { 14815, -18624, 17909, -12844, 4658, 0, 0, 0 },
402};
403
404// LGT8 name: lgt8_100_000w5
405// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
406// Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
407static const tran_high_t lgt8_100_000w5[8][8] = {
408 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
409 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
410 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
411 { 5567, 10683, 14933, 17974, 19559, 0, 0, 0 },
412 { 14933, 19559, 10683, -5567, -17974, 0, 0, 0 },
413 { 19559, 5567, -17974, -10683, 14933, 0, 0, 0 },
414 { 17974, -14933, -5567, 19559, -10683, 0, 0, 0 },
415 { 10683, -17974, 19559, -14933, 5567, 0, 0, 0 },
416};
417
418// LGT8 name: lgt8_060_000w5
419// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
420// Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
421static const tran_high_t lgt8_060_000w5[8][8] = {
422 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
423 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
424 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
425 { 7650, 11741, 15069, 17415, 18628, 0, 0, 0 },
426 { 17824, 18002, 7558, -7345, -17914, 0, 0, 0 },
427 { 19547, 569, -19303, -8852, 15505, 0, 0, 0 },
428 { 15592, -17548, -2862, 19625, -11374, 0, 0, 0 },
429 { 8505, -17423, 20218, -15907, 6006, 0, 0, 0 },
430};
431
432// LGT8 name: lgt8_000w5
433// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
434// Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
435static const tran_high_t lgt8_000w5[8][8] = {
436 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
437 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
438 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
439 { 14654, 14654, 14654, 14654, 14654, 0, 0, 0 },
440 { 19710, 12181, 0, -12181, -19710, 0, 0, 0 },
441 { 16766, -6404, -20724, -6404, 16766, 0, 0, 0 },
442 { 12181, -19710, 0, 19710, -12181, 0, 0, 0 },
443 { 6404, -16766, 20724, -16766, 6404, 0, 0, 0 },
444};
445
446// LGT8 name: lgt8_150_000w4
447// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
448// Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
449static const tran_high_t lgt8_150_000w4[8][8] = {
450 { 5655, 13343, 19159, 22286, 0, 0, 0, 0 },
451 { 15706, 21362, 2667, -19068, 0, 0, 0, 0 },
452 { 21580, -1459, -20752, 13238, 0, 0, 0, 0 },
453 { 18148, -20910, 16399, -6183, 0, 0, 0, 0 },
454 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
455 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
456 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
457 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
458};
459
460// LGT8 name: lgt8_100_000w4
461// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
462// Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
463static const tran_high_t lgt8_100_000w4[8][8] = {
464 { 7472, 14042, 18919, 21513, 0, 0, 0, 0 },
465 { 18919, 18919, 0, -18919, 0, 0, 0, 0 },
466 { 21513, -7472, -18919, 14042, 0, 0, 0, 0 },
467 { 14042, -21513, 18919, -7472, 0, 0, 0, 0 },
468 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
469 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
470 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
471 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
472};
473
474// LGT8 name: lgt8_060_000w4
475// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
476// Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
477static const tran_high_t lgt8_060_000w4[8][8] = {
478 { 9858, 14855, 18470, 20365, 0, 0, 0, 0 },
479 { 21127, 15855, -2886, -19175, 0, 0, 0, 0 },
480 { 19935, -11679, -17764, 14980, 0, 0, 0, 0 },
481 { 11525, -21570, 20217, -8180, 0, 0, 0, 0 },
482 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
483 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
484 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
485 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
486};
487
488// LGT8 name: lgt8_000w4
489// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
490// Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
491static const tran_high_t lgt8_000w4[8][8] = {
492 { 16384, 16384, 16384, 16384, 0, 0, 0, 0 },
493 { 21407, 8867, -8867, -21407, 0, 0, 0, 0 },
494 { 16384, -16384, -16384, 16384, 0, 0, 0, 0 },
495 { 8867, -21407, 21407, -8867, 0, 0, 0, 0 },
496 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
497 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
498 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
499 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
500};
501
502// LGT8 name: lgt8_150_000w3
503// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
504// Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
505static const tran_high_t lgt8_150_000w3[8][8] = {
506 { 8473, 19144, 25209, 0, 0, 0, 0, 0 },
507 { 21942, 15257, -18961, 0, 0, 0, 0, 0 },
508 { 22815, -21783, 8874, 0, 0, 0, 0, 0 },
509 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
510 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
511 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
512 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
513 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
514};
515
516// LGT8 name: lgt8_100_000w3
517// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
518// Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
519static const tran_high_t lgt8_100_000w3[8][8] = {
520 { 10747, 19366, 24149, 0, 0, 0, 0, 0 },
521 { 24149, 10747, -19366, 0, 0, 0, 0, 0 },
522 { 19366, -24149, 10747, 0, 0, 0, 0, 0 },
523 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
524 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
525 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
526 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
527 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
528};
529
530// LGT8 name: lgt8_060_000w3
531// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
532// Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
533static const tran_high_t lgt8_060_000w3[8][8] = {
534 { 13363, 19452, 22733, 0, 0, 0, 0, 0 },
535 { 24815, 6704, -20323, 0, 0, 0, 0, 0 },
536 { 16715, -25503, 11997, 0, 0, 0, 0, 0 },
537 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
538 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
539 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
540 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
541 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
542};
543
544// LGT8 name: lgt8_000w3
545// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
546// Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
547static const tran_high_t lgt8_000w3[8][8] = {
548 { 18919, 18919, 18919, 0, 0, 0, 0, 0 },
549 { 23170, 0, -23170, 0, 0, 0, 0, 0 },
550 { 13377, -26755, 13377, 0, 0, 0, 0, 0 },
551 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
552 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
553 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
554 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
555 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
556};
557
558// LGT8 name: lgt8_150_000w2
559// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
560// Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
561static const tran_high_t lgt8_150_000w2[8][8] = {
562 { 14654, 29309, 0, 0, 0, 0, 0, 0 },
563 { 29309, -14654, 0, 0, 0, 0, 0, 0 },
564 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
565 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
566 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
567 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
568 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
569 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
570};
571
572// LGT8 name: lgt8_100_000w2
573// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
574// Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
575static const tran_high_t lgt8_100_000w2[8][8] = {
576 { 17227, 27874, 0, 0, 0, 0, 0, 0 },
577 { 27874, -17227, 0, 0, 0, 0, 0, 0 },
578 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
579 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
580 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
581 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
582 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
583 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
584};
585
586// LGT8 name: lgt8_060_000w2
587// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
588// Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
589static const tran_high_t lgt8_060_000w2[8][8] = {
590 { 19560, 26290, 0, 0, 0, 0, 0, 0 },
591 { 26290, -19560, 0, 0, 0, 0, 0, 0 },
592 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
593 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
594 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
595 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
596 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
597 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
598};
599
600// LGT8 name: lgt8_000w2
601// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
602// Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
603static const tran_high_t lgt8_000w2[8][8] = {
604 { 23170, 23170, 0, 0, 0, 0, 0, 0 },
605 { 23170, -23170, 0, 0, 0, 0, 0, 0 },
606 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
607 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
608 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
609 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
610 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
611 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
612};
613
614// LGT8 name: lgt8_150_000w1
615// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
616// Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
617static const tran_high_t lgt8_150_000w1[8][8] = {
618 { 32768, 0, 0, 0, 0, 0, 0, 0 },
619 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
620 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
621 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
622 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
623 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
624 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
625 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
626};
627
628// LGT8 name: lgt8_100_000w1
629// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
630// Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
631static const tran_high_t lgt8_100_000w1[8][8] = {
632 { 32768, 0, 0, 0, 0, 0, 0, 0 },
633 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
634 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
635 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
636 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
637 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
638 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
639 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
640};
641
642// LGT8 name: lgt8_060_000w1
643// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
644// Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
645static const tran_high_t lgt8_060_000w1[8][8] = {
646 { 32768, 0, 0, 0, 0, 0, 0, 0 },
647 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
648 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
649 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
650 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
651 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
652 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
653 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
654};
655
656// LGT8 name: lgt8_000w1
657// Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
658// Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
659static const tran_high_t lgt8_000w1[8][8] = {
660 { 32768, 0, 0, 0, 0, 0, 0, 0 },
661 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
662 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
663 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
664 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
665 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
666 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
667 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
668};
669
670// LGT8 name: lgt8_060
671// Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
672// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
673static const tran_high_t lgt8_060[8][8] = {
674 { 4295, 6746, 8999, 10987, 12653, 13947, 14832, 15280 },
675 { 11303, 15101, 14912, 10786, 3812, -4168, -11047, -15010 },
676 { 15051, 13208, 1823, -10879, -15721, -9207, 3959, 14265 },
677 { 15871, 3800, -13441, -12395, 5516, 15922, 4665, -12939 },
678 { 14630, -7269, -13926, 8618, 13091, -9886, -12133, 11062 },
679 { 12008, -14735, 180, 14586, -12245, -4458, 15932, -8720 },
680 { 8472, -15623, 14088, -4721, -7272, 15221, -14708, 6018 },
681 { 4372, -9862, 13927, -15981, 15727, -13202, 8770, -3071 },
682};
683
684// LGT8 name: lgt8_100
685// Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
686// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
687static const tran_high_t lgt8_100[8][8] = {
688 { 2921, 5742, 8368, 10708, 12684, 14228, 15288, 15827 },
689 { 8368, 14228, 15827, 12684, 5742, -2921, -10708, -15288 },
690 { 12684, 15288, 5742, -8368, -15827, -10708, 2921, 14228 },
691 { 15288, 8368, -10708, -14228, 2921, 15827, 5742, -12684 },
692 { 15827, -2921, -15288, 5742, 14228, -8368, -12684, 10708 },
693 { 14228, -12684, -2921, 15288, -10708, -5742, 15827, -8368 },
694 { 10708, -15827, 12684, -2921, -8368, 15288, -14228, 5742 },
695 { 5742, -10708, 14228, -15827, 15288, -12684, 8368, -2921 },
696};
697#endif // CONFIG_LGT_FROM_PRED
698
699#if CONFIG_LGT || CONFIG_LGT_FROM_PRED
Lester Lu708c1ec2017-06-14 14:54:49 -0700[diff] [blame]700// LGT4 name: lgt4_170
701// Self loops: 1.700, 0.000, 0.000, 0.000
702// Edges: 1.000, 1.000, 1.000
703static const tran_high_t lgt4_170[4][4] = {
704 { 3636, 9287, 13584, 15902 },
705 { 10255, 15563, 2470, -13543 },
706 { 14786, 711, -15249, 9231 },
707 { 14138, -14420, 10663, -3920 },
708};
709
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]710// LGT4 name: lgt4_140
711// Self loops: 1.400, 0.000, 0.000, 0.000
712// Edges: 1.000, 1.000, 1.000
713static const tran_high_t lgt4_140[4][4] = {
714 { 4206, 9518, 13524, 15674 },
715 { 11552, 14833, 1560, -13453 },
716 { 15391, -1906, -14393, 9445 },
717 { 12201, -14921, 12016, -4581 },
Lester Lu708c1ec2017-06-14 14:54:49 -0700[diff] [blame]718};
719
Lester Luad8290b2017-06-12 18:26:18 -0700[diff] [blame]720// LGT8 name: lgt8_170
721// Self loops: 1.700, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
722// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
Lester Lu708c1ec2017-06-14 14:54:49 -0700[diff] [blame]723static const tran_high_t lgt8_170[8][8] = {
Lester Luad8290b2017-06-12 18:26:18 -0700[diff] [blame]724 { 1858, 4947, 7850, 10458, 12672, 14411, 15607, 16217 },
725 { 5494, 13022, 16256, 14129, 7343, -1864, -10456, -15601 },
726 { 8887, 16266, 9500, -5529, -15749, -12273, 1876, 14394 },
727 { 11870, 13351, -6199, -15984, -590, 15733, 7273, -12644 },
728 { 14248, 5137, -15991, 291, 15893, -5685, -13963, 10425 },
729 { 15716, -5450, -10010, 15929, -6665, -8952, 16036, -7835 },
730 { 15533, -13869, 6559, 3421, -12009, 15707, -13011, 5018 },
731 { 11357, -13726, 14841, -14600, 13025, -10259, 6556, -2254 },
732};
Lester Lu432012f2017-08-17 14:39:29 -0700[diff] [blame]733
734// LGT8 name: lgt8_150
735// Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
736// Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
737static const tran_high_t lgt8_150[8][8] = {
738 { 2075, 5110, 7958, 10511, 12677, 14376, 15544, 16140 },
739 { 6114, 13307, 16196, 13845, 7015, -2084, -10509, -15534 },
740 { 9816, 16163, 8717, -6168, -15790, -11936, 2104, 14348 },
741 { 12928, 12326, -7340, -15653, 242, 15763, 6905, -12632 },
742 { 15124, 3038, -16033, 1758, 15507, -6397, -13593, 10463 },
743 { 15895, -7947, -7947, 15895, -7947, -7947, 15895, -7947 },
744 { 14325, -15057, 9030, 1050, -10659, 15483, -13358, 5236 },
745 { 9054, -12580, 14714, -15220, 14043, -11312, 7330, -2537 },
746};
747#endif // CONFIG_LGT || CONFIG_LGT_FROM_PRED
Yaowu Xuf883b422016-08-30 14:01:10 -0700[diff] [blame]748#endif // AOM_DSP_TXFM_COMMON_H_