blob: 7ee8ba330edf584a87f80f8e61d544cc7c3ee2d4 [file] [log] [blame]
Yaowu Xuc27fc142016-08-22 16:08:15 -07001/*
Yaowu Xu9c01aa12016-09-01 14:32:49 -07002 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xuc27fc142016-08-22 16:08:15 -07003 *
Yaowu Xu9c01aa12016-09-01 14:32:49 -07004 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xuc27fc142016-08-22 16:08:15 -070010 */
11
12#include <emmintrin.h> // SSE2
13
Tom Finegan44702c82018-05-22 13:00:39 -070014#include "config/aom_dsp_rtcd.h"
15
Yaowu Xuc27fc142016-08-22 16:08:15 -070016#include "aom_dsp/txfm_common.h"
17#include "aom_dsp/x86/fwd_txfm_sse2.h"
18#include "aom_dsp/x86/txfm_common_sse2.h"
19#include "aom_ports/mem.h"
20
21// TODO(jingning) The high bit-depth functions need rework for performance.
22// After we properly fix the high bit-depth function implementations, this
23// file's dependency should be substantially simplified.
24#if DCT_HIGH_BIT_DEPTH
25#define ADD_EPI16 _mm_adds_epi16
26#define SUB_EPI16 _mm_subs_epi16
27
28#else
29#define ADD_EPI16 _mm_add_epi16
30#define SUB_EPI16 _mm_sub_epi16
31#endif
32
Jerome Jiang4e8bbd12020-02-03 10:40:12 -080033static void FDCT4x4_2D_HELPER(const int16_t *input, int stride, __m128i *in0,
34 __m128i *in1) {
35 // Constants
36 // These are the coefficients used for the multiplies.
37 // In the comments, pN means cos(N pi /64) and mN is -cos(N pi /64),
38 // where cospi_N_64 = cos(N pi /64)
39 const __m128i k__cospi_A =
40 octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64,
41 cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64);
42 const __m128i k__cospi_B =
43 octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64,
44 cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64);
45 const __m128i k__cospi_C =
46 octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64,
47 cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64);
48 const __m128i k__cospi_D =
49 octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64,
50 cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64);
51 const __m128i k__cospi_E =
52 octa_set_epi16(cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64,
53 cospi_16_64, cospi_16_64, cospi_16_64, cospi_16_64);
54 const __m128i k__cospi_F =
55 octa_set_epi16(cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64,
56 cospi_16_64, -cospi_16_64, cospi_16_64, -cospi_16_64);
57 const __m128i k__cospi_G =
58 octa_set_epi16(cospi_8_64, cospi_24_64, cospi_8_64, cospi_24_64,
59 -cospi_8_64, -cospi_24_64, -cospi_8_64, -cospi_24_64);
60 const __m128i k__cospi_H =
61 octa_set_epi16(cospi_24_64, -cospi_8_64, cospi_24_64, -cospi_8_64,
62 -cospi_24_64, cospi_8_64, -cospi_24_64, cospi_8_64);
63
64 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
65 // This second rounding constant saves doing some extra adds at the end
66 const __m128i k__DCT_CONST_ROUNDING2 =
67 _mm_set1_epi32(DCT_CONST_ROUNDING + (DCT_CONST_ROUNDING << 1));
68 const int DCT_CONST_BITS2 = DCT_CONST_BITS + 2;
69 const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
70 const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
71
72 // Load inputs.
73 *in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride));
74 *in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride));
75 *in1 = _mm_unpacklo_epi64(
76 *in1, _mm_loadl_epi64((const __m128i *)(input + 2 * stride)));
77 *in0 = _mm_unpacklo_epi64(
78 *in0, _mm_loadl_epi64((const __m128i *)(input + 3 * stride)));
79 // in0 = [i0 i1 i2 i3 iC iD iE iF]
80 // in1 = [i4 i5 i6 i7 i8 i9 iA iB]
81 // multiply by 16 to give some extra precision
82 *in0 = _mm_slli_epi16(*in0, 4);
83 *in1 = _mm_slli_epi16(*in1, 4);
84 // if (i == 0 && input[0]) input[0] += 1;
85 // add 1 to the upper left pixel if it is non-zero, which helps reduce
86 // the round-trip error
87 {
88 // The mask will only contain whether the first value is zero, all
89 // other comparison will fail as something shifted by 4 (above << 4)
90 // can never be equal to one. To increment in the non-zero case, we
91 // add the mask and one for the first element:
92 // - if zero, mask = -1, v = v - 1 + 1 = v
93 // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1
94 __m128i mask = _mm_cmpeq_epi16(*in0, k__nonzero_bias_a);
95 *in0 = _mm_add_epi16(*in0, mask);
96 *in0 = _mm_add_epi16(*in0, k__nonzero_bias_b);
97 }
98 // There are 4 total stages, alternating between an add/subtract stage
99 // followed by an multiply-and-add stage.
100 {
101 // Stage 1: Add/subtract
102
103 // in0 = [i0 i1 i2 i3 iC iD iE iF]
104 // in1 = [i4 i5 i6 i7 i8 i9 iA iB]
105 const __m128i r0 = _mm_unpacklo_epi16(*in0, *in1);
106 const __m128i r1 = _mm_unpackhi_epi16(*in0, *in1);
107 // r0 = [i0 i4 i1 i5 i2 i6 i3 i7]
108 // r1 = [iC i8 iD i9 iE iA iF iB]
109 const __m128i r2 = _mm_shuffle_epi32(r0, 0xB4);
110 const __m128i r3 = _mm_shuffle_epi32(r1, 0xB4);
111 // r2 = [i0 i4 i1 i5 i3 i7 i2 i6]
112 // r3 = [iC i8 iD i9 iF iB iE iA]
113
114 const __m128i t0 = _mm_add_epi16(r2, r3);
115 const __m128i t1 = _mm_sub_epi16(r2, r3);
116 // t0 = [a0 a4 a1 a5 a3 a7 a2 a6]
117 // t1 = [aC a8 aD a9 aF aB aE aA]
118
119 // Stage 2: multiply by constants (which gets us into 32 bits).
120 // The constants needed here are:
121 // k__cospi_A = [p16 p16 p16 p16 p16 m16 p16 m16]
122 // k__cospi_B = [p16 m16 p16 m16 p16 p16 p16 p16]
123 // k__cospi_C = [p08 p24 p08 p24 p24 m08 p24 m08]
124 // k__cospi_D = [p24 m08 p24 m08 p08 p24 p08 p24]
125 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_A);
126 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_B);
127 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_C);
128 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_D);
129 // Then add and right-shift to get back to 16-bit range
130 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
131 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
132 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
133 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
134 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
135 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
136 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
137 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
138 // w0 = [b0 b1 b7 b6]
139 // w1 = [b8 b9 bF bE]
140 // w2 = [b4 b5 b3 b2]
141 // w3 = [bC bD bB bA]
142 const __m128i x0 = _mm_packs_epi32(w0, w1);
143 const __m128i x1 = _mm_packs_epi32(w2, w3);
144
145 // x0 = [b0 b1 b7 b6 b8 b9 bF bE]
146 // x1 = [b4 b5 b3 b2 bC bD bB bA]
147 *in0 = _mm_shuffle_epi32(x0, 0xD8);
148 *in1 = _mm_shuffle_epi32(x1, 0x8D);
149 // in0 = [b0 b1 b8 b9 b7 b6 bF bE]
150 // in1 = [b3 b2 bB bA b4 b5 bC bD]
151 }
152 {
153 // vertical DCTs finished. Now we do the horizontal DCTs.
154 // Stage 3: Add/subtract
155
156 const __m128i t0 = ADD_EPI16(*in0, *in1);
157 const __m128i t1 = SUB_EPI16(*in0, *in1);
158
159 // Stage 4: multiply by constants (which gets us into 32 bits).
160 {
161 // The constants needed here are:
162 // k__cospi_E = [p16 p16 p16 p16 p16 p16 p16 p16]
163 // k__cospi_F = [p16 m16 p16 m16 p16 m16 p16 m16]
164 // k__cospi_G = [p08 p24 p08 p24 m08 m24 m08 m24]
165 // k__cospi_H = [p24 m08 p24 m08 m24 p08 m24 p08]
166 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_E);
167 const __m128i u1 = _mm_madd_epi16(t0, k__cospi_F);
168 const __m128i u2 = _mm_madd_epi16(t1, k__cospi_G);
169 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_H);
170 // Then add and right-shift to get back to 16-bit range
171 // but this combines the final right-shift as well to save operations
172 // This unusual rounding operations is to maintain bit-accurate
173 // compatibility with the c version of this function which has two
174 // rounding steps in a row.
175 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING2);
176 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING2);
177 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING2);
178 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING2);
179 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS2);
180 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS2);
181 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS2);
182 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS2);
Kyle Siefring7bb4c542022-12-09 10:27:41 -0500183 *in0 = _mm_packs_epi32(w0, w2);
184 *in1 = _mm_packs_epi32(w1, w3);
Jerome Jiang4e8bbd12020-02-03 10:40:12 -0800185 }
186 }
187}
188
189void FDCT4x4_2D(const int16_t *input, tran_low_t *output, int stride) {
190 // This 2D transform implements 4 vertical 1D transforms followed
191 // by 4 horizontal 1D transforms. The multiplies and adds are as given
192 // by Chen, Smith and Fralick ('77). The commands for moving the data
193 // around have been minimized by hand.
194 // For the purposes of the comments, the 16 inputs are referred to at i0
195 // through iF (in raster order), intermediate variables are a0, b0, c0
196 // through f, and correspond to the in-place computations mapped to input
197 // locations. The outputs, o0 through oF are labeled according to the
198 // output locations.
199 __m128i in0, in1;
200 FDCT4x4_2D_HELPER(input, stride, &in0, &in1);
201
202 // Post-condition (v + 1) >> 2 is now incorporated into previous
203 // add and right-shift commands. Only 2 store instructions needed
204 // because we are using the fact that 1/3 are stored just after 0/2.
205 storeu_output(&in0, output + 0 * 4);
206 storeu_output(&in1, output + 2 * 4);
207}
208
209void FDCT4x4_2D_LP(const int16_t *input, int16_t *output, int stride) {
210 __m128i in0, in1;
211 FDCT4x4_2D_HELPER(input, stride, &in0, &in1);
212 _mm_storeu_si128((__m128i *)(output + 0 * 4), in0);
213 _mm_storeu_si128((__m128i *)(output + 2 * 4), in1);
214}
215
James Zern0e7ea552023-03-28 15:19:00 -0700216#if CONFIG_INTERNAL_STATS
Yaowu Xuc27fc142016-08-22 16:08:15 -0700217void FDCT8x8_2D(const int16_t *input, tran_low_t *output, int stride) {
218 int pass;
219 // Constants
220 // When we use them, in one case, they are all the same. In all others
221 // it's a pair of them that we need to repeat four times. This is done
222 // by constructing the 32 bit constant corresponding to that pair.
223 const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64);
224 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
225 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
226 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
227 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
228 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
229 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
230 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
231 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
232#if DCT_HIGH_BIT_DEPTH
233 int overflow;
234#endif
235 // Load input
236 __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
237 __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
238 __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
239 __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
240 __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
241 __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
242 __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
243 __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
244 // Pre-condition input (shift by two)
245 in0 = _mm_slli_epi16(in0, 2);
246 in1 = _mm_slli_epi16(in1, 2);
247 in2 = _mm_slli_epi16(in2, 2);
248 in3 = _mm_slli_epi16(in3, 2);
249 in4 = _mm_slli_epi16(in4, 2);
250 in5 = _mm_slli_epi16(in5, 2);
251 in6 = _mm_slli_epi16(in6, 2);
252 in7 = _mm_slli_epi16(in7, 2);
253
254 // We do two passes, first the columns, then the rows. The results of the
255 // first pass are transposed so that the same column code can be reused. The
256 // results of the second pass are also transposed so that the rows (processed
257 // as columns) are put back in row positions.
258 for (pass = 0; pass < 2; pass++) {
259 // To store results of each pass before the transpose.
260 __m128i res0, res1, res2, res3, res4, res5, res6, res7;
261 // Add/subtract
262 const __m128i q0 = ADD_EPI16(in0, in7);
263 const __m128i q1 = ADD_EPI16(in1, in6);
264 const __m128i q2 = ADD_EPI16(in2, in5);
265 const __m128i q3 = ADD_EPI16(in3, in4);
266 const __m128i q4 = SUB_EPI16(in3, in4);
267 const __m128i q5 = SUB_EPI16(in2, in5);
268 const __m128i q6 = SUB_EPI16(in1, in6);
269 const __m128i q7 = SUB_EPI16(in0, in7);
270#if DCT_HIGH_BIT_DEPTH
271 if (pass == 1) {
272 overflow =
273 check_epi16_overflow_x8(&q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7);
274 if (overflow) {
Yaowu Xuf883b422016-08-30 14:01:10 -0700275 aom_highbd_fdct8x8_c(input, output, stride);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700276 return;
277 }
278 }
279#endif // DCT_HIGH_BIT_DEPTH
280 // Work on first four results
281 {
282 // Add/subtract
283 const __m128i r0 = ADD_EPI16(q0, q3);
284 const __m128i r1 = ADD_EPI16(q1, q2);
285 const __m128i r2 = SUB_EPI16(q1, q2);
286 const __m128i r3 = SUB_EPI16(q0, q3);
287#if DCT_HIGH_BIT_DEPTH
288 overflow = check_epi16_overflow_x4(&r0, &r1, &r2, &r3);
289 if (overflow) {
Yaowu Xuf883b422016-08-30 14:01:10 -0700290 aom_highbd_fdct8x8_c(input, output, stride);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700291 return;
292 }
293#endif // DCT_HIGH_BIT_DEPTH
294 // Interleave to do the multiply by constants which gets us into 32bits
295 {
296 const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
297 const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
298 const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
299 const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
300 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
301 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
302 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
303 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
304 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
305 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
306 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
307 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
308 // dct_const_round_shift
309 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
310 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
311 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
312 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
313 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
314 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
315 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
316 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
317 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
318 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
319 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
320 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
321 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
322 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
323 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
324 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
325 // Combine
326 res0 = _mm_packs_epi32(w0, w1);
327 res4 = _mm_packs_epi32(w2, w3);
328 res2 = _mm_packs_epi32(w4, w5);
329 res6 = _mm_packs_epi32(w6, w7);
330#if DCT_HIGH_BIT_DEPTH
331 overflow = check_epi16_overflow_x4(&res0, &res4, &res2, &res6);
332 if (overflow) {
Yaowu Xuf883b422016-08-30 14:01:10 -0700333 aom_highbd_fdct8x8_c(input, output, stride);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700334 return;
335 }
336#endif // DCT_HIGH_BIT_DEPTH
337 }
338 }
339 // Work on next four results
340 {
341 // Interleave to do the multiply by constants which gets us into 32bits
342 const __m128i d0 = _mm_unpacklo_epi16(q6, q5);
343 const __m128i d1 = _mm_unpackhi_epi16(q6, q5);
344 const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16);
345 const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16);
346 const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16);
347 const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16);
348 // dct_const_round_shift
349 const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING);
350 const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING);
351 const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING);
352 const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING);
353 const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS);
354 const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS);
355 const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS);
356 const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS);
357 // Combine
358 const __m128i r0 = _mm_packs_epi32(s0, s1);
359 const __m128i r1 = _mm_packs_epi32(s2, s3);
360#if DCT_HIGH_BIT_DEPTH
361 overflow = check_epi16_overflow_x2(&r0, &r1);
362 if (overflow) {
Yaowu Xuf883b422016-08-30 14:01:10 -0700363 aom_highbd_fdct8x8_c(input, output, stride);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700364 return;
365 }
366#endif // DCT_HIGH_BIT_DEPTH
367 {
368 // Add/subtract
369 const __m128i x0 = ADD_EPI16(q4, r0);
370 const __m128i x1 = SUB_EPI16(q4, r0);
371 const __m128i x2 = SUB_EPI16(q7, r1);
372 const __m128i x3 = ADD_EPI16(q7, r1);
373#if DCT_HIGH_BIT_DEPTH
374 overflow = check_epi16_overflow_x4(&x0, &x1, &x2, &x3);
375 if (overflow) {
Yaowu Xuf883b422016-08-30 14:01:10 -0700376 aom_highbd_fdct8x8_c(input, output, stride);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700377 return;
378 }
379#endif // DCT_HIGH_BIT_DEPTH
380 // Interleave to do the multiply by constants which gets us into 32bits
381 {
382 const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
383 const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
384 const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
385 const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
386 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
387 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
388 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
389 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
390 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
391 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
392 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
393 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
394 // dct_const_round_shift
395 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
396 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
397 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
398 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
399 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
400 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
401 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
402 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
403 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
404 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
405 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
406 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
407 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
408 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
409 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
410 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
411 // Combine
412 res1 = _mm_packs_epi32(w0, w1);
413 res7 = _mm_packs_epi32(w2, w3);
414 res5 = _mm_packs_epi32(w4, w5);
415 res3 = _mm_packs_epi32(w6, w7);
416#if DCT_HIGH_BIT_DEPTH
417 overflow = check_epi16_overflow_x4(&res1, &res7, &res5, &res3);
418 if (overflow) {
Yaowu Xuf883b422016-08-30 14:01:10 -0700419 aom_highbd_fdct8x8_c(input, output, stride);
Yaowu Xuc27fc142016-08-22 16:08:15 -0700420 return;
421 }
422#endif // DCT_HIGH_BIT_DEPTH
423 }
424 }
425 }
426 // Transpose the 8x8.
427 {
428 // 00 01 02 03 04 05 06 07
429 // 10 11 12 13 14 15 16 17
430 // 20 21 22 23 24 25 26 27
431 // 30 31 32 33 34 35 36 37
432 // 40 41 42 43 44 45 46 47
433 // 50 51 52 53 54 55 56 57
434 // 60 61 62 63 64 65 66 67
435 // 70 71 72 73 74 75 76 77
436 const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
437 const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3);
438 const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1);
439 const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3);
440 const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5);
441 const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7);
442 const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5);
443 const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7);
444 // 00 10 01 11 02 12 03 13
445 // 20 30 21 31 22 32 23 33
446 // 04 14 05 15 06 16 07 17
447 // 24 34 25 35 26 36 27 37
448 // 40 50 41 51 42 52 43 53
449 // 60 70 61 71 62 72 63 73
450 // 54 54 55 55 56 56 57 57
451 // 64 74 65 75 66 76 67 77
452 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
453 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
454 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
455 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
456 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
457 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
458 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
459 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
460 // 00 10 20 30 01 11 21 31
461 // 40 50 60 70 41 51 61 71
462 // 02 12 22 32 03 13 23 33
463 // 42 52 62 72 43 53 63 73
464 // 04 14 24 34 05 15 21 36
465 // 44 54 64 74 45 55 61 76
466 // 06 16 26 36 07 17 27 37
467 // 46 56 66 76 47 57 67 77
468 in0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
469 in1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
470 in2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
471 in3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
472 in4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
473 in5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
474 in6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
475 in7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
476 // 00 10 20 30 40 50 60 70
477 // 01 11 21 31 41 51 61 71
478 // 02 12 22 32 42 52 62 72
479 // 03 13 23 33 43 53 63 73
480 // 04 14 24 34 44 54 64 74
481 // 05 15 25 35 45 55 65 75
482 // 06 16 26 36 46 56 66 76
483 // 07 17 27 37 47 57 67 77
484 }
485 }
486 // Post-condition output and store it
487 {
488 // Post-condition (division by two)
489 // division of two 16 bits signed numbers using shifts
490 // n / 2 = (n - (n >> 15)) >> 1
491 const __m128i sign_in0 = _mm_srai_epi16(in0, 15);
492 const __m128i sign_in1 = _mm_srai_epi16(in1, 15);
493 const __m128i sign_in2 = _mm_srai_epi16(in2, 15);
494 const __m128i sign_in3 = _mm_srai_epi16(in3, 15);
495 const __m128i sign_in4 = _mm_srai_epi16(in4, 15);
496 const __m128i sign_in5 = _mm_srai_epi16(in5, 15);
497 const __m128i sign_in6 = _mm_srai_epi16(in6, 15);
498 const __m128i sign_in7 = _mm_srai_epi16(in7, 15);
499 in0 = _mm_sub_epi16(in0, sign_in0);
500 in1 = _mm_sub_epi16(in1, sign_in1);
501 in2 = _mm_sub_epi16(in2, sign_in2);
502 in3 = _mm_sub_epi16(in3, sign_in3);
503 in4 = _mm_sub_epi16(in4, sign_in4);
504 in5 = _mm_sub_epi16(in5, sign_in5);
505 in6 = _mm_sub_epi16(in6, sign_in6);
506 in7 = _mm_sub_epi16(in7, sign_in7);
507 in0 = _mm_srai_epi16(in0, 1);
508 in1 = _mm_srai_epi16(in1, 1);
509 in2 = _mm_srai_epi16(in2, 1);
510 in3 = _mm_srai_epi16(in3, 1);
511 in4 = _mm_srai_epi16(in4, 1);
512 in5 = _mm_srai_epi16(in5, 1);
513 in6 = _mm_srai_epi16(in6, 1);
514 in7 = _mm_srai_epi16(in7, 1);
515 // store results
516 store_output(&in0, (output + 0 * 8));
517 store_output(&in1, (output + 1 * 8));
518 store_output(&in2, (output + 2 * 8));
519 store_output(&in3, (output + 3 * 8));
520 store_output(&in4, (output + 4 * 8));
521 store_output(&in5, (output + 5 * 8));
522 store_output(&in6, (output + 6 * 8));
523 store_output(&in7, (output + 7 * 8));
524 }
525}
James Zern0e7ea552023-03-28 15:19:00 -0700526#endif // CONFIG_INTERNAL_STATS
Yaowu Xuc27fc142016-08-22 16:08:15 -0700527
Yaowu Xuc27fc142016-08-22 16:08:15 -0700528#undef ADD_EPI16
529#undef SUB_EPI16