/*
 * Copyright (c) 2017, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <tmmintrin.h>
#include <immintrin.h>
#include "./av1_rtcd.h"
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "av1/common/daala_tx.h"
#include "av1/common/daala_inv_txfm.h"
#include "av1/common/idct.h"

#if CONFIG_DAALA_TX

static INLINE __m128i od_unbiased_rshift1_epi16(__m128i a) {
  return _mm_srai_epi16(_mm_add_epi16(_mm_srli_epi16(a, 15), a), 1);
}

static INLINE __m256i od_mm256_unbiased_rshift1_epi16(__m256i a) {
  return _mm256_srai_epi16(_mm256_add_epi16(_mm256_srli_epi16(a, 15), a), 1);
}

static INLINE __m256i od_mm256_unbiased_rshift1_epi32(__m256i a) {
  return _mm256_srai_epi32(_mm256_add_epi32(_mm256_srli_epi32(a, 31), a), 1);
}

static INLINE __m128i od_avg_epi16(__m128i a, __m128i b) {
  __m128i sign_bit;
  /*x86 only provides an unsigned PAVGW with a bias (ARM is better here).
    We emulate a signed one by adding an offset to convert to unsigned and
    back. We use XOR instead of addition/subtraction because it dispatches
    better on older processors.*/
  sign_bit = _mm_set1_epi16(0x8000);
  return _mm_xor_si128(
      _mm_avg_epu16(_mm_xor_si128(a, sign_bit), _mm_xor_si128(b, sign_bit)),
      sign_bit);
}

static INLINE __m256i od_mm256_avg_epi16(__m256i a, __m256i b) {
  __m256i sign_bit;
  sign_bit = _mm256_set1_epi16(0x8000);
  return _mm256_xor_si256(_mm256_avg_epu16(_mm256_xor_si256(a, sign_bit),
                                           _mm256_xor_si256(b, sign_bit)),
                          sign_bit);
}

static INLINE __m256i od_mm256_avg_epi32(__m256i a, __m256i b) {
  __m256i neg1;
  /* It's cheaper to generate -1's than 1's. */
  neg1 = _mm256_set1_epi64x(-1);
  /* There is no corresponding PAVGD, but we are not in danger of overflowing
     a 32-bit register. */
  return _mm256_srai_epi32(_mm256_add_epi32(a, _mm256_sub_epi32(b, neg1)), 1);
}

/*Like the above, but does (a - b + 1) >> 1 instead.*/
static INLINE __m128i od_hrsub_epi16(__m128i a, __m128i b) {
  __m128i sign_bit;
  sign_bit = _mm_set1_epi16(0x8000);
  return _mm_xor_si128(
      _mm_avg_epu16(_mm_xor_si128(a, sign_bit), _mm_sub_epi16(sign_bit, b)),
      sign_bit);
}

static INLINE __m256i od_mm256_hrsub_epi16(__m256i a, __m256i b) {
  __m256i sign_bit;
  sign_bit = _mm256_set1_epi16(0x8000);
  return _mm256_xor_si256(_mm256_avg_epu16(_mm256_xor_si256(a, sign_bit),
                                           _mm256_sub_epi16(sign_bit, b)),
                          sign_bit);
}

static INLINE __m256i od_mm256_hrsub_epi32(__m256i a, __m256i b) {
  __m256i neg1;
  /* It's cheaper to generate -1's than 1's. */
  neg1 = _mm256_set1_epi64x(-1);
  /* There is no corresponding PAVGD, but we are not in danger of overflowing
     a 32-bit register. */
  return _mm256_srai_epi32(_mm256_sub_epi32(a, _mm256_add_epi32(b, neg1)), 1);
}

static INLINE void od_swap_si128(__m128i *q0, __m128i *q1) {
  __m128i t;
  t = *q0;
  *q0 = *q1;
  *q1 = t;
}

static INLINE void od_mm256_swap_si256(__m256i *q0, __m256i *q1) {
  __m256i t;
  t = *q0;
  *q0 = *q1;
  *q1 = t;
}

static INLINE __m128i od_mulhrs_epi16(__m128i a, int16_t b) {
  return _mm_mulhrs_epi16(a, _mm_set1_epi16(b));
}

static INLINE __m128i od_mul_epi16(__m128i a, int32_t b, int r) {
  int32_t b_q15;
  b_q15 = b << (15 - r);
  /* b and r are in all cases compile-time constants, so these branches
     disappear when this function gets inlined. */
  if (b_q15 > 32767) {
    return _mm_add_epi16(a, od_mulhrs_epi16(a, (int16_t)(b_q15 - 32768)));
  } else if (b_q15 < -32767) {
    return _mm_sub_epi16(od_mulhrs_epi16(a, (int16_t)(32768 + b_q15)), a);
  } else {
    return od_mulhrs_epi16(a, b_q15);
  }
}

static INLINE __m256i od_mm256_mulhrs_epi16(__m256i a, int16_t b) {
  return _mm256_mulhrs_epi16(a, _mm256_set1_epi16(b));
}

static INLINE __m256i od_mm256_mul_epi16(__m256i a, int32_t b, int r) {
  int32_t b_q15;
  b_q15 = b << (15 - r);
  /* b and r are in all cases compile-time constants, so these branches
     disappear when this function gets inlined. */
  if (b_q15 > 32767) {
    return _mm256_add_epi16(a,
                            od_mm256_mulhrs_epi16(a, (int16_t)(b_q15 - 32768)));
  } else if (b_q15 < -32767) {
    return _mm256_sub_epi16(od_mm256_mulhrs_epi16(a, (int16_t)(32768 + b_q15)),
                            a);
  } else {
    return od_mm256_mulhrs_epi16(a, b_q15);
  }
}

static INLINE __m256i od_mm256_mul_epi32(__m256i a, int32_t b, int r) {
  __m256i neg1;
  /* It's cheaper to generate -1's than 1's. */
  neg1 = _mm256_set1_epi64x(-1);
  /* There's no 32-bit version of PMULHRSW on x86 like there is on ARM .*/
  a = _mm256_mullo_epi32(a, _mm256_set1_epi32(b));
  a = _mm256_srai_epi32(a, r - 1);
  a = _mm256_sub_epi32(a, neg1);
  return _mm256_srai_epi32(a, 1);
}

static INLINE __m128i od_hbd_max_epi16(int bd) {
  return _mm_set1_epi16((1 << bd) - 1);
}

static INLINE __m256i od_mm256_hbd_max_epi16(int bd) {
  return _mm256_set1_epi16((1 << bd) - 1);
}

static INLINE __m128i od_hbd_clamp_epi16(__m128i a, __m128i max) {
  return _mm_max_epi16(_mm_setzero_si128(), _mm_min_epi16(a, max));
}

static INLINE __m256i od_mm256_hbd_clamp_epi16(__m256i a, __m256i max) {
  return _mm256_max_epi16(_mm256_setzero_si256(), _mm256_min_epi16(a, max));
}

/* Loads a 4x4 buffer of 32-bit values into four SSE registers. */
static INLINE void od_load_buffer_4x4_epi32(__m128i *q0, __m128i *q1,
                                            __m128i *q2, __m128i *q3,
                                            const tran_low_t *in) {
  *q0 = _mm_loadu_si128((const __m128i *)in + 0);
  *q1 = _mm_loadu_si128((const __m128i *)in + 1);
  *q2 = _mm_loadu_si128((const __m128i *)in + 2);
  *q3 = _mm_loadu_si128((const __m128i *)in + 3);
}

/* Loads a 4x4 buffer of 16-bit values into four SSE registers. */
static INLINE void od_load_buffer_4x4_epi16(__m128i *q0, __m128i *q1,
                                            __m128i *q2, __m128i *q3,
                                            const int16_t *in) {
  *q0 = _mm_loadu_si128((const __m128i *)in + 0);
  *q1 = _mm_unpackhi_epi64(*q0, *q0);
  *q2 = _mm_loadu_si128((const __m128i *)in + 1);
  *q3 = _mm_unpackhi_epi64(*q2, *q2);
}

/* Loads an 8x4 buffer of 16-bit values into four SSE registers. */
static INLINE void od_load_buffer_8x4_epi16(__m128i *q0, __m128i *q1,
                                            __m128i *q2, __m128i *q3,
                                            const int16_t *in, int in_stride) {
  *q0 = _mm_loadu_si128((const __m128i *)(in + 0 * in_stride));
  *q1 = _mm_loadu_si128((const __m128i *)(in + 1 * in_stride));
  *q2 = _mm_loadu_si128((const __m128i *)(in + 2 * in_stride));
  *q3 = _mm_loadu_si128((const __m128i *)(in + 3 * in_stride));
}

/* Loads an 8x4 buffer of 32-bit values and packs them into 16-bit values in
   four SSE registers. */
static INLINE void od_load_pack_buffer_8x4_epi32(__m128i *r0, __m128i *r1,
                                                 __m128i *r2, __m128i *r3,
                                                 const tran_low_t *in) {
  __m128i r4;
  __m128i r5;
  __m128i r6;
  __m128i r7;
  *r0 = _mm_loadu_si128((const __m128i *)in + 0);
  r4 = _mm_loadu_si128((const __m128i *)in + 1);
  *r1 = _mm_loadu_si128((const __m128i *)in + 2);
  r5 = _mm_loadu_si128((const __m128i *)in + 3);
  *r2 = _mm_loadu_si128((const __m128i *)in + 4);
  r6 = _mm_loadu_si128((const __m128i *)in + 5);
  *r3 = _mm_loadu_si128((const __m128i *)in + 6);
  r7 = _mm_loadu_si128((const __m128i *)in + 7);
  *r0 = _mm_packs_epi32(*r0, r4);
  *r1 = _mm_packs_epi32(*r1, r5);
  *r2 = _mm_packs_epi32(*r2, r6);
  *r3 = _mm_packs_epi32(*r3, r7);
}

/* Loads an 8x4 buffer of 32-bit values into four AVX registers. */
static INLINE void od_load_buffer_8x4_epi32(__m256i *r0, __m256i *r1,
                                            __m256i *r2, __m256i *r3,
                                            const tran_low_t *in) {
  *r0 = _mm256_loadu_si256((const __m256i *)in + 0);
  *r1 = _mm256_loadu_si256((const __m256i *)in + 1);
  *r2 = _mm256_loadu_si256((const __m256i *)in + 2);
  *r3 = _mm256_loadu_si256((const __m256i *)in + 3);
}

/* Loads a 16x4 buffer of 16-bit values into four AVX registers. */
static INLINE void od_load_buffer_16x4_epi16(__m256i *r0, __m256i *r1,
                                             __m256i *r2, __m256i *r3,
                                             const int16_t *in, int in_stride) {
  *r0 = _mm256_loadu_si256((const __m256i *)(in + 0 * in_stride));
  *r1 = _mm256_loadu_si256((const __m256i *)(in + 1 * in_stride));
  *r2 = _mm256_loadu_si256((const __m256i *)(in + 2 * in_stride));
  *r3 = _mm256_loadu_si256((const __m256i *)(in + 3 * in_stride));
}

/* Stores a 4x4 buffer of 16-bit values from two SSE registers.
   Each register holds two rows of values. */
static INLINE void od_store_buffer_4x4_epi16(int16_t *out, __m128i q0,
                                             __m128i q1) {
  _mm_storeu_si128((__m128i *)out + 0, q0);
  _mm_storeu_si128((__m128i *)out + 1, q1);
}

/* Stores a 4x8 buffer of 16-bit values from four SSE registers.
   Each register holds two rows of values. */
static INLINE void od_store_buffer_4x8_epi16(int16_t *out, __m128i q0,
                                             __m128i q1, __m128i q2,
                                             __m128i q3) {
  _mm_storeu_si128((__m128i *)out + 0, q0);
  _mm_storeu_si128((__m128i *)out + 1, q1);
  _mm_storeu_si128((__m128i *)out + 2, q2);
  _mm_storeu_si128((__m128i *)out + 3, q3);
}

static INLINE void od_store_buffer_2x16_epi16(int16_t *out, __m256i r0,
                                              __m256i r1) {
  _mm256_storeu_si256((__m256i *)out + 0, r0);
  _mm256_storeu_si256((__m256i *)out + 1, r1);
}

/* Loads a 4x4 buffer of 16-bit values, adds a 4x4 block of 16-bit values to
   them, clamps to high bit depth, and stores the sum back. */
static INLINE void od_add_store_buffer_hbd_4x4_epi16(void *output_pixels,
                                                     int output_stride,
                                                     __m128i q0, __m128i q1,
                                                     __m128i q2, __m128i q3,
                                                     int bd) {
  uint16_t *output_pixels16;
  __m128i p0;
  __m128i p1;
  __m128i p2;
  __m128i p3;
  __m128i max;
  __m128i round;
  int downshift;
  output_pixels16 = CONVERT_TO_SHORTPTR(output_pixels);
  max = od_hbd_max_epi16(bd);
  downshift = TX_COEFF_DEPTH - bd;
  round = _mm_set1_epi16((1 << downshift) >> 1);
  p0 = _mm_loadl_epi64((const __m128i *)(output_pixels16 + 0 * output_stride));
  p1 = _mm_loadl_epi64((const __m128i *)(output_pixels16 + 1 * output_stride));
  p2 = _mm_loadl_epi64((const __m128i *)(output_pixels16 + 2 * output_stride));
  p3 = _mm_loadl_epi64((const __m128i *)(output_pixels16 + 3 * output_stride));
  q0 = _mm_srai_epi16(_mm_add_epi16(q0, round), downshift);
  q1 = _mm_srai_epi16(_mm_add_epi16(q1, round), downshift);
  q2 = _mm_srai_epi16(_mm_add_epi16(q2, round), downshift);
  q3 = _mm_srai_epi16(_mm_add_epi16(q3, round), downshift);
  p0 = od_hbd_clamp_epi16(_mm_add_epi16(p0, q0), max);
  p1 = od_hbd_clamp_epi16(_mm_add_epi16(p1, q1), max);
  p2 = od_hbd_clamp_epi16(_mm_add_epi16(p2, q2), max);
  p3 = od_hbd_clamp_epi16(_mm_add_epi16(p3, q3), max);
  _mm_storel_epi64((__m128i *)(output_pixels16 + 0 * output_stride), p0);
  _mm_storel_epi64((__m128i *)(output_pixels16 + 1 * output_stride), p1);
  _mm_storel_epi64((__m128i *)(output_pixels16 + 2 * output_stride), p2);
  _mm_storel_epi64((__m128i *)(output_pixels16 + 3 * output_stride), p3);
}

/* Loads an 8x4 buffer of 16-bit values, adds a 8x4 block of 16-bit values to
   them, clamps to the high bit depth max, and stores the sum back. */
static INLINE void od_add_store_buffer_hbd_8x4_epi16(void *output_pixels,
                                                     int output_stride,
                                                     __m128i q0, __m128i q1,
                                                     __m128i q2, __m128i q3,
                                                     int bd) {
  uint16_t *output_pixels16;
  __m128i p0;
  __m128i p1;
  __m128i p2;
  __m128i p3;
  __m128i max;
  __m128i round;
  int downshift;
  output_pixels16 = CONVERT_TO_SHORTPTR(output_pixels);
  max = od_hbd_max_epi16(bd);
  downshift = TX_COEFF_DEPTH - bd;
  round = _mm_set1_epi16((1 << downshift) >> 1);
  p0 = _mm_loadu_si128((const __m128i *)(output_pixels16 + 0 * output_stride));
  p1 = _mm_loadu_si128((const __m128i *)(output_pixels16 + 1 * output_stride));
  p2 = _mm_loadu_si128((const __m128i *)(output_pixels16 + 2 * output_stride));
  p3 = _mm_loadu_si128((const __m128i *)(output_pixels16 + 3 * output_stride));
  q0 = _mm_srai_epi16(_mm_add_epi16(q0, round), downshift);
  q1 = _mm_srai_epi16(_mm_add_epi16(q1, round), downshift);
  q2 = _mm_srai_epi16(_mm_add_epi16(q2, round), downshift);
  q3 = _mm_srai_epi16(_mm_add_epi16(q3, round), downshift);
  p0 = od_hbd_clamp_epi16(_mm_add_epi16(p0, q0), max);
  p1 = od_hbd_clamp_epi16(_mm_add_epi16(p1, q1), max);
  p2 = od_hbd_clamp_epi16(_mm_add_epi16(p2, q2), max);
  p3 = od_hbd_clamp_epi16(_mm_add_epi16(p3, q3), max);
  _mm_storeu_si128((__m128i *)(output_pixels16 + 0 * output_stride), p0);
  _mm_storeu_si128((__m128i *)(output_pixels16 + 1 * output_stride), p1);
  _mm_storeu_si128((__m128i *)(output_pixels16 + 2 * output_stride), p2);
  _mm_storeu_si128((__m128i *)(output_pixels16 + 3 * output_stride), p3);
}

static INLINE void od_add_store_buffer_hbd_16x4_epi16(void *output_pixels,
                                                      int output_stride,
                                                      __m256i r0, __m256i r1,
                                                      __m256i r2, __m256i r3,
                                                      int bd) {
  uint16_t *output_pixels16;
  __m256i p0;
  __m256i p1;
  __m256i p2;
  __m256i p3;
  __m256i max;
  __m256i round;
  int downshift;
  output_pixels16 = CONVERT_TO_SHORTPTR(output_pixels);
  max = od_mm256_hbd_max_epi16(bd);
  downshift = TX_COEFF_DEPTH - bd;
  round = _mm256_set1_epi16((1 << downshift) >> 1);
  p0 = _mm256_loadu_si256(
      (const __m256i *)(output_pixels16 + 0 * output_stride));
  p1 = _mm256_loadu_si256(
      (const __m256i *)(output_pixels16 + 1 * output_stride));
  p2 = _mm256_loadu_si256(
      (const __m256i *)(output_pixels16 + 2 * output_stride));
  p3 = _mm256_loadu_si256(
      (const __m256i *)(output_pixels16 + 3 * output_stride));
  r0 = _mm256_srai_epi16(_mm256_add_epi16(r0, round), downshift);
  r1 = _mm256_srai_epi16(_mm256_add_epi16(r1, round), downshift);
  r2 = _mm256_srai_epi16(_mm256_add_epi16(r2, round), downshift);
  r3 = _mm256_srai_epi16(_mm256_add_epi16(r3, round), downshift);
  p0 = od_mm256_hbd_clamp_epi16(_mm256_add_epi16(p0, r0), max);
  p1 = od_mm256_hbd_clamp_epi16(_mm256_add_epi16(p1, r1), max);
  p2 = od_mm256_hbd_clamp_epi16(_mm256_add_epi16(p2, r2), max);
  p3 = od_mm256_hbd_clamp_epi16(_mm256_add_epi16(p3, r3), max);
  _mm256_storeu_si256((__m256i *)(output_pixels16 + 0 * output_stride), p0);
  _mm256_storeu_si256((__m256i *)(output_pixels16 + 1 * output_stride), p1);
  _mm256_storeu_si256((__m256i *)(output_pixels16 + 2 * output_stride), p2);
  _mm256_storeu_si256((__m256i *)(output_pixels16 + 3 * output_stride), p3);
}

static INLINE void od_transpose_pack4x4(__m128i *q0, __m128i *q1, __m128i *q2,
                                        __m128i *q3) {
  __m128i a;
  __m128i b;
  __m128i c;
  __m128i d;
  /* Input:
     q0: q30 q20 q10 q00
     q1: q31 q21 q11 q01
     q2: q32 q22 q12 q02
     q3: q33 q23 q13 q03
  */
  /* a: q32 q22 q12 q02 q30 q20 q10 q00 */
  a = _mm_packs_epi32(*q0, *q2);
  /* b: q33 q23 q13 q03 q31 q21 q11 q01 */
  b = _mm_packs_epi32(*q1, *q3);
  /* c: q31 q30 q21 q20 q11 q10 q01 q00 */
  c = _mm_unpacklo_epi16(a, b);
  /* d: q33 q32 q23 q22 q13 q12 q03 q02 */
  d = _mm_unpackhi_epi16(a, b);
  /* We don't care about the contents of the high half of each register. */
  /* q0: q13 q12 q11 q10 [q03 q02 q01 q00] */
  *q0 = _mm_unpacklo_epi32(c, d);
  /* q1: q13 q12 q11 q10 [q13 q12 q11 q10] */
  *q1 = _mm_unpackhi_epi64(*q0, *q0);
  /* q2: q33 q32 q31 q30 [q23 q22 q21 q20] */
  *q2 = _mm_unpackhi_epi32(c, d);
  /* q3: q33 q32 q31 q30 [q33 q32 q31 q30] */
  *q3 = _mm_unpackhi_epi64(*q2, *q2);
}

static INLINE void od_transpose4x4(__m128i *q0, __m128i q1, __m128i *q2,
                                   __m128i q3) {
  __m128i a;
  __m128i b;
  /* Input:
     q0: ... ... ... ... q30 q20 q10 q00
     q1: ... ... ... ... q31 q21 q11 q01
     q2: ... ... ... ... q32 q22 q12 q02
     q3: ... ... ... ... q33 q23 q13 q03
  */
  /* a: q31 q30 q21 q20 q11 q10 q01 q00 */
  a = _mm_unpacklo_epi16(*q0, q1);
  /* b: q33 q32 q23 q22 q13 q12 q03 q02 */
  b = _mm_unpacklo_epi16(*q2, q3);
  /* q0: q13 q12 q11 q10 | q03 q02 q01 q00 */
  *q0 = _mm_unpacklo_epi32(a, b);
  /* q2: q33 q32 q31 q30 | q23 q22 q21 q20 */
  *q2 = _mm_unpackhi_epi32(a, b);
}

static inline void od_transpose4x8(__m128i *r0, __m128i r1, __m128i *r2,
                                   __m128i r3, __m128i *r4, __m128i r5,
                                   __m128i *r6, __m128i r7) {
  __m128i a;
  __m128i b;
  /* Input:
     q0: ... ... ... ... q30 q20 q10 q00
     q1: ... ... ... ... q31 q21 q11 q01
     q2: ... ... ... ... q32 q22 q12 q02
     q3: ... ... ... ... q33 q23 q13 q03
     q4: ... ... ... ... q34 q24 q14 q04
     q5: ... ... ... ... q35 q25 q15 q05
     q6: ... ... ... ... q36 q26 q16 q06
     q7: ... ... ... ... q37 q27 q17 q07
  */
  /* r0: r13 r12 11 r10 r03 r02 r01 r00
     r2: r33 r32 31 r30 r23 r22 r21 r20 */
  od_transpose4x4(r0, r1, r2, r3);
  /* r4: r17 r16 15 r14 r07 r06 r05 r04
     r6: r37 r36 35 r34 r27 r26 r25 r24 */
  od_transpose4x4(r4, r5, r6, r7);
  a = *r0;
  b = *r2;
  /* r0: r07 r06 r05 r04 r04 r02 r01 r00 */
  *r0 = _mm_unpacklo_epi64(a, *r4);
  /* r2: r17 r16 r15 r14 r14 r12 r11 r10 */
  *r2 = _mm_unpackhi_epi64(a, *r4);
  /* r4: r27 r26 r25 r24 r24 r22 r21 r20 */
  *r4 = _mm_unpacklo_epi64(b, *r6);
  /* r6: r37 r36 r35 r34 r34 r32 r31 r30 */
  *r6 = _mm_unpackhi_epi64(b, *r6);
}

static INLINE void od_transpose8x4(__m128i *q0, __m128i *q1, __m128i *q2,
                                   __m128i *q3) {
  __m128i a;
  __m128i b;
  __m128i c;
  __m128i d;
  /* Input:
     q0: q07 q06 q05 q04 q03 q02 q01 q00
     q1: q17 q16 q15 q14 q13 q12 q11 q10
     q2: q27 q26 q25 q24 q23 q22 q21 q20
     q3: q37 q36 q35 q34 q33 q32 q31 q30
  */
  /* a: q13 q03 q12 q02 q11 q01 q10 q00 */
  a = _mm_unpacklo_epi16(*q0, *q1);
  /* b: q17 q07 q16 q06 q15 q05 q14 q04 */
  b = _mm_unpackhi_epi16(*q0, *q1);
  /* c: q33 q23 q32 q22 q31 q21 q30 q20 */
  c = _mm_unpacklo_epi16(*q2, *q3);
  /* d: q37 q27 q36 q26 q35 q25 q34 q24 */
  d = _mm_unpackhi_epi16(*q2, *q3);
  /* q0: q31 q21 q11 q01 | q30 q20 q10 q00 */
  *q0 = _mm_unpacklo_epi32(a, c);
  /* q1: q33 q23 q13 q03 | q32 q22 q12 q02 */
  *q1 = _mm_unpackhi_epi32(a, c);
  /* q2: q35 q25 q15 q05 | q34 q24 q14 q04 */
  *q2 = _mm_unpacklo_epi32(b, d);
  /* q3: q37 q27 q17 q07 | q36 q26 q16 q06 */
  *q3 = _mm_unpackhi_epi32(b, d);
}

static INLINE void od_transpose_pack4x8(__m128i *q0, __m128i *q1, __m128i *q2,
                                        __m128i *q3, __m128i q4, __m128i q5,
                                        __m128i q6, __m128i q7) {
  __m128i a;
  __m128i b;
  __m128i c;
  __m128i d;
  /* Input:
     q0: q30 q20 q10 q00
     q1: q31 q21 q11 q01
     q2: q32 q22 q12 q02
     q3: q33 q23 q13 q03
     q4: q34 q24 q14 q04
     q5: q35 q25 q15 q05
     q6: q36 q26 q16 q06
     q7: q37 q27 q17 q07
  */
  /* a: q34 q24 q14 q04 q30 q20 q10 q00 */
  a = _mm_packs_epi32(*q0, q4);
  /* b: q35 q25 q15 q05 q31 q21 q11 q01 */
  b = _mm_packs_epi32(*q1, q5);
  /* c: q36 q26 q16 q06 q32 q22 q12 q02 */
  c = _mm_packs_epi32(*q2, q6);
  /* d: q37 q27 q17 q07 q33 q23 q13 q03 */
  d = _mm_packs_epi32(*q3, q7);
  /* a: q13 q12 q11 q10 q03 q02 q01 q00
     b: q33 q32 q31 q30 q33 q22 q21 q20
     c: q53 q52 q51 q50 q43 q42 q41 q40
     d: q73 q72 q71 q70 q63 q62 q61 q60 */
  od_transpose8x4(&a, &b, &c, &d);
  /* q0: q07 q06 q05 q04 q03 q02 q01 q00 */
  *q0 = _mm_unpacklo_epi64(a, c);
  /* q1: q17 q16 q15 q14 q13 q12 q11 q10 */
  *q1 = _mm_unpackhi_epi64(a, c);
  /* q2: q27 q26 q25 q24 q23 q22 q21 q20 */
  *q2 = _mm_unpacklo_epi64(b, d);
  /* q3: q37 q36 q35 q34 q33 q32 q31 q30 */
  *q3 = _mm_unpackhi_epi64(b, d);
}

static INLINE void od_transpose_pack8x4(__m128i *r0, __m128i *r1, __m128i *r2,
                                        __m128i *r3, __m128i *r4, __m128i *r5,
                                        __m128i *r6, __m128i *r7) {
  /* Input:
     r1: r07 r06 r05 r04  r0: r03 r02 r01 r00
     r3: r17 r16 r15 r14  r2: r13 r12 r11 r10
     r5: r27 r26 r25 r24  r4: r23 r22 r21 r20
     r7: r37 r36 r35 r34  r6: r33 r32 r31 r30
  */
  /* r0: r07 r06 r05 r04 r03 r02 r01 r00 */
  *r0 = _mm_packs_epi32(*r0, *r1);
  /* r2: r17 r16 r15 r14 r13 r12 r11 r10 */
  *r2 = _mm_packs_epi32(*r2, *r3);
  /* r4: r27 r26 r25 r24 r23 r22 r21 r20 */
  *r4 = _mm_packs_epi32(*r4, *r5);
  /* r6: r37 r36 r35 r34 r33 r32 r31 r30 */
  *r6 = _mm_packs_epi32(*r6, *r7);
  /* r0: r31 r21 r11 r01 [r30 r20 r10 r00]
     r2: r33 r23 r13 r03 [r32 r22 r12 r02]
     r4: r35 r25 r15 r05 [r34 r24 r14 r04]
     r6: r37 r27 r17 r07 [r36 r26 r16 r06] */
  od_transpose8x4(r0, r2, r4, r6);
  /* We don't care about the contents of the high half of each register. */
  /* r1: r31 r21 r11 r01 [r31 r21 r11 r01] */
  *r1 = _mm_unpackhi_epi64(*r0, *r0);
  /* r3: r33 r23 r13 r03 [r33 r23 r13 r03] */
  *r3 = _mm_unpackhi_epi64(*r2, *r2);
  /* r5: r35 r25 r15 r05 [r35 r25 r15 r05] */
  *r5 = _mm_unpackhi_epi64(*r4, *r4);
  /* r7: r37 r27 r17 r07 [r37 r27 r17 r07] */
  *r7 = _mm_unpackhi_epi64(*r6, *r6);
}

static INLINE void od_transpose8x8_epi16(__m128i *r0, __m128i *r1, __m128i *r2,
                                         __m128i *r3, __m128i *r4, __m128i *r5,
                                         __m128i *r6, __m128i *r7) {
  __m128i r8;
  /*8x8 transpose with only 1 temporary register that takes the rows in order
    and returns the columns in order. The compiler's own register allocator
    will probably screw this up, but that's no reason not to pretend we might
    be able to have nice things. This only matters when we port to pre-AVX
    instruction sets without 3-operand instructions.*/
  r8 = *r4;
  *r4 = _mm_unpacklo_epi16(*r4, *r5);
  r8 = _mm_unpackhi_epi16(r8, *r5);
  *r5 = *r0;
  *r0 = _mm_unpacklo_epi16(*r0, *r1);
  *r5 = _mm_unpackhi_epi16(*r5, *r1);
  *r1 = *r6;
  *r6 = _mm_unpacklo_epi16(*r6, *r7);
  *r1 = _mm_unpackhi_epi16(*r1, *r7);
  *r7 = *r2;
  *r2 = _mm_unpackhi_epi16(*r2, *r3);
  *r7 = _mm_unpacklo_epi16(*r7, *r3);
  *r3 = *r0;
  *r0 = _mm_unpacklo_epi32(*r0, *r7);
  *r3 = _mm_unpackhi_epi32(*r3, *r7);
  *r7 = *r5;
  *r5 = _mm_unpacklo_epi32(*r5, *r2);
  *r7 = _mm_unpackhi_epi32(*r7, *r2);
  *r2 = *r4;
  *r4 = _mm_unpackhi_epi32(*r4, *r6);
  *r2 = _mm_unpacklo_epi32(*r2, *r6);
  *r6 = r8;
  r8 = _mm_unpackhi_epi32(r8, *r1);
  *r6 = _mm_unpacklo_epi32(*r6, *r1);
  *r1 = *r0;
  *r0 = _mm_unpacklo_epi64(*r0, *r2);
  *r1 = _mm_unpackhi_epi64(*r1, *r2);
  *r2 = *r3;
  *r3 = _mm_unpackhi_epi64(*r3, *r4);
  *r2 = _mm_unpacklo_epi64(*r2, *r4);
  *r4 = *r5;
  *r5 = _mm_unpackhi_epi64(*r5, *r6);
  *r4 = _mm_unpacklo_epi64(*r4, *r6);
  *r6 = *r7;
  *r7 = _mm_unpackhi_epi64(*r7, r8);
  *r6 = _mm_unpacklo_epi64(*r6, r8);
}

static INLINE void od_transpose8x8_epi32(__m256i *r0, __m256i *r1, __m256i *r2,
                                         __m256i *r3, __m256i *r4, __m256i *r5,
                                         __m256i *r6, __m256i *r7) {
  __m256i a;
  __m256i b;
  __m256i c;
  __m256i d;
  __m256i e;
  __m256i f;
  __m256i g;
  __m256i h;
  __m256i x;
  __m256i y;
  a = _mm256_unpacklo_epi32(*r0, *r1);
  b = _mm256_unpacklo_epi32(*r2, *r3);
  c = _mm256_unpackhi_epi32(*r0, *r1);
  d = _mm256_unpackhi_epi32(*r2, *r3);
  e = _mm256_unpacklo_epi32(*r4, *r5);
  f = _mm256_unpacklo_epi32(*r6, *r7);
  g = _mm256_unpackhi_epi32(*r4, *r5);
  h = _mm256_unpackhi_epi32(*r6, *r7);
  x = _mm256_unpacklo_epi64(a, b);
  y = _mm256_unpacklo_epi64(e, f);
  *r0 = _mm256_permute2x128_si256(x, y, 0 | (2 << 4));
  *r4 = _mm256_permute2x128_si256(x, y, 1 | (3 << 4));
  x = _mm256_unpackhi_epi64(a, b);
  y = _mm256_unpackhi_epi64(e, f);
  *r1 = _mm256_permute2x128_si256(x, y, 0 | (2 << 4));
  *r5 = _mm256_permute2x128_si256(x, y, 1 | (3 << 4));
  x = _mm256_unpacklo_epi64(c, d);
  y = _mm256_unpacklo_epi64(g, h);
  *r2 = _mm256_permute2x128_si256(x, y, 0 | (2 << 4));
  *r6 = _mm256_permute2x128_si256(x, y, 1 | (3 << 4));
  x = _mm256_unpackhi_epi64(c, d);
  y = _mm256_unpackhi_epi64(g, h);
  *r3 = _mm256_permute2x128_si256(x, y, 0 | (2 << 4));
  *r7 = _mm256_permute2x128_si256(x, y, 1 | (3 << 4));
}

/* Packs two blocks of 4x8 32-bit words into 16-bit words and returns the
   transpose of each packed into the high and low halves of each register. */
static INLINE void od_transpose_pack4x8x2_epi32(__m256i *out0, __m256i *out1,
                                                __m256i *out2, __m256i *out3,
                                                __m256i rr0, __m256i rr1,
                                                __m256i rr2, __m256i rr3,
                                                __m256i rr4, __m256i rr5,
                                                __m256i rr6, __m256i rr7) {
  __m256i a;
  __m256i b;
  __m256i c;
  __m256i d;
  __m256i w;
  __m256i x;
  __m256i y;
  __m256i z;
  /* a: r47 r46 r45 r44 r07 r06 r05 r04 | r43 r42 r41 r40 r03 r02 r01 r00 */
  a = _mm256_packs_epi32(rr0, rr4);
  /* b: r57 r56 r55 r54 r17 r16 r15 r14 | r53 r52 r51 r50 r13 r12 r11 r10 */
  b = _mm256_packs_epi32(rr1, rr5);
  /* c: r67 r66 r65 r64 r27 r26 r25 r24 | r63 r62 r61 r60 r23 r22 r21 r20 */
  c = _mm256_packs_epi32(rr2, rr6);
  /* d: r77 r76 r75 r74 r37 r36 r35 r34 | r73 r72 r71 r70 r33 r32 r31 r30 */
  d = _mm256_packs_epi32(rr3, rr7);
  /* w: r17 r07 r16 r06 r15 r05 r14 r04 | r13 r03 r12 r02 r11 r01 r10 r00 */
  w = _mm256_unpacklo_epi16(a, b);
  /* x: r57 r47 r56 r46 r55 r45 r54 r44 | r53 r43 r52 r42 r51 r41 r50 r40 */
  x = _mm256_unpackhi_epi16(a, b);
  /* y: r37 r27 r36 r26 r35 r25 r34 r24 | r33 r23 r32 r22 r31 r21 r30 r20 */
  y = _mm256_unpacklo_epi16(c, d);
  /* z: r77 r67 r76 r66 r75 r65 r74 r64 | r73 r63 r72 r62 r71 r61 r70 r60 */
  z = _mm256_unpackhi_epi16(c, d);
  /* a: r35 r25 r15 r05 r34 r24 r14 r04 | r31 r21 r11 r01 r30 r20 r10 r00 */
  a = _mm256_unpacklo_epi32(w, y);
  /* b: r77 r67 r57 r47 r76 r66 r56 r46 | r33 r23 r13 r03 r32 r22 r12 r02 */
  b = _mm256_unpackhi_epi32(w, y);
  /* c: r75 r65 r55 r45 r74 r64 r54 r44 | r71 r61 r51 r41 r70 r60 r50 r40 */
  c = _mm256_unpacklo_epi32(x, z);
  /* d: r77 r67 r57 r47 r76 r66 r56 r46 | r73 r63 r53 r43 r72 r62 r52 r42 */
  d = _mm256_unpackhi_epi32(x, z);
  /* out0: r74 r64 r54 r44 r34 r24 r14 r04 | r70 r60 r50 r40 r30 r20 r10 r00 */
  *out0 = _mm256_unpacklo_epi64(a, c);
  /* out1: r75 r65 r55 r45 r35 r25 r15 r05 | r71 r61 r51 r41 r31 r21 r11 r01 */
  *out1 = _mm256_unpackhi_epi64(a, c);
  /* out2: r76 r66 r56 r46 r36 r26 r16 r06 | r72 r62 r52 r42 r32 r22 r12 r02 */
  *out2 = _mm256_unpacklo_epi64(b, d);
  /* out3: r77 r67 r57 r47 r37 r27 r17 r07 | r73 r63 r53 r43 r33 r23 r13 r03 */
  *out3 = _mm256_unpackhi_epi64(b, d);
}

static INLINE void od_transpose_pack8x8_epi32(__m256i *rr0, __m256i *rr1,
                                              __m256i *rr2, __m256i *rr3,
                                              __m256i rr4, __m256i rr5,
                                              __m256i rr6, __m256i rr7) {
  __m256i w;
  __m256i x;
  __m256i y;
  __m256i z;
  /* w: r74 r64 r54 r44 r34 r24 r14 r04 | r70 r60 r50 r40 r30 r20 r10 r00
     x: r75 r65 r55 r45 r35 r25 r15 r05 | r71 r61 r51 r41 r31 r21 r11 r01
     y: r76 r66 r56 r46 r36 r26 r16 r06 | r72 r62 r52 r42 r32 r22 r12 r02
     z: r77 r67 r57 r47 r37 r27 r17 r07 | r73 r63 r53 r43 r33 r23 r13 r03 */
  od_transpose_pack4x8x2_epi32(&w, &x, &y, &z, *rr0, *rr1, *rr2, *rr3, rr4, rr5,
                               rr6, rr7);
  /* rr0: r71 r61 r51 r41 r31 r21 r11 r01 | r70 r60 r50 r40 r30 r20 r10 r00 */
  *rr0 = _mm256_permute2x128_si256(w, x, 0 | (2 << 4));
  /* rr1: r73 r63 r53 r43 r33 r23 r13 r03 | r72 r62 r52 r42 r32 r22 r12 r02 */
  *rr1 = _mm256_permute2x128_si256(y, z, 0 | (2 << 4));
  /* rr2: r75 r65 r55 r45 r35 r25 r15 r05 r74 r64 r54 r44 r34 r24 r14 r04 */
  *rr2 = _mm256_permute2x128_si256(w, x, 1 | (3 << 4));
  /* rr3: r77 r67 r57 r47 r37 r27 r17 r07 r76 r66 r56 r46 r36 r26 r16 r06 */
  *rr3 = _mm256_permute2x128_si256(y, z, 1 | (3 << 4));
}

static INLINE void od_transpose_pack8x16_epi32(
    __m256i *ss0, __m256i *ss1, __m256i *ss2, __m256i *ss3, __m256i *ss4,
    __m256i *ss5, __m256i *ss6, __m256i *ss7, __m256i ss8, __m256i ss9,
    __m256i ssa, __m256i ssb, __m256i ssc, __m256i ssd, __m256i sse,
    __m256i ssf) {
  __m256i a;
  __m256i b;
  __m256i c;
  __m256i d;
  __m256i e;
  __m256i f;
  __m256i g;
  __m256i h;
  /* ss0: s74 s64 s54 s44 s34 s24 s14 s04 | s70 s60 s50 s40 s30 s20 s10 s00
     ss2: s75 s65 s55 s45 s35 s25 s15 s05 | s71 s61 s51 s41 s31 s21 s11 s01
     ss4: s76 s66 s56 s46 s36 s26 s16 s06 | s72 s62 s52 s42 s32 s22 s12 s02
     ss6: s77 s67 s57 s47 s37 s27 s17 s07 | s73 s63 s53 s43 s33 s23 s13 s03 */
  od_transpose_pack4x8x2_epi32(&a, &b, &c, &d, *ss0, *ss1, *ss2, *ss3, *ss4,
                               *ss5, *ss6, *ss7);
  /* ss8: sf4 se4 sd4 sc4 sb4 sa4 s94 s84 | sf0 se0 sd0 sc0 sb0 sa0 s90 s80
     ssa: sf5 se5 sd5 sc5 sb5 sa5 s95 s85 | sf1 se1 sd1 sc1 sb1 sa1 s91 s81
     ssc: sf6 se6 sd6 sc6 sb6 sa6 s96 s86 | sf2 se2 sd2 sc2 sb2 sa2 s92 s82
     sse: sf7 se7 sd7 sc7 sb7 sa7 s97 s87 | sf3 se3 sd3 sc3 sb3 sa3 s93 s83 */
  od_transpose_pack4x8x2_epi32(&e, &f, &g, &h, ss8, ss9, ssa, ssb, ssc, ssd,
                               sse, ssf);
  /* ss0: sf0 se0 sd0 sc0 sb0 sa0 s90 s80 | s70 s60 s50 s40 s30 s20 s10 s00 */
  *ss0 = _mm256_permute2x128_si256(a, e, 0 | (2 << 4));
  /* ss1: sf1 se1 sd1 sc1 sb1 sa1 s91 s81 | s71 s61 s51 s41 s31 s21 s11 s01 */
  *ss1 = _mm256_permute2x128_si256(b, f, 0 | (2 << 4));
  /* ss2: sf2 se2 sd2 sc2 sb2 sa2 s92 s82 | s72 s62 s52 s42 s32 s22 s12 s02 */
  *ss2 = _mm256_permute2x128_si256(c, g, 0 | (2 << 4));
  /* ss3: sf3 se3 sd3 sc3 sb3 sa3 s93 s83 | s73 s63 s53 s43 s33 s23 s13 s03 */
  *ss3 = _mm256_permute2x128_si256(d, h, 0 | (2 << 4));
  /* ss4: sf4 se4 sd4 sc4 sb4 sa4 s94 s84 | s74 s64 s54 s44 s34 s24 s14 s04 */
  *ss4 = _mm256_permute2x128_si256(a, e, 1 | (3 << 4));
  /* ss5: sf5 se5 sd5 sc5 sb5 sa5 s95 s85 | s75 s65 s55 s45 s35 s25 s15 s05 */
  *ss5 = _mm256_permute2x128_si256(b, f, 1 | (3 << 4));
  /* ss6: rf6 re6 rd6 rc6 rb6 ra6 r96 r82 | r76 r66 r56 r46 r36 r26 r16 r06 */
  *ss6 = _mm256_permute2x128_si256(c, g, 1 | (3 << 4));
  /* ss7: rf7 re7 rd7 rc7 rb7 ra7 r97 r87 | r77 r67 r57 r47 r37 r27 r17 r07 */
  *ss7 = _mm256_permute2x128_si256(d, h, 1 | (3 << 4));
}

#undef OD_KERNEL
#undef OD_WORD
#undef OD_REG
#undef OD_ADD
#undef OD_SUB
#undef OD_RSHIFT1
#undef OD_AVG
#undef OD_HRSUB
#undef OD_MUL
#undef OD_SWAP

/* Define 8-wide 16-bit SSSE3 kernels. */

#define OD_KERNEL kernel8
#define OD_WORD epi16
#define OD_REG __m128i
#define OD_ADD _mm_add_epi16
#define OD_SUB _mm_sub_epi16
#define OD_RSHIFT1 od_unbiased_rshift1_epi16
#define OD_AVG od_avg_epi16
#define OD_HRSUB od_hrsub_epi16
#define OD_MUL od_mul_epi16
#define OD_SWAP od_swap_si128

#include "av1/common/x86/daala_tx_kernels.h"

#undef OD_KERNEL
#undef OD_REG
#undef OD_ADD
#undef OD_SUB
#undef OD_RSHIFT1
#undef OD_AVG
#undef OD_HRSUB
#undef OD_MUL
#undef OD_SWAP

/* Define 16-wide 16-bit AVX2 kernels. */

#define OD_KERNEL kernel16
#define OD_REG __m256i
#define OD_ADD _mm256_add_epi16
#define OD_SUB _mm256_sub_epi16
#define OD_RSHIFT1 od_mm256_unbiased_rshift1_epi16
#define OD_AVG od_mm256_avg_epi16
#define OD_HRSUB od_mm256_hrsub_epi16
#define OD_MUL od_mm256_mul_epi16
#define OD_SWAP od_mm256_swap_si256

#include "av1/common/x86/daala_tx_kernels.h"  // NOLINT

/* Define 8-wide 32-bit AVX2 kernels. */

#undef OD_KERNEL
#undef OD_WORD
#undef OD_ADD
#undef OD_SUB
#undef OD_RSHIFT1
#undef OD_AVG
#undef OD_HRSUB
#undef OD_MUL

#define OD_KERNEL kernel8
#define OD_WORD epi32
#define OD_ADD _mm256_add_epi32
#define OD_SUB _mm256_sub_epi32
#define OD_RSHIFT1 od_mm256_unbiased_rshift1_epi32
#define OD_AVG od_mm256_avg_epi32
#define OD_HRSUB od_mm256_hrsub_epi32
#define OD_MUL od_mm256_mul_epi32

#include "av1/common/x86/daala_tx_kernels.h"  // NOLINT

static void od_row_iidtx_avx2(int16_t *out, int coeffs, const tran_low_t *in) {
  int c;
  /* The number of rows and number of columns are both multiples of 4, so the
     total number of coefficients should be a multiple of 16. */
  assert(!(coeffs & 0xF));
  /* TODO(any): Use AVX2 for larger block sizes. */
  for (c = 0; c < coeffs; c += 16) {
    __m128i q0;
    __m128i q1;
    __m128i q2;
    __m128i q3;
    od_load_buffer_4x4_epi32(&q0, &q1, &q2, &q3, in + c);
    q0 = _mm_packs_epi32(q0, q1);
    q2 = _mm_packs_epi32(q2, q3);
    od_store_buffer_4x4_epi16(out + c, q0, q2);
  }
}

static void od_col_iidtx_add_hbd_avx2(unsigned char *output_pixels,
                                      int output_stride, int rows, int cols,
                                      const int16_t *in, int bd) {
  __m128i q0;
  __m128i q1;
  __m128i q2;
  __m128i q3;
  if (cols <= 4) {
    uint16_t *output_pixels16;
    __m128i p0;
    __m128i p1;
    __m128i p2;
    __m128i p3;
    __m128i max;
    __m128i round;
    int downshift;
    int hr;
    output_pixels16 = CONVERT_TO_SHORTPTR(output_pixels);
    max = od_hbd_max_epi16(bd);
    downshift = TX_COEFF_DEPTH - bd;
    round = _mm_set1_epi16((1 << downshift) >> 1);
    /* Here hr counts half the number of rows, to simplify address calculations
       when loading two rows of coefficients at once. */
    for (hr = 0; 2 * hr < rows; hr += 2) {
      q0 = _mm_loadu_si128((const __m128i *)in + hr + 0);
      q2 = _mm_loadu_si128((const __m128i *)in + hr + 1);
      p0 = _mm_loadl_epi64(
          (const __m128i *)(output_pixels16 + (2 * hr + 0) * output_stride));
      p1 = _mm_loadl_epi64(
          (const __m128i *)(output_pixels16 + (2 * hr + 1) * output_stride));
      p2 = _mm_loadl_epi64(
          (const __m128i *)(output_pixels16 + (2 * hr + 2) * output_stride));
      p3 = _mm_loadl_epi64(
          (const __m128i *)(output_pixels16 + (2 * hr + 3) * output_stride));
      q0 = _mm_srai_epi16(_mm_add_epi16(q0, round), downshift);
      q2 = _mm_srai_epi16(_mm_add_epi16(q2, round), downshift);
      q1 = _mm_unpackhi_epi64(q0, q0);
      q3 = _mm_unpackhi_epi64(q2, q2);
      p0 = od_hbd_clamp_epi16(_mm_add_epi16(p0, q0), max);
      p1 = od_hbd_clamp_epi16(_mm_add_epi16(p1, q1), max);
      p2 = od_hbd_clamp_epi16(_mm_add_epi16(p2, q2), max);
      p3 = od_hbd_clamp_epi16(_mm_add_epi16(p3, q3), max);
      _mm_storel_epi64(
          (__m128i *)(output_pixels16 + (2 * hr + 0) * output_stride), p0);
      _mm_storel_epi64(
          (__m128i *)(output_pixels16 + (2 * hr + 1) * output_stride), p1);
      _mm_storel_epi64(
          (__m128i *)(output_pixels16 + (2 * hr + 2) * output_stride), p2);
      _mm_storel_epi64(
          (__m128i *)(output_pixels16 + (2 * hr + 3) * output_stride), p3);
    }
  } else {
    int r;
    for (r = 0; r < rows; r += 4) {
      int c;
      /* TODO(any): Use AVX2 for larger column counts. */
      for (c = 0; c < cols; c += 8) {
        od_load_buffer_8x4_epi16(&q0, &q1, &q2, &q3, in + r * cols + c, cols);
        od_add_store_buffer_hbd_8x4_epi16(output_pixels + r * output_stride + c,
                                          output_stride, q0, q1, q2, q3, bd);
      }
    }
  }
}

typedef void (*od_tx4_kernel8_epi16)(__m128i *q0, __m128i *q2, __m128i *q1,
                                     __m128i *q3);

static void od_row_tx4_avx2(int16_t *out, int rows, const tran_low_t *in,
                            od_tx4_kernel8_epi16 kernel8) {
  __m128i q0;
  __m128i q1;
  __m128i q2;
  __m128i q3;
  if (rows <= 4) {
    od_load_buffer_4x4_epi32(&q0, &q1, &q2, &q3, in);
    /*TODO(any): Merge this transpose with coefficient scanning.*/
    od_transpose_pack4x4(&q0, &q1, &q2, &q3);
    kernel8(&q0, &q1, &q2, &q3);
    od_transpose4x4(&q0, q2, &q1, q3);
    od_store_buffer_4x4_epi16(out, q0, q1);
  } else {
    int r;
    /* Higher row counts require 32-bit precision. */
    assert(rows <= 16);
    for (r = 0; r < rows; r += 8) {
      __m128i q4;
      __m128i q5;
      __m128i q6;
      __m128i q7;
      od_load_buffer_4x4_epi32(&q0, &q1, &q2, &q3, in + 4 * r);
      od_load_buffer_4x4_epi32(&q4, &q5, &q6, &q7, in + 4 * r + 16);
      /*TODO(any): Merge this transpose with coefficient scanning.*/
      od_transpose_pack4x8(&q0, &q1, &q2, &q3, q4, q5, q6, q7);
      kernel8(&q0, &q1, &q2, &q3);
      od_transpose8x4(&q0, &q2, &q1, &q3);
      od_store_buffer_4x8_epi16(out + 4 * r, q0, q2, q1, q3);
    }
  }
}

static void od_col_tx4_add_hbd_avx2(unsigned char *output_pixels,
                                    int output_stride, int cols,
                                    const int16_t *in, int bd,
                                    od_tx4_kernel8_epi16 kernel8) {
  __m128i q0;
  __m128i q1;
  __m128i q2;
  __m128i q3;
  if (cols <= 4) {
    od_load_buffer_4x4_epi16(&q0, &q1, &q2, &q3, in);
    kernel8(&q0, &q1, &q2, &q3);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels, output_stride, q0, q2, q1,
                                      q3, bd);
  } else {
    int c;
    for (c = 0; c < cols; c += 8) {
      od_load_buffer_8x4_epi16(&q0, &q1, &q2, &q3, in + c, cols);
      kernel8(&q0, &q1, &q2, &q3);
      od_add_store_buffer_hbd_8x4_epi16(output_pixels + c, output_stride, q0,
                                        q2, q1, q3, bd);
    }
  }
}

static void od_row_idct4_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_tx4_avx2(out, rows, in, od_idct4_kernel8_epi16);
}

static void od_col_idct4_add_hbd_avx2(unsigned char *output_pixels,
                                      int output_stride, int cols,
                                      const int16_t *in, int bd) {
  od_col_tx4_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                          od_idct4_kernel8_epi16);
}

static void od_row_idst4_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_tx4_avx2(out, rows, in, od_idst_vii4_kernel8_epi16);
}

static void od_col_idst4_add_hbd_avx2(unsigned char *output_pixels,
                                      int output_stride, int cols,
                                      const int16_t *in, int bd) {
  od_col_tx4_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                          od_idst_vii4_kernel8_epi16);
}

static void od_row_flip_idst4_avx2(int16_t *out, int rows,
                                   const tran_low_t *in) {
  od_row_tx4_avx2(out, rows, in, od_flip_idst_vii4_kernel8_epi16);
}

static void od_col_flip_idst4_add_hbd_avx2(unsigned char *output_pixels,
                                           int output_stride, int cols,
                                           const int16_t *in, int bd) {
  od_col_tx4_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                          od_flip_idst_vii4_kernel8_epi16);
}

static void od_row_iidtx4_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_iidtx_avx2(out, rows * 4, in);
}

static void od_col_iidtx4_add_hbd_avx2(unsigned char *output_pixels,
                                       int output_stride, int cols,
                                       const int16_t *in, int bd) {
  od_col_iidtx_add_hbd_avx2(output_pixels, output_stride, 4, cols, in, bd);
}

typedef void (*od_tx8_kernel8_epi16)(__m128i *r0, __m128i *r4, __m128i *r2,
                                     __m128i *r6, __m128i *r1, __m128i *r5,
                                     __m128i *r3, __m128i *r7);

typedef void (*od_tx8_mm256_kernel)(__m256i *r0, __m256i *r4, __m256i *r2,
                                    __m256i *r6, __m256i *r1, __m256i *r5,
                                    __m256i *r3, __m256i *r7);

static void od_row_tx8_avx2(int16_t *out, int rows, const tran_low_t *in,
                            od_tx8_kernel8_epi16 kernel8_epi16,
                            od_tx8_mm256_kernel kernel8_epi32) {
  __m128i r0;
  __m128i r1;
  __m128i r2;
  __m128i r3;
  __m128i r4;
  __m128i r5;
  __m128i r6;
  __m128i r7;
  if (rows <= 4) {
    od_load_buffer_4x4_epi32(&r0, &r1, &r2, &r3, in);
    od_load_buffer_4x4_epi32(&r4, &r5, &r6, &r7, in + 16);
    /*TODO(any): Merge this transpose with coefficient scanning.*/
    od_transpose_pack8x4(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7);
    kernel8_epi16(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7);
    od_transpose4x8(&r0, r4, &r2, r6, &r1, r5, &r3, r7);
    od_store_buffer_4x4_epi16(out, r0, r2);
    od_store_buffer_4x4_epi16(out + 16, r1, r3);
  } else if (rows <= 8) {
    od_load_pack_buffer_8x4_epi32(&r0, &r1, &r2, &r3, in);
    od_load_pack_buffer_8x4_epi32(&r4, &r5, &r6, &r7, in + 32);
    /*TODO(any): Merge this transpose with coefficient scanning.*/
    od_transpose8x8_epi16(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7);
    kernel8_epi16(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7);
    od_transpose8x8_epi16(&r0, &r4, &r2, &r6, &r1, &r5, &r3, &r7);
    od_store_buffer_4x8_epi16(out, r0, r4, r2, r6);
    od_store_buffer_4x8_epi16(out + 32, r1, r5, r3, r7);
  } else {
    int r;
    /* 16 or more rows requires 32-bit precision.
       TODO(any): If the column TX is IDTX, then we can still use 16 bits. */
    for (r = 0; r < rows; r += 8) {
      __m256i rr0;
      __m256i rr1;
      __m256i rr2;
      __m256i rr3;
      __m256i rr4;
      __m256i rr5;
      __m256i rr6;
      __m256i rr7;
      od_load_buffer_8x4_epi32(&rr0, &rr1, &rr2, &rr3, in + r * 8);
      od_load_buffer_8x4_epi32(&rr4, &rr5, &rr6, &rr7, in + r * 8 + 32);
      od_transpose8x8_epi32(&rr0, &rr1, &rr2, &rr3, &rr4, &rr5, &rr6, &rr7);
      kernel8_epi32(&rr0, &rr1, &rr2, &rr3, &rr4, &rr5, &rr6, &rr7);
      od_transpose_pack8x8_epi32(&rr0, &rr4, &rr2, &rr6, rr1, rr5, rr3, rr7);
      od_store_buffer_2x16_epi16(out + r * 8, rr0, rr4);
      od_store_buffer_2x16_epi16(out + r * 8 + 32, rr2, rr6);
    }
  }
}

static void od_col_tx8_add_hbd_avx2(unsigned char *output_pixels,
                                    int output_stride, int cols,
                                    const int16_t *in, int bd,
                                    od_tx8_kernel8_epi16 kernel8_epi16,
                                    od_tx8_mm256_kernel kernel16_epi16) {
  __m128i r0;
  __m128i r1;
  __m128i r2;
  __m128i r3;
  __m128i r4;
  __m128i r5;
  __m128i r6;
  __m128i r7;
  if (cols <= 4) {
    od_load_buffer_4x4_epi16(&r0, &r1, &r2, &r3, in);
    od_load_buffer_4x4_epi16(&r4, &r5, &r6, &r7, in + 16);
    kernel8_epi16(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels, output_stride, r0, r4, r2,
                                      r6, bd);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels + 4 * output_stride,
                                      output_stride, r1, r5, r3, r7, bd);
  } else if (cols <= 8) {
    od_load_buffer_8x4_epi16(&r0, &r1, &r2, &r3, in, cols);
    od_load_buffer_8x4_epi16(&r4, &r5, &r6, &r7, in + 32, cols);
    kernel8_epi16(&r0, &r1, &r2, &r3, &r4, &r5, &r6, &r7);
    od_add_store_buffer_hbd_8x4_epi16(output_pixels, output_stride, r0, r4, r2,
                                      r6, bd);
    od_add_store_buffer_hbd_8x4_epi16(output_pixels + 4 * output_stride,
                                      output_stride, r1, r5, r3, r7, bd);
  } else {
    __m256i rr0;
    __m256i rr1;
    __m256i rr2;
    __m256i rr3;
    __m256i rr4;
    __m256i rr5;
    __m256i rr6;
    __m256i rr7;
    int c;
    for (c = 0; c < cols; c += 16) {
      od_load_buffer_16x4_epi16(&rr0, &rr1, &rr2, &rr3, in + c, cols);
      od_load_buffer_16x4_epi16(&rr4, &rr5, &rr6, &rr7, in + 4 * cols + c,
                                cols);
      kernel16_epi16(&rr0, &rr1, &rr2, &rr3, &rr4, &rr5, &rr6, &rr7);
      od_add_store_buffer_hbd_16x4_epi16(output_pixels, output_stride, rr0, rr4,
                                         rr2, rr6, bd);
      od_add_store_buffer_hbd_16x4_epi16(output_pixels + 4 * output_stride,
                                         output_stride, rr1, rr5, rr3, rr7, bd);
    }
  }
}

static void od_row_idct8_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_tx8_avx2(out, rows, in, od_idct8_kernel8_epi16,
                  od_idct8_kernel8_epi32);
}

static void od_col_idct8_add_hbd_avx2(unsigned char *output_pixels,
                                      int output_stride, int cols,
                                      const int16_t *in, int bd) {
  od_col_tx8_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                          od_idct8_kernel8_epi16, od_idct8_kernel16_epi16);
}

static void od_row_idst8_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_tx8_avx2(out, rows, in, od_idst8_kernel8_epi16,
                  od_idst8_kernel8_epi32);
}

static void od_col_idst8_add_hbd_avx2(unsigned char *output_pixels,
                                      int output_stride, int cols,
                                      const int16_t *in, int bd) {
  od_col_tx8_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                          od_idst8_kernel8_epi16, od_idst8_kernel16_epi16);
}

static void od_row_flip_idst8_avx2(int16_t *out, int rows,
                                   const tran_low_t *in) {
  od_row_tx8_avx2(out, rows, in, od_flip_idst8_kernel8_epi16,
                  od_flip_idst8_kernel8_epi32);
}

static void od_col_flip_idst8_add_hbd_avx2(unsigned char *output_pixels,
                                           int output_stride, int cols,
                                           const int16_t *in, int bd) {
  od_col_tx8_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                          od_flip_idst8_kernel8_epi16,
                          od_flip_idst8_kernel16_epi16);
}

static void od_row_iidtx8_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_iidtx_avx2(out, rows * 8, in);
}

static void od_col_iidtx8_add_hbd_avx2(unsigned char *output_pixels,
                                       int output_stride, int cols,
                                       const int16_t *in, int bd) {
  od_col_iidtx_add_hbd_avx2(output_pixels, output_stride, 8, cols, in, bd);
}

typedef void (*od_tx16_kernel8_epi16)(__m128i *s0, __m128i *s4, __m128i *s2,
                                      __m128i *s6, __m128i *s1, __m128i *s5,
                                      __m128i *s3, __m128i *s7, __m128i *s8,
                                      __m128i *s9, __m128i *sa, __m128i *sb,
                                      __m128i *sc, __m128i *sd, __m128i *se,
                                      __m128i *sf);

typedef void (*od_tx16_mm256_kernel)(__m256i *s0, __m256i *s4, __m256i *s2,
                                     __m256i *s6, __m256i *s1, __m256i *s5,
                                     __m256i *s3, __m256i *s7, __m256i *s8,
                                     __m256i *s9, __m256i *sa, __m256i *sb,
                                     __m256i *sc, __m256i *sd, __m256i *se,
                                     __m256i *sf);

static void od_row_tx16_avx2(int16_t *out, int rows, const tran_low_t *in,
#if CONFIG_RECT_TX_EXT
                             od_tx16_kernel8_epi16 kernel8_epi16,
#endif
                             od_tx16_mm256_kernel kernel8_epi32) {
#if CONFIG_RECT_TX_EXT
  if (rows <= 4) {
    __m128i s0;
    __m128i s1;
    __m128i s2;
    __m128i s3;
    __m128i s4;
    __m128i s5;
    __m128i s6;
    __m128i s7;
    __m128i s8;
    __m128i s9;
    __m128i sa;
    __m128i sb;
    __m128i sc;
    __m128i sd;
    __m128i se;
    __m128i sf;
    od_load_buffer_4x4_epi32(&s0, &s1, &s8, &s9, in);
    od_load_buffer_4x4_epi32(&s2, &s3, &sa, &sb, in + 16);
    od_load_buffer_4x4_epi32(&s4, &s5, &sc, &sd, in + 32);
    od_load_buffer_4x4_epi32(&s6, &s7, &se, &sf, in + 48);
    /*TODO(any): Merge this transpose with coefficient scanning.*/
    od_transpose_pack8x4(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
    od_transpose_pack8x4(&s8, &s9, &sa, &sb, &sc, &sd, &se, &sf);
    kernel8_epi16(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8, &s9, &sa, &sb,
                  &sc, &sd, &se, &sf);
    od_transpose4x8(&s0, s8, &s4, sc, &s2, sa, &s6, se);
    od_transpose4x8(&s1, s9, &s5, sd, &s3, sb, &s7, sf);
    od_store_buffer_4x4_epi16(out, s0, s1);
    od_store_buffer_4x4_epi16(out + 16, s4, s5);
    od_store_buffer_4x4_epi16(out + 32, s2, s3);
    od_store_buffer_4x4_epi16(out + 48, s6, s7);
    return;
  }
#endif  // CONFIG_RECT_TX_EXT
  {
    int r;
    /* 8 or more rows requires 32-bit precision.
       TODO(any): If the column TX is IDTX, then we can still use 16 bits. */
    for (r = 0; r < rows; r += 8) {
      __m256i ss0;
      __m256i ss1;
      __m256i ss2;
      __m256i ss3;
      __m256i ss4;
      __m256i ss5;
      __m256i ss6;
      __m256i ss7;
      __m256i ss8;
      __m256i ss9;
      __m256i ssa;
      __m256i ssb;
      __m256i ssc;
      __m256i ssd;
      __m256i sse;
      __m256i ssf;
      od_load_buffer_8x4_epi32(&ss0, &ss8, &ss1, &ss9, in + r * 16);
      od_load_buffer_8x4_epi32(&ss2, &ssa, &ss3, &ssb, in + r * 16 + 32);
      od_load_buffer_8x4_epi32(&ss4, &ssc, &ss5, &ssd, in + r * 16 + 64);
      od_load_buffer_8x4_epi32(&ss6, &sse, &ss7, &ssf, in + r * 16 + 96);
      od_transpose8x8_epi32(&ss0, &ss1, &ss2, &ss3, &ss4, &ss5, &ss6, &ss7);
      od_transpose8x8_epi32(&ss8, &ss9, &ssa, &ssb, &ssc, &ssd, &sse, &ssf);
      kernel8_epi32(&ss0, &ss1, &ss2, &ss3, &ss4, &ss5, &ss6, &ss7, &ss8, &ss9,
                    &ssa, &ssb, &ssc, &ssd, &sse, &ssf);
      od_transpose_pack8x16_epi32(&ss0, &ss8, &ss4, &ssc, &ss2, &ssa, &ss6,
                                  &sse, ss1, ss9, ss5, ssd, ss3, ssb, ss7, ssf);
      od_store_buffer_2x16_epi16(out + r * 16, ss0, ss8);
      od_store_buffer_2x16_epi16(out + r * 16 + 32, ss4, ssc);
      od_store_buffer_2x16_epi16(out + r * 16 + 64, ss2, ssa);
      od_store_buffer_2x16_epi16(out + r * 16 + 96, ss6, sse);
    }
  }
}

static void od_col_tx16_add_hbd_avx2(unsigned char *output_pixels,
                                     int output_stride, int cols,
                                     const int16_t *in, int bd,
                                     od_tx16_kernel8_epi16 kernel8_epi16,
                                     od_tx16_mm256_kernel kernel16_epi16) {
  __m128i s0;
  __m128i s1;
  __m128i s2;
  __m128i s3;
  __m128i s4;
  __m128i s5;
  __m128i s6;
  __m128i s7;
  __m128i s8;
  __m128i s9;
  __m128i sa;
  __m128i sb;
  __m128i sc;
  __m128i sd;
  __m128i se;
  __m128i sf;
#if CONFIG_RECT_TX_EXT
  if (cols <= 4) {
    od_load_buffer_4x4_epi16(&s0, &s1, &s2, &s3, in);
    od_load_buffer_4x4_epi16(&s4, &s5, &s6, &s7, in + 16);
    od_load_buffer_4x4_epi16(&s8, &s9, &sa, &sb, in + 32);
    od_load_buffer_4x4_epi16(&sc, &sd, &se, &sf, in + 48);
    kernel8_epi16(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8, &s9, &sa, &sb,
                  &sc, &sd, &se, &sf);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels, output_stride, s0, s8, s4,
                                      sc, bd);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels + 4 * output_stride,
                                      output_stride, s2, sa, s6, se, bd);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels + 8 * output_stride,
                                      output_stride, s1, s9, s5, sd, bd);
    od_add_store_buffer_hbd_4x4_epi16(output_pixels + 12 * output_stride,
                                      output_stride, s3, sb, s7, sf, bd);
    return;
  }
#endif  // CONFIG_RECT_TX_EXT
  if (cols <= 8) {
    od_load_buffer_8x4_epi16(&s0, &s1, &s2, &s3, in, cols);
    od_load_buffer_8x4_epi16(&s4, &s5, &s6, &s7, in + 32, cols);
    od_load_buffer_8x4_epi16(&s8, &s9, &sa, &sb, in + 64, cols);
    od_load_buffer_8x4_epi16(&sc, &sd, &se, &sf, in + 96, cols);
    kernel8_epi16(&s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7, &s8, &s9, &sa, &sb,
                  &sc, &sd, &se, &sf);
    od_add_store_buffer_hbd_8x4_epi16(output_pixels, output_stride, s0, s8, s4,
                                      sc, bd);
    od_add_store_buffer_hbd_8x4_epi16(output_pixels + 4 * output_stride,
                                      output_stride, s2, sa, s6, se, bd);
    od_add_store_buffer_hbd_8x4_epi16(output_pixels + 8 * output_stride,
                                      output_stride, s1, s9, s5, sd, bd);
    od_add_store_buffer_hbd_8x4_epi16(output_pixels + 12 * output_stride,
                                      output_stride, s3, sb, s7, sf, bd);
  } else {
    __m256i ss0;
    __m256i ss1;
    __m256i ss2;
    __m256i ss3;
    __m256i ss4;
    __m256i ss5;
    __m256i ss6;
    __m256i ss7;
    __m256i ss8;
    __m256i ss9;
    __m256i ssa;
    __m256i ssb;
    __m256i ssc;
    __m256i ssd;
    __m256i sse;
    __m256i ssf;
    int c;
    for (c = 0; c < cols; c += 16) {
      od_load_buffer_16x4_epi16(&ss0, &ss1, &ss2, &ss3, in + c, cols);
      od_load_buffer_16x4_epi16(&ss4, &ss5, &ss6, &ss7, in + 4 * cols + c,
                                cols);
      od_load_buffer_16x4_epi16(&ss8, &ss9, &ssa, &ssb, in + 8 * cols + c,
                                cols);
      od_load_buffer_16x4_epi16(&ssc, &ssd, &sse, &ssf, in + 12 * cols + c,
                                cols);
      kernel16_epi16(&ss0, &ss1, &ss2, &ss3, &ss4, &ss5, &ss6, &ss7, &ss8, &ss9,
                     &ssa, &ssb, &ssc, &ssd, &sse, &ssf);
      od_add_store_buffer_hbd_16x4_epi16(output_pixels, output_stride, ss0, ss8,
                                         ss4, ssc, bd);
      od_add_store_buffer_hbd_16x4_epi16(output_pixels + 4 * output_stride,
                                         output_stride, ss2, ssa, ss6, sse, bd);
      od_add_store_buffer_hbd_16x4_epi16(output_pixels + 8 * output_stride,
                                         output_stride, ss1, ss9, ss5, ssd, bd);
      od_add_store_buffer_hbd_16x4_epi16(output_pixels + 12 * output_stride,
                                         output_stride, ss3, ssb, ss7, ssf, bd);
    }
  }
}

static void od_row_idct16_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_tx16_avx2(out, rows, in,
#if CONFIG_RECT_TX_EXT
                   od_idct16_kernel8_epi16,
#endif
                   od_idct16_kernel8_epi32);
}

static void od_col_idct16_add_hbd_avx2(unsigned char *output_pixels,
                                       int output_stride, int cols,
                                       const int16_t *in, int bd) {
  od_col_tx16_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                           od_idct16_kernel8_epi16, od_idct16_kernel16_epi16);
}

static void od_row_idst16_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_tx16_avx2(out, rows, in,
#if CONFIG_RECT_TX_EXT
                   od_idst16_kernel8_epi16,
#endif
                   od_idst16_kernel8_epi32);
}

static void od_col_idst16_add_hbd_avx2(unsigned char *output_pixels,
                                       int output_stride, int cols,
                                       const int16_t *in, int bd) {
  od_col_tx16_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                           od_idst16_kernel8_epi16, od_idst16_kernel16_epi16);
}

static void od_row_flip_idst16_avx2(int16_t *out, int rows,
                                    const tran_low_t *in) {
  od_row_tx16_avx2(out, rows, in,
#if CONFIG_RECT_TX_EXT
                   od_flip_idst16_kernel8_epi16,
#endif
                   od_flip_idst16_kernel8_epi32);
}

static void od_col_flip_idst16_add_hbd_avx2(unsigned char *output_pixels,
                                            int output_stride, int cols,
                                            const int16_t *in, int bd) {
  od_col_tx16_add_hbd_avx2(output_pixels, output_stride, cols, in, bd,
                           od_flip_idst16_kernel8_epi16,
                           od_flip_idst16_kernel16_epi16);
}

static void od_row_iidtx16_avx2(int16_t *out, int rows, const tran_low_t *in) {
  od_row_iidtx_avx2(out, rows * 16, in);
}

static void od_col_iidtx16_add_hbd_avx2(unsigned char *output_pixels,
                                        int output_stride, int cols,
                                        const int16_t *in, int bd) {
  od_col_iidtx_add_hbd_avx2(output_pixels, output_stride, 16, cols, in, bd);
}

typedef void (*daala_row_itx)(int16_t *out, int rows, const tran_low_t *in);
typedef void (*daala_col_itx_add)(unsigned char *output_pixels,
                                  int output_stride, int cols,
                                  const int16_t *in, int bd);

static const daala_row_itx TX_ROW_MAP[TX_SIZES][TX_TYPES] = {
  // 4-point transforms
  { od_row_idct4_avx2, od_row_idst4_avx2, od_row_flip_idst4_avx2,
    od_row_iidtx4_avx2 },
  // 8-point transforms
  { od_row_idct8_avx2, od_row_idst8_avx2, od_row_flip_idst8_avx2,
    od_row_iidtx8_avx2 },
  // 16-point transforms
  { od_row_idct16_avx2, od_row_idst16_avx2, od_row_flip_idst16_avx2,
    od_row_iidtx16_avx2 },
  // 32-point transforms
  { NULL, NULL, NULL, NULL },
#if CONFIG_TX64X64
  // 64-point transforms
  { NULL, NULL, NULL, NULL },
#endif
};

static const daala_col_itx_add TX_COL_MAP[2][TX_SIZES][TX_TYPES] = {
  // Low bit depth output
  {
      // 4-point transforms
      { NULL, NULL, NULL, NULL },
      // 8-point transforms
      { NULL, NULL, NULL, NULL },
      // 16-point transforms
      { NULL, NULL, NULL, NULL },
      // 32-point transforms
      { NULL, NULL, NULL, NULL },
#if CONFIG_TX64X64
      // 64-point transforms
      { NULL, NULL, NULL, NULL },
#endif
  },
  // High bit depth output
  {
      // 4-point transforms
      { od_col_idct4_add_hbd_avx2, od_col_idst4_add_hbd_avx2,
        od_col_flip_idst4_add_hbd_avx2, od_col_iidtx4_add_hbd_avx2 },
      // 8-point transforms
      { od_col_idct8_add_hbd_avx2, od_col_idst8_add_hbd_avx2,
        od_col_flip_idst8_add_hbd_avx2, od_col_iidtx8_add_hbd_avx2 },
      // 16-point transforms
      { od_col_idct16_add_hbd_avx2, od_col_idst16_add_hbd_avx2,
        od_col_flip_idst16_add_hbd_avx2, od_col_iidtx16_add_hbd_avx2 },
      // 32-point transforms
      { NULL, NULL, NULL, NULL },
#if CONFIG_TX64X64
      // 64-point transforms
      { NULL, NULL, NULL, NULL },
#endif
  }
};

/* Define this to verify the SIMD against the C versions of the transforms.
   This is intended to be replaced by real unit tests in the future. */
#undef DAALA_TX_VERIFY_SIMD

void daala_inv_txfm_add_avx2(const tran_low_t *input_coeffs,
                             void *output_pixels, int output_stride,
                             TxfmParam *txfm_param) {
  const TX_SIZE tx_size = txfm_param->tx_size;
  const TX_TYPE tx_type = txfm_param->tx_type;
  assert(tx_size <= TX_SIZES_ALL);
  assert(tx_type <= TX_TYPES);

  if (txfm_param->lossless) {
    daala_inv_txfm_add_c(input_coeffs, output_pixels, output_stride,
                         txfm_param);
  } else {
    // General TX case
    assert(sizeof(tran_low_t) == sizeof(od_coeff));
    assert(sizeof(tran_low_t) >= 4);

    // Hook into existing map translation infrastructure to select
    // appropriate TX functions
    const TX_SIZE col_idx = txsize_vert_map[tx_size];
    const TX_SIZE row_idx = txsize_horz_map[tx_size];
    assert(col_idx <= TX_SIZES);
    assert(row_idx <= TX_SIZES);
    assert(vtx_tab[tx_type] <= (int)TX_TYPES_1D);
    assert(htx_tab[tx_type] <= (int)TX_TYPES_1D);
    daala_row_itx row_tx = TX_ROW_MAP[row_idx][htx_tab[tx_type]];
    daala_col_itx_add col_tx =
        TX_COL_MAP[txfm_param->is_hbd][col_idx][vtx_tab[tx_type]];
    int16_t tmpsq[MAX_TX_SQUARE];

    if (row_tx == NULL || col_tx == NULL) {
      daala_inv_txfm_add_c(input_coeffs, output_pixels, output_stride,
                           txfm_param);
    } else {
      const int cols = tx_size_wide[tx_size];
      const int rows = tx_size_high[tx_size];
#if defined(DAALA_TX_VERIFY_SIMD)
      unsigned char out_check_buf8[MAX_TX_SQUARE];
      int16_t out_check_buf16[MAX_TX_SQUARE];
      unsigned char *out_check_buf;
      {
        if (txfm_param->is_hbd) {
          uint16_t *output_pixels16;
          int r;
          output_pixels16 = CONVERT_TO_SHORTPTR(output_pixels);
          for (r = 0; r < rows; r++) {
            memcpy(out_check_buf16 + r * cols,
                   output_pixels16 + r * output_stride,
                   cols * sizeof(*out_check_buf16));
          }
          out_check_buf = CONVERT_TO_BYTEPTR(out_check_buf16);
        } else {
          unsigned char *output_pixels8;
          int r;
          output_pixels8 = (unsigned char *)output_pixels;
          for (r = 0; r < rows; r++) {
            memcpy(out_check_buf8 + r * cols,
                   output_pixels8 + r * output_stride,
                   cols * sizeof(*out_check_buf8));
          }
          out_check_buf = out_check_buf8;
        }
      }
      daala_inv_txfm_add_c(input_coeffs, out_check_buf, cols, txfm_param);
#endif
      // Inverse-transform rows
      row_tx(tmpsq, rows, input_coeffs);
      // Inverse-transform columns and sum with destination
      col_tx(output_pixels, output_stride, cols, tmpsq, txfm_param->bd);
#if defined(DAALA_TX_VERIFY_SIMD)
      {
        if (txfm_param->is_hbd) {
          uint16_t *output_pixels16;
          int r;
          output_pixels16 = CONVERT_TO_SHORTPTR(output_pixels);
          for (r = 0; r < rows; r++) {
            if (memcmp(out_check_buf16 + r * cols,
                       output_pixels16 + r * output_stride,
                       cols * sizeof(*out_check_buf16))) {
              fprintf(stderr, "%s(%i): Inverse %ix%i %i_%i TX SIMD mismatch.\n",
                      __FILE__, __LINE__, rows, cols, vtx_tab[tx_type],
                      htx_tab[tx_type]);
              assert(0);
              exit(EXIT_FAILURE);
            }
          }
        } else {
          unsigned char *output_pixels8;
          int r;
          output_pixels8 = (unsigned char *)output_pixels;
          for (r = 0; r < rows; r++) {
            if (memcmp(out_check_buf8 + r * cols,
                       output_pixels8 + r * output_stride,
                       cols * sizeof(*out_check_buf8))) {
              fprintf(stderr, "%s(%i): Inverse %ix%i %i_%i TX SIMD mismatch.\n",
                      __FILE__, __LINE__, rows, cols, vtx_tab[tx_type],
                      htx_tab[tx_type]);
              assert(0);
              exit(EXIT_FAILURE);
            }
          }
        }
      }
#endif
    }
  }
}

#endif