blob: 6a71d5e2cd663c048509f95f1796b52df8cf8f48 [file] [edit]
/*
* Copyright (c) 2026, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
#include "av1/encoder/cost.h"
// Vector form of av1_cost_symbol_clamped for 8 lanes. Bit-exact with the
// scalar path:
// p15c = clamp(p15, EC_MIN_PROB, CDF_PROB_TOP - 1)
// shift = (CDF_PROB_BITS - 1) - floor(log2(p15c)) // via float exponent
// idx = min(((p15c << shift) + 64) >> 7, 255) - 128
// cost = av1_prob_cost[idx] + shift * 512
// The scalar path can reach idx 128, which duplicates idx 127, so clamping to
// 127 gives the same cost and keeps the 32-bit gather below within the table.
static inline __m256i cost_symbol_x8(__m256i p15) {
p15 = _mm256_max_epi32(p15, _mm256_set1_epi32(EC_MIN_PROB));
p15 = _mm256_min_epi32(p15, _mm256_set1_epi32(CDF_PROB_TOP - 1));
// floor(log2(p15)) = (biased exponent) - 127; exact since p15 < 2^24.
const __m256i bits = _mm256_castps_si256(_mm256_cvtepi32_ps(p15));
const __m256i exp =
_mm256_and_si256(_mm256_srli_epi32(bits, 23), _mm256_set1_epi32(0xFF));
// shift = (CDF_PROB_BITS - 1) - (exp - 127) = 141 - exp.
const __m256i shift = _mm256_sub_epi32(_mm256_set1_epi32(141), exp);
const __m256i norm = _mm256_sllv_epi32(p15, shift);
__m256i prob =
_mm256_srli_epi32(_mm256_add_epi32(norm, _mm256_set1_epi32(64)), 7);
prob = _mm256_min_epi32(prob, _mm256_set1_epi32(255));
const __m256i idx = _mm256_sub_epi32(prob, _mm256_set1_epi32(128));
// 16-bit table read via 32-bit gather; mask off the adjacent entry (idx + 1
// is at most 128, which is inside the 129-entry table).
__m256i tbl = _mm256_i32gather_epi32((const int *)av1_prob_cost, idx, 2);
tbl = _mm256_and_si256(tbl, _mm256_set1_epi32(0xFFFF));
const __m256i lit = _mm256_slli_epi32(shift, AV1_PROB_COST_SHIFT);
return _mm256_add_epi32(tbl, lit);
}
void av1_cost_tokens_from_cdf_4_batch_avx2(int *costs, int costs_stride,
const uint16_t *cdf, int cdf_stride,
int n) {
const int s = cdf_stride;
// Per-lane element offset of the 8 CDFs processed together.
const __m256i lane =
_mm256_setr_epi32(0, s, 2 * s, 3 * s, 4 * s, 5 * s, 6 * s, 7 * s);
const __m256i mask16 = _mm256_set1_epi32(0xFFFF);
const __m256i top = _mm256_set1_epi32(CDF_PROB_TOP);
int c = 0;
for (; c + 8 <= n; c += 8) {
const __m256i base = _mm256_add_epi32(_mm256_set1_epi32(c * s), lane);
// Gather {c0,c1} and {c2,c3} pairs (16-bit values packed in 32-bit lanes).
const __m256i g01 = _mm256_i32gather_epi32((const int *)cdf, base, 2);
const __m256i g23 = _mm256_i32gather_epi32(
(const int *)cdf, _mm256_add_epi32(base, _mm256_set1_epi32(2)), 2);
// ICDF(x) = CDF_PROB_TOP - x; p15[i] = ICDF[i] - ICDF[i-1], ICDF[-1] = 0.
const __m256i i0 = _mm256_sub_epi32(top, _mm256_and_si256(g01, mask16));
const __m256i i1 = _mm256_sub_epi32(top, _mm256_srli_epi32(g01, 16));
const __m256i i2 = _mm256_sub_epi32(top, _mm256_and_si256(g23, mask16));
const __m256i i3 = _mm256_sub_epi32(top, _mm256_srli_epi32(g23, 16));
const __m256i r0 = cost_symbol_x8(i0);
const __m256i r1 = cost_symbol_x8(_mm256_sub_epi32(i1, i0));
const __m256i r2 = cost_symbol_x8(_mm256_sub_epi32(i2, i1));
const __m256i r3 = cost_symbol_x8(_mm256_sub_epi32(i3, i2));
// Transpose the 4 SoA cost vectors (8 ctx each) into 8 AoS rows of 4.
const __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
const __m256i t1 = _mm256_unpackhi_epi32(r0, r1);
const __m256i t2 = _mm256_unpacklo_epi32(r2, r3);
const __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
const __m256i q0 = _mm256_unpacklo_epi64(t0, t2); // ctx 0 | ctx 4
const __m256i q1 = _mm256_unpackhi_epi64(t0, t2); // ctx 1 | ctx 5
const __m256i q2 = _mm256_unpacklo_epi64(t1, t3); // ctx 2 | ctx 6
const __m256i q3 = _mm256_unpackhi_epi64(t1, t3); // ctx 3 | ctx 7
int *o = costs + c * costs_stride;
_mm_storeu_si128((__m128i *)(o + 0 * costs_stride),
_mm256_castsi256_si128(q0));
_mm_storeu_si128((__m128i *)(o + 1 * costs_stride),
_mm256_castsi256_si128(q1));
_mm_storeu_si128((__m128i *)(o + 2 * costs_stride),
_mm256_castsi256_si128(q2));
_mm_storeu_si128((__m128i *)(o + 3 * costs_stride),
_mm256_castsi256_si128(q3));
_mm_storeu_si128((__m128i *)(o + 4 * costs_stride),
_mm256_extracti128_si256(q0, 1));
_mm_storeu_si128((__m128i *)(o + 5 * costs_stride),
_mm256_extracti128_si256(q1, 1));
_mm_storeu_si128((__m128i *)(o + 6 * costs_stride),
_mm256_extracti128_si256(q2, 1));
_mm_storeu_si128((__m128i *)(o + 7 * costs_stride),
_mm256_extracti128_si256(q3, 1));
}
for (; c < n; ++c)
av1_cost_tokens_from_cdf_4(costs + c * costs_stride, cdf + c * cdf_stride);
}
void av1_cost_tokens_from_cdf_2_batch_avx2(int *costs, int costs_stride,
const uint16_t *cdf, int cdf_stride,
int n) {
const int s = cdf_stride;
const __m256i lane =
_mm256_setr_epi32(0, s, 2 * s, 3 * s, 4 * s, 5 * s, 6 * s, 7 * s);
const __m256i top = _mm256_set1_epi32(CDF_PROB_TOP);
int c = 0;
for (; c + 8 <= n; c += 8) {
const __m256i base = _mm256_add_epi32(_mm256_set1_epi32(c * s), lane);
// Only cdf[0] is needed for a 2-symbol CDF; the 32-bit read's high half is
// the terminator and is masked away.
const __m256i g = _mm256_i32gather_epi32((const int *)cdf, base, 2);
const __m256i c0 = _mm256_and_si256(g, _mm256_set1_epi32(0xFFFF));
// p15[0] = ICDF[0] = CDF_PROB_TOP - cdf[0]; p15[1] = cdf[0].
const __m256i r0 = cost_symbol_x8(_mm256_sub_epi32(top, c0));
const __m256i r1 = cost_symbol_x8(c0);
// Transpose the 2 SoA cost vectors into 8 AoS pairs and store 2 ints/ctx.
const __m256i t0 = _mm256_unpacklo_epi32(r0, r1); // ctx 0,1 | ctx 4,5
const __m256i t1 = _mm256_unpackhi_epi32(r0, r1); // ctx 2,3 | ctx 6,7
const __m128i lo0 = _mm256_castsi256_si128(t0);
const __m128i hi0 = _mm256_extracti128_si256(t0, 1);
const __m128i lo1 = _mm256_castsi256_si128(t1);
const __m128i hi1 = _mm256_extracti128_si256(t1, 1);
int *o = costs + c * costs_stride;
_mm_storel_epi64((__m128i *)(o + 0 * costs_stride), lo0);
_mm_storel_epi64((__m128i *)(o + 1 * costs_stride),
_mm_unpackhi_epi64(lo0, lo0));
_mm_storel_epi64((__m128i *)(o + 2 * costs_stride), lo1);
_mm_storel_epi64((__m128i *)(o + 3 * costs_stride),
_mm_unpackhi_epi64(lo1, lo1));
_mm_storel_epi64((__m128i *)(o + 4 * costs_stride), hi0);
_mm_storel_epi64((__m128i *)(o + 5 * costs_stride),
_mm_unpackhi_epi64(hi0, hi0));
_mm_storel_epi64((__m128i *)(o + 6 * costs_stride), hi1);
_mm_storel_epi64((__m128i *)(o + 7 * costs_stride),
_mm_unpackhi_epi64(hi1, hi1));
}
for (; c < n; ++c)
av1_cost_tokens_from_cdf_2(costs + c * costs_stride, cdf + c * cdf_stride);
}