Merge "third_party/libwebm: pull from upstream"
diff --git a/examples/vp9_spatial_scalable_encoder.c b/examples/vp9_spatial_scalable_encoder.c
index 64e62ef..983f52d 100644
--- a/examples/vp9_spatial_scalable_encoder.c
+++ b/examples/vp9_spatial_scalable_encoder.c
@@ -344,7 +344,7 @@
}
res = vpx_svc_encode(&svc_ctx, &codec, (end_of_stream ? NULL : &raw),
- pts, frame_duration, VPX_DL_REALTIME);
+ pts, frame_duration, VPX_DL_GOOD_QUALITY);
printf("%s", vpx_svc_get_message(&svc_ctx));
if (res != VPX_CODEC_OK) {
die_codec(&codec, "Failed to encode frame");
diff --git a/test/fdct4x4_test.cc b/test/fdct4x4_test.cc
index 127775c..e2ba8a8 100644
--- a/test/fdct4x4_test.cc
+++ b/test/fdct4x4_test.cc
@@ -48,6 +48,10 @@
vp9_fht4x4_c(in, out, stride, tx_type);
}
+void fwht4x4_ref(const int16_t *in, int16_t *out, int stride, int tx_type) {
+ vp9_fwht4x4_c(in, out, stride);
+}
+
class Trans4x4TestBase {
public:
virtual ~Trans4x4TestBase() {}
@@ -57,7 +61,7 @@
virtual void RunInvTxfm(const int16_t *out, uint8_t *dst, int stride) = 0;
- void RunAccuracyCheck() {
+ void RunAccuracyCheck(int limit) {
ACMRandom rnd(ACMRandom::DeterministicSeed());
uint32_t max_error = 0;
int64_t total_error = 0;
@@ -88,11 +92,13 @@
}
}
- EXPECT_GE(1u, max_error)
- << "Error: 4x4 FHT/IHT has an individual round trip error > 1";
+ EXPECT_GE(static_cast<uint32_t>(limit), max_error)
+ << "Error: 4x4 FHT/IHT has an individual round trip error > "
+ << limit;
- EXPECT_GE(count_test_block , total_error)
- << "Error: 4x4 FHT/IHT has average round trip error > 1 per block";
+ EXPECT_GE(count_test_block * limit, total_error)
+ << "Error: 4x4 FHT/IHT has average round trip error > " << limit
+ << " per block";
}
void RunCoeffCheck() {
@@ -150,7 +156,7 @@
}
}
- void RunInvAccuracyCheck() {
+ void RunInvAccuracyCheck(int limit) {
ACMRandom rnd(ACMRandom::DeterministicSeed());
const int count_test_block = 1000;
DECLARE_ALIGNED_ARRAY(16, int16_t, in, kNumCoeffs);
@@ -173,8 +179,8 @@
for (int j = 0; j < kNumCoeffs; ++j) {
const uint32_t diff = dst[j] - src[j];
const uint32_t error = diff * diff;
- EXPECT_GE(1u, error)
- << "Error: 16x16 IDCT has error " << error
+ EXPECT_GE(static_cast<uint32_t>(limit), error)
+ << "Error: 4x4 IDCT has error " << error
<< " at index " << j;
}
}
@@ -213,7 +219,7 @@
};
TEST_P(Trans4x4DCT, AccuracyCheck) {
- RunAccuracyCheck();
+ RunAccuracyCheck(1);
}
TEST_P(Trans4x4DCT, CoeffCheck) {
@@ -225,7 +231,7 @@
}
TEST_P(Trans4x4DCT, InvAccuracyCheck) {
- RunInvAccuracyCheck();
+ RunInvAccuracyCheck(1);
}
class Trans4x4HT
@@ -257,7 +263,7 @@
};
TEST_P(Trans4x4HT, AccuracyCheck) {
- RunAccuracyCheck();
+ RunAccuracyCheck(1);
}
TEST_P(Trans4x4HT, CoeffCheck) {
@@ -269,9 +275,51 @@
}
TEST_P(Trans4x4HT, InvAccuracyCheck) {
- RunInvAccuracyCheck();
+ RunInvAccuracyCheck(1);
}
+class Trans4x4WHT
+ : public Trans4x4TestBase,
+ public ::testing::TestWithParam<dct_4x4_param_t> {
+ public:
+ virtual ~Trans4x4WHT() {}
+
+ virtual void SetUp() {
+ fwd_txfm_ = GET_PARAM(0);
+ inv_txfm_ = GET_PARAM(1);
+ tx_type_ = GET_PARAM(2);
+ pitch_ = 4;
+ fwd_txfm_ref = fwht4x4_ref;
+ }
+ virtual void TearDown() { libvpx_test::ClearSystemState(); }
+
+ protected:
+ void RunFwdTxfm(const int16_t *in, int16_t *out, int stride) {
+ fwd_txfm_(in, out, stride);
+ }
+ void RunInvTxfm(const int16_t *out, uint8_t *dst, int stride) {
+ inv_txfm_(out, dst, stride);
+ }
+
+ fdct_t fwd_txfm_;
+ idct_t inv_txfm_;
+};
+
+TEST_P(Trans4x4WHT, AccuracyCheck) {
+ RunAccuracyCheck(0);
+}
+
+TEST_P(Trans4x4WHT, CoeffCheck) {
+ RunCoeffCheck();
+}
+
+TEST_P(Trans4x4WHT, MemCheck) {
+ RunMemCheck();
+}
+
+TEST_P(Trans4x4WHT, InvAccuracyCheck) {
+ RunInvAccuracyCheck(0);
+}
using std::tr1::make_tuple;
INSTANTIATE_TEST_CASE_P(
@@ -285,6 +333,10 @@
make_tuple(&vp9_fht4x4_c, &vp9_iht4x4_16_add_c, 1),
make_tuple(&vp9_fht4x4_c, &vp9_iht4x4_16_add_c, 2),
make_tuple(&vp9_fht4x4_c, &vp9_iht4x4_16_add_c, 3)));
+INSTANTIATE_TEST_CASE_P(
+ C, Trans4x4WHT,
+ ::testing::Values(
+ make_tuple(&vp9_fwht4x4_c, &vp9_iwht4x4_16_add_c, 0)));
#if HAVE_NEON
INSTANTIATE_TEST_CASE_P(
@@ -301,6 +353,13 @@
make_tuple(&vp9_fht4x4_c, &vp9_iht4x4_16_add_neon, 3)));
#endif
+#if HAVE_MMX
+INSTANTIATE_TEST_CASE_P(
+ MMX, Trans4x4WHT,
+ ::testing::Values(
+ make_tuple(&vp9_fwht4x4_mmx, &vp9_iwht4x4_16_add_c, 0)));
+#endif
+
#if HAVE_SSE2
INSTANTIATE_TEST_CASE_P(
SSE2, Trans4x4DCT,
diff --git a/test/postproc.sh b/test/postproc.sh
new file mode 100755
index 0000000..050a368
--- /dev/null
+++ b/test/postproc.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+##
+## Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+##
+## Use of this source code is governed by a BSD-style license
+## that can be found in the LICENSE file in the root of the source
+## tree. An additional intellectual property rights grant can be found
+## in the file PATENTS. All contributing project authors may
+## be found in the AUTHORS file in the root of the source tree.
+##
+## This file tests the libvpx postproc example code. To add new tests to this
+## file, do the following:
+## 1. Write a shell function (this is your test).
+## 2. Add the function to postproc_tests (on a new line).
+##
+. $(dirname $0)/tools_common.sh
+
+# Environment check: Make sure input is available:
+# $VP8_IVF_FILE and $VP9_IVF_FILE are required.
+postproc_verify_environment() {
+ if [ ! -e "${VP8_IVF_FILE}" ] || [ ! -e "${VP9_IVF_FILE}" ]; then
+ echo "Libvpx test data must exist in LIBVPX_TEST_DATA_PATH."
+ return 1
+ fi
+}
+
+# Runs postproc using $1 as input file. $2 is the codec name, and is used
+# solely to name the output file.
+postproc() {
+ local decoder="${LIBVPX_BIN_PATH}/postproc${VPX_TEST_EXE_SUFFIX}"
+ local input_file="$1"
+ local codec="$2"
+ local output_file="${VPX_TEST_OUTPUT_DIR}/postproc_${codec}.raw"
+
+ [ -x "${decoder}" ] || return 1
+
+ eval "${decoder}" "${input_file}" "${output_file}" ${devnull}
+
+ [ -e "${output_file}" ] || return 1
+}
+
+postproc_vp8() {
+ if [ "$(vp8_decode_available)" = "yes" ]; then
+ postproc "${VP8_IVF_FILE}" vp8 || return 1
+ fi
+}
+
+postproc_vp9() {
+ if [ "$(vpx_config_option_enabled CONFIG_VP9_POSTPROC)" = "yes" ]; then
+ if [ "$(vp9_decode_available)" = "yes" ]; then
+ postproc "${VP9_IVF_FILE}" vp9 || return 1
+ fi
+ fi
+}
+
+postproc_tests="postproc_vp8
+ postproc_vp9"
+
+run_tests postproc_verify_environment "${postproc_tests}"
diff --git a/test/register_state_check.h b/test/register_state_check.h
index 7e3d053..5987fe3 100644
--- a/test/register_state_check.h
+++ b/test/register_state_check.h
@@ -82,8 +82,8 @@
} // namespace libvpx_test
-#elif defined(CONFIG_SHARED) && defined(HAVE_NEON) \
- && !CONFIG_SHARED && HAVE_NEON
+#elif defined(CONFIG_SHARED) && defined(HAVE_NEON) && defined(CONFIG_VP9) \
+ && !CONFIG_SHARED && HAVE_NEON && CONFIG_VP9
#include "vpx/vpx_integer.h"
diff --git a/vp8/common/arm/neon/idct_dequant_0_2x_neon.asm b/vp8/common/arm/neon/idct_dequant_0_2x_neon.asm
deleted file mode 100644
index 3a39210..0000000
--- a/vp8/common/arm/neon/idct_dequant_0_2x_neon.asm
+++ /dev/null
@@ -1,81 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license and patent
-; grant that can be found in the LICENSE file in the root of the source
-; tree. All contributing project authors may be found in the AUTHORS
-; file in the root of the source tree.
-;
-
-
- EXPORT |idct_dequant_0_2x_neon|
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-;void idct_dequant_0_2x_neon(short *q, short dq,
-; unsigned char *dst, int stride);
-; r0 *q
-; r1 dq
-; r2 *dst
-; r3 stride
-|idct_dequant_0_2x_neon| PROC
- push {r4, r5}
- vpush {d8-d15}
-
- add r12, r2, #4
- vld1.32 {d2[0]}, [r2], r3
- vld1.32 {d8[0]}, [r12], r3
- vld1.32 {d2[1]}, [r2], r3
- vld1.32 {d8[1]}, [r12], r3
- vld1.32 {d4[0]}, [r2], r3
- vld1.32 {d10[0]}, [r12], r3
- vld1.32 {d4[1]}, [r2], r3
- vld1.32 {d10[1]}, [r12], r3
-
- ldrh r12, [r0] ; lo q
- ldrh r4, [r0, #32] ; hi q
- mov r5, #0
- strh r5, [r0]
- strh r5, [r0, #32]
-
- sxth r12, r12 ; lo
- mul r0, r12, r1
- add r0, r0, #4
- asr r0, r0, #3
- vdup.16 q0, r0
- sxth r4, r4 ; hi
- mul r0, r4, r1
- add r0, r0, #4
- asr r0, r0, #3
- vdup.16 q3, r0
-
- vaddw.u8 q1, q0, d2 ; lo
- vaddw.u8 q2, q0, d4
- vaddw.u8 q4, q3, d8 ; hi
- vaddw.u8 q5, q3, d10
-
- sub r2, r2, r3, lsl #2 ; dst - 4*stride
- add r0, r2, #4
-
- vqmovun.s16 d2, q1 ; lo
- vqmovun.s16 d4, q2
- vqmovun.s16 d8, q4 ; hi
- vqmovun.s16 d10, q5
-
- vst1.32 {d2[0]}, [r2], r3 ; lo
- vst1.32 {d8[0]}, [r0], r3 ; hi
- vst1.32 {d2[1]}, [r2], r3
- vst1.32 {d8[1]}, [r0], r3
- vst1.32 {d4[0]}, [r2], r3
- vst1.32 {d10[0]}, [r0], r3
- vst1.32 {d4[1]}, [r2]
- vst1.32 {d10[1]}, [r0]
-
- vpop {d8-d15}
- pop {r4, r5}
- bx lr
-
- ENDP ; |idct_dequant_0_2x_neon|
- END
diff --git a/vp8/common/arm/neon/idct_dequant_0_2x_neon.c b/vp8/common/arm/neon/idct_dequant_0_2x_neon.c
new file mode 100644
index 0000000..967c322
--- /dev/null
+++ b/vp8/common/arm/neon/idct_dequant_0_2x_neon.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+void idct_dequant_0_2x_neon(
+ int16_t *q,
+ int16_t dq,
+ unsigned char *dst,
+ int stride) {
+ unsigned char *dst0;
+ int i, a0, a1;
+ int16x8x2_t q2Add;
+ int32x2_t d2s32, d4s32;
+ uint8x8_t d2u8, d4u8;
+ uint16x8_t q1u16, q2u16;
+
+ a0 = ((q[0] * dq) + 4) >> 3;
+ a1 = ((q[16] * dq) + 4) >> 3;
+ q[0] = q[16] = 0;
+ q2Add.val[0] = vdupq_n_s16((int16_t)a0);
+ q2Add.val[1] = vdupq_n_s16((int16_t)a1);
+
+ for (i = 0; i < 2; i++, dst += 4) {
+ dst0 = dst;
+ d2s32 = vld1_lane_s32((const int32_t *)dst0, d2s32, 0);
+ dst0 += stride;
+ d2s32 = vld1_lane_s32((const int32_t *)dst0, d2s32, 1);
+ dst0 += stride;
+ d4s32 = vld1_lane_s32((const int32_t *)dst0, d4s32, 0);
+ dst0 += stride;
+ d4s32 = vld1_lane_s32((const int32_t *)dst0, d4s32, 1);
+
+ q1u16 = vaddw_u8(vreinterpretq_u16_s16(q2Add.val[i]),
+ vreinterpret_u8_s32(d2s32));
+ q2u16 = vaddw_u8(vreinterpretq_u16_s16(q2Add.val[i]),
+ vreinterpret_u8_s32(d4s32));
+
+ d2u8 = vqmovun_s16(vreinterpretq_s16_u16(q1u16));
+ d4u8 = vqmovun_s16(vreinterpretq_s16_u16(q2u16));
+
+ d2s32 = vreinterpret_s32_u8(d2u8);
+ d4s32 = vreinterpret_s32_u8(d4u8);
+
+ dst0 = dst;
+ vst1_lane_s32((int32_t *)dst0, d2s32, 0);
+ dst0 += stride;
+ vst1_lane_s32((int32_t *)dst0, d2s32, 1);
+ dst0 += stride;
+ vst1_lane_s32((int32_t *)dst0, d4s32, 0);
+ dst0 += stride;
+ vst1_lane_s32((int32_t *)dst0, d4s32, 1);
+ }
+ return;
+}
diff --git a/vp8/common/arm/neon/iwalsh_neon.asm b/vp8/common/arm/neon/iwalsh_neon.asm
deleted file mode 100644
index e8ea2a6..0000000
--- a/vp8/common/arm/neon/iwalsh_neon.asm
+++ /dev/null
@@ -1,87 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
- EXPORT |vp8_short_inv_walsh4x4_neon|
-
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA |.text|, CODE, READONLY ; name this block of code
-
-;short vp8_short_inv_walsh4x4_neon(short *input, short *mb_dqcoeff)
-|vp8_short_inv_walsh4x4_neon| PROC
-
- ; read in all four lines of values: d0->d3
- vld1.i16 {q0-q1}, [r0@128]
-
- ; first for loop
- vadd.s16 d4, d0, d3 ;a = [0] + [12]
- vadd.s16 d6, d1, d2 ;b = [4] + [8]
- vsub.s16 d5, d0, d3 ;d = [0] - [12]
- vsub.s16 d7, d1, d2 ;c = [4] - [8]
-
- vadd.s16 q0, q2, q3 ; a+b d+c
- vsub.s16 q1, q2, q3 ; a-b d-c
-
- vtrn.32 d0, d2 ;d0: 0 1 8 9
- ;d2: 2 3 10 11
- vtrn.32 d1, d3 ;d1: 4 5 12 13
- ;d3: 6 7 14 15
-
- vtrn.16 d0, d1 ;d0: 0 4 8 12
- ;d1: 1 5 9 13
- vtrn.16 d2, d3 ;d2: 2 6 10 14
- ;d3: 3 7 11 15
-
- ; second for loop
-
- vadd.s16 d4, d0, d3 ;a = [0] + [3]
- vadd.s16 d6, d1, d2 ;b = [1] + [2]
- vsub.s16 d5, d0, d3 ;d = [0] - [3]
- vsub.s16 d7, d1, d2 ;c = [1] - [2]
-
- vmov.i16 q8, #3
-
- vadd.s16 q0, q2, q3 ; a+b d+c
- vsub.s16 q1, q2, q3 ; a-b d-c
-
- vadd.i16 q0, q0, q8 ;e/f += 3
- vadd.i16 q1, q1, q8 ;g/h += 3
-
- vshr.s16 q0, q0, #3 ;e/f >> 3
- vshr.s16 q1, q1, #3 ;g/h >> 3
-
- mov r2, #64
- add r3, r1, #32
-
- vst1.i16 d0[0], [r1],r2
- vst1.i16 d1[0], [r3],r2
- vst1.i16 d2[0], [r1],r2
- vst1.i16 d3[0], [r3],r2
-
- vst1.i16 d0[1], [r1],r2
- vst1.i16 d1[1], [r3],r2
- vst1.i16 d2[1], [r1],r2
- vst1.i16 d3[1], [r3],r2
-
- vst1.i16 d0[2], [r1],r2
- vst1.i16 d1[2], [r3],r2
- vst1.i16 d2[2], [r1],r2
- vst1.i16 d3[2], [r3],r2
-
- vst1.i16 d0[3], [r1],r2
- vst1.i16 d1[3], [r3],r2
- vst1.i16 d2[3], [r1]
- vst1.i16 d3[3], [r3]
-
- bx lr
- ENDP ; |vp8_short_inv_walsh4x4_neon|
-
- END
diff --git a/vp8/common/arm/neon/iwalsh_neon.c b/vp8/common/arm/neon/iwalsh_neon.c
new file mode 100644
index 0000000..6ea9dd7
--- /dev/null
+++ b/vp8/common/arm/neon/iwalsh_neon.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+void vp8_short_inv_walsh4x4_neon(
+ int16_t *input,
+ int16_t *mb_dqcoeff) {
+ int16x8_t q0s16, q1s16, q2s16, q3s16;
+ int16x4_t d4s16, d5s16, d6s16, d7s16;
+ int16x4x2_t v2tmp0, v2tmp1;
+ int32x2x2_t v2tmp2, v2tmp3;
+ int16x8_t qAdd3;
+
+ q0s16 = vld1q_s16(input);
+ q1s16 = vld1q_s16(input + 8);
+
+ // 1st for loop
+ d4s16 = vadd_s16(vget_low_s16(q0s16), vget_high_s16(q1s16));
+ d6s16 = vadd_s16(vget_high_s16(q0s16), vget_low_s16(q1s16));
+ d5s16 = vsub_s16(vget_low_s16(q0s16), vget_high_s16(q1s16));
+ d7s16 = vsub_s16(vget_high_s16(q0s16), vget_low_s16(q1s16));
+
+ q2s16 = vcombine_s16(d4s16, d5s16);
+ q3s16 = vcombine_s16(d6s16, d7s16);
+
+ q0s16 = vaddq_s16(q2s16, q3s16);
+ q1s16 = vsubq_s16(q2s16, q3s16);
+
+ v2tmp2 = vtrn_s32(vreinterpret_s32_s16(vget_low_s16(q0s16)),
+ vreinterpret_s32_s16(vget_low_s16(q1s16)));
+ v2tmp3 = vtrn_s32(vreinterpret_s32_s16(vget_high_s16(q0s16)),
+ vreinterpret_s32_s16(vget_high_s16(q1s16)));
+ v2tmp0 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[0]),
+ vreinterpret_s16_s32(v2tmp3.val[0]));
+ v2tmp1 = vtrn_s16(vreinterpret_s16_s32(v2tmp2.val[1]),
+ vreinterpret_s16_s32(v2tmp3.val[1]));
+
+ // 2nd for loop
+ d4s16 = vadd_s16(v2tmp0.val[0], v2tmp1.val[1]);
+ d6s16 = vadd_s16(v2tmp0.val[1], v2tmp1.val[0]);
+ d5s16 = vsub_s16(v2tmp0.val[0], v2tmp1.val[1]);
+ d7s16 = vsub_s16(v2tmp0.val[1], v2tmp1.val[0]);
+ q2s16 = vcombine_s16(d4s16, d5s16);
+ q3s16 = vcombine_s16(d6s16, d7s16);
+
+ qAdd3 = vdupq_n_s16(3);
+
+ q0s16 = vaddq_s16(q2s16, q3s16);
+ q1s16 = vsubq_s16(q2s16, q3s16);
+
+ q0s16 = vaddq_s16(q0s16, qAdd3);
+ q1s16 = vaddq_s16(q1s16, qAdd3);
+
+ q0s16 = vshrq_n_s16(q0s16, 3);
+ q1s16 = vshrq_n_s16(q1s16, 3);
+
+ // store
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q0s16), 0);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q0s16), 0);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q1s16), 0);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q1s16), 0);
+ mb_dqcoeff += 16;
+
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q0s16), 1);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q0s16), 1);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q1s16), 1);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q1s16), 1);
+ mb_dqcoeff += 16;
+
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q0s16), 2);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q0s16), 2);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q1s16), 2);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q1s16), 2);
+ mb_dqcoeff += 16;
+
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q0s16), 3);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q0s16), 3);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_low_s16(q1s16), 3);
+ mb_dqcoeff += 16;
+ vst1_lane_s16(mb_dqcoeff, vget_high_s16(q1s16), 3);
+ mb_dqcoeff += 16;
+ return;
+}
diff --git a/vp8/common/arm/neon/loopfilter_neon.asm b/vp8/common/arm/neon/loopfilter_neon.asm
deleted file mode 100644
index c4f09c7..0000000
--- a/vp8/common/arm/neon/loopfilter_neon.asm
+++ /dev/null
@@ -1,409 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_loop_filter_horizontal_edge_y_neon|
- EXPORT |vp8_loop_filter_horizontal_edge_uv_neon|
- EXPORT |vp8_loop_filter_vertical_edge_y_neon|
- EXPORT |vp8_loop_filter_vertical_edge_uv_neon|
- ARM
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-; r0 unsigned char *src
-; r1 int pitch
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-|vp8_loop_filter_horizontal_edge_y_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- vdup.u8 q0, r2 ; duplicate blimit
- vdup.u8 q1, r3 ; duplicate limit
- sub r2, r0, r1, lsl #2 ; move src pointer down by 4 lines
- ldr r3, [sp, #68] ; load thresh
- add r12, r2, r1
- add r1, r1, r1
-
- vdup.u8 q2, r3 ; duplicate thresh
-
- vld1.u8 {q3}, [r2@128], r1 ; p3
- vld1.u8 {q4}, [r12@128], r1 ; p2
- vld1.u8 {q5}, [r2@128], r1 ; p1
- vld1.u8 {q6}, [r12@128], r1 ; p0
- vld1.u8 {q7}, [r2@128], r1 ; q0
- vld1.u8 {q8}, [r12@128], r1 ; q1
- vld1.u8 {q9}, [r2@128] ; q2
- vld1.u8 {q10}, [r12@128] ; q3
-
- sub r2, r2, r1, lsl #1
- sub r12, r12, r1, lsl #1
-
- bl vp8_loop_filter_neon
-
- vst1.u8 {q5}, [r2@128], r1 ; store op1
- vst1.u8 {q6}, [r12@128], r1 ; store op0
- vst1.u8 {q7}, [r2@128], r1 ; store oq0
- vst1.u8 {q8}, [r12@128], r1 ; store oq1
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_loop_filter_horizontal_edge_y_neon|
-
-
-; r0 unsigned char *u,
-; r1 int pitch,
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-; sp+4 unsigned char *v
-|vp8_loop_filter_horizontal_edge_uv_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- vdup.u8 q0, r2 ; duplicate blimit
- vdup.u8 q1, r3 ; duplicate limit
- ldr r12, [sp, #68] ; load thresh
- ldr r2, [sp, #72] ; load v ptr
- vdup.u8 q2, r12 ; duplicate thresh
-
- sub r3, r0, r1, lsl #2 ; move u pointer down by 4 lines
- sub r12, r2, r1, lsl #2 ; move v pointer down by 4 lines
-
- vld1.u8 {d6}, [r3@64], r1 ; p3
- vld1.u8 {d7}, [r12@64], r1 ; p3
- vld1.u8 {d8}, [r3@64], r1 ; p2
- vld1.u8 {d9}, [r12@64], r1 ; p2
- vld1.u8 {d10}, [r3@64], r1 ; p1
- vld1.u8 {d11}, [r12@64], r1 ; p1
- vld1.u8 {d12}, [r3@64], r1 ; p0
- vld1.u8 {d13}, [r12@64], r1 ; p0
- vld1.u8 {d14}, [r3@64], r1 ; q0
- vld1.u8 {d15}, [r12@64], r1 ; q0
- vld1.u8 {d16}, [r3@64], r1 ; q1
- vld1.u8 {d17}, [r12@64], r1 ; q1
- vld1.u8 {d18}, [r3@64], r1 ; q2
- vld1.u8 {d19}, [r12@64], r1 ; q2
- vld1.u8 {d20}, [r3@64] ; q3
- vld1.u8 {d21}, [r12@64] ; q3
-
- bl vp8_loop_filter_neon
-
- sub r0, r0, r1, lsl #1
- sub r2, r2, r1, lsl #1
-
- vst1.u8 {d10}, [r0@64], r1 ; store u op1
- vst1.u8 {d11}, [r2@64], r1 ; store v op1
- vst1.u8 {d12}, [r0@64], r1 ; store u op0
- vst1.u8 {d13}, [r2@64], r1 ; store v op0
- vst1.u8 {d14}, [r0@64], r1 ; store u oq0
- vst1.u8 {d15}, [r2@64], r1 ; store v oq0
- vst1.u8 {d16}, [r0@64] ; store u oq1
- vst1.u8 {d17}, [r2@64] ; store v oq1
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_loop_filter_horizontal_edge_uv_neon|
-
-; void vp8_loop_filter_vertical_edge_y_neon(unsigned char *src, int pitch,
-; const signed char *flimit,
-; const signed char *limit,
-; const signed char *thresh,
-; int count)
-; r0 unsigned char *src
-; r1 int pitch
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-
-|vp8_loop_filter_vertical_edge_y_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- vdup.u8 q0, r2 ; duplicate blimit
- vdup.u8 q1, r3 ; duplicate limit
- sub r2, r0, #4 ; src ptr down by 4 columns
- add r1, r1, r1
- ldr r3, [sp, #68] ; load thresh
- add r12, r2, r1, asr #1
-
- vld1.u8 {d6}, [r2], r1
- vld1.u8 {d8}, [r12], r1
- vld1.u8 {d10}, [r2], r1
- vld1.u8 {d12}, [r12], r1
- vld1.u8 {d14}, [r2], r1
- vld1.u8 {d16}, [r12], r1
- vld1.u8 {d18}, [r2], r1
- vld1.u8 {d20}, [r12], r1
-
- vld1.u8 {d7}, [r2], r1 ; load second 8-line src data
- vld1.u8 {d9}, [r12], r1
- vld1.u8 {d11}, [r2], r1
- vld1.u8 {d13}, [r12], r1
- vld1.u8 {d15}, [r2], r1
- vld1.u8 {d17}, [r12], r1
- vld1.u8 {d19}, [r2]
- vld1.u8 {d21}, [r12]
-
- ;transpose to 8x16 matrix
- vtrn.32 q3, q7
- vtrn.32 q4, q8
- vtrn.32 q5, q9
- vtrn.32 q6, q10
-
- vdup.u8 q2, r3 ; duplicate thresh
-
- vtrn.16 q3, q5
- vtrn.16 q4, q6
- vtrn.16 q7, q9
- vtrn.16 q8, q10
-
- vtrn.8 q3, q4
- vtrn.8 q5, q6
- vtrn.8 q7, q8
- vtrn.8 q9, q10
-
- bl vp8_loop_filter_neon
-
- vswp d12, d11
- vswp d16, d13
-
- sub r0, r0, #2 ; dst ptr
-
- vswp d14, d12
- vswp d16, d15
-
- add r12, r0, r1, asr #1
-
- ;store op1, op0, oq0, oq1
- vst4.8 {d10[0], d11[0], d12[0], d13[0]}, [r0], r1
- vst4.8 {d10[1], d11[1], d12[1], d13[1]}, [r12], r1
- vst4.8 {d10[2], d11[2], d12[2], d13[2]}, [r0], r1
- vst4.8 {d10[3], d11[3], d12[3], d13[3]}, [r12], r1
- vst4.8 {d10[4], d11[4], d12[4], d13[4]}, [r0], r1
- vst4.8 {d10[5], d11[5], d12[5], d13[5]}, [r12], r1
- vst4.8 {d10[6], d11[6], d12[6], d13[6]}, [r0], r1
- vst4.8 {d10[7], d11[7], d12[7], d13[7]}, [r12], r1
-
- vst4.8 {d14[0], d15[0], d16[0], d17[0]}, [r0], r1
- vst4.8 {d14[1], d15[1], d16[1], d17[1]}, [r12], r1
- vst4.8 {d14[2], d15[2], d16[2], d17[2]}, [r0], r1
- vst4.8 {d14[3], d15[3], d16[3], d17[3]}, [r12], r1
- vst4.8 {d14[4], d15[4], d16[4], d17[4]}, [r0], r1
- vst4.8 {d14[5], d15[5], d16[5], d17[5]}, [r12], r1
- vst4.8 {d14[6], d15[6], d16[6], d17[6]}, [r0]
- vst4.8 {d14[7], d15[7], d16[7], d17[7]}, [r12]
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_loop_filter_vertical_edge_y_neon|
-
-; void vp8_loop_filter_vertical_edge_uv_neon(unsigned char *u, int pitch
-; const signed char *flimit,
-; const signed char *limit,
-; const signed char *thresh,
-; unsigned char *v)
-; r0 unsigned char *u,
-; r1 int pitch,
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-; sp+4 unsigned char *v
-|vp8_loop_filter_vertical_edge_uv_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- vdup.u8 q0, r2 ; duplicate blimit
- sub r12, r0, #4 ; move u pointer down by 4 columns
- ldr r2, [sp, #72] ; load v ptr
- vdup.u8 q1, r3 ; duplicate limit
- sub r3, r2, #4 ; move v pointer down by 4 columns
-
- vld1.u8 {d6}, [r12], r1 ;load u data
- vld1.u8 {d7}, [r3], r1 ;load v data
- vld1.u8 {d8}, [r12], r1
- vld1.u8 {d9}, [r3], r1
- vld1.u8 {d10}, [r12], r1
- vld1.u8 {d11}, [r3], r1
- vld1.u8 {d12}, [r12], r1
- vld1.u8 {d13}, [r3], r1
- vld1.u8 {d14}, [r12], r1
- vld1.u8 {d15}, [r3], r1
- vld1.u8 {d16}, [r12], r1
- vld1.u8 {d17}, [r3], r1
- vld1.u8 {d18}, [r12], r1
- vld1.u8 {d19}, [r3], r1
- vld1.u8 {d20}, [r12]
- vld1.u8 {d21}, [r3]
-
- ldr r12, [sp, #68] ; load thresh
-
- ;transpose to 8x16 matrix
- vtrn.32 q3, q7
- vtrn.32 q4, q8
- vtrn.32 q5, q9
- vtrn.32 q6, q10
-
- vdup.u8 q2, r12 ; duplicate thresh
-
- vtrn.16 q3, q5
- vtrn.16 q4, q6
- vtrn.16 q7, q9
- vtrn.16 q8, q10
-
- vtrn.8 q3, q4
- vtrn.8 q5, q6
- vtrn.8 q7, q8
- vtrn.8 q9, q10
-
- bl vp8_loop_filter_neon
-
- vswp d12, d11
- vswp d16, d13
- vswp d14, d12
- vswp d16, d15
-
- sub r0, r0, #2
- sub r2, r2, #2
-
- ;store op1, op0, oq0, oq1
- vst4.8 {d10[0], d11[0], d12[0], d13[0]}, [r0], r1
- vst4.8 {d14[0], d15[0], d16[0], d17[0]}, [r2], r1
- vst4.8 {d10[1], d11[1], d12[1], d13[1]}, [r0], r1
- vst4.8 {d14[1], d15[1], d16[1], d17[1]}, [r2], r1
- vst4.8 {d10[2], d11[2], d12[2], d13[2]}, [r0], r1
- vst4.8 {d14[2], d15[2], d16[2], d17[2]}, [r2], r1
- vst4.8 {d10[3], d11[3], d12[3], d13[3]}, [r0], r1
- vst4.8 {d14[3], d15[3], d16[3], d17[3]}, [r2], r1
- vst4.8 {d10[4], d11[4], d12[4], d13[4]}, [r0], r1
- vst4.8 {d14[4], d15[4], d16[4], d17[4]}, [r2], r1
- vst4.8 {d10[5], d11[5], d12[5], d13[5]}, [r0], r1
- vst4.8 {d14[5], d15[5], d16[5], d17[5]}, [r2], r1
- vst4.8 {d10[6], d11[6], d12[6], d13[6]}, [r0], r1
- vst4.8 {d14[6], d15[6], d16[6], d17[6]}, [r2], r1
- vst4.8 {d10[7], d11[7], d12[7], d13[7]}, [r0]
- vst4.8 {d14[7], d15[7], d16[7], d17[7]}, [r2]
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_loop_filter_vertical_edge_uv_neon|
-
-; void vp8_loop_filter_neon();
-; This is a helper function for the loopfilters. The invidual functions do the
-; necessary load, transpose (if necessary) and store.
-
-; r0-r3 PRESERVE
-; q0 flimit
-; q1 limit
-; q2 thresh
-; q3 p3
-; q4 p2
-; q5 p1
-; q6 p0
-; q7 q0
-; q8 q1
-; q9 q2
-; q10 q3
-|vp8_loop_filter_neon| PROC
-
- ; vp8_filter_mask
- vabd.u8 q11, q3, q4 ; abs(p3 - p2)
- vabd.u8 q12, q4, q5 ; abs(p2 - p1)
- vabd.u8 q13, q5, q6 ; abs(p1 - p0)
- vabd.u8 q14, q8, q7 ; abs(q1 - q0)
- vabd.u8 q3, q9, q8 ; abs(q2 - q1)
- vabd.u8 q4, q10, q9 ; abs(q3 - q2)
-
- vmax.u8 q11, q11, q12
- vmax.u8 q12, q13, q14
- vmax.u8 q3, q3, q4
- vmax.u8 q15, q11, q12
-
- vabd.u8 q9, q6, q7 ; abs(p0 - q0)
-
- ; vp8_hevmask
- vcgt.u8 q13, q13, q2 ; (abs(p1 - p0) > thresh)*-1
- vcgt.u8 q14, q14, q2 ; (abs(q1 - q0) > thresh)*-1
- vmax.u8 q15, q15, q3
-
- vmov.u8 q10, #0x80 ; 0x80
-
- vabd.u8 q2, q5, q8 ; a = abs(p1 - q1)
- vqadd.u8 q9, q9, q9 ; b = abs(p0 - q0) * 2
-
- vcge.u8 q15, q1, q15
-
- ; vp8_filter() function
- ; convert to signed
- veor q7, q7, q10 ; qs0
- vshr.u8 q2, q2, #1 ; a = a / 2
- veor q6, q6, q10 ; ps0
-
- veor q5, q5, q10 ; ps1
- vqadd.u8 q9, q9, q2 ; a = b + a
-
- veor q8, q8, q10 ; qs1
-
- vmov.u8 q10, #3 ; #3
-
- vsubl.s8 q2, d14, d12 ; ( qs0 - ps0)
- vsubl.s8 q11, d15, d13
-
- vcge.u8 q9, q0, q9 ; (a > flimit * 2 + limit) * -1
-
- vmovl.u8 q4, d20
-
- vqsub.s8 q1, q5, q8 ; vp8_filter = clamp(ps1-qs1)
- vorr q14, q13, q14 ; vp8_hevmask
-
- vmul.i16 q2, q2, q4 ; 3 * ( qs0 - ps0)
- vmul.i16 q11, q11, q4
-
- vand q1, q1, q14 ; vp8_filter &= hev
- vand q15, q15, q9 ; vp8_filter_mask
-
- vaddw.s8 q2, q2, d2
- vaddw.s8 q11, q11, d3
-
- vmov.u8 q9, #4 ; #4
-
- ; vp8_filter = clamp(vp8_filter + 3 * ( qs0 - ps0))
- vqmovn.s16 d2, q2
- vqmovn.s16 d3, q11
- vand q1, q1, q15 ; vp8_filter &= mask
-
- vqadd.s8 q2, q1, q10 ; Filter2 = clamp(vp8_filter+3)
- vqadd.s8 q1, q1, q9 ; Filter1 = clamp(vp8_filter+4)
- vshr.s8 q2, q2, #3 ; Filter2 >>= 3
- vshr.s8 q1, q1, #3 ; Filter1 >>= 3
-
-
- vqadd.s8 q11, q6, q2 ; u = clamp(ps0 + Filter2)
- vqsub.s8 q10, q7, q1 ; u = clamp(qs0 - Filter1)
-
- ; outer tap adjustments: ++vp8_filter >> 1
- vrshr.s8 q1, q1, #1
- vbic q1, q1, q14 ; vp8_filter &= ~hev
- vmov.u8 q0, #0x80 ; 0x80
- vqadd.s8 q13, q5, q1 ; u = clamp(ps1 + vp8_filter)
- vqsub.s8 q12, q8, q1 ; u = clamp(qs1 - vp8_filter)
-
- veor q6, q11, q0 ; *op0 = u^0x80
- veor q7, q10, q0 ; *oq0 = u^0x80
- veor q5, q13, q0 ; *op1 = u^0x80
- veor q8, q12, q0 ; *oq1 = u^0x80
-
- bx lr
- ENDP ; |vp8_loop_filter_horizontal_edge_y_neon|
-
-;-----------------
-
- END
diff --git a/vp8/common/arm/neon/loopfilter_neon.c b/vp8/common/arm/neon/loopfilter_neon.c
new file mode 100644
index 0000000..be77e67
--- /dev/null
+++ b/vp8/common/arm/neon/loopfilter_neon.c
@@ -0,0 +1,539 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+#include "./vpx_config.h"
+
+static INLINE void vp8_loop_filter_neon(
+ uint8x16_t qblimit, // flimit
+ uint8x16_t qlimit, // limit
+ uint8x16_t qthresh, // thresh
+ uint8x16_t q3, // p3
+ uint8x16_t q4, // p2
+ uint8x16_t q5, // p1
+ uint8x16_t q6, // p0
+ uint8x16_t q7, // q0
+ uint8x16_t q8, // q1
+ uint8x16_t q9, // q2
+ uint8x16_t q10, // q3
+ uint8x16_t *q5r, // p1
+ uint8x16_t *q6r, // p0
+ uint8x16_t *q7r, // q0
+ uint8x16_t *q8r) { // q1
+ uint8x16_t q0u8, q1u8, q2u8, q11u8, q12u8, q13u8, q14u8, q15u8;
+ int16x8_t q2s16, q11s16;
+ uint16x8_t q4u16;
+ int8x16_t q1s8, q2s8, q10s8, q11s8, q12s8, q13s8;
+ int8x8_t d2s8, d3s8;
+
+ q11u8 = vabdq_u8(q3, q4);
+ q12u8 = vabdq_u8(q4, q5);
+ q13u8 = vabdq_u8(q5, q6);
+ q14u8 = vabdq_u8(q8, q7);
+ q3 = vabdq_u8(q9, q8);
+ q4 = vabdq_u8(q10, q9);
+
+ q11u8 = vmaxq_u8(q11u8, q12u8);
+ q12u8 = vmaxq_u8(q13u8, q14u8);
+ q3 = vmaxq_u8(q3, q4);
+ q15u8 = vmaxq_u8(q11u8, q12u8);
+
+ q9 = vabdq_u8(q6, q7);
+
+ // vp8_hevmask
+ q13u8 = vcgtq_u8(q13u8, qthresh);
+ q14u8 = vcgtq_u8(q14u8, qthresh);
+ q15u8 = vmaxq_u8(q15u8, q3);
+
+ q2u8 = vabdq_u8(q5, q8);
+ q9 = vqaddq_u8(q9, q9);
+
+ q15u8 = vcgeq_u8(qlimit, q15u8);
+
+ // vp8_filter() function
+ // convert to signed
+ q10 = vdupq_n_u8(0x80);
+ q8 = veorq_u8(q8, q10);
+ q7 = veorq_u8(q7, q10);
+ q6 = veorq_u8(q6, q10);
+ q5 = veorq_u8(q5, q10);
+
+ q2u8 = vshrq_n_u8(q2u8, 1);
+ q9 = vqaddq_u8(q9, q2u8);
+
+ q10 = vdupq_n_u8(3);
+
+ q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q7)),
+ vget_low_s8(vreinterpretq_s8_u8(q6)));
+ q11s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q7)),
+ vget_high_s8(vreinterpretq_s8_u8(q6)));
+
+ q9 = vcgeq_u8(qblimit, q9);
+
+ q1s8 = vqsubq_s8(vreinterpretq_s8_u8(q5),
+ vreinterpretq_s8_u8(q8));
+
+ q14u8 = vorrq_u8(q13u8, q14u8);
+
+ q4u16 = vmovl_u8(vget_low_u8(q10));
+ q2s16 = vmulq_s16(q2s16, vreinterpretq_s16_u16(q4u16));
+ q11s16 = vmulq_s16(q11s16, vreinterpretq_s16_u16(q4u16));
+
+ q1u8 = vandq_u8(vreinterpretq_u8_s8(q1s8), q14u8);
+ q15u8 = vandq_u8(q15u8, q9);
+
+ q1s8 = vreinterpretq_s8_u8(q1u8);
+ q2s16 = vaddw_s8(q2s16, vget_low_s8(q1s8));
+ q11s16 = vaddw_s8(q11s16, vget_high_s8(q1s8));
+
+ q9 = vdupq_n_u8(4);
+ // vp8_filter = clamp(vp8_filter + 3 * ( qs0 - ps0))
+ d2s8 = vqmovn_s16(q2s16);
+ d3s8 = vqmovn_s16(q11s16);
+ q1s8 = vcombine_s8(d2s8, d3s8);
+ q1u8 = vandq_u8(vreinterpretq_u8_s8(q1s8), q15u8);
+ q1s8 = vreinterpretq_s8_u8(q1u8);
+
+ q2s8 = vqaddq_s8(q1s8, vreinterpretq_s8_u8(q10));
+ q1s8 = vqaddq_s8(q1s8, vreinterpretq_s8_u8(q9));
+ q2s8 = vshrq_n_s8(q2s8, 3);
+ q1s8 = vshrq_n_s8(q1s8, 3);
+
+ q11s8 = vqaddq_s8(vreinterpretq_s8_u8(q6), q2s8);
+ q10s8 = vqsubq_s8(vreinterpretq_s8_u8(q7), q1s8);
+
+ q1s8 = vrshrq_n_s8(q1s8, 1);
+ q1s8 = vbicq_s8(q1s8, vreinterpretq_s8_u8(q14u8));
+
+ q13s8 = vqaddq_s8(vreinterpretq_s8_u8(q5), q1s8);
+ q12s8 = vqsubq_s8(vreinterpretq_s8_u8(q8), q1s8);
+
+ q0u8 = vdupq_n_u8(0x80);
+ *q8r = veorq_u8(vreinterpretq_u8_s8(q12s8), q0u8);
+ *q7r = veorq_u8(vreinterpretq_u8_s8(q10s8), q0u8);
+ *q6r = veorq_u8(vreinterpretq_u8_s8(q11s8), q0u8);
+ *q5r = veorq_u8(vreinterpretq_u8_s8(q13s8), q0u8);
+ return;
+}
+
+void vp8_loop_filter_horizontal_edge_y_neon(
+ unsigned char *src,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh) {
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+ src -= (pitch << 2);
+
+ q3 = vld1q_u8(src);
+ src += pitch;
+ q4 = vld1q_u8(src);
+ src += pitch;
+ q5 = vld1q_u8(src);
+ src += pitch;
+ q6 = vld1q_u8(src);
+ src += pitch;
+ q7 = vld1q_u8(src);
+ src += pitch;
+ q8 = vld1q_u8(src);
+ src += pitch;
+ q9 = vld1q_u8(src);
+ src += pitch;
+ q10 = vld1q_u8(src);
+
+ vp8_loop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q5, &q6, &q7, &q8);
+
+ src -= (pitch * 5);
+ vst1q_u8(src, q5);
+ src += pitch;
+ vst1q_u8(src, q6);
+ src += pitch;
+ vst1q_u8(src, q7);
+ src += pitch;
+ vst1q_u8(src, q8);
+ return;
+}
+
+void vp8_loop_filter_horizontal_edge_uv_neon(
+ unsigned char *u,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh,
+ unsigned char *v) {
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+ uint8x8_t d6, d7, d8, d9, d10, d11, d12, d13, d14;
+ uint8x8_t d15, d16, d17, d18, d19, d20, d21;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ u -= (pitch << 2);
+ v -= (pitch << 2);
+
+ d6 = vld1_u8(u);
+ u += pitch;
+ d7 = vld1_u8(v);
+ v += pitch;
+ d8 = vld1_u8(u);
+ u += pitch;
+ d9 = vld1_u8(v);
+ v += pitch;
+ d10 = vld1_u8(u);
+ u += pitch;
+ d11 = vld1_u8(v);
+ v += pitch;
+ d12 = vld1_u8(u);
+ u += pitch;
+ d13 = vld1_u8(v);
+ v += pitch;
+ d14 = vld1_u8(u);
+ u += pitch;
+ d15 = vld1_u8(v);
+ v += pitch;
+ d16 = vld1_u8(u);
+ u += pitch;
+ d17 = vld1_u8(v);
+ v += pitch;
+ d18 = vld1_u8(u);
+ u += pitch;
+ d19 = vld1_u8(v);
+ v += pitch;
+ d20 = vld1_u8(u);
+ d21 = vld1_u8(v);
+
+ q3 = vcombine_u8(d6, d7);
+ q4 = vcombine_u8(d8, d9);
+ q5 = vcombine_u8(d10, d11);
+ q6 = vcombine_u8(d12, d13);
+ q7 = vcombine_u8(d14, d15);
+ q8 = vcombine_u8(d16, d17);
+ q9 = vcombine_u8(d18, d19);
+ q10 = vcombine_u8(d20, d21);
+
+ vp8_loop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q5, &q6, &q7, &q8);
+
+ u -= (pitch * 5);
+ vst1_u8(u, vget_low_u8(q5));
+ u += pitch;
+ vst1_u8(u, vget_low_u8(q6));
+ u += pitch;
+ vst1_u8(u, vget_low_u8(q7));
+ u += pitch;
+ vst1_u8(u, vget_low_u8(q8));
+
+ v -= (pitch * 5);
+ vst1_u8(v, vget_high_u8(q5));
+ v += pitch;
+ vst1_u8(v, vget_high_u8(q6));
+ v += pitch;
+ vst1_u8(v, vget_high_u8(q7));
+ v += pitch;
+ vst1_u8(v, vget_high_u8(q8));
+ return;
+}
+
+void vp8_loop_filter_vertical_edge_y_neon(
+ unsigned char *src,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh) {
+ unsigned char *s, *d;
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+ uint8x8_t d6, d7, d8, d9, d10, d11, d12, d13, d14;
+ uint8x8_t d15, d16, d17, d18, d19, d20, d21;
+ uint32x4x2_t q2tmp0, q2tmp1, q2tmp2, q2tmp3;
+ uint16x8x2_t q2tmp4, q2tmp5, q2tmp6, q2tmp7;
+ uint8x16x2_t q2tmp8, q2tmp9, q2tmp10, q2tmp11;
+ uint8x8x4_t q4ResultH, q4ResultL;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ s = src - 4;
+ d6 = vld1_u8(s);
+ s += pitch;
+ d8 = vld1_u8(s);
+ s += pitch;
+ d10 = vld1_u8(s);
+ s += pitch;
+ d12 = vld1_u8(s);
+ s += pitch;
+ d14 = vld1_u8(s);
+ s += pitch;
+ d16 = vld1_u8(s);
+ s += pitch;
+ d18 = vld1_u8(s);
+ s += pitch;
+ d20 = vld1_u8(s);
+ s += pitch;
+ d7 = vld1_u8(s);
+ s += pitch;
+ d9 = vld1_u8(s);
+ s += pitch;
+ d11 = vld1_u8(s);
+ s += pitch;
+ d13 = vld1_u8(s);
+ s += pitch;
+ d15 = vld1_u8(s);
+ s += pitch;
+ d17 = vld1_u8(s);
+ s += pitch;
+ d19 = vld1_u8(s);
+ s += pitch;
+ d21 = vld1_u8(s);
+
+ q3 = vcombine_u8(d6, d7);
+ q4 = vcombine_u8(d8, d9);
+ q5 = vcombine_u8(d10, d11);
+ q6 = vcombine_u8(d12, d13);
+ q7 = vcombine_u8(d14, d15);
+ q8 = vcombine_u8(d16, d17);
+ q9 = vcombine_u8(d18, d19);
+ q10 = vcombine_u8(d20, d21);
+
+ q2tmp0 = vtrnq_u32(vreinterpretq_u32_u8(q3), vreinterpretq_u32_u8(q7));
+ q2tmp1 = vtrnq_u32(vreinterpretq_u32_u8(q4), vreinterpretq_u32_u8(q8));
+ q2tmp2 = vtrnq_u32(vreinterpretq_u32_u8(q5), vreinterpretq_u32_u8(q9));
+ q2tmp3 = vtrnq_u32(vreinterpretq_u32_u8(q6), vreinterpretq_u32_u8(q10));
+
+ q2tmp4 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[0]),
+ vreinterpretq_u16_u32(q2tmp2.val[0]));
+ q2tmp5 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[0]),
+ vreinterpretq_u16_u32(q2tmp3.val[0]));
+ q2tmp6 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[1]),
+ vreinterpretq_u16_u32(q2tmp2.val[1]));
+ q2tmp7 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[1]),
+ vreinterpretq_u16_u32(q2tmp3.val[1]));
+
+ q2tmp8 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[0]),
+ vreinterpretq_u8_u16(q2tmp5.val[0]));
+ q2tmp9 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[1]),
+ vreinterpretq_u8_u16(q2tmp5.val[1]));
+ q2tmp10 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[0]),
+ vreinterpretq_u8_u16(q2tmp7.val[0]));
+ q2tmp11 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[1]),
+ vreinterpretq_u8_u16(q2tmp7.val[1]));
+
+ q3 = q2tmp8.val[0];
+ q4 = q2tmp8.val[1];
+ q5 = q2tmp9.val[0];
+ q6 = q2tmp9.val[1];
+ q7 = q2tmp10.val[0];
+ q8 = q2tmp10.val[1];
+ q9 = q2tmp11.val[0];
+ q10 = q2tmp11.val[1];
+
+ vp8_loop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q5, &q6, &q7, &q8);
+
+ q4ResultL.val[0] = vget_low_u8(q5); // d10
+ q4ResultL.val[1] = vget_low_u8(q6); // d12
+ q4ResultL.val[2] = vget_low_u8(q7); // d14
+ q4ResultL.val[3] = vget_low_u8(q8); // d16
+ q4ResultH.val[0] = vget_high_u8(q5); // d11
+ q4ResultH.val[1] = vget_high_u8(q6); // d13
+ q4ResultH.val[2] = vget_high_u8(q7); // d15
+ q4ResultH.val[3] = vget_high_u8(q8); // d17
+
+ d = src - 2;
+ vst4_lane_u8(d, q4ResultL, 0);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 1);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 2);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 3);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 4);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 5);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 6);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultL, 7);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 0);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 1);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 2);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 3);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 4);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 5);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 6);
+ d += pitch;
+ vst4_lane_u8(d, q4ResultH, 7);
+ return;
+}
+
+void vp8_loop_filter_vertical_edge_uv_neon(
+ unsigned char *u,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh,
+ unsigned char *v) {
+ unsigned char *us, *ud;
+ unsigned char *vs, *vd;
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+ uint8x8_t d6, d7, d8, d9, d10, d11, d12, d13, d14;
+ uint8x8_t d15, d16, d17, d18, d19, d20, d21;
+ uint32x4x2_t q2tmp0, q2tmp1, q2tmp2, q2tmp3;
+ uint16x8x2_t q2tmp4, q2tmp5, q2tmp6, q2tmp7;
+ uint8x16x2_t q2tmp8, q2tmp9, q2tmp10, q2tmp11;
+ uint8x8x4_t q4ResultH, q4ResultL;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ us = u - 4;
+ d6 = vld1_u8(us);
+ us += pitch;
+ d8 = vld1_u8(us);
+ us += pitch;
+ d10 = vld1_u8(us);
+ us += pitch;
+ d12 = vld1_u8(us);
+ us += pitch;
+ d14 = vld1_u8(us);
+ us += pitch;
+ d16 = vld1_u8(us);
+ us += pitch;
+ d18 = vld1_u8(us);
+ us += pitch;
+ d20 = vld1_u8(us);
+
+ vs = v - 4;
+ d7 = vld1_u8(vs);
+ vs += pitch;
+ d9 = vld1_u8(vs);
+ vs += pitch;
+ d11 = vld1_u8(vs);
+ vs += pitch;
+ d13 = vld1_u8(vs);
+ vs += pitch;
+ d15 = vld1_u8(vs);
+ vs += pitch;
+ d17 = vld1_u8(vs);
+ vs += pitch;
+ d19 = vld1_u8(vs);
+ vs += pitch;
+ d21 = vld1_u8(vs);
+
+ q3 = vcombine_u8(d6, d7);
+ q4 = vcombine_u8(d8, d9);
+ q5 = vcombine_u8(d10, d11);
+ q6 = vcombine_u8(d12, d13);
+ q7 = vcombine_u8(d14, d15);
+ q8 = vcombine_u8(d16, d17);
+ q9 = vcombine_u8(d18, d19);
+ q10 = vcombine_u8(d20, d21);
+
+ q2tmp0 = vtrnq_u32(vreinterpretq_u32_u8(q3), vreinterpretq_u32_u8(q7));
+ q2tmp1 = vtrnq_u32(vreinterpretq_u32_u8(q4), vreinterpretq_u32_u8(q8));
+ q2tmp2 = vtrnq_u32(vreinterpretq_u32_u8(q5), vreinterpretq_u32_u8(q9));
+ q2tmp3 = vtrnq_u32(vreinterpretq_u32_u8(q6), vreinterpretq_u32_u8(q10));
+
+ q2tmp4 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[0]),
+ vreinterpretq_u16_u32(q2tmp2.val[0]));
+ q2tmp5 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[0]),
+ vreinterpretq_u16_u32(q2tmp3.val[0]));
+ q2tmp6 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[1]),
+ vreinterpretq_u16_u32(q2tmp2.val[1]));
+ q2tmp7 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[1]),
+ vreinterpretq_u16_u32(q2tmp3.val[1]));
+
+ q2tmp8 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[0]),
+ vreinterpretq_u8_u16(q2tmp5.val[0]));
+ q2tmp9 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[1]),
+ vreinterpretq_u8_u16(q2tmp5.val[1]));
+ q2tmp10 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[0]),
+ vreinterpretq_u8_u16(q2tmp7.val[0]));
+ q2tmp11 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[1]),
+ vreinterpretq_u8_u16(q2tmp7.val[1]));
+
+ q3 = q2tmp8.val[0];
+ q4 = q2tmp8.val[1];
+ q5 = q2tmp9.val[0];
+ q6 = q2tmp9.val[1];
+ q7 = q2tmp10.val[0];
+ q8 = q2tmp10.val[1];
+ q9 = q2tmp11.val[0];
+ q10 = q2tmp11.val[1];
+
+ vp8_loop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q5, &q6, &q7, &q8);
+
+ q4ResultL.val[0] = vget_low_u8(q5); // d10
+ q4ResultL.val[1] = vget_low_u8(q6); // d12
+ q4ResultL.val[2] = vget_low_u8(q7); // d14
+ q4ResultL.val[3] = vget_low_u8(q8); // d16
+ ud = u - 2;
+ vst4_lane_u8(ud, q4ResultL, 0);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 1);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 2);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 3);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 4);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 5);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 6);
+ ud += pitch;
+ vst4_lane_u8(ud, q4ResultL, 7);
+
+ q4ResultH.val[0] = vget_high_u8(q5); // d11
+ q4ResultH.val[1] = vget_high_u8(q6); // d13
+ q4ResultH.val[2] = vget_high_u8(q7); // d15
+ q4ResultH.val[3] = vget_high_u8(q8); // d17
+ vd = v - 2;
+ vst4_lane_u8(vd, q4ResultH, 0);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 1);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 2);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 3);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 4);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 5);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 6);
+ vd += pitch;
+ vst4_lane_u8(vd, q4ResultH, 7);
+ return;
+}
diff --git a/vp8/common/arm/neon/loopfiltersimplehorizontaledge_neon.asm b/vp8/common/arm/neon/loopfiltersimplehorizontaledge_neon.asm
deleted file mode 100644
index 6eb0651..0000000
--- a/vp8/common/arm/neon/loopfiltersimplehorizontaledge_neon.asm
+++ /dev/null
@@ -1,117 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_loop_filter_bhs_neon|
- EXPORT |vp8_loop_filter_mbhs_neon|
- ARM
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-; r0 unsigned char *s, PRESERVE
-; r1 int p, PRESERVE
-; q1 limit, PRESERVE
-
-|vp8_loop_filter_simple_horizontal_edge_neon| PROC
- vpush {d8-d15}
- sub r3, r0, r1, lsl #1 ; move src pointer down by 2 lines
-
- vld1.u8 {q7}, [r0@128], r1 ; q0
- vld1.u8 {q5}, [r3@128], r1 ; p0
- vld1.u8 {q8}, [r0@128] ; q1
- vld1.u8 {q6}, [r3@128] ; p1
-
- vabd.u8 q15, q6, q7 ; abs(p0 - q0)
- vabd.u8 q14, q5, q8 ; abs(p1 - q1)
-
- vqadd.u8 q15, q15, q15 ; abs(p0 - q0) * 2
- vshr.u8 q14, q14, #1 ; abs(p1 - q1) / 2
- vmov.u8 q0, #0x80 ; 0x80
- vmov.s16 q13, #3
- vqadd.u8 q15, q15, q14 ; abs(p0 - q0) * 2 + abs(p1 - q1) / 2
-
- veor q7, q7, q0 ; qs0: q0 offset to convert to a signed value
- veor q6, q6, q0 ; ps0: p0 offset to convert to a signed value
- veor q5, q5, q0 ; ps1: p1 offset to convert to a signed value
- veor q8, q8, q0 ; qs1: q1 offset to convert to a signed value
-
- vcge.u8 q15, q1, q15 ; (abs(p0 - q0)*2 + abs(p1-q1)/2 > limit)*-1
-
- vsubl.s8 q2, d14, d12 ; ( qs0 - ps0)
- vsubl.s8 q3, d15, d13
-
- vqsub.s8 q4, q5, q8 ; q4: vp8_filter = vp8_signed_char_clamp(ps1-qs1)
-
- vmul.s16 q2, q2, q13 ; 3 * ( qs0 - ps0)
- vmul.s16 q3, q3, q13
-
- vmov.u8 q10, #0x03 ; 0x03
- vmov.u8 q9, #0x04 ; 0x04
-
- vaddw.s8 q2, q2, d8 ; vp8_filter + 3 * ( qs0 - ps0)
- vaddw.s8 q3, q3, d9
-
- vqmovn.s16 d8, q2 ; vp8_filter = vp8_signed_char_clamp(vp8_filter + 3 * ( qs0 - ps0))
- vqmovn.s16 d9, q3
-
- vand q14, q4, q15 ; vp8_filter &= mask
-
- vqadd.s8 q2, q14, q10 ; Filter2 = vp8_signed_char_clamp(vp8_filter+3)
- vqadd.s8 q3, q14, q9 ; Filter1 = vp8_signed_char_clamp(vp8_filter+4)
- vshr.s8 q2, q2, #3 ; Filter2 >>= 3
- vshr.s8 q4, q3, #3 ; Filter1 >>= 3
-
- sub r0, r0, r1
-
- ;calculate output
- vqadd.s8 q11, q6, q2 ; u = vp8_signed_char_clamp(ps0 + Filter2)
- vqsub.s8 q10, q7, q4 ; u = vp8_signed_char_clamp(qs0 - Filter1)
-
- veor q6, q11, q0 ; *op0 = u^0x80
- veor q7, q10, q0 ; *oq0 = u^0x80
-
- vst1.u8 {q6}, [r3@128] ; store op0
- vst1.u8 {q7}, [r0@128] ; store oq0
-
- vpop {d8-d15}
- bx lr
- ENDP ; |vp8_loop_filter_simple_horizontal_edge_neon|
-
-; r0 unsigned char *y
-; r1 int ystride
-; r2 const unsigned char *blimit
-
-|vp8_loop_filter_bhs_neon| PROC
- push {r4, lr}
- ldrb r3, [r2] ; load blim from mem
- vdup.s8 q1, r3 ; duplicate blim
-
- add r0, r0, r1, lsl #2 ; src = y_ptr + 4 * y_stride
- bl vp8_loop_filter_simple_horizontal_edge_neon
- ; vp8_loop_filter_simple_horizontal_edge_neon preserves r0, r1 and q1
- add r0, r0, r1, lsl #2 ; src = y_ptr + 8* y_stride
- bl vp8_loop_filter_simple_horizontal_edge_neon
- add r0, r0, r1, lsl #2 ; src = y_ptr + 12 * y_stride
- pop {r4, lr}
- b vp8_loop_filter_simple_horizontal_edge_neon
- ENDP ;|vp8_loop_filter_bhs_neon|
-
-; r0 unsigned char *y
-; r1 int ystride
-; r2 const unsigned char *blimit
-
-|vp8_loop_filter_mbhs_neon| PROC
- ldrb r3, [r2] ; load blim from mem
- vdup.s8 q1, r3 ; duplicate mblim
- b vp8_loop_filter_simple_horizontal_edge_neon
- ENDP ;|vp8_loop_filter_bhs_neon|
-
- END
diff --git a/vp8/common/arm/neon/loopfiltersimplehorizontaledge_neon.c b/vp8/common/arm/neon/loopfiltersimplehorizontaledge_neon.c
new file mode 100644
index 0000000..b25686f
--- /dev/null
+++ b/vp8/common/arm/neon/loopfiltersimplehorizontaledge_neon.c
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+#include "./vpx_config.h"
+
+static INLINE void vp8_loop_filter_simple_horizontal_edge_neon(
+ unsigned char *s,
+ int p,
+ const unsigned char *blimit) {
+ uint8_t *sp;
+ uint8x16_t qblimit, q0u8;
+ uint8x16_t q5u8, q6u8, q7u8, q8u8, q9u8, q10u8, q14u8, q15u8;
+ int16x8_t q2s16, q3s16, q13s16;
+ int8x8_t d8s8, d9s8;
+ int8x16_t q2s8, q3s8, q4s8, q10s8, q11s8, q14s8;
+
+ qblimit = vdupq_n_u8(*blimit);
+
+ sp = s - (p << 1);
+ q5u8 = vld1q_u8(sp);
+ sp += p;
+ q6u8 = vld1q_u8(sp);
+ sp += p;
+ q7u8 = vld1q_u8(sp);
+ sp += p;
+ q8u8 = vld1q_u8(sp);
+
+ q15u8 = vabdq_u8(q6u8, q7u8);
+ q14u8 = vabdq_u8(q5u8, q8u8);
+
+ q15u8 = vqaddq_u8(q15u8, q15u8);
+ q14u8 = vshrq_n_u8(q14u8, 1);
+ q0u8 = vdupq_n_u8(0x80);
+ q13s16 = vdupq_n_s16(3);
+ q15u8 = vqaddq_u8(q15u8, q14u8);
+
+ q5u8 = veorq_u8(q5u8, q0u8);
+ q6u8 = veorq_u8(q6u8, q0u8);
+ q7u8 = veorq_u8(q7u8, q0u8);
+ q8u8 = veorq_u8(q8u8, q0u8);
+
+ q15u8 = vcgeq_u8(qblimit, q15u8);
+
+ q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q7u8)),
+ vget_low_s8(vreinterpretq_s8_u8(q6u8)));
+ q3s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q7u8)),
+ vget_high_s8(vreinterpretq_s8_u8(q6u8)));
+
+ q4s8 = vqsubq_s8(vreinterpretq_s8_u8(q5u8),
+ vreinterpretq_s8_u8(q8u8));
+
+ q2s16 = vmulq_s16(q2s16, q13s16);
+ q3s16 = vmulq_s16(q3s16, q13s16);
+
+ q10u8 = vdupq_n_u8(3);
+ q9u8 = vdupq_n_u8(4);
+
+ q2s16 = vaddw_s8(q2s16, vget_low_s8(q4s8));
+ q3s16 = vaddw_s8(q3s16, vget_high_s8(q4s8));
+
+ d8s8 = vqmovn_s16(q2s16);
+ d9s8 = vqmovn_s16(q3s16);
+ q4s8 = vcombine_s8(d8s8, d9s8);
+
+ q14s8 = vandq_s8(q4s8, vreinterpretq_s8_u8(q15u8));
+
+ q2s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q10u8));
+ q3s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q9u8));
+ q2s8 = vshrq_n_s8(q2s8, 3);
+ q3s8 = vshrq_n_s8(q3s8, 3);
+
+ q11s8 = vqaddq_s8(vreinterpretq_s8_u8(q6u8), q2s8);
+ q10s8 = vqsubq_s8(vreinterpretq_s8_u8(q7u8), q3s8);
+
+ q6u8 = veorq_u8(vreinterpretq_u8_s8(q11s8), q0u8);
+ q7u8 = veorq_u8(vreinterpretq_u8_s8(q10s8), q0u8);
+
+ vst1q_u8(s, q7u8);
+ s -= p;
+ vst1q_u8(s, q6u8);
+ return;
+}
+
+void vp8_loop_filter_bhs_neon(
+ unsigned char *y_ptr,
+ int y_stride,
+ const unsigned char *blimit) {
+ y_ptr += y_stride * 4;
+ vp8_loop_filter_simple_horizontal_edge_neon(y_ptr, y_stride, blimit);
+ y_ptr += y_stride * 4;
+ vp8_loop_filter_simple_horizontal_edge_neon(y_ptr, y_stride, blimit);
+ y_ptr += y_stride * 4;
+ vp8_loop_filter_simple_horizontal_edge_neon(y_ptr, y_stride, blimit);
+ return;
+}
+
+void vp8_loop_filter_mbhs_neon(
+ unsigned char *y_ptr,
+ int y_stride,
+ const unsigned char *blimit) {
+ vp8_loop_filter_simple_horizontal_edge_neon(y_ptr, y_stride, blimit);
+ return;
+}
diff --git a/vp8/common/arm/neon/loopfiltersimpleverticaledge_neon.asm b/vp8/common/arm/neon/loopfiltersimpleverticaledge_neon.asm
deleted file mode 100644
index 78d13c8..0000000
--- a/vp8/common/arm/neon/loopfiltersimpleverticaledge_neon.asm
+++ /dev/null
@@ -1,156 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_loop_filter_bvs_neon|
- EXPORT |vp8_loop_filter_mbvs_neon|
- ARM
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-; r0 unsigned char *s, PRESERVE
-; r1 int p, PRESERVE
-; q1 limit, PRESERVE
-
-|vp8_loop_filter_simple_vertical_edge_neon| PROC
- vpush {d8-d15}
-
- sub r0, r0, #2 ; move src pointer down by 2 columns
- add r12, r1, r1
- add r3, r0, r1
-
- vld4.8 {d6[0], d7[0], d8[0], d9[0]}, [r0], r12
- vld4.8 {d6[1], d7[1], d8[1], d9[1]}, [r3], r12
- vld4.8 {d6[2], d7[2], d8[2], d9[2]}, [r0], r12
- vld4.8 {d6[3], d7[3], d8[3], d9[3]}, [r3], r12
- vld4.8 {d6[4], d7[4], d8[4], d9[4]}, [r0], r12
- vld4.8 {d6[5], d7[5], d8[5], d9[5]}, [r3], r12
- vld4.8 {d6[6], d7[6], d8[6], d9[6]}, [r0], r12
- vld4.8 {d6[7], d7[7], d8[7], d9[7]}, [r3], r12
-
- vld4.8 {d10[0], d11[0], d12[0], d13[0]}, [r0], r12
- vld4.8 {d10[1], d11[1], d12[1], d13[1]}, [r3], r12
- vld4.8 {d10[2], d11[2], d12[2], d13[2]}, [r0], r12
- vld4.8 {d10[3], d11[3], d12[3], d13[3]}, [r3], r12
- vld4.8 {d10[4], d11[4], d12[4], d13[4]}, [r0], r12
- vld4.8 {d10[5], d11[5], d12[5], d13[5]}, [r3], r12
- vld4.8 {d10[6], d11[6], d12[6], d13[6]}, [r0], r12
- vld4.8 {d10[7], d11[7], d12[7], d13[7]}, [r3]
-
- vswp d7, d10
- vswp d12, d9
-
- ;vp8_filter_mask() function
- ;vp8_hevmask() function
- sub r0, r0, r1, lsl #4
- vabd.u8 q15, q5, q4 ; abs(p0 - q0)
- vabd.u8 q14, q3, q6 ; abs(p1 - q1)
-
- vqadd.u8 q15, q15, q15 ; abs(p0 - q0) * 2
- vshr.u8 q14, q14, #1 ; abs(p1 - q1) / 2
- vmov.u8 q0, #0x80 ; 0x80
- vmov.s16 q11, #3
- vqadd.u8 q15, q15, q14 ; abs(p0 - q0) * 2 + abs(p1 - q1) / 2
-
- veor q4, q4, q0 ; qs0: q0 offset to convert to a signed value
- veor q5, q5, q0 ; ps0: p0 offset to convert to a signed value
- veor q3, q3, q0 ; ps1: p1 offset to convert to a signed value
- veor q6, q6, q0 ; qs1: q1 offset to convert to a signed value
-
- vcge.u8 q15, q1, q15 ; abs(p0 - q0)*2 + abs(p1-q1)/2 > flimit*2 + limit)*-1
-
- vsubl.s8 q2, d8, d10 ; ( qs0 - ps0)
- vsubl.s8 q13, d9, d11
-
- vqsub.s8 q14, q3, q6 ; vp8_filter = vp8_signed_char_clamp(ps1-qs1)
-
- vmul.s16 q2, q2, q11 ; 3 * ( qs0 - ps0)
- vmul.s16 q13, q13, q11
-
- vmov.u8 q11, #0x03 ; 0x03
- vmov.u8 q12, #0x04 ; 0x04
-
- vaddw.s8 q2, q2, d28 ; vp8_filter + 3 * ( qs0 - ps0)
- vaddw.s8 q13, q13, d29
-
- vqmovn.s16 d28, q2 ; vp8_filter = vp8_signed_char_clamp(vp8_filter + 3 * ( qs0 - ps0))
- vqmovn.s16 d29, q13
-
- add r0, r0, #1
- add r3, r0, r1
-
- vand q14, q14, q15 ; vp8_filter &= mask
-
- vqadd.s8 q2, q14, q11 ; Filter2 = vp8_signed_char_clamp(vp8_filter+3)
- vqadd.s8 q3, q14, q12 ; Filter1 = vp8_signed_char_clamp(vp8_filter+4)
- vshr.s8 q2, q2, #3 ; Filter2 >>= 3
- vshr.s8 q14, q3, #3 ; Filter1 >>= 3
-
- ;calculate output
- vqadd.s8 q11, q5, q2 ; u = vp8_signed_char_clamp(ps0 + Filter2)
- vqsub.s8 q10, q4, q14 ; u = vp8_signed_char_clamp(qs0 - Filter1)
-
- veor q6, q11, q0 ; *op0 = u^0x80
- veor q7, q10, q0 ; *oq0 = u^0x80
- add r12, r1, r1
- vswp d13, d14
-
- ;store op1, op0, oq0, oq1
- vst2.8 {d12[0], d13[0]}, [r0], r12
- vst2.8 {d12[1], d13[1]}, [r3], r12
- vst2.8 {d12[2], d13[2]}, [r0], r12
- vst2.8 {d12[3], d13[3]}, [r3], r12
- vst2.8 {d12[4], d13[4]}, [r0], r12
- vst2.8 {d12[5], d13[5]}, [r3], r12
- vst2.8 {d12[6], d13[6]}, [r0], r12
- vst2.8 {d12[7], d13[7]}, [r3], r12
- vst2.8 {d14[0], d15[0]}, [r0], r12
- vst2.8 {d14[1], d15[1]}, [r3], r12
- vst2.8 {d14[2], d15[2]}, [r0], r12
- vst2.8 {d14[3], d15[3]}, [r3], r12
- vst2.8 {d14[4], d15[4]}, [r0], r12
- vst2.8 {d14[5], d15[5]}, [r3], r12
- vst2.8 {d14[6], d15[6]}, [r0], r12
- vst2.8 {d14[7], d15[7]}, [r3]
-
- vpop {d8-d15}
- bx lr
- ENDP ; |vp8_loop_filter_simple_vertical_edge_neon|
-
-; r0 unsigned char *y
-; r1 int ystride
-; r2 const unsigned char *blimit
-
-|vp8_loop_filter_bvs_neon| PROC
- push {r4, lr}
- ldrb r3, [r2] ; load blim from mem
- mov r4, r0
- add r0, r0, #4
- vdup.s8 q1, r3 ; duplicate blim
- bl vp8_loop_filter_simple_vertical_edge_neon
- ; vp8_loop_filter_simple_vertical_edge_neon preserves r1 and q1
- add r0, r4, #8
- bl vp8_loop_filter_simple_vertical_edge_neon
- add r0, r4, #12
- pop {r4, lr}
- b vp8_loop_filter_simple_vertical_edge_neon
- ENDP ;|vp8_loop_filter_bvs_neon|
-
-; r0 unsigned char *y
-; r1 int ystride
-; r2 const unsigned char *blimit
-
-|vp8_loop_filter_mbvs_neon| PROC
- ldrb r3, [r2] ; load mblim from mem
- vdup.s8 q1, r3 ; duplicate mblim
- b vp8_loop_filter_simple_vertical_edge_neon
- ENDP ;|vp8_loop_filter_bvs_neon|
- END
diff --git a/vp8/common/arm/neon/loopfiltersimpleverticaledge_neon.c b/vp8/common/arm/neon/loopfiltersimpleverticaledge_neon.c
new file mode 100644
index 0000000..b0952b5
--- /dev/null
+++ b/vp8/common/arm/neon/loopfiltersimpleverticaledge_neon.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+#include "./vpx_config.h"
+
+static INLINE void vp8_loop_filter_simple_vertical_edge_neon(
+ unsigned char *s,
+ int p,
+ const unsigned char *blimit) {
+ int pitch;
+ unsigned char *src1, *src2;
+ uint8x16_t qblimit, q0u8;
+ uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8, q11u8, q12u8, q14u8, q15u8;
+ int16x8_t q2s16, q13s16, q11s16;
+ int8x8_t d28s8, d29s8;
+ int8x16_t q2s8, q3s8, q10s8, q11s8, q14s8;
+ uint8x8x4_t d0u8x4; // d6, d7, d8, d9
+ uint8x8x4_t d1u8x4; // d10, d11, d12, d13
+ uint8x8x2_t d2u8x2; // d12, d13
+ uint8x8x2_t d3u8x2; // d14, d15
+
+ pitch = p << 1;
+ qblimit = vdupq_n_u8(*blimit);
+
+ src1 = s - 2;
+ d0u8x4 = vld4_lane_u8(src1, d0u8x4, 0);
+ src1 += pitch;
+ d0u8x4 = vld4_lane_u8(src1, d0u8x4, 2);
+ src1 += pitch;
+ d0u8x4 = vld4_lane_u8(src1, d0u8x4, 4);
+ src1 += pitch;
+ d0u8x4 = vld4_lane_u8(src1, d0u8x4, 6);
+ src1 += pitch;
+ d1u8x4 = vld4_lane_u8(src1, d1u8x4, 0);
+ src1 += pitch;
+ d1u8x4 = vld4_lane_u8(src1, d1u8x4, 2);
+ src1 += pitch;
+ d1u8x4 = vld4_lane_u8(src1, d1u8x4, 4);
+ src1 += pitch;
+ d1u8x4 = vld4_lane_u8(src1, d1u8x4, 6);
+
+ src2 = s - 2 + p;
+ d0u8x4 = vld4_lane_u8(src2, d0u8x4, 1);
+ src2 += pitch;
+ d0u8x4 = vld4_lane_u8(src2, d0u8x4, 3);
+ src2 += pitch;
+ d0u8x4 = vld4_lane_u8(src2, d0u8x4, 5);
+ src2 += pitch;
+ d0u8x4 = vld4_lane_u8(src2, d0u8x4, 7);
+ src2 += pitch;
+ d1u8x4 = vld4_lane_u8(src2, d1u8x4, 1);
+ src2 += pitch;
+ d1u8x4 = vld4_lane_u8(src2, d1u8x4, 3);
+ src2 += pitch;
+ d1u8x4 = vld4_lane_u8(src2, d1u8x4, 5);
+ src2 += pitch;
+ d1u8x4 = vld4_lane_u8(src2, d1u8x4, 7);
+
+ // vswp d7, d10
+ // vswp d12, d9
+ q3u8 = vcombine_u8(d0u8x4.val[0], d1u8x4.val[0]); // d6 d10
+ q4u8 = vcombine_u8(d0u8x4.val[2], d1u8x4.val[2]); // d8 d12
+ q5u8 = vcombine_u8(d0u8x4.val[1], d1u8x4.val[1]); // d7 d11
+ q6u8 = vcombine_u8(d0u8x4.val[3], d1u8x4.val[3]); // d9 d13
+
+ q15u8 = vabdq_u8(q5u8, q4u8);
+ q14u8 = vabdq_u8(q3u8, q6u8);
+
+ q15u8 = vqaddq_u8(q15u8, q15u8);
+ q14u8 = vshrq_n_u8(q14u8, 1);
+ q0u8 = vdupq_n_u8(0x80);
+ q11s16 = vdupq_n_s16(3);
+ q15u8 = vqaddq_u8(q15u8, q14u8);
+
+ q3u8 = veorq_u8(q3u8, q0u8);
+ q4u8 = veorq_u8(q4u8, q0u8);
+ q5u8 = veorq_u8(q5u8, q0u8);
+ q6u8 = veorq_u8(q6u8, q0u8);
+
+ q15u8 = vcgeq_u8(qblimit, q15u8);
+
+ q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q4u8)),
+ vget_low_s8(vreinterpretq_s8_u8(q5u8)));
+ q13s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q4u8)),
+ vget_high_s8(vreinterpretq_s8_u8(q5u8)));
+
+ q14s8 = vqsubq_s8(vreinterpretq_s8_u8(q3u8),
+ vreinterpretq_s8_u8(q6u8));
+
+ q2s16 = vmulq_s16(q2s16, q11s16);
+ q13s16 = vmulq_s16(q13s16, q11s16);
+
+ q11u8 = vdupq_n_u8(3);
+ q12u8 = vdupq_n_u8(4);
+
+ q2s16 = vaddw_s8(q2s16, vget_low_s8(q14s8));
+ q13s16 = vaddw_s8(q13s16, vget_high_s8(q14s8));
+
+ d28s8 = vqmovn_s16(q2s16);
+ d29s8 = vqmovn_s16(q13s16);
+ q14s8 = vcombine_s8(d28s8, d29s8);
+
+ q14s8 = vandq_s8(q14s8, vreinterpretq_s8_u8(q15u8));
+
+ q2s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q11u8));
+ q3s8 = vqaddq_s8(q14s8, vreinterpretq_s8_u8(q12u8));
+ q2s8 = vshrq_n_s8(q2s8, 3);
+ q14s8 = vshrq_n_s8(q3s8, 3);
+
+ q11s8 = vqaddq_s8(vreinterpretq_s8_u8(q5u8), q2s8);
+ q10s8 = vqsubq_s8(vreinterpretq_s8_u8(q4u8), q14s8);
+
+ q6u8 = veorq_u8(vreinterpretq_u8_s8(q11s8), q0u8);
+ q7u8 = veorq_u8(vreinterpretq_u8_s8(q10s8), q0u8);
+
+ d2u8x2.val[0] = vget_low_u8(q6u8); // d12
+ d2u8x2.val[1] = vget_low_u8(q7u8); // d14
+ d3u8x2.val[0] = vget_high_u8(q6u8); // d13
+ d3u8x2.val[1] = vget_high_u8(q7u8); // d15
+
+ src1 = s - 1;
+ vst2_lane_u8(src1, d2u8x2, 0);
+ src1 += pitch;
+ vst2_lane_u8(src1, d2u8x2, 2);
+ src1 += pitch;
+ vst2_lane_u8(src1, d2u8x2, 4);
+ src1 += pitch;
+ vst2_lane_u8(src1, d2u8x2, 6);
+ src1 += pitch;
+ vst2_lane_u8(src1, d3u8x2, 0);
+ src1 += pitch;
+ vst2_lane_u8(src1, d3u8x2, 2);
+ src1 += pitch;
+ vst2_lane_u8(src1, d3u8x2, 4);
+ src1 += pitch;
+ vst2_lane_u8(src1, d3u8x2, 6);
+
+ src2 = s - 1 + p;
+ vst2_lane_u8(src2, d2u8x2, 1);
+ src2 += pitch;
+ vst2_lane_u8(src2, d2u8x2, 3);
+ src2 += pitch;
+ vst2_lane_u8(src2, d2u8x2, 5);
+ src2 += pitch;
+ vst2_lane_u8(src2, d2u8x2, 7);
+ src2 += pitch;
+ vst2_lane_u8(src2, d3u8x2, 1);
+ src2 += pitch;
+ vst2_lane_u8(src2, d3u8x2, 3);
+ src2 += pitch;
+ vst2_lane_u8(src2, d3u8x2, 5);
+ src2 += pitch;
+ vst2_lane_u8(src2, d3u8x2, 7);
+ return;
+}
+
+void vp8_loop_filter_bvs_neon(
+ unsigned char *y_ptr,
+ int y_stride,
+ const unsigned char *blimit) {
+ y_ptr += 4;
+ vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
+ y_ptr += 4;
+ vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
+ y_ptr += 4;
+ vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
+ return;
+}
+
+void vp8_loop_filter_mbvs_neon(
+ unsigned char *y_ptr,
+ int y_stride,
+ const unsigned char *blimit) {
+ vp8_loop_filter_simple_vertical_edge_neon(y_ptr, y_stride, blimit);
+ return;
+}
diff --git a/vp8/common/arm/neon/mbloopfilter_neon.asm b/vp8/common/arm/neon/mbloopfilter_neon.asm
deleted file mode 100644
index d200c30..0000000
--- a/vp8/common/arm/neon/mbloopfilter_neon.asm
+++ /dev/null
@@ -1,481 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_mbloop_filter_horizontal_edge_y_neon|
- EXPORT |vp8_mbloop_filter_horizontal_edge_uv_neon|
- EXPORT |vp8_mbloop_filter_vertical_edge_y_neon|
- EXPORT |vp8_mbloop_filter_vertical_edge_uv_neon|
- ARM
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-; void vp8_mbloop_filter_horizontal_edge_y_neon(unsigned char *src, int pitch,
-; const unsigned char *blimit,
-; const unsigned char *limit,
-; const unsigned char *thresh)
-; r0 unsigned char *src,
-; r1 int pitch,
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-|vp8_mbloop_filter_horizontal_edge_y_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- add r1, r1, r1 ; double stride
- ldr r12, [sp, #68] ; load thresh
- sub r0, r0, r1, lsl #1 ; move src pointer down by 4 lines
- vdup.u8 q2, r12 ; thresh
- add r12, r0, r1, lsr #1 ; move src pointer up by 1 line
-
- vld1.u8 {q3}, [r0@128], r1 ; p3
- vld1.u8 {q4}, [r12@128], r1 ; p2
- vld1.u8 {q5}, [r0@128], r1 ; p1
- vld1.u8 {q6}, [r12@128], r1 ; p0
- vld1.u8 {q7}, [r0@128], r1 ; q0
- vld1.u8 {q8}, [r12@128], r1 ; q1
- vld1.u8 {q9}, [r0@128], r1 ; q2
- vld1.u8 {q10}, [r12@128], r1 ; q3
-
- bl vp8_mbloop_filter_neon
-
- sub r12, r12, r1, lsl #2
- add r0, r12, r1, lsr #1
-
- vst1.u8 {q4}, [r12@128],r1 ; store op2
- vst1.u8 {q5}, [r0@128],r1 ; store op1
- vst1.u8 {q6}, [r12@128], r1 ; store op0
- vst1.u8 {q7}, [r0@128],r1 ; store oq0
- vst1.u8 {q8}, [r12@128] ; store oq1
- vst1.u8 {q9}, [r0@128] ; store oq2
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_mbloop_filter_horizontal_edge_y_neon|
-
-; void vp8_mbloop_filter_horizontal_edge_uv_neon(unsigned char *u, int pitch,
-; const unsigned char *blimit,
-; const unsigned char *limit,
-; const unsigned char *thresh,
-; unsigned char *v)
-; r0 unsigned char *u,
-; r1 int pitch,
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-; sp+4 unsigned char *v
-
-|vp8_mbloop_filter_horizontal_edge_uv_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- ldr r12, [sp, #68] ; load thresh
- sub r0, r0, r1, lsl #2 ; move u pointer down by 4 lines
- vdup.u8 q2, r12 ; thresh
- ldr r12, [sp, #72] ; load v ptr
- sub r12, r12, r1, lsl #2 ; move v pointer down by 4 lines
-
- vld1.u8 {d6}, [r0@64], r1 ; p3
- vld1.u8 {d7}, [r12@64], r1 ; p3
- vld1.u8 {d8}, [r0@64], r1 ; p2
- vld1.u8 {d9}, [r12@64], r1 ; p2
- vld1.u8 {d10}, [r0@64], r1 ; p1
- vld1.u8 {d11}, [r12@64], r1 ; p1
- vld1.u8 {d12}, [r0@64], r1 ; p0
- vld1.u8 {d13}, [r12@64], r1 ; p0
- vld1.u8 {d14}, [r0@64], r1 ; q0
- vld1.u8 {d15}, [r12@64], r1 ; q0
- vld1.u8 {d16}, [r0@64], r1 ; q1
- vld1.u8 {d17}, [r12@64], r1 ; q1
- vld1.u8 {d18}, [r0@64], r1 ; q2
- vld1.u8 {d19}, [r12@64], r1 ; q2
- vld1.u8 {d20}, [r0@64], r1 ; q3
- vld1.u8 {d21}, [r12@64], r1 ; q3
-
- bl vp8_mbloop_filter_neon
-
- sub r0, r0, r1, lsl #3
- sub r12, r12, r1, lsl #3
-
- add r0, r0, r1
- add r12, r12, r1
-
- vst1.u8 {d8}, [r0@64], r1 ; store u op2
- vst1.u8 {d9}, [r12@64], r1 ; store v op2
- vst1.u8 {d10}, [r0@64], r1 ; store u op1
- vst1.u8 {d11}, [r12@64], r1 ; store v op1
- vst1.u8 {d12}, [r0@64], r1 ; store u op0
- vst1.u8 {d13}, [r12@64], r1 ; store v op0
- vst1.u8 {d14}, [r0@64], r1 ; store u oq0
- vst1.u8 {d15}, [r12@64], r1 ; store v oq0
- vst1.u8 {d16}, [r0@64], r1 ; store u oq1
- vst1.u8 {d17}, [r12@64], r1 ; store v oq1
- vst1.u8 {d18}, [r0@64], r1 ; store u oq2
- vst1.u8 {d19}, [r12@64], r1 ; store v oq2
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_mbloop_filter_horizontal_edge_uv_neon|
-
-; void vp8_mbloop_filter_vertical_edge_y_neon(unsigned char *src, int pitch,
-; const unsigned char *blimit,
-; const unsigned char *limit,
-; const unsigned char *thresh)
-; r0 unsigned char *src,
-; r1 int pitch,
-; r2 unsigned char blimit
-; r3 unsigned char limit
-; sp unsigned char thresh,
-|vp8_mbloop_filter_vertical_edge_y_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- ldr r12, [sp, #68] ; load thresh
- sub r0, r0, #4 ; move src pointer down by 4 columns
- vdup.s8 q2, r12 ; thresh
- add r12, r0, r1, lsl #3 ; move src pointer down by 8 lines
-
- vld1.u8 {d6}, [r0], r1 ; load first 8-line src data
- vld1.u8 {d7}, [r12], r1 ; load second 8-line src data
- vld1.u8 {d8}, [r0], r1
- vld1.u8 {d9}, [r12], r1
- vld1.u8 {d10}, [r0], r1
- vld1.u8 {d11}, [r12], r1
- vld1.u8 {d12}, [r0], r1
- vld1.u8 {d13}, [r12], r1
- vld1.u8 {d14}, [r0], r1
- vld1.u8 {d15}, [r12], r1
- vld1.u8 {d16}, [r0], r1
- vld1.u8 {d17}, [r12], r1
- vld1.u8 {d18}, [r0], r1
- vld1.u8 {d19}, [r12], r1
- vld1.u8 {d20}, [r0], r1
- vld1.u8 {d21}, [r12], r1
-
- ;transpose to 8x16 matrix
- vtrn.32 q3, q7
- vtrn.32 q4, q8
- vtrn.32 q5, q9
- vtrn.32 q6, q10
-
- vtrn.16 q3, q5
- vtrn.16 q4, q6
- vtrn.16 q7, q9
- vtrn.16 q8, q10
-
- vtrn.8 q3, q4
- vtrn.8 q5, q6
- vtrn.8 q7, q8
- vtrn.8 q9, q10
-
- sub r0, r0, r1, lsl #3
-
- bl vp8_mbloop_filter_neon
-
- sub r12, r12, r1, lsl #3
-
- ;transpose to 16x8 matrix
- vtrn.32 q3, q7
- vtrn.32 q4, q8
- vtrn.32 q5, q9
- vtrn.32 q6, q10
-
- vtrn.16 q3, q5
- vtrn.16 q4, q6
- vtrn.16 q7, q9
- vtrn.16 q8, q10
-
- vtrn.8 q3, q4
- vtrn.8 q5, q6
- vtrn.8 q7, q8
- vtrn.8 q9, q10
-
- ;store op2, op1, op0, oq0, oq1, oq2
- vst1.8 {d6}, [r0], r1
- vst1.8 {d7}, [r12], r1
- vst1.8 {d8}, [r0], r1
- vst1.8 {d9}, [r12], r1
- vst1.8 {d10}, [r0], r1
- vst1.8 {d11}, [r12], r1
- vst1.8 {d12}, [r0], r1
- vst1.8 {d13}, [r12], r1
- vst1.8 {d14}, [r0], r1
- vst1.8 {d15}, [r12], r1
- vst1.8 {d16}, [r0], r1
- vst1.8 {d17}, [r12], r1
- vst1.8 {d18}, [r0], r1
- vst1.8 {d19}, [r12], r1
- vst1.8 {d20}, [r0]
- vst1.8 {d21}, [r12]
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_mbloop_filter_vertical_edge_y_neon|
-
-; void vp8_mbloop_filter_vertical_edge_uv_neon(unsigned char *u, int pitch,
-; const unsigned char *blimit,
-; const unsigned char *limit,
-; const unsigned char *thresh,
-; unsigned char *v)
-; r0 unsigned char *u,
-; r1 int pitch,
-; r2 const signed char *flimit,
-; r3 const signed char *limit,
-; sp const signed char *thresh,
-; sp+4 unsigned char *v
-|vp8_mbloop_filter_vertical_edge_uv_neon| PROC
- push {lr}
- vpush {d8-d15}
-
- ldr r12, [sp, #68] ; load thresh
- sub r0, r0, #4 ; move u pointer down by 4 columns
- vdup.u8 q2, r12 ; thresh
- ldr r12, [sp, #72] ; load v ptr
- sub r12, r12, #4 ; move v pointer down by 4 columns
-
- vld1.u8 {d6}, [r0], r1 ;load u data
- vld1.u8 {d7}, [r12], r1 ;load v data
- vld1.u8 {d8}, [r0], r1
- vld1.u8 {d9}, [r12], r1
- vld1.u8 {d10}, [r0], r1
- vld1.u8 {d11}, [r12], r1
- vld1.u8 {d12}, [r0], r1
- vld1.u8 {d13}, [r12], r1
- vld1.u8 {d14}, [r0], r1
- vld1.u8 {d15}, [r12], r1
- vld1.u8 {d16}, [r0], r1
- vld1.u8 {d17}, [r12], r1
- vld1.u8 {d18}, [r0], r1
- vld1.u8 {d19}, [r12], r1
- vld1.u8 {d20}, [r0], r1
- vld1.u8 {d21}, [r12], r1
-
- ;transpose to 8x16 matrix
- vtrn.32 q3, q7
- vtrn.32 q4, q8
- vtrn.32 q5, q9
- vtrn.32 q6, q10
-
- vtrn.16 q3, q5
- vtrn.16 q4, q6
- vtrn.16 q7, q9
- vtrn.16 q8, q10
-
- vtrn.8 q3, q4
- vtrn.8 q5, q6
- vtrn.8 q7, q8
- vtrn.8 q9, q10
-
- sub r0, r0, r1, lsl #3
-
- bl vp8_mbloop_filter_neon
-
- sub r12, r12, r1, lsl #3
-
- ;transpose to 16x8 matrix
- vtrn.32 q3, q7
- vtrn.32 q4, q8
- vtrn.32 q5, q9
- vtrn.32 q6, q10
-
- vtrn.16 q3, q5
- vtrn.16 q4, q6
- vtrn.16 q7, q9
- vtrn.16 q8, q10
-
- vtrn.8 q3, q4
- vtrn.8 q5, q6
- vtrn.8 q7, q8
- vtrn.8 q9, q10
-
- ;store op2, op1, op0, oq0, oq1, oq2
- vst1.8 {d6}, [r0], r1
- vst1.8 {d7}, [r12], r1
- vst1.8 {d8}, [r0], r1
- vst1.8 {d9}, [r12], r1
- vst1.8 {d10}, [r0], r1
- vst1.8 {d11}, [r12], r1
- vst1.8 {d12}, [r0], r1
- vst1.8 {d13}, [r12], r1
- vst1.8 {d14}, [r0], r1
- vst1.8 {d15}, [r12], r1
- vst1.8 {d16}, [r0], r1
- vst1.8 {d17}, [r12], r1
- vst1.8 {d18}, [r0], r1
- vst1.8 {d19}, [r12], r1
- vst1.8 {d20}, [r0]
- vst1.8 {d21}, [r12]
-
- vpop {d8-d15}
- pop {pc}
- ENDP ; |vp8_mbloop_filter_vertical_edge_uv_neon|
-
-; void vp8_mbloop_filter_neon()
-; This is a helper function for the macroblock loopfilters. The individual
-; functions do the necessary load, transpose (if necessary), preserve (if
-; necessary) and store.
-
-; r0,r1 PRESERVE
-; r2 mblimit
-; r3 limit
-
-; q2 thresh
-; q3 p3 PRESERVE
-; q4 p2
-; q5 p1
-; q6 p0
-; q7 q0
-; q8 q1
-; q9 q2
-; q10 q3 PRESERVE
-
-|vp8_mbloop_filter_neon| PROC
-
- ; vp8_filter_mask
- vabd.u8 q11, q3, q4 ; abs(p3 - p2)
- vabd.u8 q12, q4, q5 ; abs(p2 - p1)
- vabd.u8 q13, q5, q6 ; abs(p1 - p0)
- vabd.u8 q14, q8, q7 ; abs(q1 - q0)
- vabd.u8 q1, q9, q8 ; abs(q2 - q1)
- vabd.u8 q0, q10, q9 ; abs(q3 - q2)
-
- vmax.u8 q11, q11, q12
- vmax.u8 q12, q13, q14
- vmax.u8 q1, q1, q0
- vmax.u8 q15, q11, q12
-
- vabd.u8 q12, q6, q7 ; abs(p0 - q0)
-
- ; vp8_hevmask
- vcgt.u8 q13, q13, q2 ; (abs(p1 - p0) > thresh) * -1
- vcgt.u8 q14, q14, q2 ; (abs(q1 - q0) > thresh) * -1
- vmax.u8 q15, q15, q1
-
- vdup.u8 q1, r3 ; limit
- vdup.u8 q2, r2 ; mblimit
-
- vmov.u8 q0, #0x80 ; 0x80
-
- vcge.u8 q15, q1, q15
-
- vabd.u8 q1, q5, q8 ; a = abs(p1 - q1)
- vqadd.u8 q12, q12, q12 ; b = abs(p0 - q0) * 2
- vmov.u16 q11, #3 ; #3
-
- ; vp8_filter
- ; convert to signed
- veor q7, q7, q0 ; qs0
- vshr.u8 q1, q1, #1 ; a = a / 2
- veor q6, q6, q0 ; ps0
- veor q5, q5, q0 ; ps1
-
- vqadd.u8 q12, q12, q1 ; a = b + a
-
- veor q8, q8, q0 ; qs1
- veor q4, q4, q0 ; ps2
- veor q9, q9, q0 ; qs2
-
- vorr q14, q13, q14 ; vp8_hevmask
-
- vcge.u8 q12, q2, q12 ; (a > flimit * 2 + limit) * -1
-
- vsubl.s8 q2, d14, d12 ; qs0 - ps0
- vsubl.s8 q13, d15, d13
-
- vqsub.s8 q1, q5, q8 ; vp8_filter = clamp(ps1-qs1)
-
- vmul.i16 q2, q2, q11 ; 3 * ( qs0 - ps0)
-
- vand q15, q15, q12 ; vp8_filter_mask
-
- vmul.i16 q13, q13, q11
-
- vmov.u8 q12, #3 ; #3
-
- vaddw.s8 q2, q2, d2 ; vp8_filter + 3 * ( qs0 - ps0)
- vaddw.s8 q13, q13, d3
-
- vmov.u8 q11, #4 ; #4
-
- ; vp8_filter = clamp(vp8_filter + 3 * ( qs0 - ps0))
- vqmovn.s16 d2, q2
- vqmovn.s16 d3, q13
-
- vand q1, q1, q15 ; vp8_filter &= mask
-
- vmov.u16 q15, #63 ; #63
-
- vand q13, q1, q14 ; Filter2 &= hev
-
- vqadd.s8 q2, q13, q11 ; Filter1 = clamp(Filter2+4)
- vqadd.s8 q13, q13, q12 ; Filter2 = clamp(Filter2+3)
-
- vmov q0, q15
-
- vshr.s8 q2, q2, #3 ; Filter1 >>= 3
- vshr.s8 q13, q13, #3 ; Filter2 >>= 3
-
- vmov q11, q15
- vmov q12, q15
-
- vqsub.s8 q7, q7, q2 ; qs0 = clamp(qs0 - Filter1)
-
- vqadd.s8 q6, q6, q13 ; ps0 = clamp(ps0 + Filter2)
-
- vbic q1, q1, q14 ; vp8_filter &= ~hev
-
- ; roughly 1/7th difference across boundary
- ; roughly 2/7th difference across boundary
- ; roughly 3/7th difference across boundary
-
- vmov.u8 d5, #9 ; #9
- vmov.u8 d4, #18 ; #18
-
- vmov q13, q15
- vmov q14, q15
-
- vmlal.s8 q0, d2, d5 ; 63 + Filter2 * 9
- vmlal.s8 q11, d3, d5
- vmov.u8 d5, #27 ; #27
- vmlal.s8 q12, d2, d4 ; 63 + Filter2 * 18
- vmlal.s8 q13, d3, d4
- vmlal.s8 q14, d2, d5 ; 63 + Filter2 * 27
- vmlal.s8 q15, d3, d5
-
- vqshrn.s16 d0, q0, #7 ; u = clamp((63 + Filter2 * 9)>>7)
- vqshrn.s16 d1, q11, #7
- vqshrn.s16 d24, q12, #7 ; u = clamp((63 + Filter2 * 18)>>7)
- vqshrn.s16 d25, q13, #7
- vqshrn.s16 d28, q14, #7 ; u = clamp((63 + Filter2 * 27)>>7)
- vqshrn.s16 d29, q15, #7
-
- vmov.u8 q1, #0x80 ; 0x80
-
- vqsub.s8 q11, q9, q0 ; s = clamp(qs2 - u)
- vqadd.s8 q0, q4, q0 ; s = clamp(ps2 + u)
- vqsub.s8 q13, q8, q12 ; s = clamp(qs1 - u)
- vqadd.s8 q12, q5, q12 ; s = clamp(ps1 + u)
- vqsub.s8 q15, q7, q14 ; s = clamp(qs0 - u)
- vqadd.s8 q14, q6, q14 ; s = clamp(ps0 + u)
-
- veor q9, q11, q1 ; *oq2 = s^0x80
- veor q4, q0, q1 ; *op2 = s^0x80
- veor q8, q13, q1 ; *oq1 = s^0x80
- veor q5, q12, q1 ; *op2 = s^0x80
- veor q7, q15, q1 ; *oq0 = s^0x80
- veor q6, q14, q1 ; *op0 = s^0x80
-
- bx lr
- ENDP ; |vp8_mbloop_filter_neon|
-
-;-----------------
-
- END
diff --git a/vp8/common/arm/neon/mbloopfilter_neon.c b/vp8/common/arm/neon/mbloopfilter_neon.c
new file mode 100644
index 0000000..5351f4b
--- /dev/null
+++ b/vp8/common/arm/neon/mbloopfilter_neon.c
@@ -0,0 +1,625 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+#include "./vpx_config.h"
+
+static INLINE void vp8_mbloop_filter_neon(
+ uint8x16_t qblimit, // mblimit
+ uint8x16_t qlimit, // limit
+ uint8x16_t qthresh, // thresh
+ uint8x16_t q3, // p2
+ uint8x16_t q4, // p2
+ uint8x16_t q5, // p1
+ uint8x16_t q6, // p0
+ uint8x16_t q7, // q0
+ uint8x16_t q8, // q1
+ uint8x16_t q9, // q2
+ uint8x16_t q10, // q3
+ uint8x16_t *q4r, // p1
+ uint8x16_t *q5r, // p1
+ uint8x16_t *q6r, // p0
+ uint8x16_t *q7r, // q0
+ uint8x16_t *q8r, // q1
+ uint8x16_t *q9r) { // q1
+ uint8x16_t q0u8, q1u8, q11u8, q12u8, q13u8, q14u8, q15u8;
+ int16x8_t q0s16, q2s16, q11s16, q12s16, q13s16, q14s16, q15s16;
+ int8x16_t q1s8, q6s8, q7s8, q2s8, q11s8, q13s8;
+ uint16x8_t q0u16, q11u16, q12u16, q13u16, q14u16, q15u16;
+ int8x16_t q0s8, q12s8, q14s8, q15s8;
+ int8x8_t d0, d1, d2, d3, d4, d5, d24, d25, d28, d29;
+
+ q11u8 = vabdq_u8(q3, q4);
+ q12u8 = vabdq_u8(q4, q5);
+ q13u8 = vabdq_u8(q5, q6);
+ q14u8 = vabdq_u8(q8, q7);
+ q1u8 = vabdq_u8(q9, q8);
+ q0u8 = vabdq_u8(q10, q9);
+
+ q11u8 = vmaxq_u8(q11u8, q12u8);
+ q12u8 = vmaxq_u8(q13u8, q14u8);
+ q1u8 = vmaxq_u8(q1u8, q0u8);
+ q15u8 = vmaxq_u8(q11u8, q12u8);
+
+ q12u8 = vabdq_u8(q6, q7);
+
+ // vp8_hevmask
+ q13u8 = vcgtq_u8(q13u8, qthresh);
+ q14u8 = vcgtq_u8(q14u8, qthresh);
+ q15u8 = vmaxq_u8(q15u8, q1u8);
+
+ q15u8 = vcgeq_u8(qlimit, q15u8);
+
+ q1u8 = vabdq_u8(q5, q8);
+ q12u8 = vqaddq_u8(q12u8, q12u8);
+
+ // vp8_filter() function
+ // convert to signed
+ q0u8 = vdupq_n_u8(0x80);
+ q9 = veorq_u8(q9, q0u8);
+ q8 = veorq_u8(q8, q0u8);
+ q7 = veorq_u8(q7, q0u8);
+ q6 = veorq_u8(q6, q0u8);
+ q5 = veorq_u8(q5, q0u8);
+ q4 = veorq_u8(q4, q0u8);
+
+ q1u8 = vshrq_n_u8(q1u8, 1);
+ q12u8 = vqaddq_u8(q12u8, q1u8);
+
+ q14u8 = vorrq_u8(q13u8, q14u8);
+ q12u8 = vcgeq_u8(qblimit, q12u8);
+
+ q2s16 = vsubl_s8(vget_low_s8(vreinterpretq_s8_u8(q7)),
+ vget_low_s8(vreinterpretq_s8_u8(q6)));
+ q13s16 = vsubl_s8(vget_high_s8(vreinterpretq_s8_u8(q7)),
+ vget_high_s8(vreinterpretq_s8_u8(q6)));
+
+ q1s8 = vqsubq_s8(vreinterpretq_s8_u8(q5),
+ vreinterpretq_s8_u8(q8));
+
+ q11s16 = vdupq_n_s16(3);
+ q2s16 = vmulq_s16(q2s16, q11s16);
+ q13s16 = vmulq_s16(q13s16, q11s16);
+
+ q15u8 = vandq_u8(q15u8, q12u8);
+
+ q2s16 = vaddw_s8(q2s16, vget_low_s8(q1s8));
+ q13s16 = vaddw_s8(q13s16, vget_high_s8(q1s8));
+
+ q12u8 = vdupq_n_u8(3);
+ q11u8 = vdupq_n_u8(4);
+ // vp8_filter = clamp(vp8_filter + 3 * ( qs0 - ps0))
+ d2 = vqmovn_s16(q2s16);
+ d3 = vqmovn_s16(q13s16);
+ q1s8 = vcombine_s8(d2, d3);
+ q1s8 = vandq_s8(q1s8, vreinterpretq_s8_u8(q15u8));
+ q13s8 = vandq_s8(q1s8, vreinterpretq_s8_u8(q14u8));
+
+ q2s8 = vqaddq_s8(q13s8, vreinterpretq_s8_u8(q11u8));
+ q13s8 = vqaddq_s8(q13s8, vreinterpretq_s8_u8(q12u8));
+ q2s8 = vshrq_n_s8(q2s8, 3);
+ q13s8 = vshrq_n_s8(q13s8, 3);
+
+ q7s8 = vqsubq_s8(vreinterpretq_s8_u8(q7), q2s8);
+ q6s8 = vqaddq_s8(vreinterpretq_s8_u8(q6), q13s8);
+
+ q1s8 = vbicq_s8(q1s8, vreinterpretq_s8_u8(q14u8));
+
+ q0u16 = q11u16 = q12u16 = q13u16 = q14u16 = q15u16 = vdupq_n_u16(63);
+ d5 = vdup_n_s8(9);
+ d4 = vdup_n_s8(18);
+
+ q0s16 = vmlal_s8(vreinterpretq_s16_u16(q0u16), vget_low_s8(q1s8), d5);
+ q11s16 = vmlal_s8(vreinterpretq_s16_u16(q11u16), vget_high_s8(q1s8), d5);
+ d5 = vdup_n_s8(27);
+ q12s16 = vmlal_s8(vreinterpretq_s16_u16(q12u16), vget_low_s8(q1s8), d4);
+ q13s16 = vmlal_s8(vreinterpretq_s16_u16(q13u16), vget_high_s8(q1s8), d4);
+ q14s16 = vmlal_s8(vreinterpretq_s16_u16(q14u16), vget_low_s8(q1s8), d5);
+ q15s16 = vmlal_s8(vreinterpretq_s16_u16(q15u16), vget_high_s8(q1s8), d5);
+
+ d0 = vqshrn_n_s16(q0s16 , 7);
+ d1 = vqshrn_n_s16(q11s16, 7);
+ d24 = vqshrn_n_s16(q12s16, 7);
+ d25 = vqshrn_n_s16(q13s16, 7);
+ d28 = vqshrn_n_s16(q14s16, 7);
+ d29 = vqshrn_n_s16(q15s16, 7);
+
+ q0s8 = vcombine_s8(d0, d1);
+ q12s8 = vcombine_s8(d24, d25);
+ q14s8 = vcombine_s8(d28, d29);
+
+ q11s8 = vqsubq_s8(vreinterpretq_s8_u8(q9), q0s8);
+ q0s8 = vqaddq_s8(vreinterpretq_s8_u8(q4), q0s8);
+ q13s8 = vqsubq_s8(vreinterpretq_s8_u8(q8), q12s8);
+ q12s8 = vqaddq_s8(vreinterpretq_s8_u8(q5), q12s8);
+ q15s8 = vqsubq_s8((q7s8), q14s8);
+ q14s8 = vqaddq_s8((q6s8), q14s8);
+
+ q1u8 = vdupq_n_u8(0x80);
+ *q9r = veorq_u8(vreinterpretq_u8_s8(q11s8), q1u8);
+ *q8r = veorq_u8(vreinterpretq_u8_s8(q13s8), q1u8);
+ *q7r = veorq_u8(vreinterpretq_u8_s8(q15s8), q1u8);
+ *q6r = veorq_u8(vreinterpretq_u8_s8(q14s8), q1u8);
+ *q5r = veorq_u8(vreinterpretq_u8_s8(q12s8), q1u8);
+ *q4r = veorq_u8(vreinterpretq_u8_s8(q0s8), q1u8);
+ return;
+}
+
+void vp8_mbloop_filter_horizontal_edge_y_neon(
+ unsigned char *src,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh) {
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ src -= (pitch << 2);
+
+ q3 = vld1q_u8(src);
+ src += pitch;
+ q4 = vld1q_u8(src);
+ src += pitch;
+ q5 = vld1q_u8(src);
+ src += pitch;
+ q6 = vld1q_u8(src);
+ src += pitch;
+ q7 = vld1q_u8(src);
+ src += pitch;
+ q8 = vld1q_u8(src);
+ src += pitch;
+ q9 = vld1q_u8(src);
+ src += pitch;
+ q10 = vld1q_u8(src);
+
+ vp8_mbloop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q4, &q5, &q6, &q7, &q8, &q9);
+
+ src -= (pitch * 6);
+ vst1q_u8(src, q4);
+ src += pitch;
+ vst1q_u8(src, q5);
+ src += pitch;
+ vst1q_u8(src, q6);
+ src += pitch;
+ vst1q_u8(src, q7);
+ src += pitch;
+ vst1q_u8(src, q8);
+ src += pitch;
+ vst1q_u8(src, q9);
+ return;
+}
+
+void vp8_mbloop_filter_horizontal_edge_uv_neon(
+ unsigned char *u,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh,
+ unsigned char *v) {
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+ uint8x8_t d6, d7, d8, d9, d10, d11, d12, d13, d14;
+ uint8x8_t d15, d16, d17, d18, d19, d20, d21;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ u -= (pitch << 2);
+ v -= (pitch << 2);
+
+ d6 = vld1_u8(u);
+ u += pitch;
+ d7 = vld1_u8(v);
+ v += pitch;
+ d8 = vld1_u8(u);
+ u += pitch;
+ d9 = vld1_u8(v);
+ v += pitch;
+ d10 = vld1_u8(u);
+ u += pitch;
+ d11 = vld1_u8(v);
+ v += pitch;
+ d12 = vld1_u8(u);
+ u += pitch;
+ d13 = vld1_u8(v);
+ v += pitch;
+ d14 = vld1_u8(u);
+ u += pitch;
+ d15 = vld1_u8(v);
+ v += pitch;
+ d16 = vld1_u8(u);
+ u += pitch;
+ d17 = vld1_u8(v);
+ v += pitch;
+ d18 = vld1_u8(u);
+ u += pitch;
+ d19 = vld1_u8(v);
+ v += pitch;
+ d20 = vld1_u8(u);
+ d21 = vld1_u8(v);
+
+ q3 = vcombine_u8(d6, d7);
+ q4 = vcombine_u8(d8, d9);
+ q5 = vcombine_u8(d10, d11);
+ q6 = vcombine_u8(d12, d13);
+ q7 = vcombine_u8(d14, d15);
+ q8 = vcombine_u8(d16, d17);
+ q9 = vcombine_u8(d18, d19);
+ q10 = vcombine_u8(d20, d21);
+
+ vp8_mbloop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q4, &q5, &q6, &q7, &q8, &q9);
+
+ u -= (pitch * 6);
+ v -= (pitch * 6);
+ vst1_u8(u, vget_low_u8(q4));
+ u += pitch;
+ vst1_u8(v, vget_high_u8(q4));
+ v += pitch;
+ vst1_u8(u, vget_low_u8(q5));
+ u += pitch;
+ vst1_u8(v, vget_high_u8(q5));
+ v += pitch;
+ vst1_u8(u, vget_low_u8(q6));
+ u += pitch;
+ vst1_u8(v, vget_high_u8(q6));
+ v += pitch;
+ vst1_u8(u, vget_low_u8(q7));
+ u += pitch;
+ vst1_u8(v, vget_high_u8(q7));
+ v += pitch;
+ vst1_u8(u, vget_low_u8(q8));
+ u += pitch;
+ vst1_u8(v, vget_high_u8(q8));
+ v += pitch;
+ vst1_u8(u, vget_low_u8(q9));
+ vst1_u8(v, vget_high_u8(q9));
+ return;
+}
+
+void vp8_mbloop_filter_vertical_edge_y_neon(
+ unsigned char *src,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh) {
+ unsigned char *s1, *s2;
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+ uint8x8_t d6, d7, d8, d9, d10, d11, d12, d13, d14;
+ uint8x8_t d15, d16, d17, d18, d19, d20, d21;
+ uint32x4x2_t q2tmp0, q2tmp1, q2tmp2, q2tmp3;
+ uint16x8x2_t q2tmp4, q2tmp5, q2tmp6, q2tmp7;
+ uint8x16x2_t q2tmp8, q2tmp9, q2tmp10, q2tmp11;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ s1 = src - 4;
+ s2 = s1 + 8 * pitch;
+ d6 = vld1_u8(s1);
+ s1 += pitch;
+ d7 = vld1_u8(s2);
+ s2 += pitch;
+ d8 = vld1_u8(s1);
+ s1 += pitch;
+ d9 = vld1_u8(s2);
+ s2 += pitch;
+ d10 = vld1_u8(s1);
+ s1 += pitch;
+ d11 = vld1_u8(s2);
+ s2 += pitch;
+ d12 = vld1_u8(s1);
+ s1 += pitch;
+ d13 = vld1_u8(s2);
+ s2 += pitch;
+ d14 = vld1_u8(s1);
+ s1 += pitch;
+ d15 = vld1_u8(s2);
+ s2 += pitch;
+ d16 = vld1_u8(s1);
+ s1 += pitch;
+ d17 = vld1_u8(s2);
+ s2 += pitch;
+ d18 = vld1_u8(s1);
+ s1 += pitch;
+ d19 = vld1_u8(s2);
+ s2 += pitch;
+ d20 = vld1_u8(s1);
+ d21 = vld1_u8(s2);
+
+ q3 = vcombine_u8(d6, d7);
+ q4 = vcombine_u8(d8, d9);
+ q5 = vcombine_u8(d10, d11);
+ q6 = vcombine_u8(d12, d13);
+ q7 = vcombine_u8(d14, d15);
+ q8 = vcombine_u8(d16, d17);
+ q9 = vcombine_u8(d18, d19);
+ q10 = vcombine_u8(d20, d21);
+
+ q2tmp0 = vtrnq_u32(vreinterpretq_u32_u8(q3), vreinterpretq_u32_u8(q7));
+ q2tmp1 = vtrnq_u32(vreinterpretq_u32_u8(q4), vreinterpretq_u32_u8(q8));
+ q2tmp2 = vtrnq_u32(vreinterpretq_u32_u8(q5), vreinterpretq_u32_u8(q9));
+ q2tmp3 = vtrnq_u32(vreinterpretq_u32_u8(q6), vreinterpretq_u32_u8(q10));
+
+ q2tmp4 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[0]),
+ vreinterpretq_u16_u32(q2tmp2.val[0]));
+ q2tmp5 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[0]),
+ vreinterpretq_u16_u32(q2tmp3.val[0]));
+ q2tmp6 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[1]),
+ vreinterpretq_u16_u32(q2tmp2.val[1]));
+ q2tmp7 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[1]),
+ vreinterpretq_u16_u32(q2tmp3.val[1]));
+
+ q2tmp8 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[0]),
+ vreinterpretq_u8_u16(q2tmp5.val[0]));
+ q2tmp9 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[1]),
+ vreinterpretq_u8_u16(q2tmp5.val[1]));
+ q2tmp10 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[0]),
+ vreinterpretq_u8_u16(q2tmp7.val[0]));
+ q2tmp11 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[1]),
+ vreinterpretq_u8_u16(q2tmp7.val[1]));
+
+ q3 = q2tmp8.val[0];
+ q4 = q2tmp8.val[1];
+ q5 = q2tmp9.val[0];
+ q6 = q2tmp9.val[1];
+ q7 = q2tmp10.val[0];
+ q8 = q2tmp10.val[1];
+ q9 = q2tmp11.val[0];
+ q10 = q2tmp11.val[1];
+
+ vp8_mbloop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q4, &q5, &q6, &q7, &q8, &q9);
+
+ q2tmp0 = vtrnq_u32(vreinterpretq_u32_u8(q3), vreinterpretq_u32_u8(q7));
+ q2tmp1 = vtrnq_u32(vreinterpretq_u32_u8(q4), vreinterpretq_u32_u8(q8));
+ q2tmp2 = vtrnq_u32(vreinterpretq_u32_u8(q5), vreinterpretq_u32_u8(q9));
+ q2tmp3 = vtrnq_u32(vreinterpretq_u32_u8(q6), vreinterpretq_u32_u8(q10));
+
+ q2tmp4 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[0]),
+ vreinterpretq_u16_u32(q2tmp2.val[0]));
+ q2tmp5 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[0]),
+ vreinterpretq_u16_u32(q2tmp3.val[0]));
+ q2tmp6 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[1]),
+ vreinterpretq_u16_u32(q2tmp2.val[1]));
+ q2tmp7 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[1]),
+ vreinterpretq_u16_u32(q2tmp3.val[1]));
+
+ q2tmp8 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[0]),
+ vreinterpretq_u8_u16(q2tmp5.val[0]));
+ q2tmp9 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[1]),
+ vreinterpretq_u8_u16(q2tmp5.val[1]));
+ q2tmp10 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[0]),
+ vreinterpretq_u8_u16(q2tmp7.val[0]));
+ q2tmp11 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[1]),
+ vreinterpretq_u8_u16(q2tmp7.val[1]));
+
+ q3 = q2tmp8.val[0];
+ q4 = q2tmp8.val[1];
+ q5 = q2tmp9.val[0];
+ q6 = q2tmp9.val[1];
+ q7 = q2tmp10.val[0];
+ q8 = q2tmp10.val[1];
+ q9 = q2tmp11.val[0];
+ q10 = q2tmp11.val[1];
+
+ s1 -= 7 * pitch;
+ s2 -= 7 * pitch;
+
+ vst1_u8(s1, vget_low_u8(q3));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q3));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q4));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q4));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q5));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q5));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q6));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q6));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q7));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q7));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q8));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q8));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q9));
+ s1 += pitch;
+ vst1_u8(s2, vget_high_u8(q9));
+ s2 += pitch;
+ vst1_u8(s1, vget_low_u8(q10));
+ vst1_u8(s2, vget_high_u8(q10));
+ return;
+}
+
+void vp8_mbloop_filter_vertical_edge_uv_neon(
+ unsigned char *u,
+ int pitch,
+ unsigned char blimit,
+ unsigned char limit,
+ unsigned char thresh,
+ unsigned char *v) {
+ unsigned char *us, *ud;
+ unsigned char *vs, *vd;
+ uint8x16_t qblimit, qlimit, qthresh, q3, q4;
+ uint8x16_t q5, q6, q7, q8, q9, q10;
+ uint8x8_t d6, d7, d8, d9, d10, d11, d12, d13, d14;
+ uint8x8_t d15, d16, d17, d18, d19, d20, d21;
+ uint32x4x2_t q2tmp0, q2tmp1, q2tmp2, q2tmp3;
+ uint16x8x2_t q2tmp4, q2tmp5, q2tmp6, q2tmp7;
+ uint8x16x2_t q2tmp8, q2tmp9, q2tmp10, q2tmp11;
+
+ qblimit = vdupq_n_u8(blimit);
+ qlimit = vdupq_n_u8(limit);
+ qthresh = vdupq_n_u8(thresh);
+
+ us = u - 4;
+ vs = v - 4;
+ d6 = vld1_u8(us);
+ us += pitch;
+ d7 = vld1_u8(vs);
+ vs += pitch;
+ d8 = vld1_u8(us);
+ us += pitch;
+ d9 = vld1_u8(vs);
+ vs += pitch;
+ d10 = vld1_u8(us);
+ us += pitch;
+ d11 = vld1_u8(vs);
+ vs += pitch;
+ d12 = vld1_u8(us);
+ us += pitch;
+ d13 = vld1_u8(vs);
+ vs += pitch;
+ d14 = vld1_u8(us);
+ us += pitch;
+ d15 = vld1_u8(vs);
+ vs += pitch;
+ d16 = vld1_u8(us);
+ us += pitch;
+ d17 = vld1_u8(vs);
+ vs += pitch;
+ d18 = vld1_u8(us);
+ us += pitch;
+ d19 = vld1_u8(vs);
+ vs += pitch;
+ d20 = vld1_u8(us);
+ d21 = vld1_u8(vs);
+
+ q3 = vcombine_u8(d6, d7);
+ q4 = vcombine_u8(d8, d9);
+ q5 = vcombine_u8(d10, d11);
+ q6 = vcombine_u8(d12, d13);
+ q7 = vcombine_u8(d14, d15);
+ q8 = vcombine_u8(d16, d17);
+ q9 = vcombine_u8(d18, d19);
+ q10 = vcombine_u8(d20, d21);
+
+ q2tmp0 = vtrnq_u32(vreinterpretq_u32_u8(q3), vreinterpretq_u32_u8(q7));
+ q2tmp1 = vtrnq_u32(vreinterpretq_u32_u8(q4), vreinterpretq_u32_u8(q8));
+ q2tmp2 = vtrnq_u32(vreinterpretq_u32_u8(q5), vreinterpretq_u32_u8(q9));
+ q2tmp3 = vtrnq_u32(vreinterpretq_u32_u8(q6), vreinterpretq_u32_u8(q10));
+
+ q2tmp4 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[0]),
+ vreinterpretq_u16_u32(q2tmp2.val[0]));
+ q2tmp5 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[0]),
+ vreinterpretq_u16_u32(q2tmp3.val[0]));
+ q2tmp6 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[1]),
+ vreinterpretq_u16_u32(q2tmp2.val[1]));
+ q2tmp7 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[1]),
+ vreinterpretq_u16_u32(q2tmp3.val[1]));
+
+ q2tmp8 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[0]),
+ vreinterpretq_u8_u16(q2tmp5.val[0]));
+ q2tmp9 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[1]),
+ vreinterpretq_u8_u16(q2tmp5.val[1]));
+ q2tmp10 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[0]),
+ vreinterpretq_u8_u16(q2tmp7.val[0]));
+ q2tmp11 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[1]),
+ vreinterpretq_u8_u16(q2tmp7.val[1]));
+
+ q3 = q2tmp8.val[0];
+ q4 = q2tmp8.val[1];
+ q5 = q2tmp9.val[0];
+ q6 = q2tmp9.val[1];
+ q7 = q2tmp10.val[0];
+ q8 = q2tmp10.val[1];
+ q9 = q2tmp11.val[0];
+ q10 = q2tmp11.val[1];
+
+ vp8_mbloop_filter_neon(qblimit, qlimit, qthresh, q3, q4,
+ q5, q6, q7, q8, q9, q10,
+ &q4, &q5, &q6, &q7, &q8, &q9);
+
+ q2tmp0 = vtrnq_u32(vreinterpretq_u32_u8(q3), vreinterpretq_u32_u8(q7));
+ q2tmp1 = vtrnq_u32(vreinterpretq_u32_u8(q4), vreinterpretq_u32_u8(q8));
+ q2tmp2 = vtrnq_u32(vreinterpretq_u32_u8(q5), vreinterpretq_u32_u8(q9));
+ q2tmp3 = vtrnq_u32(vreinterpretq_u32_u8(q6), vreinterpretq_u32_u8(q10));
+
+ q2tmp4 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[0]),
+ vreinterpretq_u16_u32(q2tmp2.val[0]));
+ q2tmp5 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[0]),
+ vreinterpretq_u16_u32(q2tmp3.val[0]));
+ q2tmp6 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp0.val[1]),
+ vreinterpretq_u16_u32(q2tmp2.val[1]));
+ q2tmp7 = vtrnq_u16(vreinterpretq_u16_u32(q2tmp1.val[1]),
+ vreinterpretq_u16_u32(q2tmp3.val[1]));
+
+ q2tmp8 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[0]),
+ vreinterpretq_u8_u16(q2tmp5.val[0]));
+ q2tmp9 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp4.val[1]),
+ vreinterpretq_u8_u16(q2tmp5.val[1]));
+ q2tmp10 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[0]),
+ vreinterpretq_u8_u16(q2tmp7.val[0]));
+ q2tmp11 = vtrnq_u8(vreinterpretq_u8_u16(q2tmp6.val[1]),
+ vreinterpretq_u8_u16(q2tmp7.val[1]));
+
+ q3 = q2tmp8.val[0];
+ q4 = q2tmp8.val[1];
+ q5 = q2tmp9.val[0];
+ q6 = q2tmp9.val[1];
+ q7 = q2tmp10.val[0];
+ q8 = q2tmp10.val[1];
+ q9 = q2tmp11.val[0];
+ q10 = q2tmp11.val[1];
+
+ ud = u - 4;
+ vst1_u8(ud, vget_low_u8(q3));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q4));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q5));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q6));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q7));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q8));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q9));
+ ud += pitch;
+ vst1_u8(ud, vget_low_u8(q10));
+
+ vd = v - 4;
+ vst1_u8(vd, vget_high_u8(q3));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q4));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q5));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q6));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q7));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q8));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q9));
+ vd += pitch;
+ vst1_u8(vd, vget_high_u8(q10));
+ return;
+}
diff --git a/vp8/common/arm/neon/sad16_neon.asm b/vp8/common/arm/neon/sad16_neon.asm
deleted file mode 100644
index 7197e56..0000000
--- a/vp8/common/arm/neon/sad16_neon.asm
+++ /dev/null
@@ -1,212 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_sad16x16_neon|
- EXPORT |vp8_sad16x8_neon|
-
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-; r0 unsigned char *src_ptr
-; r1 int src_stride
-; r2 unsigned char *ref_ptr
-; r3 int ref_stride
-|vp8_sad16x16_neon| PROC
-;;
- vpush {d8-d15}
- vld1.8 {q0}, [r0], r1
- vld1.8 {q4}, [r2], r3
-
- vld1.8 {q1}, [r0], r1
- vld1.8 {q5}, [r2], r3
-
- vabdl.u8 q12, d0, d8
- vabdl.u8 q13, d1, d9
-
- vld1.8 {q2}, [r0], r1
- vld1.8 {q6}, [r2], r3
-
- vabal.u8 q12, d2, d10
- vabal.u8 q13, d3, d11
-
- vld1.8 {q3}, [r0], r1
- vld1.8 {q7}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q13, d5, d13
-
-;;
- vld1.8 {q0}, [r0], r1
- vld1.8 {q4}, [r2], r3
-
- vabal.u8 q12, d6, d14
- vabal.u8 q13, d7, d15
-
- vld1.8 {q1}, [r0], r1
- vld1.8 {q5}, [r2], r3
-
- vabal.u8 q12, d0, d8
- vabal.u8 q13, d1, d9
-
- vld1.8 {q2}, [r0], r1
- vld1.8 {q6}, [r2], r3
-
- vabal.u8 q12, d2, d10
- vabal.u8 q13, d3, d11
-
- vld1.8 {q3}, [r0], r1
- vld1.8 {q7}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q13, d5, d13
-
-;;
- vld1.8 {q0}, [r0], r1
- vld1.8 {q4}, [r2], r3
-
- vabal.u8 q12, d6, d14
- vabal.u8 q13, d7, d15
-
- vld1.8 {q1}, [r0], r1
- vld1.8 {q5}, [r2], r3
-
- vabal.u8 q12, d0, d8
- vabal.u8 q13, d1, d9
-
- vld1.8 {q2}, [r0], r1
- vld1.8 {q6}, [r2], r3
-
- vabal.u8 q12, d2, d10
- vabal.u8 q13, d3, d11
-
- vld1.8 {q3}, [r0], r1
- vld1.8 {q7}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q13, d5, d13
-
-;;
- vld1.8 {q0}, [r0], r1
- vld1.8 {q4}, [r2], r3
-
- vabal.u8 q12, d6, d14
- vabal.u8 q13, d7, d15
-
- vld1.8 {q1}, [r0], r1
- vld1.8 {q5}, [r2], r3
-
- vabal.u8 q12, d0, d8
- vabal.u8 q13, d1, d9
-
- vld1.8 {q2}, [r0], r1
- vld1.8 {q6}, [r2], r3
-
- vabal.u8 q12, d2, d10
- vabal.u8 q13, d3, d11
-
- vld1.8 {q3}, [r0]
- vld1.8 {q7}, [r2]
-
- vabal.u8 q12, d4, d12
- vabal.u8 q13, d5, d13
-
- vabal.u8 q12, d6, d14
- vabal.u8 q13, d7, d15
-
- vadd.u16 q0, q12, q13
-
- vpaddl.u16 q1, q0
- vpaddl.u32 q0, q1
-
- vadd.u32 d0, d0, d1
-
- vmov.32 r0, d0[0]
-
- vpop {d8-d15}
- bx lr
-
- ENDP
-
-;==============================
-;unsigned int vp8_sad16x8_c(
-; unsigned char *src_ptr,
-; int src_stride,
-; unsigned char *ref_ptr,
-; int ref_stride)
-|vp8_sad16x8_neon| PROC
- vpush {d8-d15}
-
- vld1.8 {q0}, [r0], r1
- vld1.8 {q4}, [r2], r3
-
- vld1.8 {q1}, [r0], r1
- vld1.8 {q5}, [r2], r3
-
- vabdl.u8 q12, d0, d8
- vabdl.u8 q13, d1, d9
-
- vld1.8 {q2}, [r0], r1
- vld1.8 {q6}, [r2], r3
-
- vabal.u8 q12, d2, d10
- vabal.u8 q13, d3, d11
-
- vld1.8 {q3}, [r0], r1
- vld1.8 {q7}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q13, d5, d13
-
- vld1.8 {q0}, [r0], r1
- vld1.8 {q4}, [r2], r3
-
- vabal.u8 q12, d6, d14
- vabal.u8 q13, d7, d15
-
- vld1.8 {q1}, [r0], r1
- vld1.8 {q5}, [r2], r3
-
- vabal.u8 q12, d0, d8
- vabal.u8 q13, d1, d9
-
- vld1.8 {q2}, [r0], r1
- vld1.8 {q6}, [r2], r3
-
- vabal.u8 q12, d2, d10
- vabal.u8 q13, d3, d11
-
- vld1.8 {q3}, [r0], r1
- vld1.8 {q7}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q13, d5, d13
-
- vabal.u8 q12, d6, d14
- vabal.u8 q13, d7, d15
-
- vadd.u16 q0, q12, q13
-
- vpaddl.u16 q1, q0
- vpaddl.u32 q0, q1
-
- vadd.u32 d0, d0, d1
-
- vmov.32 r0, d0[0]
-
- vpop {d8-d15}
- bx lr
-
- ENDP
-
- END
diff --git a/vp8/common/arm/neon/sad8_neon.asm b/vp8/common/arm/neon/sad8_neon.asm
deleted file mode 100644
index 6b849d9..0000000
--- a/vp8/common/arm/neon/sad8_neon.asm
+++ /dev/null
@@ -1,215 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_sad8x8_neon|
- EXPORT |vp8_sad8x16_neon|
- EXPORT |vp8_sad4x4_neon|
-
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-; unsigned int vp8_sad8x8_c(
-; unsigned char *src_ptr,
-; int src_stride,
-; unsigned char *ref_ptr,
-; int ref_stride)
-
-|vp8_sad8x8_neon| PROC
- vpush {d8-d15}
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabdl.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
-
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vabal.u8 q12, d6, d14
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabal.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q12, d6, d14
-
- vpaddl.u16 q1, q12
- vpaddl.u32 q0, q1
- vadd.u32 d0, d0, d1
-
- vmov.32 r0, d0[0]
-
- vpop {d8-d15}
- bx lr
-
- ENDP
-
-;============================
-;unsigned int vp8_sad8x16_c(
-; unsigned char *src_ptr,
-; int src_stride,
-; unsigned char *ref_ptr,
-; int ref_stride)
-
-|vp8_sad8x16_neon| PROC
- vpush {d8-d15}
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabdl.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
-
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vabal.u8 q12, d6, d14
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabal.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
-
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vabal.u8 q12, d6, d14
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabal.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
-
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vabal.u8 q12, d6, d14
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabal.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q12, d6, d14
-
- vpaddl.u16 q1, q12
- vpaddl.u32 q0, q1
- vadd.u32 d0, d0, d1
-
- vmov.32 r0, d0[0]
-
- vpop {d8-d15}
- bx lr
-
- ENDP
-
-;===========================
-;unsigned int vp8_sad4x4_c(
-; unsigned char *src_ptr,
-; int src_stride,
-; unsigned char *ref_ptr,
-; int ref_stride)
-
-|vp8_sad4x4_neon| PROC
- vpush {d8-d15}
- vld1.8 {d0}, [r0], r1
- vld1.8 {d8}, [r2], r3
-
- vld1.8 {d2}, [r0], r1
- vld1.8 {d10}, [r2], r3
-
- vabdl.u8 q12, d0, d8
-
- vld1.8 {d4}, [r0], r1
- vld1.8 {d12}, [r2], r3
-
- vabal.u8 q12, d2, d10
-
- vld1.8 {d6}, [r0], r1
- vld1.8 {d14}, [r2], r3
-
- vabal.u8 q12, d4, d12
- vabal.u8 q12, d6, d14
-
- vpaddl.u16 d1, d24
- vpaddl.u32 d0, d1
- vmov.32 r0, d0[0]
-
- vpop {d8-d15}
- bx lr
-
- ENDP
-
- END
diff --git a/vp8/common/arm/neon/sad_neon.c b/vp8/common/arm/neon/sad_neon.c
new file mode 100644
index 0000000..6595ac0
--- /dev/null
+++ b/vp8/common/arm/neon/sad_neon.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+unsigned int vp8_sad8x8_neon(
+ unsigned char *src_ptr,
+ int src_stride,
+ unsigned char *ref_ptr,
+ int ref_stride) {
+ uint8x8_t d0, d8;
+ uint16x8_t q12;
+ uint32x4_t q1;
+ uint64x2_t q3;
+ uint32x2_t d5;
+ int i;
+
+ d0 = vld1_u8(src_ptr);
+ src_ptr += src_stride;
+ d8 = vld1_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabdl_u8(d0, d8);
+
+ for (i = 0; i < 7; i++) {
+ d0 = vld1_u8(src_ptr);
+ src_ptr += src_stride;
+ d8 = vld1_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabal_u8(q12, d0, d8);
+ }
+
+ q1 = vpaddlq_u16(q12);
+ q3 = vpaddlq_u32(q1);
+ d5 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(q3)),
+ vreinterpret_u32_u64(vget_high_u64(q3)));
+
+ return vget_lane_u32(d5, 0);
+}
+
+unsigned int vp8_sad8x16_neon(
+ unsigned char *src_ptr,
+ int src_stride,
+ unsigned char *ref_ptr,
+ int ref_stride) {
+ uint8x8_t d0, d8;
+ uint16x8_t q12;
+ uint32x4_t q1;
+ uint64x2_t q3;
+ uint32x2_t d5;
+ int i;
+
+ d0 = vld1_u8(src_ptr);
+ src_ptr += src_stride;
+ d8 = vld1_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabdl_u8(d0, d8);
+
+ for (i = 0; i < 15; i++) {
+ d0 = vld1_u8(src_ptr);
+ src_ptr += src_stride;
+ d8 = vld1_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabal_u8(q12, d0, d8);
+ }
+
+ q1 = vpaddlq_u16(q12);
+ q3 = vpaddlq_u32(q1);
+ d5 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(q3)),
+ vreinterpret_u32_u64(vget_high_u64(q3)));
+
+ return vget_lane_u32(d5, 0);
+}
+
+unsigned int vp8_sad4x4_neon(
+ unsigned char *src_ptr,
+ int src_stride,
+ unsigned char *ref_ptr,
+ int ref_stride) {
+ uint8x8_t d0, d8;
+ uint16x8_t q12;
+ uint32x2_t d1;
+ uint64x1_t d3;
+ int i;
+
+ d0 = vld1_u8(src_ptr);
+ src_ptr += src_stride;
+ d8 = vld1_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabdl_u8(d0, d8);
+
+ for (i = 0; i < 3; i++) {
+ d0 = vld1_u8(src_ptr);
+ src_ptr += src_stride;
+ d8 = vld1_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabal_u8(q12, d0, d8);
+ }
+
+ d1 = vpaddl_u16(vget_low_u16(q12));
+ d3 = vpaddl_u32(d1);
+
+ return vget_lane_u32(vreinterpret_u32_u64(d3), 0);
+}
+
+unsigned int vp8_sad16x16_neon(
+ unsigned char *src_ptr,
+ int src_stride,
+ unsigned char *ref_ptr,
+ int ref_stride) {
+ uint8x16_t q0, q4;
+ uint16x8_t q12, q13;
+ uint32x4_t q1;
+ uint64x2_t q3;
+ uint32x2_t d5;
+ int i;
+
+ q0 = vld1q_u8(src_ptr);
+ src_ptr += src_stride;
+ q4 = vld1q_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabdl_u8(vget_low_u8(q0), vget_low_u8(q4));
+ q13 = vabdl_u8(vget_high_u8(q0), vget_high_u8(q4));
+
+ for (i = 0; i < 15; i++) {
+ q0 = vld1q_u8(src_ptr);
+ src_ptr += src_stride;
+ q4 = vld1q_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabal_u8(q12, vget_low_u8(q0), vget_low_u8(q4));
+ q13 = vabal_u8(q13, vget_high_u8(q0), vget_high_u8(q4));
+ }
+
+ q12 = vaddq_u16(q12, q13);
+ q1 = vpaddlq_u16(q12);
+ q3 = vpaddlq_u32(q1);
+ d5 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(q3)),
+ vreinterpret_u32_u64(vget_high_u64(q3)));
+
+ return vget_lane_u32(d5, 0);
+}
+
+unsigned int vp8_sad16x8_neon(
+ unsigned char *src_ptr,
+ int src_stride,
+ unsigned char *ref_ptr,
+ int ref_stride) {
+ uint8x16_t q0, q4;
+ uint16x8_t q12, q13;
+ uint32x4_t q1;
+ uint64x2_t q3;
+ uint32x2_t d5;
+ int i;
+
+ q0 = vld1q_u8(src_ptr);
+ src_ptr += src_stride;
+ q4 = vld1q_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabdl_u8(vget_low_u8(q0), vget_low_u8(q4));
+ q13 = vabdl_u8(vget_high_u8(q0), vget_high_u8(q4));
+
+ for (i = 0; i < 7; i++) {
+ q0 = vld1q_u8(src_ptr);
+ src_ptr += src_stride;
+ q4 = vld1q_u8(ref_ptr);
+ ref_ptr += ref_stride;
+ q12 = vabal_u8(q12, vget_low_u8(q0), vget_low_u8(q4));
+ q13 = vabal_u8(q13, vget_high_u8(q0), vget_high_u8(q4));
+ }
+
+ q12 = vaddq_u16(q12, q13);
+ q1 = vpaddlq_u16(q12);
+ q3 = vpaddlq_u32(q1);
+ d5 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(q3)),
+ vreinterpret_u32_u64(vget_high_u64(q3)));
+
+ return vget_lane_u32(d5, 0);
+}
diff --git a/vp8/common/arm/neon/shortidct4x4llm_neon.asm b/vp8/common/arm/neon/shortidct4x4llm_neon.asm
deleted file mode 100644
index 87ca887..0000000
--- a/vp8/common/arm/neon/shortidct4x4llm_neon.asm
+++ /dev/null
@@ -1,142 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_short_idct4x4llm_neon|
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-;*************************************************************
-;void vp8_short_idct4x4llm_c(short *input, unsigned char *pred, int pitch,
-; unsigned char *dst, int stride)
-;r0 short * input
-;r1 short * pred
-;r2 int pitch
-;r3 unsigned char dst
-;sp int stride
-;*************************************************************
-
-; static const int cospi8sqrt2minus1=20091;
-; static const int sinpi8sqrt2 =35468;
-; static const int rounding = 0;
-
-; Optimization note: The resulted data from dequantization are signed
-; 13-bit data that is in the range of [-4096, 4095]. This allows to
-; use "vqdmulh"(neon) instruction since it won't go out of range
-; (13+16+1=30bits<32bits). This instruction gives the high half
-; result of the multiplication that is needed in IDCT.
-
-|vp8_short_idct4x4llm_neon| PROC
- vpush {d8-d15}
-
- adr r12, idct_coeff
- vld1.16 {q1, q2}, [r0]
- vld1.16 {d0}, [r12]
-
- vswp d3, d4 ;q2(vp[4] vp[12])
- ldr r0, [sp, #64] ; stride
-
- vqdmulh.s16 q3, q2, d0[2]
- vqdmulh.s16 q4, q2, d0[0]
-
- vqadd.s16 d12, d2, d3 ;a1
- vqsub.s16 d13, d2, d3 ;b1
-
- vshr.s16 q3, q3, #1
- vshr.s16 q4, q4, #1
-
- vqadd.s16 q3, q3, q2 ;modify since sinpi8sqrt2 > 65536/2 (negtive number)
- vqadd.s16 q4, q4, q2
-
- ;d6 - c1:temp1
- ;d7 - d1:temp2
- ;d8 - d1:temp1
- ;d9 - c1:temp2
-
- vqsub.s16 d10, d6, d9 ;c1
- vqadd.s16 d11, d7, d8 ;d1
-
- vqadd.s16 d2, d12, d11
- vqadd.s16 d3, d13, d10
- vqsub.s16 d4, d13, d10
- vqsub.s16 d5, d12, d11
-
- vtrn.32 d2, d4
- vtrn.32 d3, d5
- vtrn.16 d2, d3
- vtrn.16 d4, d5
-
- vswp d3, d4
-
- vqdmulh.s16 q3, q2, d0[2]
- vqdmulh.s16 q4, q2, d0[0]
-
- vqadd.s16 d12, d2, d3 ;a1
- vqsub.s16 d13, d2, d3 ;b1
-
- vshr.s16 q3, q3, #1
- vshr.s16 q4, q4, #1
-
- vqadd.s16 q3, q3, q2 ;modify since sinpi8sqrt2 > 65536/2 (negtive number)
- vqadd.s16 q4, q4, q2
-
- vqsub.s16 d10, d6, d9 ;c1
- vqadd.s16 d11, d7, d8 ;d1
-
- vqadd.s16 d2, d12, d11
- vqadd.s16 d3, d13, d10
- vqsub.s16 d4, d13, d10
- vqsub.s16 d5, d12, d11
-
- vrshr.s16 d2, d2, #3
- vrshr.s16 d3, d3, #3
- vrshr.s16 d4, d4, #3
- vrshr.s16 d5, d5, #3
-
- vtrn.32 d2, d4
- vtrn.32 d3, d5
- vtrn.16 d2, d3
- vtrn.16 d4, d5
-
- ; load prediction data
- vld1.32 d6[0], [r1], r2
- vld1.32 d6[1], [r1], r2
- vld1.32 d7[0], [r1], r2
- vld1.32 d7[1], [r1], r2
-
- ; add prediction and residual
- vaddw.u8 q1, q1, d6
- vaddw.u8 q2, q2, d7
-
- vqmovun.s16 d1, q1
- vqmovun.s16 d2, q2
-
- ; store to destination
- vst1.32 d1[0], [r3], r0
- vst1.32 d1[1], [r3], r0
- vst1.32 d2[0], [r3], r0
- vst1.32 d2[1], [r3], r0
-
- vpop {d8-d15}
- bx lr
-
- ENDP
-
-;-----------------
-
-idct_coeff
- DCD 0x4e7b4e7b, 0x8a8c8a8c
-
-;20091, 20091, 35468, 35468
-
- END
diff --git a/vp8/common/arm/neon/shortidct4x4llm_neon.c b/vp8/common/arm/neon/shortidct4x4llm_neon.c
new file mode 100644
index 0000000..373afa6
--- /dev/null
+++ b/vp8/common/arm/neon/shortidct4x4llm_neon.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+static const int16_t cospi8sqrt2minus1 = 20091;
+static const int16_t sinpi8sqrt2 = 35468;
+
+void vp8_short_idct4x4llm_neon(
+ int16_t *input,
+ unsigned char *pred_ptr,
+ int pred_stride,
+ unsigned char *dst_ptr,
+ int dst_stride) {
+ int i;
+ uint32x2_t d6u32 = vdup_n_u32(0);
+ uint8x8_t d1u8;
+ int16x4_t d2, d3, d4, d5, d10, d11, d12, d13;
+ uint16x8_t q1u16;
+ int16x8_t q1s16, q2s16, q3s16, q4s16;
+ int32x2x2_t v2tmp0, v2tmp1;
+ int16x4x2_t v2tmp2, v2tmp3;
+
+ d2 = vld1_s16(input);
+ d3 = vld1_s16(input + 4);
+ d4 = vld1_s16(input + 8);
+ d5 = vld1_s16(input + 12);
+
+ // 1st for loop
+ q1s16 = vcombine_s16(d2, d4); // Swap d3 d4 here
+ q2s16 = vcombine_s16(d3, d5);
+
+ q3s16 = vqdmulhq_n_s16(q2s16, sinpi8sqrt2);
+ q4s16 = vqdmulhq_n_s16(q2s16, cospi8sqrt2minus1);
+
+ d12 = vqadd_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // a1
+ d13 = vqsub_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // b1
+
+ q3s16 = vshrq_n_s16(q3s16, 1);
+ q4s16 = vshrq_n_s16(q4s16, 1);
+
+ q3s16 = vqaddq_s16(q3s16, q2s16);
+ q4s16 = vqaddq_s16(q4s16, q2s16);
+
+ d10 = vqsub_s16(vget_low_s16(q3s16), vget_high_s16(q4s16)); // c1
+ d11 = vqadd_s16(vget_high_s16(q3s16), vget_low_s16(q4s16)); // d1
+
+ d2 = vqadd_s16(d12, d11);
+ d3 = vqadd_s16(d13, d10);
+ d4 = vqsub_s16(d13, d10);
+ d5 = vqsub_s16(d12, d11);
+
+ v2tmp0 = vtrn_s32(vreinterpret_s32_s16(d2), vreinterpret_s32_s16(d4));
+ v2tmp1 = vtrn_s32(vreinterpret_s32_s16(d3), vreinterpret_s32_s16(d5));
+ v2tmp2 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[0]),
+ vreinterpret_s16_s32(v2tmp1.val[0]));
+ v2tmp3 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[1]),
+ vreinterpret_s16_s32(v2tmp1.val[1]));
+
+ // 2nd for loop
+ q1s16 = vcombine_s16(v2tmp2.val[0], v2tmp3.val[0]);
+ q2s16 = vcombine_s16(v2tmp2.val[1], v2tmp3.val[1]);
+
+ q3s16 = vqdmulhq_n_s16(q2s16, sinpi8sqrt2);
+ q4s16 = vqdmulhq_n_s16(q2s16, cospi8sqrt2minus1);
+
+ d12 = vqadd_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // a1
+ d13 = vqsub_s16(vget_low_s16(q1s16), vget_high_s16(q1s16)); // b1
+
+ q3s16 = vshrq_n_s16(q3s16, 1);
+ q4s16 = vshrq_n_s16(q4s16, 1);
+
+ q3s16 = vqaddq_s16(q3s16, q2s16);
+ q4s16 = vqaddq_s16(q4s16, q2s16);
+
+ d10 = vqsub_s16(vget_low_s16(q3s16), vget_high_s16(q4s16)); // c1
+ d11 = vqadd_s16(vget_high_s16(q3s16), vget_low_s16(q4s16)); // d1
+
+ d2 = vqadd_s16(d12, d11);
+ d3 = vqadd_s16(d13, d10);
+ d4 = vqsub_s16(d13, d10);
+ d5 = vqsub_s16(d12, d11);
+
+ d2 = vrshr_n_s16(d2, 3);
+ d3 = vrshr_n_s16(d3, 3);
+ d4 = vrshr_n_s16(d4, 3);
+ d5 = vrshr_n_s16(d5, 3);
+
+ v2tmp0 = vtrn_s32(vreinterpret_s32_s16(d2), vreinterpret_s32_s16(d4));
+ v2tmp1 = vtrn_s32(vreinterpret_s32_s16(d3), vreinterpret_s32_s16(d5));
+ v2tmp2 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[0]),
+ vreinterpret_s16_s32(v2tmp1.val[0]));
+ v2tmp3 = vtrn_s16(vreinterpret_s16_s32(v2tmp0.val[1]),
+ vreinterpret_s16_s32(v2tmp1.val[1]));
+
+ q1s16 = vcombine_s16(v2tmp2.val[0], v2tmp2.val[1]);
+ q2s16 = vcombine_s16(v2tmp3.val[0], v2tmp3.val[1]);
+
+ // dc_only_idct_add
+ for (i = 0; i < 2; i++, q1s16 = q2s16) {
+ d6u32 = vld1_lane_u32((const uint32_t *)pred_ptr, d6u32, 0);
+ pred_ptr += pred_stride;
+ d6u32 = vld1_lane_u32((const uint32_t *)pred_ptr, d6u32, 1);
+ pred_ptr += pred_stride;
+
+ q1u16 = vaddw_u8(vreinterpretq_u16_s16(q1s16),
+ vreinterpret_u8_u32(d6u32));
+ d1u8 = vqmovun_s16(vreinterpretq_s16_u16(q1u16));
+
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d1u8), 0);
+ dst_ptr += dst_stride;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d1u8), 1);
+ dst_ptr += dst_stride;
+ }
+ return;
+}
diff --git a/vp8/common/arm/neon/sixtappredict16x16_neon.asm b/vp8/common/arm/neon/sixtappredict16x16_neon.asm
deleted file mode 100644
index dd27719..0000000
--- a/vp8/common/arm/neon/sixtappredict16x16_neon.asm
+++ /dev/null
@@ -1,495 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_sixtap_predict16x16_neon|
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-filter16_coeff
- DCD 0, 0, 128, 0, 0, 0, 0, 0
- DCD 0, -6, 123, 12, -1, 0, 0, 0
- DCD 2, -11, 108, 36, -8, 1, 0, 0
- DCD 0, -9, 93, 50, -6, 0, 0, 0
- DCD 3, -16, 77, 77, -16, 3, 0, 0
- DCD 0, -6, 50, 93, -9, 0, 0, 0
- DCD 1, -8, 36, 108, -11, 2, 0, 0
- DCD 0, -1, 12, 123, -6, 0, 0, 0
-
-; r0 unsigned char *src_ptr,
-; r1 int src_pixels_per_line,
-; r2 int xoffset,
-; r3 int yoffset,
-; r4 unsigned char *dst_ptr,
-; stack(r5) int dst_pitch
-
-;Note: To take advantage of 8-bit mulplication instruction in NEON. First apply abs() to
-; filter coeffs to make them u8. Then, use vmlsl for negtive coeffs. After multiplication,
-; the result can be negtive. So, I treat the result as s16. But, since it is also possible
-; that the result can be a large positive number (> 2^15-1), which could be confused as a
-; negtive number. To avoid that error, apply filter coeffs in the order of 0, 1, 4 ,5 ,2,
-; which ensures that the result stays in s16 range. Finally, saturated add the result by
-; applying 3rd filter coeff. Same applys to other filter functions.
-
-|vp8_sixtap_predict16x16_neon| PROC
- push {r4-r5, lr}
- vpush {d8-d15}
-
- adr r12, filter16_coeff
- ldr r4, [sp, #76] ;load parameters from stack
- ldr r5, [sp, #80] ;load parameters from stack
-
- cmp r2, #0 ;skip first_pass filter if xoffset=0
- beq secondpass_filter16x16_only
-
- add r2, r12, r2, lsl #5 ;calculate filter location
-
- cmp r3, #0 ;skip second_pass filter if yoffset=0
-
- vld1.s32 {q14, q15}, [r2] ;load first_pass filter
-
- beq firstpass_filter16x16_only
-
- sub sp, sp, #336 ;reserve space on stack for temporary storage
- mov lr, sp
-
- vabs.s32 q12, q14
- vabs.s32 q13, q15
-
- mov r2, #7 ;loop counter
- sub r0, r0, #2 ;move srcptr back to (line-2) and (column-2)
- sub r0, r0, r1, lsl #1
-
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vdup.8 d1, d24[4]
- vdup.8 d2, d25[0]
- vdup.8 d3, d25[4]
- vdup.8 d4, d26[0]
- vdup.8 d5, d26[4]
-
-;First Pass: output_height lines x output_width columns (21x16)
-filt_blk2d_fp16x16_loop_neon
- vld1.u8 {d6, d7, d8}, [r0], r1 ;load src data
- vld1.u8 {d9, d10, d11}, [r0], r1
- vld1.u8 {d12, d13, d14}, [r0], r1
-
- pld [r0]
- pld [r0, r1]
- pld [r0, r1, lsl #1]
-
- vmull.u8 q8, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q9, d7, d0
- vmull.u8 q10, d9, d0
- vmull.u8 q11, d10, d0
- vmull.u8 q12, d12, d0
- vmull.u8 q13, d13, d0
-
- vext.8 d28, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d29, d9, d10, #1
- vext.8 d30, d12, d13, #1
-
- vmlsl.u8 q8, d28, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q10, d29, d1
- vmlsl.u8 q12, d30, d1
-
- vext.8 d28, d7, d8, #1
- vext.8 d29, d10, d11, #1
- vext.8 d30, d13, d14, #1
-
- vmlsl.u8 q9, d28, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q11, d29, d1
- vmlsl.u8 q13, d30, d1
-
- vext.8 d28, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d29, d9, d10, #4
- vext.8 d30, d12, d13, #4
-
- vmlsl.u8 q8, d28, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q10, d29, d4
- vmlsl.u8 q12, d30, d4
-
- vext.8 d28, d7, d8, #4
- vext.8 d29, d10, d11, #4
- vext.8 d30, d13, d14, #4
-
- vmlsl.u8 q9, d28, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q11, d29, d4
- vmlsl.u8 q13, d30, d4
-
- vext.8 d28, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d29, d9, d10, #5
- vext.8 d30, d12, d13, #5
-
- vmlal.u8 q8, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q10, d29, d5
- vmlal.u8 q12, d30, d5
-
- vext.8 d28, d7, d8, #5
- vext.8 d29, d10, d11, #5
- vext.8 d30, d13, d14, #5
-
- vmlal.u8 q9, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q11, d29, d5
- vmlal.u8 q13, d30, d5
-
- vext.8 d28, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d29, d9, d10, #2
- vext.8 d30, d12, d13, #2
-
- vmlal.u8 q8, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q10, d29, d2
- vmlal.u8 q12, d30, d2
-
- vext.8 d28, d7, d8, #2
- vext.8 d29, d10, d11, #2
- vext.8 d30, d13, d14, #2
-
- vmlal.u8 q9, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q11, d29, d2
- vmlal.u8 q13, d30, d2
-
- vext.8 d28, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d29, d9, d10, #3
- vext.8 d30, d12, d13, #3
-
- vext.8 d15, d7, d8, #3
- vext.8 d31, d10, d11, #3
- vext.8 d6, d13, d14, #3
-
- vmull.u8 q4, d28, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q5, d29, d3
- vmull.u8 q6, d30, d3
-
- vqadd.s16 q8, q4 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q10, q5
- vqadd.s16 q12, q6
-
- vmull.u8 q6, d15, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q7, d31, d3
- vmull.u8 q3, d6, d3
-
- subs r2, r2, #1
-
- vqadd.s16 q9, q6
- vqadd.s16 q11, q7
- vqadd.s16 q13, q3
-
- vqrshrun.s16 d6, q8, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q9, #7
- vqrshrun.s16 d8, q10, #7
- vqrshrun.s16 d9, q11, #7
- vqrshrun.s16 d10, q12, #7
- vqrshrun.s16 d11, q13, #7
-
- vst1.u8 {d6, d7, d8}, [lr]! ;store result
- vst1.u8 {d9, d10, d11}, [lr]!
-
- bne filt_blk2d_fp16x16_loop_neon
-
-;Second pass: 16x16
-;secondpass_filter - do first 8-columns and then second 8-columns
- add r3, r12, r3, lsl #5
- sub lr, lr, #336
-
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
- mov r3, #2 ;loop counter
-
- vabs.s32 q7, q5
- vabs.s32 q8, q6
-
- mov r2, #16
-
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vdup.8 d1, d14[4]
- vdup.8 d2, d15[0]
- vdup.8 d3, d15[4]
- vdup.8 d4, d16[0]
- vdup.8 d5, d16[4]
-
-filt_blk2d_sp16x16_outloop_neon
- vld1.u8 {d18}, [lr], r2 ;load src data
- vld1.u8 {d19}, [lr], r2
- vld1.u8 {d20}, [lr], r2
- vld1.u8 {d21}, [lr], r2
- mov r12, #4 ;loop counter
- vld1.u8 {d22}, [lr], r2
-
-secondpass_inner_loop_neon
- vld1.u8 {d23}, [lr], r2 ;load src data
- vld1.u8 {d24}, [lr], r2
- vld1.u8 {d25}, [lr], r2
- vld1.u8 {d26}, [lr], r2
-
- vmull.u8 q3, d18, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d19, d0
- vmull.u8 q5, d20, d0
- vmull.u8 q6, d21, d0
-
- vmlsl.u8 q3, d19, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q4, d20, d1
- vmlsl.u8 q5, d21, d1
- vmlsl.u8 q6, d22, d1
-
- vmlsl.u8 q3, d22, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d23, d4
- vmlsl.u8 q5, d24, d4
- vmlsl.u8 q6, d25, d4
-
- vmlal.u8 q3, d20, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d21, d2
- vmlal.u8 q5, d22, d2
- vmlal.u8 q6, d23, d2
-
- vmlal.u8 q3, d23, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q4, d24, d5
- vmlal.u8 q5, d25, d5
- vmlal.u8 q6, d26, d5
-
- vmull.u8 q7, d21, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q8, d22, d3
- vmull.u8 q9, d23, d3
- vmull.u8 q10, d24, d3
-
- subs r12, r12, #1
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d6, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q8, #7
- vqrshrun.s16 d8, q9, #7
- vqrshrun.s16 d9, q10, #7
-
- vst1.u8 {d6}, [r4], r5 ;store result
- vmov q9, q11
- vst1.u8 {d7}, [r4], r5
- vmov q10, q12
- vst1.u8 {d8}, [r4], r5
- vmov d22, d26
- vst1.u8 {d9}, [r4], r5
-
- bne secondpass_inner_loop_neon
-
- subs r3, r3, #1
- sub lr, lr, #336
- add lr, lr, #8
-
- sub r4, r4, r5, lsl #4
- add r4, r4, #8
-
- bne filt_blk2d_sp16x16_outloop_neon
-
- add sp, sp, #336
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
-;--------------------
-firstpass_filter16x16_only
- vabs.s32 q12, q14
- vabs.s32 q13, q15
-
- mov r2, #8 ;loop counter
- sub r0, r0, #2 ;move srcptr back to (column-2)
-
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vdup.8 d1, d24[4]
- vdup.8 d2, d25[0]
- vdup.8 d3, d25[4]
- vdup.8 d4, d26[0]
- vdup.8 d5, d26[4]
-
-;First Pass: output_height lines x output_width columns (16x16)
-filt_blk2d_fpo16x16_loop_neon
- vld1.u8 {d6, d7, d8}, [r0], r1 ;load src data
- vld1.u8 {d9, d10, d11}, [r0], r1
-
- pld [r0]
- pld [r0, r1]
-
- vmull.u8 q6, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q7, d7, d0
- vmull.u8 q8, d9, d0
- vmull.u8 q9, d10, d0
-
- vext.8 d20, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d21, d9, d10, #1
- vext.8 d22, d7, d8, #1
- vext.8 d23, d10, d11, #1
- vext.8 d24, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d25, d9, d10, #4
- vext.8 d26, d7, d8, #4
- vext.8 d27, d10, d11, #4
- vext.8 d28, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d29, d9, d10, #5
-
- vmlsl.u8 q6, d20, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d21, d1
- vmlsl.u8 q7, d22, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q9, d23, d1
- vmlsl.u8 q6, d24, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d25, d4
- vmlsl.u8 q7, d26, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q9, d27, d4
- vmlal.u8 q6, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q8, d29, d5
-
- vext.8 d20, d7, d8, #5
- vext.8 d21, d10, d11, #5
- vext.8 d22, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d23, d9, d10, #2
- vext.8 d24, d7, d8, #2
- vext.8 d25, d10, d11, #2
-
- vext.8 d26, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d27, d9, d10, #3
- vext.8 d28, d7, d8, #3
- vext.8 d29, d10, d11, #3
-
- vmlal.u8 q7, d20, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q9, d21, d5
- vmlal.u8 q6, d22, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d23, d2
- vmlal.u8 q7, d24, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q9, d25, d2
-
- vmull.u8 q10, d26, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q11, d27, d3
- vmull.u8 q12, d28, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q15, d29, d3
-
- vqadd.s16 q6, q10 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q11
- vqadd.s16 q7, q12
- vqadd.s16 q9, q15
-
- subs r2, r2, #1
-
- vqrshrun.s16 d6, q6, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q7, #7
- vqrshrun.s16 d8, q8, #7
- vqrshrun.s16 d9, q9, #7
-
- vst1.u8 {q3}, [r4], r5 ;store result
- vst1.u8 {q4}, [r4], r5
-
- bne filt_blk2d_fpo16x16_loop_neon
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
-;--------------------
-secondpass_filter16x16_only
-;Second pass: 16x16
- add r3, r12, r3, lsl #5
- sub r0, r0, r1, lsl #1
-
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
- mov r3, #2 ;loop counter
-
- vabs.s32 q7, q5
- vabs.s32 q8, q6
-
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vdup.8 d1, d14[4]
- vdup.8 d2, d15[0]
- vdup.8 d3, d15[4]
- vdup.8 d4, d16[0]
- vdup.8 d5, d16[4]
-
-filt_blk2d_spo16x16_outloop_neon
- vld1.u8 {d18}, [r0], r1 ;load src data
- vld1.u8 {d19}, [r0], r1
- vld1.u8 {d20}, [r0], r1
- vld1.u8 {d21}, [r0], r1
- mov r12, #4 ;loop counter
- vld1.u8 {d22}, [r0], r1
-
-secondpass_only_inner_loop_neon
- vld1.u8 {d23}, [r0], r1 ;load src data
- vld1.u8 {d24}, [r0], r1
- vld1.u8 {d25}, [r0], r1
- vld1.u8 {d26}, [r0], r1
-
- vmull.u8 q3, d18, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d19, d0
- vmull.u8 q5, d20, d0
- vmull.u8 q6, d21, d0
-
- vmlsl.u8 q3, d19, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q4, d20, d1
- vmlsl.u8 q5, d21, d1
- vmlsl.u8 q6, d22, d1
-
- vmlsl.u8 q3, d22, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d23, d4
- vmlsl.u8 q5, d24, d4
- vmlsl.u8 q6, d25, d4
-
- vmlal.u8 q3, d20, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d21, d2
- vmlal.u8 q5, d22, d2
- vmlal.u8 q6, d23, d2
-
- vmlal.u8 q3, d23, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q4, d24, d5
- vmlal.u8 q5, d25, d5
- vmlal.u8 q6, d26, d5
-
- vmull.u8 q7, d21, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q8, d22, d3
- vmull.u8 q9, d23, d3
- vmull.u8 q10, d24, d3
-
- subs r12, r12, #1
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d6, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q8, #7
- vqrshrun.s16 d8, q9, #7
- vqrshrun.s16 d9, q10, #7
-
- vst1.u8 {d6}, [r4], r5 ;store result
- vmov q9, q11
- vst1.u8 {d7}, [r4], r5
- vmov q10, q12
- vst1.u8 {d8}, [r4], r5
- vmov d22, d26
- vst1.u8 {d9}, [r4], r5
-
- bne secondpass_only_inner_loop_neon
-
- subs r3, r3, #1
- sub r0, r0, r1, lsl #4
- sub r0, r0, r1, lsl #2
- sub r0, r0, r1
- add r0, r0, #8
-
- sub r4, r4, r5, lsl #4
- add r4, r4, #8
-
- bne filt_blk2d_spo16x16_outloop_neon
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
- ENDP
-
-;-----------------
- END
diff --git a/vp8/common/arm/neon/sixtappredict4x4_neon.asm b/vp8/common/arm/neon/sixtappredict4x4_neon.asm
deleted file mode 100644
index e32e713..0000000
--- a/vp8/common/arm/neon/sixtappredict4x4_neon.asm
+++ /dev/null
@@ -1,426 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_sixtap_predict4x4_neon|
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-filter4_coeff
- DCD 0, 0, 128, 0, 0, 0, 0, 0
- DCD 0, -6, 123, 12, -1, 0, 0, 0
- DCD 2, -11, 108, 36, -8, 1, 0, 0
- DCD 0, -9, 93, 50, -6, 0, 0, 0
- DCD 3, -16, 77, 77, -16, 3, 0, 0
- DCD 0, -6, 50, 93, -9, 0, 0, 0
- DCD 1, -8, 36, 108, -11, 2, 0, 0
- DCD 0, -1, 12, 123, -6, 0, 0, 0
-
-; r0 unsigned char *src_ptr,
-; r1 int src_pixels_per_line,
-; r2 int xoffset,
-; r3 int yoffset,
-; stack(r4) unsigned char *dst_ptr,
-; stack(lr) int dst_pitch
-
-|vp8_sixtap_predict4x4_neon| PROC
- push {r4, lr}
- vpush {d8-d15}
-
- adr r12, filter4_coeff
- ldr r4, [sp, #72] ;load parameters from stack
- ldr lr, [sp, #76] ;load parameters from stack
-
- cmp r2, #0 ;skip first_pass filter if xoffset=0
- beq secondpass_filter4x4_only
-
- add r2, r12, r2, lsl #5 ;calculate filter location
-
- cmp r3, #0 ;skip second_pass filter if yoffset=0
- vld1.s32 {q14, q15}, [r2] ;load first_pass filter
-
- beq firstpass_filter4x4_only
-
- vabs.s32 q12, q14 ;get abs(filer_parameters)
- vabs.s32 q13, q15
-
- sub r0, r0, #2 ;go back 2 columns of src data
- sub r0, r0, r1, lsl #1 ;go back 2 lines of src data
-
-;First pass: output_height lines x output_width columns (9x4)
- vld1.u8 {q3}, [r0], r1 ;load first 4-line src data
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vld1.u8 {q4}, [r0], r1
- vdup.8 d1, d24[4]
- vld1.u8 {q5}, [r0], r1
- vdup.8 d2, d25[0]
- vld1.u8 {q6}, [r0], r1
- vdup.8 d3, d25[4]
- vdup.8 d4, d26[0]
- vdup.8 d5, d26[4]
-
- pld [r0]
- pld [r0, r1]
- pld [r0, r1, lsl #1]
-
- vext.8 d18, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d19, d8, d9, #5
- vext.8 d20, d10, d11, #5
- vext.8 d21, d12, d13, #5
-
- vswp d7, d8 ;discard 2nd half data after src_ptr[3] is done
- vswp d11, d12
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[3])
- vzip.32 d20, d21
- vmull.u8 q7, d18, d5 ;(src_ptr[3] * vp8_filter[5])
- vmull.u8 q8, d20, d5
-
- vmov q4, q3 ;keep original src data in q4 q6
- vmov q6, q5
-
- vzip.32 d6, d7 ;construct src_ptr[-2], and put 2-line data together
- vzip.32 d10, d11
- vshr.u64 q9, q4, #8 ;construct src_ptr[-1]
- vshr.u64 q10, q6, #8
- vmlal.u8 q7, d6, d0 ;+(src_ptr[-2] * vp8_filter[0])
- vmlal.u8 q8, d10, d0
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[-1])
- vzip.32 d20, d21
- vshr.u64 q3, q4, #32 ;construct src_ptr[2]
- vshr.u64 q5, q6, #32
- vmlsl.u8 q7, d18, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d20, d1
-
- vzip.32 d6, d7 ;put 2-line data in 1 register (src_ptr[2])
- vzip.32 d10, d11
- vshr.u64 q9, q4, #16 ;construct src_ptr[0]
- vshr.u64 q10, q6, #16
- vmlsl.u8 q7, d6, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d10, d4
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[0])
- vzip.32 d20, d21
- vshr.u64 q3, q4, #24 ;construct src_ptr[1]
- vshr.u64 q5, q6, #24
- vmlal.u8 q7, d18, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d20, d2
-
- vzip.32 d6, d7 ;put 2-line data in 1 register (src_ptr[1])
- vzip.32 d10, d11
- vmull.u8 q9, d6, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q10, d10, d3
-
- vld1.u8 {q3}, [r0], r1 ;load rest 5-line src data
- vld1.u8 {q4}, [r0], r1
-
- vqadd.s16 q7, q9 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q10
-
- vld1.u8 {q5}, [r0], r1
- vld1.u8 {q6}, [r0], r1
-
- vqrshrun.s16 d27, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d28, q8, #7
-
- ;First Pass on rest 5-line data
- vld1.u8 {q11}, [r0], r1
-
- vext.8 d18, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d19, d8, d9, #5
- vext.8 d20, d10, d11, #5
- vext.8 d21, d12, d13, #5
-
- vswp d7, d8 ;discard 2nd half data after src_ptr[3] is done
- vswp d11, d12
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[3])
- vzip.32 d20, d21
- vext.8 d31, d22, d23, #5 ;construct src_ptr[3]
- vmull.u8 q7, d18, d5 ;(src_ptr[3] * vp8_filter[5])
- vmull.u8 q8, d20, d5
- vmull.u8 q12, d31, d5 ;(src_ptr[3] * vp8_filter[5])
-
- vmov q4, q3 ;keep original src data in q4 q6
- vmov q6, q5
-
- vzip.32 d6, d7 ;construct src_ptr[-2], and put 2-line data together
- vzip.32 d10, d11
- vshr.u64 q9, q4, #8 ;construct src_ptr[-1]
- vshr.u64 q10, q6, #8
-
- vmlal.u8 q7, d6, d0 ;+(src_ptr[-2] * vp8_filter[0])
- vmlal.u8 q8, d10, d0
- vmlal.u8 q12, d22, d0 ;(src_ptr[-2] * vp8_filter[0])
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[-1])
- vzip.32 d20, d21
- vshr.u64 q3, q4, #32 ;construct src_ptr[2]
- vshr.u64 q5, q6, #32
- vext.8 d31, d22, d23, #1 ;construct src_ptr[-1]
-
- vmlsl.u8 q7, d18, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d20, d1
- vmlsl.u8 q12, d31, d1 ;-(src_ptr[-1] * vp8_filter[1])
-
- vzip.32 d6, d7 ;put 2-line data in 1 register (src_ptr[2])
- vzip.32 d10, d11
- vshr.u64 q9, q4, #16 ;construct src_ptr[0]
- vshr.u64 q10, q6, #16
- vext.8 d31, d22, d23, #4 ;construct src_ptr[2]
-
- vmlsl.u8 q7, d6, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d10, d4
- vmlsl.u8 q12, d31, d4 ;-(src_ptr[2] * vp8_filter[4])
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[0])
- vzip.32 d20, d21
- vshr.u64 q3, q4, #24 ;construct src_ptr[1]
- vshr.u64 q5, q6, #24
- vext.8 d31, d22, d23, #2 ;construct src_ptr[0]
-
- vmlal.u8 q7, d18, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d20, d2
- vmlal.u8 q12, d31, d2 ;(src_ptr[0] * vp8_filter[2])
-
- vzip.32 d6, d7 ;put 2-line data in 1 register (src_ptr[1])
- vzip.32 d10, d11
- vext.8 d31, d22, d23, #3 ;construct src_ptr[1]
- vmull.u8 q9, d6, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q10, d10, d3
- vmull.u8 q11, d31, d3 ;(src_ptr[1] * vp8_filter[3])
-
- add r3, r12, r3, lsl #5
-
- vqadd.s16 q7, q9 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q10
- vqadd.s16 q12, q11
-
- vext.8 d23, d27, d28, #4
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
-
- vqrshrun.s16 d29, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d30, q8, #7
- vqrshrun.s16 d31, q12, #7
-
-;Second pass: 4x4
- vabs.s32 q7, q5
- vabs.s32 q8, q6
-
- vext.8 d24, d28, d29, #4
- vext.8 d25, d29, d30, #4
- vext.8 d26, d30, d31, #4
-
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vdup.8 d1, d14[4]
- vdup.8 d2, d15[0]
- vdup.8 d3, d15[4]
- vdup.8 d4, d16[0]
- vdup.8 d5, d16[4]
-
- vmull.u8 q3, d27, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d28, d0
-
- vmull.u8 q5, d25, d5 ;(src_ptr[3] * vp8_filter[5])
- vmull.u8 q6, d26, d5
-
- vmlsl.u8 q3, d29, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d30, d4
-
- vmlsl.u8 q5, d23, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q6, d24, d1
-
- vmlal.u8 q3, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d29, d2
-
- vmlal.u8 q5, d24, d3 ;(src_ptr[1] * vp8_filter[3])
- vmlal.u8 q6, d25, d3
-
- add r0, r4, lr
- add r1, r0, lr
- add r2, r1, lr
-
- vqadd.s16 q5, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q6, q4
-
- vqrshrun.s16 d3, q5, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d4, q6, #7
-
- vst1.32 {d3[0]}, [r4] ;store result
- vst1.32 {d3[1]}, [r0]
- vst1.32 {d4[0]}, [r1]
- vst1.32 {d4[1]}, [r2]
-
- vpop {d8-d15}
- pop {r4, pc}
-
-
-;---------------------
-firstpass_filter4x4_only
- vabs.s32 q12, q14 ;get abs(filer_parameters)
- vabs.s32 q13, q15
-
- sub r0, r0, #2 ;go back 2 columns of src data
-
-;First pass: output_height lines x output_width columns (4x4)
- vld1.u8 {q3}, [r0], r1 ;load first 4-line src data
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vld1.u8 {q4}, [r0], r1
- vdup.8 d1, d24[4]
- vld1.u8 {q5}, [r0], r1
- vdup.8 d2, d25[0]
- vld1.u8 {q6}, [r0], r1
-
- vdup.8 d3, d25[4]
- vdup.8 d4, d26[0]
- vdup.8 d5, d26[4]
-
- vext.8 d18, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d19, d8, d9, #5
- vext.8 d20, d10, d11, #5
- vext.8 d21, d12, d13, #5
-
- vswp d7, d8 ;discard 2nd half data after src_ptr[3] is done
- vswp d11, d12
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[3])
- vzip.32 d20, d21
- vmull.u8 q7, d18, d5 ;(src_ptr[3] * vp8_filter[5])
- vmull.u8 q8, d20, d5
-
- vmov q4, q3 ;keep original src data in q4 q6
- vmov q6, q5
-
- vzip.32 d6, d7 ;construct src_ptr[-2], and put 2-line data together
- vzip.32 d10, d11
- vshr.u64 q9, q4, #8 ;construct src_ptr[-1]
- vshr.u64 q10, q6, #8
- vmlal.u8 q7, d6, d0 ;+(src_ptr[-2] * vp8_filter[0])
- vmlal.u8 q8, d10, d0
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[-1])
- vzip.32 d20, d21
- vshr.u64 q3, q4, #32 ;construct src_ptr[2]
- vshr.u64 q5, q6, #32
- vmlsl.u8 q7, d18, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d20, d1
-
- vzip.32 d6, d7 ;put 2-line data in 1 register (src_ptr[2])
- vzip.32 d10, d11
- vshr.u64 q9, q4, #16 ;construct src_ptr[0]
- vshr.u64 q10, q6, #16
- vmlsl.u8 q7, d6, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d10, d4
-
- vzip.32 d18, d19 ;put 2-line data in 1 register (src_ptr[0])
- vzip.32 d20, d21
- vshr.u64 q3, q4, #24 ;construct src_ptr[1]
- vshr.u64 q5, q6, #24
- vmlal.u8 q7, d18, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d20, d2
-
- vzip.32 d6, d7 ;put 2-line data in 1 register (src_ptr[1])
- vzip.32 d10, d11
- vmull.u8 q9, d6, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q10, d10, d3
-
- add r0, r4, lr
- add r1, r0, lr
- add r2, r1, lr
-
- vqadd.s16 q7, q9 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q10
-
- vqrshrun.s16 d27, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d28, q8, #7
-
- vst1.32 {d27[0]}, [r4] ;store result
- vst1.32 {d27[1]}, [r0]
- vst1.32 {d28[0]}, [r1]
- vst1.32 {d28[1]}, [r2]
-
- vpop {d8-d15}
- pop {r4, pc}
-
-
-;---------------------
-secondpass_filter4x4_only
- sub r0, r0, r1, lsl #1
- add r3, r12, r3, lsl #5
-
- vld1.32 {d27[0]}, [r0], r1 ;load src data
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
- vld1.32 {d27[1]}, [r0], r1
- vabs.s32 q7, q5
- vld1.32 {d28[0]}, [r0], r1
- vabs.s32 q8, q6
- vld1.32 {d28[1]}, [r0], r1
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vld1.32 {d29[0]}, [r0], r1
- vdup.8 d1, d14[4]
- vld1.32 {d29[1]}, [r0], r1
- vdup.8 d2, d15[0]
- vld1.32 {d30[0]}, [r0], r1
- vdup.8 d3, d15[4]
- vld1.32 {d30[1]}, [r0], r1
- vdup.8 d4, d16[0]
- vld1.32 {d31[0]}, [r0], r1
- vdup.8 d5, d16[4]
-
- vext.8 d23, d27, d28, #4
- vext.8 d24, d28, d29, #4
- vext.8 d25, d29, d30, #4
- vext.8 d26, d30, d31, #4
-
- vmull.u8 q3, d27, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d28, d0
-
- vmull.u8 q5, d25, d5 ;(src_ptr[3] * vp8_filter[5])
- vmull.u8 q6, d26, d5
-
- vmlsl.u8 q3, d29, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d30, d4
-
- vmlsl.u8 q5, d23, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q6, d24, d1
-
- vmlal.u8 q3, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d29, d2
-
- vmlal.u8 q5, d24, d3 ;(src_ptr[1] * vp8_filter[3])
- vmlal.u8 q6, d25, d3
-
- add r0, r4, lr
- add r1, r0, lr
- add r2, r1, lr
-
- vqadd.s16 q5, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q6, q4
-
- vqrshrun.s16 d3, q5, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d4, q6, #7
-
- vst1.32 {d3[0]}, [r4] ;store result
- vst1.32 {d3[1]}, [r0]
- vst1.32 {d4[0]}, [r1]
- vst1.32 {d4[1]}, [r2]
-
- vpop {d8-d15}
- pop {r4, pc}
-
- ENDP
-
-;-----------------
-
- END
diff --git a/vp8/common/arm/neon/sixtappredict8x4_neon.asm b/vp8/common/arm/neon/sixtappredict8x4_neon.asm
deleted file mode 100644
index d19bf89..0000000
--- a/vp8/common/arm/neon/sixtappredict8x4_neon.asm
+++ /dev/null
@@ -1,478 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_sixtap_predict8x4_neon|
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-filter8_coeff
- DCD 0, 0, 128, 0, 0, 0, 0, 0
- DCD 0, -6, 123, 12, -1, 0, 0, 0
- DCD 2, -11, 108, 36, -8, 1, 0, 0
- DCD 0, -9, 93, 50, -6, 0, 0, 0
- DCD 3, -16, 77, 77, -16, 3, 0, 0
- DCD 0, -6, 50, 93, -9, 0, 0, 0
- DCD 1, -8, 36, 108, -11, 2, 0, 0
- DCD 0, -1, 12, 123, -6, 0, 0, 0
-
-; r0 unsigned char *src_ptr,
-; r1 int src_pixels_per_line,
-; r2 int xoffset,
-; r3 int yoffset,
-; r4 unsigned char *dst_ptr,
-; stack(r5) int dst_pitch
-
-|vp8_sixtap_predict8x4_neon| PROC
- push {r4-r5, lr}
- vpush {d8-d15}
-
- adr r12, filter8_coeff
- ldr r4, [sp, #76] ;load parameters from stack
- ldr r5, [sp, #80] ;load parameters from stack
-
- cmp r2, #0 ;skip first_pass filter if xoffset=0
- beq secondpass_filter8x4_only
-
- add r2, r12, r2, lsl #5 ;calculate filter location
-
- cmp r3, #0 ;skip second_pass filter if yoffset=0
-
- vld1.s32 {q14, q15}, [r2] ;load first_pass filter
-
- beq firstpass_filter8x4_only
-
- sub sp, sp, #32 ;reserve space on stack for temporary storage
- vabs.s32 q12, q14
- vabs.s32 q13, q15
-
- sub r0, r0, #2 ;move srcptr back to (line-2) and (column-2)
- mov lr, sp
- sub r0, r0, r1, lsl #1
-
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vdup.8 d1, d24[4]
- vdup.8 d2, d25[0]
-
-;First pass: output_height lines x output_width columns (9x8)
- vld1.u8 {q3}, [r0], r1 ;load src data
- vdup.8 d3, d25[4]
- vld1.u8 {q4}, [r0], r1
- vdup.8 d4, d26[0]
- vld1.u8 {q5}, [r0], r1
- vdup.8 d5, d26[4]
- vld1.u8 {q6}, [r0], r1
-
- pld [r0]
- pld [r0, r1]
- pld [r0, r1, lsl #1]
-
- vmull.u8 q7, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q8, d8, d0
- vmull.u8 q9, d10, d0
- vmull.u8 q10, d12, d0
-
- vext.8 d28, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d29, d8, d9, #1
- vext.8 d30, d10, d11, #1
- vext.8 d31, d12, d13, #1
-
- vmlsl.u8 q7, d28, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d29, d1
- vmlsl.u8 q9, d30, d1
- vmlsl.u8 q10, d31, d1
-
- vext.8 d28, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d29, d8, d9, #4
- vext.8 d30, d10, d11, #4
- vext.8 d31, d12, d13, #4
-
- vmlsl.u8 q7, d28, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d29, d4
- vmlsl.u8 q9, d30, d4
- vmlsl.u8 q10, d31, d4
-
- vext.8 d28, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d29, d8, d9, #2
- vext.8 d30, d10, d11, #2
- vext.8 d31, d12, d13, #2
-
- vmlal.u8 q7, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d29, d2
- vmlal.u8 q9, d30, d2
- vmlal.u8 q10, d31, d2
-
- vext.8 d28, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d29, d8, d9, #5
- vext.8 d30, d10, d11, #5
- vext.8 d31, d12, d13, #5
-
- vmlal.u8 q7, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q8, d29, d5
- vmlal.u8 q9, d30, d5
- vmlal.u8 q10, d31, d5
-
- vext.8 d28, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d29, d8, d9, #3
- vext.8 d30, d10, d11, #3
- vext.8 d31, d12, d13, #3
-
- vmull.u8 q3, d28, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q4, d29, d3
- vmull.u8 q5, d30, d3
- vmull.u8 q6, d31, d3
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vld1.u8 {q3}, [r0], r1 ;load src data
-
- vqrshrun.s16 d22, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d23, q8, #7
- vqrshrun.s16 d24, q9, #7
- vqrshrun.s16 d25, q10, #7
-
- vld1.u8 {q4}, [r0], r1
- vst1.u8 {d22}, [lr]! ;store result
- vld1.u8 {q5}, [r0], r1
- vst1.u8 {d23}, [lr]!
- vld1.u8 {q6}, [r0], r1
- vst1.u8 {d24}, [lr]!
- vld1.u8 {q7}, [r0], r1
- vst1.u8 {d25}, [lr]!
-
- ;first_pass filtering on the rest 5-line data
- vmull.u8 q8, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q9, d8, d0
- vmull.u8 q10, d10, d0
- vmull.u8 q11, d12, d0
- vmull.u8 q12, d14, d0
-
- vext.8 d27, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d28, d8, d9, #1
- vext.8 d29, d10, d11, #1
- vext.8 d30, d12, d13, #1
- vext.8 d31, d14, d15, #1
-
- vmlsl.u8 q8, d27, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q9, d28, d1
- vmlsl.u8 q10, d29, d1
- vmlsl.u8 q11, d30, d1
- vmlsl.u8 q12, d31, d1
-
- vext.8 d27, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d28, d8, d9, #4
- vext.8 d29, d10, d11, #4
- vext.8 d30, d12, d13, #4
- vext.8 d31, d14, d15, #4
-
- vmlsl.u8 q8, d27, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q9, d28, d4
- vmlsl.u8 q10, d29, d4
- vmlsl.u8 q11, d30, d4
- vmlsl.u8 q12, d31, d4
-
- vext.8 d27, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d28, d8, d9, #2
- vext.8 d29, d10, d11, #2
- vext.8 d30, d12, d13, #2
- vext.8 d31, d14, d15, #2
-
- vmlal.u8 q8, d27, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q9, d28, d2
- vmlal.u8 q10, d29, d2
- vmlal.u8 q11, d30, d2
- vmlal.u8 q12, d31, d2
-
- vext.8 d27, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d28, d8, d9, #5
- vext.8 d29, d10, d11, #5
- vext.8 d30, d12, d13, #5
- vext.8 d31, d14, d15, #5
-
- vmlal.u8 q8, d27, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q9, d28, d5
- vmlal.u8 q10, d29, d5
- vmlal.u8 q11, d30, d5
- vmlal.u8 q12, d31, d5
-
- vext.8 d27, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d28, d8, d9, #3
- vext.8 d29, d10, d11, #3
- vext.8 d30, d12, d13, #3
- vext.8 d31, d14, d15, #3
-
- vmull.u8 q3, d27, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q4, d28, d3
- vmull.u8 q5, d29, d3
- vmull.u8 q6, d30, d3
- vmull.u8 q7, d31, d3
-
- vqadd.s16 q8, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q9, q4
- vqadd.s16 q10, q5
- vqadd.s16 q11, q6
- vqadd.s16 q12, q7
-
- vqrshrun.s16 d26, q8, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d27, q9, #7
- vqrshrun.s16 d28, q10, #7
- vqrshrun.s16 d29, q11, #7 ;load intermediate data from stack
- vqrshrun.s16 d30, q12, #7
-
-;Second pass: 8x4
-;secondpass_filter
- add r3, r12, r3, lsl #5
- sub lr, lr, #32
-
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
- vld1.u8 {q11}, [lr]!
-
- vabs.s32 q7, q5
- vabs.s32 q8, q6
-
- vld1.u8 {q12}, [lr]!
-
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vdup.8 d1, d14[4]
- vdup.8 d2, d15[0]
- vdup.8 d3, d15[4]
- vdup.8 d4, d16[0]
- vdup.8 d5, d16[4]
-
- vmull.u8 q3, d22, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d23, d0
- vmull.u8 q5, d24, d0
- vmull.u8 q6, d25, d0
-
- vmlsl.u8 q3, d23, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q4, d24, d1
- vmlsl.u8 q5, d25, d1
- vmlsl.u8 q6, d26, d1
-
- vmlsl.u8 q3, d26, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d27, d4
- vmlsl.u8 q5, d28, d4
- vmlsl.u8 q6, d29, d4
-
- vmlal.u8 q3, d24, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d25, d2
- vmlal.u8 q5, d26, d2
- vmlal.u8 q6, d27, d2
-
- vmlal.u8 q3, d27, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q4, d28, d5
- vmlal.u8 q5, d29, d5
- vmlal.u8 q6, d30, d5
-
- vmull.u8 q7, d25, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q8, d26, d3
- vmull.u8 q9, d27, d3
- vmull.u8 q10, d28, d3
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d6, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q8, #7
- vqrshrun.s16 d8, q9, #7
- vqrshrun.s16 d9, q10, #7
-
- vst1.u8 {d6}, [r4], r5 ;store result
- vst1.u8 {d7}, [r4], r5
- vst1.u8 {d8}, [r4], r5
- vst1.u8 {d9}, [r4], r5
-
- add sp, sp, #32
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
-;--------------------
-firstpass_filter8x4_only
- vabs.s32 q12, q14
- vabs.s32 q13, q15
-
- sub r0, r0, #2 ;move srcptr back to (line-2) and (column-2)
- vld1.u8 {q3}, [r0], r1 ;load src data
-
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vld1.u8 {q4}, [r0], r1
- vdup.8 d1, d24[4]
- vld1.u8 {q5}, [r0], r1
- vdup.8 d2, d25[0]
- vld1.u8 {q6}, [r0], r1
- vdup.8 d3, d25[4]
- vdup.8 d4, d26[0]
- vdup.8 d5, d26[4]
-
-;First pass: output_height lines x output_width columns (4x8)
- pld [r0]
- pld [r0, r1]
- pld [r0, r1, lsl #1]
-
- vmull.u8 q7, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q8, d8, d0
- vmull.u8 q9, d10, d0
- vmull.u8 q10, d12, d0
-
- vext.8 d28, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d29, d8, d9, #1
- vext.8 d30, d10, d11, #1
- vext.8 d31, d12, d13, #1
-
- vmlsl.u8 q7, d28, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d29, d1
- vmlsl.u8 q9, d30, d1
- vmlsl.u8 q10, d31, d1
-
- vext.8 d28, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d29, d8, d9, #4
- vext.8 d30, d10, d11, #4
- vext.8 d31, d12, d13, #4
-
- vmlsl.u8 q7, d28, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d29, d4
- vmlsl.u8 q9, d30, d4
- vmlsl.u8 q10, d31, d4
-
- vext.8 d28, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d29, d8, d9, #2
- vext.8 d30, d10, d11, #2
- vext.8 d31, d12, d13, #2
-
- vmlal.u8 q7, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d29, d2
- vmlal.u8 q9, d30, d2
- vmlal.u8 q10, d31, d2
-
- vext.8 d28, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d29, d8, d9, #5
- vext.8 d30, d10, d11, #5
- vext.8 d31, d12, d13, #5
-
- vmlal.u8 q7, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q8, d29, d5
- vmlal.u8 q9, d30, d5
- vmlal.u8 q10, d31, d5
-
- vext.8 d28, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d29, d8, d9, #3
- vext.8 d30, d10, d11, #3
- vext.8 d31, d12, d13, #3
-
- vmull.u8 q3, d28, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q4, d29, d3
- vmull.u8 q5, d30, d3
- vmull.u8 q6, d31, d3
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d22, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d23, q8, #7
- vqrshrun.s16 d24, q9, #7
- vqrshrun.s16 d25, q10, #7
-
- vst1.u8 {d22}, [r4], r5 ;store result
- vst1.u8 {d23}, [r4], r5
- vst1.u8 {d24}, [r4], r5
- vst1.u8 {d25}, [r4], r5
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
-;---------------------
-secondpass_filter8x4_only
-;Second pass: 8x4
- add r3, r12, r3, lsl #5
- sub r0, r0, r1, lsl #1
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
- vabs.s32 q7, q5
- vabs.s32 q8, q6
-
- vld1.u8 {d22}, [r0], r1
- vld1.u8 {d23}, [r0], r1
- vld1.u8 {d24}, [r0], r1
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vld1.u8 {d25}, [r0], r1
- vdup.8 d1, d14[4]
- vld1.u8 {d26}, [r0], r1
- vdup.8 d2, d15[0]
- vld1.u8 {d27}, [r0], r1
- vdup.8 d3, d15[4]
- vld1.u8 {d28}, [r0], r1
- vdup.8 d4, d16[0]
- vld1.u8 {d29}, [r0], r1
- vdup.8 d5, d16[4]
- vld1.u8 {d30}, [r0], r1
-
- vmull.u8 q3, d22, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d23, d0
- vmull.u8 q5, d24, d0
- vmull.u8 q6, d25, d0
-
- vmlsl.u8 q3, d23, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q4, d24, d1
- vmlsl.u8 q5, d25, d1
- vmlsl.u8 q6, d26, d1
-
- vmlsl.u8 q3, d26, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d27, d4
- vmlsl.u8 q5, d28, d4
- vmlsl.u8 q6, d29, d4
-
- vmlal.u8 q3, d24, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d25, d2
- vmlal.u8 q5, d26, d2
- vmlal.u8 q6, d27, d2
-
- vmlal.u8 q3, d27, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q4, d28, d5
- vmlal.u8 q5, d29, d5
- vmlal.u8 q6, d30, d5
-
- vmull.u8 q7, d25, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q8, d26, d3
- vmull.u8 q9, d27, d3
- vmull.u8 q10, d28, d3
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d6, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q8, #7
- vqrshrun.s16 d8, q9, #7
- vqrshrun.s16 d9, q10, #7
-
- vst1.u8 {d6}, [r4], r5 ;store result
- vst1.u8 {d7}, [r4], r5
- vst1.u8 {d8}, [r4], r5
- vst1.u8 {d9}, [r4], r5
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
- ENDP
-
-;-----------------
-
- END
diff --git a/vp8/common/arm/neon/sixtappredict8x8_neon.asm b/vp8/common/arm/neon/sixtappredict8x8_neon.asm
deleted file mode 100644
index 4b04925..0000000
--- a/vp8/common/arm/neon/sixtappredict8x8_neon.asm
+++ /dev/null
@@ -1,528 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_sixtap_predict8x8_neon|
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-filter8_coeff
- DCD 0, 0, 128, 0, 0, 0, 0, 0
- DCD 0, -6, 123, 12, -1, 0, 0, 0
- DCD 2, -11, 108, 36, -8, 1, 0, 0
- DCD 0, -9, 93, 50, -6, 0, 0, 0
- DCD 3, -16, 77, 77, -16, 3, 0, 0
- DCD 0, -6, 50, 93, -9, 0, 0, 0
- DCD 1, -8, 36, 108, -11, 2, 0, 0
- DCD 0, -1, 12, 123, -6, 0, 0, 0
-
-; r0 unsigned char *src_ptr,
-; r1 int src_pixels_per_line,
-; r2 int xoffset,
-; r3 int yoffset,
-; stack(r4) unsigned char *dst_ptr,
-; stack(r5) int dst_pitch
-
-|vp8_sixtap_predict8x8_neon| PROC
- push {r4-r5, lr}
- vpush {d8-d15}
- adr r12, filter8_coeff
-
- ldr r4, [sp, #76] ;load parameters from stack
- ldr r5, [sp, #80] ;load parameters from stack
-
- cmp r2, #0 ;skip first_pass filter if xoffset=0
- beq secondpass_filter8x8_only
-
- add r2, r12, r2, lsl #5 ;calculate filter location
-
- cmp r3, #0 ;skip second_pass filter if yoffset=0
-
- vld1.s32 {q14, q15}, [r2] ;load first_pass filter
-
- beq firstpass_filter8x8_only
-
- sub sp, sp, #64 ;reserve space on stack for temporary storage
- mov lr, sp
-
- vabs.s32 q12, q14
- vabs.s32 q13, q15
-
- mov r2, #2 ;loop counter
- sub r0, r0, #2 ;move srcptr back to (line-2) and (column-2)
- sub r0, r0, r1, lsl #1
-
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vdup.8 d1, d24[4]
- vdup.8 d2, d25[0]
-
-;First pass: output_height lines x output_width columns (13x8)
- vld1.u8 {q3}, [r0], r1 ;load src data
- vdup.8 d3, d25[4]
- vld1.u8 {q4}, [r0], r1
- vdup.8 d4, d26[0]
- vld1.u8 {q5}, [r0], r1
- vdup.8 d5, d26[4]
- vld1.u8 {q6}, [r0], r1
-
-filt_blk2d_fp8x8_loop_neon
- pld [r0]
- pld [r0, r1]
- pld [r0, r1, lsl #1]
-
- vmull.u8 q7, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q8, d8, d0
- vmull.u8 q9, d10, d0
- vmull.u8 q10, d12, d0
-
- vext.8 d28, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d29, d8, d9, #1
- vext.8 d30, d10, d11, #1
- vext.8 d31, d12, d13, #1
-
- vmlsl.u8 q7, d28, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d29, d1
- vmlsl.u8 q9, d30, d1
- vmlsl.u8 q10, d31, d1
-
- vext.8 d28, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d29, d8, d9, #4
- vext.8 d30, d10, d11, #4
- vext.8 d31, d12, d13, #4
-
- vmlsl.u8 q7, d28, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d29, d4
- vmlsl.u8 q9, d30, d4
- vmlsl.u8 q10, d31, d4
-
- vext.8 d28, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d29, d8, d9, #2
- vext.8 d30, d10, d11, #2
- vext.8 d31, d12, d13, #2
-
- vmlal.u8 q7, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d29, d2
- vmlal.u8 q9, d30, d2
- vmlal.u8 q10, d31, d2
-
- vext.8 d28, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d29, d8, d9, #5
- vext.8 d30, d10, d11, #5
- vext.8 d31, d12, d13, #5
-
- vmlal.u8 q7, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q8, d29, d5
- vmlal.u8 q9, d30, d5
- vmlal.u8 q10, d31, d5
-
- vext.8 d28, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d29, d8, d9, #3
- vext.8 d30, d10, d11, #3
- vext.8 d31, d12, d13, #3
-
- vmull.u8 q3, d28, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q4, d29, d3
- vmull.u8 q5, d30, d3
- vmull.u8 q6, d31, d3
-
- subs r2, r2, #1
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vld1.u8 {q3}, [r0], r1 ;load src data
-
- vqrshrun.s16 d22, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d23, q8, #7
- vqrshrun.s16 d24, q9, #7
- vqrshrun.s16 d25, q10, #7
-
- vst1.u8 {d22}, [lr]! ;store result
- vld1.u8 {q4}, [r0], r1
- vst1.u8 {d23}, [lr]!
- vld1.u8 {q5}, [r0], r1
- vst1.u8 {d24}, [lr]!
- vld1.u8 {q6}, [r0], r1
- vst1.u8 {d25}, [lr]!
-
- bne filt_blk2d_fp8x8_loop_neon
-
- ;first_pass filtering on the rest 5-line data
- ;vld1.u8 {q3}, [r0], r1 ;load src data
- ;vld1.u8 {q4}, [r0], r1
- ;vld1.u8 {q5}, [r0], r1
- ;vld1.u8 {q6}, [r0], r1
- vld1.u8 {q7}, [r0], r1
-
- vmull.u8 q8, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q9, d8, d0
- vmull.u8 q10, d10, d0
- vmull.u8 q11, d12, d0
- vmull.u8 q12, d14, d0
-
- vext.8 d27, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d28, d8, d9, #1
- vext.8 d29, d10, d11, #1
- vext.8 d30, d12, d13, #1
- vext.8 d31, d14, d15, #1
-
- vmlsl.u8 q8, d27, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q9, d28, d1
- vmlsl.u8 q10, d29, d1
- vmlsl.u8 q11, d30, d1
- vmlsl.u8 q12, d31, d1
-
- vext.8 d27, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d28, d8, d9, #4
- vext.8 d29, d10, d11, #4
- vext.8 d30, d12, d13, #4
- vext.8 d31, d14, d15, #4
-
- vmlsl.u8 q8, d27, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q9, d28, d4
- vmlsl.u8 q10, d29, d4
- vmlsl.u8 q11, d30, d4
- vmlsl.u8 q12, d31, d4
-
- vext.8 d27, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d28, d8, d9, #2
- vext.8 d29, d10, d11, #2
- vext.8 d30, d12, d13, #2
- vext.8 d31, d14, d15, #2
-
- vmlal.u8 q8, d27, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q9, d28, d2
- vmlal.u8 q10, d29, d2
- vmlal.u8 q11, d30, d2
- vmlal.u8 q12, d31, d2
-
- vext.8 d27, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d28, d8, d9, #5
- vext.8 d29, d10, d11, #5
- vext.8 d30, d12, d13, #5
- vext.8 d31, d14, d15, #5
-
- vmlal.u8 q8, d27, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q9, d28, d5
- vmlal.u8 q10, d29, d5
- vmlal.u8 q11, d30, d5
- vmlal.u8 q12, d31, d5
-
- vext.8 d27, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d28, d8, d9, #3
- vext.8 d29, d10, d11, #3
- vext.8 d30, d12, d13, #3
- vext.8 d31, d14, d15, #3
-
- vmull.u8 q3, d27, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q4, d28, d3
- vmull.u8 q5, d29, d3
- vmull.u8 q6, d30, d3
- vmull.u8 q7, d31, d3
-
- vqadd.s16 q8, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q9, q4
- vqadd.s16 q10, q5
- vqadd.s16 q11, q6
- vqadd.s16 q12, q7
-
- add r3, r12, r3, lsl #5
-
- vqrshrun.s16 d26, q8, #7 ;shift/round/saturate to u8
- sub lr, lr, #64
- vqrshrun.s16 d27, q9, #7
- vld1.u8 {q9}, [lr]! ;load intermediate data from stack
- vqrshrun.s16 d28, q10, #7
- vld1.u8 {q10}, [lr]!
-
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
-
- vqrshrun.s16 d29, q11, #7
- vld1.u8 {q11}, [lr]!
-
- vabs.s32 q7, q5
- vabs.s32 q8, q6
-
- vqrshrun.s16 d30, q12, #7
- vld1.u8 {q12}, [lr]!
-
-;Second pass: 8x8
- mov r3, #2 ;loop counter
-
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vdup.8 d1, d14[4]
- vdup.8 d2, d15[0]
- vdup.8 d3, d15[4]
- vdup.8 d4, d16[0]
- vdup.8 d5, d16[4]
-
-filt_blk2d_sp8x8_loop_neon
- vmull.u8 q3, d18, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d19, d0
- vmull.u8 q5, d20, d0
- vmull.u8 q6, d21, d0
-
- vmlsl.u8 q3, d19, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q4, d20, d1
- vmlsl.u8 q5, d21, d1
- vmlsl.u8 q6, d22, d1
-
- vmlsl.u8 q3, d22, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d23, d4
- vmlsl.u8 q5, d24, d4
- vmlsl.u8 q6, d25, d4
-
- vmlal.u8 q3, d20, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d21, d2
- vmlal.u8 q5, d22, d2
- vmlal.u8 q6, d23, d2
-
- vmlal.u8 q3, d23, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q4, d24, d5
- vmlal.u8 q5, d25, d5
- vmlal.u8 q6, d26, d5
-
- vmull.u8 q7, d21, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q8, d22, d3
- vmull.u8 q9, d23, d3
- vmull.u8 q10, d24, d3
-
- subs r3, r3, #1
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d6, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q8, #7
- vqrshrun.s16 d8, q9, #7
- vqrshrun.s16 d9, q10, #7
-
- vmov q9, q11
- vst1.u8 {d6}, [r4], r5 ;store result
- vmov q10, q12
- vst1.u8 {d7}, [r4], r5
- vmov q11, q13
- vst1.u8 {d8}, [r4], r5
- vmov q12, q14
- vst1.u8 {d9}, [r4], r5
- vmov d26, d30
-
- bne filt_blk2d_sp8x8_loop_neon
-
- add sp, sp, #64
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
-;---------------------
-firstpass_filter8x8_only
- ;add r2, r12, r2, lsl #5 ;calculate filter location
- ;vld1.s32 {q14, q15}, [r2] ;load first_pass filter
- vabs.s32 q12, q14
- vabs.s32 q13, q15
-
- mov r2, #2 ;loop counter
- sub r0, r0, #2 ;move srcptr back to (line-2) and (column-2)
-
- vdup.8 d0, d24[0] ;first_pass filter (d0-d5)
- vdup.8 d1, d24[4]
- vdup.8 d2, d25[0]
- vdup.8 d3, d25[4]
- vdup.8 d4, d26[0]
- vdup.8 d5, d26[4]
-
-;First pass: output_height lines x output_width columns (8x8)
-filt_blk2d_fpo8x8_loop_neon
- vld1.u8 {q3}, [r0], r1 ;load src data
- vld1.u8 {q4}, [r0], r1
- vld1.u8 {q5}, [r0], r1
- vld1.u8 {q6}, [r0], r1
-
- pld [r0]
- pld [r0, r1]
- pld [r0, r1, lsl #1]
-
- vmull.u8 q7, d6, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q8, d8, d0
- vmull.u8 q9, d10, d0
- vmull.u8 q10, d12, d0
-
- vext.8 d28, d6, d7, #1 ;construct src_ptr[-1]
- vext.8 d29, d8, d9, #1
- vext.8 d30, d10, d11, #1
- vext.8 d31, d12, d13, #1
-
- vmlsl.u8 q7, d28, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q8, d29, d1
- vmlsl.u8 q9, d30, d1
- vmlsl.u8 q10, d31, d1
-
- vext.8 d28, d6, d7, #4 ;construct src_ptr[2]
- vext.8 d29, d8, d9, #4
- vext.8 d30, d10, d11, #4
- vext.8 d31, d12, d13, #4
-
- vmlsl.u8 q7, d28, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q8, d29, d4
- vmlsl.u8 q9, d30, d4
- vmlsl.u8 q10, d31, d4
-
- vext.8 d28, d6, d7, #2 ;construct src_ptr[0]
- vext.8 d29, d8, d9, #2
- vext.8 d30, d10, d11, #2
- vext.8 d31, d12, d13, #2
-
- vmlal.u8 q7, d28, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q8, d29, d2
- vmlal.u8 q9, d30, d2
- vmlal.u8 q10, d31, d2
-
- vext.8 d28, d6, d7, #5 ;construct src_ptr[3]
- vext.8 d29, d8, d9, #5
- vext.8 d30, d10, d11, #5
- vext.8 d31, d12, d13, #5
-
- vmlal.u8 q7, d28, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q8, d29, d5
- vmlal.u8 q9, d30, d5
- vmlal.u8 q10, d31, d5
-
- vext.8 d28, d6, d7, #3 ;construct src_ptr[1]
- vext.8 d29, d8, d9, #3
- vext.8 d30, d10, d11, #3
- vext.8 d31, d12, d13, #3
-
- vmull.u8 q3, d28, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q4, d29, d3
- vmull.u8 q5, d30, d3
- vmull.u8 q6, d31, d3
- ;
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- subs r2, r2, #1
-
- vqrshrun.s16 d22, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d23, q8, #7
- vqrshrun.s16 d24, q9, #7
- vqrshrun.s16 d25, q10, #7
-
- vst1.u8 {d22}, [r4], r5 ;store result
- vst1.u8 {d23}, [r4], r5
- vst1.u8 {d24}, [r4], r5
- vst1.u8 {d25}, [r4], r5
-
- bne filt_blk2d_fpo8x8_loop_neon
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
-;---------------------
-secondpass_filter8x8_only
- sub r0, r0, r1, lsl #1
- add r3, r12, r3, lsl #5
-
- vld1.u8 {d18}, [r0], r1 ;load src data
- vld1.s32 {q5, q6}, [r3] ;load second_pass filter
- vld1.u8 {d19}, [r0], r1
- vabs.s32 q7, q5
- vld1.u8 {d20}, [r0], r1
- vabs.s32 q8, q6
- vld1.u8 {d21}, [r0], r1
- mov r3, #2 ;loop counter
- vld1.u8 {d22}, [r0], r1
- vdup.8 d0, d14[0] ;second_pass filter parameters (d0-d5)
- vld1.u8 {d23}, [r0], r1
- vdup.8 d1, d14[4]
- vld1.u8 {d24}, [r0], r1
- vdup.8 d2, d15[0]
- vld1.u8 {d25}, [r0], r1
- vdup.8 d3, d15[4]
- vld1.u8 {d26}, [r0], r1
- vdup.8 d4, d16[0]
- vld1.u8 {d27}, [r0], r1
- vdup.8 d5, d16[4]
- vld1.u8 {d28}, [r0], r1
- vld1.u8 {d29}, [r0], r1
- vld1.u8 {d30}, [r0], r1
-
-;Second pass: 8x8
-filt_blk2d_spo8x8_loop_neon
- vmull.u8 q3, d18, d0 ;(src_ptr[-2] * vp8_filter[0])
- vmull.u8 q4, d19, d0
- vmull.u8 q5, d20, d0
- vmull.u8 q6, d21, d0
-
- vmlsl.u8 q3, d19, d1 ;-(src_ptr[-1] * vp8_filter[1])
- vmlsl.u8 q4, d20, d1
- vmlsl.u8 q5, d21, d1
- vmlsl.u8 q6, d22, d1
-
- vmlsl.u8 q3, d22, d4 ;-(src_ptr[2] * vp8_filter[4])
- vmlsl.u8 q4, d23, d4
- vmlsl.u8 q5, d24, d4
- vmlsl.u8 q6, d25, d4
-
- vmlal.u8 q3, d20, d2 ;(src_ptr[0] * vp8_filter[2])
- vmlal.u8 q4, d21, d2
- vmlal.u8 q5, d22, d2
- vmlal.u8 q6, d23, d2
-
- vmlal.u8 q3, d23, d5 ;(src_ptr[3] * vp8_filter[5])
- vmlal.u8 q4, d24, d5
- vmlal.u8 q5, d25, d5
- vmlal.u8 q6, d26, d5
-
- vmull.u8 q7, d21, d3 ;(src_ptr[1] * vp8_filter[3])
- vmull.u8 q8, d22, d3
- vmull.u8 q9, d23, d3
- vmull.u8 q10, d24, d3
-
- subs r3, r3, #1
-
- vqadd.s16 q7, q3 ;sum of all (src_data*filter_parameters)
- vqadd.s16 q8, q4
- vqadd.s16 q9, q5
- vqadd.s16 q10, q6
-
- vqrshrun.s16 d6, q7, #7 ;shift/round/saturate to u8
- vqrshrun.s16 d7, q8, #7
- vqrshrun.s16 d8, q9, #7
- vqrshrun.s16 d9, q10, #7
-
- vmov q9, q11
- vst1.u8 {d6}, [r4], r5 ;store result
- vmov q10, q12
- vst1.u8 {d7}, [r4], r5
- vmov q11, q13
- vst1.u8 {d8}, [r4], r5
- vmov q12, q14
- vst1.u8 {d9}, [r4], r5
- vmov d26, d30
-
- bne filt_blk2d_spo8x8_loop_neon
-
- vpop {d8-d15}
- pop {r4-r5,pc}
-
- ENDP
-
-;-----------------
-
- END
diff --git a/vp8/common/arm/neon/sixtappredict_neon.c b/vp8/common/arm/neon/sixtappredict_neon.c
new file mode 100644
index 0000000..22932e9
--- /dev/null
+++ b/vp8/common/arm/neon/sixtappredict_neon.c
@@ -0,0 +1,1748 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+static const int8_t vp8_sub_pel_filters[8][8] = {
+ {0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positionyys are */
+ {0, -6, 123, 12, -1, 0, 0, 0}, /* just as per alpha -0.5 bicubic */
+ {2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
+ {0, -9, 93, 50, -6, 0, 0, 0},
+ {3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
+ {0, -6, 50, 93, -9, 0, 0, 0},
+ {1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
+ {0, -1, 12, 123, -6, 0, 0, 0},
+};
+
+void vp8_sixtap_predict4x4_neon(
+ unsigned char *src_ptr,
+ int src_pixels_per_line,
+ int xoffset,
+ int yoffset,
+ unsigned char *dst_ptr,
+ int dst_pitch) {
+ unsigned char *src;
+ uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d18u8, d19u8, d20u8, d21u8;
+ uint8x8_t d23u8, d24u8, d25u8, d26u8, d27u8, d28u8, d29u8, d30u8, d31u8;
+ int8x8_t dtmps8, d0s8, d1s8, d2s8, d3s8, d4s8, d5s8;
+ uint32x2_t d27u32, d28u32, d29u32, d30u32, d31u32;
+ uint16x8_t q3u16, q4u16, q5u16, q6u16, q7u16;
+ uint16x8_t q8u16, q9u16, q10u16, q11u16, q12u16;
+ int16x8_t q3s16, q4s16, q5s16, q6s16, q7s16;
+ int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16;
+ uint8x16_t q3u8, q4u8, q5u8, q6u8, q11u8;
+ uint64x2_t q3u64, q4u64, q5u64, q6u64, q9u64, q10u64;
+ uint32x2x2_t d0u32x2, d1u32x2;
+
+ if (xoffset == 0) { // secondpass_filter4x4_only
+ // load second_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // load src data
+ src = src_ptr - src_pixels_per_line * 2;
+ d27u32 = vld1_lane_u32((const uint32_t *)src, d27u32, 0);
+ src += src_pixels_per_line;
+ d27u32 = vld1_lane_u32((const uint32_t *)src, d27u32, 1);
+ src += src_pixels_per_line;
+ d28u32 = vld1_lane_u32((const uint32_t *)src, d28u32, 0);
+ src += src_pixels_per_line;
+ d28u32 = vld1_lane_u32((const uint32_t *)src, d28u32, 1);
+ src += src_pixels_per_line;
+ d29u32 = vld1_lane_u32((const uint32_t *)src, d29u32, 0);
+ src += src_pixels_per_line;
+ d29u32 = vld1_lane_u32((const uint32_t *)src, d29u32, 1);
+ src += src_pixels_per_line;
+ d30u32 = vld1_lane_u32((const uint32_t *)src, d30u32, 0);
+ src += src_pixels_per_line;
+ d30u32 = vld1_lane_u32((const uint32_t *)src, d30u32, 1);
+ src += src_pixels_per_line;
+ d31u32 = vld1_lane_u32((const uint32_t *)src, d31u32, 0);
+
+ d27u8 = vreinterpret_u8_u32(d27u32);
+ d28u8 = vreinterpret_u8_u32(d28u32);
+ d29u8 = vreinterpret_u8_u32(d29u32);
+ d30u8 = vreinterpret_u8_u32(d30u32);
+ d31u8 = vreinterpret_u8_u32(d31u32);
+
+ d23u8 = vext_u8(d27u8, d28u8, 4);
+ d24u8 = vext_u8(d28u8, d29u8, 4);
+ d25u8 = vext_u8(d29u8, d30u8, 4);
+ d26u8 = vext_u8(d30u8, d31u8, 4);
+
+ q3u16 = vmull_u8(d27u8, d0u8);
+ q4u16 = vmull_u8(d28u8, d0u8);
+ q5u16 = vmull_u8(d25u8, d5u8);
+ q6u16 = vmull_u8(d26u8, d5u8);
+
+ q3u16 = vmlsl_u8(q3u16, d29u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d30u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d23u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d24u8, d1u8);
+
+ q3u16 = vmlal_u8(q3u16, d28u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d29u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d24u8, d3u8);
+ q6u16 = vmlal_u8(q6u16, d25u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+
+ q5s16 = vqaddq_s16(q5s16, q3s16);
+ q6s16 = vqaddq_s16(q6s16, q4s16);
+
+ d3u8 = vqrshrun_n_s16(q5s16, 7);
+ d4u8 = vqrshrun_n_s16(q6s16, 7);
+
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d3u8), 0);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d3u8), 1);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d4u8), 0);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d4u8), 1);
+ return;
+ }
+
+ // load first_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[xoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // First pass: output_height lines x output_width columns (9x4)
+
+ if (yoffset == 0) // firstpass_filter4x4_only
+ src = src_ptr - 2;
+ else
+ src = src_ptr - 2 - (src_pixels_per_line * 2);
+
+ q3u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q4u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q5u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q6u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+
+ d18u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
+ d19u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
+ d20u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
+ d21u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
+
+ // vswp here
+ q3u8 = vcombine_u8(vget_low_u8(q3u8), vget_low_u8(q4u8));
+ q5u8 = vcombine_u8(vget_low_u8(q5u8), vget_low_u8(q6u8));
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u8(d18u8), // d18 d19
+ vreinterpret_u32_u8(d19u8));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u8(d20u8), // d20 d21
+ vreinterpret_u32_u8(d21u8));
+ q7u16 = vmull_u8(vreinterpret_u8_u32(d0u32x2.val[0]), d5u8);
+ q8u16 = vmull_u8(vreinterpret_u8_u32(d1u32x2.val[0]), d5u8);
+
+ // keep original src data in q4 q6
+ q4u64 = vreinterpretq_u64_u8(q3u8);
+ q6u64 = vreinterpretq_u64_u8(q5u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u8(vget_low_u8(q3u8)), // d6 d7
+ vreinterpret_u32_u8(vget_high_u8(q3u8)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u8(vget_low_u8(q5u8)), // d10 d11
+ vreinterpret_u32_u8(vget_high_u8(q5u8)));
+ q9u64 = vshrq_n_u64(q4u64, 8);
+ q10u64 = vshrq_n_u64(q6u64, 8);
+ q7u16 = vmlal_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d0u8);
+ q8u16 = vmlal_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d0u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q9u64)), // d18 d19
+ vreinterpret_u32_u64(vget_high_u64(q9u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q10u64)), // d20 d211
+ vreinterpret_u32_u64(vget_high_u64(q10u64)));
+ q3u64 = vshrq_n_u64(q4u64, 32);
+ q5u64 = vshrq_n_u64(q6u64, 32);
+ q7u16 = vmlsl_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d1u8);
+ q8u16 = vmlsl_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d1u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q3u64)), // d6 d7
+ vreinterpret_u32_u64(vget_high_u64(q3u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q5u64)), // d10 d11
+ vreinterpret_u32_u64(vget_high_u64(q5u64)));
+ q9u64 = vshrq_n_u64(q4u64, 16);
+ q10u64 = vshrq_n_u64(q6u64, 16);
+ q7u16 = vmlsl_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d4u8);
+ q8u16 = vmlsl_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d4u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q9u64)), // d18 d19
+ vreinterpret_u32_u64(vget_high_u64(q9u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q10u64)), // d20 d211
+ vreinterpret_u32_u64(vget_high_u64(q10u64)));
+ q3u64 = vshrq_n_u64(q4u64, 24);
+ q5u64 = vshrq_n_u64(q6u64, 24);
+ q7u16 = vmlal_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d2u8);
+ q8u16 = vmlal_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d2u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q3u64)), // d6 d7
+ vreinterpret_u32_u64(vget_high_u64(q3u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q5u64)), // d10 d11
+ vreinterpret_u32_u64(vget_high_u64(q5u64)));
+ q9u16 = vmull_u8(vreinterpret_u8_u32(d0u32x2.val[0]), d3u8);
+ q10u16 = vmull_u8(vreinterpret_u8_u32(d1u32x2.val[0]), d3u8);
+
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+ q7s16 = vqaddq_s16(q7s16, q9s16);
+ q8s16 = vqaddq_s16(q8s16, q10s16);
+
+ d27u8 = vqrshrun_n_s16(q7s16, 7);
+ d28u8 = vqrshrun_n_s16(q8s16, 7);
+
+ if (yoffset == 0) { // firstpass_filter4x4_only
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d27u8), 0);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d27u8), 1);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d28u8), 0);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d28u8), 1);
+ return;
+ }
+
+ // First Pass on rest 5-line data
+ q3u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q4u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q5u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q6u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q11u8 = vld1q_u8(src);
+
+ d18u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
+ d19u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
+ d20u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
+ d21u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
+
+ // vswp here
+ q3u8 = vcombine_u8(vget_low_u8(q3u8), vget_low_u8(q4u8));
+ q5u8 = vcombine_u8(vget_low_u8(q5u8), vget_low_u8(q6u8));
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u8(d18u8), // d18 d19
+ vreinterpret_u32_u8(d19u8));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u8(d20u8), // d20 d21
+ vreinterpret_u32_u8(d21u8));
+ d31u8 = vext_u8(vget_low_u8(q11u8), vget_high_u8(q11u8), 5);
+ q7u16 = vmull_u8(vreinterpret_u8_u32(d0u32x2.val[0]), d5u8);
+ q8u16 = vmull_u8(vreinterpret_u8_u32(d1u32x2.val[0]), d5u8);
+ q12u16 = vmull_u8(d31u8, d5u8);
+
+ q4u64 = vreinterpretq_u64_u8(q3u8);
+ q6u64 = vreinterpretq_u64_u8(q5u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u8(vget_low_u8(q3u8)), // d6 d7
+ vreinterpret_u32_u8(vget_high_u8(q3u8)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u8(vget_low_u8(q5u8)), // d10 d11
+ vreinterpret_u32_u8(vget_high_u8(q5u8)));
+ q9u64 = vshrq_n_u64(q4u64, 8);
+ q10u64 = vshrq_n_u64(q6u64, 8);
+ q7u16 = vmlal_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d0u8);
+ q8u16 = vmlal_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d0u8);
+ q12u16 = vmlal_u8(q12u16, vget_low_u8(q11u8), d0u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q9u64)), // d18 d19
+ vreinterpret_u32_u64(vget_high_u64(q9u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q10u64)), // d20 d211
+ vreinterpret_u32_u64(vget_high_u64(q10u64)));
+ q3u64 = vshrq_n_u64(q4u64, 32);
+ q5u64 = vshrq_n_u64(q6u64, 32);
+ d31u8 = vext_u8(vget_low_u8(q11u8), vget_high_u8(q11u8), 1);
+ q7u16 = vmlsl_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d1u8);
+ q8u16 = vmlsl_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d1u8);
+ q12u16 = vmlsl_u8(q12u16, d31u8, d1u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q3u64)), // d6 d7
+ vreinterpret_u32_u64(vget_high_u64(q3u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q5u64)), // d10 d11
+ vreinterpret_u32_u64(vget_high_u64(q5u64)));
+ q9u64 = vshrq_n_u64(q4u64, 16);
+ q10u64 = vshrq_n_u64(q6u64, 16);
+ d31u8 = vext_u8(vget_low_u8(q11u8), vget_high_u8(q11u8), 4);
+ q7u16 = vmlsl_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d4u8);
+ q8u16 = vmlsl_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d4u8);
+ q12u16 = vmlsl_u8(q12u16, d31u8, d4u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q9u64)), // d18 d19
+ vreinterpret_u32_u64(vget_high_u64(q9u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q10u64)), // d20 d211
+ vreinterpret_u32_u64(vget_high_u64(q10u64)));
+ q3u64 = vshrq_n_u64(q4u64, 24);
+ q5u64 = vshrq_n_u64(q6u64, 24);
+ d31u8 = vext_u8(vget_low_u8(q11u8), vget_high_u8(q11u8), 2);
+ q7u16 = vmlal_u8(q7u16, vreinterpret_u8_u32(d0u32x2.val[0]), d2u8);
+ q8u16 = vmlal_u8(q8u16, vreinterpret_u8_u32(d1u32x2.val[0]), d2u8);
+ q12u16 = vmlal_u8(q12u16, d31u8, d2u8);
+
+ d0u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q3u64)), // d6 d7
+ vreinterpret_u32_u64(vget_high_u64(q3u64)));
+ d1u32x2 = vzip_u32(vreinterpret_u32_u64(vget_low_u64(q5u64)), // d10 d11
+ vreinterpret_u32_u64(vget_high_u64(q5u64)));
+ d31u8 = vext_u8(vget_low_u8(q11u8), vget_high_u8(q11u8), 3);
+ q9u16 = vmull_u8(vreinterpret_u8_u32(d0u32x2.val[0]), d3u8);
+ q10u16 = vmull_u8(vreinterpret_u8_u32(d1u32x2.val[0]), d3u8);
+ q11u16 = vmull_u8(d31u8, d3u8);
+
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+ q11s16 = vreinterpretq_s16_u16(q11u16);
+ q12s16 = vreinterpretq_s16_u16(q12u16);
+ q7s16 = vqaddq_s16(q7s16, q9s16);
+ q8s16 = vqaddq_s16(q8s16, q10s16);
+ q12s16 = vqaddq_s16(q12s16, q11s16);
+
+ d29u8 = vqrshrun_n_s16(q7s16, 7);
+ d30u8 = vqrshrun_n_s16(q8s16, 7);
+ d31u8 = vqrshrun_n_s16(q12s16, 7);
+
+ // Second pass: 4x4
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ d23u8 = vext_u8(d27u8, d28u8, 4);
+ d24u8 = vext_u8(d28u8, d29u8, 4);
+ d25u8 = vext_u8(d29u8, d30u8, 4);
+ d26u8 = vext_u8(d30u8, d31u8, 4);
+
+ q3u16 = vmull_u8(d27u8, d0u8);
+ q4u16 = vmull_u8(d28u8, d0u8);
+ q5u16 = vmull_u8(d25u8, d5u8);
+ q6u16 = vmull_u8(d26u8, d5u8);
+
+ q3u16 = vmlsl_u8(q3u16, d29u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d30u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d23u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d24u8, d1u8);
+
+ q3u16 = vmlal_u8(q3u16, d28u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d29u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d24u8, d3u8);
+ q6u16 = vmlal_u8(q6u16, d25u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+
+ q5s16 = vqaddq_s16(q5s16, q3s16);
+ q6s16 = vqaddq_s16(q6s16, q4s16);
+
+ d3u8 = vqrshrun_n_s16(q5s16, 7);
+ d4u8 = vqrshrun_n_s16(q6s16, 7);
+
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d3u8), 0);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d3u8), 1);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d4u8), 0);
+ dst_ptr += dst_pitch;
+ vst1_lane_u32((uint32_t *)dst_ptr, vreinterpret_u32_u8(d4u8), 1);
+ return;
+}
+
+void vp8_sixtap_predict8x4_neon(
+ unsigned char *src_ptr,
+ int src_pixels_per_line,
+ int xoffset,
+ int yoffset,
+ unsigned char *dst_ptr,
+ int dst_pitch) {
+ unsigned char *src;
+ uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
+ uint8x8_t d22u8, d23u8, d24u8, d25u8, d26u8;
+ uint8x8_t d27u8, d28u8, d29u8, d30u8, d31u8;
+ int8x8_t dtmps8, d0s8, d1s8, d2s8, d3s8, d4s8, d5s8;
+ uint16x8_t q3u16, q4u16, q5u16, q6u16, q7u16;
+ uint16x8_t q8u16, q9u16, q10u16, q11u16, q12u16;
+ int16x8_t q3s16, q4s16, q5s16, q6s16, q7s16;
+ int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16;
+ uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8;
+
+ if (xoffset == 0) { // secondpass_filter8x4_only
+ // load second_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // load src data
+ src = src_ptr - src_pixels_per_line * 2;
+ d22u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d23u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d24u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d25u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d26u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d27u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d28u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d29u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d30u8 = vld1_u8(src);
+
+ q3u16 = vmull_u8(d22u8, d0u8);
+ q4u16 = vmull_u8(d23u8, d0u8);
+ q5u16 = vmull_u8(d24u8, d0u8);
+ q6u16 = vmull_u8(d25u8, d0u8);
+
+ q3u16 = vmlsl_u8(q3u16, d23u8, d1u8);
+ q4u16 = vmlsl_u8(q4u16, d24u8, d1u8);
+ q5u16 = vmlsl_u8(q5u16, d25u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d26u8, d1u8);
+
+ q3u16 = vmlsl_u8(q3u16, d26u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d27u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d28u8, d4u8);
+ q6u16 = vmlsl_u8(q6u16, d29u8, d4u8);
+
+ q3u16 = vmlal_u8(q3u16, d24u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d25u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d26u8, d2u8);
+ q6u16 = vmlal_u8(q6u16, d27u8, d2u8);
+
+ q3u16 = vmlal_u8(q3u16, d27u8, d5u8);
+ q4u16 = vmlal_u8(q4u16, d28u8, d5u8);
+ q5u16 = vmlal_u8(q5u16, d29u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d30u8, d5u8);
+
+ q7u16 = vmull_u8(d25u8, d3u8);
+ q8u16 = vmull_u8(d26u8, d3u8);
+ q9u16 = vmull_u8(d27u8, d3u8);
+ q10u16 = vmull_u8(d28u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d6u8 = vqrshrun_n_s16(q7s16, 7);
+ d7u8 = vqrshrun_n_s16(q8s16, 7);
+ d8u8 = vqrshrun_n_s16(q9s16, 7);
+ d9u8 = vqrshrun_n_s16(q10s16, 7);
+
+ vst1_u8(dst_ptr, d6u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d7u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d8u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d9u8);
+ return;
+ }
+
+ // load first_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[xoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // First pass: output_height lines x output_width columns (9x4)
+ if (yoffset == 0) // firstpass_filter4x4_only
+ src = src_ptr - 2;
+ else
+ src = src_ptr - 2 - (src_pixels_per_line * 2);
+ q3u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q4u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q5u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q6u8 = vld1q_u8(src);
+
+ q7u16 = vmull_u8(vget_low_u8(q3u8), d0u8);
+ q8u16 = vmull_u8(vget_low_u8(q4u8), d0u8);
+ q9u16 = vmull_u8(vget_low_u8(q5u8), d0u8);
+ q10u16 = vmull_u8(vget_low_u8(q6u8), d0u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 1);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 1);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 1);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 1);
+
+ q7u16 = vmlsl_u8(q7u16, d28u8, d1u8);
+ q8u16 = vmlsl_u8(q8u16, d29u8, d1u8);
+ q9u16 = vmlsl_u8(q9u16, d30u8, d1u8);
+ q10u16 = vmlsl_u8(q10u16, d31u8, d1u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 4);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 4);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 4);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 4);
+
+ q7u16 = vmlsl_u8(q7u16, d28u8, d4u8);
+ q8u16 = vmlsl_u8(q8u16, d29u8, d4u8);
+ q9u16 = vmlsl_u8(q9u16, d30u8, d4u8);
+ q10u16 = vmlsl_u8(q10u16, d31u8, d4u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 2);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 2);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 2);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 2);
+
+ q7u16 = vmlal_u8(q7u16, d28u8, d2u8);
+ q8u16 = vmlal_u8(q8u16, d29u8, d2u8);
+ q9u16 = vmlal_u8(q9u16, d30u8, d2u8);
+ q10u16 = vmlal_u8(q10u16, d31u8, d2u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
+
+ q7u16 = vmlal_u8(q7u16, d28u8, d5u8);
+ q8u16 = vmlal_u8(q8u16, d29u8, d5u8);
+ q9u16 = vmlal_u8(q9u16, d30u8, d5u8);
+ q10u16 = vmlal_u8(q10u16, d31u8, d5u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 3);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 3);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 3);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 3);
+
+ q3u16 = vmull_u8(d28u8, d3u8);
+ q4u16 = vmull_u8(d29u8, d3u8);
+ q5u16 = vmull_u8(d30u8, d3u8);
+ q6u16 = vmull_u8(d31u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d22u8 = vqrshrun_n_s16(q7s16, 7);
+ d23u8 = vqrshrun_n_s16(q8s16, 7);
+ d24u8 = vqrshrun_n_s16(q9s16, 7);
+ d25u8 = vqrshrun_n_s16(q10s16, 7);
+
+ if (yoffset == 0) { // firstpass_filter8x4_only
+ vst1_u8(dst_ptr, d22u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d23u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d24u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d25u8);
+ return;
+ }
+
+ // First Pass on rest 5-line data
+ src += src_pixels_per_line;
+ q3u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q4u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q5u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q6u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q7u8 = vld1q_u8(src);
+
+ q8u16 = vmull_u8(vget_low_u8(q3u8), d0u8);
+ q9u16 = vmull_u8(vget_low_u8(q4u8), d0u8);
+ q10u16 = vmull_u8(vget_low_u8(q5u8), d0u8);
+ q11u16 = vmull_u8(vget_low_u8(q6u8), d0u8);
+ q12u16 = vmull_u8(vget_low_u8(q7u8), d0u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 1);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 1);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 1);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 1);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 1);
+
+ q8u16 = vmlsl_u8(q8u16, d27u8, d1u8);
+ q9u16 = vmlsl_u8(q9u16, d28u8, d1u8);
+ q10u16 = vmlsl_u8(q10u16, d29u8, d1u8);
+ q11u16 = vmlsl_u8(q11u16, d30u8, d1u8);
+ q12u16 = vmlsl_u8(q12u16, d31u8, d1u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 4);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 4);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 4);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 4);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 4);
+
+ q8u16 = vmlsl_u8(q8u16, d27u8, d4u8);
+ q9u16 = vmlsl_u8(q9u16, d28u8, d4u8);
+ q10u16 = vmlsl_u8(q10u16, d29u8, d4u8);
+ q11u16 = vmlsl_u8(q11u16, d30u8, d4u8);
+ q12u16 = vmlsl_u8(q12u16, d31u8, d4u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 2);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 2);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 2);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 2);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 2);
+
+ q8u16 = vmlal_u8(q8u16, d27u8, d2u8);
+ q9u16 = vmlal_u8(q9u16, d28u8, d2u8);
+ q10u16 = vmlal_u8(q10u16, d29u8, d2u8);
+ q11u16 = vmlal_u8(q11u16, d30u8, d2u8);
+ q12u16 = vmlal_u8(q12u16, d31u8, d2u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 5);
+
+ q8u16 = vmlal_u8(q8u16, d27u8, d5u8);
+ q9u16 = vmlal_u8(q9u16, d28u8, d5u8);
+ q10u16 = vmlal_u8(q10u16, d29u8, d5u8);
+ q11u16 = vmlal_u8(q11u16, d30u8, d5u8);
+ q12u16 = vmlal_u8(q12u16, d31u8, d5u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 3);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 3);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 3);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 3);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 3);
+
+ q3u16 = vmull_u8(d27u8, d3u8);
+ q4u16 = vmull_u8(d28u8, d3u8);
+ q5u16 = vmull_u8(d29u8, d3u8);
+ q6u16 = vmull_u8(d30u8, d3u8);
+ q7u16 = vmull_u8(d31u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+ q11s16 = vreinterpretq_s16_u16(q11u16);
+ q12s16 = vreinterpretq_s16_u16(q12u16);
+
+ q8s16 = vqaddq_s16(q8s16, q3s16);
+ q9s16 = vqaddq_s16(q9s16, q4s16);
+ q10s16 = vqaddq_s16(q10s16, q5s16);
+ q11s16 = vqaddq_s16(q11s16, q6s16);
+ q12s16 = vqaddq_s16(q12s16, q7s16);
+
+ d26u8 = vqrshrun_n_s16(q8s16, 7);
+ d27u8 = vqrshrun_n_s16(q9s16, 7);
+ d28u8 = vqrshrun_n_s16(q10s16, 7);
+ d29u8 = vqrshrun_n_s16(q11s16, 7);
+ d30u8 = vqrshrun_n_s16(q12s16, 7);
+
+ // Second pass: 8x4
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ q3u16 = vmull_u8(d22u8, d0u8);
+ q4u16 = vmull_u8(d23u8, d0u8);
+ q5u16 = vmull_u8(d24u8, d0u8);
+ q6u16 = vmull_u8(d25u8, d0u8);
+
+ q3u16 = vmlsl_u8(q3u16, d23u8, d1u8);
+ q4u16 = vmlsl_u8(q4u16, d24u8, d1u8);
+ q5u16 = vmlsl_u8(q5u16, d25u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d26u8, d1u8);
+
+ q3u16 = vmlsl_u8(q3u16, d26u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d27u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d28u8, d4u8);
+ q6u16 = vmlsl_u8(q6u16, d29u8, d4u8);
+
+ q3u16 = vmlal_u8(q3u16, d24u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d25u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d26u8, d2u8);
+ q6u16 = vmlal_u8(q6u16, d27u8, d2u8);
+
+ q3u16 = vmlal_u8(q3u16, d27u8, d5u8);
+ q4u16 = vmlal_u8(q4u16, d28u8, d5u8);
+ q5u16 = vmlal_u8(q5u16, d29u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d30u8, d5u8);
+
+ q7u16 = vmull_u8(d25u8, d3u8);
+ q8u16 = vmull_u8(d26u8, d3u8);
+ q9u16 = vmull_u8(d27u8, d3u8);
+ q10u16 = vmull_u8(d28u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d6u8 = vqrshrun_n_s16(q7s16, 7);
+ d7u8 = vqrshrun_n_s16(q8s16, 7);
+ d8u8 = vqrshrun_n_s16(q9s16, 7);
+ d9u8 = vqrshrun_n_s16(q10s16, 7);
+
+ vst1_u8(dst_ptr, d6u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d7u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d8u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d9u8);
+ return;
+}
+
+void vp8_sixtap_predict8x8_neon(
+ unsigned char *src_ptr,
+ int src_pixels_per_line,
+ int xoffset,
+ int yoffset,
+ unsigned char *dst_ptr,
+ int dst_pitch) {
+ unsigned char *src, *tmpp;
+ unsigned char tmp[64];
+ int i;
+ uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
+ uint8x8_t d18u8, d19u8, d20u8, d21u8, d22u8, d23u8, d24u8, d25u8;
+ uint8x8_t d26u8, d27u8, d28u8, d29u8, d30u8, d31u8;
+ int8x8_t dtmps8, d0s8, d1s8, d2s8, d3s8, d4s8, d5s8;
+ uint16x8_t q3u16, q4u16, q5u16, q6u16, q7u16;
+ uint16x8_t q8u16, q9u16, q10u16, q11u16, q12u16;
+ int16x8_t q3s16, q4s16, q5s16, q6s16, q7s16;
+ int16x8_t q8s16, q9s16, q10s16, q11s16, q12s16;
+ uint8x16_t q3u8, q4u8, q5u8, q6u8, q7u8, q9u8, q10u8, q11u8, q12u8;
+
+ if (xoffset == 0) { // secondpass_filter8x8_only
+ // load second_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // load src data
+ src = src_ptr - src_pixels_per_line * 2;
+ d18u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d19u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d20u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d21u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d22u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d23u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d24u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d25u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d26u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d27u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d28u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d29u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d30u8 = vld1_u8(src);
+
+ for (i = 2; i > 0; i--) {
+ q3u16 = vmull_u8(d18u8, d0u8);
+ q4u16 = vmull_u8(d19u8, d0u8);
+ q5u16 = vmull_u8(d20u8, d0u8);
+ q6u16 = vmull_u8(d21u8, d0u8);
+
+ q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
+ q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
+ q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
+
+ q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
+ q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
+
+ q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
+ q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
+
+ q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
+ q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
+ q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
+
+ q7u16 = vmull_u8(d21u8, d3u8);
+ q8u16 = vmull_u8(d22u8, d3u8);
+ q9u16 = vmull_u8(d23u8, d3u8);
+ q10u16 = vmull_u8(d24u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d6u8 = vqrshrun_n_s16(q7s16, 7);
+ d7u8 = vqrshrun_n_s16(q8s16, 7);
+ d8u8 = vqrshrun_n_s16(q9s16, 7);
+ d9u8 = vqrshrun_n_s16(q10s16, 7);
+
+ d18u8 = d22u8;
+ d19u8 = d23u8;
+ d20u8 = d24u8;
+ d21u8 = d25u8;
+ d22u8 = d26u8;
+ d23u8 = d27u8;
+ d24u8 = d28u8;
+ d25u8 = d29u8;
+ d26u8 = d30u8;
+
+ vst1_u8(dst_ptr, d6u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d7u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d8u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d9u8);
+ dst_ptr += dst_pitch;
+ }
+ return;
+ }
+
+ // load first_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[xoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // First pass: output_height lines x output_width columns (9x4)
+ if (yoffset == 0) // firstpass_filter4x4_only
+ src = src_ptr - 2;
+ else
+ src = src_ptr - 2 - (src_pixels_per_line * 2);
+
+ tmpp = tmp;
+ for (i = 2; i > 0; i--) {
+ q3u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q4u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q5u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q6u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+
+ __builtin_prefetch(src);
+ __builtin_prefetch(src + src_pixels_per_line);
+ __builtin_prefetch(src + src_pixels_per_line * 2);
+
+ q7u16 = vmull_u8(vget_low_u8(q3u8), d0u8);
+ q8u16 = vmull_u8(vget_low_u8(q4u8), d0u8);
+ q9u16 = vmull_u8(vget_low_u8(q5u8), d0u8);
+ q10u16 = vmull_u8(vget_low_u8(q6u8), d0u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 1);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 1);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 1);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 1);
+
+ q7u16 = vmlsl_u8(q7u16, d28u8, d1u8);
+ q8u16 = vmlsl_u8(q8u16, d29u8, d1u8);
+ q9u16 = vmlsl_u8(q9u16, d30u8, d1u8);
+ q10u16 = vmlsl_u8(q10u16, d31u8, d1u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 4);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 4);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 4);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 4);
+
+ q7u16 = vmlsl_u8(q7u16, d28u8, d4u8);
+ q8u16 = vmlsl_u8(q8u16, d29u8, d4u8);
+ q9u16 = vmlsl_u8(q9u16, d30u8, d4u8);
+ q10u16 = vmlsl_u8(q10u16, d31u8, d4u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 2);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 2);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 2);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 2);
+
+ q7u16 = vmlal_u8(q7u16, d28u8, d2u8);
+ q8u16 = vmlal_u8(q8u16, d29u8, d2u8);
+ q9u16 = vmlal_u8(q9u16, d30u8, d2u8);
+ q10u16 = vmlal_u8(q10u16, d31u8, d2u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
+
+ q7u16 = vmlal_u8(q7u16, d28u8, d5u8);
+ q8u16 = vmlal_u8(q8u16, d29u8, d5u8);
+ q9u16 = vmlal_u8(q9u16, d30u8, d5u8);
+ q10u16 = vmlal_u8(q10u16, d31u8, d5u8);
+
+ d28u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 3);
+ d29u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 3);
+ d30u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 3);
+ d31u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 3);
+
+ q3u16 = vmull_u8(d28u8, d3u8);
+ q4u16 = vmull_u8(d29u8, d3u8);
+ q5u16 = vmull_u8(d30u8, d3u8);
+ q6u16 = vmull_u8(d31u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d22u8 = vqrshrun_n_s16(q7s16, 7);
+ d23u8 = vqrshrun_n_s16(q8s16, 7);
+ d24u8 = vqrshrun_n_s16(q9s16, 7);
+ d25u8 = vqrshrun_n_s16(q10s16, 7);
+
+ if (yoffset == 0) { // firstpass_filter8x4_only
+ vst1_u8(dst_ptr, d22u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d23u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d24u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d25u8);
+ dst_ptr += dst_pitch;
+ } else {
+ vst1_u8(tmpp, d22u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d23u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d24u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d25u8);
+ tmpp += 8;
+ }
+ }
+ if (yoffset == 0)
+ return;
+
+ // First Pass on rest 5-line data
+ q3u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q4u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q5u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q6u8 = vld1q_u8(src);
+ src += src_pixels_per_line;
+ q7u8 = vld1q_u8(src);
+
+ q8u16 = vmull_u8(vget_low_u8(q3u8), d0u8);
+ q9u16 = vmull_u8(vget_low_u8(q4u8), d0u8);
+ q10u16 = vmull_u8(vget_low_u8(q5u8), d0u8);
+ q11u16 = vmull_u8(vget_low_u8(q6u8), d0u8);
+ q12u16 = vmull_u8(vget_low_u8(q7u8), d0u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 1);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 1);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 1);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 1);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 1);
+
+ q8u16 = vmlsl_u8(q8u16, d27u8, d1u8);
+ q9u16 = vmlsl_u8(q9u16, d28u8, d1u8);
+ q10u16 = vmlsl_u8(q10u16, d29u8, d1u8);
+ q11u16 = vmlsl_u8(q11u16, d30u8, d1u8);
+ q12u16 = vmlsl_u8(q12u16, d31u8, d1u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 4);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 4);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 4);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 4);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 4);
+
+ q8u16 = vmlsl_u8(q8u16, d27u8, d4u8);
+ q9u16 = vmlsl_u8(q9u16, d28u8, d4u8);
+ q10u16 = vmlsl_u8(q10u16, d29u8, d4u8);
+ q11u16 = vmlsl_u8(q11u16, d30u8, d4u8);
+ q12u16 = vmlsl_u8(q12u16, d31u8, d4u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 2);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 2);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 2);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 2);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 2);
+
+ q8u16 = vmlal_u8(q8u16, d27u8, d2u8);
+ q9u16 = vmlal_u8(q9u16, d28u8, d2u8);
+ q10u16 = vmlal_u8(q10u16, d29u8, d2u8);
+ q11u16 = vmlal_u8(q11u16, d30u8, d2u8);
+ q12u16 = vmlal_u8(q12u16, d31u8, d2u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 5);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 5);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 5);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 5);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 5);
+
+ q8u16 = vmlal_u8(q8u16, d27u8, d5u8);
+ q9u16 = vmlal_u8(q9u16, d28u8, d5u8);
+ q10u16 = vmlal_u8(q10u16, d29u8, d5u8);
+ q11u16 = vmlal_u8(q11u16, d30u8, d5u8);
+ q12u16 = vmlal_u8(q12u16, d31u8, d5u8);
+
+ d27u8 = vext_u8(vget_low_u8(q3u8), vget_high_u8(q3u8), 3);
+ d28u8 = vext_u8(vget_low_u8(q4u8), vget_high_u8(q4u8), 3);
+ d29u8 = vext_u8(vget_low_u8(q5u8), vget_high_u8(q5u8), 3);
+ d30u8 = vext_u8(vget_low_u8(q6u8), vget_high_u8(q6u8), 3);
+ d31u8 = vext_u8(vget_low_u8(q7u8), vget_high_u8(q7u8), 3);
+
+ q3u16 = vmull_u8(d27u8, d3u8);
+ q4u16 = vmull_u8(d28u8, d3u8);
+ q5u16 = vmull_u8(d29u8, d3u8);
+ q6u16 = vmull_u8(d30u8, d3u8);
+ q7u16 = vmull_u8(d31u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+ q11s16 = vreinterpretq_s16_u16(q11u16);
+ q12s16 = vreinterpretq_s16_u16(q12u16);
+
+ q8s16 = vqaddq_s16(q8s16, q3s16);
+ q9s16 = vqaddq_s16(q9s16, q4s16);
+ q10s16 = vqaddq_s16(q10s16, q5s16);
+ q11s16 = vqaddq_s16(q11s16, q6s16);
+ q12s16 = vqaddq_s16(q12s16, q7s16);
+
+ d26u8 = vqrshrun_n_s16(q8s16, 7);
+ d27u8 = vqrshrun_n_s16(q9s16, 7);
+ d28u8 = vqrshrun_n_s16(q10s16, 7);
+ d29u8 = vqrshrun_n_s16(q11s16, 7);
+ d30u8 = vqrshrun_n_s16(q12s16, 7);
+
+ // Second pass: 8x8
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ tmpp = tmp;
+ q9u8 = vld1q_u8(tmpp);
+ tmpp += 16;
+ q10u8 = vld1q_u8(tmpp);
+ tmpp += 16;
+ q11u8 = vld1q_u8(tmpp);
+ tmpp += 16;
+ q12u8 = vld1q_u8(tmpp);
+
+ d18u8 = vget_low_u8(q9u8);
+ d19u8 = vget_high_u8(q9u8);
+ d20u8 = vget_low_u8(q10u8);
+ d21u8 = vget_high_u8(q10u8);
+ d22u8 = vget_low_u8(q11u8);
+ d23u8 = vget_high_u8(q11u8);
+ d24u8 = vget_low_u8(q12u8);
+ d25u8 = vget_high_u8(q12u8);
+
+ for (i = 2; i > 0; i--) {
+ q3u16 = vmull_u8(d18u8, d0u8);
+ q4u16 = vmull_u8(d19u8, d0u8);
+ q5u16 = vmull_u8(d20u8, d0u8);
+ q6u16 = vmull_u8(d21u8, d0u8);
+
+ q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
+ q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
+ q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
+
+ q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
+ q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
+
+ q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
+ q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
+
+ q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
+ q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
+ q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
+
+ q7u16 = vmull_u8(d21u8, d3u8);
+ q8u16 = vmull_u8(d22u8, d3u8);
+ q9u16 = vmull_u8(d23u8, d3u8);
+ q10u16 = vmull_u8(d24u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d6u8 = vqrshrun_n_s16(q7s16, 7);
+ d7u8 = vqrshrun_n_s16(q8s16, 7);
+ d8u8 = vqrshrun_n_s16(q9s16, 7);
+ d9u8 = vqrshrun_n_s16(q10s16, 7);
+
+ d18u8 = d22u8;
+ d19u8 = d23u8;
+ d20u8 = d24u8;
+ d21u8 = d25u8;
+ d22u8 = d26u8;
+ d23u8 = d27u8;
+ d24u8 = d28u8;
+ d25u8 = d29u8;
+ d26u8 = d30u8;
+
+ vst1_u8(dst_ptr, d6u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d7u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d8u8);
+ dst_ptr += dst_pitch;
+ vst1_u8(dst_ptr, d9u8);
+ dst_ptr += dst_pitch;
+ }
+ return;
+}
+
+void vp8_sixtap_predict16x16_neon(
+ unsigned char *src_ptr,
+ int src_pixels_per_line,
+ int xoffset,
+ int yoffset,
+ unsigned char *dst_ptr,
+ int dst_pitch) {
+ unsigned char *src, *src_tmp, *dst, *tmpp;
+ unsigned char tmp[336];
+ int i, j;
+ uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8, d8u8, d9u8;
+ uint8x8_t d10u8, d11u8, d12u8, d13u8, d14u8, d15u8, d18u8, d19u8;
+ uint8x8_t d20u8, d21u8, d22u8, d23u8, d24u8, d25u8, d26u8, d27u8;
+ uint8x8_t d28u8, d29u8, d30u8, d31u8;
+ int8x8_t dtmps8, d0s8, d1s8, d2s8, d3s8, d4s8, d5s8;
+ uint8x16_t q3u8, q4u8;
+ uint16x8_t q3u16, q4u16, q5u16, q6u16, q7u16, q8u16, q9u16, q10u16;
+ uint16x8_t q11u16, q12u16, q13u16, q15u16;
+ int16x8_t q3s16, q4s16, q5s16, q6s16, q7s16, q8s16, q9s16, q10s16;
+ int16x8_t q11s16, q12s16, q13s16, q15s16;
+
+ if (xoffset == 0) { // secondpass_filter8x8_only
+ // load second_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // load src data
+ src_tmp = src_ptr - src_pixels_per_line * 2;
+ for (i = 0; i < 2; i++) {
+ src = src_tmp + i * 8;
+ dst = dst_ptr + i * 8;
+ d18u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d19u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d20u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d21u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d22u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ for (j = 0; j < 4; j++) {
+ d23u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d24u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d25u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+ d26u8 = vld1_u8(src);
+ src += src_pixels_per_line;
+
+ q3u16 = vmull_u8(d18u8, d0u8);
+ q4u16 = vmull_u8(d19u8, d0u8);
+ q5u16 = vmull_u8(d20u8, d0u8);
+ q6u16 = vmull_u8(d21u8, d0u8);
+
+ q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
+ q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
+ q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
+
+ q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
+ q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
+
+ q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
+ q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
+
+ q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
+ q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
+ q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
+
+ q7u16 = vmull_u8(d21u8, d3u8);
+ q8u16 = vmull_u8(d22u8, d3u8);
+ q9u16 = vmull_u8(d23u8, d3u8);
+ q10u16 = vmull_u8(d24u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d6u8 = vqrshrun_n_s16(q7s16, 7);
+ d7u8 = vqrshrun_n_s16(q8s16, 7);
+ d8u8 = vqrshrun_n_s16(q9s16, 7);
+ d9u8 = vqrshrun_n_s16(q10s16, 7);
+
+ d18u8 = d22u8;
+ d19u8 = d23u8;
+ d20u8 = d24u8;
+ d21u8 = d25u8;
+ d22u8 = d26u8;
+
+ vst1_u8(dst, d6u8);
+ dst += dst_pitch;
+ vst1_u8(dst, d7u8);
+ dst += dst_pitch;
+ vst1_u8(dst, d8u8);
+ dst += dst_pitch;
+ vst1_u8(dst, d9u8);
+ dst += dst_pitch;
+ }
+ }
+ return;
+ }
+
+ // load first_pass filter
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[xoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ // First pass: output_height lines x output_width columns (9x4)
+ if (yoffset == 0) { // firstpass_filter4x4_only
+ src = src_ptr - 2;
+ dst = dst_ptr;
+ for (i = 0; i < 8; i++) {
+ d6u8 = vld1_u8(src);
+ d7u8 = vld1_u8(src + 8);
+ d8u8 = vld1_u8(src + 16);
+ src += src_pixels_per_line;
+ d9u8 = vld1_u8(src);
+ d10u8 = vld1_u8(src + 8);
+ d11u8 = vld1_u8(src + 16);
+ src += src_pixels_per_line;
+
+ __builtin_prefetch(src);
+ __builtin_prefetch(src + src_pixels_per_line);
+
+ q6u16 = vmull_u8(d6u8, d0u8);
+ q7u16 = vmull_u8(d7u8, d0u8);
+ q8u16 = vmull_u8(d9u8, d0u8);
+ q9u16 = vmull_u8(d10u8, d0u8);
+
+ d20u8 = vext_u8(d6u8, d7u8, 1);
+ d21u8 = vext_u8(d9u8, d10u8, 1);
+ d22u8 = vext_u8(d7u8, d8u8, 1);
+ d23u8 = vext_u8(d10u8, d11u8, 1);
+ d24u8 = vext_u8(d6u8, d7u8, 4);
+ d25u8 = vext_u8(d9u8, d10u8, 4);
+ d26u8 = vext_u8(d7u8, d8u8, 4);
+ d27u8 = vext_u8(d10u8, d11u8, 4);
+ d28u8 = vext_u8(d6u8, d7u8, 5);
+ d29u8 = vext_u8(d9u8, d10u8, 5);
+
+ q6u16 = vmlsl_u8(q6u16, d20u8, d1u8);
+ q8u16 = vmlsl_u8(q8u16, d21u8, d1u8);
+ q7u16 = vmlsl_u8(q7u16, d22u8, d1u8);
+ q9u16 = vmlsl_u8(q9u16, d23u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d24u8, d4u8);
+ q8u16 = vmlsl_u8(q8u16, d25u8, d4u8);
+ q7u16 = vmlsl_u8(q7u16, d26u8, d4u8);
+ q9u16 = vmlsl_u8(q9u16, d27u8, d4u8);
+ q6u16 = vmlal_u8(q6u16, d28u8, d5u8);
+ q8u16 = vmlal_u8(q8u16, d29u8, d5u8);
+
+ d20u8 = vext_u8(d7u8, d8u8, 5);
+ d21u8 = vext_u8(d10u8, d11u8, 5);
+ d22u8 = vext_u8(d6u8, d7u8, 2);
+ d23u8 = vext_u8(d9u8, d10u8, 2);
+ d24u8 = vext_u8(d7u8, d8u8, 2);
+ d25u8 = vext_u8(d10u8, d11u8, 2);
+ d26u8 = vext_u8(d6u8, d7u8, 3);
+ d27u8 = vext_u8(d9u8, d10u8, 3);
+ d28u8 = vext_u8(d7u8, d8u8, 3);
+ d29u8 = vext_u8(d10u8, d11u8, 3);
+
+ q7u16 = vmlal_u8(q7u16, d20u8, d5u8);
+ q9u16 = vmlal_u8(q9u16, d21u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d22u8, d2u8);
+ q8u16 = vmlal_u8(q8u16, d23u8, d2u8);
+ q7u16 = vmlal_u8(q7u16, d24u8, d2u8);
+ q9u16 = vmlal_u8(q9u16, d25u8, d2u8);
+
+ q10u16 = vmull_u8(d26u8, d3u8);
+ q11u16 = vmull_u8(d27u8, d3u8);
+ q12u16 = vmull_u8(d28u8, d3u8);
+ q15u16 = vmull_u8(d29u8, d3u8);
+
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+ q11s16 = vreinterpretq_s16_u16(q11u16);
+ q12s16 = vreinterpretq_s16_u16(q12u16);
+ q15s16 = vreinterpretq_s16_u16(q15u16);
+
+ q6s16 = vqaddq_s16(q6s16, q10s16);
+ q8s16 = vqaddq_s16(q8s16, q11s16);
+ q7s16 = vqaddq_s16(q7s16, q12s16);
+ q9s16 = vqaddq_s16(q9s16, q15s16);
+
+ d6u8 = vqrshrun_n_s16(q6s16, 7);
+ d7u8 = vqrshrun_n_s16(q7s16, 7);
+ d8u8 = vqrshrun_n_s16(q8s16, 7);
+ d9u8 = vqrshrun_n_s16(q9s16, 7);
+
+ q3u8 = vcombine_u8(d6u8, d7u8);
+ q4u8 = vcombine_u8(d8u8, d9u8);
+ vst1q_u8(dst, q3u8);
+ dst += dst_pitch;
+ vst1q_u8(dst, q4u8);
+ dst += dst_pitch;
+ }
+ return;
+ }
+
+ src = src_ptr - 2 - src_pixels_per_line * 2;
+ tmpp = tmp;
+ for (i = 0; i < 7; i++) {
+ d6u8 = vld1_u8(src);
+ d7u8 = vld1_u8(src + 8);
+ d8u8 = vld1_u8(src + 16);
+ src += src_pixels_per_line;
+ d9u8 = vld1_u8(src);
+ d10u8 = vld1_u8(src + 8);
+ d11u8 = vld1_u8(src + 16);
+ src += src_pixels_per_line;
+ d12u8 = vld1_u8(src);
+ d13u8 = vld1_u8(src + 8);
+ d14u8 = vld1_u8(src + 16);
+ src += src_pixels_per_line;
+
+ __builtin_prefetch(src);
+ __builtin_prefetch(src + src_pixels_per_line);
+ __builtin_prefetch(src + src_pixels_per_line * 2);
+
+ q8u16 = vmull_u8(d6u8, d0u8);
+ q9u16 = vmull_u8(d7u8, d0u8);
+ q10u16 = vmull_u8(d9u8, d0u8);
+ q11u16 = vmull_u8(d10u8, d0u8);
+ q12u16 = vmull_u8(d12u8, d0u8);
+ q13u16 = vmull_u8(d13u8, d0u8);
+
+ d28u8 = vext_u8(d6u8, d7u8, 1);
+ d29u8 = vext_u8(d9u8, d10u8, 1);
+ d30u8 = vext_u8(d12u8, d13u8, 1);
+ q8u16 = vmlsl_u8(q8u16, d28u8, d1u8);
+ q10u16 = vmlsl_u8(q10u16, d29u8, d1u8);
+ q12u16 = vmlsl_u8(q12u16, d30u8, d1u8);
+ d28u8 = vext_u8(d7u8, d8u8, 1);
+ d29u8 = vext_u8(d10u8, d11u8, 1);
+ d30u8 = vext_u8(d13u8, d14u8, 1);
+ q9u16 = vmlsl_u8(q9u16, d28u8, d1u8);
+ q11u16 = vmlsl_u8(q11u16, d29u8, d1u8);
+ q13u16 = vmlsl_u8(q13u16, d30u8, d1u8);
+
+ d28u8 = vext_u8(d6u8, d7u8, 4);
+ d29u8 = vext_u8(d9u8, d10u8, 4);
+ d30u8 = vext_u8(d12u8, d13u8, 4);
+ q8u16 = vmlsl_u8(q8u16, d28u8, d4u8);
+ q10u16 = vmlsl_u8(q10u16, d29u8, d4u8);
+ q12u16 = vmlsl_u8(q12u16, d30u8, d4u8);
+ d28u8 = vext_u8(d7u8, d8u8, 4);
+ d29u8 = vext_u8(d10u8, d11u8, 4);
+ d30u8 = vext_u8(d13u8, d14u8, 4);
+ q9u16 = vmlsl_u8(q9u16, d28u8, d4u8);
+ q11u16 = vmlsl_u8(q11u16, d29u8, d4u8);
+ q13u16 = vmlsl_u8(q13u16, d30u8, d4u8);
+
+ d28u8 = vext_u8(d6u8, d7u8, 5);
+ d29u8 = vext_u8(d9u8, d10u8, 5);
+ d30u8 = vext_u8(d12u8, d13u8, 5);
+ q8u16 = vmlal_u8(q8u16, d28u8, d5u8);
+ q10u16 = vmlal_u8(q10u16, d29u8, d5u8);
+ q12u16 = vmlal_u8(q12u16, d30u8, d5u8);
+ d28u8 = vext_u8(d7u8, d8u8, 5);
+ d29u8 = vext_u8(d10u8, d11u8, 5);
+ d30u8 = vext_u8(d13u8, d14u8, 5);
+ q9u16 = vmlal_u8(q9u16, d28u8, d5u8);
+ q11u16 = vmlal_u8(q11u16, d29u8, d5u8);
+ q13u16 = vmlal_u8(q13u16, d30u8, d5u8);
+
+ d28u8 = vext_u8(d6u8, d7u8, 2);
+ d29u8 = vext_u8(d9u8, d10u8, 2);
+ d30u8 = vext_u8(d12u8, d13u8, 2);
+ q8u16 = vmlal_u8(q8u16, d28u8, d2u8);
+ q10u16 = vmlal_u8(q10u16, d29u8, d2u8);
+ q12u16 = vmlal_u8(q12u16, d30u8, d2u8);
+ d28u8 = vext_u8(d7u8, d8u8, 2);
+ d29u8 = vext_u8(d10u8, d11u8, 2);
+ d30u8 = vext_u8(d13u8, d14u8, 2);
+ q9u16 = vmlal_u8(q9u16, d28u8, d2u8);
+ q11u16 = vmlal_u8(q11u16, d29u8, d2u8);
+ q13u16 = vmlal_u8(q13u16, d30u8, d2u8);
+
+ d28u8 = vext_u8(d6u8, d7u8, 3);
+ d29u8 = vext_u8(d9u8, d10u8, 3);
+ d30u8 = vext_u8(d12u8, d13u8, 3);
+ d15u8 = vext_u8(d7u8, d8u8, 3);
+ d31u8 = vext_u8(d10u8, d11u8, 3);
+ d6u8 = vext_u8(d13u8, d14u8, 3);
+ q4u16 = vmull_u8(d28u8, d3u8);
+ q5u16 = vmull_u8(d29u8, d3u8);
+ q6u16 = vmull_u8(d30u8, d3u8);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+ q12s16 = vreinterpretq_s16_u16(q12u16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q10s16 = vqaddq_s16(q10s16, q5s16);
+ q12s16 = vqaddq_s16(q12s16, q6s16);
+
+ q6u16 = vmull_u8(d15u8, d3u8);
+ q7u16 = vmull_u8(d31u8, d3u8);
+ q3u16 = vmull_u8(d6u8, d3u8);
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q11s16 = vreinterpretq_s16_u16(q11u16);
+ q13s16 = vreinterpretq_s16_u16(q13u16);
+ q9s16 = vqaddq_s16(q9s16, q6s16);
+ q11s16 = vqaddq_s16(q11s16, q7s16);
+ q13s16 = vqaddq_s16(q13s16, q3s16);
+
+ d6u8 = vqrshrun_n_s16(q8s16, 7);
+ d7u8 = vqrshrun_n_s16(q9s16, 7);
+ d8u8 = vqrshrun_n_s16(q10s16, 7);
+ d9u8 = vqrshrun_n_s16(q11s16, 7);
+ d10u8 = vqrshrun_n_s16(q12s16, 7);
+ d11u8 = vqrshrun_n_s16(q13s16, 7);
+
+ vst1_u8(tmpp, d6u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d7u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d8u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d9u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d10u8);
+ tmpp += 8;
+ vst1_u8(tmpp, d11u8);
+ tmpp += 8;
+ }
+
+ // Second pass: 16x16
+ dtmps8 = vld1_s8(vp8_sub_pel_filters[yoffset]);
+ d0s8 = vdup_lane_s8(dtmps8, 0);
+ d1s8 = vdup_lane_s8(dtmps8, 1);
+ d2s8 = vdup_lane_s8(dtmps8, 2);
+ d3s8 = vdup_lane_s8(dtmps8, 3);
+ d4s8 = vdup_lane_s8(dtmps8, 4);
+ d5s8 = vdup_lane_s8(dtmps8, 5);
+ d0u8 = vreinterpret_u8_s8(vabs_s8(d0s8));
+ d1u8 = vreinterpret_u8_s8(vabs_s8(d1s8));
+ d2u8 = vreinterpret_u8_s8(vabs_s8(d2s8));
+ d3u8 = vreinterpret_u8_s8(vabs_s8(d3s8));
+ d4u8 = vreinterpret_u8_s8(vabs_s8(d4s8));
+ d5u8 = vreinterpret_u8_s8(vabs_s8(d5s8));
+
+ for (i = 0; i < 2; i++) {
+ dst = dst_ptr + 8 * i;
+ tmpp = tmp + 8 * i;
+ d18u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d19u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d20u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d21u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d22u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ for (j = 0; j < 4; j++) {
+ d23u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d24u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d25u8 = vld1_u8(tmpp);
+ tmpp += 16;
+ d26u8 = vld1_u8(tmpp);
+ tmpp += 16;
+
+ q3u16 = vmull_u8(d18u8, d0u8);
+ q4u16 = vmull_u8(d19u8, d0u8);
+ q5u16 = vmull_u8(d20u8, d0u8);
+ q6u16 = vmull_u8(d21u8, d0u8);
+
+ q3u16 = vmlsl_u8(q3u16, d19u8, d1u8);
+ q4u16 = vmlsl_u8(q4u16, d20u8, d1u8);
+ q5u16 = vmlsl_u8(q5u16, d21u8, d1u8);
+ q6u16 = vmlsl_u8(q6u16, d22u8, d1u8);
+
+ q3u16 = vmlsl_u8(q3u16, d22u8, d4u8);
+ q4u16 = vmlsl_u8(q4u16, d23u8, d4u8);
+ q5u16 = vmlsl_u8(q5u16, d24u8, d4u8);
+ q6u16 = vmlsl_u8(q6u16, d25u8, d4u8);
+
+ q3u16 = vmlal_u8(q3u16, d20u8, d2u8);
+ q4u16 = vmlal_u8(q4u16, d21u8, d2u8);
+ q5u16 = vmlal_u8(q5u16, d22u8, d2u8);
+ q6u16 = vmlal_u8(q6u16, d23u8, d2u8);
+
+ q3u16 = vmlal_u8(q3u16, d23u8, d5u8);
+ q4u16 = vmlal_u8(q4u16, d24u8, d5u8);
+ q5u16 = vmlal_u8(q5u16, d25u8, d5u8);
+ q6u16 = vmlal_u8(q6u16, d26u8, d5u8);
+
+ q7u16 = vmull_u8(d21u8, d3u8);
+ q8u16 = vmull_u8(d22u8, d3u8);
+ q9u16 = vmull_u8(d23u8, d3u8);
+ q10u16 = vmull_u8(d24u8, d3u8);
+
+ q3s16 = vreinterpretq_s16_u16(q3u16);
+ q4s16 = vreinterpretq_s16_u16(q4u16);
+ q5s16 = vreinterpretq_s16_u16(q5u16);
+ q6s16 = vreinterpretq_s16_u16(q6u16);
+ q7s16 = vreinterpretq_s16_u16(q7u16);
+ q8s16 = vreinterpretq_s16_u16(q8u16);
+ q9s16 = vreinterpretq_s16_u16(q9u16);
+ q10s16 = vreinterpretq_s16_u16(q10u16);
+
+ q7s16 = vqaddq_s16(q7s16, q3s16);
+ q8s16 = vqaddq_s16(q8s16, q4s16);
+ q9s16 = vqaddq_s16(q9s16, q5s16);
+ q10s16 = vqaddq_s16(q10s16, q6s16);
+
+ d6u8 = vqrshrun_n_s16(q7s16, 7);
+ d7u8 = vqrshrun_n_s16(q8s16, 7);
+ d8u8 = vqrshrun_n_s16(q9s16, 7);
+ d9u8 = vqrshrun_n_s16(q10s16, 7);
+
+ d18u8 = d22u8;
+ d19u8 = d23u8;
+ d20u8 = d24u8;
+ d21u8 = d25u8;
+ d22u8 = d26u8;
+
+ vst1_u8(dst, d6u8);
+ dst += dst_pitch;
+ vst1_u8(dst, d7u8);
+ dst += dst_pitch;
+ vst1_u8(dst, d8u8);
+ dst += dst_pitch;
+ vst1_u8(dst, d9u8);
+ dst += dst_pitch;
+ }
+ }
+ return;
+}
diff --git a/vp8/common/arm/neon/variance_neon.asm b/vp8/common/arm/neon/variance_neon.asm
deleted file mode 100644
index 8ecad72..0000000
--- a/vp8/common/arm/neon/variance_neon.asm
+++ /dev/null
@@ -1,291 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license
-; that can be found in the LICENSE file in the root of the source
-; tree. An additional intellectual property rights grant can be found
-; in the file PATENTS. All contributing project authors may
-; be found in the AUTHORS file in the root of the source tree.
-;
-
-
- EXPORT |vp8_variance16x16_neon|
- EXPORT |vp8_variance16x8_neon|
- EXPORT |vp8_variance8x16_neon|
- EXPORT |vp8_variance8x8_neon|
-
- ARM
- REQUIRE8
- PRESERVE8
-
- AREA ||.text||, CODE, READONLY, ALIGN=2
-
-; r0 unsigned char *src_ptr
-; r1 int source_stride
-; r2 unsigned char *ref_ptr
-; r3 int recon_stride
-; stack unsigned int *sse
-|vp8_variance16x16_neon| PROC
- vpush {q5}
- vmov.i8 q8, #0 ;q8 - sum
- vmov.i8 q9, #0 ;q9, q10 - sse
- vmov.i8 q10, #0
-
- mov r12, #8
-
-variance16x16_neon_loop
- vld1.8 {q0}, [r0], r1 ;Load up source and reference
- vld1.8 {q2}, [r2], r3
- vld1.8 {q1}, [r0], r1
- vld1.8 {q3}, [r2], r3
-
- vsubl.u8 q11, d0, d4 ;calculate diff
- vsubl.u8 q12, d1, d5
- vsubl.u8 q13, d2, d6
- vsubl.u8 q14, d3, d7
-
- ;VPADAL adds adjacent pairs of elements of a vector, and accumulates
- ;the results into the elements of the destination vector. The explanation
- ;in ARM guide is wrong.
- vpadal.s16 q8, q11 ;calculate sum
- vmlal.s16 q9, d22, d22 ;calculate sse
- vmlal.s16 q10, d23, d23
-
- subs r12, r12, #1
-
- vpadal.s16 q8, q12
- vmlal.s16 q9, d24, d24
- vmlal.s16 q10, d25, d25
- vpadal.s16 q8, q13
- vmlal.s16 q9, d26, d26
- vmlal.s16 q10, d27, d27
- vpadal.s16 q8, q14
- vmlal.s16 q9, d28, d28
- vmlal.s16 q10, d29, d29
-
- bne variance16x16_neon_loop
-
- vadd.u32 q10, q9, q10 ;accumulate sse
- vpaddl.s32 q0, q8 ;accumulate sum
-
- ldr r12, [sp, #16] ;load *sse from stack
-
- vpaddl.u32 q1, q10
- vadd.s64 d0, d0, d1
- vadd.u64 d1, d2, d3
-
- ;vmov.32 r0, d0[0] ;this instruction costs a lot
- ;vmov.32 r1, d1[0]
- ;mul r0, r0, r0
- ;str r1, [r12]
- ;sub r0, r1, r0, lsr #8
-
- ; while sum is signed, sum * sum is always positive and must be treated as
- ; unsigned to avoid propagating the sign bit.
- vmull.s32 q5, d0, d0
- vst1.32 {d1[0]}, [r12] ;store sse
- vshr.u32 d10, d10, #8
- vsub.u32 d0, d1, d10
-
- vmov.32 r0, d0[0] ;return
-
- vpop {q5}
- bx lr
-
- ENDP
-
-;================================
-;unsigned int vp8_variance16x8_c(
-; unsigned char *src_ptr,
-; int source_stride,
-; unsigned char *ref_ptr,
-; int recon_stride,
-; unsigned int *sse)
-|vp8_variance16x8_neon| PROC
- vpush {q5}
-
- vmov.i8 q8, #0 ;q8 - sum
- vmov.i8 q9, #0 ;q9, q10 - sse
- vmov.i8 q10, #0
-
- mov r12, #4
-
-variance16x8_neon_loop
- vld1.8 {q0}, [r0], r1 ;Load up source and reference
- vld1.8 {q2}, [r2], r3
- vld1.8 {q1}, [r0], r1
- vld1.8 {q3}, [r2], r3
-
- vsubl.u8 q11, d0, d4 ;calculate diff
- vsubl.u8 q12, d1, d5
- vsubl.u8 q13, d2, d6
- vsubl.u8 q14, d3, d7
-
- vpadal.s16 q8, q11 ;calculate sum
- vmlal.s16 q9, d22, d22 ;calculate sse
- vmlal.s16 q10, d23, d23
-
- subs r12, r12, #1
-
- vpadal.s16 q8, q12
- vmlal.s16 q9, d24, d24
- vmlal.s16 q10, d25, d25
- vpadal.s16 q8, q13
- vmlal.s16 q9, d26, d26
- vmlal.s16 q10, d27, d27
- vpadal.s16 q8, q14
- vmlal.s16 q9, d28, d28
- vmlal.s16 q10, d29, d29
-
- bne variance16x8_neon_loop
-
- vadd.u32 q10, q9, q10 ;accumulate sse
- vpaddl.s32 q0, q8 ;accumulate sum
-
- ldr r12, [sp, #16] ;load *sse from stack
-
- vpaddl.u32 q1, q10
- vadd.s64 d0, d0, d1
- vadd.u64 d1, d2, d3
-
- vmull.s32 q5, d0, d0
- vst1.32 {d1[0]}, [r12] ;store sse
- vshr.u32 d10, d10, #7
- vsub.u32 d0, d1, d10
-
- vmov.32 r0, d0[0] ;return
-
- vpop {q5}
- bx lr
-
- ENDP
-
-;=================================
-;unsigned int vp8_variance8x16_c(
-; unsigned char *src_ptr,
-; int source_stride,
-; unsigned char *ref_ptr,
-; int recon_stride,
-; unsigned int *sse)
-
-|vp8_variance8x16_neon| PROC
- vpush {q5}
-
- vmov.i8 q8, #0 ;q8 - sum
- vmov.i8 q9, #0 ;q9, q10 - sse
- vmov.i8 q10, #0
-
- mov r12, #8
-
-variance8x16_neon_loop
- vld1.8 {d0}, [r0], r1 ;Load up source and reference
- vld1.8 {d4}, [r2], r3
- vld1.8 {d2}, [r0], r1
- vld1.8 {d6}, [r2], r3
-
- vsubl.u8 q11, d0, d4 ;calculate diff
- vsubl.u8 q12, d2, d6
-
- vpadal.s16 q8, q11 ;calculate sum
- vmlal.s16 q9, d22, d22 ;calculate sse
- vmlal.s16 q10, d23, d23
-
- subs r12, r12, #1
-
- vpadal.s16 q8, q12
- vmlal.s16 q9, d24, d24
- vmlal.s16 q10, d25, d25
-
- bne variance8x16_neon_loop
-
- vadd.u32 q10, q9, q10 ;accumulate sse
- vpaddl.s32 q0, q8 ;accumulate sum
-
- ldr r12, [sp, #16] ;load *sse from stack
-
- vpaddl.u32 q1, q10
- vadd.s64 d0, d0, d1
- vadd.u64 d1, d2, d3
-
- vmull.s32 q5, d0, d0
- vst1.32 {d1[0]}, [r12] ;store sse
- vshr.u32 d10, d10, #7
- vsub.u32 d0, d1, d10
-
- vmov.32 r0, d0[0] ;return
-
- vpop {q5}
- bx lr
-
- ENDP
-
-;==================================
-; r0 unsigned char *src_ptr
-; r1 int source_stride
-; r2 unsigned char *ref_ptr
-; r3 int recon_stride
-; stack unsigned int *sse
-|vp8_variance8x8_neon| PROC
- vpush {q5}
-
- vmov.i8 q8, #0 ;q8 - sum
- vmov.i8 q9, #0 ;q9, q10 - sse
- vmov.i8 q10, #0
-
- mov r12, #2
-
-variance8x8_neon_loop
- vld1.8 {d0}, [r0], r1 ;Load up source and reference
- vld1.8 {d4}, [r2], r3
- vld1.8 {d1}, [r0], r1
- vld1.8 {d5}, [r2], r3
- vld1.8 {d2}, [r0], r1
- vld1.8 {d6}, [r2], r3
- vld1.8 {d3}, [r0], r1
- vld1.8 {d7}, [r2], r3
-
- vsubl.u8 q11, d0, d4 ;calculate diff
- vsubl.u8 q12, d1, d5
- vsubl.u8 q13, d2, d6
- vsubl.u8 q14, d3, d7
-
- vpadal.s16 q8, q11 ;calculate sum
- vmlal.s16 q9, d22, d22 ;calculate sse
- vmlal.s16 q10, d23, d23
-
- subs r12, r12, #1
-
- vpadal.s16 q8, q12
- vmlal.s16 q9, d24, d24
- vmlal.s16 q10, d25, d25
- vpadal.s16 q8, q13
- vmlal.s16 q9, d26, d26
- vmlal.s16 q10, d27, d27
- vpadal.s16 q8, q14
- vmlal.s16 q9, d28, d28
- vmlal.s16 q10, d29, d29
-
- bne variance8x8_neon_loop
-
- vadd.u32 q10, q9, q10 ;accumulate sse
- vpaddl.s32 q0, q8 ;accumulate sum
-
- ldr r12, [sp, #16] ;load *sse from stack
-
- vpaddl.u32 q1, q10
- vadd.s64 d0, d0, d1
- vadd.u64 d1, d2, d3
-
- vmull.s32 q5, d0, d0
- vst1.32 {d1[0]}, [r12] ;store sse
- vshr.u32 d10, d10, #6
- vsub.u32 d0, d1, d10
-
- vmov.32 r0, d0[0] ;return
-
- vpop {q5}
- bx lr
-
- ENDP
-
- END
diff --git a/vp8/common/arm/neon/variance_neon.c b/vp8/common/arm/neon/variance_neon.c
new file mode 100644
index 0000000..3e25f7d
--- /dev/null
+++ b/vp8/common/arm/neon/variance_neon.c
@@ -0,0 +1,319 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <arm_neon.h>
+
+unsigned int vp8_variance16x16_neon(
+ const unsigned char *src_ptr,
+ int source_stride,
+ const unsigned char *ref_ptr,
+ int recon_stride,
+ unsigned int *sse) {
+ int i;
+ int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
+ uint32x2_t d0u32, d10u32;
+ int64_t d0s64, d1s64;
+ uint8x16_t q0u8, q1u8, q2u8, q3u8;
+ uint16x8_t q11u16, q12u16, q13u16, q14u16;
+ int32x4_t q8s32, q9s32, q10s32;
+ int64x2_t q0s64, q1s64, q5s64;
+
+ q8s32 = vdupq_n_s32(0);
+ q9s32 = vdupq_n_s32(0);
+ q10s32 = vdupq_n_s32(0);
+
+ for (i = 0; i < 8; i++) {
+ q0u8 = vld1q_u8(src_ptr);
+ src_ptr += source_stride;
+ q1u8 = vld1q_u8(src_ptr);
+ src_ptr += source_stride;
+ __builtin_prefetch(src_ptr);
+
+ q2u8 = vld1q_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ q3u8 = vld1q_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ __builtin_prefetch(ref_ptr);
+
+ q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
+ q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
+ q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
+ q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
+
+ d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
+ d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
+ q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
+ q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
+
+ d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
+ d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
+ q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
+ q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
+
+ d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
+ d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
+ q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
+ q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
+
+ d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
+ d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
+ q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
+ q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
+ }
+
+ q10s32 = vaddq_s32(q10s32, q9s32);
+ q0s64 = vpaddlq_s32(q8s32);
+ q1s64 = vpaddlq_s32(q10s32);
+
+ d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64));
+ d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
+
+ q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
+ vreinterpret_s32_s64(d0s64));
+ vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
+
+ d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 8);
+ d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
+
+ return vget_lane_u32(d0u32, 0);
+}
+
+unsigned int vp8_variance16x8_neon(
+ const unsigned char *src_ptr,
+ int source_stride,
+ const unsigned char *ref_ptr,
+ int recon_stride,
+ unsigned int *sse) {
+ int i;
+ int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
+ uint32x2_t d0u32, d10u32;
+ int64_t d0s64, d1s64;
+ uint8x16_t q0u8, q1u8, q2u8, q3u8;
+ uint16x8_t q11u16, q12u16, q13u16, q14u16;
+ int32x4_t q8s32, q9s32, q10s32;
+ int64x2_t q0s64, q1s64, q5s64;
+
+ q8s32 = vdupq_n_s32(0);
+ q9s32 = vdupq_n_s32(0);
+ q10s32 = vdupq_n_s32(0);
+
+ for (i = 0; i < 4; i++) { // variance16x8_neon_loop
+ q0u8 = vld1q_u8(src_ptr);
+ src_ptr += source_stride;
+ q1u8 = vld1q_u8(src_ptr);
+ src_ptr += source_stride;
+ __builtin_prefetch(src_ptr);
+
+ q2u8 = vld1q_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ q3u8 = vld1q_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ __builtin_prefetch(ref_ptr);
+
+ q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8));
+ q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8));
+ q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8));
+ q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8));
+
+ d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
+ d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
+ q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
+ q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
+
+ d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
+ d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
+ q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
+ q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
+
+ d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
+ d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
+ q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
+ q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
+
+ d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
+ d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
+ q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
+ q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
+ }
+
+ q10s32 = vaddq_s32(q10s32, q9s32);
+ q0s64 = vpaddlq_s32(q8s32);
+ q1s64 = vpaddlq_s32(q10s32);
+
+ d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64));
+ d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
+
+ q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
+ vreinterpret_s32_s64(d0s64));
+ vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
+
+ d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 7);
+ d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
+
+ return vget_lane_u32(d0u32, 0);
+}
+
+unsigned int vp8_variance8x16_neon(
+ const unsigned char *src_ptr,
+ int source_stride,
+ const unsigned char *ref_ptr,
+ int recon_stride,
+ unsigned int *sse) {
+ int i;
+ uint8x8_t d0u8, d2u8, d4u8, d6u8;
+ int16x4_t d22s16, d23s16, d24s16, d25s16;
+ uint32x2_t d0u32, d10u32;
+ int64_t d0s64, d1s64;
+ uint16x8_t q11u16, q12u16;
+ int32x4_t q8s32, q9s32, q10s32;
+ int64x2_t q0s64, q1s64, q5s64;
+
+ q8s32 = vdupq_n_s32(0);
+ q9s32 = vdupq_n_s32(0);
+ q10s32 = vdupq_n_s32(0);
+
+ for (i = 0; i < 8; i++) { // variance8x16_neon_loop
+ d0u8 = vld1_u8(src_ptr);
+ src_ptr += source_stride;
+ d2u8 = vld1_u8(src_ptr);
+ src_ptr += source_stride;
+ __builtin_prefetch(src_ptr);
+
+ d4u8 = vld1_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ d6u8 = vld1_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ __builtin_prefetch(ref_ptr);
+
+ q11u16 = vsubl_u8(d0u8, d4u8);
+ q12u16 = vsubl_u8(d2u8, d6u8);
+
+ d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
+ d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
+ q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
+ q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
+
+ d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
+ d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
+ q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
+ q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
+ }
+
+ q10s32 = vaddq_s32(q10s32, q9s32);
+ q0s64 = vpaddlq_s32(q8s32);
+ q1s64 = vpaddlq_s32(q10s32);
+
+ d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64));
+ d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
+
+ q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
+ vreinterpret_s32_s64(d0s64));
+ vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
+
+ d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 7);
+ d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
+
+ return vget_lane_u32(d0u32, 0);
+}
+
+unsigned int vp8_variance8x8_neon(
+ const unsigned char *src_ptr,
+ int source_stride,
+ const unsigned char *ref_ptr,
+ int recon_stride,
+ unsigned int *sse) {
+ int i;
+ uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8;
+ int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16;
+ uint32x2_t d0u32, d10u32;
+ int64_t d0s64, d1s64;
+ uint16x8_t q11u16, q12u16, q13u16, q14u16;
+ int32x4_t q8s32, q9s32, q10s32;
+ int64x2_t q0s64, q1s64, q5s64;
+
+ q8s32 = vdupq_n_s32(0);
+ q9s32 = vdupq_n_s32(0);
+ q10s32 = vdupq_n_s32(0);
+
+ for (i = 0; i < 2; i++) { // variance8x8_neon_loop
+ d0u8 = vld1_u8(src_ptr);
+ src_ptr += source_stride;
+ d1u8 = vld1_u8(src_ptr);
+ src_ptr += source_stride;
+ d2u8 = vld1_u8(src_ptr);
+ src_ptr += source_stride;
+ d3u8 = vld1_u8(src_ptr);
+ src_ptr += source_stride;
+
+ d4u8 = vld1_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ d5u8 = vld1_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ d6u8 = vld1_u8(ref_ptr);
+ ref_ptr += recon_stride;
+ d7u8 = vld1_u8(ref_ptr);
+ ref_ptr += recon_stride;
+
+ q11u16 = vsubl_u8(d0u8, d4u8);
+ q12u16 = vsubl_u8(d1u8, d5u8);
+ q13u16 = vsubl_u8(d2u8, d6u8);
+ q14u16 = vsubl_u8(d3u8, d7u8);
+
+ d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16));
+ d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q11u16));
+ q9s32 = vmlal_s16(q9s32, d22s16, d22s16);
+ q10s32 = vmlal_s16(q10s32, d23s16, d23s16);
+
+ d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16));
+ d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q12u16));
+ q9s32 = vmlal_s16(q9s32, d24s16, d24s16);
+ q10s32 = vmlal_s16(q10s32, d25s16, d25s16);
+
+ d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16));
+ d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q13u16));
+ q9s32 = vmlal_s16(q9s32, d26s16, d26s16);
+ q10s32 = vmlal_s16(q10s32, d27s16, d27s16);
+
+ d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16));
+ d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16));
+ q8s32 = vpadalq_s16(q8s32, vreinterpretq_s16_u16(q14u16));
+ q9s32 = vmlal_s16(q9s32, d28s16, d28s16);
+ q10s32 = vmlal_s16(q10s32, d29s16, d29s16);
+ }
+
+ q10s32 = vaddq_s32(q10s32, q9s32);
+ q0s64 = vpaddlq_s32(q8s32);
+ q1s64 = vpaddlq_s32(q10s32);
+
+ d0s64 = vadd_s64(vget_low_s64(q0s64), vget_high_s64(q0s64));
+ d1s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64));
+
+ q5s64 = vmull_s32(vreinterpret_s32_s64(d0s64),
+ vreinterpret_s32_s64(d0s64));
+ vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d1s64), 0);
+
+ d10u32 = vshr_n_u32(vreinterpret_u32_s64(vget_low_s64(q5s64)), 6);
+ d0u32 = vsub_u32(vreinterpret_u32_s64(d1s64), d10u32);
+
+ return vget_lane_u32(d0u32, 0);
+}
diff --git a/vp8/common/postproc.c b/vp8/common/postproc.c
index 7d0fbf6..8e546d5 100644
--- a/vp8/common/postproc.c
+++ b/vp8/common/postproc.c
@@ -310,6 +310,7 @@
}
}
+#if CONFIG_POSTPROC
static void vp8_de_mblock(YV12_BUFFER_CONFIG *post,
int q)
{
@@ -382,6 +383,7 @@
vp8_yv12_copy_frame(source, post);
}
}
+#endif
#if !(CONFIG_TEMPORAL_DENOISING)
void vp8_de_noise(VP8_COMMON *cm,
diff --git a/vp8/common/rtcd_defs.pl b/vp8/common/rtcd_defs.pl
index 130d965..789b0de5 100644
--- a/vp8/common/rtcd_defs.pl
+++ b/vp8/common/rtcd_defs.pl
@@ -532,7 +532,7 @@
# Denoiser filter
#
if (vpx_config("CONFIG_TEMPORAL_DENOISING") eq "yes") {
- add_proto qw/int vp8_denoiser_filter/, "struct yv12_buffer_config* mc_running_avg, struct yv12_buffer_config* running_avg, struct macroblock* signal, unsigned int motion_magnitude2, int y_offset, int uv_offset";
+ add_proto qw/int vp8_denoiser_filter/, "unsigned char *mc_running_avg_y, int mc_avg_y_stride, unsigned char *running_avg_y, int avg_y_stride, unsigned char *sig, int sig_stride, unsigned int motion_magnitude";
specialize qw/vp8_denoiser_filter sse2 neon/;
}
diff --git a/vp8/encoder/arm/neon/denoising_neon.c b/vp8/encoder/arm/neon/denoising_neon.c
index 23dc0a9..1bebe8f 100644
--- a/vp8/encoder/arm/neon/denoising_neon.c
+++ b/vp8/encoder/arm/neon/denoising_neon.c
@@ -45,10 +45,12 @@
* [16, 255] 3 6 7
*/
-int vp8_denoiser_filter_neon(YV12_BUFFER_CONFIG *mc_running_avg,
- YV12_BUFFER_CONFIG *running_avg,
- MACROBLOCK *signal, unsigned int motion_magnitude,
- int y_offset, int uv_offset) {
+int vp8_denoiser_filter_neon(unsigned char *mc_running_avg_y,
+ int mc_running_avg_y_stride,
+ unsigned char *running_avg_y,
+ int running_avg_y_stride,
+ unsigned char *sig, int sig_stride,
+ unsigned int motion_magnitude) {
/* If motion_magnitude is small, making the denoiser more aggressive by
* increasing the adjustment for each level, level1 adjustment is
* increased, the deltas stay the same.
@@ -60,14 +62,6 @@
const uint8x16_t v_level1_threshold = vdupq_n_u8(4);
const uint8x16_t v_level2_threshold = vdupq_n_u8(8);
const uint8x16_t v_level3_threshold = vdupq_n_u8(16);
-
- /* Local variables for array pointers and strides. */
- unsigned char *sig = signal->thismb;
- int sig_stride = 16;
- unsigned char *mc_running_avg_y = mc_running_avg->y_buffer + y_offset;
- int mc_running_avg_y_stride = mc_running_avg->y_stride;
- unsigned char *running_avg_y = running_avg->y_buffer + y_offset;
- int running_avg_y_stride = running_avg->y_stride;
int64x2_t v_sum_diff_total = vdupq_n_s64(0);
/* Go over lines. */
diff --git a/vp8/encoder/denoising.c b/vp8/encoder/denoising.c
index 7819265..bfce280 100644
--- a/vp8/encoder/denoising.c
+++ b/vp8/encoder/denoising.c
@@ -51,17 +51,13 @@
* [16, 255] 6 7
*/
-int vp8_denoiser_filter_c(YV12_BUFFER_CONFIG *mc_running_avg,
- YV12_BUFFER_CONFIG *running_avg, MACROBLOCK *signal,
- unsigned int motion_magnitude, int y_offset,
- int uv_offset)
+int vp8_denoiser_filter_c(unsigned char *mc_running_avg_y, int mc_avg_y_stride,
+ unsigned char *running_avg_y, int avg_y_stride,
+ unsigned char *sig, int sig_stride,
+ unsigned int motion_magnitude)
{
- unsigned char *sig = signal->thismb;
- int sig_stride = 16;
- unsigned char *mc_running_avg_y = mc_running_avg->y_buffer + y_offset;
- int mc_avg_y_stride = mc_running_avg->y_stride;
- unsigned char *running_avg_y = running_avg->y_buffer + y_offset;
- int avg_y_stride = running_avg->y_stride;
+ unsigned char *running_avg_y_start = running_avg_y;
+ unsigned char *sig_start = sig;
int r, c, i;
int sum_diff = 0;
int adj_val[3] = {3, 4, 6};
@@ -130,8 +126,7 @@
if (abs(sum_diff) > SUM_DIFF_THRESHOLD)
return COPY_BLOCK;
- vp8_copy_mem16x16(running_avg->y_buffer + y_offset, avg_y_stride,
- signal->thismb, sig_stride);
+ vp8_copy_mem16x16(running_avg_y_start, avg_y_stride, sig_start, sig_stride);
return FILTER_BLOCK;
}
@@ -285,12 +280,17 @@
if (decision == FILTER_BLOCK)
{
+ unsigned char *mc_running_avg_y =
+ denoiser->yv12_mc_running_avg.y_buffer + recon_yoffset;
+ int mc_avg_y_stride = denoiser->yv12_mc_running_avg.y_stride;
+ unsigned char *running_avg_y =
+ denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset;
+ int avg_y_stride = denoiser->yv12_running_avg[INTRA_FRAME].y_stride;
+
/* Filter. */
- decision = vp8_denoiser_filter(&denoiser->yv12_mc_running_avg,
- &denoiser->yv12_running_avg[INTRA_FRAME],
- x,
- motion_magnitude2,
- recon_yoffset, recon_uvoffset);
+ decision = vp8_denoiser_filter(mc_running_avg_y, mc_avg_y_stride,
+ running_avg_y, avg_y_stride,
+ x->thismb, 16, motion_magnitude2);
}
if (decision == COPY_BLOCK)
{
diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 560134e..e95e44f 100644
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -5227,7 +5227,7 @@
int y_samples = orig->y_height * orig->y_width ;
int uv_samples = orig->uv_height * orig->uv_width ;
int t_samples = y_samples + 2 * uv_samples;
- double sq_error, sq_error2;
+ double sq_error;
ye = calc_plane_error(orig->y_buffer, orig->y_stride,
recon->y_buffer, recon->y_stride, orig->y_width, orig->y_height);
@@ -5250,6 +5250,7 @@
#if CONFIG_POSTPROC
{
YV12_BUFFER_CONFIG *pp = &cm->post_proc_buffer;
+ double sq_error2;
double frame_psnr2, frame_ssim2 = 0;
double weight = 0;
diff --git a/vp8/encoder/temporal_filter.c b/vp8/encoder/temporal_filter.c
index 513b2bf..4dc0d95 100644
--- a/vp8/encoder/temporal_filter.c
+++ b/vp8/encoder/temporal_filter.c
@@ -98,6 +98,7 @@
unsigned int i, j, k;
int modifier;
int byte = 0;
+ const int rounding = strength > 0 ? 1 << (strength - 1) : 0;
for (i = 0,k = 0; i < block_size; i++)
{
@@ -114,7 +115,7 @@
*/
modifier *= modifier;
modifier *= 3;
- modifier += 1 << (strength - 1);
+ modifier += rounding;
modifier >>= strength;
if (modifier > 16)
diff --git a/vp8/encoder/x86/denoising_sse2.c b/vp8/encoder/x86/denoising_sse2.c
index cceb826..d1f76b2 100644
--- a/vp8/encoder/x86/denoising_sse2.c
+++ b/vp8/encoder/x86/denoising_sse2.c
@@ -22,17 +22,14 @@
signed char e[16];
};
-int vp8_denoiser_filter_sse2(YV12_BUFFER_CONFIG *mc_running_avg,
- YV12_BUFFER_CONFIG *running_avg,
- MACROBLOCK *signal, unsigned int motion_magnitude,
- int y_offset, int uv_offset)
+int vp8_denoiser_filter_sse2(unsigned char *mc_running_avg_y,
+ int mc_avg_y_stride,
+ unsigned char *running_avg_y, int avg_y_stride,
+ unsigned char *sig, int sig_stride,
+ unsigned int motion_magnitude)
{
- unsigned char *sig = signal->thismb;
- int sig_stride = 16;
- unsigned char *mc_running_avg_y = mc_running_avg->y_buffer + y_offset;
- int mc_avg_y_stride = mc_running_avg->y_stride;
- unsigned char *running_avg_y = running_avg->y_buffer + y_offset;
- int avg_y_stride = running_avg->y_stride;
+ unsigned char *running_avg_y_start = running_avg_y;
+ unsigned char *sig_start = sig;
int r;
__m128i acc_diff = _mm_setzero_si128();
const __m128i k_0 = _mm_setzero_si128();
@@ -114,7 +111,6 @@
}
}
- vp8_copy_mem16x16(running_avg->y_buffer + y_offset, avg_y_stride,
- signal->thismb, sig_stride);
+ vp8_copy_mem16x16(running_avg_y_start, avg_y_stride, sig_start, sig_stride);
return FILTER_BLOCK;
}
diff --git a/vp8/encoder/x86/quantize_ssse3.asm b/vp8/encoder/x86/quantize_ssse3.asm
deleted file mode 100644
index 7b1dc11..0000000
--- a/vp8/encoder/x86/quantize_ssse3.asm
+++ /dev/null
@@ -1,138 +0,0 @@
-;
-; Copyright (c) 2010 The WebM project authors. All Rights Reserved.
-;
-; Use of this source code is governed by a BSD-style license and patent
-; grant that can be found in the LICENSE file in the root of the source
-; tree. All contributing project authors may be found in the AUTHORS
-; file in the root of the source tree.
-;
-
-
-%include "vpx_ports/x86_abi_support.asm"
-%include "vp8_asm_enc_offsets.asm"
-
-
-; void vp8_fast_quantize_b_ssse3 | arg
-; (BLOCK *b, | 0
-; BLOCKD *d) | 1
-;
-
-global sym(vp8_fast_quantize_b_ssse3) PRIVATE
-sym(vp8_fast_quantize_b_ssse3):
- push rbp
- mov rbp, rsp
- GET_GOT rbx
-
-%if ABI_IS_32BIT
- push rdi
- push rsi
-%else
- %if LIBVPX_YASM_WIN64
- push rdi
- push rsi
- %endif
-%endif
- ; end prolog
-
-%if ABI_IS_32BIT
- mov rdi, arg(0) ; BLOCK *b
- mov rsi, arg(1) ; BLOCKD *d
-%else
- %if LIBVPX_YASM_WIN64
- mov rdi, rcx ; BLOCK *b
- mov rsi, rdx ; BLOCKD *d
- %else
- ;mov rdi, rdi ; BLOCK *b
- ;mov rsi, rsi ; BLOCKD *d
- %endif
-%endif
-
- mov rax, [rdi + vp8_block_coeff]
- mov rcx, [rdi + vp8_block_round]
- mov rdx, [rdi + vp8_block_quant_fast]
-
- ; coeff
- movdqa xmm0, [rax]
- movdqa xmm4, [rax + 16]
-
- ; round
- movdqa xmm2, [rcx]
- movdqa xmm3, [rcx + 16]
-
- movdqa xmm1, xmm0
- movdqa xmm5, xmm4
-
- ; sz = z >> 15
- psraw xmm0, 15
- psraw xmm4, 15
-
- pabsw xmm1, xmm1
- pabsw xmm5, xmm5
-
- paddw xmm1, xmm2
- paddw xmm5, xmm3
-
- ; quant_fast
- pmulhw xmm1, [rdx]
- pmulhw xmm5, [rdx + 16]
-
- mov rax, [rsi + vp8_blockd_qcoeff]
- mov rdi, [rsi + vp8_blockd_dequant]
- mov rcx, [rsi + vp8_blockd_dqcoeff]
-
- movdqa xmm2, xmm1 ;store y for getting eob
- movdqa xmm3, xmm5
-
- pxor xmm1, xmm0
- pxor xmm5, xmm4
- psubw xmm1, xmm0
- psubw xmm5, xmm4
-
- movdqa [rax], xmm1
- movdqa [rax + 16], xmm5
-
- movdqa xmm0, [rdi]
- movdqa xmm4, [rdi + 16]
-
- pmullw xmm0, xmm1
- pmullw xmm4, xmm5
- pxor xmm1, xmm1
-
- pcmpgtw xmm2, xmm1 ;calculate eob
- pcmpgtw xmm3, xmm1
- packsswb xmm2, xmm3
- pshufb xmm2, [GLOBAL(zz_shuf)]
-
- pmovmskb edx, xmm2
-
- movdqa [rcx], xmm0 ;store dqcoeff
- movdqa [rcx + 16], xmm4 ;store dqcoeff
- mov rcx, [rsi + vp8_blockd_eob]
-
- bsr eax, edx ;count 0
- add eax, 1
-
- cmp edx, 0 ;if all 0, eob=0
- cmove eax, edx
-
- mov BYTE PTR [rcx], al ;store eob
-
- ; begin epilog
-%if ABI_IS_32BIT
- pop rsi
- pop rdi
-%else
- %if LIBVPX_YASM_WIN64
- pop rsi
- pop rdi
- %endif
-%endif
-
- RESTORE_GOT
- pop rbp
- ret
-
-SECTION_RODATA
-align 16
-zz_shuf:
- db 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15
diff --git a/vp8/encoder/x86/quantize_ssse3.c b/vp8/encoder/x86/quantize_ssse3.c
new file mode 100644
index 0000000..9b4471d
--- /dev/null
+++ b/vp8/encoder/x86/quantize_ssse3.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2012 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <tmmintrin.h> /* SSSE3 */
+
+#include "vp8/encoder/block.h"
+
+/* bitscan reverse (bsr) */
+#if defined(_MSC_VER)
+#include <intrin.h>
+#pragma intrinsic(_BitScanReverse)
+static int bsr(int mask) {
+ int eob;
+ _BitScanReverse(&eob, mask);
+ eob++;
+ if (mask == 0)
+ eob = 0;
+ return eob;
+}
+#else
+static int bsr(int mask) {
+ int eob;
+ asm volatile("bsr %1, %0" : "=r" (eob) : "r" (mask) : "flags");
+ eob++;
+ if (mask == 0)
+ eob = 0;
+ return eob;
+}
+#endif
+
+void vp8_fast_quantize_b_ssse3(BLOCK *b, BLOCKD *d) {
+ int eob, mask;
+
+ __m128i z0 = _mm_load_si128((__m128i *)(b->coeff));
+ __m128i z1 = _mm_load_si128((__m128i *)(b->coeff + 8));
+ __m128i round0 = _mm_load_si128((__m128i *)(b->round));
+ __m128i round1 = _mm_load_si128((__m128i *)(b->round + 8));
+ __m128i quant_fast0 = _mm_load_si128((__m128i *)(b->quant_fast));
+ __m128i quant_fast1 = _mm_load_si128((__m128i *)(b->quant_fast + 8));
+ __m128i dequant0 = _mm_load_si128((__m128i *)(d->dequant));
+ __m128i dequant1 = _mm_load_si128((__m128i *)(d->dequant + 8));
+
+ __m128i sz0, sz1, x, x0, x1, y0, y1, zeros, abs0, abs1;
+
+ DECLARE_ALIGNED(16, const uint8_t, pshufb_zig_zag_mask[16]) =
+ { 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 };
+ __m128i zig_zag = _mm_load_si128((const __m128i *)pshufb_zig_zag_mask);
+
+ /* sign of z: z >> 15 */
+ sz0 = _mm_srai_epi16(z0, 15);
+ sz1 = _mm_srai_epi16(z1, 15);
+
+ /* x = abs(z) */
+ x0 = _mm_abs_epi16(z0);
+ x1 = _mm_abs_epi16(z1);
+
+ /* x += round */
+ x0 = _mm_add_epi16(x0, round0);
+ x1 = _mm_add_epi16(x1, round1);
+
+ /* y = (x * quant) >> 16 */
+ y0 = _mm_mulhi_epi16(x0, quant_fast0);
+ y1 = _mm_mulhi_epi16(x1, quant_fast1);
+
+ /* ASM saves Y for EOB */
+ /* I think we can ignore that because adding the sign doesn't change anything
+ * and multiplying 0 by dequant is OK as well */
+ abs0 = y0;
+ abs1 = y1;
+
+ /* Restore the sign bit. */
+ y0 = _mm_xor_si128(y0, sz0);
+ y1 = _mm_xor_si128(y1, sz1);
+ x0 = _mm_sub_epi16(y0, sz0);
+ x1 = _mm_sub_epi16(y1, sz1);
+
+ /* qcoeff = x */
+ _mm_store_si128((__m128i *)(d->qcoeff), x0);
+ _mm_store_si128((__m128i *)(d->qcoeff + 8), x1);
+
+ /* x * dequant */
+ x0 = _mm_mullo_epi16(x0, dequant0);
+ x1 = _mm_mullo_epi16(x1, dequant1);
+
+ /* dqcoeff = x * dequant */
+ _mm_store_si128((__m128i *)(d->dqcoeff), x0);
+ _mm_store_si128((__m128i *)(d->dqcoeff + 8), x1);
+
+ zeros = _mm_setzero_si128();
+
+ x0 = _mm_cmpgt_epi16(abs0, zeros);
+ x1 = _mm_cmpgt_epi16(abs1, zeros);
+
+ x = _mm_packs_epi16(x0, x1);
+
+ x = _mm_shuffle_epi8(x, zig_zag);
+
+ mask = _mm_movemask_epi8(x);
+
+ eob = bsr(mask);
+
+ *d->eob = 0xFF & eob;
+}
diff --git a/vp8/vp8_common.mk b/vp8/vp8_common.mk
index cdb2716..52e35f2 100644
--- a/vp8/vp8_common.mk
+++ b/vp8/vp8_common.mk
@@ -159,22 +159,8 @@
VP8_COMMON_SRCS-$(HAVE_MEDIA) += common/arm/armv6/vp8_variance_halfpixvar16x16_hv_armv6$(ASM)
# common (neon)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/iwalsh_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/loopfilter_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/loopfiltersimplehorizontaledge_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/loopfiltersimpleverticaledge_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/mbloopfilter_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/shortidct4x4llm_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sad8_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sad16_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sixtappredict4x4_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sixtappredict8x4_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sixtappredict8x8_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sixtappredict16x16_neon$(ASM)
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/buildintrapredictorsmby_neon$(ASM)
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/idct_dequant_0_2x_neon$(ASM)
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/idct_blk_neon.c
-VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/variance_neon$(ASM)
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp8_subpixelvariance8x8_neon$(ASM)
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp8_subpixelvariance16x16_neon$(ASM)
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp8_subpixelvariance16x16s_neon$(ASM)
@@ -186,6 +172,15 @@
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/dequant_idct_neon.c
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/dequantizeb_neon.c
VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/idct_dequant_full_2x_neon.c
-
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/iwalsh_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/loopfilter_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/loopfiltersimplehorizontaledge_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/loopfiltersimpleverticaledge_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/mbloopfilter_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sad_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/shortidct4x4llm_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/sixtappredict_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/idct_dequant_0_2x_neon.c
+VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/variance_neon.c
$(eval $(call rtcd_h_template,vp8_rtcd,vp8/common/rtcd_defs.pl))
diff --git a/vp8/vp8cx.mk b/vp8/vp8cx.mk
index d7c6dd1..607382b 100644
--- a/vp8/vp8cx.mk
+++ b/vp8/vp8cx.mk
@@ -88,6 +88,7 @@
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/dct_sse2.asm
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/fwalsh_sse2.asm
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/quantize_sse2.c
+VP8_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/quantize_ssse3.c
ifeq ($(CONFIG_TEMPORAL_DENOISING),yes)
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/denoising_sse2.c
@@ -96,7 +97,6 @@
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/subtract_sse2.asm
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/temporal_filter_apply_sse2.asm
VP8_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp8_enc_stubs_sse2.c
-VP8_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/quantize_ssse3.asm
VP8_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/quantize_sse4.asm
VP8_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64) += encoder/x86/quantize_mmx.asm
VP8_CX_SRCS-$(ARCH_X86)$(ARCH_X86_64) += encoder/x86/encodeopt.asm
diff --git a/vp9/common/vp9_rtcd_defs.pl b/vp9/common/vp9_rtcd_defs.pl
index d4c3065..de69772 100644
--- a/vp9/common/vp9_rtcd_defs.pl
+++ b/vp9/common/vp9_rtcd_defs.pl
@@ -310,7 +310,7 @@
specialize qw/vp9_idct8x8_1_add sse2 neon dspr2/;
add_proto qw/void vp9_idct8x8_64_add/, "const int16_t *input, uint8_t *dest, int dest_stride";
-specialize qw/vp9_idct8x8_64_add sse2 neon dspr2/;
+specialize qw/vp9_idct8x8_64_add sse2 neon dspr2/, "$ssse3_x86_64";
add_proto qw/void vp9_idct8x8_10_add/, "const int16_t *input, uint8_t *dest, int dest_stride";
specialize qw/vp9_idct8x8_10_add sse2 neon dspr2/;
@@ -379,10 +379,6 @@
add_proto qw/unsigned int vp9_variance16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vp9_variance16x16 mmx/, "$sse2_x86inc", "$avx2_x86inc";
-add_proto qw/void vp9_get_sse_sum_16x16/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum";
-specialize qw/vp9_get_sse_sum_16x16 sse2/;
-$vp9_get_sse_sum_16x16_sse2=vp9_get16x16var_sse2;
-
add_proto qw/unsigned int vp9_variance16x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vp9_variance16x8 mmx/, "$sse2_x86inc";
@@ -392,10 +388,6 @@
add_proto qw/unsigned int vp9_variance8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vp9_variance8x8 mmx/, "$sse2_x86inc";
-add_proto qw/void vp9_get_sse_sum_8x8/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum";
-specialize qw/vp9_get_sse_sum_8x8 sse2/;
-$vp9_get_sse_sum_8x8_sse2=vp9_get8x8var_sse2;
-
add_proto qw/unsigned int vp9_variance8x4/, "const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse";
specialize qw/vp9_variance8x4/, "$sse2_x86inc";
@@ -701,7 +693,7 @@
specialize qw/vp9_fht16x16 sse2 avx2/;
add_proto qw/void vp9_fwht4x4/, "const int16_t *input, int16_t *output, int stride";
-specialize qw/vp9_fwht4x4/;
+specialize qw/vp9_fwht4x4 mmx/;
add_proto qw/void vp9_fdct4x4/, "const int16_t *input, int16_t *output, int stride";
specialize qw/vp9_fdct4x4 sse2 avx2/;
diff --git a/vp9/common/x86/vp9_idct_ssse3.asm b/vp9/common/x86/vp9_idct_ssse3.asm
new file mode 100644
index 0000000..f2a120f
--- /dev/null
+++ b/vp9/common/x86/vp9_idct_ssse3.asm
@@ -0,0 +1,162 @@
+;
+; Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+;
+; Use of this source code is governed by a BSD-style license
+; that can be found in the LICENSE file in the root of the source
+; tree. An additional intellectual property rights grant can be found
+; in the file PATENTS. All contributing project authors may
+; be found in the AUTHORS file in the root of the source tree.
+;
+%include "third_party/x86inc/x86inc.asm"
+
+; This file provides SSSE3 version of the inverse transformation. Part
+; of the functions are originally derived from the ffmpeg project.
+; Note that the current version applies to x86 64-bit only.
+
+SECTION_RODATA
+
+pw_11585x2: times 8 dw 23170
+pd_8192: times 4 dd 8192
+pw_16: times 8 dw 16
+
+%macro TRANSFORM_COEFFS 2
+pw_%1_%2: dw %1, %2, %1, %2, %1, %2, %1, %2
+pw_m%2_%1: dw -%2, %1, -%2, %1, -%2, %1, -%2, %1
+%endmacro
+
+TRANSFORM_COEFFS 6270, 15137
+TRANSFORM_COEFFS 3196, 16069
+TRANSFORM_COEFFS 13623, 9102
+
+SECTION .text
+
+%if ARCH_X86_64
+%macro SUM_SUB 3
+ psubw m%3, m%1, m%2
+ paddw m%1, m%2
+ SWAP %2, %3
+%endmacro
+
+; butterfly operation
+%macro MUL_ADD_2X 6 ; dst1, dst2, src, round, coefs1, coefs2
+ pmaddwd m%1, m%3, %5
+ pmaddwd m%2, m%3, %6
+ paddd m%1, %4
+ paddd m%2, %4
+ psrad m%1, 14
+ psrad m%2, 14
+%endmacro
+
+%macro BUTTERFLY_4X 7 ; dst1, dst2, coef1, coef2, round, tmp1, tmp2
+ punpckhwd m%6, m%2, m%1
+ MUL_ADD_2X %7, %6, %6, %5, [pw_m%4_%3], [pw_%3_%4]
+ punpcklwd m%2, m%1
+ MUL_ADD_2X %1, %2, %2, %5, [pw_m%4_%3], [pw_%3_%4]
+ packssdw m%1, m%7
+ packssdw m%2, m%6
+%endmacro
+
+; matrix transpose
+%macro INTERLEAVE_2X 4
+ punpckh%1 m%4, m%2, m%3
+ punpckl%1 m%2, m%3
+ SWAP %3, %4
+%endmacro
+
+%macro TRANSPOSE8X8 9
+ INTERLEAVE_2X wd, %1, %2, %9
+ INTERLEAVE_2X wd, %3, %4, %9
+ INTERLEAVE_2X wd, %5, %6, %9
+ INTERLEAVE_2X wd, %7, %8, %9
+
+ INTERLEAVE_2X dq, %1, %3, %9
+ INTERLEAVE_2X dq, %2, %4, %9
+ INTERLEAVE_2X dq, %5, %7, %9
+ INTERLEAVE_2X dq, %6, %8, %9
+
+ INTERLEAVE_2X qdq, %1, %5, %9
+ INTERLEAVE_2X qdq, %3, %7, %9
+ INTERLEAVE_2X qdq, %2, %6, %9
+ INTERLEAVE_2X qdq, %4, %8, %9
+
+ SWAP %2, %5
+ SWAP %4, %7
+%endmacro
+
+%macro IDCT8_1D 0
+ SUM_SUB 0, 4, 9
+ BUTTERFLY_4X 2, 6, 6270, 15137, m8, 9, 10
+ pmulhrsw m0, m12
+ pmulhrsw m4, m12
+ BUTTERFLY_4X 1, 7, 3196, 16069, m8, 9, 10
+ BUTTERFLY_4X 5, 3, 13623, 9102, m8, 9, 10
+
+ SUM_SUB 1, 5, 9
+ SUM_SUB 7, 3, 9
+ SUM_SUB 0, 6, 9
+ SUM_SUB 4, 2, 9
+ SUM_SUB 3, 5, 9
+ pmulhrsw m3, m12
+ pmulhrsw m5, m12
+
+ SUM_SUB 0, 7, 9
+ SUM_SUB 4, 3, 9
+ SUM_SUB 2, 5, 9
+ SUM_SUB 6, 1, 9
+
+ SWAP 3, 6
+ SWAP 1, 4
+%endmacro
+
+; This macro handles 8 pixels per line
+%macro ADD_STORE_8P_2X 5; src1, src2, tmp1, tmp2, zero
+ paddw m%1, m11
+ paddw m%2, m11
+ psraw m%1, 5
+ psraw m%2, 5
+
+ movh m%3, [outputq]
+ movh m%4, [outputq + strideq]
+ punpcklbw m%3, m%5
+ punpcklbw m%4, m%5
+ paddw m%3, m%1
+ paddw m%4, m%2
+ packuswb m%3, m%5
+ packuswb m%4, m%5
+ movh [outputq], m%3
+ movh [outputq + strideq], m%4
+%endmacro
+
+INIT_XMM ssse3
+cglobal idct8x8_64_add, 3, 5, 13, input, output, stride
+ mova m8, [pd_8192]
+ mova m11, [pw_16]
+ mova m12, [pw_11585x2]
+
+ lea r3, [2 * strideq]
+
+ mova m0, [inputq + 0]
+ mova m1, [inputq + 16]
+ mova m2, [inputq + 32]
+ mova m3, [inputq + 48]
+ mova m4, [inputq + 64]
+ mova m5, [inputq + 80]
+ mova m6, [inputq + 96]
+ mova m7, [inputq + 112]
+
+ TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9
+ IDCT8_1D
+ TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9
+ IDCT8_1D
+
+ pxor m12, m12
+ ADD_STORE_8P_2X 0, 1, 9, 10, 12
+ lea outputq, [outputq + r3]
+ ADD_STORE_8P_2X 2, 3, 9, 10, 12
+ lea outputq, [outputq + r3]
+ ADD_STORE_8P_2X 4, 5, 9, 10, 12
+ lea outputq, [outputq + r3]
+ ADD_STORE_8P_2X 6, 7, 9, 10, 12
+
+ RET
+%endif
diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index c5a85c9..35d2ecf 100644
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -1006,9 +1006,10 @@
found = cm->width == cfg->y_crop_width &&
cm->height == cfg->y_crop_height;
- // TODO(ivan): This prevents a bug while more than 3 buffers are used. Do it
- // in a better way.
- if (cpi->use_svc) {
+ // Set "found" to 0 for temporal svc and for spatial svc key frame
+ if (cpi->use_svc &&
+ (cpi->svc.number_spatial_layers == 1 ||
+ cpi->svc.layer_context[cpi->svc.spatial_layer_id].is_key_frame)) {
found = 0;
}
vp9_wb_write_bit(wb, found);
diff --git a/vp9/encoder/vp9_block.h b/vp9/encoder/vp9_block.h
index f35a85f..2ccf4f8 100644
--- a/vp9/encoder/vp9_block.h
+++ b/vp9/encoder/vp9_block.h
@@ -129,12 +129,6 @@
int *nmvsadcost_hp[2];
int **mvsadcost;
- int mbmode_cost[INTRA_MODES];
- unsigned inter_mode_cost[INTER_MODE_CONTEXTS][INTER_MODES];
- int intra_uv_mode_cost[FRAME_TYPES][INTRA_MODES];
- int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
- int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
-
// These define limits to motion vector components to prevent them
// from extending outside the UMV borders
int mv_col_min;
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 19aa592..ef1d4e6 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -70,6 +70,18 @@
128, 128, 128, 128, 128, 128, 128, 128
};
+static void get_sse_sum_8x8(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ unsigned int *sse, int *sum) {
+ variance(src, src_stride, ref, ref_stride, 8, 8, sse, sum);
+}
+
+static void get_sse_sum_16x16(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
+ unsigned int *sse, int *sum) {
+ variance(src, src_stride, ref, ref_stride, 16, 16, sse, sum);
+}
+
static unsigned int get_sby_perpixel_variance(VP9_COMP *cpi,
const struct buf_2d *ref,
BLOCK_SIZE bs) {
@@ -475,8 +487,8 @@
unsigned int sse = 0;
int sum = 0;
if (x_idx < pixels_wide && y_idx < pixels_high)
- vp9_get_sse_sum_8x8(s + y_idx * sp + x_idx, sp,
- d + y_idx * dp + x_idx, dp, &sse, &sum);
+ get_sse_sum_8x8(s + y_idx * sp + x_idx, sp,
+ d + y_idx * dp + x_idx, dp, &sse, &sum);
fill_variance(sse, sum, 64, &vst->split[k].part_variances.none);
}
}
@@ -1211,10 +1223,9 @@
int b_offset = b_mi_row * MI_SIZE * src_stride +
b_mi_col * MI_SIZE;
- vp9_get_sse_sum_16x16(src + b_offset,
- src_stride,
- pre_src + b_offset,
- pre_stride, &d16[j].sse, &d16[j].sum);
+ get_sse_sum_16x16(src + b_offset, src_stride,
+ pre_src + b_offset, pre_stride,
+ &d16[j].sse, &d16[j].sum);
d16[j].var = d16[j].sse -
(((uint32_t)d16[j].sum * d16[j].sum) >> 8);
diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h
index 364ea3a..8732095 100644
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -499,6 +499,12 @@
search_site_config ss_cfg;
+ int mbmode_cost[INTRA_MODES];
+ unsigned inter_mode_cost[INTER_MODE_CONTEXTS][INTER_MODES];
+ int intra_uv_mode_cost[FRAME_TYPES][INTRA_MODES];
+ int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
+ int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
+
#if CONFIG_MULTIPLE_ARF
// ARF tracking variables.
int multi_arf_enabled;
diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c
index bbec4da..15afa1a 100644
--- a/vp9/encoder/vp9_mcomp.c
+++ b/vp9/encoder/vp9_mcomp.c
@@ -878,89 +878,67 @@
const vp9_variance_fn_ptr_t *fn_ptr,
const MV *center_mv) {
const MACROBLOCKD *const xd = &x->e_mbd;
- const uint8_t *what = x->plane[0].src.buf;
- const int what_stride = x->plane[0].src.stride;
- const uint8_t *in_what;
- const int in_what_stride = xd->plane[0].pre[0].stride;
-
- unsigned int bestsad = INT_MAX;
- int ref_row, ref_col;
-
- unsigned int thissad;
+ const struct buf_2d *const what = &x->plane[0].src;
+ const struct buf_2d *const in_what = &xd->plane[0].pre[0];
+ const int range = 64;
const MV fcenter_mv = {center_mv->row >> 3, center_mv->col >> 3};
- int tr, tc;
- int best_tr = 0;
- int best_tc = 0;
- int range = 64;
-
- int start_col, end_col;
- int start_row, end_row;
- int i;
+ unsigned int best_sad = INT_MAX;
+ int r, c, i;
+ int start_col, end_col, start_row, end_row;
clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max);
- ref_row = ref_mv->row;
- ref_col = ref_mv->col;
+ *best_mv = *ref_mv;
*num00 = 11;
- best_mv->row = ref_row;
- best_mv->col = ref_col;
+ best_sad = fn_ptr->sdf(what->buf, what->stride,
+ get_buf_from_mv(in_what, ref_mv), in_what->stride,
+ 0x7fffffff) +
+ mvsad_err_cost(x, ref_mv, &fcenter_mv, sad_per_bit);
+ start_row = MAX(-range, x->mv_row_min - ref_mv->row);
+ start_col = MAX(-range, x->mv_col_min - ref_mv->col);
+ end_row = MIN(range, x->mv_row_max - ref_mv->row);
+ end_col = MIN(range, x->mv_col_max - ref_mv->col);
- // Work out the start point for the search
- in_what = xd->plane[0].pre[0].buf + ref_row * in_what_stride + ref_col;
+ for (r = start_row; r <= end_row; ++r) {
+ for (c = start_col; c <= end_col; c += 4) {
+ if (c + 3 <= end_col) {
+ unsigned int sads[4];
+ const uint8_t *addrs[4];
+ for (i = 0; i < 4; ++i) {
+ const MV mv = {ref_mv->row + r, ref_mv->col + c + i};
+ addrs[i] = get_buf_from_mv(in_what, &mv);
+ }
- // Check the starting position
- bestsad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride, 0x7fffffff)
- + mvsad_err_cost(x, best_mv, &fcenter_mv, sad_per_bit);
-
- start_row = MAX(-range, x->mv_row_min - ref_row);
- start_col = MAX(-range, x->mv_col_min - ref_col);
- end_row = MIN(range, x->mv_row_max - ref_row);
- end_col = MIN(range, x->mv_col_max - ref_col);
-
- for (tr = start_row; tr <= end_row; ++tr) {
- for (tc = start_col; tc <= end_col; tc += 4) {
- if ((tc + 3) <= end_col) {
- unsigned int sad_array[4];
- unsigned char const *addr_ref[4];
- for (i = 0; i < 4; ++i)
- addr_ref[i] = in_what + tr * in_what_stride + tc + i;
-
- fn_ptr->sdx4df(what, what_stride, addr_ref, in_what_stride, sad_array);
+ fn_ptr->sdx4df(what->buf, what->stride, addrs, in_what->stride, sads);
for (i = 0; i < 4; ++i) {
- if (sad_array[i] < bestsad) {
- const MV this_mv = {ref_row + tr, ref_col + tc + i};
- thissad = sad_array[i] +
- mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit);
- if (thissad < bestsad) {
- bestsad = thissad;
- best_tr = tr;
- best_tc = tc + i;
+ if (sads[i] < best_sad) {
+ const MV mv = {ref_mv->row + r, ref_mv->col + c + i};
+ const unsigned int sad = sads[i] +
+ mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit);
+ if (sad < best_sad) {
+ best_sad = sad;
+ *best_mv = mv;
}
}
}
} else {
- for (i = 0; i < end_col - tc; ++i) {
- const uint8_t *check_here = in_what + tr * in_what_stride + tc + i;
- thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride,
- bestsad);
-
- if (thissad < bestsad) {
- const MV this_mv = {ref_row + tr, ref_col + tc + i};
- thissad += mvsad_err_cost(x, &this_mv, &fcenter_mv, sad_per_bit);
-
- if (thissad < bestsad) {
- bestsad = thissad;
- best_tr = tr;
- best_tc = tc + i;
+ for (i = 0; i < end_col - c; ++i) {
+ const MV mv = {ref_mv->row + r, ref_mv->col + c + i};
+ unsigned int sad = fn_ptr->sdf(what->buf, what->stride,
+ get_buf_from_mv(in_what, &mv), in_what->stride, best_sad);
+ if (sad < best_sad) {
+ sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit);
+ if (sad < best_sad) {
+ best_sad = sad;
+ *best_mv = mv;
}
}
}
}
}
}
- best_mv->row += best_tr;
- best_mv->col += best_tc;
- return bestsad;
+
+ return best_sad;
}
int vp9_diamond_search_sad_c(const MACROBLOCK *x,
diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 56eb944..df520f2 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -320,8 +320,8 @@
if (frame_mv[NEWMV][ref_frame].as_int == INVALID_MV)
continue;
- rate_mode = x->inter_mode_cost[mbmi->mode_context[ref_frame]]
- [INTER_OFFSET(this_mode)];
+ rate_mode = cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
+ [INTER_OFFSET(this_mode)];
if (RDCOST(x->rdmult, x->rddiv, rate_mv + rate_mode, 0) > best_rd)
continue;
@@ -355,24 +355,24 @@
model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP],
&pf_dist[EIGHTTAP]);
tmp_rdcost1 = RDCOST(x->rdmult, x->rddiv,
- vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP],
+ vp9_get_switchable_rate(cpi) + pf_rate[EIGHTTAP],
pf_dist[EIGHTTAP]);
mbmi->interp_filter = EIGHTTAP_SHARP;
vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP_SHARP],
&pf_dist[EIGHTTAP_SHARP]);
- tmp_rdcost2 = RDCOST(x->rdmult, x->rddiv,
- vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP_SHARP],
- pf_dist[EIGHTTAP_SHARP]);
+ tmp_rdcost2 = RDCOST(x->rdmult, x->rddiv, vp9_get_switchable_rate(cpi) +
+ pf_rate[EIGHTTAP_SHARP],
+ pf_dist[EIGHTTAP_SHARP]);
mbmi->interp_filter = EIGHTTAP_SMOOTH;
vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP_SMOOTH],
&pf_dist[EIGHTTAP_SMOOTH]);
- tmp_rdcost3 = RDCOST(x->rdmult, x->rddiv,
- vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP_SMOOTH],
- pf_dist[EIGHTTAP_SMOOTH]);
+ tmp_rdcost3 = RDCOST(x->rdmult, x->rddiv, vp9_get_switchable_rate(cpi) +
+ pf_rate[EIGHTTAP_SMOOTH],
+ pf_dist[EIGHTTAP_SMOOTH]);
if (tmp_rdcost2 < tmp_rdcost1) {
if (tmp_rdcost2 < tmp_rdcost3)
@@ -395,7 +395,7 @@
}
rate += rate_mv;
- rate += x->inter_mode_cost[mbmi->mode_context[ref_frame]]
+ rate += cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
[INTER_OFFSET(this_mode)];
this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
@@ -426,7 +426,7 @@
&pd->dst.buf[0], pd->dst.stride, 0, 0, 0);
model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist);
- rate += x->mbmode_cost[this_mode];
+ rate += cpi->mbmode_cost[this_mode];
rate += intra_cost_penalty;
this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c
index 24e75ae..985743d 100644
--- a/vp9/encoder/vp9_ratectrl.c
+++ b/vp9/encoder/vp9_ratectrl.c
@@ -1308,11 +1308,26 @@
cpi->oxcf.key_freq == 0))) {
cm->frame_type = KEY_FRAME;
rc->source_alt_ref_active = 0;
+
+ if (cpi->use_svc && cpi->svc.number_temporal_layers == 1) {
+ cpi->svc.layer_context[cpi->svc.spatial_layer_id].is_key_frame = 1;
+ }
+
if (cpi->pass == 0 && cpi->oxcf.rc_mode == RC_MODE_CBR) {
target = calc_iframe_target_size_one_pass_cbr(cpi);
}
} else {
cm->frame_type = INTER_FRAME;
+
+ if (cpi->use_svc && cpi->svc.number_temporal_layers == 1) {
+ LAYER_CONTEXT *lc = &cpi->svc.layer_context[cpi->svc.spatial_layer_id];
+ if (cpi->svc.spatial_layer_id == 0) {
+ lc->is_key_frame = 0;
+ } else {
+ lc->is_key_frame = cpi->svc.layer_context[0].is_key_frame;
+ }
+ }
+
if (cpi->pass == 0 && cpi->oxcf.rc_mode == RC_MODE_CBR) {
target = calc_pframe_target_size_one_pass_cbr(cpi);
}
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index f854356..857015e 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -152,24 +152,23 @@
}
static void fill_mode_costs(VP9_COMP *cpi) {
- MACROBLOCK *const x = &cpi->mb;
const FRAME_CONTEXT *const fc = &cpi->common.fc;
int i, j;
for (i = 0; i < INTRA_MODES; i++)
for (j = 0; j < INTRA_MODES; j++)
- vp9_cost_tokens((int *)x->y_mode_costs[i][j], vp9_kf_y_mode_prob[i][j],
+ vp9_cost_tokens(cpi->y_mode_costs[i][j], vp9_kf_y_mode_prob[i][j],
vp9_intra_mode_tree);
// TODO(rbultje) separate tables for superblock costing?
- vp9_cost_tokens(x->mbmode_cost, fc->y_mode_prob[1], vp9_intra_mode_tree);
- vp9_cost_tokens(x->intra_uv_mode_cost[KEY_FRAME],
+ vp9_cost_tokens(cpi->mbmode_cost, fc->y_mode_prob[1], vp9_intra_mode_tree);
+ vp9_cost_tokens(cpi->intra_uv_mode_cost[KEY_FRAME],
vp9_kf_uv_mode_prob[TM_PRED], vp9_intra_mode_tree);
- vp9_cost_tokens(x->intra_uv_mode_cost[INTER_FRAME],
+ vp9_cost_tokens(cpi->intra_uv_mode_cost[INTER_FRAME],
fc->uv_mode_prob[TM_PRED], vp9_intra_mode_tree);
for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
- vp9_cost_tokens((int *)x->switchable_interp_costs[i],
+ vp9_cost_tokens(cpi->switchable_interp_costs[i],
fc->switchable_interp_prob[i], vp9_switchable_interp_tree);
}
@@ -313,7 +312,7 @@
&cm->fc.nmvc, cm->allow_high_precision_mv);
for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
- vp9_cost_tokens((int *)x->inter_mode_cost[i],
+ vp9_cost_tokens((int *)cpi->inter_mode_cost[i],
cm->fc.inter_mode_probs[i], vp9_inter_mode_tree);
}
}
@@ -1187,7 +1186,7 @@
int tot_rate_y = 0;
int64_t total_rd = 0;
ENTROPY_CONTEXT t_above[4], t_left[4];
- const int *bmode_costs = mb->mbmode_cost;
+ const int *bmode_costs = cpi->mbmode_cost;
vpx_memcpy(t_above, xd->plane[0].above_context, sizeof(t_above));
vpx_memcpy(t_left, xd->plane[0].left_context, sizeof(t_left));
@@ -1203,7 +1202,7 @@
const PREDICTION_MODE A = vp9_above_block_mode(mic, above_mi, i);
const PREDICTION_MODE L = vp9_left_block_mode(mic, left_mi, i);
- bmode_costs = mb->y_mode_costs[A][L];
+ bmode_costs = cpi->y_mode_costs[A][L];
}
this_rd = rd_pick_intra4x4block(cpi, mb, i, &best_mode, bmode_costs,
@@ -1250,7 +1249,7 @@
int64_t this_distortion, this_rd;
TX_SIZE best_tx = TX_4X4;
int i;
- int *bmode_costs = x->mbmode_cost;
+ int *bmode_costs = cpi->mbmode_cost;
if (cpi->sf.tx_size_search_method == USE_FULL_RD)
for (i = 0; i < TX_MODES; i++)
@@ -1269,7 +1268,7 @@
const PREDICTION_MODE A = vp9_above_block_mode(mic, above_mi, 0);
const PREDICTION_MODE L = vp9_left_block_mode(mic, left_mi, 0);
- bmode_costs = x->y_mode_costs[A][L];
+ bmode_costs = cpi->y_mode_costs[A][L];
}
mic->mbmi.mode = mode;
@@ -1378,7 +1377,7 @@
if (this_rate_tokenonly == INT_MAX)
continue;
this_rate = this_rate_tokenonly +
- x->intra_uv_mode_cost[cpi->common.frame_type][mode];
+ cpi->intra_uv_mode_cost[cpi->common.frame_type][mode];
this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);
if (this_rd < best_rd) {
@@ -1426,7 +1425,7 @@
x->e_mbd.mi[0]->mbmi.uv_mode = DC_PRED;
super_block_uvrd(cpi, x, rate_tokenonly, distortion,
skippable, &unused, bsize, INT64_MAX);
- *rate = *rate_tokenonly + x->intra_uv_mode_cost[cm->frame_type][DC_PRED];
+ *rate = *rate_tokenonly + cpi->intra_uv_mode_cost[cm->frame_type][DC_PRED];
return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
}
@@ -1460,7 +1459,7 @@
// Don't account for mode here if segment skip is enabled.
if (!vp9_segfeature_active(&cpi->common.seg, segment_id, SEG_LVL_SKIP)) {
assert(is_inter_mode(mode));
- return x->inter_mode_cost[mode_context][INTER_OFFSET(mode)];
+ return cpi->inter_mode_cost[mode_context][INTER_OFFSET(mode)];
} else {
return 0;
}
@@ -2334,12 +2333,12 @@
return (scaled_idx != ref_idx) ? &cm->frame_bufs[scaled_idx].buf : NULL;
}
-int vp9_get_switchable_rate(const MACROBLOCK *x) {
- const MACROBLOCKD *const xd = &x->e_mbd;
+int vp9_get_switchable_rate(const VP9_COMP *cpi) {
+ const MACROBLOCKD *const xd = &cpi->mb.e_mbd;
const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const int ctx = vp9_get_pred_context_switchable_interp(xd);
return SWITCHABLE_INTERP_RATE_FACTOR *
- x->switchable_interp_costs[ctx][mbmi->interp_filter];
+ cpi->switchable_interp_costs[ctx][mbmi->interp_filter];
}
static void single_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
@@ -2788,7 +2787,7 @@
int j;
int64_t rs_rd;
mbmi->interp_filter = i;
- rs = vp9_get_switchable_rate(x);
+ rs = vp9_get_switchable_rate(cpi);
rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
if (i > 0 && intpel_mv) {
@@ -2858,7 +2857,7 @@
// Set the appropriate filter
mbmi->interp_filter = cm->interp_filter != SWITCHABLE ?
cm->interp_filter : *best_filter;
- rs = cm->interp_filter == SWITCHABLE ? vp9_get_switchable_rate(x) : 0;
+ rs = cm->interp_filter == SWITCHABLE ? vp9_get_switchable_rate(cpi) : 0;
if (pred_exists) {
if (best_needs_copy) {
@@ -2888,7 +2887,7 @@
}
if (cm->interp_filter == SWITCHABLE)
- *rate2 += vp9_get_switchable_rate(x);
+ *rate2 += vp9_get_switchable_rate(cpi);
if (!is_comp_pred) {
if (!x->in_active_map) {
@@ -3435,7 +3434,7 @@
skippable = skippable && skip_uv[uv_tx];
mbmi->uv_mode = mode_uv[uv_tx];
- rate2 = rate_y + x->mbmode_cost[mbmi->mode] + rate_uv_intra[uv_tx];
+ rate2 = rate_y + cpi->mbmode_cost[mbmi->mode] + rate_uv_intra[uv_tx];
if (this_mode != DC_PRED && this_mode != TM_PRED)
rate2 += intra_cost_penalty;
distortion2 = distortion_y + distortion_uv;
@@ -4034,7 +4033,7 @@
if (tmp_rd == INT64_MAX)
continue;
- rs = vp9_get_switchable_rate(x);
+ rs = vp9_get_switchable_rate(cpi);
rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
rd_opt->filter_cache[switchable_filter_index] = tmp_rd;
rd_opt->filter_cache[SWITCHABLE_FILTERS] =
@@ -4112,7 +4111,7 @@
distortion2 += distortion;
if (cm->interp_filter == SWITCHABLE)
- rate2 += vp9_get_switchable_rate(x);
+ rate2 += vp9_get_switchable_rate(cpi);
if (!mode_excluded)
mode_excluded = comp_pred ? cm->reference_mode == SINGLE_REFERENCE
diff --git a/vp9/encoder/vp9_rdopt.h b/vp9/encoder/vp9_rdopt.h
index cd622d6..d016ebd 100644
--- a/vp9/encoder/vp9_rdopt.h
+++ b/vp9/encoder/vp9_rdopt.h
@@ -40,7 +40,7 @@
unsigned int qstep, int *rate,
int64_t *dist);
-int vp9_get_switchable_rate(const MACROBLOCK *x);
+int vp9_get_switchable_rate(const VP9_COMP *cpi);
void vp9_setup_buffer_inter(VP9_COMP *cpi, MACROBLOCK *x,
const TileInfo *const tile,
diff --git a/vp9/encoder/vp9_temporal_filter.c b/vp9/encoder/vp9_temporal_filter.c
index a5234cd..6eff200 100644
--- a/vp9/encoder/vp9_temporal_filter.c
+++ b/vp9/encoder/vp9_temporal_filter.c
@@ -99,6 +99,7 @@
unsigned int i, j, k;
int modifier;
int byte = 0;
+ const int rounding = strength > 0 ? 1 << (strength - 1) : 0;
for (i = 0, k = 0; i < block_size; i++) {
for (j = 0; j < block_size; j++, k++) {
@@ -111,7 +112,7 @@
// modifier = (int)roundf(coeff > 16 ? 0 : 16-coeff);
modifier *= modifier;
modifier *= 3;
- modifier += 1 << (strength - 1);
+ modifier += rounding;
modifier >>= strength;
if (modifier > 16)
diff --git a/vp9/encoder/vp9_variance.c b/vp9/encoder/vp9_variance.c
index ae3c86a..91d8ea4 100644
--- a/vp9/encoder/vp9_variance.c
+++ b/vp9/encoder/vp9_variance.c
@@ -113,12 +113,9 @@
unsigned int vp9_variance##W##x##H##_c(const uint8_t *a, int a_stride, \
const uint8_t *b, int b_stride, \
unsigned int *sse) { \
- unsigned int var; \
- int avg; \
-\
- variance(a, a_stride, b, b_stride, W, H, &var, &avg); \
- *sse = var; \
- return var - (((int64_t)avg * avg) / (W * H)); \
+ int sum; \
+ variance(a, a_stride, b, b_stride, W, H, sse, &sum); \
+ return *sse - (((int64_t)sum * sum) / (W * H)); \
}
#define SUBPIX_VAR(W, H) \
@@ -159,69 +156,36 @@
return vp9_variance##W##x##H##_c(temp3, W, dst, dst_stride, sse); \
}
-
-void vp9_get_sse_sum_16x16_c(const uint8_t *src_ptr, int source_stride,
- const uint8_t *ref_ptr, int ref_stride,
- unsigned int *sse, int *sum) {
- variance(src_ptr, source_stride, ref_ptr, ref_stride, 16, 16, sse, sum);
-}
-
-void vp9_get_sse_sum_8x8_c(const uint8_t *src_ptr, int source_stride,
- const uint8_t *ref_ptr, int ref_stride,
- unsigned int *sse, int *sum) {
- variance(src_ptr, source_stride, ref_ptr, ref_stride, 8, 8, sse, sum);
-}
-
-unsigned int vp9_mse16x16_c(const uint8_t *src_ptr,
- int source_stride,
- const uint8_t *ref_ptr,
- int recon_stride,
+unsigned int vp9_mse16x16_c(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
unsigned int *sse) {
- unsigned int var;
- int avg;
-
- variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg);
- *sse = var;
- return var;
+ int sum;
+ variance(src, src_stride, ref, ref_stride, 16, 16, sse, &sum);
+ return *sse;
}
-unsigned int vp9_mse16x8_c(const uint8_t *src_ptr,
- int source_stride,
- const uint8_t *ref_ptr,
- int recon_stride,
+unsigned int vp9_mse16x8_c(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
unsigned int *sse) {
- unsigned int var;
- int avg;
-
- variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg);
- *sse = var;
- return var;
+ int sum;
+ variance(src, src_stride, ref, ref_stride, 16, 8, sse, &sum);
+ return *sse;
}
-unsigned int vp9_mse8x16_c(const uint8_t *src_ptr,
- int source_stride,
- const uint8_t *ref_ptr,
- int recon_stride,
+unsigned int vp9_mse8x16_c(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
unsigned int *sse) {
- unsigned int var;
- int avg;
-
- variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg);
- *sse = var;
- return var;
+ int sum;
+ variance(src, src_stride, ref, ref_stride, 8, 16, sse, &sum);
+ return *sse;
}
-unsigned int vp9_mse8x8_c(const uint8_t *src_ptr,
- int source_stride,
- const uint8_t *ref_ptr,
- int recon_stride,
+unsigned int vp9_mse8x8_c(const uint8_t *src, int src_stride,
+ const uint8_t *ref, int ref_stride,
unsigned int *sse) {
- unsigned int var;
- int avg;
-
- variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg);
- *sse = var;
- return var;
+ int sum;
+ variance(src, src_stride, ref, ref_stride, 8, 8, sse, &sum);
+ return *sse;
}
VAR(4, 4)
diff --git a/vp9/encoder/x86/vp9_dct_mmx.c b/vp9/encoder/x86/vp9_dct_mmx.c
new file mode 100644
index 0000000..4524ba4
--- /dev/null
+++ b/vp9/encoder/x86/vp9_dct_mmx.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <mmintrin.h>
+#include <stdint.h>
+
+#include "./vpx_config.h"
+
+static void INLINE transpose_4x4_mmx(__m64* a, __m64* b, __m64* c, __m64* d) {
+ __m64 w, x, y, z;
+ w = _mm_unpacklo_pi16(*a, *b);
+ x = _mm_unpackhi_pi16(*a, *b);
+ y = _mm_unpacklo_pi16(*c, *d);
+ z = _mm_unpackhi_pi16(*c, *d);
+ *a = _mm_unpacklo_pi32(w, y);
+ *b = _mm_unpackhi_pi32(w, y);
+ *c = _mm_unpacklo_pi32(x, z);
+ *d = _mm_unpackhi_pi32(x, z);
+}
+
+static void INLINE fwht_4x4_cols(__m64* out0,
+ __m64* out1,
+ __m64* out2,
+ __m64* out3,
+ __m64 a1,
+ __m64 b1,
+ __m64 c1,
+ __m64 d1) {
+ __m64 e1;
+
+ a1 = _mm_add_pi16(a1, b1);
+ d1 = _mm_sub_pi16(d1, c1);
+ e1 = _mm_sub_pi16(a1, d1);
+ e1 = _mm_srai_pi16(e1, 1);
+ b1 = _mm_sub_pi16(e1, b1);
+ c1 = _mm_sub_pi16(e1, c1);
+ a1 = _mm_sub_pi16(a1, c1);
+ d1 = _mm_add_pi16(d1, b1);
+ *out0 = a1;
+ *out1 = c1;
+ *out2 = d1;
+ *out3 = b1;
+}
+
+void vp9_fwht4x4_mmx(const int16_t* input, int16_t* output, int stride) {
+ __m64 a1 = *(const __m64*)input;
+ __m64 b1 = *(const __m64*)(input + stride);
+ __m64 c1 = *(const __m64*)(input + 2 * stride);
+ __m64 d1 = *(const __m64*)(input + 3 * stride);
+
+ fwht_4x4_cols(&a1, &b1, &c1, &d1, a1, b1, c1, d1);
+ transpose_4x4_mmx(&a1, &b1, &c1, &d1);
+ fwht_4x4_cols(&a1, &b1, &c1, &d1, a1, b1, c1, d1);
+ transpose_4x4_mmx(&a1, &b1, &c1, &d1);
+
+ *(__m64*)output = _mm_slli_pi16(a1, 2);
+ *(__m64*)(output + 4) = _mm_slli_pi16(b1, 2);
+ *(__m64*)(output + 8) = _mm_slli_pi16(c1, 2);
+ *(__m64*)(output + 12) = _mm_slli_pi16(d1, 2);
+}
diff --git a/vp9/encoder/x86/vp9_dct_ssse3.asm b/vp9/encoder/x86/vp9_dct_ssse3.asm
index 1400071..8723a71 100644
--- a/vp9/encoder/x86/vp9_dct_ssse3.asm
+++ b/vp9/encoder/x86/vp9_dct_ssse3.asm
@@ -10,7 +10,7 @@
%include "third_party/x86inc/x86inc.asm"
; This file provides SSSE3 version of the forward transformation. Part
-; of the macro definitions are originally derived from ffmpeg project.
+; of the macro definitions are originally derived from the ffmpeg project.
; The current version applies to x86 64-bit only.
SECTION_RODATA
diff --git a/vp9/vp9_common.mk b/vp9/vp9_common.mk
index b1ba0b1..eaff60a 100644
--- a/vp9/vp9_common.mk
+++ b/vp9/vp9_common.mk
@@ -120,6 +120,10 @@
VP9_COMMON_SRCS-$(HAVE_SSE2) += common/x86/vp9_idct_intrin_sse2.c
+ifeq ($(ARCH_X86_64), yes)
+VP9_COMMON_SRCS-$(HAVE_SSSE3) += common/x86/vp9_idct_ssse3.asm
+endif
+
VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_convolve_neon.c
VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_idct16x16_neon.c
VP9_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/vp9_loopfilter_16_neon.c
diff --git a/vp9/vp9cx.mk b/vp9/vp9cx.mk
index 5e88793..154fdae 100644
--- a/vp9/vp9cx.mk
+++ b/vp9/vp9cx.mk
@@ -92,6 +92,7 @@
VP9_CX_SRCS-$(HAVE_MMX) += encoder/x86/vp9_variance_mmx.c
VP9_CX_SRCS-$(HAVE_MMX) += encoder/x86/vp9_variance_impl_mmx.asm
VP9_CX_SRCS-$(HAVE_MMX) += encoder/x86/vp9_sad_mmx.asm
+VP9_CX_SRCS-$(HAVE_MMX) += encoder/x86/vp9_dct_mmx.c
VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_variance_impl_sse2.asm
VP9_CX_SRCS-$(HAVE_AVX2) += encoder/x86/vp9_variance_impl_intrin_avx2.c
VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_sad4d_sse2.asm
diff --git a/vpx/src/svc_encodeframe.c b/vpx/src/svc_encodeframe.c
index 76aacd2..38c2d26 100644
--- a/vpx/src/svc_encodeframe.c
+++ b/vpx/src/svc_encodeframe.c
@@ -1000,8 +1000,10 @@
(int)si->frame_size, (int)pts);
}
}
- ++si->frame_within_gop;
- ++si->encode_frame_count;
+ if (rawimg != NULL) {
+ ++si->frame_within_gop;
+ ++si->encode_frame_count;
+ }
return VPX_CODEC_OK;
}
