aom_dsp/arm/intrapred_neon.c - aom - Git at Google

 /*
  * Copyright (c) 2016, Alliance for Open Media. All rights reserved
  *
  * This source code is subject to the terms of the BSD 2 Clause License and
  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
  * was not distributed with this source code in the LICENSE file, you can
  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
  * Media Patent License 1.0 was not distributed with this source code in the
  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
  */

 #include <arm_neon.h>

 #include "./aom_config.h"
 #include "./aom_dsp_rtcd.h"
 #include "aom/aom_integer.h"

 //------------------------------------------------------------------------------
 // DC 4x4

 // 'do_above' and 'do_left' facilitate branch removal when inlined.
 static INLINE void dc_4x4(uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
                           const uint8_t *left, int do_above, int do_left) {
   uint16x8_t sum_top;
   uint16x8_t sum_left;
   uint8x8_t dc0;

   if (do_above) {
     const uint8x8_t A = vld1_u8(above);  // top row
     const uint16x4_t p0 = vpaddl_u8(A);  // cascading summation of the top
     const uint16x4_t p1 = vpadd_u16(p0, p0);
     sum_top = vcombine_u16(p1, p1);
   }

   if (do_left) {
     const uint8x8_t L = vld1_u8(left);   // left border
     const uint16x4_t p0 = vpaddl_u8(L);  // cascading summation of the left
     const uint16x4_t p1 = vpadd_u16(p0, p0);
     sum_left = vcombine_u16(p1, p1);
   }

   if (do_above && do_left) {
     const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
     dc0 = vrshrn_n_u16(sum, 3);
   } else if (do_above) {
     dc0 = vrshrn_n_u16(sum_top, 2);
   } else if (do_left) {
     dc0 = vrshrn_n_u16(sum_left, 2);
   } else {
     dc0 = vdup_n_u8(0x80);
   }

   {
     const uint8x8_t dc = vdup_lane_u8(dc0, 0);
     int i;
     for (i = 0; i < 4; ++i) {
       vst1_lane_u32((uint32_t *)(dst + i * stride), vreinterpret_u32_u8(dc), 0);
     }
   }
 }

 void aom_dc_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                                const uint8_t *above, const uint8_t *left) {
   dc_4x4(dst, stride, above, left, 1, 1);
 }

 void aom_dc_left_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                                     const uint8_t *above, const uint8_t *left) {
   (void)above;
   dc_4x4(dst, stride, NULL, left, 0, 1);
 }

 void aom_dc_top_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                                    const uint8_t *above, const uint8_t *left) {
   (void)left;
   dc_4x4(dst, stride, above, NULL, 1, 0);
 }

 void aom_dc_128_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                                    const uint8_t *above, const uint8_t *left) {
   (void)above;
   (void)left;
   dc_4x4(dst, stride, NULL, NULL, 0, 0);
 }

 //------------------------------------------------------------------------------
 // DC 8x8

 // 'do_above' and 'do_left' facilitate branch removal when inlined.
 static INLINE void dc_8x8(uint8_t *dst, ptrdiff_t stride, const uint8_t *above,
                           const uint8_t *left, int do_above, int do_left) {
   uint16x8_t sum_top;
   uint16x8_t sum_left;
   uint8x8_t dc0;

   if (do_above) {
     const uint8x8_t A = vld1_u8(above);  // top row
     const uint16x4_t p0 = vpaddl_u8(A);  // cascading summation of the top
     const uint16x4_t p1 = vpadd_u16(p0, p0);
     const uint16x4_t p2 = vpadd_u16(p1, p1);
     sum_top = vcombine_u16(p2, p2);
   }

   if (do_left) {
     const uint8x8_t L = vld1_u8(left);   // left border
     const uint16x4_t p0 = vpaddl_u8(L);  // cascading summation of the left
     const uint16x4_t p1 = vpadd_u16(p0, p0);
     const uint16x4_t p2 = vpadd_u16(p1, p1);
     sum_left = vcombine_u16(p2, p2);
   }

   if (do_above && do_left) {
     const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
     dc0 = vrshrn_n_u16(sum, 4);
   } else if (do_above) {
     dc0 = vrshrn_n_u16(sum_top, 3);
   } else if (do_left) {
     dc0 = vrshrn_n_u16(sum_left, 3);
   } else {
     dc0 = vdup_n_u8(0x80);
   }

   {
     const uint8x8_t dc = vdup_lane_u8(dc0, 0);
     int i;
     for (i = 0; i < 8; ++i) {
       vst1_u32((uint32_t *)(dst + i * stride), vreinterpret_u32_u8(dc));
     }
   }
 }

 void aom_dc_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                const uint8_t *above, const uint8_t *left) {
   dc_8x8(dst, stride, above, left, 1, 1);
 }

 void aom_dc_left_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                     const uint8_t *above, const uint8_t *left) {
   (void)above;
   dc_8x8(dst, stride, NULL, left, 0, 1);
 }

 void aom_dc_top_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                    const uint8_t *above, const uint8_t *left) {
   (void)left;
   dc_8x8(dst, stride, above, NULL, 1, 0);
 }

 void aom_dc_128_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                                    const uint8_t *above, const uint8_t *left) {
   (void)above;
   (void)left;
   dc_8x8(dst, stride, NULL, NULL, 0, 0);
 }

 //------------------------------------------------------------------------------
 // DC 16x16

 // 'do_above' and 'do_left' facilitate branch removal when inlined.
 static INLINE void dc_16x16(uint8_t *dst, ptrdiff_t stride,
                             const uint8_t *above, const uint8_t *left,
                             int do_above, int do_left) {
   uint16x8_t sum_top;
   uint16x8_t sum_left;
   uint8x8_t dc0;

   if (do_above) {
     const uint8x16_t A = vld1q_u8(above);  // top row
     const uint16x8_t p0 = vpaddlq_u8(A);   // cascading summation of the top
     const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0));
     const uint16x4_t p2 = vpadd_u16(p1, p1);
     const uint16x4_t p3 = vpadd_u16(p2, p2);
     sum_top = vcombine_u16(p3, p3);
   }

   if (do_left) {
     const uint8x16_t L = vld1q_u8(left);  // left row
     const uint16x8_t p0 = vpaddlq_u8(L);  // cascading summation of the left
     const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0));
     const uint16x4_t p2 = vpadd_u16(p1, p1);
     const uint16x4_t p3 = vpadd_u16(p2, p2);
     sum_left = vcombine_u16(p3, p3);
   }

   if (do_above && do_left) {
     const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
     dc0 = vrshrn_n_u16(sum, 5);
   } else if (do_above) {
     dc0 = vrshrn_n_u16(sum_top, 4);
   } else if (do_left) {
     dc0 = vrshrn_n_u16(sum_left, 4);
   } else {
     dc0 = vdup_n_u8(0x80);
   }

   {
     const uint8x16_t dc = vdupq_lane_u8(dc0, 0);
     int i;
     for (i = 0; i < 16; ++i) {
       vst1q_u8(dst + i * stride, dc);
     }
   }
 }

 void aom_dc_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                  const uint8_t *above, const uint8_t *left) {
   dc_16x16(dst, stride, above, left, 1, 1);
 }

 void aom_dc_left_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above,
                                       const uint8_t *left) {
   (void)above;
   dc_16x16(dst, stride, NULL, left, 0, 1);
 }

 void aom_dc_top_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
   (void)left;
   dc_16x16(dst, stride, above, NULL, 1, 0);
 }

 void aom_dc_128_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
   (void)above;
   (void)left;
   dc_16x16(dst, stride, NULL, NULL, 0, 0);
 }

 //------------------------------------------------------------------------------
 // DC 32x32

 // 'do_above' and 'do_left' facilitate branch removal when inlined.
 static INLINE void dc_32x32(uint8_t *dst, ptrdiff_t stride,
                             const uint8_t *above, const uint8_t *left,
                             int do_above, int do_left) {
   uint16x8_t sum_top;
   uint16x8_t sum_left;
   uint8x8_t dc0;

   if (do_above) {
     const uint8x16_t A0 = vld1q_u8(above);  // top row
     const uint8x16_t A1 = vld1q_u8(above + 16);
     const uint16x8_t p0 = vpaddlq_u8(A0);  // cascading summation of the top
     const uint16x8_t p1 = vpaddlq_u8(A1);
     const uint16x8_t p2 = vaddq_u16(p0, p1);
     const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2));
     const uint16x4_t p4 = vpadd_u16(p3, p3);
     const uint16x4_t p5 = vpadd_u16(p4, p4);
     sum_top = vcombine_u16(p5, p5);
   }

   if (do_left) {
     const uint8x16_t L0 = vld1q_u8(left);  // left row
     const uint8x16_t L1 = vld1q_u8(left + 16);
     const uint16x8_t p0 = vpaddlq_u8(L0);  // cascading summation of the left
     const uint16x8_t p1 = vpaddlq_u8(L1);
     const uint16x8_t p2 = vaddq_u16(p0, p1);
     const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2));
     const uint16x4_t p4 = vpadd_u16(p3, p3);
     const uint16x4_t p5 = vpadd_u16(p4, p4);
     sum_left = vcombine_u16(p5, p5);
   }

   if (do_above && do_left) {
     const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
     dc0 = vrshrn_n_u16(sum, 6);
   } else if (do_above) {
     dc0 = vrshrn_n_u16(sum_top, 5);
   } else if (do_left) {
     dc0 = vrshrn_n_u16(sum_left, 5);
   } else {
     dc0 = vdup_n_u8(0x80);
   }

   {
     const uint8x16_t dc = vdupq_lane_u8(dc0, 0);
     int i;
     for (i = 0; i < 32; ++i) {
       vst1q_u8(dst + i * stride, dc);
       vst1q_u8(dst + i * stride + 16, dc);
     }
   }
 }

 void aom_dc_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                  const uint8_t *above, const uint8_t *left) {
   dc_32x32(dst, stride, above, left, 1, 1);
 }

 void aom_dc_left_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                       const uint8_t *above,
                                       const uint8_t *left) {
   (void)above;
   dc_32x32(dst, stride, NULL, left, 0, 1);
 }

 void aom_dc_top_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
   (void)left;
   dc_32x32(dst, stride, above, NULL, 1, 0);
 }

 void aom_dc_128_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                      const uint8_t *above,
                                      const uint8_t *left) {
   (void)above;
   (void)left;
   dc_32x32(dst, stride, NULL, NULL, 0, 0);
 }

 // -----------------------------------------------------------------------------

 void aom_d135_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                                  const uint8_t *above, const uint8_t *left) {
   const uint8x8_t XABCD_u8 = vld1_u8(above - 1);
   const uint64x1_t XABCD = vreinterpret_u64_u8(XABCD_u8);
   const uint64x1_t ____XABC = vshl_n_u64(XABCD, 32);
   const uint32x2_t zero = vdup_n_u32(0);
   const uint32x2_t IJKL = vld1_lane_u32((const uint32_t *)left, zero, 0);
   const uint8x8_t IJKL_u8 = vreinterpret_u8_u32(IJKL);
   const uint64x1_t LKJI____ = vreinterpret_u64_u8(vrev32_u8(IJKL_u8));
   const uint64x1_t LKJIXABC = vorr_u64(LKJI____, ____XABC);
   const uint8x8_t KJIXABC_ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 8));
   const uint8x8_t JIXABC__ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 16));
   const uint8_t D = vget_lane_u8(XABCD_u8, 4);
   const uint8x8_t JIXABCD_ = vset_lane_u8(D, JIXABC__, 6);
   const uint8x8_t LKJIXABC_u8 = vreinterpret_u8_u64(LKJIXABC);
   const uint8x8_t avg1 = vhadd_u8(JIXABCD_, LKJIXABC_u8);
   const uint8x8_t avg2 = vrhadd_u8(avg1, KJIXABC_);
   const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2);
   const uint32x2_t r3 = vreinterpret_u32_u8(avg2);
   const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8));
   const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16));
   const uint32x2_t r0 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24));
   vst1_lane_u32((uint32_t *)(dst + 0 * stride), r0, 0);
   vst1_lane_u32((uint32_t *)(dst + 1 * stride), r1, 0);
   vst1_lane_u32((uint32_t *)(dst + 2 * stride), r2, 0);
   vst1_lane_u32((uint32_t *)(dst + 3 * stride), r3, 0);
 }

 #if !HAVE_NEON_ASM

 void aom_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                               const uint8_t *above, const uint8_t *left) {
   int i;
   uint32x2_t d0u32 = vdup_n_u32(0);
   (void)left;

   d0u32 = vld1_lane_u32((const uint32_t *)above, d0u32, 0);
   for (i = 0; i < 4; i++, dst += stride)
     vst1_lane_u32((uint32_t *)dst, d0u32, 0);
 }

 void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                               const uint8_t *above, const uint8_t *left) {
   int i;
   uint8x8_t d0u8 = vdup_n_u8(0);
   (void)left;

   d0u8 = vld1_u8(above);
   for (i = 0; i < 8; i++, dst += stride) vst1_u8(dst, d0u8);
 }

 void aom_v_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                 const uint8_t *above, const uint8_t *left) {
   int i;
   uint8x16_t q0u8 = vdupq_n_u8(0);
   (void)left;

   q0u8 = vld1q_u8(above);
   for (i = 0; i < 16; i++, dst += stride) vst1q_u8(dst, q0u8);
 }

 void aom_v_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                 const uint8_t *above, const uint8_t *left) {
   int i;
   uint8x16_t q0u8 = vdupq_n_u8(0);
   uint8x16_t q1u8 = vdupq_n_u8(0);
   (void)left;

   q0u8 = vld1q_u8(above);
   q1u8 = vld1q_u8(above + 16);
   for (i = 0; i < 32; i++, dst += stride) {
     vst1q_u8(dst, q0u8);
     vst1q_u8(dst + 16, q1u8);
   }
 }

 void aom_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
                               const uint8_t *above, const uint8_t *left) {
   uint8x8_t d0u8 = vdup_n_u8(0);
   uint32x2_t d1u32 = vdup_n_u32(0);
   (void)above;

   d1u32 = vld1_lane_u32((const uint32_t *)left, d1u32, 0);

   d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 0);
   vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 1);
   vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 2);
   vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 3);
   vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
 }

 void aom_h_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
                               const uint8_t *above, const uint8_t *left) {
   uint8x8_t d0u8 = vdup_n_u8(0);
   uint64x1_t d1u64 = vdup_n_u64(0);
   (void)above;

   d1u64 = vld1_u64((const uint64_t *)left);

   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 0);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 1);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 2);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 3);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 4);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 5);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 6);
   vst1_u8(dst, d0u8);
   dst += stride;
   d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 7);
   vst1_u8(dst, d0u8);
 }

 void aom_h_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
                                 const uint8_t *above, const uint8_t *left) {
   int j;
   uint8x8_t d2u8 = vdup_n_u8(0);
   uint8x16_t q0u8 = vdupq_n_u8(0);
   uint8x16_t q1u8 = vdupq_n_u8(0);
   (void)above;

   q1u8 = vld1q_u8(left);
   d2u8 = vget_low_u8(q1u8);
   for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) {
     q0u8 = vdupq_lane_u8(d2u8, 0);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 1);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 2);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 3);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 4);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 5);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 6);
     vst1q_u8(dst, q0u8);
     dst += stride;
     q0u8 = vdupq_lane_u8(d2u8, 7);
     vst1q_u8(dst, q0u8);
     dst += stride;
   }
 }

 void aom_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
                                 const uint8_t *above, const uint8_t *left) {
   int j, k;
   uint8x8_t d2u8 = vdup_n_u8(0);
   uint8x16_t q0u8 = vdupq_n_u8(0);
   uint8x16_t q1u8 = vdupq_n_u8(0);
   (void)above;

   for (k = 0; k < 2; k++, left += 16) {
     q1u8 = vld1q_u8(left);
     d2u8 = vget_low_u8(q1u8);
     for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) {
       q0u8 = vdupq_lane_u8(d2u8, 0);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 1);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 2);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 3);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 4);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 5);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 6);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
       q0u8 = vdupq_lane_u8(d2u8, 7);
       vst1q_u8(dst, q0u8);
       vst1q_u8(dst + 16, q0u8);
       dst += stride;
     }
   }
 }
 #endif  // !HAVE_NEON_ASM
	/*
	* Copyright (c) 2016, Alliance for Open Media. All rights reserved
	*
	* This source code is subject to the terms of the BSD 2 Clause License and
	* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
	* was not distributed with this source code in the LICENSE file, you can
	* obtain it at www.aomedia.org/license/software. If the Alliance for Open
	* Media Patent License 1.0 was not distributed with this source code in the
	* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
	*/

	#include <arm_neon.h>

	#include "./aom_config.h"
	#include "./aom_dsp_rtcd.h"
	#include "aom/aom_integer.h"

	//------------------------------------------------------------------------------
	// DC 4x4

	// 'do_above' and 'do_left' facilitate branch removal when inlined.
	static INLINE void dc_4x4(uint8_t dst, ptrdiff_t stride, const uint8_t above,
	const uint8_t *left, int do_above, int do_left) {
	uint16x8_t sum_top;
	uint16x8_t sum_left;
	uint8x8_t dc0;

	if (do_above) {
	const uint8x8_t A = vld1_u8(above); // top row
	const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top
	const uint16x4_t p1 = vpadd_u16(p0, p0);
	sum_top = vcombine_u16(p1, p1);
	}

	if (do_left) {
	const uint8x8_t L = vld1_u8(left); // left border
	const uint16x4_t p0 = vpaddl_u8(L); // cascading summation of the left
	const uint16x4_t p1 = vpadd_u16(p0, p0);
	sum_left = vcombine_u16(p1, p1);
	}

	if (do_above && do_left) {
	const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
	dc0 = vrshrn_n_u16(sum, 3);
	} else if (do_above) {
	dc0 = vrshrn_n_u16(sum_top, 2);
	} else if (do_left) {
	dc0 = vrshrn_n_u16(sum_left, 2);
	} else {
	dc0 = vdup_n_u8(0x80);
	}

	{
	const uint8x8_t dc = vdup_lane_u8(dc0, 0);
	int i;
	for (i = 0; i < 4; ++i) {
	vst1_lane_u32((uint32_t )(dst + i stride), vreinterpret_u32_u8(dc), 0);
	}
	}
	}

	void aom_dc_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	dc_4x4(dst, stride, above, left, 1, 1);
	}

	void aom_dc_left_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	(void)above;
	dc_4x4(dst, stride, NULL, left, 0, 1);
	}

	void aom_dc_top_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	(void)left;
	dc_4x4(dst, stride, above, NULL, 1, 0);
	}

	void aom_dc_128_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	(void)above;
	(void)left;
	dc_4x4(dst, stride, NULL, NULL, 0, 0);
	}

	//------------------------------------------------------------------------------
	// DC 8x8

	// 'do_above' and 'do_left' facilitate branch removal when inlined.
	static INLINE void dc_8x8(uint8_t dst, ptrdiff_t stride, const uint8_t above,
	const uint8_t *left, int do_above, int do_left) {
	uint16x8_t sum_top;
	uint16x8_t sum_left;
	uint8x8_t dc0;

	if (do_above) {
	const uint8x8_t A = vld1_u8(above); // top row
	const uint16x4_t p0 = vpaddl_u8(A); // cascading summation of the top
	const uint16x4_t p1 = vpadd_u16(p0, p0);
	const uint16x4_t p2 = vpadd_u16(p1, p1);
	sum_top = vcombine_u16(p2, p2);
	}

	if (do_left) {
	const uint8x8_t L = vld1_u8(left); // left border
	const uint16x4_t p0 = vpaddl_u8(L); // cascading summation of the left
	const uint16x4_t p1 = vpadd_u16(p0, p0);
	const uint16x4_t p2 = vpadd_u16(p1, p1);
	sum_left = vcombine_u16(p2, p2);
	}

	if (do_above && do_left) {
	const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
	dc0 = vrshrn_n_u16(sum, 4);
	} else if (do_above) {
	dc0 = vrshrn_n_u16(sum_top, 3);
	} else if (do_left) {
	dc0 = vrshrn_n_u16(sum_left, 3);
	} else {
	dc0 = vdup_n_u8(0x80);
	}

	{
	const uint8x8_t dc = vdup_lane_u8(dc0, 0);
	int i;
	for (i = 0; i < 8; ++i) {
	vst1_u32((uint32_t )(dst + i stride), vreinterpret_u32_u8(dc));
	}
	}
	}

	void aom_dc_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	dc_8x8(dst, stride, above, left, 1, 1);
	}

	void aom_dc_left_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	(void)above;
	dc_8x8(dst, stride, NULL, left, 0, 1);
	}

	void aom_dc_top_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	(void)left;
	dc_8x8(dst, stride, above, NULL, 1, 0);
	}

	void aom_dc_128_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	(void)above;
	(void)left;
	dc_8x8(dst, stride, NULL, NULL, 0, 0);
	}

	//------------------------------------------------------------------------------
	// DC 16x16

	// 'do_above' and 'do_left' facilitate branch removal when inlined.
	static INLINE void dc_16x16(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left,
	int do_above, int do_left) {
	uint16x8_t sum_top;
	uint16x8_t sum_left;
	uint8x8_t dc0;

	if (do_above) {
	const uint8x16_t A = vld1q_u8(above); // top row
	const uint16x8_t p0 = vpaddlq_u8(A); // cascading summation of the top
	const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0));
	const uint16x4_t p2 = vpadd_u16(p1, p1);
	const uint16x4_t p3 = vpadd_u16(p2, p2);
	sum_top = vcombine_u16(p3, p3);
	}

	if (do_left) {
	const uint8x16_t L = vld1q_u8(left); // left row
	const uint16x8_t p0 = vpaddlq_u8(L); // cascading summation of the left
	const uint16x4_t p1 = vadd_u16(vget_low_u16(p0), vget_high_u16(p0));
	const uint16x4_t p2 = vpadd_u16(p1, p1);
	const uint16x4_t p3 = vpadd_u16(p2, p2);
	sum_left = vcombine_u16(p3, p3);
	}

	if (do_above && do_left) {
	const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
	dc0 = vrshrn_n_u16(sum, 5);
	} else if (do_above) {
	dc0 = vrshrn_n_u16(sum_top, 4);
	} else if (do_left) {
	dc0 = vrshrn_n_u16(sum_left, 4);
	} else {
	dc0 = vdup_n_u8(0x80);
	}

	{
	const uint8x16_t dc = vdupq_lane_u8(dc0, 0);
	int i;
	for (i = 0; i < 16; ++i) {
	vst1q_u8(dst + i * stride, dc);
	}
	}
	}

	void aom_dc_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	dc_16x16(dst, stride, above, left, 1, 1);
	}

	void aom_dc_left_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t *above,
	const uint8_t *left) {
	(void)above;
	dc_16x16(dst, stride, NULL, left, 0, 1);
	}

	void aom_dc_top_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t *above,
	const uint8_t *left) {
	(void)left;
	dc_16x16(dst, stride, above, NULL, 1, 0);
	}

	void aom_dc_128_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t *above,
	const uint8_t *left) {
	(void)above;
	(void)left;
	dc_16x16(dst, stride, NULL, NULL, 0, 0);
	}

	//------------------------------------------------------------------------------
	// DC 32x32

	// 'do_above' and 'do_left' facilitate branch removal when inlined.
	static INLINE void dc_32x32(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left,
	int do_above, int do_left) {
	uint16x8_t sum_top;
	uint16x8_t sum_left;
	uint8x8_t dc0;

	if (do_above) {
	const uint8x16_t A0 = vld1q_u8(above); // top row
	const uint8x16_t A1 = vld1q_u8(above + 16);
	const uint16x8_t p0 = vpaddlq_u8(A0); // cascading summation of the top
	const uint16x8_t p1 = vpaddlq_u8(A1);
	const uint16x8_t p2 = vaddq_u16(p0, p1);
	const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2));
	const uint16x4_t p4 = vpadd_u16(p3, p3);
	const uint16x4_t p5 = vpadd_u16(p4, p4);
	sum_top = vcombine_u16(p5, p5);
	}

	if (do_left) {
	const uint8x16_t L0 = vld1q_u8(left); // left row
	const uint8x16_t L1 = vld1q_u8(left + 16);
	const uint16x8_t p0 = vpaddlq_u8(L0); // cascading summation of the left
	const uint16x8_t p1 = vpaddlq_u8(L1);
	const uint16x8_t p2 = vaddq_u16(p0, p1);
	const uint16x4_t p3 = vadd_u16(vget_low_u16(p2), vget_high_u16(p2));
	const uint16x4_t p4 = vpadd_u16(p3, p3);
	const uint16x4_t p5 = vpadd_u16(p4, p4);
	sum_left = vcombine_u16(p5, p5);
	}

	if (do_above && do_left) {
	const uint16x8_t sum = vaddq_u16(sum_left, sum_top);
	dc0 = vrshrn_n_u16(sum, 6);
	} else if (do_above) {
	dc0 = vrshrn_n_u16(sum_top, 5);
	} else if (do_left) {
	dc0 = vrshrn_n_u16(sum_left, 5);
	} else {
	dc0 = vdup_n_u8(0x80);
	}

	{
	const uint8x16_t dc = vdupq_lane_u8(dc0, 0);
	int i;
	for (i = 0; i < 32; ++i) {
	vst1q_u8(dst + i * stride, dc);
	vst1q_u8(dst + i * stride + 16, dc);
	}
	}
	}

	void aom_dc_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	dc_32x32(dst, stride, above, left, 1, 1);
	}

	void aom_dc_left_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t *above,
	const uint8_t *left) {
	(void)above;
	dc_32x32(dst, stride, NULL, left, 0, 1);
	}

	void aom_dc_top_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t *above,
	const uint8_t *left) {
	(void)left;
	dc_32x32(dst, stride, above, NULL, 1, 0);
	}

	void aom_dc_128_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t *above,
	const uint8_t *left) {
	(void)above;
	(void)left;
	dc_32x32(dst, stride, NULL, NULL, 0, 0);
	}

	// -----------------------------------------------------------------------------

	void aom_d135_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	const uint8x8_t XABCD_u8 = vld1_u8(above - 1);
	const uint64x1_t XABCD = vreinterpret_u64_u8(XABCD_u8);
	const uint64x1_t ____XABC = vshl_n_u64(XABCD, 32);
	const uint32x2_t zero = vdup_n_u32(0);
	const uint32x2_t IJKL = vld1_lane_u32((const uint32_t *)left, zero, 0);
	const uint8x8_t IJKL_u8 = vreinterpret_u8_u32(IJKL);
	const uint64x1_t LKJI____ = vreinterpret_u64_u8(vrev32_u8(IJKL_u8));
	const uint64x1_t LKJIXABC = vorr_u64(LKJI____, ____XABC);
	const uint8x8_t KJIXABC_ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 8));
	const uint8x8_t JIXABC__ = vreinterpret_u8_u64(vshr_n_u64(LKJIXABC, 16));
	const uint8_t D = vget_lane_u8(XABCD_u8, 4);
	const uint8x8_t JIXABCD_ = vset_lane_u8(D, JIXABC__, 6);
	const uint8x8_t LKJIXABC_u8 = vreinterpret_u8_u64(LKJIXABC);
	const uint8x8_t avg1 = vhadd_u8(JIXABCD_, LKJIXABC_u8);
	const uint8x8_t avg2 = vrhadd_u8(avg1, KJIXABC_);
	const uint64x1_t avg2_u64 = vreinterpret_u64_u8(avg2);
	const uint32x2_t r3 = vreinterpret_u32_u8(avg2);
	const uint32x2_t r2 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 8));
	const uint32x2_t r1 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 16));
	const uint32x2_t r0 = vreinterpret_u32_u64(vshr_n_u64(avg2_u64, 24));
	vst1_lane_u32((uint32_t )(dst + 0 stride), r0, 0);
	vst1_lane_u32((uint32_t )(dst + 1 stride), r1, 0);
	vst1_lane_u32((uint32_t )(dst + 2 stride), r2, 0);
	vst1_lane_u32((uint32_t )(dst + 3 stride), r3, 0);
	}

	#if !HAVE_NEON_ASM

	void aom_v_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	int i;
	uint32x2_t d0u32 = vdup_n_u32(0);
	(void)left;

	d0u32 = vld1_lane_u32((const uint32_t *)above, d0u32, 0);
	for (i = 0; i < 4; i++, dst += stride)
	vst1_lane_u32((uint32_t *)dst, d0u32, 0);
	}

	void aom_v_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	int i;
	uint8x8_t d0u8 = vdup_n_u8(0);
	(void)left;

	d0u8 = vld1_u8(above);
	for (i = 0; i < 8; i++, dst += stride) vst1_u8(dst, d0u8);
	}

	void aom_v_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	int i;
	uint8x16_t q0u8 = vdupq_n_u8(0);
	(void)left;

	q0u8 = vld1q_u8(above);
	for (i = 0; i < 16; i++, dst += stride) vst1q_u8(dst, q0u8);
	}

	void aom_v_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	int i;
	uint8x16_t q0u8 = vdupq_n_u8(0);
	uint8x16_t q1u8 = vdupq_n_u8(0);
	(void)left;

	q0u8 = vld1q_u8(above);
	q1u8 = vld1q_u8(above + 16);
	for (i = 0; i < 32; i++, dst += stride) {
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q1u8);
	}
	}

	void aom_h_predictor_4x4_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	uint8x8_t d0u8 = vdup_n_u8(0);
	uint32x2_t d1u32 = vdup_n_u32(0);
	(void)above;

	d1u32 = vld1_lane_u32((const uint32_t *)left, d1u32, 0);

	d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 0);
	vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 1);
	vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 2);
	vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u32(d1u32), 3);
	vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d0u8), 0);
	}

	void aom_h_predictor_8x8_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	uint8x8_t d0u8 = vdup_n_u8(0);
	uint64x1_t d1u64 = vdup_n_u64(0);
	(void)above;

	d1u64 = vld1_u64((const uint64_t *)left);

	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 0);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 1);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 2);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 3);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 4);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 5);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 6);
	vst1_u8(dst, d0u8);
	dst += stride;
	d0u8 = vdup_lane_u8(vreinterpret_u8_u64(d1u64), 7);
	vst1_u8(dst, d0u8);
	}

	void aom_h_predictor_16x16_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	int j;
	uint8x8_t d2u8 = vdup_n_u8(0);
	uint8x16_t q0u8 = vdupq_n_u8(0);
	uint8x16_t q1u8 = vdupq_n_u8(0);
	(void)above;

	q1u8 = vld1q_u8(left);
	d2u8 = vget_low_u8(q1u8);
	for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) {
	q0u8 = vdupq_lane_u8(d2u8, 0);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 1);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 2);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 3);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 4);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 5);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 6);
	vst1q_u8(dst, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 7);
	vst1q_u8(dst, q0u8);
	dst += stride;
	}
	}

	void aom_h_predictor_32x32_neon(uint8_t *dst, ptrdiff_t stride,
	const uint8_t above, const uint8_t left) {
	int j, k;
	uint8x8_t d2u8 = vdup_n_u8(0);
	uint8x16_t q0u8 = vdupq_n_u8(0);
	uint8x16_t q1u8 = vdupq_n_u8(0);
	(void)above;

	for (k = 0; k < 2; k++, left += 16) {
	q1u8 = vld1q_u8(left);
	d2u8 = vget_low_u8(q1u8);
	for (j = 0; j < 2; j++, d2u8 = vget_high_u8(q1u8)) {
	q0u8 = vdupq_lane_u8(d2u8, 0);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 1);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 2);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 3);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 4);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 5);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 6);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	q0u8 = vdupq_lane_u8(d2u8, 7);
	vst1q_u8(dst, q0u8);
	vst1q_u8(dst + 16, q0u8);
	dst += stride;
	}
	}
	}
	#endif // !HAVE_NEON_ASM