Don't read 8 bytes from a 3-byte array
The elements of the av1_fwd_txfm_shift_ls array point to 3-byte (global)
arrays. It is unsafe to read from a 3-byte array using vld1_s8(), which
reads 8 bytes.
This bug was introduced when av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
was added in https://aomedia-review.googlesource.com/c/aom/+/110221.
ASan reported global-buffer-overflow errors.
Change-Id: I53f06dba6b6be3e7216c4f2359e1060cf9d7fd87
diff --git a/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c b/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
index 249b3a6..3640cf1 100644
--- a/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
+++ b/av1/encoder/arm/neon/av1_fwd_txfm2d_neon.c
@@ -2317,10 +2317,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip) {
load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
} else {
@@ -2360,10 +2359,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip) {
load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
} else {
@@ -2401,10 +2399,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip) {
load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
} else {
@@ -2446,10 +2443,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip)
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
else
@@ -2486,10 +2482,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip)
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
else
@@ -2526,10 +2521,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
} else {
@@ -2571,10 +2565,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
} else {
@@ -2619,10 +2612,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
for (int i = 0; i < 2; i++) {
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
@@ -2663,10 +2655,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
for (int i = 0; i < 2; i++) {
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
@@ -2706,10 +2697,9 @@
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
for (int i = 0; i < 2; i++) {
if (ud_flip) {
@@ -2755,10 +2745,9 @@
if (col_txfm != NULL && row_txfm != NULL) {
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
for (int i = 0; i < 2; i++) {
if (ud_flip) {
@@ -2809,10 +2798,9 @@
if (col_txfm != NULL && row_txfm != NULL) {
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
for (int i = 0; i < 4; i++) {
if (ud_flip) {
@@ -2858,10 +2846,9 @@
const transform_1d_lbd_neon row_txfm = row_txfm8x32_arr[tx_type];
if (col_txfm != NULL && row_txfm != NULL) {
- const int16x4_t v_shifts = vget_low_s16(vmovl_s8(vld1_s8(&shift[0])));
- const int16x8_t v_shift0 = vdupq_lane_s16(v_shifts, 0);
- const int16x8_t v_shift1 = vdupq_lane_s16(v_shifts, 1);
- const int16x8_t v_shift2 = vdupq_lane_s16(v_shifts, 2);
+ const int16x8_t v_shift0 = vdupq_n_s16(shift[0]);
+ const int16x8_t v_shift1 = vdupq_n_s16(shift[1]);
+ const int16x8_t v_shift2 = vdupq_n_s16(shift[2]);
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
