commit 22850493b60b70b647c94e8d0a310442fecc3248
author Peter de Rivaz <peter.derivaz@argondesign.com> Tue Dec 08 15:58:21 2015 +0000
committer Peter de Rivaz <peter.derivaz@argondesign.com> Tue Dec 08 15:58:21 2015 +0000
tree faa8356836a3e60f141f3bc4a909d7724c40ddb3
parent 69f49300410a6815494a5361f49fca82bd3f337c

Fix for crash when using high bitdepth and var-tx

Change-Id: Ide48fa4312f7828f99290f7a2be878f5673fa716

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index ff71889..f92f720 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1889,7 +1889,12 @@
   int src_stride = p->src.stride;
   uint8_t *src = &p->src.buf[4 * blk_row * src_stride + 4 * blk_col];
   uint8_t *dst = &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col];
+#if CONFIG_VP9_HIGHBITDEPTH
+  DECLARE_ALIGNED(16, uint16_t, rec_buffer_alloc_16[32 * 32]);
+  uint8_t *rec_buffer;
+#else
   DECLARE_ALIGNED(16, uint8_t, rec_buffer[32 * 32]);
+#endif
 
   int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
   int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];
@@ -1901,8 +1906,20 @@
 
   vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size);
 
+#if CONFIG_VP9_HIGHBITDEPTH
+  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
+    rec_buffer = CONVERT_TO_BYTEPTR(rec_buffer_alloc_16);
+    vpx_highbd_convolve_copy(dst, pd->dst.stride, rec_buffer, 32,
+                             NULL, 0, NULL, 0, bh, bh, xd->bd);
+  } else {
+    rec_buffer = (uint8_t *)rec_buffer_alloc_16;
+    vpx_convolve_copy(dst, pd->dst.stride, rec_buffer, 32,
+                      NULL, 0, NULL, 0, bh, bh);
+  }
+#else
   vpx_convolve_copy(dst, pd->dst.stride, rec_buffer, 32,
                     NULL, 0, NULL, 0, bh, bh);
+#endif
 
   if (blk_row + (bh >> 2) > max_blocks_high ||
       blk_col + (bh >> 2) > max_blocks_wide) {