commit 79dd1ee907936f3a0afa816f0d488e5925a1a518
author Sarah Parker <sarahparker@google.com> Wed Mar 16 16:06:19 2016 -0700
committer Sarah Parker <sarahparker@google.com> Thu Mar 17 09:58:29 2016 -0700
tree 792eef599d4b45eb453d4e5250e0e31e16c67440
parent f330444a098ceb2d877d41e694bc3a141f09be34

Fix hbd segfault for prune_one and prune_two

Change-Id: I71da102550aa7d81961e5f10c71058d5fa8dc6c4

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 16deebf..5ac930a 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -377,16 +377,35 @@
   unsigned int var[16];
   double total = 0;
   const int f_index = bsize - 6;
+
   if (f_index < 0) {
     int i, j, index;
     int w_shift = bw == 8 ? 1 : 2;
     int h_shift = bh == 8 ? 1 : 2;
-    for (i = 0; i < bh; ++i)
-      for (j = 0; j < bw; ++j) {
-        index = (j >> w_shift) + ((i >> h_shift) << 2);
-        esq[index] += (src[j + i * src_stride] - dst[j + i * dst_stride]) *
-                      (src[j + i * src_stride] - dst[j + i * dst_stride]);
-      }
+#if CONFIG_VP9_HIGHBITDEPTH
+    if (cpi->common.use_highbitdepth) {
+      uint16_t *src16 = CONVERT_TO_SHORTPTR(src);
+      uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
+      for (i = 0; i < bh; ++i)
+        for (j = 0; j < bw; ++j) {
+          index = (j >> w_shift) + ((i >> h_shift) << 2);
+          esq[index] += (src16[j + i * src_stride] -
+                        dst16[j + i * dst_stride]) *
+                        (src16[j + i * src_stride] -
+                        dst16[j + i * dst_stride]);
+        }
+    } else {
+#endif  // CONFIG_VP9_HIGHBITDEPTH
+
+      for (i = 0; i < bh; ++i)
+        for (j = 0; j < bw; ++j) {
+          index = (j >> w_shift) + ((i >> h_shift) << 2);
+          esq[index] += (src[j + i * src_stride] - dst[j + i * dst_stride]) *
+                        (src[j + i * src_stride] - dst[j + i * dst_stride]);
+        }
+#if CONFIG_VP9_HIGHBITDEPTH
+    }
+#endif  // CONFIG_VP9_HIGHBITDEPTH
   } else {
     var[0] = cpi->fn_ptr[f_index].vf(src, src_stride,
                                      dst, dst_stride, &esq[0]);