commit d57d9238fb5de60f1a3747267fff26012c459eae
author Cheng Chen <chengchen@google.com> Sun Feb 25 21:56:15 2018 -0800
committer Cheng Chen <chengchen@google.com> Mon Feb 26 17:31:46 2018 +0000
tree 09cf2a225b5f5d3e722784fbbfc7b030344f9b9d
parent bbc0c21b73bce3527878972c892156514f3216be

Fix high bit-depth convolve function

Fix high bit depth c function, to make is compatible with
convolve functions which make intermediate results 16 bits.

Change-Id: I3844b69098168c09f8a120d39ee3f4b326031063

av1/common/convolve.c

diff --git a/av1/common/convolve.c b/av1/common/convolve.c
index 674ee3f..6c8531f 100644
--- a/av1/common/convolve.c
+++ b/av1/common/convolve.c
@@ -1438,15 +1438,20 @@
       res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0);
       if (conv_params->use_jnt_comp_avg) {
         if (conv_params->do_average) {
-          dst[y * dst_stride + x] += res * conv_params->bck_offset;
+          int32_t tmp = dst[y * dst_stride + x];
+          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+          dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
         } else {
-          dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+          dst[y * dst_stride + x] = res;
         }
       } else {
-        if (conv_params->do_average)
-          dst[y * dst_stride + x] += res;
-        else
+        if (conv_params->do_average) {
+          int32_t tmp = dst[y * dst_stride + x];
+          tmp += res;
+          dst[y * dst_stride + x] = tmp >> 1;
+        } else {
           dst[y * dst_stride + x] = res;
+        }
       }
     }
   }
@@ -1481,15 +1486,20 @@
       res = ROUND_POWER_OF_TWO(res, conv_params->round_1);
       if (conv_params->use_jnt_comp_avg) {
         if (conv_params->do_average) {
-          dst[y * dst_stride + x] += res * conv_params->bck_offset;
+          int32_t tmp = dst[y * dst_stride + x];
+          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+          dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
         } else {
-          dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+          dst[y * dst_stride + x] = res;
         }
       } else {
-        if (conv_params->do_average)
-          dst[y * dst_stride + x] += res;
-        else
+        if (conv_params->do_average) {
+          int32_t tmp = dst[y * dst_stride + x];
+          tmp += res;
+          dst[y * dst_stride + x] = tmp >> 1;
+        } else {
           dst[y * dst_stride + x] = res;
+        }
       }
     }
   }
@@ -1518,15 +1528,20 @@
       CONV_BUF_TYPE res = src[y * src_stride + x] << bits;
       if (conv_params->use_jnt_comp_avg) {
         if (conv_params->do_average) {
-          dst[y * dst_stride + x] += res * conv_params->bck_offset;
+          int32_t tmp = dst[y * dst_stride + x];
+          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+          dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
         } else {
-          dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+          dst[y * dst_stride + x] = res;
         }
       } else {
-        if (conv_params->do_average)
-          dst[y * dst_stride + x] += res;
-        else
+        if (conv_params->do_average) {
+          int32_t tmp = dst[y * dst_stride + x];
+          tmp += res;
+          dst[y * dst_stride + x] = tmp >> 1;
+        } else {
           dst[y * dst_stride + x] = res;
+        }
       }
     }
   }