commit c67b895fa4a0dcb5c897a44fcc57c79153975879
author Jean-Marc Valin <jmvalin@mozilla.com> Tue Oct 18 17:39:03 2016 -0400
committer Yaowu Xu <yaowu@google.com> Thu Oct 27 12:37:52 2016 -0700
tree 8ea4507d9465cc1163f4220fb19d1ac277fcfec1
parent f0f98578df4b39070c721c97988b8baa9a44651a

Deringing support for 4:2:2 by not deringing chroma

No change in output for 4:2:0 and 4:4:4

Change-Id: Ic46753d23a5b5f90b611a3da1a4574870519957c

av1/common/dering.c

diff --git a/av1/common/dering.c b/av1/common/dering.c
index fd5167a..4519031 100644
--- a/av1/common/dering.c
+++ b/av1/common/dering.c
@@ -60,18 +60,24 @@
   int dec_y[3];
   int pli;
   int coeff_shift = AOMMAX(cm->bit_depth - 8, 0);
+  int nplanes;
+  if (xd->plane[1].subsampling_x == xd->plane[1].subsampling_y &&
+      xd->plane[2].subsampling_x == xd->plane[2].subsampling_y)
+    nplanes = 3;
+  else
+    nplanes = 1;
   nvsb = (cm->mi_rows + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE;
   nhsb = (cm->mi_cols + MAX_MIB_SIZE - 1) / MAX_MIB_SIZE;
   bskip = aom_malloc(sizeof(*bskip) * cm->mi_rows * cm->mi_cols);
   av1_setup_dst_planes(xd->plane, frame, 0, 0);
-  for (pli = 0; pli < 3; pli++) {
+  for (pli = 0; pli < nplanes; pli++) {
     dec_x[pli] = xd->plane[pli].subsampling_x;
     dec_y[pli] = xd->plane[pli].subsampling_y;
     bsize_x[pli] = 8 >> dec_x[pli];
     bsize_y[pli] = 8 >> dec_y[pli];
   }
   stride = bsize_x[0] * cm->mi_cols;
-  for (pli = 0; pli < 3; pli++) {
+  for (pli = 0; pli < nplanes; pli++) {
     src[pli] = aom_malloc(sizeof(*src) * cm->mi_rows * cm->mi_cols * 64);
     for (r = 0; r < bsize_y[pli] * cm->mi_rows; ++r) {
       for (c = 0; c < bsize_x[pli] * cm->mi_cols; ++c) {
@@ -108,7 +114,7 @@
                             ->mbmi.dering_gain);
       if (level == 0 || sb_all_skip(cm, sbr * MAX_MIB_SIZE, sbc * MAX_MIB_SIZE))
         continue;
-      for (pli = 0; pli < 3; pli++) {
+      for (pli = 0; pli < nplanes; pli++) {
         int16_t dst[MAX_MIB_SIZE * MAX_MIB_SIZE * 8 * 8];
         int threshold;
         /* FIXME: This is a temporary hack that uses more conservative
@@ -149,7 +155,7 @@
       }
     }
   }
-  for (pli = 0; pli < 3; pli++) {
+  for (pli = 0; pli < nplanes; pli++) {
     aom_free(src[pli]);
   }
   aom_free(bskip);