commit aff479fac9ee1c5785f7129a59b59c62f6b06f14
author Cheng Chen <chengchen@google.com> Thu Oct 26 16:53:10 2017 -0700
committer Cheng Chen <chengchen@google.com> Fri Oct 27 16:44:40 2017 +0000
tree f8d38c70b6d3ecb0e03fa5a32d857bd420dd2ac5
parent 3392c4367b6ba3760067561b61388c4af22c7725

Clamp loop filter levels

Clamp in range [-MAX_LOOP_FILTER, MAX_LOOP_FILTER], for experiment LOOPFILTER_LEVELS.

BUG=aomedia:970

Change-Id: Ica52242e969dcea49565f118155de5eef32a4cc4

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index f9ca985..6f127e6 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -1184,17 +1184,21 @@
 #if CONFIG_LOOPFILTER_LEVEL
       if (cm->delta_lf_multi) {
         for (int lf_id = 0; lf_id < FRAME_LF_COUNT; ++lf_id) {
-          mbmi->curr_delta_lf[lf_id] = xd->curr_delta_lf[lf_id] =
+          const int tmp_lvl =
               xd->prev_delta_lf[lf_id] +
               read_delta_lflevel(cm, xd, r, lf_id, mbmi, mi_col, mi_row) *
                   cm->delta_lf_res;
+          mbmi->curr_delta_lf[lf_id] = xd->curr_delta_lf[lf_id] =
+              clamp(tmp_lvl, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
           xd->prev_delta_lf[lf_id] = xd->curr_delta_lf[lf_id];
         }
       } else {
-        mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base =
+        const int tmp_lvl =
             xd->prev_delta_lf_from_base +
             read_delta_lflevel(cm, xd, r, -1, mbmi, mi_col, mi_row) *
                 cm->delta_lf_res;
+        mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base =
+            clamp(tmp_lvl, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
         xd->prev_delta_lf_from_base = xd->current_delta_lf_from_base;
       }
 #else
@@ -1203,7 +1207,7 @@
           read_delta_lflevel(cm, xd, r, mbmi, mi_col, mi_row) *
               cm->delta_lf_res;
       mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base =
-          clamp(current_delta_lf_from_base, 0, MAX_LOOP_FILTER);
+          clamp(current_delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
       xd->prev_delta_lf_from_base = xd->current_delta_lf_from_base;
 #endif  // CONFIG_LOOPFILTER_LEVEL
     }
@@ -2716,17 +2720,21 @@
 #if CONFIG_LOOPFILTER_LEVEL
       if (cm->delta_lf_multi) {
         for (int lf_id = 0; lf_id < FRAME_LF_COUNT; ++lf_id) {
-          mbmi->curr_delta_lf[lf_id] = xd->curr_delta_lf[lf_id] =
+          const int tmp_lvl =
               xd->prev_delta_lf[lf_id] +
               read_delta_lflevel(cm, xd, r, lf_id, mbmi, mi_col, mi_row) *
                   cm->delta_lf_res;
+          mbmi->curr_delta_lf[lf_id] = xd->curr_delta_lf[lf_id] =
+              clamp(tmp_lvl, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
           xd->prev_delta_lf[lf_id] = xd->curr_delta_lf[lf_id];
         }
       } else {
-        mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base =
+        const int tmp_lvl =
             xd->prev_delta_lf_from_base +
             read_delta_lflevel(cm, xd, r, -1, mbmi, mi_col, mi_row) *
                 cm->delta_lf_res;
+        mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base =
+            clamp(tmp_lvl, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
         xd->prev_delta_lf_from_base = xd->current_delta_lf_from_base;
       }
 #else
@@ -2735,7 +2743,7 @@
           read_delta_lflevel(cm, xd, r, mbmi, mi_col, mi_row) *
               cm->delta_lf_res;
       mbmi->current_delta_lf_from_base = xd->current_delta_lf_from_base =
-          clamp(current_delta_lf_from_base, 0, MAX_LOOP_FILTER);
+          clamp(current_delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
       xd->prev_delta_lf_from_base = xd->current_delta_lf_from_base;
 #endif  // CONFIG_LOOPFILTER_LEVEL
     }

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 9099737..c8bae01 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -3389,12 +3389,14 @@
         for (j = 0; j < AOMMIN(cm->mib_size, cm->mi_rows - mi_row); j++) {
           for (k = 0; k < AOMMIN(cm->mib_size, cm->mi_cols - mi_col); k++) {
             cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)]
-                .mbmi.current_delta_lf_from_base =
-                clamp(current_delta_lf_from_base, 0, MAX_LOOP_FILTER);
+                .mbmi.current_delta_lf_from_base = clamp(
+                current_delta_lf_from_base, -MAX_LOOP_FILTER, MAX_LOOP_FILTER);
 #if CONFIG_LOOPFILTER_LEVEL
             for (int lf_id = 0; lf_id < FRAME_LF_COUNT; ++lf_id) {
               cm->mi[(mi_row + j) * cm->mi_stride + (mi_col + k)]
-                  .mbmi.curr_delta_lf[lf_id] = current_delta_lf_from_base;
+                  .mbmi.curr_delta_lf[lf_id] =
+                  clamp(current_delta_lf_from_base, -MAX_LOOP_FILTER,
+                        MAX_LOOP_FILTER);
             }
 #endif  // CONFIG_LOOPFILTER_LEVEL
           }