commit 6a0245c53bfd48db15ef7593375e2ed46c311182
author Rostislav Pehlivanov <rpehlivanov@mozilla.com> Wed Jan 17 18:30:00 2018 +0000
committer Rostislav Pehlivanov <rpehlivanov@mozilla.com> Fri Jan 19 22:44:19 2018 +0000
tree a270f5ade900ed60a140329cd06131a2d9f2be3f
parent 1f309004134c63be3742926b9825495bd1f3256c

spatial_segmentation: improve edge case handling

Change-Id: If47ac212044a977b12c8df1322759d173dd29ed5

av1/common/pred_common.h

diff --git a/av1/common/pred_common.h b/av1/common/pred_common.h
index 7c3b042..e381610 100644
--- a/av1/common/pred_common.h
+++ b/av1/common/pred_common.h
@@ -21,8 +21,10 @@
 #endif
 
 #if CONFIG_SPATIAL_SEGMENTATION
-/* Picks CDFs based on number of matching segment IDs */
+/* Picks CDFs based on number of matching/out-of-bounds segment IDs */
 static INLINE int pick_spatial_seg_cdf(int prev_ul, int prev_u, int prev_l) {
+  if (prev_ul < 0 || prev_u < 0 || prev_l < 0) /* Edge case */
+    return 0;
   if ((prev_ul == prev_u) && (prev_ul == prev_l))
     return 2;
   else if ((prev_ul == prev_u) || (prev_ul == prev_l) || (prev_u == prev_l))
@@ -31,8 +33,12 @@
     return 0;
 }
 
+/* If 2 or more are identical returns that as predictor, otherwise prev_l */
 static INLINE int pick_spatial_seg_pred(int prev_ul, int prev_u, int prev_l) {
-  /* If 2 or more are identical returns that as predictor, otherwise prev_l */
+  if (prev_u == -1) /* Edge case */
+    return prev_l == -1 ? 0 : prev_l;
+  if (prev_l == -1) /* Edge case */
+    return prev_u;
   return (prev_ul == prev_u) ? prev_u : prev_l;
 }
 

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index e1a69e1..4b6694e 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -350,9 +350,9 @@
                            int mi_row, int mi_col, aom_reader *r, int skip) {
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
   struct segmentation_probs *const segp = &ec_ctx->seg;
-  int prev_ul = 0; /* Top left segment_id */
-  int prev_l = 0;  /* Current left segment_id */
-  int prev_u = 0;  /* Current top segment_id */
+  int prev_ul = -1; /* Top left segment_id */
+  int prev_l = -1;  /* Current left segment_id */
+  int prev_u = -1;  /* Current top segment_id */
 
   if ((xd->up_available) && (xd->left_available))
     prev_ul = get_segment_id(cm, cm->current_frame_seg_map, BLOCK_4X4,

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 8aa8bb0..25eac0a 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -593,9 +593,9 @@
                              int mi_col, int skip) {
   AV1_COMMON *const cm = &cpi->common;
   MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
-  int prev_ul = 0; /* Top left segment_id */
-  int prev_l = 0;  /* Current left segment_id */
-  int prev_u = 0;  /* Current top segment_id */
+  int prev_ul = -1; /* Top left segment_id */
+  int prev_l = -1;  /* Current left segment_id */
+  int prev_u = -1;  /* Current top segment_id */
 
   if (!seg->enabled || !seg->update_map) return;