rtc: Fix prediction on keyframes for spatial layers

On superframes whose base is key: fix the reference
predicton structure so that the SL layer (for SL > 0)
only does the inter-layer prediction from the SL-1 layer.
This requires fixes to the sample encoder and in the
nonrd_pickmode.

In the current code SL2 was either predicting only from
SL0 (for ksvc), or it was predicting from both SL0 & SL1
(in non-ksvc mode), which was not the intended behavior.
So this fix improves the key frame quality for ksvc and
reduces the encode_time for non-ksvc.

Change-Id: Ifde750c741b53b42e4f27d493945036fb7e0e334

av1/encoder/nonrd_pickmode.c

diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 8d96b86..2b3adcb 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -1798,8 +1798,9 @@
   int use_alt_ref_frame =
       cpi->ppi->use_svc || cpi->sf.rt_sf.use_nonrd_altref_frame;
   int use_golden_ref_frame = 1;
+  int use_last_ref_frame = 1;
 
-  use_ref_frame[LAST_FRAME] = 1;  // we never skip LAST
+  use_ref_frame[LAST_FRAME] = 1;
 
   if (cpi->rc.frames_since_golden == 0 && gf_temporal_ref) {
     use_golden_ref_frame = 0;
@@ -1836,8 +1837,14 @@
   use_golden_ref_frame =
       cpi->ref_frame_flags & AOM_GOLD_FLAG ? use_golden_ref_frame : 0;
 
+  if (cpi->ppi->use_svc)
+    use_last_ref_frame =
+        cpi->ref_frame_flags & AOM_LAST_FLAG ? use_last_ref_frame : 0;
+
   use_ref_frame[ALTREF_FRAME] = use_alt_ref_frame;
   use_ref_frame[GOLDEN_FRAME] = use_golden_ref_frame;
+  use_ref_frame[LAST_FRAME] = use_last_ref_frame;
+  assert(use_last_ref_frame || use_golden_ref_frame || use_alt_ref_frame);
 }
 
 /*!\brief Estimates best intra mode for inter mode search

examples/svc_encoder_rtc.c

diff --git a/examples/svc_encoder_rtc.c b/examples/svc_encoder_rtc.c
index ac1238d..d745b79 100644
--- a/examples/svc_encoder_rtc.c
+++ b/examples/svc_encoder_rtc.c
@@ -579,11 +579,9 @@
   for (i = 0; i < REF_FRAMES; i++) ref_frame_config->refresh[i] = 0;
 
   if (ksvc_mode) {
-    // Same pattern as case 9.
+    // Same pattern as case 9, but the reference strucutre will be constrained
+    // below.
     layering_mode = 9;
-    if (!is_key_frame)
-      // No inter-layer prediction on inter-frames.
-      ref_frame_config->reference[SVC_LAST_FRAME] = 1;
   }
   switch (layering_mode) {
     case 0:
@@ -852,10 +850,6 @@
           ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 3;
         }
       }
-      if (layer_id->spatial_layer_id > 0 && !ksvc_mode) {
-        // Reference GOLDEN.
-        ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
-      }
       break;
     case 8:
       // 3 spatial and 3 temporal layer.
@@ -985,9 +979,20 @@
           ref_frame_config->ref_idx[SVC_GOLDEN_FRAME] = 4;
         }
       }
-      if (layer_id->spatial_layer_id > 0 && !ksvc_mode)
-        // Reference GOLDEN.
+      if (layer_id->spatial_layer_id > 0) {
+        // Always reference GOLDEN (inter-layer prediction).
         ref_frame_config->reference[SVC_GOLDEN_FRAME] = 1;
+        if (ksvc_mode) {
+          // KSVC: only keep the inter-layer reference (GOLDEN) for
+          // superframes whose base is key.
+          if (!is_key_frame) ref_frame_config->reference[SVC_GOLDEN_FRAME] = 0;
+        }
+        if (is_key_frame && layer_id->spatial_layer_id > 1) {
+          // On superframes whose base is key: remove LAST to avoid prediction
+          // off layer two levels below.
+          ref_frame_config->reference[SVC_LAST_FRAME] = 0;
+        }
+      }
       // For 3 spatial layer case 8 (where there is free buffer slot):
       // allow for top spatial layer to use additional temporal reference.
       // Additional reference is only updated on base temporal layer, every
@@ -1305,7 +1310,8 @@
       aom_codec_iter_t iter = NULL;
       const aom_codec_cx_pkt_t *pkt;
       int layer = 0;
-      int is_key_frame = (frame_cnt % cfg.kf_max_dist) == 0 && slx == 0;
+      // Flag for superframe whose base is key.
+      int is_key_frame = (frame_cnt % cfg.kf_max_dist) == 0;
       // For flexible mode:
       if (app_input.layering_mode >= 0) {
         // Set the reference/update flags, layer_id, and reference_map