Merge "Various rect-tx fixes" into nextgenv2
diff --git a/vp10/common/warped_motion.c b/vp10/common/warped_motion.c
index 59999f7..7d5b7a2 100644
--- a/vp10/common/warped_motion.c
+++ b/vp10/common/warped_motion.c
@@ -562,3 +562,30 @@
                p_height, p_stride, subsampling_x, subsampling_y, x_scale,
                y_scale);
 }
+
+void vp10_integerize_model(const double *model, TransformationType wmtype,
+                           WarpedMotionParams *wm) {
+  wm->wmtype = wmtype;
+  switch (wmtype) {
+    case HOMOGRAPHY:
+      assert(fabs(model[8] - 1.0) < 1e-12);
+      wm->wmmat[7] =
+          (int)lrint(model[7] * (1 << WARPEDMODEL_ROW3HOMO_PREC_BITS));
+      wm->wmmat[6] =
+          (int)lrint(model[6] * (1 << WARPEDMODEL_ROW3HOMO_PREC_BITS));
+    /* fallthrough intended */
+    case AFFINE:
+      wm->wmmat[5] = (int)lrint(model[5] * (1 << WARPEDMODEL_PREC_BITS));
+      wm->wmmat[4] = (int)lrint(model[4] * (1 << WARPEDMODEL_PREC_BITS));
+    /* fallthrough intended */
+    case ROTZOOM:
+      wm->wmmat[3] = (int)lrint(model[3] * (1 << WARPEDMODEL_PREC_BITS));
+      wm->wmmat[2] = (int)lrint(model[2] * (1 << WARPEDMODEL_PREC_BITS));
+    /* fallthrough intended */
+    case TRANSLATION:
+      wm->wmmat[1] = (int)lrint(model[1] * (1 << WARPEDMODEL_PREC_BITS));
+      wm->wmmat[0] = (int)lrint(model[0] * (1 << WARPEDMODEL_PREC_BITS));
+      break;
+    default: assert(0 && "Invalid TransformationType");
+  }
+}
diff --git a/vp10/common/warped_motion.h b/vp10/common/warped_motion.h
index a9b53a5..3e566b6 100644
--- a/vp10/common/warped_motion.h
+++ b/vp10/common/warped_motion.h
@@ -62,4 +62,8 @@
                      uint8_t *pred, int p_col, int p_row, int p_width,
                      int p_height, int p_stride, int subsampling_x,
                      int subsampling_y, int x_scale, int y_scale);
+
+// Integerize model into the WarpedMotionParams structure
+void vp10_integerize_model(const double *model, TransformationType wmtype,
+                           WarpedMotionParams *wm);
 #endif  // VP10_COMMON_WARPED_MOTION_H
diff --git a/vp10/encoder/bitstream.h b/vp10/encoder/bitstream.h
index 5b6d8f0..d6e8895 100644
--- a/vp10/encoder/bitstream.h
+++ b/vp10/encoder/bitstream.h
@@ -24,8 +24,8 @@
 static INLINE int vp10_preserve_existing_gf(VP10_COMP *cpi) {
 #if CONFIG_EXT_REFS
   // Do not swap gf and arf indices for internal overlay frames
-  return !cpi->multi_arf_allowed &&
-         cpi->rc.is_src_frame_alt_ref && !cpi->rc.is_src_frame_ext_arf;
+  return !cpi->multi_arf_allowed && cpi->rc.is_src_frame_alt_ref &&
+         !cpi->rc.is_src_frame_ext_arf;
 #else
   return !cpi->multi_arf_allowed && cpi->refresh_golden_frame &&
          cpi->rc.is_src_frame_alt_ref;
diff --git a/vp10/encoder/encoder.c b/vp10/encoder/encoder.c
index 266c02d..640e953f 100644
--- a/vp10/encoder/encoder.c
+++ b/vp10/encoder/encoder.c
@@ -3162,14 +3162,14 @@
     // We need to modify the mapping accordingly
     cpi->arf_map[0] = cpi->alt_fb_idx;
 #endif
-    // TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to
-    // cpi->interp_filter_selected[GOLDEN_FRAME]?
+// TODO(zoeliu): Do we need to copy cpi->interp_filter_selected[0] over to
+// cpi->interp_filter_selected[GOLDEN_FRAME]?
 #if CONFIG_EXT_REFS
   } else if (cpi->rc.is_last_bipred_frame) {
     // Refresh the LAST_FRAME with the BWDREF_FRAME and retire the LAST3_FRAME
     // by updating the virtual indices. Note that the frame BWDREF_FRAME points
     // to now should be retired, and it should not be used before refreshed.
-    int tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES-1];
+    int tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES - 1];
 
     shift_last_ref_frames(cpi);
 
@@ -3180,7 +3180,7 @@
              sizeof(cpi->interp_filter_selected[BWDREF_FRAME]));
     }
     cpi->bwd_fb_idx = tmp;
-#endif     // CONFIG_EXT_REFS
+#endif  // CONFIG_EXT_REFS
 #if CONFIG_EXT_REFS
   } else if (cpi->rc.is_src_frame_ext_arf && cm->show_existing_frame) {
     // Deal with the special case for showing existing internal ALTREF_FRAME
@@ -3188,7 +3188,7 @@
     // by updating the virtual indices.
     const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
     int which_arf = gf_group->arf_ref_idx[gf_group->index];
-    int tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES-1];
+    int tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES - 1];
 
     shift_last_ref_frames(cpi);
 
@@ -3200,7 +3200,7 @@
     cpi->alt_fb_idx = tmp;
     // We need to modify the mapping accordingly
     cpi->arf_map[which_arf] = cpi->alt_fb_idx;
-#endif  // CONFIG_EXT_REFS
+#endif     // CONFIG_EXT_REFS
   } else { /* For non key/golden frames */
     if (cpi->refresh_alt_ref_frame) {
       int arf_idx = cpi->alt_fb_idx;
@@ -3334,7 +3334,7 @@
             &cpi->upsampled_ref_idx[cpi->lst_fb_idxes[LAST_REF_FRAMES - 1]],
             new_uidx);
 
-      tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES-1];
+      tmp = cpi->lst_fb_idxes[LAST_REF_FRAMES - 1];
 
       shift_last_ref_frames(cpi);
 
@@ -4334,7 +4334,7 @@
 #if CONFIG_EXT_REFS
   // Disable the use of BWDREF_FRAME for non-bipredictive frames.
   if (!(cpi->rc.is_bipred_frame || cpi->rc.is_last_bipred_frame ||
-      (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs)))
+        (cpi->rc.is_bwd_ref_frame && cpi->num_extra_arfs)))
     flags &= ~VPX_BWD_FLAG;
 #endif  // CONFIG_EXT_REFS
 
@@ -4415,7 +4415,7 @@
   // The arf_sign_bias will be one for internal ARFs'
   arf_sign_bias = cpi->rc.source_alt_ref_active &&
                   (!cpi->refresh_alt_ref_frame ||
-                  (gf_group->rf_level[gf_group->index] == GF_ARF_LOW));
+                   (gf_group->rf_level[gf_group->index] == GF_ARF_LOW));
 #else
   if ((cpi->oxcf.pass == 2) && cpi->multi_arf_allowed) {
     const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
@@ -4458,8 +4458,8 @@
     ref_total[ref] += cpi->interp_filter_selected[arf_idx][ifilter];
 #else
   for (ref = LAST_FRAME; ref <= ALTREF_FRAME; ++ref)
-      for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter)
-        ref_total[ref] += cpi->interp_filter_selected[ref][ifilter];
+    for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter)
+      ref_total[ref] += cpi->interp_filter_selected[ref][ifilter];
 #endif
 
   for (ifilter = EIGHTTAP_REGULAR; ifilter < SWITCHABLE_FILTERS; ++ifilter) {
@@ -4483,7 +4483,7 @@
 #endif  // CONFIG_EXT_REFS
         (ref_total[ALTREF_FRAME] == 0 ||
          cpi->interp_filter_selected[arf_idx][ifilter] * 50 <
-         ref_total[ALTREF_FRAME]))
+             ref_total[ALTREF_FRAME]))
       mask |= 1 << ifilter;
   }
   return mask;
@@ -5049,9 +5049,9 @@
     const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
     rc->is_src_frame_alt_ref =
 #if CONFIG_EXT_REFS
-      (gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE) ||
+        (gf_group->update_type[gf_group->index] == INTNL_OVERLAY_UPDATE) ||
 #endif
-      (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE);
+        (gf_group->update_type[gf_group->index] == OVERLAY_UPDATE);
   } else {
     rc->is_src_frame_alt_ref =
         cpi->alt_ref_source && (source == cpi->alt_ref_source);
diff --git a/vp10/encoder/firstpass.c b/vp10/encoder/firstpass.c
index 1d1bf52..c7838fe 100644
--- a/vp10/encoder/firstpass.c
+++ b/vp10/encoder/firstpass.c
@@ -1595,11 +1595,11 @@
           (rc->baseline_gf_interval - rc->source_alt_ref_pending);
   int bipred_group_end = 0;
   int bipred_frame_index = 0;
-  int arf_pos[MAX_EXT_ARFS+1];
-  const unsigned char ext_arf_interval = (unsigned char)
-      (rc->baseline_gf_interval / (cpi->num_extra_arfs + 1) - 1);
+  int arf_pos[MAX_EXT_ARFS + 1];
+  const unsigned char ext_arf_interval =
+      (unsigned char)(rc->baseline_gf_interval / (cpi->num_extra_arfs + 1) - 1);
   int which_arf = cpi->num_extra_arfs;
-  int subgroup_interval[MAX_EXT_ARFS+1];
+  int subgroup_interval[MAX_EXT_ARFS + 1];
   int ext_arf_boost[MAX_EXT_ARFS];
   int is_sg_bipred_enabled = is_bipred_enabled;
   int accumulative_subgroup_interval = 0;
@@ -1684,15 +1684,16 @@
     // We index ALTREF's as: KEY ----- ALT2 ----- ALT1 ----- ALT0
     // but code them in the following order:
     // KEY-ALT0-ALT2 ----- OVERLAY2-ALT1 ----- OVERLAY1 ----- OVERLAY0
-    arf_pos[0] = frame_index + cpi->num_extra_arfs +
-                 gf_group->arf_src_offset[1] + 1;
+    arf_pos[0] =
+        frame_index + cpi->num_extra_arfs + gf_group->arf_src_offset[1] + 1;
     for (i = 0; i < cpi->num_extra_arfs; ++i) {
-      arf_pos[i + 1] = frame_index +
-                       (cpi->num_extra_arfs - i)*(ext_arf_interval + 2);
+      arf_pos[i + 1] =
+          frame_index + (cpi->num_extra_arfs - i) * (ext_arf_interval + 2);
       subgroup_interval[i] = arf_pos[i] - arf_pos[i + 1] - (i == 0 ? 1 : 2);
     }
     subgroup_interval[cpi->num_extra_arfs] = arf_pos[cpi->num_extra_arfs] -
-        frame_index - (cpi->num_extra_arfs == 0 ? 1 : 2);
+                                             frame_index -
+                                             (cpi->num_extra_arfs == 0 ? 1 : 2);
 #endif  // CONFIG_EXT_REFS
 
     ++frame_index;
@@ -1766,7 +1767,8 @@
     // If we are going to have ARFs, check if we can have BWDREF in this
     // subgroup.
     if (rc->source_alt_ref_pending) {
-      is_sg_bipred_enabled =  is_bipred_enabled &&
+      is_sg_bipred_enabled =
+          is_bipred_enabled &&
           (subgroup_interval[which_arf] > rc->bipred_group_interval);
     }
     // NOTE: BIDIR_PRED is only enabled when the length of the bi-predictive
@@ -1801,7 +1803,7 @@
       // In addition, we need to avoid coding a BRF right before an ARF.
       if (bipred_frame_index == 1 &&
           (i + 2 + cur_brf_src_offset) >= accumulative_subgroup_interval) {
-          bipred_group_end = 1;
+        bipred_group_end = 1;
       }
     } else {
 #endif  // CONFIG_EXT_REFS
@@ -1857,10 +1859,10 @@
 #endif
   }
 
-  // Note:
-  // We need to configure the frame at the end of the sequence + 1 that will be
-  // the start frame for the next group. Otherwise prior to the call to
-  // vp10_rc_get_second_pass_params() the data will be undefined.
+// Note:
+// We need to configure the frame at the end of the sequence + 1 that will be
+// the start frame for the next group. Otherwise prior to the call to
+// vp10_rc_get_second_pass_params() the data will be undefined.
 #if CONFIG_EXT_REFS
   gf_group->arf_update_idx[frame_index] = 0;
   gf_group->arf_ref_idx[frame_index] = 0;
@@ -2109,9 +2111,8 @@
 
 #if CONFIG_EXT_REFS
   // Compute how many extra alt_refs we can have
-  cpi->num_extra_arfs =
-      get_number_of_extra_arfs(rc->baseline_gf_interval,
-                               rc->source_alt_ref_pending);
+  cpi->num_extra_arfs = get_number_of_extra_arfs(rc->baseline_gf_interval,
+                                                 rc->source_alt_ref_pending);
   // Currently at maximum two extra ARFs' are allowed
   assert(cpi->num_extra_arfs <= 2);
 #endif
@@ -2566,29 +2567,29 @@
 
   switch (twopass->gf_group.update_type[twopass->gf_group.index]) {
     case KF_UPDATE:
-      cpi->refresh_last_frame = 1;
-      cpi->refresh_golden_frame = 1;
 #if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 1;
 #endif  // CONFIG_EXT_REFS
+      cpi->refresh_last_frame = 1;
+      cpi->refresh_golden_frame = 1;
       cpi->refresh_alt_ref_frame = 1;
       break;
 
     case LF_UPDATE:
-      cpi->refresh_last_frame = 1;
-      cpi->refresh_golden_frame = 0;
 #if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
 #endif  // CONFIG_EXT_REFS
+      cpi->refresh_last_frame = 1;
+      cpi->refresh_golden_frame = 0;
       cpi->refresh_alt_ref_frame = 0;
       break;
 
     case GF_UPDATE:
-      cpi->refresh_last_frame = 1;
-      cpi->refresh_golden_frame = 1;
 #if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 0;
 #endif  // CONFIG_EXT_REFS
+      cpi->refresh_last_frame = 1;
+      cpi->refresh_golden_frame = 1;
       cpi->refresh_alt_ref_frame = 0;
       break;
 
@@ -2603,11 +2604,11 @@
       break;
 
     case ARF_UPDATE:
-      cpi->refresh_last_frame = 0;
-      cpi->refresh_golden_frame = 0;
 #if CONFIG_EXT_REFS
       cpi->refresh_bwd_ref_frame = 1;
 #endif  // CONFIG_EXT_REFS
+      cpi->refresh_last_frame = 0;
+      cpi->refresh_golden_frame = 0;
       cpi->refresh_alt_ref_frame = 1;
       break;
 
diff --git a/vp10/encoder/firstpass.h b/vp10/encoder/firstpass.h
index 156fa2c..aacd095 100644
--- a/vp10/encoder/firstpass.h
+++ b/vp10/encoder/firstpass.h
@@ -43,12 +43,12 @@
 // Length of the bi-predictive frame group (BFG)
 // NOTE: Currently each BFG contains one backward ref (BWF) frame plus a certain
 //       number of bi-predictive frames.
-#define BFG_INTERVAL          2
+#define BFG_INTERVAL 2
 // The maximum number of extra ALT_REF's
 // NOTE: This number cannot be greater than 2 or the reference frame buffer will
 //       overflow.
-#define MAX_EXT_ARFS          2
-#define MIN_EXT_ARF_INTERVAL  4
+#define MAX_EXT_ARFS 2
+#define MIN_EXT_ARF_INTERVAL 4
 #endif  // CONFIG_EXT_REFS
 
 #define VLOW_MOTION_THRESHOLD 950
@@ -185,8 +185,11 @@
 #if CONFIG_EXT_REFS
 static inline int get_number_of_extra_arfs(int interval, int arf_pending) {
   if (arf_pending && MAX_EXT_ARFS > 0)
-    return interval >= MIN_EXT_ARF_INTERVAL*(MAX_EXT_ARFS+1) ? MAX_EXT_ARFS :
-           interval >= MIN_EXT_ARF_INTERVAL*MAX_EXT_ARFS ? MAX_EXT_ARFS - 1 : 0;
+    return interval >= MIN_EXT_ARF_INTERVAL * (MAX_EXT_ARFS + 1)
+               ? MAX_EXT_ARFS
+               : interval >= MIN_EXT_ARF_INTERVAL * MAX_EXT_ARFS
+                     ? MAX_EXT_ARFS - 1
+                     : 0;
   else
     return 0;
 }
diff --git a/vp10/encoder/ratectrl.c b/vp10/encoder/ratectrl.c
index c7e4348..4e7dd64 100644
--- a/vp10/encoder/ratectrl.c
+++ b/vp10/encoder/ratectrl.c
@@ -920,7 +920,7 @@
 #else
     1.00,  // INTER_HIGH
     1.50,  // GF_ARF_LOW
-#endif  // CONFIG_EXT_REFS
+#endif     // CONFIG_EXT_REFS
     2.00,  // GF_ARF_STD
     2.00,  // KF_STD
   };
diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 63d8e9b..15f5ce6 100644
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -225,7 +225,8 @@
   128, 128, 128
   // TODO(weitinglin): We should investigate if the values should be the same
   //                   as the value used by OVERLAY frame
-  , 144
+  ,
+  144
 #endif  // CONFIG_EXT_REFS
 };
 
diff --git a/vp10/encoder/temporal_filter.c b/vp10/encoder/temporal_filter.c
index 691e157..263177c 100644
--- a/vp10/encoder/temporal_filter.c
+++ b/vp10/encoder/temporal_filter.c
@@ -631,10 +631,10 @@
 
   // Apply context specific adjustments to the arnr filter parameters.
   adjust_arnr_filter(cpi, distance, rc->gfu_boost, &frames_to_blur, &strength);
-  // TODO(weitinglin): Currently, we enforce the filtering strength on
-  //                   extra ARFs' to be zeros. We should investigate in which
-  //                   case it is more beneficial to use non-zero strength
-  //                   filtering.
+// TODO(weitinglin): Currently, we enforce the filtering strength on
+//                   extra ARFs' to be zeros. We should investigate in which
+//                   case it is more beneficial to use non-zero strength
+//                   filtering.
 #if CONFIG_EXT_REFS
   if (gf_group->rf_level[gf_group->index] == GF_ARF_LOW) {
     strength = 0;