Add encoder option to enable/disable 64-pt txfm

Allows turning on / off 64-pt DCT for testing.

Change-Id: Ib6dd54c95ffc1646d6dafd7e7374560962239af3
diff --git a/aom/aomcx.h b/aom/aomcx.h
index ef3131e..f665cea 100644
--- a/aom/aomcx.h
+++ b/aom/aomcx.h
@@ -739,6 +739,14 @@
    */
   AV1E_SET_ENABLE_ORDER_HINT,
 
+  /*!\brief Codec control function to turn on / off 64-length transforms.
+   *
+   * This will enable or disable usage of length 64 transforms in any
+   * direction. The default value is 1.
+   *
+   */
+  AV1E_SET_ENABLE_TX64,
+
   /*!\brief Codec control function to turn on / off dist-wtd compound mode
    * at sequence level.
    *
@@ -1151,6 +1159,9 @@
 AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_ORDER_HINT, unsigned int)
 #define AOM_CTRL_AV1E_SET_ENABLE_ORDER_HINT
 
+AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_TX64, unsigned int)
+#define AOM_CTRL_AV1E_SET_ENABLE_TX64
+
 AOM_CTRL_USE_TYPE(AV1E_SET_ENABLE_DIST_WTD_COMP, unsigned int)
 #define AOM_CTRL_AV1E_SET_ENABLE_DIST_WTD_COMP
 
diff --git a/apps/aomenc.c b/apps/aomenc.c
index 7038f52..14fc469 100644
--- a/apps/aomenc.c
+++ b/apps/aomenc.c
@@ -453,6 +453,9 @@
     ARG_DEF(NULL, "enable-order-hint", 1,
             "Enable order hint "
             "(0: false, 1: true (default))");
+static const arg_def_t enable_tx64 =
+    ARG_DEF(NULL, "enable-tx64", 1,
+            "Enable 64-pt transform (0: false, 1: true (default))");
 static const arg_def_t enable_dist_wtd_comp =
     ARG_DEF(NULL, "enable-dist-wtd-comp", 1,
             "Enable distance-weighted compound "
@@ -708,6 +711,7 @@
                                        &enable_dual_filter,
                                        &enable_intra_edge_filter,
                                        &enable_order_hint,
+                                       &enable_tx64,
                                        &enable_dist_wtd_comp,
                                        &enable_masked_comp,
                                        &enable_interintra_comp,
@@ -784,6 +788,7 @@
                                         AV1E_SET_ENABLE_DUAL_FILTER,
                                         AV1E_SET_ENABLE_INTRA_EDGE_FILTER,
                                         AV1E_SET_ENABLE_ORDER_HINT,
+                                        AV1E_SET_ENABLE_TX64,
                                         AV1E_SET_ENABLE_DIST_WTD_COMP,
                                         AV1E_SET_ENABLE_MASKED_COMP,
                                         AV1E_SET_ENABLE_INTERINTRA_COMP,
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 0b37fc3..60bcf60 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -98,6 +98,7 @@
   int enable_rect_partitions;    // enable rectangular partitions for sequence
   int enable_intra_edge_filter;  // enable intra-edge filter for sequence
   int enable_order_hint;         // enable order hint for sequence
+  int enable_tx64;               // enable 64-pt transform usage for sequence
   int enable_dist_wtd_comp;      // enable dist wtd compound for sequence
   int enable_ref_frame_mvs;      // sequence level
   int allow_ref_frame_mvs;       // frame level
@@ -186,6 +187,7 @@
   1,                            // enable rectangular partitions
   1,                            // enable intra edge filter
   1,                            // frame order hint
+  1,                            // enable 64-pt transform usage
   1,                            // dist-wtd compound
   1,                            // enable_ref_frame_mvs sequence level
   1,                            // allow ref_frame_mvs frame level
@@ -695,6 +697,7 @@
   oxcf->enable_dual_filter = extra_cfg->enable_dual_filter;
   oxcf->enable_rect_partitions = extra_cfg->enable_rect_partitions;
   oxcf->enable_intra_edge_filter = extra_cfg->enable_intra_edge_filter;
+  oxcf->enable_tx64 = extra_cfg->enable_tx64;
   oxcf->enable_order_hint = extra_cfg->enable_order_hint;
   oxcf->enable_dist_wtd_comp =
       extra_cfg->enable_dist_wtd_comp & extra_cfg->enable_order_hint;
@@ -1091,6 +1094,13 @@
   return update_extra_cfg(ctx, &extra_cfg);
 }
 
+static aom_codec_err_t ctrl_set_enable_tx64(aom_codec_alg_priv_t *ctx,
+                                            va_list args) {
+  struct av1_extracfg extra_cfg = ctx->extra_cfg;
+  extra_cfg.enable_tx64 = CAST(AV1E_SET_ENABLE_TX64, args);
+  return update_extra_cfg(ctx, &extra_cfg);
+}
+
 static aom_codec_err_t ctrl_set_enable_dist_wtd_comp(aom_codec_alg_priv_t *ctx,
                                                      va_list args) {
   struct av1_extracfg extra_cfg = ctx->extra_cfg;
@@ -1939,6 +1949,7 @@
   { AV1E_SET_ENABLE_DUAL_FILTER, ctrl_set_enable_dual_filter },
   { AV1E_SET_ENABLE_INTRA_EDGE_FILTER, ctrl_set_enable_intra_edge_filter },
   { AV1E_SET_ENABLE_ORDER_HINT, ctrl_set_enable_order_hint },
+  { AV1E_SET_ENABLE_TX64, ctrl_set_enable_tx64 },
   { AV1E_SET_ENABLE_DIST_WTD_COMP, ctrl_set_enable_dist_wtd_comp },
   { AV1E_SET_ENABLE_REF_FRAME_MVS, ctrl_set_enable_ref_frame_mvs },
   { AV1E_SET_ALLOW_REF_FRAME_MVS, ctrl_set_allow_ref_frame_mvs },
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 19fec97..f478843 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -325,6 +325,7 @@
   const cfg_options_t *cfg;
   int enable_rect_partitions;
   int enable_intra_edge_filter;
+  int enable_tx64;
   int enable_order_hint;
   int enable_dist_wtd_comp;
   int enable_ref_frame_mvs;
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 089c221..31a38cd 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3436,6 +3436,11 @@
   args.this_rd = this_rd;
   av1_init_rd_stats(&args.rd_stats);
 
+  if (!cpi->oxcf.enable_tx64 && txsize_sqr_up_map[tx_size] == TX_64X64) {
+    av1_invalid_rd_stats(rd_stats);
+    return;
+  }
+
   if (plane == 0) xd->mi[0]->tx_size = tx_size;
 
   av1_get_entropy_contexts(bsize, pd, args.t_above, args.t_left);
@@ -3658,6 +3663,8 @@
       if (tx_size_wide[n] < 8 || tx_size_high[n] < 8) continue;
     }
 #endif
+    if (!cpi->oxcf.enable_tx64 && txsize_sqr_up_map[n] == TX_64X64) continue;
+
     RD_STATS this_rd_stats;
     const int64_t rd =
         txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, n, FTXS_NONE);
@@ -4933,7 +4940,8 @@
                                          mbmi->sb_type, tx_size);
   struct macroblock_plane *const p = &x->plane[0];
 
-  const int try_no_split = 1;
+  const int try_no_split =
+      cpi->oxcf.enable_tx64 || txsize_sqr_up_map[tx_size] != TX_64X64;
   int try_split = tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH;
 #if CONFIG_DIST_8X8
   if (x->using_dist_8x8)