Add encoder options for reduced tx type set

Adds a command line flag and control option for a reduced transform
type set.

Change-Id: I392c4f7f9b24f7d0451d1417fe680896ec2cf447
diff --git a/aom/aomcx.h b/aom/aomcx.h
index 360d302..b4f7a7a 100644
--- a/aom/aomcx.h
+++ b/aom/aomcx.h
@@ -864,6 +864,9 @@
 
   /*!\brief Sets the chroma subsampling y value */
   AV1E_SET_CHROMA_SUBSAMPLING_Y,
+
+  /*!\brief Control to use a reduced tx type set */
+  AV1E_SET_REDUCED_TX_TYPE_SET,
 };
 
 /*!\brief aom 1-D scaling mode
@@ -1193,6 +1196,9 @@
 AOM_CTRL_USE_TYPE(AV1E_SET_CHROMA_SUBSAMPLING_Y, unsigned int)
 #define AOM_CTRL_AV1E_SET_CHROMA_SUBSAMPLING_Y
 
+AOM_CTRL_USE_TYPE(AV1E_SET_REDUCED_TX_TYPE_SET, unsigned int)
+#define AOM_CTRL_AV1E_SET_REDUCED_TX_TYPE_SET
+
 /*!\endcond */
 /*! @} - end defgroup aom_encoder */
 #ifdef __cplusplus
diff --git a/apps/aomenc.c b/apps/aomenc.c
index 343e852..d187e1b 100644
--- a/apps/aomenc.c
+++ b/apps/aomenc.c
@@ -470,6 +470,8 @@
     NULL, "qm-min", 1, "Min quant matrix flatness (0..15), default is 8");
 static const arg_def_t qm_max = ARG_DEF(
     NULL, "qm-max", 1, "Max quant matrix flatness (0..15), default is 15");
+static const arg_def_t reduced_tx_type_set = ARG_DEF(
+    NULL, "reduced-tx-type-set", 1, "Use reduced set of transform types");
 #if CONFIG_DIST_8X8
 static const arg_def_t enable_dist_8x8 =
     ARG_DEF(NULL, "enable-dist-8x8", 1,
@@ -670,6 +672,7 @@
                                        &enable_qm,
                                        &qm_min,
                                        &qm_max,
+                                       &reduced_tx_type_set,
 #if CONFIG_DIST_8X8
                                        &enable_dist_8x8,
 #endif
@@ -734,6 +737,7 @@
                                         AV1E_SET_ENABLE_QM,
                                         AV1E_SET_QM_MIN,
                                         AV1E_SET_QM_MAX,
+                                        AV1E_SET_REDUCED_TX_TYPE_SET,
 #if CONFIG_DIST_8X8
                                         AV1E_SET_ENABLE_DIST_8X8,
 #endif
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 7a094ea..0cee628 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -111,6 +111,7 @@
 
   unsigned int chroma_subsampling_x;
   unsigned int chroma_subsampling_y;
+  int reduced_tx_type_set;
 };
 
 static struct av1_extracfg default_extra_cfg = {
@@ -188,6 +189,7 @@
 #endif
   0,  // chroma_subsampling_x
   0,  // chroma_subsampling_y
+  0,  // reduced_tx_type_set
 };
 
 struct aom_codec_alg_priv {
@@ -538,6 +540,7 @@
   oxcf->qm_v = extra_cfg->qm_v;
   oxcf->qm_minlevel = extra_cfg->qm_min;
   oxcf->qm_maxlevel = extra_cfg->qm_max;
+  oxcf->reduced_tx_type_set = extra_cfg->reduced_tx_type_set;
 #if CONFIG_DIST_8X8
   oxcf->using_dist_8x8 = extra_cfg->enable_dist_8x8;
   if (extra_cfg->tuning == AOM_TUNE_CDEF_DIST ||
@@ -1140,6 +1143,13 @@
   return update_extra_cfg(ctx, &extra_cfg);
 }
 
+static aom_codec_err_t ctrl_set_reduced_tx_type_set(aom_codec_alg_priv_t *ctx,
+                                                    va_list args) {
+  struct av1_extracfg extra_cfg = ctx->extra_cfg;
+  extra_cfg.reduced_tx_type_set = CAST(AV1E_SET_REDUCED_TX_TYPE_SET, args);
+  return update_extra_cfg(ctx, &extra_cfg);
+}
+
 static aom_codec_err_t ctrl_set_film_grain_test_vector(
     aom_codec_alg_priv_t *ctx, va_list args) {
   struct av1_extracfg extra_cfg = ctx->extra_cfg;
@@ -1837,6 +1847,7 @@
   { AV1E_SET_ALLOW_WARPED_MOTION, ctrl_set_allow_warped_motion },
   { AV1E_SET_ENABLE_SUPERRES, ctrl_set_enable_superres },
   { AV1E_SET_AQ_MODE, ctrl_set_aq_mode },
+  { AV1E_SET_REDUCED_TX_TYPE_SET, ctrl_set_reduced_tx_type_set },
   { AV1E_SET_DELTAQ_MODE, ctrl_set_deltaq_mode },
   { AV1E_SET_FRAME_PERIODIC_BOOST, ctrl_set_frame_periodic_boost },
   { AV1E_SET_TUNE_CONTENT, ctrl_set_tune_content },
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 33fd852..94c2a52 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -6022,7 +6022,7 @@
   const int num_planes = av1_num_planes(cm);
   // Indicates whether or not to use a default reduced set for ext-tx
   // rather than the potential full set of 16 transforms
-  cm->reduced_tx_set_used = 0;
+  cm->reduced_tx_set_used = cpi->oxcf.reduced_tx_type_set;
 
   if (cm->show_frame == 0) {
     int arf_offset = AOMMIN(
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 3fbb0ce..41cd62e 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -341,6 +341,7 @@
 
   unsigned int chroma_subsampling_x;
   unsigned int chroma_subsampling_y;
+  int reduced_tx_type_set;
 } AV1EncoderConfig;
 
 static INLINE int is_lossless_requested(const AV1EncoderConfig *cfg) {