svc: Check for invalid params for svc layer id setting

Return AOM_CODEC_INVALID_PARAM if the spatial or
temporal layer id is set to be below 0 or above
the configured setting. Update the documentation
of the control.

Bug: 503975732, 503993984, 503993985
Change-Id: Idb30133e116bddb8ac869bb7dcf37a4b99c12f64

aom/aomcx.h

diff --git a/aom/aomcx.h b/aom/aomcx.h
index 7c7ad7b..1c344f7 100644
--- a/aom/aomcx.h
+++ b/aom/aomcx.h
@@ -198,7 +198,9 @@
   AOME_SET_SCALEMODE = 11,
 
   /*!\brief Codec control function to set encoder spatial layer id, int
-   * parameter.
+   * parameter. Spatial layer id must be within valid range of 0 to the
+   * allowed number of spatial layers, set via the control
+   * AV1E_SET_SVC_PARAMS, or via AOME_SET_SPATIAL_LAYER_ID.
    */
   AOME_SET_SPATIAL_LAYER_ID = 12,
 
@@ -1289,7 +1291,9 @@
   /* NOTE: enums 145-149 unused */
 
   /*!\brief Codec control function to set the layer id, aom_svc_layer_id_t*
-   * parameter
+   * parameter. Layer id for spatial or temporal layer must be within valid
+   * range of 0 to the allowed number of spatial or temporal layers, set via
+   * the control AV1E_SET_SVC_PARAMS, or via AOME_SET_SPATIAL_LAYER_ID.
    */
   AV1E_SET_SVC_LAYER_ID = 131,
 

av1/av1_cx_iface.c

diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 3b63df5..1ab5822 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -4034,7 +4034,8 @@
 static aom_codec_err_t ctrl_set_spatial_layer_id(aom_codec_alg_priv_t *ctx,
                                                  va_list args) {
   const int spatial_layer_id = va_arg(args, int);
-  if (spatial_layer_id >= MAX_NUM_SPATIAL_LAYERS)
+  if (spatial_layer_id < 0 ||
+      spatial_layer_id >= (int)ctx->ppi->number_spatial_layers)
     return AOM_CODEC_INVALID_PARAM;
   ctx->ppi->cpi->common.spatial_layer_id = spatial_layer_id;
   return AOM_CODEC_OK;
@@ -4064,6 +4065,11 @@
 static aom_codec_err_t ctrl_set_layer_id(aom_codec_alg_priv_t *ctx,
                                          va_list args) {
   aom_svc_layer_id_t *const data = va_arg(args, aom_svc_layer_id_t *);
+  if (data->spatial_layer_id < 0 || data->temporal_layer_id < 0 ||
+      data->spatial_layer_id >= (int)ctx->ppi->number_spatial_layers ||
+      data->temporal_layer_id >= (int)ctx->ppi->number_temporal_layers) {
+    return AOM_CODEC_INVALID_PARAM;
+  }
   ctx->ppi->cpi->common.spatial_layer_id = data->spatial_layer_id;
   ctx->ppi->cpi->common.temporal_layer_id = data->temporal_layer_id;
   ctx->ppi->cpi->svc.spatial_layer_id = data->spatial_layer_id;