Add color range detection in tune=butteraugli mode

Use the correct coefficients for YUV->RGB conversion based on the color
range. Get about additional -0.2% Butteraugli gains in
tune=butteraugli mode.

Performance of tune=butteraugli mode. Baseline: tune=psnr, speed 6
                        PSNR    SSIM    Butteraugli
HDRplus (limited range) 4.0%   -2.5%    -18.1%
HDRplus (full range)    3.4%   -2.0%    -19.2%

Change-Id: If0920d75c31863022081c3fe340940c59d228a0d
diff --git a/aom_dsp/butteraugli.c b/aom_dsp/butteraugli.c
index dff9fed..038efcd 100644
--- a/aom_dsp/butteraugli.c
+++ b/aom_dsp/butteraugli.c
@@ -18,7 +18,8 @@
 
 int aom_calc_butteraugli(const YV12_BUFFER_CONFIG *source,
                          const YV12_BUFFER_CONFIG *distorted, int bit_depth,
-                         float *dist_map) {
+                         aom_matrix_coefficients_t matrix_coefficients,
+                         aom_color_range_t color_range, float *dist_map) {
   (void)bit_depth;
   assert(bit_depth == 8);
   const int width = source->y_crop_width;
@@ -26,6 +27,15 @@
   const int ss_x = source->subsampling_x;
   const int ss_y = source->subsampling_y;
 
+  const struct YuvConstants *yuv_constants;
+  if (matrix_coefficients == AOM_CICP_MC_BT_709) {
+    if (color_range == AOM_CR_FULL_RANGE) return 0;
+    yuv_constants = &kYuvH709Constants;
+  } else {
+    yuv_constants = color_range == AOM_CR_FULL_RANGE ? &kYuvJPEGConstants
+                                                     : &kYuvI601Constants;
+  }
+
   const size_t stride_argb = width * 4;
   const size_t buffer_size = height * stride_argb;
   uint8_t *src_argb = (uint8_t *)aom_malloc(buffer_size);
@@ -39,27 +49,27 @@
   if (ss_x == 1 && ss_y == 1) {
     I420ToARGBMatrix(source->y_buffer, source->y_stride, source->u_buffer,
                      source->uv_stride, source->v_buffer, source->uv_stride,
-                     src_argb, stride_argb, &kYuvH709Constants, width, height);
+                     src_argb, stride_argb, yuv_constants, width, height);
     I420ToARGBMatrix(distorted->y_buffer, distorted->y_stride,
                      distorted->u_buffer, distorted->uv_stride,
                      distorted->v_buffer, distorted->uv_stride, distorted_argb,
-                     stride_argb, &kYuvH709Constants, width, height);
+                     stride_argb, yuv_constants, width, height);
   } else if (ss_x == 1 && ss_y == 0) {
     I422ToARGBMatrix(source->y_buffer, source->y_stride, source->u_buffer,
                      source->uv_stride, source->v_buffer, source->uv_stride,
-                     src_argb, stride_argb, &kYuvH709Constants, width, height);
+                     src_argb, stride_argb, yuv_constants, width, height);
     I422ToARGBMatrix(distorted->y_buffer, distorted->y_stride,
                      distorted->u_buffer, distorted->uv_stride,
                      distorted->v_buffer, distorted->uv_stride, distorted_argb,
-                     stride_argb, &kYuvH709Constants, width, height);
+                     stride_argb, yuv_constants, width, height);
   } else if (ss_x == 0 && ss_y == 0) {
     I444ToARGBMatrix(source->y_buffer, source->y_stride, source->u_buffer,
                      source->uv_stride, source->v_buffer, source->uv_stride,
-                     src_argb, stride_argb, &kYuvH709Constants, width, height);
+                     src_argb, stride_argb, yuv_constants, width, height);
     I444ToARGBMatrix(distorted->y_buffer, distorted->y_stride,
                      distorted->u_buffer, distorted->uv_stride,
                      distorted->v_buffer, distorted->uv_stride, distorted_argb,
-                     stride_argb, &kYuvH709Constants, width, height);
+                     stride_argb, yuv_constants, width, height);
   } else {
     aom_free(src_argb);
     aom_free(distorted_argb);
diff --git a/aom_dsp/butteraugli.h b/aom_dsp/butteraugli.h
index 06402aa..5304092 100644
--- a/aom_dsp/butteraugli.h
+++ b/aom_dsp/butteraugli.h
@@ -14,8 +14,10 @@
 
 #include "aom_scale/yv12config.h"
 
+// Returns a boolean that indicates success/failure.
 int aom_calc_butteraugli(const YV12_BUFFER_CONFIG *source,
                          const YV12_BUFFER_CONFIG *distorted, int bit_depth,
-                         float *dist_map);
+                         aom_matrix_coefficients_t matrix_coefficients,
+                         aom_color_range_t color_range, float *dist_map);
 
 #endif  // AOM_AOM_DSP_BUTTERAUGLI_H_
diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 679a574..fa23738 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -764,10 +764,11 @@
     if (img->bit_depth > 8) {
       ERROR("Only 8 bit depth images supported in tune=butteraugli mode.");
     }
-    if ((img->cp != 0 && img->cp != AOM_CICP_CP_BT_709) ||
-        (img->tc != 0 && img->tc != AOM_CICP_TC_BT_709) ||
-        (img->mc != 0 && img->mc != AOM_CICP_MC_BT_709)) {
-      ERROR("Only BT.709 images supported in tune=butteraugli mode.");
+    if (img->mc != 0 && img->mc != AOM_CICP_MC_BT_709 &&
+        img->mc != AOM_CICP_MC_BT_601 && img->mc != AOM_CICP_MC_BT_470_B_G) {
+      ERROR(
+          "Only BT.709 and BT.601 matrix coefficients supported in "
+          "tune=butteraugli mode. Identity matrix is treated as BT.601.");
     }
   }
 #endif
diff --git a/av1/encoder/tune_butteraugli.c b/av1/encoder/tune_butteraugli.c
index 359e5b5..b5691f9 100644
--- a/av1/encoder/tune_butteraugli.c
+++ b/av1/encoder/tune_butteraugli.c
@@ -24,7 +24,10 @@
                                               const YV12_BUFFER_CONFIG *source,
                                               const YV12_BUFFER_CONFIG *recon) {
   AV1_COMMON *const cm = &cpi->common;
+  SequenceHeader *const seq_params = &cm->seq_params;
   const CommonModeInfoParams *const mi_params = &cm->mi_params;
+  const aom_color_range_t color_range =
+      seq_params->color_range != 0 ? AOM_CR_FULL_RANGE : AOM_CR_STUDIO_RANGE;
   const int bit_depth = cpi->td.mb.e_mbd.bd;
   const int width = source->y_crop_width;
   const int height = source->y_crop_height;
@@ -33,7 +36,9 @@
 
   float *diffmap;
   CHECK_MEM_ERROR(cm, diffmap, aom_malloc(width * height * sizeof(*diffmap)));
-  if (!aom_calc_butteraugli(source, recon, bit_depth, diffmap)) {
+  if (!aom_calc_butteraugli(source, recon, bit_depth,
+                            seq_params->matrix_coefficients, color_range,
+                            diffmap)) {
     aom_internal_error(&cm->error, AOM_CODEC_ERROR,
                        "Failed to calculate Butteraugli distances.");
   }