Replace libvmaf with libvmaf_rc when CONFIG_USE_VMAF_RC enabled
This CL also replaces VMAF score with VMAF_NEG score during RDO in
tune=vmaf_neg mode. Got additional -0.3% VMAF_NEG gains.
baseline=psnr
avg_psnr ssim vmaf vmaf_neg
lowres_bd10 11.633 4.393 -20.757 -8.446
midres_bd10 10.772 5.555 -21.276 -7.924
midres 10.778 6.660 -17.250 -6.720
hdres 9.958 6.408 -20.309 -8.138
Change-Id: I9be2b95ab226226180a7df368d8f3dd71d55b846
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7e1f730..af89576 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -438,19 +438,6 @@
if(CONFIG_TUNE_VMAF)
find_package(PkgConfig)
- if(PKG_CONFIG_FOUND)
- pkg_check_modules(VMAF REQUIRED libvmaf)
- else()
- find_library(VMAF_LIBRARIES vmaf)
- find_path(VMAF_INCLUDE_DIRS libvmaf.h PATH_SUFFIXES libvmaf)
- if(VMAF_LIBRARIES AND VMAF_INCLUDE_DIRS)
- message(STATUS "Found VMAF library: ${VMAF_LIBRARIES}")
- message(STATUS "Found VMAF include: ${VMAF_INCLUDE_DIRS}")
- else()
- message(FATAL_ERROR "VMAF library not found.")
- endif()
- endif()
-
if(CONFIG_USE_VMAF_RC)
if(PKG_CONFIG_FOUND)
pkg_check_modules(VMAF_RC REQUIRED libvmaf_rc)
@@ -463,17 +450,29 @@
else()
message(FATAL_ERROR "CONFIG_USE_VMAF_RC error: pkg-config not found.")
endif()
+ else()
+ if(PKG_CONFIG_FOUND)
+ pkg_check_modules(VMAF REQUIRED libvmaf)
+ else()
+ find_library(VMAF_LIBRARIES vmaf)
+ find_path(VMAF_INCLUDE_DIRS libvmaf.h PATH_SUFFIXES libvmaf)
+ if(VMAF_LIBRARIES AND VMAF_INCLUDE_DIRS)
+ message(STATUS "Found VMAF library: ${VMAF_LIBRARIES}")
+ message(STATUS "Found VMAF include: ${VMAF_INCLUDE_DIRS}")
+ else()
+ message(FATAL_ERROR "VMAF library not found.")
+ endif()
+ endif()
+ target_link_libraries(aom PRIVATE ${VMAF_LDFLAGS} ${VMAF_LIBRARIES})
+ target_include_directories(aom PRIVATE ${VMAF_INCLUDE_DIRS})
+ if(VMAF_CFLAGS)
+ append_compiler_flag("${VMAF_CFLAGS}")
+ endif()
endif()
-
set_target_properties(aom PROPERTIES LINKER_LANGUAGE CXX)
if(BUILD_SHARED_LIBS)
set_target_properties(aom_static PROPERTIES LINKER_LANGUAGE CXX)
endif()
- target_link_libraries(aom PRIVATE ${VMAF_LDFLAGS} ${VMAF_LIBRARIES})
- target_include_directories(aom PRIVATE ${VMAF_INCLUDE_DIRS})
- if(VMAF_CFLAGS)
- append_compiler_flag("${VMAF_CFLAGS}")
- endif()
endif()
endif()
diff --git a/aom_dsp/vmaf.c b/aom_dsp/vmaf.c
index b0be482..ee72e57 100644
--- a/aom_dsp/vmaf.c
+++ b/aom_dsp/vmaf.c
@@ -12,7 +12,9 @@
#include "aom_dsp/vmaf.h"
#include <assert.h>
+#if !CONFIG_USE_VMAF_RC
#include <libvmaf.h>
+#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -29,6 +31,12 @@
#include "aom_dsp/blend.h"
#include "aom_ports/system_state.h"
+static void vmaf_fatal_error(const char *message) {
+ fprintf(stderr, "Fatal error: %s\n", message);
+ exit(EXIT_FAILURE);
+}
+
+#if !CONFIG_USE_VMAF_RC
typedef struct FrameData {
const YV12_BUFFER_CONFIG *source;
const YV12_BUFFER_CONFIG *distorted;
@@ -36,11 +44,6 @@
int bit_depth;
} FrameData;
-static void vmaf_fatal_error(const char *message) {
- fprintf(stderr, "Fatal error: %s\n", message);
- exit(EXIT_FAILURE);
-}
-
// A callback function used to pass data to VMAF.
// Returns 0 after reading a frame.
// Returns 2 when there is no more frame to read.
@@ -173,6 +176,7 @@
aom_clear_system_state();
}
+#endif
#if CONFIG_USE_VMAF_RC
void aom_init_vmaf_rc(VmafModel **vmaf_model, const char *model_path) {
diff --git a/aom_dsp/vmaf.h b/aom_dsp/vmaf.h
index 775c9cb..607d4bd 100644
--- a/aom_dsp/vmaf.h
+++ b/aom_dsp/vmaf.h
@@ -46,6 +46,15 @@
#endif
} TuneVMAFInfo;
+#if CONFIG_USE_VMAF_RC
+void aom_init_vmaf_rc(VmafModel **vmaf_model, const char *model_path);
+
+void aom_calc_vmaf_rc(VmafModel *vmaf_model, const YV12_BUFFER_CONFIG *source,
+ const YV12_BUFFER_CONFIG *distorted, int bit_depth,
+ int cal_vmaf_neg, double *vmaf);
+
+void aom_close_vmaf_rc(VmafModel *vmaf_model);
+#else
void aom_calc_vmaf(const char *model_path, const YV12_BUFFER_CONFIG *source,
const YV12_BUFFER_CONFIG *distorted, int bit_depth,
double *vmaf);
@@ -55,15 +64,6 @@
int (*read_frame)(float *ref_data, float *main_data, float *temp_data,
int stride_byte, void *user_data),
int frame_width, int frame_height, int bit_depth, double *vmaf);
-
-#if CONFIG_USE_VMAF_RC
-void aom_init_vmaf_rc(VmafModel **vmaf_model, const char *model_path);
-
-void aom_calc_vmaf_rc(VmafModel *vmaf_model, const YV12_BUFFER_CONFIG *source,
- const YV12_BUFFER_CONFIG *distorted, int bit_depth,
- int cal_vmaf_neg, double *vmaf);
-
-void aom_close_vmaf_rc(VmafModel *vmaf_model);
#endif // CONFIG_USE_VMAF_RC
#endif // AOM_AOM_DSP_VMAF_H_
diff --git a/av1/encoder/tune_vmaf.c b/av1/encoder/tune_vmaf.c
index 62c29d4..ec63379 100644
--- a/av1/encoder/tune_vmaf.c
+++ b/av1/encoder/tune_vmaf.c
@@ -16,6 +16,9 @@
#include "aom_ports/system_state.h"
#include "av1/encoder/extend.h"
#include "av1/encoder/rdopt.h"
+#if CONFIG_USE_VMAF_RC
+#include "config/aom_scale_rtcd.h"
+#endif
static const double kBaselineVmaf = 97.42773;
@@ -160,8 +163,16 @@
YV12_BUFFER_CONFIG *const sharpened) {
const int bit_depth = cpi->td.mb.e_mbd.bd;
double new_vmaf;
+
+#if CONFIG_USE_VMAF_RC
+ aom_calc_vmaf_rc(cpi->vmaf_info.vmaf_model, source, sharpened, bit_depth,
+ cpi->oxcf.tune_cfg.tuning == AOM_TUNE_VMAF_NEG_MAX_GAIN,
+ &new_vmaf);
+#else
aom_calc_vmaf(cpi->oxcf.tune_cfg.vmaf_model_path, source, sharpened,
bit_depth, &new_vmaf);
+#endif
+
const double sharpened_var = frame_average_variance(cpi, sharpened);
return source_variance / sharpened_var * (new_vmaf - kBaselineVmaf);
}
@@ -447,6 +458,7 @@
aom_clear_system_state();
}
+#if !CONFIG_USE_VMAF_RC
typedef struct FrameData {
const YV12_BUFFER_CONFIG *source, *blurred;
int block_w, block_h, num_rows, num_cols, row, col, bit_depth;
@@ -531,6 +543,7 @@
return 2;
}
}
+#endif
void av1_set_mb_vmaf_rdmult_scaling(AV1_COMP *cpi) {
AV1_COMMON *cm = &cpi->common;
@@ -567,6 +580,15 @@
cm->features.byte_alignment);
gaussian_blur(bit_depth, &resized_source, &blurred);
+#if CONFIG_USE_VMAF_RC
+ YV12_BUFFER_CONFIG recon;
+ memset(&recon, 0, sizeof(recon));
+ aom_alloc_frame_buffer(&recon, resized_y_width, resized_y_height, 1, 1,
+ cm->seq_params.use_highbitdepth,
+ cpi->oxcf.border_in_pixels,
+ cm->features.byte_alignment);
+ aom_yv12_copy_frame(&resized_source, &recon, 1);
+#else
double *scores = aom_malloc(sizeof(*scores) * (num_rows * num_cols));
memset(scores, 0, sizeof(*scores) * (num_rows * num_cols));
FrameData frame_data;
@@ -582,6 +604,7 @@
aom_calc_vmaf_multi_frame(&frame_data, cpi->oxcf.tune_cfg.vmaf_model_path,
update_frame, resized_y_width, resized_y_height,
bit_depth, scores);
+#endif
// Loop through each 'block_size' block.
for (int row = 0; row < num_rows; ++row) {
@@ -596,12 +619,47 @@
uint8_t *const blurred_buf =
blurred.y_buffer + row_offset_y * blurred.y_stride + col_offset_y;
- const double vmaf = scores[index];
- const double dvmaf = kBaselineVmaf - vmaf;
unsigned int sse;
cpi->fn_ptr[resized_block_size].vf(orig_buf, resized_source.y_stride,
blurred_buf, blurred.y_stride, &sse);
+#if CONFIG_USE_VMAF_RC
+ uint8_t *const recon_buf =
+ recon.y_buffer + row_offset_y * recon.y_stride + col_offset_y;
+ // Set recon buf
+ if (cpi->common.seq_params.use_highbitdepth) {
+ highbd_unsharp_rect(CONVERT_TO_SHORTPTR(blurred_buf), blurred.y_stride,
+ CONVERT_TO_SHORTPTR(blurred_buf), blurred.y_stride,
+ CONVERT_TO_SHORTPTR(recon_buf), recon.y_stride,
+ resized_block_w, resized_block_h, 0.0, bit_depth);
+ } else {
+ unsharp_rect(blurred_buf, blurred.y_stride, blurred_buf,
+ blurred.y_stride, recon_buf, recon.y_stride,
+ resized_block_w, resized_block_h, 0.0);
+ }
+
+ double vmaf;
+ aom_calc_vmaf_rc(
+ cpi->vmaf_info.vmaf_model, &resized_source, &recon, bit_depth,
+ cpi->oxcf.tune_cfg.tuning == AOM_TUNE_VMAF_NEG_MAX_GAIN, &vmaf);
+
+ // Restore recon buf
+ if (cpi->common.seq_params.use_highbitdepth) {
+ highbd_unsharp_rect(
+ CONVERT_TO_SHORTPTR(orig_buf), resized_source.y_stride,
+ CONVERT_TO_SHORTPTR(orig_buf), resized_source.y_stride,
+ CONVERT_TO_SHORTPTR(recon_buf), recon.y_stride, resized_block_w,
+ resized_block_h, 0.0, bit_depth);
+ } else {
+ unsharp_rect(orig_buf, resized_source.y_stride, orig_buf,
+ resized_source.y_stride, recon_buf, recon.y_stride,
+ resized_block_w, resized_block_h, 0.0);
+ }
+#else
+ const double vmaf = scores[index];
+#endif
+ const double dvmaf = kBaselineVmaf - vmaf;
+
const double mse =
(double)sse / (double)(resized_y_width * resized_y_height);
double weight;
@@ -620,7 +678,9 @@
aom_free_frame_buffer(&resized_source);
aom_free_frame_buffer(&blurred);
+#if !CONFIG_USE_VMAF_RC
aom_free(scores);
+#endif
aom_clear_system_state();
}
@@ -882,8 +942,14 @@
void av1_update_vmaf_curve(AV1_COMP *cpi, YV12_BUFFER_CONFIG *source,
YV12_BUFFER_CONFIG *recon) {
const int bit_depth = cpi->td.mb.e_mbd.bd;
+#if CONFIG_USE_VMAF_RC
+ aom_calc_vmaf_rc(cpi->vmaf_info.vmaf_model, source, recon, bit_depth,
+ cpi->oxcf.tune_cfg.tuning == AOM_TUNE_VMAF_NEG_MAX_GAIN,
+ &cpi->vmaf_info.last_frame_vmaf);
+#else
aom_calc_vmaf(cpi->oxcf.tune_cfg.vmaf_model_path, source, recon, bit_depth,
&cpi->vmaf_info.last_frame_vmaf);
+#endif
if (cpi->common.seq_params.use_highbitdepth) {
assert(source->flags & YV12_FLAG_HIGHBITDEPTH);
assert(recon->flags & YV12_FLAG_HIGHBITDEPTH);
