Google Git
Sign in
aomedia / aom / 92960a83754e1f6b248bf5c29f6e76469c1eafd8 / . / av1 / encoder / saliency_map.c
blob: 33768464b96841a5ffb2a90b4f0a241652739954 [file] [log] [blame]
/*
* Copyright (c) 2023, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <float.h>
#include <string.h>
#include "av1/encoder/encoder.h"
#include "av1/encoder/encoder_utils.h"
#include "av1/encoder/firstpass.h"
#include "av1/encoder/rdopt.h"
#include "av1/encoder/saliency_map.h"
// The Gabor filter is generated by setting the parameters as:
// ksize = 9
// sigma = 1
// theta = y*np.pi/4, where y /in {0, 1, 2, 3}, i.e., 0, 45, 90, 135 degree
// lambda1 = 1
// gamma=0.8
// phi =0
static const double kGaborFilter[4][9][9] = { // [angle: 0, 45, 90, 135
// degree][ksize][ksize]
{ { 2.0047323e-06, 6.6387620e-05, 8.0876675e-04, 3.6246411e-03, 5.9760227e-03,
3.6246411e-03, 8.0876675e-04, 6.6387620e-05, 2.0047323e-06 },
{ 1.8831115e-05, 6.2360091e-04, 7.5970138e-03, 3.4047455e-02, 5.6134764e-02,
3.4047455e-02, 7.5970138e-03, 6.2360091e-04, 1.8831115e-05 },
{ 9.3271126e-05, 3.0887155e-03, 3.7628256e-02, 1.6863814e-01, 2.7803731e-01,
1.6863814e-01, 3.7628256e-02, 3.0887155e-03, 9.3271126e-05 },
{ 2.4359586e-04, 8.0667874e-03, 9.8273583e-02, 4.4043165e-01, 7.2614902e-01,
4.4043165e-01, 9.8273583e-02, 8.0667874e-03, 2.4359586e-04 },
{ 3.3546262e-04, 1.1108996e-02, 1.3533528e-01, 6.0653067e-01, 1.0000000e+00,
6.0653067e-01, 1.3533528e-01, 1.1108996e-02, 3.3546262e-04 },
{ 2.4359586e-04, 8.0667874e-03, 9.8273583e-02, 4.4043165e-01, 7.2614902e-01,
4.4043165e-01, 9.8273583e-02, 8.0667874e-03, 2.4359586e-04 },
{ 9.3271126e-05, 3.0887155e-03, 3.7628256e-02, 1.6863814e-01, 2.7803731e-01,
1.6863814e-01, 3.7628256e-02, 3.0887155e-03, 9.3271126e-05 },
{ 1.8831115e-05, 6.2360091e-04, 7.5970138e-03, 3.4047455e-02, 5.6134764e-02,
3.4047455e-02, 7.5970138e-03, 6.2360091e-04, 1.8831115e-05 },
{ 2.0047323e-06, 6.6387620e-05, 8.0876675e-04, 3.6246411e-03, 5.9760227e-03,
3.6246411e-03, 8.0876675e-04, 6.6387620e-05, 2.0047323e-06 } },
{ { -6.2165498e-08, 3.8760313e-06, 3.0079011e-06, -4.4602581e-04,
6.6981313e-04, 1.3962291e-03, -9.9486928e-04, -8.1631159e-05,
3.5712848e-05 },
{ 3.8760313e-06, 5.7044272e-06, -1.6041942e-03, 4.5687673e-03,
1.8061366e-02, -2.4406660e-02, -3.7979286e-03, 3.1511115e-03,
-8.1631159e-05 },
{ 3.0079011e-06, -1.6041942e-03, 8.6645801e-03, 6.4960226e-02,
-1.6647682e-01, -4.9129307e-02, 7.7304743e-02, -3.7979286e-03,
-9.9486928e-04 },
{ -4.4602581e-04, 4.5687673e-03, 6.4960226e-02, -3.1572008e-01,
-1.7670043e-01, 5.2729243e-01, -4.9129307e-02, -2.4406660e-02,
1.3962291e-03 },
{ 6.6981313e-04, 1.8061366e-02, -1.6647682e-01, -1.7670043e-01,
1.0000000e+00, -1.7670043e-01, -1.6647682e-01, 1.8061366e-02,
6.6981313e-04 },
{ 1.3962291e-03, -2.4406660e-02, -4.9129307e-02, 5.2729243e-01,
-1.7670043e-01, -3.1572008e-01, 6.4960226e-02, 4.5687673e-03,
-4.4602581e-04 },
{ -9.9486928e-04, -3.7979286e-03, 7.7304743e-02, -4.9129307e-02,
-1.6647682e-01, 6.4960226e-02, 8.6645801e-03, -1.6041942e-03,
3.0079011e-06 },
{ -8.1631159e-05, 3.1511115e-03, -3.7979286e-03, -2.4406660e-02,
1.8061366e-02, 4.5687673e-03, -1.6041942e-03, 5.7044272e-06,
3.8760313e-06 },
{ 3.5712848e-05, -8.1631159e-05, -9.9486928e-04, 1.3962291e-03,
6.6981313e-04, -4.4602581e-04, 3.0079011e-06, 3.8760313e-06,
-6.2165498e-08 } },
{ { 2.0047323e-06, 1.8831115e-05, 9.3271126e-05, 2.4359586e-04, 3.3546262e-04,
2.4359586e-04, 9.3271126e-05, 1.8831115e-05, 2.0047323e-06 },
{ 6.6387620e-05, 6.2360091e-04, 3.0887155e-03, 8.0667874e-03, 1.1108996e-02,
8.0667874e-03, 3.0887155e-03, 6.2360091e-04, 6.6387620e-05 },
{ 8.0876675e-04, 7.5970138e-03, 3.7628256e-02, 9.8273583e-02, 1.3533528e-01,
9.8273583e-02, 3.7628256e-02, 7.5970138e-03, 8.0876675e-04 },
{ 3.6246411e-03, 3.4047455e-02, 1.6863814e-01, 4.4043165e-01, 6.0653067e-01,
4.4043165e-01, 1.6863814e-01, 3.4047455e-02, 3.6246411e-03 },
{ 5.9760227e-03, 5.6134764e-02, 2.7803731e-01, 7.2614902e-01, 1.0000000e+00,
7.2614902e-01, 2.7803731e-01, 5.6134764e-02, 5.9760227e-03 },
{ 3.6246411e-03, 3.4047455e-02, 1.6863814e-01, 4.4043165e-01, 6.0653067e-01,
4.4043165e-01, 1.6863814e-01, 3.4047455e-02, 3.6246411e-03 },
{ 8.0876675e-04, 7.5970138e-03, 3.7628256e-02, 9.8273583e-02, 1.3533528e-01,
9.8273583e-02, 3.7628256e-02, 7.5970138e-03, 8.0876675e-04 },
{ 6.6387620e-05, 6.2360091e-04, 3.0887155e-03, 8.0667874e-03, 1.1108996e-02,
8.0667874e-03, 3.0887155e-03, 6.2360091e-04, 6.6387620e-05 },
{ 2.0047323e-06, 1.8831115e-05, 9.3271126e-05, 2.4359586e-04, 3.3546262e-04,
2.4359586e-04, 9.3271126e-05, 1.8831115e-05, 2.0047323e-06 } },
{ { 3.5712848e-05, -8.1631159e-05, -9.9486928e-04, 1.3962291e-03,
6.6981313e-04, -4.4602581e-04, 3.0079011e-06, 3.8760313e-06,
-6.2165498e-08 },
{ -8.1631159e-05, 3.1511115e-03, -3.7979286e-03, -2.4406660e-02,
1.8061366e-02, 4.5687673e-03, -1.6041942e-03, 5.7044272e-06,
3.8760313e-06 },
{ -9.9486928e-04, -3.7979286e-03, 7.7304743e-02, -4.9129307e-02,
-1.6647682e-01, 6.4960226e-02, 8.6645801e-03, -1.6041942e-03,
3.0079011e-06 },
{ 1.3962291e-03, -2.4406660e-02, -4.9129307e-02, 5.2729243e-01,
-1.7670043e-01, -3.1572008e-01, 6.4960226e-02, 4.5687673e-03,
-4.4602581e-04 },
{ 6.6981313e-04, 1.8061366e-02, -1.6647682e-01, -1.7670043e-01,
1.0000000e+00, -1.7670043e-01, -1.6647682e-01, 1.8061366e-02,
6.6981313e-04 },
{ -4.4602581e-04, 4.5687673e-03, 6.4960226e-02, -3.1572008e-01,
-1.7670043e-01, 5.2729243e-01, -4.9129307e-02, -2.4406660e-02,
1.3962291e-03 },
{ 3.0079011e-06, -1.6041942e-03, 8.6645801e-03, 6.4960226e-02,
-1.6647682e-01, -4.9129307e-02, 7.7304743e-02, -3.7979286e-03,
-9.9486928e-04 },
{ 3.8760313e-06, 5.7044272e-06, -1.6041942e-03, 4.5687673e-03,
1.8061366e-02, -2.4406660e-02, -3.7979286e-03, 3.1511115e-03,
-8.1631159e-05 },
{ -6.2165498e-08, 3.8760313e-06, 3.0079011e-06, -4.4602581e-04,
6.6981313e-04, 1.3962291e-03, -9.9486928e-04, -8.1631159e-05,
3.5712848e-05 } }
};
// This function is to extract red/green/blue channels, and calculate intensity
// = (r+g+b)/3. Note that it only handles 8bits case now.
// TODO(linzhen): add high bitdepth support.
static void get_color_intensity(const YV12_BUFFER_CONFIG *src,
int subsampling_x, int subsampling_y,
double *cr, double *cg, double *cb,
double *intensity) {
const uint8_t *y = src->buffers[0];
const uint8_t *u = src->buffers[1];
const uint8_t *v = src->buffers[2];
const int y_height = src->crop_heights[0];
const int y_width = src->crop_widths[0];
const int y_stride = src->strides[0];
const int c_stride = src->strides[1];
for (int i = 0; i < y_height; ++i) {
for (int j = 0; j < y_width; ++j) {
cr[i * y_width + j] =
fclamp((double)y[i * y_stride + j] +
1.370 * (double)(v[(i >> subsampling_y) * c_stride +
(j >> subsampling_x)] -
128),
0, 255);
cg[i * y_width + j] =
fclamp((double)y[i * y_stride + j] -
0.698 * (double)(u[(i >> subsampling_y) * c_stride +
(j >> subsampling_x)] -
128) -
0.337 * (double)(v[(i >> subsampling_y) * c_stride +
(j >> subsampling_x)] -
128),
0, 255);
cb[i * y_width + j] =
fclamp((double)y[i * y_stride + j] +
1.732 * (double)(u[(i >> subsampling_y) * c_stride +
(j >> subsampling_x)] -
128),
0, 255);
intensity[i * y_width + j] =
(cr[i * y_width + j] + cg[i * y_width + j] + cb[i * y_width + j]) /
3.0;
assert(intensity[i * y_width + j] >= 0 &&
intensity[i * y_width + j] <= 255);
intensity[i * y_width + j] /= 256;
cr[i * y_width + j] /= 256;
cg[i * y_width + j] /= 256;
cb[i * y_width + j] /= 256;
}
}
}
static INLINE double convolve_map(const double *filter, const double *map,
const int size) {
double result = 0;
for (int i = 0; i < size; ++i) {
result += filter[i] * map[i]; // symmetric filter is used
}
return result;
}
// This function is to decimate the map by half, and apply Gaussian filter on
// top of the downsampled map.
static INLINE void decimate_map(const double *map, int height, int width,
int stride, double *downsampled_map) {
const int new_width = width / 2;
const int window_size = 5;
const double gaussian_filter[25] = {
1. / 256, 1.0 / 64, 3. / 128, 1. / 64, 1. / 256, 1. / 64, 1. / 16,
3. / 32, 1. / 16, 1. / 64, 3. / 128, 3. / 32, 9. / 64, 3. / 32,
3. / 128, 1. / 64, 1. / 16, 3. / 32, 1. / 16, 1. / 64, 1. / 256,
1. / 64, 3. / 128, 1. / 64, 1. / 256
};
double map_region[25];
for (int y = 0; y < height - 1; y += 2) {
for (int x = 0; x < width - 1; x += 2) {
int i = 0;
for (int yy = y - window_size / 2; yy <= y + window_size / 2; ++yy) {
for (int xx = x - window_size / 2; xx <= x + window_size / 2; ++xx) {
int yvalue = clamp(yy, 0, height - 1);
int xvalue = clamp(xx, 0, width - 1);
map_region[i++] = map[yvalue * stride + xvalue];
}
}
downsampled_map[(y / 2) * new_width + (x / 2)] =
convolve_map(gaussian_filter, map_region, window_size * window_size);
}
}
}
// This function is to upscale the map from in_level size to out_level size.
// Note that the map at "level-1" will upscale the map at "level" by x2.
static INLINE int upscale_map(const double *input, int in_level, int out_level,
int height[9], int width[9], double *output) {
for (int level = in_level; level > out_level; level--) {
const int cur_width = width[level];
const int cur_height = height[level];
const int cur_stride = width[level];
double *original = (level == in_level) ? (double *)input : output;
assert(level > 0);
const int h_upscale = height[level - 1];
const int w_upscale = width[level - 1];
const int s_upscale = width[level - 1];
double *upscale = aom_malloc(h_upscale * w_upscale * sizeof(*upscale));
if (!upscale) {
return 0;
}
for (int i = 0; i < h_upscale; ++i) {
for (int j = 0; j < w_upscale; ++j) {
const int ii = clamp((i >> 1), 0, cur_height - 1);
const int jj = clamp((j >> 1), 0, cur_width - 1);
upscale[j + i * s_upscale] = (double)original[jj + ii * cur_stride];
}
}
memcpy(output, upscale, h_upscale * w_upscale * sizeof(double));
aom_free(upscale);
}
return 1;
}
// This function calculates the differences between a fine scale c and a
// coarser scale s yielding the feature maps. c \in {2, 3, 4}, and s = c +
// delta, where delta \in {3, 4}.
static int center_surround_diff(const double *input[9], int height[9],
int width[9], saliency_feature_map *output[6]) {
int j = 0;
for (int k = 2; k < 5; ++k) {
int cur_height = height[k];
int cur_width = width[k];
if (upscale_map(input[k + 3], k + 3, k, height, width, output[j]->buf) ==
0) {
return 0;
}
for (int r = 0; r < cur_height; ++r) {
for (int c = 0; c < cur_width; ++c) {
output[j]->buf[r * cur_width + c] =
fabs((double)(input[k][r * cur_width + c] -
output[j]->buf[r * cur_width + c]));
}
}
if (upscale_map(input[k + 4], k + 4, k, height, width,
output[j + 1]->buf) == 0) {
return 0;
}
for (int r = 0; r < cur_height; ++r) {
for (int c = 0; c < cur_width; ++c) {
output[j + 1]->buf[r * cur_width + c] =
fabs(input[k][r * cur_width + c] -
output[j + 1]->buf[r * cur_width + c]);
}
}
j += 2;
}
return 1;
}
// For color channels, the differences is calculated based on "color
// double-opponency". For example, the RG feature map is constructed between a
// fine scale c of R-G component and a coarser scale s of G-R component.
static int center_surround_diff_rgb(const double *input_1[9],
const double *input_2[9], int height[9],
int width[9],
saliency_feature_map *output[6]) {
int j = 0;
for (int k = 2; k < 5; ++k) {
int cur_height = height[k];
int cur_width = width[k];
if (upscale_map(input_2[k + 3], k + 3, k, height, width, output[j]->buf) ==
0) {
return 0;
}
for (int r = 0; r < cur_height; ++r) {
for (int c = 0; c < cur_width; ++c) {
output[j]->buf[r * cur_width + c] =
fabs((double)(input_1[k][r * cur_width + c] -
output[j]->buf[r * cur_width + c]));
}
}
if (upscale_map(input_2[k + 4], k + 4, k, height, width,
output[j + 1]->buf) == 0) {
return 0;
}
for (int r = 0; r < cur_height; ++r) {
for (int c = 0; c < cur_width; ++c) {
output[j + 1]->buf[r * cur_width + c] =
fabs(input_1[k][r * cur_width + c] -
output[j + 1]->buf[r * cur_width + c]);
}
}
j += 2;
}
return 1;
}
// This function is to generate Gaussian pyramid images with indexes from 0 to
// 8, and construct the feature maps from calculating the center-surround
// differences.
static int gaussian_pyramid(const double *src, int width[9], int height[9],
saliency_feature_map *dst[6]) {
double *gaussian_map[9]; // scale = 9
gaussian_map[0] =
(double *)aom_malloc(width[0] * height[0] * sizeof(*gaussian_map[0]));
if (!gaussian_map[0]) {
return 0;
}
memcpy(gaussian_map[0], src, width[0] * height[0] * sizeof(double));
for (int i = 1; i < 9; ++i) {
int stride = width[i - 1];
int new_width = width[i];
int new_height = height[i];
gaussian_map[i] =
(double *)aom_malloc(new_width * new_height * sizeof(*gaussian_map[i]));
if (!gaussian_map[i]) {
for (int l = 0; l < i; ++l) {
aom_free(gaussian_map[l]);
}
return 0;
}
memset(gaussian_map[i], 0, new_width * new_height * sizeof(double));
decimate_map(gaussian_map[i - 1], height[i - 1], width[i - 1], stride,
gaussian_map[i]);
}
if (center_surround_diff((const double **)gaussian_map, height, width, dst) ==
0) {
for (int l = 0; l < 9; ++l) {
aom_free(gaussian_map[l]);
}
return 0;
}
for (int i = 0; i < 9; ++i) {
aom_free(gaussian_map[i]);
}
return 1;
}
static int gaussian_pyramid_rgb(double *src_1, double *src_2, int width[9],
int height[9], saliency_feature_map *dst[6]) {
double *gaussian_map[2][9]; // scale = 9
double *src[2];
src[0] = src_1;
src[1] = src_2;
for (int k = 0; k < 2; ++k) {
gaussian_map[k][0] = (double *)aom_malloc(width[0] * height[0] *
sizeof(*gaussian_map[k][0]));
if (!gaussian_map[k][0]) {
for (int l = 0; l < k; ++l) {
aom_free(gaussian_map[l][0]);
}
return 0;
}
memcpy(gaussian_map[k][0], src[k], width[0] * height[0] * sizeof(double));
for (int i = 1; i < 9; ++i) {
int stride = width[i - 1];
int new_width = width[i];
int new_height = height[i];
gaussian_map[k][i] = (double *)aom_malloc(new_width * new_height *
sizeof(*gaussian_map[k][i]));
if (!gaussian_map[k][i]) {
for (int l = 0; l < k; ++l) {
aom_free(gaussian_map[l][i]);
}
return 0;
}
memset(gaussian_map[k][i], 0, new_width * new_height * sizeof(double));
decimate_map(gaussian_map[k][i - 1], height[i - 1], width[i - 1], stride,
gaussian_map[k][i]);
}
}
if (center_surround_diff_rgb((const double **)gaussian_map[0],
(const double **)gaussian_map[1], height, width,
dst) == 0) {
for (int l = 0; l < 2; ++l) {
for (int i = 0; i < 9; ++i) {
aom_free(gaussian_map[l][i]);
}
}
return 0;
}
for (int l = 0; l < 2; ++l) {
for (int i = 0; i < 9; ++i) {
aom_free(gaussian_map[l][i]);
}
}
return 1;
}
static int get_feature_map_intensity(double *intensity, int width[9],
int height[9],
saliency_feature_map *i_map[6]) {
if (gaussian_pyramid(intensity, width, height, i_map) == 0) {
return 0;
}
return 1;
}
static int get_feature_map_rgb(double *cr, double *cg, double *cb, int width[9],
int height[9], saliency_feature_map *rg_map[6],
saliency_feature_map *by_map[6]) {
double *rg_mat = aom_malloc(height[0] * width[0] * sizeof(*rg_mat));
double *by_mat = aom_malloc(height[0] * width[0] * sizeof(*by_mat));
double *gr_mat = aom_malloc(height[0] * width[0] * sizeof(*gr_mat));
double *yb_mat = aom_malloc(height[0] * width[0] * sizeof(*yb_mat));
if (!rg_mat || !by_mat || !gr_mat || !yb_mat) {
aom_free(rg_mat);
aom_free(by_mat);
aom_free(gr_mat);
aom_free(yb_mat);
return 0;
}
double r, g, b, y;
for (int i = 0; i < height[0]; ++i) {
for (int j = 0; j < width[0]; ++j) {
r = AOMMAX(0, cr[i * width[0] + j] -
(cg[i * width[0] + j] + cb[i * width[0] + j]) / 2);
g = AOMMAX(0, cg[i * width[0] + j] -
(cr[i * width[0] + j] + cb[i * width[0] + j]) / 2);
b = AOMMAX(0, cb[i * width[0] + j] -
(cr[i * width[0] + j] + cg[i * width[0] + j]) / 2);
y = AOMMAX(0, (cr[i * width[0] + j] + cg[i * width[0] + j]) / 2 -
fabs(cr[i * width[0] + j] - cg[i * width[0] + j]) / 2 -
cb[i * width[0] + j]);
rg_mat[i * width[0] + j] = r - g;
by_mat[i * width[0] + j] = b - y;
gr_mat[i * width[0] + j] = g - r;
yb_mat[i * width[0] + j] = y - b;
}
}
if (gaussian_pyramid_rgb(rg_mat, gr_mat, width, height, rg_map) == 0 ||
gaussian_pyramid_rgb(by_mat, yb_mat, width, height, by_map) == 0) {
aom_free(rg_mat);
aom_free(by_mat);
aom_free(gr_mat);
aom_free(yb_mat);
return 0;
}
aom_free(rg_mat);
aom_free(by_mat);
aom_free(gr_mat);
aom_free(yb_mat);
return 1;
}
static INLINE void filter2d(const double *input, const double kernel[9][9],
int width, int height, double *output) {
const int window_size = 9;
double map_section[81];
for (int y = 0; y <= height - 1; ++y) {
for (int x = 0; x <= width - 1; ++x) {
int i = 0;
for (int yy = y - window_size / 2; yy <= y + window_size / 2; ++yy) {
for (int xx = x - window_size / 2; xx <= x + window_size / 2; ++xx) {
int yvalue = clamp(yy, 0, height - 1);
int xvalue = clamp(xx, 0, width - 1);
map_section[i++] = input[yvalue * width + xvalue];
}
}
output[y * width + x] = 0;
for (int k = 0; k < window_size; ++k) {
for (int l = 0; l < window_size; ++l) {
output[y * width + x] +=
kernel[k][l] * map_section[k * window_size + l];
}
}
}
}
}
static int get_feature_map_orientation(const double *intensity, int width[9],
int height[9],
saliency_feature_map *dst[24]) {
double *gaussian_map[9];
gaussian_map[0] =
(double *)aom_malloc(width[0] * height[0] * sizeof(*gaussian_map[0]));
if (!gaussian_map[0]) {
return 0;
}
memcpy(gaussian_map[0], intensity, width[0] * height[0] * sizeof(double));
for (int i = 1; i < 9; ++i) {
int stride = width[i - 1];
int new_width = width[i];
int new_height = height[i];
gaussian_map[i] =
(double *)aom_malloc(new_width * new_height * sizeof(*gaussian_map[i]));
if (!gaussian_map[i]) {
for (int l = 0; l < i; ++l) {
aom_free(gaussian_map[l]);
}
return 0;
}
memset(gaussian_map[i], 0, new_width * new_height * sizeof(double));
decimate_map(gaussian_map[i - 1], height[i - 1], width[i - 1], stride,
gaussian_map[i]);
}
double *tempGaborOutput[4][9]; //[angle: 0, 45, 90, 135 degree][filter_size]
for (int i = 2; i < 9; ++i) {
const int cur_height = height[i];
const int cur_width = width[i];
for (int j = 0; j < 4; ++j) {
tempGaborOutput[j][i] = (double *)aom_malloc(
cur_height * cur_width * sizeof(*tempGaborOutput[j][i]));
if (!tempGaborOutput[j][i]) {
for (int l = 0; l < 9; ++l) {
aom_free(gaussian_map[l]);
}
for (int h = 0; h < 4; ++h) {
for (int g = 2; g < 9; ++g) {
aom_free(tempGaborOutput[h][g]);
}
}
return 0;
}
filter2d(gaussian_map[i], kGaborFilter[j], cur_width, cur_height,
tempGaborOutput[j][i]);
}
}
for (int i = 0; i < 9; ++i) {
aom_free(gaussian_map[i]);
}
saliency_feature_map
*tmp[4][6]; //[angle: 0, 45, 90, 135 degree][filter_size]
for (int i = 0; i < 6; ++i) {
for (int j = 0; j < 4; ++j) {
tmp[j][i] = dst[j * 6 + i];
}
}
for (int j = 0; j < 4; ++j) {
if (center_surround_diff((const double **)tempGaborOutput[j], height, width,
tmp[j]) == 0) {
for (int h = 0; h < 4; ++h) {
for (int g = 2; g < 9; ++g) {
aom_free(tempGaborOutput[h][g]);
}
}
return 0;
}
}
for (int i = 2; i < 9; ++i) {
for (int j = 0; j < 4; ++j) {
aom_free(tempGaborOutput[j][i]);
}
}
return 1;
}
static INLINE void find_min_max(const saliency_feature_map *input,
double *max_value, double *min_value) {
assert(input && input->buf);
*min_value = DBL_MAX;
*max_value = 0.0;
for (int i = 0; i < input->height; ++i) {
for (int j = 0; j < input->width; ++j) {
assert(input->buf[i * input->width + j] >= 0.0);
*min_value = fmin(input->buf[i * input->width + j], *min_value);
*max_value = fmax(input->buf[i * input->width + j], *max_value);
}
}
}
static INLINE double average_local_max(const saliency_feature_map *input,
int stepsize) {
int numlocal = 0;
double lmaxmean = 0, lmax = 0, dummy = 0;
saliency_feature_map local_map;
local_map.height = stepsize;
local_map.width = stepsize;
local_map.buf =
(double *)aom_malloc(stepsize * stepsize * sizeof(*local_map.buf));
if (!local_map.buf) {
return -1;
}
for (int y = 0; y < input->height - stepsize; y += stepsize) {
for (int x = 0; x < input->width - stepsize; x += stepsize) {
for (int i = 0; i < stepsize; ++i) {
for (int j = 0; j < stepsize; ++j) {
local_map.buf[i * stepsize + j] =
input->buf[(y + i) * input->width + x + j];
}
}
find_min_max(&local_map, &lmax, &dummy);
lmaxmean += lmax;
numlocal++;
}
}
aom_free(local_map.buf);
return lmaxmean / numlocal;
}
// Linear normalization the values in the map to [0,1].
static void minmax_normalize(saliency_feature_map *input) {
double max_value, min_value;
find_min_max(input, &max_value, &min_value);
for (int i = 0; i < input->height; ++i) {
for (int j = 0; j < input->width; ++j) {
if (max_value != min_value) {
input->buf[i * input->width + j] =
input->buf[i * input->width + j] / (max_value - min_value) +
min_value / (min_value - max_value);
} else {
input->buf[i * input->width + j] -= min_value;
}
}
}
}
// This function is to promote meaningful “activation spots” in the map and
// ignores homogeneous areas.
static int nomalization_operator(saliency_feature_map *input, int stepsize) {
minmax_normalize(input);
double lmaxmean = average_local_max(input, stepsize);
if (lmaxmean < 0) {
return 0;
}
double normCoeff = (1 - lmaxmean) * (1 - lmaxmean);
for (int i = 0; i < input->height; ++i) {
for (int j = 0; j < input->width; ++j) {
input->buf[i * input->width + j] *= normCoeff;
}
}
return 1;
}
// Normalize the values in feature maps to [0,1], and then upscale all maps to
// the original frame size.
static int normalize_fm(saliency_feature_map *input[6], int width[9],
int height[9], int num_fm,
saliency_feature_map *output[6]) {
// Feature maps (FM) are generated by function "center_surround_diff()". The
// difference is between a fine scale c and a coarser scale s, where c \in {2,
// 3, 4}, and s = c + delta, where delta \in {3, 4}, and the FM size is scale
// c. Specifically, i=0: c=2 and s=5, i=1: c=2 and s=6, i=2: c=3 and s=6, i=3:
// c=3 and s=7, i=4: c=4 and s=7, i=5: c=4 and s=8.
for (int i = 0; i < num_fm; ++i) {
if (nomalization_operator(input[i], 8) == 0) {
return 0;
}
// Upscale FM to original frame size
if (upscale_map(input[i]->buf, (i / 2) + 2, 0, height, width,
output[i]->buf) == 0) {
return 0;
}
}
return 1;
}
// Combine feature maps with the same category (intensity, color, or
// orientation) into one conspicuity map.
static int normalized_map(saliency_feature_map *input[6], int width[9],
int height[9], saliency_feature_map *output) {
int num_fm = 6;
saliency_feature_map *n_input[6];
for (int i = 0; i < 6; ++i) {
n_input[i] = (saliency_feature_map *)aom_malloc(sizeof(*n_input[i]));
if (!n_input[i]) {
return 0;
}
n_input[i]->buf =
(double *)aom_malloc(width[0] * height[0] * sizeof(*n_input[i]->buf));
if (!n_input[i]->buf) {
aom_free(n_input[i]);
return 0;
}
n_input[i]->height = height[0];
n_input[i]->width = width[0];
}
if (normalize_fm(input, width, height, num_fm, n_input) == 0) {
for (int i = 0; i < num_fm; ++i) {
aom_free(n_input[i]->buf);
aom_free(n_input[i]);
}
return 0;
}
// Add up all normalized feature maps with the same category into one map.
for (int i = 0; i < num_fm; ++i) {
for (int r = 0; r < height[0]; ++r) {
for (int c = 0; c < width[0]; ++c) {
output->buf[r * width[0] + c] += n_input[i]->buf[r * width[0] + c];
}
}
}
for (int i = 0; i < num_fm; ++i) {
aom_free(n_input[i]->buf);
aom_free(n_input[i]);
}
nomalization_operator(output, 8);
return 1;
}
static int normalized_map_rgb(saliency_feature_map *rg_map[6],
saliency_feature_map *by_map[6], int width[9],
int height[9], saliency_feature_map *output) {
saliency_feature_map *color_cm[2]; // 0: color_cm_rg, 1: color_cm_by
for (int i = 0; i < 2; ++i) {
color_cm[i] = aom_malloc(sizeof(*color_cm[i]));
if (!color_cm[i]) {
return 0;
}
color_cm[i]->buf =
(double *)aom_malloc(width[0] * height[0] * sizeof(*color_cm[i]->buf));
if (!color_cm[i]->buf) {
for (int l = 0; l < i; ++l) {
aom_free(color_cm[l]->buf);
}
aom_free(color_cm[i]);
return 0;
}
color_cm[i]->width = width[0];
color_cm[i]->height = height[0];
memset(color_cm[i]->buf, 0,
width[0] * height[0] * sizeof(*color_cm[i]->buf));
}
if (normalized_map(rg_map, width, height, color_cm[0]) == 0 ||
normalized_map(by_map, width, height, color_cm[1]) == 0) {
for (int i = 0; i < 2; ++i) {
aom_free(color_cm[i]->buf);
aom_free(color_cm[i]);
}
return 0;
}
for (int r = 0; r < height[0]; ++r) {
for (int c = 0; c < width[0]; ++c) {
output->buf[r * width[0] + c] = color_cm[0]->buf[r * width[0] + c] +
color_cm[1]->buf[r * width[0] + c];
}
}
for (int i = 0; i < 2; ++i) {
aom_free(color_cm[i]->buf);
aom_free(color_cm[i]);
}
nomalization_operator(output, 8);
return 1;
}
static int normalized_map_orientation(saliency_feature_map *orientation_map[24],
int width[9], int height[9],
saliency_feature_map *output) {
int num_fms_per_angle = 6;
saliency_feature_map *ofm[4][6];
for (int i = 0; i < num_fms_per_angle; ++i) {
for (int j = 0; j < 4; ++j) {
ofm[j][i] = orientation_map[j * num_fms_per_angle + i];
}
}
// extract conspicuity map for each angle
saliency_feature_map *nofm = aom_malloc(sizeof(*nofm));
if (!nofm) {
return 0;
}
nofm->buf = (double *)aom_malloc(width[0] * height[0] * sizeof(*nofm->buf));
if (!nofm->buf) {
aom_free(nofm);
return 0;
}
nofm->height = height[0];
nofm->width = width[0];
for (int i = 0; i < 4; ++i) {
memset(nofm->buf, 0, width[0] * height[0] * sizeof(*nofm->buf));
if (normalized_map(ofm[i], width, height, nofm) == 0) {
aom_free(nofm->buf);
aom_free(nofm);
return 0;
}
for (int r = 0; r < height[0]; ++r) {
for (int c = 0; c < width[0]; ++c) {
output->buf[r * width[0] + c] += nofm->buf[r * width[0] + c];
}
}
}
aom_free(nofm->buf);
aom_free(nofm);
nomalization_operator(output, 8);
return 1;
}
// Set pixel level saliency mask based on Itti-Koch algorithm
int av1_set_saliency_map(AV1_COMP *cpi) {
AV1_COMMON *const cm = &cpi->common;
int frm_width = cm->width;
int frm_height = cm->height;
int pyr_height[9];
int pyr_width[9];
pyr_height[0] = frm_height;
pyr_width[0] = frm_width;
for (int i = 1; i < 9; ++i) {
pyr_width[i] = pyr_width[i - 1] / 2;
pyr_height[i] = pyr_height[i - 1] / 2;
}
double *cr = aom_malloc(frm_width * frm_height * sizeof(*cr));
double *cg = aom_malloc(frm_width * frm_height * sizeof(*cg));
double *cb = aom_malloc(frm_width * frm_height * sizeof(*cb));
double *intensity = aom_malloc(frm_width * frm_height * sizeof(*intensity));
if (!cr || !cg || !cb || !intensity) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
return 0;
}
// Extract red / green / blue channels and intensity component
get_color_intensity(cpi->source, cm->seq_params->subsampling_x,
cm->seq_params->subsampling_y, cr, cg, cb, intensity);
// Feature Map Extraction
// intensity map
saliency_feature_map *i_map[6];
for (int i = 0; i < 6; ++i) {
int cur_height = pyr_height[(i / 2) + 2];
int cur_width = pyr_width[(i / 2) + 2];
i_map[i] = (saliency_feature_map *)aom_malloc(sizeof(*i_map[i]));
if (!i_map[i]) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < i; ++l) {
aom_free(i_map[l]);
}
return 0;
}
i_map[i]->buf =
(double *)aom_malloc(cur_height * cur_width * sizeof(*i_map[i]->buf));
if (!i_map[i]->buf) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < i; ++l) {
aom_free(i_map[l]->buf);
aom_free(i_map[l]);
}
return 0;
}
i_map[i]->height = cur_height;
i_map[i]->width = cur_width;
}
if (get_feature_map_intensity(intensity, pyr_width, pyr_height, i_map) == 0) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(i_map[l]);
}
return 0;
}
// RGB map
saliency_feature_map *rg_map[6], *by_map[6];
for (int i = 0; i < 6; ++i) {
int cur_height = pyr_height[(i / 2) + 2];
int cur_width = pyr_width[(i / 2) + 2];
rg_map[i] = (saliency_feature_map *)aom_malloc(sizeof(*rg_map[i]));
by_map[i] = (saliency_feature_map *)aom_malloc(sizeof(*by_map[i]));
if (!rg_map[i] || !by_map[i]) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
return 0;
}
rg_map[i]->buf =
(double *)aom_malloc(cur_height * cur_width * sizeof(*rg_map[i]->buf));
by_map[i]->buf =
(double *)aom_malloc(cur_height * cur_width * sizeof(*by_map[i]->buf));
if (!by_map[i]->buf || !rg_map[i]->buf) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(i_map[l]);
}
for (int l = 0; l < i; ++l) {
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
return 0;
}
rg_map[i]->height = cur_height;
rg_map[i]->width = cur_width;
by_map[i]->height = cur_height;
by_map[i]->width = cur_width;
}
if (get_feature_map_rgb(cr, cg, cb, pyr_width, pyr_height, rg_map, by_map) ==
0) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
return 0;
}
// Orientation map
saliency_feature_map *orientation_map[24];
for (int i = 0; i < 24; ++i) {
int cur_height = pyr_height[((i % 6) / 2) + 2];
int cur_width = pyr_width[((i % 6) / 2) + 2];
orientation_map[i] =
(saliency_feature_map *)aom_malloc(sizeof(*orientation_map[i]));
if (!orientation_map[i]) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
for (int h = 0; h < i; ++h) {
aom_free(orientation_map[h]);
}
return 0;
}
orientation_map[i]->buf = (double *)aom_malloc(
cur_height * cur_width * sizeof(*orientation_map[i]->buf));
if (!orientation_map[i]->buf) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
for (int h = 0; h < i; ++h) {
aom_free(orientation_map[h]->buf);
aom_free(orientation_map[h]->buf);
aom_free(orientation_map[h]);
aom_free(orientation_map[h]);
}
return 0;
}
orientation_map[i]->height = cur_height;
orientation_map[i]->width = cur_width;
}
if (get_feature_map_orientation(intensity, pyr_width, pyr_height,
orientation_map) == 0) {
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
for (int h = 0; h < 24; ++h) {
aom_free(orientation_map[h]->buf);
aom_free(orientation_map[h]);
}
return 0;
}
aom_free(cr);
aom_free(cg);
aom_free(cb);
aom_free(intensity);
saliency_feature_map
*normalized_maps[3]; // 0: intensity, 1: color, 2: orientation
for (int i = 0; i < 3; ++i) {
normalized_maps[i] = aom_malloc(sizeof(*normalized_maps[i]));
if (!normalized_maps[i]) {
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
for (int h = 0; h < 24; ++h) {
aom_free(orientation_map[h]->buf);
aom_free(orientation_map[h]);
}
for (int l = 0; l < i; ++l) {
aom_free(normalized_maps[l]);
}
return 0;
}
normalized_maps[i]->buf = (double *)aom_malloc(
frm_width * frm_height * sizeof(*normalized_maps[i]->buf));
if (!normalized_maps[i]->buf) {
for (int l = 0; l < 6; ++l) {
aom_free(i_map[l]->buf);
aom_free(rg_map[l]->buf);
aom_free(by_map[l]->buf);
aom_free(i_map[l]);
aom_free(rg_map[l]);
aom_free(by_map[l]);
}
for (int h = 0; h < 24; ++h) {
aom_free(orientation_map[h]->buf);
aom_free(orientation_map[h]);
}
for (int l = 0; l < i; ++l) {
aom_free(normalized_maps[l]->buf);
aom_free(normalized_maps[l]);
}
return 0;
}
normalized_maps[i]->width = frm_width;
normalized_maps[i]->height = frm_height;
memset(normalized_maps[i]->buf, 0,
frm_width * frm_height * sizeof(*normalized_maps[i]->buf));
}
// Conspicuity map generation
if (normalized_map(i_map, pyr_width, pyr_height, normalized_maps[0]) == 0 ||
normalized_map_rgb(rg_map, by_map, pyr_width, pyr_height,
normalized_maps[1]) == 0 ||
normalized_map_orientation(orientation_map, pyr_width, pyr_height,
normalized_maps[2]) == 0) {
for (int i = 0; i < 6; ++i) {
aom_free(i_map[i]->buf);
aom_free(rg_map[i]->buf);
aom_free(by_map[i]->buf);
aom_free(i_map[i]);
aom_free(rg_map[i]);
aom_free(by_map[i]);
}
for (int i = 0; i < 24; ++i) {
aom_free(orientation_map[i]->buf);
aom_free(orientation_map[i]);
}
for (int i = 0; i < 3; ++i) {
aom_free(normalized_maps[i]->buf);
aom_free(normalized_maps[i]);
}
return 0;
}
for (int i = 0; i < 6; ++i) {
aom_free(i_map[i]->buf);
aom_free(rg_map[i]->buf);
aom_free(by_map[i]->buf);
aom_free(i_map[i]);
aom_free(rg_map[i]);
aom_free(by_map[i]);
}
for (int i = 0; i < 24; ++i) {
aom_free(orientation_map[i]->buf);
aom_free(orientation_map[i]);
}
// Pixel level saliency map
saliency_feature_map *combined_saliency_map =
aom_malloc(sizeof(*combined_saliency_map));
if (!combined_saliency_map) {
for (int i = 0; i < 3; ++i) {
aom_free(normalized_maps[i]->buf);
aom_free(normalized_maps[i]);
}
return 0;
}
combined_saliency_map->buf = (double *)aom_malloc(
frm_width * frm_height * sizeof(*combined_saliency_map->buf));
if (!combined_saliency_map->buf) {
for (int i = 0; i < 3; ++i) {
aom_free(normalized_maps[i]->buf);
aom_free(normalized_maps[i]);
}
aom_free(combined_saliency_map);
return 0;
}
combined_saliency_map->height = frm_height;
combined_saliency_map->width = frm_width;
double w_intensity, w_color, w_orient;
w_intensity = w_color = w_orient = (double)1 / 3;
for (int r = 0; r < frm_height; ++r) {
for (int c = 0; c < frm_width; ++c) {
combined_saliency_map->buf[r * frm_width + c] =
(w_intensity * normalized_maps[0]->buf[r * frm_width + c] +
w_color * normalized_maps[1]->buf[r * frm_width + c] +
w_orient * normalized_maps[2]->buf[r * frm_width + c]);
}
}
for (int r = 0; r < frm_height; ++r) {
for (int c = 0; c < frm_width; ++c) {
int index = r * frm_width + c;
cpi->saliency_map[index] =
(uint8_t)(combined_saliency_map->buf[index] * 255);
}
}
for (int i = 0; i < 3; ++i) {
aom_free(normalized_maps[i]->buf);
aom_free(normalized_maps[i]);
}
aom_free(combined_saliency_map->buf);
aom_free(combined_saliency_map);
return 1;
}
// Set superblock level saliency mask for rdmult scaling
int av1_setup_sm_rdmult_scaling_factor(AV1_COMP *cpi, double motion_ratio) {
AV1_COMMON *cm = &cpi->common;
saliency_feature_map *sb_saliency_map =
aom_malloc(sizeof(saliency_feature_map));
if (sb_saliency_map == NULL) {
return 0;
}
const int bsize = cm->seq_params->sb_size;
const int num_mi_w = mi_size_wide[bsize];
const int num_mi_h = mi_size_high[bsize];
const int block_width = block_size_wide[bsize];
const int block_height = block_size_high[bsize];
const int num_sb_cols = (cm->mi_params.mi_cols + num_mi_w - 1) / num_mi_w;
const int num_sb_rows = (cm->mi_params.mi_rows + num_mi_h - 1) / num_mi_h;
sb_saliency_map->height = num_sb_rows;
sb_saliency_map->width = num_sb_cols;
sb_saliency_map->buf = (double *)aom_malloc(num_sb_rows * num_sb_cols *
sizeof(*sb_saliency_map->buf));
if (sb_saliency_map->buf == NULL) {
aom_free(sb_saliency_map);
return 0;
}
for (int row = 0; row < num_sb_rows; ++row) {
for (int col = 0; col < num_sb_cols; ++col) {
const int index = row * num_sb_cols + col;
double total_pixel = 0;
double total_weight = 0;
for (int i = 0; i < block_height; i++) {
for (int j = 0; j < block_width; j++) {
if ((row * block_height + i) >= cpi->common.height ||
(col * block_width + j) >= cpi->common.width)
continue;
total_pixel++;
total_weight +=
cpi->saliency_map[(row * block_height + i) * cpi->common.width +
col * block_width + j];
}
}
assert(total_pixel > 0);
// Calculate the superblock level saliency map from pixel level saliency
// map
sb_saliency_map->buf[index] = total_weight / total_pixel;
// Further lower the superblock saliency score for boundary superblocks.
if (row < 1 || row > num_sb_rows - 2 || col < 1 ||
col > num_sb_cols - 2) {
sb_saliency_map->buf[index] /= 5;
}
}
}
// superblock level saliency map finalization
minmax_normalize(sb_saliency_map);
double log_sum = 0.0;
double sum = 0.0;
int block_count = 0;
// Calculate the average superblock sm_scaling_factor for a frame, to be used
// for clamping later.
for (int row = 0; row < num_sb_rows; ++row) {
for (int col = 0; col < num_sb_cols; ++col) {
const int index = row * num_sb_cols + col;
const double saliency = sb_saliency_map->buf[index];
cpi->sm_scaling_factor[index] = 1 - saliency;
sum += cpi->sm_scaling_factor[index];
block_count++;
}
}
assert(block_count > 0);
sum /= block_count;
// Calculate the geometric mean of superblock sm_scaling_factor for a frame,
// to be used for normalization.
for (int row = 0; row < num_sb_rows; ++row) {
for (int col = 0; col < num_sb_cols; ++col) {
const int index = row * num_sb_cols + col;
log_sum += log(fmax(cpi->sm_scaling_factor[index], 0.001));
cpi->sm_scaling_factor[index] =
fmax(cpi->sm_scaling_factor[index], 0.8 * sum);
}
}
log_sum = exp(log_sum / block_count);
// Normalize the sm_scaling_factor by geometric mean.
for (int row = 0; row < num_sb_rows; ++row) {
for (int col = 0; col < num_sb_cols; ++col) {
const int index = row * num_sb_cols + col;
assert(log_sum > 0);
cpi->sm_scaling_factor[index] /= log_sum;
// Modulate the sm_scaling_factor by frame basis motion factor
cpi->sm_scaling_factor[index] =
cpi->sm_scaling_factor[index] * motion_ratio;
}
}
aom_free(sb_saliency_map->buf);
aom_free(sb_saliency_map);
return 1;
}
// av1_setup_motion_ratio() is only enabled when CONFIG_REALTIME_ONLY is 0,
// because the computations need to access the first pass stats which are
// only available when CONFIG_REALTIME_ONLY is equal to 0.
#if !CONFIG_REALTIME_ONLY
// Set motion_ratio that reflects the motion quantities between two consecutive
// frames. Motion_ratio will be used to set up saliency_map based rdmult scaling
// factor, i.e., the less the motion quantities are, the more bits will be spent
// on this frame, and vice versa.
double av1_setup_motion_ratio(AV1_COMP *cpi) {
AV1_COMMON *cm = &cpi->common;
int frames_since_key =
cm->current_frame.display_order_hint - cpi->rc.frames_since_key;
const FIRSTPASS_STATS *cur_stats = av1_firstpass_info_peek(
&cpi->ppi->twopass.firstpass_info, frames_since_key);
assert(cur_stats != NULL);
assert(cpi->ppi->twopass.firstpass_info.total_stats.count > 0);
const double avg_intra_error =
exp(cpi->ppi->twopass.firstpass_info.total_stats.log_intra_error /
cpi->ppi->twopass.firstpass_info.total_stats.count);
const double avg_inter_error =
exp(cpi->ppi->twopass.firstpass_info.total_stats.log_coded_error /
cpi->ppi->twopass.firstpass_info.total_stats.count);
double inter_error = cur_stats->coded_error;
double error_stdev = 0;
const double avg_error =
cpi->ppi->twopass.firstpass_info.total_stats.intra_error /
cpi->ppi->twopass.firstpass_info.total_stats.count;
for (int i = 0; i < cpi->ppi->twopass.firstpass_info.total_stats.count; i++) {
const FIRSTPASS_STATS *stats =
&cpi->ppi->twopass.firstpass_info.stats_buf[i];
error_stdev +=
(stats->intra_error - avg_error) * (stats->intra_error - avg_error);
}
error_stdev =
sqrt(error_stdev / cpi->ppi->twopass.firstpass_info.total_stats.count);
double motion_ratio = 1;
if (error_stdev / fmax(avg_intra_error, 1) > 0.1) {
motion_ratio = inter_error / fmax(1, avg_inter_error);
motion_ratio = AOMMIN(motion_ratio, 1.5);
motion_ratio = AOMMAX(motion_ratio, 0.8);
}
return motion_ratio;
}
#endif // !CONFIG_REALTIME_ONLY