Separate model_rd functions from rdopt.c
Created model_rd.h to improve modularity of rdopt.c
model_rd.h : To keep model_rd related functions,
data structures and enum.
Change-Id: If089d3a72af7e54b5dc0ba4a4fbe9073c297d71b
diff --git a/av1/av1.cmake b/av1/av1.cmake
index 01a70bf..b12aca0 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -175,6 +175,7 @@
"${AOM_ROOT}/av1/encoder/mcomp.h"
"${AOM_ROOT}/av1/encoder/ml.c"
"${AOM_ROOT}/av1/encoder/ml.h"
+ "${AOM_ROOT}/av1/encoder/model_rd.h"
"${AOM_ROOT}/av1/encoder/palette.c"
"${AOM_ROOT}/av1/encoder/palette.h"
"${AOM_ROOT}/av1/encoder/partition_strategy.h"
diff --git a/av1/encoder/model_rd.h b/av1/encoder/model_rd.h
new file mode 100644
index 0000000..53c30cc
--- /dev/null
+++ b/av1/encoder/model_rd.h
@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) 2020, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#ifndef AOM_AV1_ENCODER_MODEL_RD_H_
+#define AOM_AV1_ENCODER_MODEL_RD_H_
+
+#include "aom/aom_integer.h"
+#include "av1/encoder/block.h"
+#include "av1/encoder/encoder.h"
+#include "av1/encoder/pustats.h"
+#include "av1/encoder/rdopt_utils.h"
+#include "aom_ports/system_state.h"
+#include "config/aom_dsp_rtcd.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// 0: Legacy model
+// 1: Curve fit model
+// 2: Surface fit model
+// 3: DNN regression model
+// 4: Full rd model
+#define MODELRD_TYPE_INTERP_FILTER 1
+#define MODELRD_TYPE_TX_SEARCH_PRUNE 1
+#define MODELRD_TYPE_MASKED_COMPOUND 1
+#define MODELRD_TYPE_INTERINTRA 1
+#define MODELRD_TYPE_INTRA 1
+#define MODELRD_TYPE_MOTION_MODE_RD 1
+
+typedef void (*model_rd_for_sb_type)(const AV1_COMP *const cpi,
+ BLOCK_SIZE bsize, MACROBLOCK *x,
+ MACROBLOCKD *xd, int plane_from,
+ int plane_to, int *out_rate_sum,
+ int64_t *out_dist_sum, int *skip_txfm_sb,
+ int64_t *skip_sse_sb, int *plane_rate,
+ int64_t *plane_sse, int64_t *plane_dist);
+typedef void (*model_rd_from_sse_type)(const AV1_COMP *const cpi,
+ const MACROBLOCK *const x,
+ BLOCK_SIZE plane_bsize, int plane,
+ int64_t sse, int num_samples, int *rate,
+ int64_t *dist);
+
+static int64_t calculate_sse(MACROBLOCKD *const xd,
+ const struct macroblock_plane *p,
+ struct macroblockd_plane *pd, const int bw,
+ const int bh) {
+ int64_t sse = 0;
+ const int shift = xd->bd - 8;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (is_cur_buf_hbd(xd)) {
+ sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
+ bw, bh);
+ } else {
+ sse =
+ aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
+ }
+#else
+ sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
+#endif
+ sse = ROUND_POWER_OF_TWO(sse, shift * 2);
+ return sse;
+}
+
+static AOM_INLINE void model_rd_from_sse(const AV1_COMP *const cpi,
+ const MACROBLOCK *const x,
+ BLOCK_SIZE plane_bsize, int plane,
+ int64_t sse, int num_samples,
+ int *rate, int64_t *dist) {
+ (void)num_samples;
+ const MACROBLOCKD *const xd = &x->e_mbd;
+ const struct macroblock_plane *const p = &x->plane[plane];
+ const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
+
+ // Fast approximate the modelling function.
+ if (cpi->sf.rd_sf.simple_model_rd_from_var) {
+ const int64_t square_error = sse;
+ int quantizer = p->dequant_QTX[1] >> dequant_shift;
+ if (quantizer < 120)
+ *rate = (int)AOMMIN(
+ (square_error * (280 - quantizer)) >> (16 - AV1_PROB_COST_SHIFT),
+ INT_MAX);
+ else
+ *rate = 0;
+ assert(*rate >= 0);
+ *dist = (square_error * quantizer) >> 8;
+ } else {
+ av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[plane_bsize],
+ p->dequant_QTX[1] >> dequant_shift, rate,
+ dist);
+ }
+ *dist <<= 4;
+}
+
+// Fits a curve for rate and distortion using as feature:
+// log2(sse_norm/qstep^2)
+static AOM_INLINE void model_rd_with_curvfit(const AV1_COMP *const cpi,
+ const MACROBLOCK *const x,
+ BLOCK_SIZE plane_bsize, int plane,
+ int64_t sse, int num_samples,
+ int *rate, int64_t *dist) {
+ (void)cpi;
+ (void)plane_bsize;
+ const MACROBLOCKD *const xd = &x->e_mbd;
+ const struct macroblock_plane *const p = &x->plane[plane];
+ const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
+ const int qstep = AOMMAX(p->dequant_QTX[1] >> dequant_shift, 1);
+
+ if (sse == 0) {
+ if (rate) *rate = 0;
+ if (dist) *dist = 0;
+ return;
+ }
+ aom_clear_system_state();
+ const double sse_norm = (double)sse / num_samples;
+ const double qstepsqr = (double)qstep * qstep;
+ const double xqr = log2(sse_norm / qstepsqr);
+ double rate_f, dist_by_sse_norm_f;
+ av1_model_rd_curvfit(plane_bsize, sse_norm, xqr, &rate_f,
+ &dist_by_sse_norm_f);
+
+ const double dist_f = dist_by_sse_norm_f * sse_norm;
+ int rate_i = (int)(AOMMAX(0.0, rate_f * num_samples) + 0.5);
+ int64_t dist_i = (int64_t)(AOMMAX(0.0, dist_f * num_samples) + 0.5);
+ aom_clear_system_state();
+
+ // Check if skip is better
+ if (rate_i == 0) {
+ dist_i = sse << 4;
+ } else if (RDCOST(x->rdmult, rate_i, dist_i) >=
+ RDCOST(x->rdmult, 0, sse << 4)) {
+ rate_i = 0;
+ dist_i = sse << 4;
+ }
+
+ if (rate) *rate = rate_i;
+ if (dist) *dist = dist_i;
+}
+
+static AOM_INLINE void model_rd_for_sb(
+ const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
+ int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
+ int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
+ int64_t *plane_sse, int64_t *plane_dist) {
+ // Note our transform coeffs are 8 times an orthogonal transform.
+ // Hence quantizer step is also 8 times. To get effective quantizer
+ // we need to divide by 8 before sending to modeling function.
+ int plane;
+ const int ref = xd->mi[0]->ref_frame[0];
+
+ int64_t rate_sum = 0;
+ int64_t dist_sum = 0;
+ int64_t total_sse = 0;
+
+ assert(bsize < BLOCK_SIZES_ALL);
+
+ for (plane = plane_from; plane <= plane_to; ++plane) {
+ struct macroblock_plane *const p = &x->plane[plane];
+ struct macroblockd_plane *const pd = &xd->plane[plane];
+ const BLOCK_SIZE plane_bsize =
+ get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
+ assert(plane_bsize < BLOCK_SIZES_ALL);
+ const int bw = block_size_wide[plane_bsize];
+ const int bh = block_size_high[plane_bsize];
+ int64_t sse;
+ int rate;
+ int64_t dist;
+
+ if (x->skip_chroma_rd && plane) continue;
+
+ sse = calculate_sse(xd, p, pd, bw, bh);
+
+ model_rd_from_sse(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, &dist);
+
+ if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
+
+ total_sse += sse;
+ rate_sum += rate;
+ dist_sum += dist;
+ if (plane_rate) plane_rate[plane] = rate;
+ if (plane_sse) plane_sse[plane] = sse;
+ if (plane_dist) plane_dist[plane] = dist;
+ assert(rate_sum >= 0);
+ }
+
+ if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0;
+ if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
+ rate_sum = AOMMIN(rate_sum, INT_MAX);
+ *out_rate_sum = (int)rate_sum;
+ *out_dist_sum = dist_sum;
+}
+
+static AOM_INLINE void model_rd_for_sb_with_curvfit(
+ const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
+ int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
+ int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
+ int64_t *plane_sse, int64_t *plane_dist) {
+ // Note our transform coeffs are 8 times an orthogonal transform.
+ // Hence quantizer step is also 8 times. To get effective quantizer
+ // we need to divide by 8 before sending to modeling function.
+ const int ref = xd->mi[0]->ref_frame[0];
+
+ int64_t rate_sum = 0;
+ int64_t dist_sum = 0;
+ int64_t total_sse = 0;
+
+ for (int plane = plane_from; plane <= plane_to; ++plane) {
+ struct macroblockd_plane *const pd = &xd->plane[plane];
+ const BLOCK_SIZE plane_bsize =
+ get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
+ int64_t dist, sse;
+ int rate;
+
+ if (x->skip_chroma_rd && plane) continue;
+
+ int bw, bh;
+ const struct macroblock_plane *const p = &x->plane[plane];
+ get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL,
+ &bw, &bh);
+
+ sse = calculate_sse(xd, p, pd, bw, bh);
+ model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate,
+ &dist);
+
+ if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
+
+ total_sse += sse;
+ rate_sum += rate;
+ dist_sum += dist;
+
+ if (plane_rate) plane_rate[plane] = rate;
+ if (plane_sse) plane_sse[plane] = sse;
+ if (plane_dist) plane_dist[plane] = dist;
+ }
+
+ if (skip_txfm_sb) *skip_txfm_sb = rate_sum == 0;
+ if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
+ *out_rate_sum = (int)rate_sum;
+ *out_dist_sum = dist_sum;
+}
+
+enum { MODELRD_LEGACY, MODELRD_CURVFIT, MODELRD_TYPES } UENUM1BYTE(ModelRdType);
+
+static model_rd_for_sb_type model_rd_sb_fn[MODELRD_TYPES] = {
+ model_rd_for_sb, model_rd_for_sb_with_curvfit
+};
+
+static model_rd_from_sse_type model_rd_sse_fn[MODELRD_TYPES] = {
+ model_rd_from_sse, model_rd_with_curvfit
+};
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+#endif // AOM_AV1_ENCODER_MODEL_RD_H_
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 35a7dc0..0a82562 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -52,6 +52,7 @@
#include "av1/encoder/mcomp.h"
#include "av1/encoder/ml.h"
#include "av1/encoder/mode_prune_model_weights.h"
+#include "av1/encoder/model_rd.h"
#include "av1/encoder/palette.h"
#include "av1/encoder/pustats.h"
#include "av1/encoder/random.h"
@@ -71,65 +72,6 @@
static const char av1_tx_size_data_output_file[] = "tx_size_data.txt";
#endif
-typedef void (*model_rd_for_sb_type)(const AV1_COMP *const cpi,
- BLOCK_SIZE bsize, MACROBLOCK *x,
- MACROBLOCKD *xd, int plane_from,
- int plane_to, int *out_rate_sum,
- int64_t *out_dist_sum, int *skip_txfm_sb,
- int64_t *skip_sse_sb, int *plane_rate,
- int64_t *plane_sse, int64_t *plane_dist);
-typedef void (*model_rd_from_sse_type)(const AV1_COMP *const cpi,
- const MACROBLOCK *const x,
- BLOCK_SIZE plane_bsize, int plane,
- int64_t sse, int num_samples, int *rate,
- int64_t *dist);
-
-static AOM_INLINE void model_rd_for_sb(const AV1_COMP *const cpi,
- BLOCK_SIZE bsize, MACROBLOCK *x,
- MACROBLOCKD *xd, int plane_from,
- int plane_to, int *out_rate_sum,
- int64_t *out_dist_sum, int *skip_txfm_sb,
- int64_t *skip_sse_sb, int *plane_rate,
- int64_t *plane_sse, int64_t *plane_dist);
-static AOM_INLINE void model_rd_for_sb_with_curvfit(
- const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
- int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
- int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
- int64_t *plane_sse, int64_t *plane_dist);
-
-static AOM_INLINE void model_rd_from_sse(const AV1_COMP *const cpi,
- const MACROBLOCK *const x,
- BLOCK_SIZE plane_bsize, int plane,
- int64_t sse, int num_samples,
- int *rate, int64_t *dist);
-static AOM_INLINE void model_rd_with_curvfit(const AV1_COMP *const cpi,
- const MACROBLOCK *const x,
- BLOCK_SIZE plane_bsize, int plane,
- int64_t sse, int num_samples,
- int *rate, int64_t *dist);
-
-enum { MODELRD_LEGACY, MODELRD_CURVFIT, MODELRD_TYPES } UENUM1BYTE(ModelRdType);
-
-static model_rd_for_sb_type model_rd_sb_fn[MODELRD_TYPES] = {
- model_rd_for_sb, model_rd_for_sb_with_curvfit
-};
-
-static model_rd_from_sse_type model_rd_sse_fn[MODELRD_TYPES] = {
- model_rd_from_sse, model_rd_with_curvfit
-};
-
-// 0: Legacy model
-// 1: Curve fit model
-// 2: Surface fit model
-// 3: DNN regression model
-// 4: Full rd model
-#define MODELRD_TYPE_INTERP_FILTER 1
-#define MODELRD_TYPE_TX_SEARCH_PRUNE 1
-#define MODELRD_TYPE_MASKED_COMPOUND 1
-#define MODELRD_TYPE_INTERINTRA 1
-#define MODELRD_TYPE_INTRA 1
-#define MODELRD_TYPE_MOTION_MODE_RD 1
-
#define DUAL_FILTER_SET_SIZE (SWITCHABLE_FILTERS * SWITCHABLE_FILTERS)
static const int_interpfilters filter_sets[DUAL_FILTER_SET_SIZE] = {
{ 0x00000000 }, { 0x00010000 }, { 0x00020000 }, // y = 0
@@ -1663,36 +1605,6 @@
*allowed_tx_mask = allow_bitmask;
}
-static AOM_INLINE void model_rd_from_sse(const AV1_COMP *const cpi,
- const MACROBLOCK *const x,
- BLOCK_SIZE plane_bsize, int plane,
- int64_t sse, int num_samples,
- int *rate, int64_t *dist) {
- (void)num_samples;
- const MACROBLOCKD *const xd = &x->e_mbd;
- const struct macroblock_plane *const p = &x->plane[plane];
- const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
-
- // Fast approximate the modelling function.
- if (cpi->sf.rd_sf.simple_model_rd_from_var) {
- const int64_t square_error = sse;
- int quantizer = p->dequant_QTX[1] >> dequant_shift;
- if (quantizer < 120)
- *rate = (int)AOMMIN(
- (square_error * (280 - quantizer)) >> (16 - AV1_PROB_COST_SHIFT),
- INT_MAX);
- else
- *rate = 0;
- assert(*rate >= 0);
- *dist = (square_error * quantizer) >> 8;
- } else {
- av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[plane_bsize],
- p->dequant_QTX[1] >> dequant_shift, rate,
- dist);
- }
- *dist <<= 4;
-}
-
static int64_t get_sse(const AV1_COMP *cpi, const MACROBLOCK *x) {
const AV1_COMMON *cm = &cpi->common;
const int num_planes = av1_num_planes(cm);
@@ -1716,80 +1628,6 @@
return total_sse;
}
-static int64_t calculate_sse(MACROBLOCKD *const xd,
- const struct macroblock_plane *p,
- struct macroblockd_plane *pd, const int bw,
- const int bh) {
- int64_t sse = 0;
- const int shift = xd->bd - 8;
-#if CONFIG_AV1_HIGHBITDEPTH
- if (is_cur_buf_hbd(xd)) {
- sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
- bw, bh);
- } else {
- sse =
- aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
- }
-#else
- sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
-#endif
- sse = ROUND_POWER_OF_TWO(sse, shift * 2);
- return sse;
-}
-
-static AOM_INLINE void model_rd_for_sb(
- const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
- int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
- int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
- int64_t *plane_sse, int64_t *plane_dist) {
- // Note our transform coeffs are 8 times an orthogonal transform.
- // Hence quantizer step is also 8 times. To get effective quantizer
- // we need to divide by 8 before sending to modeling function.
- int plane;
- const int ref = xd->mi[0]->ref_frame[0];
-
- int64_t rate_sum = 0;
- int64_t dist_sum = 0;
- int64_t total_sse = 0;
-
- assert(bsize < BLOCK_SIZES_ALL);
-
- for (plane = plane_from; plane <= plane_to; ++plane) {
- struct macroblock_plane *const p = &x->plane[plane];
- struct macroblockd_plane *const pd = &xd->plane[plane];
- const BLOCK_SIZE plane_bsize =
- get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
- assert(plane_bsize < BLOCK_SIZES_ALL);
- const int bw = block_size_wide[plane_bsize];
- const int bh = block_size_high[plane_bsize];
- int64_t sse;
- int rate;
- int64_t dist;
-
- if (x->skip_chroma_rd && plane) continue;
-
- sse = calculate_sse(xd, p, pd, bw, bh);
-
- model_rd_from_sse(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, &dist);
-
- if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
-
- total_sse += sse;
- rate_sum += rate;
- dist_sum += dist;
- if (plane_rate) plane_rate[plane] = rate;
- if (plane_sse) plane_sse[plane] = sse;
- if (plane_dist) plane_dist[plane] = dist;
- assert(rate_sum >= 0);
- }
-
- if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0;
- if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
- rate_sum = AOMMIN(rate_sum, INT_MAX);
- *out_rate_sum = (int)rate_sum;
- *out_dist_sum = dist_sum;
-}
-
int64_t av1_block_error_c(const tran_low_t *coeff, const tran_low_t *dqcoeff,
intptr_t block_size, int64_t *ssz) {
int i;
@@ -2516,100 +2354,6 @@
#endif // CONFIG_COLLECT_RD_STATS >= 2
#endif // CONFIG_COLLECT_RD_STATS
-// Fits a curve for rate and distortion using as feature:
-// log2(sse_norm/qstep^2)
-static AOM_INLINE void model_rd_with_curvfit(const AV1_COMP *const cpi,
- const MACROBLOCK *const x,
- BLOCK_SIZE plane_bsize, int plane,
- int64_t sse, int num_samples,
- int *rate, int64_t *dist) {
- (void)cpi;
- (void)plane_bsize;
- const MACROBLOCKD *const xd = &x->e_mbd;
- const struct macroblock_plane *const p = &x->plane[plane];
- const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
- const int qstep = AOMMAX(p->dequant_QTX[1] >> dequant_shift, 1);
-
- if (sse == 0) {
- if (rate) *rate = 0;
- if (dist) *dist = 0;
- return;
- }
- aom_clear_system_state();
- const double sse_norm = (double)sse / num_samples;
- const double qstepsqr = (double)qstep * qstep;
- const double xqr = log2(sse_norm / qstepsqr);
- double rate_f, dist_by_sse_norm_f;
- av1_model_rd_curvfit(plane_bsize, sse_norm, xqr, &rate_f,
- &dist_by_sse_norm_f);
-
- const double dist_f = dist_by_sse_norm_f * sse_norm;
- int rate_i = (int)(AOMMAX(0.0, rate_f * num_samples) + 0.5);
- int64_t dist_i = (int64_t)(AOMMAX(0.0, dist_f * num_samples) + 0.5);
- aom_clear_system_state();
-
- // Check if skip is better
- if (rate_i == 0) {
- dist_i = sse << 4;
- } else if (RDCOST(x->rdmult, rate_i, dist_i) >=
- RDCOST(x->rdmult, 0, sse << 4)) {
- rate_i = 0;
- dist_i = sse << 4;
- }
-
- if (rate) *rate = rate_i;
- if (dist) *dist = dist_i;
-}
-
-static AOM_INLINE void model_rd_for_sb_with_curvfit(
- const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
- int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
- int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
- int64_t *plane_sse, int64_t *plane_dist) {
- // Note our transform coeffs are 8 times an orthogonal transform.
- // Hence quantizer step is also 8 times. To get effective quantizer
- // we need to divide by 8 before sending to modeling function.
- const int ref = xd->mi[0]->ref_frame[0];
-
- int64_t rate_sum = 0;
- int64_t dist_sum = 0;
- int64_t total_sse = 0;
-
- for (int plane = plane_from; plane <= plane_to; ++plane) {
- struct macroblockd_plane *const pd = &xd->plane[plane];
- const BLOCK_SIZE plane_bsize =
- get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
- int64_t dist, sse;
- int rate;
-
- if (x->skip_chroma_rd && plane) continue;
-
- int bw, bh;
- const struct macroblock_plane *const p = &x->plane[plane];
- get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL,
- &bw, &bh);
-
- sse = calculate_sse(xd, p, pd, bw, bh);
- model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate,
- &dist);
-
- if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
-
- total_sse += sse;
- rate_sum += rate;
- dist_sum += dist;
-
- if (plane_rate) plane_rate[plane] = rate;
- if (plane_sse) plane_sse[plane] = sse;
- if (plane_dist) plane_dist[plane] = dist;
- }
-
- if (skip_txfm_sb) *skip_txfm_sb = rate_sum == 0;
- if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
- *out_rate_sum = (int)rate_sum;
- *out_dist_sum = dist_sum;
-}
-
static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
