Fix a hbd bug in av1_choose_var_based_partitioning
Fix a high bit depth bug in av1_choose_var_based_partitioning(). We
should not use the uint8_t array AV1_VAR_OFFS if is_cur_buf_hbd(xd) is
true.
Abstract the correct code in av1_get_perpixel_variance() into a new
helper function av1_var_offs(). In av1_var_offs(), conditionally compile
the high bit depth code only if CONFIG_AV1_HIGHBITDEPTH is equal to 1.
Bug: aomedia:3409
Change-Id: Ibea1f23a5b76e3617ed6f766c7a834ad4a41a97d
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index de197e6..5446dcf 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -81,7 +81,7 @@
// purposes of activity masking.
// Eventually this should be replaced by custom no-reference routines,
// which will be faster.
-const uint8_t AV1_VAR_OFFS[MAX_SB_SIZE] = {
+static const uint8_t AV1_VAR_OFFS[MAX_SB_SIZE] = {
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
@@ -93,6 +93,7 @@
128, 128, 128, 128, 128, 128, 128, 128
};
+#if CONFIG_AV1_HIGHBITDEPTH
static const uint16_t AV1_HIGH_VAR_OFFS_8[MAX_SB_SIZE] = {
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
@@ -145,8 +146,28 @@
128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
128 * 16, 128 * 16
};
+#endif // CONFIG_AV1_HIGHBITDEPTH
/*!\endcond */
+const uint8_t *av1_var_offs(int use_hbd, int bd) {
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (use_hbd) {
+ assert(bd == 8 || bd == 10 || bd == 12);
+ const int off_index = (bd - 8) >> 1;
+ static const uint16_t *high_var_offs[3] = { AV1_HIGH_VAR_OFFS_8,
+ AV1_HIGH_VAR_OFFS_10,
+ AV1_HIGH_VAR_OFFS_12 };
+ return CONVERT_TO_BYTEPTR(high_var_offs[off_index]);
+ }
+#else
+ (void)use_hbd;
+ (void)bd;
+ assert(!use_hbd);
+#endif
+ assert(bd == 8);
+ return AV1_VAR_OFFS;
+}
+
void av1_init_rtc_counters(MACROBLOCK *const x) {
av1_init_cyclic_refresh_counters(x);
x->cnt_zeromv = 0;
@@ -167,21 +188,9 @@
const int subsampling_y = xd->plane[plane].subsampling_y;
const BLOCK_SIZE plane_bsize =
get_plane_block_size(bsize, subsampling_x, subsampling_y);
- unsigned int var, sse;
- if (use_hbd) {
- const int bd = xd->bd;
- assert(bd == 8 || bd == 10 || bd == 12);
- const int off_index = (bd - 8) >> 1;
- static const uint16_t *high_var_offs[3] = { AV1_HIGH_VAR_OFFS_8,
- AV1_HIGH_VAR_OFFS_10,
- AV1_HIGH_VAR_OFFS_12 };
- var = cpi->ppi->fn_ptr[plane_bsize].vf(
- ref->buf, ref->stride, CONVERT_TO_BYTEPTR(high_var_offs[off_index]), 0,
- &sse);
- } else {
- var = cpi->ppi->fn_ptr[plane_bsize].vf(ref->buf, ref->stride, AV1_VAR_OFFS,
- 0, &sse);
- }
+ unsigned int sse;
+ const unsigned int var = cpi->ppi->fn_ptr[plane_bsize].vf(
+ ref->buf, ref->stride, av1_var_offs(use_hbd, xd->bd), 0, &sse);
return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[plane_bsize]);
}
diff --git a/av1/encoder/encodeframe.h b/av1/encoder/encodeframe.h
index ce32fb4..0f3511d 100644
--- a/av1/encoder/encodeframe.h
+++ b/av1/encoder/encodeframe.h
@@ -31,6 +31,11 @@
struct AV1_COMP;
struct ThreadData;
+// For the given bit depth, returns a constant array used to assist the
+// calculation of source block variance, which will then be used to decide
+// adaptive quantizers.
+const uint8_t *av1_var_offs(int use_hbd, int bd);
+
void av1_init_rtc_counters(struct macroblock *const x);
void av1_accumulate_rtc_counters(struct AV1_COMP *cpi,
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 8a5ee18..bc91ba6 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -30,8 +30,6 @@
#include "av1/encoder/var_based_part.h"
#include "av1/encoder/reconinter_enc.h"
-extern const uint8_t AV1_VAR_OFFS[];
-
// Possible values for the force_split variable while evaluating variance based
// partitioning.
enum {
@@ -1650,7 +1648,7 @@
dst_stride = xd->plane[AOM_PLANE_Y].pre[0].stride;
}
} else {
- dst_buf = AV1_VAR_OFFS;
+ dst_buf = av1_var_offs(is_cur_buf_hbd(xd), xd->bd);
dst_stride = 0;
}
