Use lower-precision filters in filter_intra
Filter coeffcients c0, c1, c2 are scaled by 8, and can be
represented by 4 bits unsigned integer (c2 is always <=0)
Change-Id: I93643bab6734214cef0b0175d6980ebabe9dfe10
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c
index a285e87..a10928a 100644
--- a/av1/common/reconintra.c
+++ b/av1/common/reconintra.c
@@ -1118,127 +1118,145 @@
#if CONFIG_FILTER_INTRA
static int filter_intra_taps_3[TX_SIZES_ALL][FILTER_INTRA_MODES][3] = {
{
- { 697, 836, -509 },
- { 993, 513, -482 },
- { 381, 984, -341 },
- { 762, 385, -123 },
- { 358, 687, -21 },
- { 883, 902, -761 },
+ { 5, 7, -4 },
+ { 8, 4, -4 },
+ { 3, 8, -3 },
+ { 6, 3, -1 },
+ { 3, 5, 0 },
+ { 7, 7, -6 },
},
{
- { 659, 816, -451 },
- { 980, 625, -581 },
- { 558, 962, -496 },
- { 778, 399, -153 },
- { 495, 641, -112 },
- { 839, 911, -726 },
+ { 5, 6, -3 },
+ { 8, 5, -5 },
+ { 4, 8, -4 },
+ { 6, 3, -1 },
+ { 4, 5, -1 },
+ { 7, 7, -6 },
},
{
- { 539, 927, -442 },
- { 1003, 714, -693 },
- { 349, 1271, -596 },
- { 780, 681, -437 },
- { 586, 795, -357 },
- { 974, 970, -920 },
+ { 4, 7, -3 },
+ { 8, 6, -6 },
+ { 3, 10, -5 },
+ { 6, 5, -3 },
+ { 5, 6, -3 },
+ { 8, 8, -8 },
},
{
- { 595, 919, -490 },
- { 945, 668, -589 },
- { 495, 962, -433 },
- { 852, 478, -306 },
- { 177, 760, 87 },
- { 846, 1010, -832 },
+ { 5, 7, -4 },
+ { 7, 5, -4 },
+ { 4, 8, -4 },
+ { 7, 4, -3 },
+ { 2, 6, 0 },
+ { 7, 8, -7 },
},
#if CONFIG_TX64X64
{
- { 595, 919, -490 },
- { 945, 668, -589 },
- { 495, 962, -433 },
- { 852, 478, -306 },
- { 177, 760, 87 },
- { 846, 1010, -832 },
+ { 5, 7, -4 },
+ { 7, 5, -4 },
+ { 4, 8, -4 },
+ { 7, 4, -3 },
+ { 2, 6, 0 },
+ { 7, 8, -7 },
},
#endif // CONFIG_TX64X64
{
- { 697, 836, -509 },
- { 993, 513, -482 },
- { 381, 984, -341 },
- { 762, 385, -123 },
- { 358, 687, -21 },
- { 883, 902, -761 },
+ { 5, 7, -4 },
+ { 8, 4, -4 },
+ { 3, 8, -3 },
+ { 6, 3, -1 },
+ { 3, 5, 0 },
+ { 7, 7, -6 },
},
{
- { 697, 836, -509 },
- { 993, 513, -482 },
- { 381, 984, -341 },
- { 762, 385, -123 },
- { 358, 687, -21 },
- { 883, 902, -761 },
+ { 5, 7, -4 },
+ { 8, 4, -4 },
+ { 3, 8, -3 },
+ { 6, 3, -1 },
+ { 3, 5, 0 },
+ { 7, 7, -6 },
},
{
- { 659, 816, -451 },
- { 980, 625, -581 },
- { 558, 962, -496 },
- { 778, 399, -153 },
- { 495, 641, -112 },
- { 839, 911, -726 },
+ { 5, 6, -3 },
+ { 8, 5, -5 },
+ { 4, 8, -4 },
+ { 6, 3, -1 },
+ { 4, 5, -1 },
+ { 7, 7, -6 },
},
{
- { 659, 816, -451 },
- { 980, 625, -581 },
- { 558, 962, -496 },
- { 778, 399, -153 },
- { 495, 641, -112 },
- { 839, 911, -726 },
+ { 5, 6, -3 },
+ { 8, 5, -5 },
+ { 4, 8, -4 },
+ { 6, 3, -1 },
+ { 4, 5, -1 },
+ { 7, 7, -6 },
},
{
- { 539, 927, -442 },
- { 1003, 714, -693 },
- { 349, 1271, -596 },
- { 780, 681, -437 },
- { 586, 795, -357 },
- { 974, 970, -920 },
+ { 4, 7, -3 },
+ { 8, 6, -6 },
+ { 3, 10, -5 },
+ { 6, 5, -3 },
+ { 5, 6, -3 },
+ { 8, 8, -8 },
},
{
- { 539, 927, -442 },
- { 1003, 714, -693 },
- { 349, 1271, -596 },
- { 780, 681, -437 },
- { 586, 795, -357 },
- { 974, 970, -920 },
+ { 4, 7, -3 },
+ { 8, 6, -6 },
+ { 3, 10, -5 },
+ { 6, 5, -3 },
+ { 5, 6, -3 },
+ { 8, 8, -8 },
+ },
+#if CONFIG_TX64X64
+ {
+ { 5, 7, -4 },
+ { 7, 5, -4 },
+ { 4, 8, -4 },
+ { 7, 4, -3 },
+ { 2, 6, 0 },
+ { 7, 8, -7 },
},
{
- { 697, 836, -509 },
- { 993, 513, -482 },
- { 381, 984, -341 },
- { 762, 385, -123 },
- { 358, 687, -21 },
- { 883, 902, -761 },
+ { 5, 7, -4 },
+ { 7, 5, -4 },
+ { 4, 8, -4 },
+ { 7, 4, -3 },
+ { 2, 6, 0 },
+ { 7, 8, -7 },
+ },
+#endif // CONFIG_TX64X64
+ {
+ { 5, 7, -4 },
+ { 8, 4, -4 },
+ { 3, 8, -3 },
+ { 6, 3, -1 },
+ { 3, 5, 0 },
+ { 7, 7, -6 },
},
{
- { 697, 836, -509 },
- { 993, 513, -482 },
- { 381, 984, -341 },
- { 762, 385, -123 },
- { 358, 687, -21 },
- { 883, 902, -761 },
+ { 5, 7, -4 },
+ { 8, 4, -4 },
+ { 3, 8, -3 },
+ { 6, 3, -1 },
+ { 3, 5, 0 },
+ { 7, 7, -6 },
},
{
- { 659, 816, -451 },
- { 980, 625, -581 },
- { 558, 962, -496 },
- { 778, 399, -153 },
- { 495, 641, -112 },
- { 839, 911, -726 },
+ { 5, 6, -3 },
+ { 8, 5, -5 },
+ { 4, 8, -4 },
+ { 6, 3, -1 },
+ { 4, 5, -1 },
+ { 7, 7, -6 },
},
{
- { 659, 816, -451 },
- { 980, 625, -581 },
- { 558, 962, -496 },
- { 778, 399, -153 },
- { 495, 641, -112 },
- { 839, 911, -726 },
- }
+ { 5, 6, -3 },
+ { 8, 5, -5 },
+ { 4, 8, -4 },
+ { 6, 3, -1 },
+ { 4, 5, -1 },
+ { 7, 7, -6 },
+ },
};
static void filter_intra_predictors_3tap(uint8_t *dst, ptrdiff_t stride,
@@ -1265,7 +1283,7 @@
for (c = 1; c < bw + 1; ++c) {
ipred = c0 * buffer[r - 1][c] + c1 * buffer[r][c - 1] +
c2 * buffer[r - 1][c - 1];
- buffer[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
+ buffer[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_SCALE_BITS);
buffer[r][c] = clip_pixel(buffer[r][c]);
}
@@ -1368,7 +1386,7 @@
for (c = 1; c < bw + 1; ++c) {
ipred = c0 * preds[r - 1][c] + c1 * preds[r][c - 1] +
c2 * preds[r - 1][c - 1];
- preds[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
+ preds[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_SCALE_BITS);
preds[r][c] = clip_pixel_highbd(preds[r][c], bd);
}
diff --git a/av1/common/reconintra.h b/av1/common/reconintra.h
index 87e030e..b821752 100644
--- a/av1/common/reconintra.h
+++ b/av1/common/reconintra.h
@@ -42,7 +42,7 @@
};
#if CONFIG_FILTER_INTRA
-#define FILTER_INTRA_PREC_BITS 10
+#define FILTER_INTRA_SCALE_BITS 3
#endif // CONFIG_FILTER_INTRA
#define CONFIG_INTRA_EDGE_UPSAMPLE CONFIG_INTRA_EDGE
