commit 4a8212a16a2f4a5cbbdfa5e8b259037c0f7906b5
author Rubén Aciego <raciego@netflix.com> Tue Jan 30 17:39:43 2024 -0800
committer Joel Sole <jsole@netflix.com> Tue Mar 26 13:46:03 2024 -0700
tree 7bc297d9b8e7e997c3b14321e85ea1619324c56f
parent f4ab0f961bdc4cd396cf205e9c0c58edaccfbbae

0 lr

av1/common/entropy.h

diff --git a/av1/common/entropy.h b/av1/common/entropy.h
index fab9004..361c2b4 100644
--- a/av1/common/entropy.h
+++ b/av1/common/entropy.h
@@ -97,7 +97,7 @@
 #define NUM_BASE_LEVELS 2
 
 #define BR_CDF_SIZE (4)
-#define COEFF_BASE_RANGE (1 * (BR_CDF_SIZE - 1))
+#define COEFF_BASE_RANGE (0 * (BR_CDF_SIZE - 1))
 
 #define COEFF_CONTEXT_BITS 3
 #define COEFF_CONTEXT_MASK ((1 << COEFF_CONTEXT_BITS) - 1)

av1/common/txb_common.h

diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h
index e3ef3f0..8f3bfae 100644
--- a/av1/common/txb_common.h
+++ b/av1/common/txb_common.h
@@ -396,12 +396,12 @@
                                           const TX_CLASS tx_class) {
   int mag;
   // Note: AOMMIN(level, 5) is useless for decoder since level < 5.
-  mag = clip_max5[levels[1]];                         // { 0, 1 }
-  mag += clip_max5[levels[(1 << bwl) + TX_PAD_HOR]];  // { 1, 0 }
+  mag = clip_max3[levels[1]];                         // { 0, 1 }
+  mag += clip_max3[levels[(1 << bwl) + TX_PAD_HOR]];  // { 1, 0 }
   if (tx_class == TX_CLASS_2D) {
-    mag += clip_max5[levels[(1 << bwl) + TX_PAD_HOR + 1]];          // { 1, 1 }
-    mag += clip_max5[levels[2]];                                    // { 0, 2 }
-    mag += clip_max5[levels[(2 << bwl) + (2 << TX_PAD_HOR_LOG2)]];  // { 2, 0 }
+    mag += clip_max3[levels[(1 << bwl) + TX_PAD_HOR + 1]];          // { 1, 1 }
+    mag += clip_max3[levels[2]];                                    // { 0, 2 }
+    mag += clip_max3[levels[(2 << bwl) + (2 << TX_PAD_HOR_LOG2)]];  // { 2, 0 }
   } else if (tx_class == TX_CLASS_VERT) {
     mag += clip_max3[levels[(2 << bwl) + (2 << TX_PAD_HOR_LOG2)]];  // { 2, 0 }
     mag += clip_max3[levels[(3 << bwl) + (3 << TX_PAD_HOR_LOG2)]];  // { 3, 0 }
@@ -706,11 +706,11 @@
   int mag;
   // Note: AOMMIN(level, 3) is useless for decoder since level < 5.
   levels = levels + get_padded_idx(coeff_idx, bwl);
-  mag = AOMMIN(levels[1], 5);                                     // { 0, 1 }
-  mag += AOMMIN(levels[(1 << bwl) + TX_PAD_HOR], 5);              // { 1, 0 }
-  mag += AOMMIN(levels[(1 << bwl) + TX_PAD_HOR + 1], 5);          // { 1, 1 }
-  mag += AOMMIN(levels[2], 5);                                    // { 0, 2 }
-  mag += AOMMIN(levels[(2 << bwl) + (2 << TX_PAD_HOR_LOG2)], 5);  // { 2, 0 }
+  mag = AOMMIN(levels[1], 3);                                     // { 0, 1 }
+  mag += AOMMIN(levels[(1 << bwl) + TX_PAD_HOR], 3);              // { 1, 0 }
+  mag += AOMMIN(levels[(1 << bwl) + TX_PAD_HOR + 1], 3);          // { 1, 1 }
+  mag += AOMMIN(levels[2], 3);                                    // { 0, 2 }
+  mag += AOMMIN(levels[(2 << bwl) + (2 << TX_PAD_HOR_LOG2)], 3);  // { 2, 0 }
   int ctx = (mag + 1) >> 1;
   const int row = coeff_idx >> bwl;
   const int col = coeff_idx - (row << bwl);