commit 483fd09bb28a8b060ec958c3a139c30850867cc8
author Rubén Aciego <raciego@netflix.com> Mon Jan 15 14:15:53 2024 -0800
committer Joel Sole <jsole@netflix.com> Tue Mar 26 13:45:35 2024 -0700
tree d9abfb82cf45e945204f71faefe4c72105a9d8a0
parent de986bff111e103afd423e0971ad8c7071e78326

br context fix

av1/common/txb_common.h

diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h
index e852873..e3ef3f0 100644
--- a/av1/common/txb_common.h
+++ b/av1/common/txb_common.h
@@ -90,8 +90,8 @@
   const int col = (c - (row << bwl)) + TX_PAD_LEFT;
   const int stride = (1 << bwl) + TX_PAD_LEFT;
   const int pos = row * stride + col;
-  int mag = levels[pos - 1];
-  mag += levels[pos - stride];
+  int mag = AOMMIN(levels[pos - 1], MAX_BASE_BR_RANGE);
+  mag += AOMMIN(levels[pos - stride], MAX_BASE_BR_RANGE);
   mag = AOMMIN(mag, 6);
   if ((row < 2) && (col < (2 + TX_PAD_LEFT))) return mag;
   return mag + 7;
@@ -245,7 +245,7 @@
   const int stride = (1 << bwl) + TX_PAD_HOR;
   const int pos = row * stride + col;
   int mag = AOMMIN(levels[pos + 1], MAX_BASE_BR_RANGE);
-  mag += levels[pos + stride];
+  mag += AOMMIN(levels[pos + stride], MAX_BASE_BR_RANGE);
   switch (tx_class) {
     case TX_CLASS_2D:
       mag += AOMMIN(levels[pos + stride + 1], MAX_BASE_BR_RANGE);
@@ -278,14 +278,14 @@
   const int col = c - (row << bwl);
   const int stride = (1 << bwl) + TX_PAD_HOR;
   const int pos = row * stride + col;
-  int mag = levels[pos + 1];
-  mag += levels[pos + stride];
+  int mag = AOMMIN(levels[pos + 1], MAX_BASE_BR_RANGE);
+  mag += AOMMIN(levels[pos + stride], MAX_BASE_BR_RANGE);
   if (tx_class == TX_CLASS_2D) {
-    mag += levels[pos + stride + 1];
+    mag += AOMMIN(levels[pos + stride + 1], MAX_BASE_BR_RANGE);
   } else if (tx_class == TX_CLASS_VERT) {
-    mag += levels[pos + (stride << 1)];
+    mag += AOMMIN(levels[pos + (stride << 1)], MAX_BASE_BR_RANGE);
   } else {
-    mag += levels[pos + 2];
+    mag += AOMMIN(levels[pos + 2], MAX_BASE_BR_RANGE);
   }
   mag = AOMMIN((mag + 1) >> 1, 6);
   return mag;