commit 80a21c19c69aceb48176f4d58f0923f73b3772c8
author Wan-Teh Chang <wtc@google.com> Thu Nov 14 16:05:41 2024 -0800
committer Wan-Teh Chang <wtc@google.com> Fri Nov 15 21:44:21 2024 +0000
tree e3a17b7ec6611cdcc481f0f9769971990c0c11d9
parent 5406c5739bfa3bfe738d21b18cfbd772961e4b88

Don't assert signed integer overflow didn't happen

These functions apparently use the assertion
  assert(error >= 0 && sqcoeff >= 0);
to assert that signed integer overflow didn't happen during the
calculation of `error` and `sqcoeff`. Since signed integer overflow is
undefined behavior, it doesn't make sense to detect undefined behavior
after it has happened.

Change-Id: I70a820d4903fc4845cdf742ecba7380e883597e8

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 71c7c45..a366390 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -672,7 +672,6 @@
     error += diff * diff;
     sqcoeff += (int64_t)coeff[i] * (int64_t)coeff[i];
   }
-  assert(error >= 0 && sqcoeff >= 0);
   error = (error + rounding) >> shift;
   sqcoeff = (sqcoeff + rounding) >> shift;
 

av1/encoder/tx_search.c

diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index 74414f0..334bce1 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -1065,7 +1065,6 @@
     sqcoeff += (cc * cc + (1 << (2 * AOM_QM_BITS - 1))) >> (2 * AOM_QM_BITS);
   }
 
-  assert(error >= 0 && sqcoeff >= 0);
   error = (error + rounding) >> shift;
   sqcoeff = (sqcoeff + rounding) >> shift;
 

av1/encoder/x86/highbd_block_error_intrin_avx2.c

diff --git a/av1/encoder/x86/highbd_block_error_intrin_avx2.c b/av1/encoder/x86/highbd_block_error_intrin_avx2.c
index fec6ea0..d329857 100644
--- a/av1/encoder/x86/highbd_block_error_intrin_avx2.c
+++ b/av1/encoder/x86/highbd_block_error_intrin_avx2.c
@@ -55,7 +55,6 @@
     error += temp1[0] + temp1[1] + temp1[2] + temp1[3];
     sqcoeff += temp1[4] + temp1[5] + temp1[6] + temp1[7];
   }
-  assert(error >= 0 && sqcoeff >= 0);
   error = (error + rounding) >> shift;
   sqcoeff = (sqcoeff + rounding) >> shift;
 

av1/encoder/x86/highbd_block_error_intrin_sse2.c

diff --git a/av1/encoder/x86/highbd_block_error_intrin_sse2.c b/av1/encoder/x86/highbd_block_error_intrin_sse2.c
index 3d843f9..c86cadd 100644
--- a/av1/encoder/x86/highbd_block_error_intrin_sse2.c
+++ b/av1/encoder/x86/highbd_block_error_intrin_sse2.c
@@ -65,7 +65,6 @@
       }
     }
   }
-  assert(error >= 0 && sqcoeff >= 0);
   error = (error + rounding) >> shift;
   sqcoeff = (sqcoeff + rounding) >> shift;