commit f34c1565c4092f428f971cbc402a1b2ad977f54e
author Wan-Teh Chang <wtc@google.com> Wed Apr 12 14:31:05 2023 -0700
committer Wan-Teh Chang <wtc@google.com> Wed Apr 12 16:04:00 2023 -0700
tree a82c60e618865479af421a0e03a88ad23c41a203
parent 901c53bbabfb5d26aa372a1dfc1aa899a85ae264

codec_aom.c: Update min/max quantizers correctly

cfg->rc_min_quantizer and cfg->rc_max_quantizer should be updated based
on encoder->minQuantizer(Alpha) and encoder->maxQuantizer(Alpha) rather
than cfg->rc_min_quantizer and cfg->rc_max_quantizer, because the
`quantizer` parameter and encoder->minQuantizer(Alpha) and
encoder->maxQuantizer(Alpha) are updated together.

src/codec_aom.c

diff --git a/src/codec_aom.c b/src/codec_aom.c
index 5ef1ce5..8b5019d 100644
--- a/src/codec_aom.c
+++ b/src/codec_aom.c
@@ -870,8 +870,19 @@
                     cfg->rc_min_quantizer = AVIF_QUANTIZER_LOSSLESS;
                     cfg->rc_max_quantizer = AVIF_QUANTIZER_LOSSLESS;
                 } else {
-                    cfg->rc_min_quantizer = AVIF_MAX(quantizer - 4, (int)cfg->rc_min_quantizer);
-                    cfg->rc_max_quantizer = AVIF_MIN(quantizer + 4, (int)cfg->rc_max_quantizer);
+                    int minQuantizer;
+                    int maxQuantizer;
+                    if (alpha) {
+                        minQuantizer = encoder->minQuantizerAlpha;
+                        maxQuantizer = encoder->maxQuantizerAlpha;
+                    } else {
+                        minQuantizer = encoder->minQuantizer;
+                        maxQuantizer = encoder->maxQuantizer;
+                    }
+                    minQuantizer = AVIF_CLAMP(minQuantizer, 0, 63);
+                    maxQuantizer = AVIF_CLAMP(maxQuantizer, 0, 63);
+                    cfg->rc_min_quantizer = AVIF_MAX(quantizer - 4, minQuantizer);
+                    cfg->rc_max_quantizer = AVIF_MIN(quantizer + 4, maxQuantizer);
                 }
                 quantizerUpdated = AVIF_TRUE;
             }