commit 2a7bd71fbc24c5f94443d476e058b1c1cad7b6be
author Debargha Mukherjee <debargha@google.com> Wed May 21 16:57:28 2025 +0000
committer Urvang Joshi <urvang@google.com> Wed May 21 16:57:28 2025 +0000
tree 3f0929b3752171c9ef242c39b6ca32619bbb5bd9
parent 7ee731b0f3cc923fa95406eba2be3dcd85ef5c16

F054 fix to setup av1_setup_pred_block() correctly

Fixes #697.

With the integration of F054, a discrepancy arises when fetching reference samples outside the picture boundary. Specifically, the reference samples used during encoder RD (via `av1_setup_pred_block()`) differ from those used during final encoding (via `av1_setup_pre_planes()`). This mismatch is due to a missing update from F054 in `av1_setup_pred_block()`.

As a result, in certain cases—such as when lossless coding is enabled—the encoder sets skip_txfm=1 based on the RD path. However, during final encoding, the prediction differs while the residual is treated as zero, leading to a reconstructed block that is no longer lossless.

This MR aligns the prediction behavior between the RD path and final encoding, ensuring consistency and avoiding incorrect skip_txfm settings.

av1/encoder/rd.c

diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 5862661..ba045f1 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -2415,11 +2415,18 @@
   const int mi_row = xd->mi_row;
   const int mi_col = xd->mi_col;
   for (int i = 0; i < num_planes; ++i) {
+#if CONFIG_F054_PIC_BOUNDARY
+    setup_pred_plane(dst + i, dst[i].buf, i ? src->uv_width : src->y_width,
+                     i ? src->uv_height : src->y_height, dst[i].stride, mi_row,
+                     mi_col, i ? scale_uv : scale, xd->plane[i].subsampling_x,
+                     xd->plane[i].subsampling_y, &xd->mi[0]->chroma_ref_info);
+#else
     setup_pred_plane(
         dst + i, dst[i].buf, i ? src->uv_crop_width : src->y_crop_width,
         i ? src->uv_crop_height : src->y_crop_height, dst[i].stride, mi_row,
         mi_col, i ? scale_uv : scale, xd->plane[i].subsampling_x,
         xd->plane[i].subsampling_y, &xd->mi[0]->chroma_ref_info);
+#endif  // CONFIG_F054_PIC_BOUNDARY
   }
 }
 

av1/encoder/reconinter_enc.c

diff --git a/av1/encoder/reconinter_enc.c b/av1/encoder/reconinter_enc.c
index 11778fe..15930d6 100644
--- a/av1/encoder/reconinter_enc.c
+++ b/av1/encoder/reconinter_enc.c
@@ -68,8 +68,10 @@
       orig_pos_y += src_mv->row * (1 << (1 - ssy));
       orig_pos_x += src_mv->col * (1 << (1 - ssx));
     }
-    int pos_y = sf->scale_value_y(orig_pos_y, sf);
-    int pos_x = sf->scale_value_x(orig_pos_x, sf);
+    int pos_y =
+        sf->scale_value_y ? sf->scale_value_y(orig_pos_y, sf) : orig_pos_y;
+    int pos_x =
+        sf->scale_value_x ? sf->scale_value_x(orig_pos_x, sf) : orig_pos_x;
     pos_x += SCALE_EXTRA_OFF;
     pos_y += SCALE_EXTRA_OFF;