commit 22ea12992ec1ffe5d89d0ee6ae8d63ed30d18d14
author Jingning Han <jingning@google.com> Tue Jan 12 13:42:37 2021 -0800
committer Jingning Han <jingning@google.com> Wed Jan 13 07:50:57 2021 +0000
tree 2aba33ad4598fb73b48f31946ce87f40314df381
parent 7b0628b217d571603f705e0c06bf6887f04be25a

Fix tpl seg fault in YUV422 mode

Properly select the scan order and address pointer in 422 mode.

Change-Id: I177feb5ac7b95b965380abf9ba4bba0b8acfcd55

av1/encoder/tpl_model.c

diff --git a/av1/encoder/tpl_model.c b/av1/encoder/tpl_model.c
index f2fbda2..c980905 100644
--- a/av1/encoder/tpl_model.c
+++ b/av1/encoder/tpl_model.c
@@ -43,7 +43,7 @@
                                           int64_t *sse) {
   const struct macroblock_plane *const p = &x->plane[plane];
   const MACROBLOCKD *xd = &x->e_mbd;
-  const SCAN_ORDER *const scan_order = &av1_default_scan_orders[tx_size];
+  const SCAN_ORDER *const scan_order = &av1_scan_orders[tx_size][DCT_DCT];
   int pix_num = 1 << num_pels_log2_lookup[txsize_to_bsize[tx_size]];
   const int shift = tx_size == TX_32X32 ? 0 : 2;
 
@@ -104,7 +104,7 @@
 }
 
 static int rate_estimator(const tran_low_t *qcoeff, int eob, TX_SIZE tx_size) {
-  const SCAN_ORDER *const scan_order = &av1_default_scan_orders[tx_size];
+  const SCAN_ORDER *const scan_order = &av1_scan_orders[tx_size][DCT_DCT];
 
   assert((1 << num_pels_log2_lookup[txsize_to_bsize[tx_size]]) >= eob);
   aom_clear_system_state();
@@ -258,8 +258,8 @@
 
     int dst_buffer_stride = rec_stride_pool[plane];
     int dst_mb_offset =
-        (mi_row * MI_SIZE * dst_buffer_stride + mi_col * MI_SIZE) >>
-        pd->subsampling_x;
+        ((mi_row * MI_SIZE * dst_buffer_stride) >> pd->subsampling_y) +
+        ((mi_col * MI_SIZE) >> pd->subsampling_x);
     uint8_t *dst_buffer = rec_buffer_pool[plane] + dst_mb_offset;
     for (int ref = 0; ref < 1 + is_compound; ++ref) {
       if (!is_inter_mode(best_mode)) {
@@ -299,8 +299,8 @@
     }
 
     int src_stride = src_stride_pool[plane];
-    int src_mb_offset =
-        (mi_row * MI_SIZE * src_stride + mi_col * MI_SIZE) >> pd->subsampling_x;
+    int src_mb_offset = ((mi_row * MI_SIZE * src_stride) >> pd->subsampling_y) +
+                        ((mi_col * MI_SIZE) >> pd->subsampling_x);
 
     int this_rate = 1;
     int64_t this_recon_error = 1;