Properly scale the distortion in high bit-depth settings

Scale the high bit-depth distortion to align with the baseline
rate-distortion optimization usage.

BUG=b/435430898

Change-Id: I7c5dc2636086cb4c9001786f77e35f82ace77c58

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 5f0edb1..b0ea574 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -632,11 +632,15 @@
   BLOCK_SIZE bsize = mbmi->bsize;
   int bw = block_size_wide[bsize];
   int bh = block_size_high[bsize];
+  const int shift = 2 * (xd->bd - 8);
 
   *rec_var = aom_highbd_calc_variance_stat(CONVERT_TO_SHORTPTR(pd->dst.buf),
                                            pd->dst.stride, bw, bh);
   *src_var = aom_highbd_calc_variance_stat(CONVERT_TO_SHORTPTR(p->src.buf),
                                            p->src.stride, bw, bh);
+
+  *rec_var = ROUND_POWER_OF_TWO(*rec_var, shift);
+  *src_var = ROUND_POWER_OF_TWO(*src_var, shift);
 }
 #endif  // CONFIG_AV1_HIGHBITDEPTH