diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index f04cd0f..93a7066 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -952,12 +952,16 @@
   assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE);
 
   if (use_planewise_strategy) {  // Commonly used for high-resolution video.
-    // TODO(any): avx2 and sse2 version should also support high bit-depth, and
-    // they should be changed to consider cross-plane information (see C
-    // function) before using.
-    av1_apply_temporal_filter_planewise_c(frame_to_filter, mbd, block_size,
+    // TODO(any): avx2 and sse2 version should also support high bit-depth.
+    if (is_frame_high_bitdepth(frame_to_filter)) {
+      av1_apply_temporal_filter_planewise_c(frame_to_filter, mbd, block_size,
+                                            mb_row, mb_col, num_planes,
+                                            noise_levels, pred, accum, count);
+    } else {
+      av1_apply_temporal_filter_planewise(frame_to_filter, mbd, block_size,
                                           mb_row, mb_col, num_planes,
                                           noise_levels, pred, accum, count);
+    }
   } else {  // Commonly used for low-resolution video.
     const int adj_strength = strength + 2 * (mbd->bd - 8);
     if (num_planes == 1) {
diff --git a/av1/encoder/x86/temporal_filter_avx2.c b/av1/encoder/x86/temporal_filter_avx2.c
index 93d9186..88a74a2 100644
--- a/av1/encoder/x86/temporal_filter_avx2.c
+++ b/av1/encoder/x86/temporal_filter_avx2.c
@@ -19,10 +19,10 @@
 #define SSE_STRIDE (BW + 2)
 
 DECLARE_ALIGNED(32, static const uint32_t, sse_bytemask[4][8]) = {
-  { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000 },
-  { 0x0000, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000, 0x0000 },
-  { 0x0000, 0x0000, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0x0000 },
-  { 0x0000, 0x0000, 0x0000, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF }
+  { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0, 0, 0 },
+  { 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0, 0 },
+  { 0, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0 },
+  { 0, 0, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }
 };
 
 DECLARE_ALIGNED(32, static const uint8_t, shufflemask_16b[2][16]) = {
@@ -104,7 +104,7 @@
     // For the last column, replicate the last element twice to the right
     v128tmp = _mm_shuffle_epi8(v128tmp, *(__m128i *)shufflemask_16b[1]);
   }
-  return _mm256_cvtepi16_epi32(v128tmp);
+  return _mm256_cvtepu16_epi32(v128tmp);
 }
 
 static AOM_FORCE_INLINE int32_t xx_mask_and_hadd(__m256i vsum, int i) {
@@ -131,16 +131,17 @@
     const uint8_t *frame1, const unsigned int stride, const uint8_t *frame2,
     const unsigned int stride2, const int block_width, const int block_height,
     const double sigma, const int decay_control, unsigned int *accumulator,
-    uint16_t *count) {
-  const double h = decay_control * (0.7 + log(sigma + 1.0));
-  const double beta = 1.0;
-
-  uint16_t frame_sse[SSE_STRIDE * BH];
-  uint32_t acc_5x5_sse[BH][BW];
-
+    uint16_t *count, uint16_t *luma_sq_error, uint16_t *chroma_sq_error,
+    int plane, int ss_x_shift, int ss_y_shift) {
   assert(TF_PLANEWISE_FILTER_WINDOW_LENGTH == 5);
   assert(((block_width == 32) && (block_height == 32)) ||
          ((block_width == 16) && (block_height == 16)));
+  if (plane > PLANE_TYPE_Y) assert(chroma_sq_error != NULL);
+
+  uint32_t acc_5x5_sse[BH][BW];
+  const double h = decay_control * (0.7 + log(sigma + 1.0));
+  uint16_t *frame_sse =
+      (plane == PLANE_TYPE_Y) ? luma_sq_error : chroma_sq_error;
 
   if (block_width == 32) {
     get_squared_error_32x32_avx2(frame1, stride, frame2, stride2, block_width,
@@ -200,17 +201,29 @@
       const int pixel_value = frame2[i * stride2 + j];
 
       int diff_sse = acc_5x5_sse[i][j];
-      diff_sse /= (TF_PLANEWISE_FILTER_WINDOW_LENGTH *
-                   TF_PLANEWISE_FILTER_WINDOW_LENGTH);
+      int num_ref_pixels =
+          TF_PLANEWISE_FILTER_WINDOW_LENGTH * TF_PLANEWISE_FILTER_WINDOW_LENGTH;
 
-      double scaled_diff = -diff_sse / (2 * beta * h * h);
-      // clamp the value to avoid underflow in exp()
-      if (scaled_diff < -15) scaled_diff = -15;
-      double w = exp(scaled_diff);
-      const int weight = (int)(w * TF_PLANEWISE_FILTER_WEIGHT_SCALE);
+      // Filter U-plane and V-plane using Y-plane. This is because motion
+      // search is only done on Y-plane, so the information from Y-plane will
+      // be more accurate.
+      if (plane != PLANE_TYPE_Y) {
+        for (int ii = 0; ii < (1 << ss_y_shift); ++ii) {
+          for (int jj = 0; jj < (1 << ss_x_shift); ++jj) {
+            const int yy = (i << ss_y_shift) + ii;  // Y-coord on Y-plane.
+            const int xx = (j << ss_x_shift) + jj;  // X-coord on Y-plane.
+            diff_sse += luma_sq_error[yy * SSE_STRIDE + xx];
+            ++num_ref_pixels;
+          }
+        }
+      }
+      const double scaled_diff =
+          AOMMAX(-(double)(diff_sse / num_ref_pixels) / (2 * h * h), -15.0);
+      const int adjusted_weight =
+          (int)(exp(scaled_diff) * TF_PLANEWISE_FILTER_WEIGHT_SCALE);
 
-      count[k] += weight;
-      accumulator[k] += weight * pixel_value;
+      count[k] += adjusted_weight;
+      accumulator[k] += adjusted_weight * pixel_value;
     }
   }
 }
@@ -232,6 +245,12 @@
   const int mb_height = block_size_high[block_size];
   const int mb_width = block_size_wide[block_size];
   const int mb_pels = mb_height * mb_width;
+  uint16_t luma_sq_error[SSE_STRIDE * BH];
+  uint16_t *chroma_sq_error =
+      (num_planes > 0)
+          ? (uint16_t *)aom_malloc(SSE_STRIDE * BH * sizeof(uint16_t))
+          : NULL;
+
   for (int plane = 0; plane < num_planes; ++plane) {
     const uint32_t plane_h = mb_height >> mbd->plane[plane].subsampling_y;
     const uint32_t plane_w = mb_width >> mbd->plane[plane].subsampling_x;
@@ -239,9 +258,16 @@
     const int frame_offset = mb_row * plane_h * frame_stride + mb_col * plane_w;
 
     const uint8_t *ref = ref_frame->buffers[plane] + frame_offset;
+    const int ss_x_shift =
+        mbd->plane[plane].subsampling_x - mbd->plane[0].subsampling_x;
+    const int ss_y_shift =
+        mbd->plane[plane].subsampling_y - mbd->plane[0].subsampling_y;
+
     apply_temporal_filter_planewise(
         ref, frame_stride, pred + mb_pels * plane, plane_w, plane_w, plane_h,
         noise_levels[plane], decay_control, accum + mb_pels * plane,
-        count + mb_pels * plane);
+        count + mb_pels * plane, luma_sq_error, chroma_sq_error, plane,
+        ss_x_shift, ss_y_shift);
   }
+  if (chroma_sq_error != NULL) aom_free(chroma_sq_error);
 }
diff --git a/av1/encoder/x86/temporal_filter_sse2.c b/av1/encoder/x86/temporal_filter_sse2.c
index 0cfa841..5b85dca 100644
--- a/av1/encoder/x86/temporal_filter_sse2.c
+++ b/av1/encoder/x86/temporal_filter_sse2.c
@@ -20,10 +20,14 @@
 #define SSE_STRIDE (BW + 4)
 
 DECLARE_ALIGNED(32, static const uint32_t, sse_bytemask_2x4[4][2][4]) = {
-  { { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF }, { 0xFFFF, 0x0000, 0x0000, 0x0000 } },
-  { { 0x0000, 0xFFFF, 0xFFFF, 0xFFFF }, { 0xFFFF, 0xFFFF, 0x0000, 0x0000 } },
-  { { 0x0000, 0x0000, 0xFFFF, 0xFFFF }, { 0xFFFF, 0xFFFF, 0xFFFF, 0x0000 } },
-  { { 0x0000, 0x0000, 0x0000, 0xFFFF }, { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF } }
+  { { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF },
+    { 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000 } },
+  { { 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF },
+    { 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 } },
+  { { 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF },
+    { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 } },
+  { { 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF },
+    { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF } }
 };
 
 static void get_squared_error(const uint8_t *frame1, const unsigned int stride,
@@ -102,16 +106,17 @@
     const uint8_t *frame1, const unsigned int stride, const uint8_t *frame2,
     const unsigned int stride2, const int block_width, const int block_height,
     const double sigma, const int decay_control, unsigned int *accumulator,
-    uint16_t *count) {
-  const double h = decay_control * (0.7 + log(sigma + 1.0));
-  const double beta = 1.0;
-
-  uint16_t frame_sse[SSE_STRIDE * BH];
-  uint32_t acc_5x5_sse[BH][BW];
-
+    uint16_t *count, uint16_t *luma_sq_error, uint16_t *chroma_sq_error,
+    int plane, int ss_x_shift, int ss_y_shift) {
   assert(TF_PLANEWISE_FILTER_WINDOW_LENGTH == 5);
   assert(((block_width == 32) && (block_height == 32)) ||
          ((block_width == 16) && (block_height == 16)));
+  if (plane > PLANE_TYPE_Y) assert(chroma_sq_error != NULL);
+
+  uint32_t acc_5x5_sse[BH][BW];
+  const double h = decay_control * (0.7 + log(sigma + 1.0));
+  uint16_t *frame_sse =
+      (plane == PLANE_TYPE_Y) ? luma_sq_error : chroma_sq_error;
 
   get_squared_error(frame1, stride, frame2, stride2, block_width, block_height,
                     frame_sse, SSE_STRIDE);
@@ -173,17 +178,31 @@
       const int pixel_value = frame2[i * stride2 + j];
 
       int diff_sse = acc_5x5_sse[i][j];
-      diff_sse /= (TF_PLANEWISE_FILTER_WINDOW_LENGTH *
-                   TF_PLANEWISE_FILTER_WINDOW_LENGTH);
+      int num_ref_pixels =
+          TF_PLANEWISE_FILTER_WINDOW_LENGTH * TF_PLANEWISE_FILTER_WINDOW_LENGTH;
 
-      double scaled_diff = -diff_sse / (2 * beta * h * h);
-      // clamp the value to avoid underflow in exp()
-      if (scaled_diff < -15) scaled_diff = -15;
-      double w = exp(scaled_diff);
-      const int weight = (int)(w * TF_PLANEWISE_FILTER_WEIGHT_SCALE);
+      // Filter U-plane and V-plane using Y-plane. This is because motion
+      // search is only done on Y-plane, so the information from Y-plane will
+      // be more accurate.
+      if (plane != PLANE_TYPE_Y) {
+        for (int ii = 0; ii < (1 << ss_y_shift); ++ii) {
+          for (int jj = 0; jj < (1 << ss_x_shift); ++jj) {
+            const int yy = (i << ss_y_shift) + ii;      // Y-coord on Y-plane.
+            const int xx = (j << ss_x_shift) + jj + 2;  // X-coord on Y-plane.
+            const int ww = SSE_STRIDE;                  // Stride of Y-plane.
+            diff_sse += luma_sq_error[yy * ww + xx];
+            ++num_ref_pixels;
+          }
+        }
+      }
 
-      count[k] += weight;
-      accumulator[k] += weight * pixel_value;
+      const double scaled_diff =
+          AOMMAX(-(double)(diff_sse / num_ref_pixels) / (2 * h * h), -15.0);
+      const int adjusted_weight =
+          (int)(exp(scaled_diff) * TF_PLANEWISE_FILTER_WEIGHT_SCALE);
+
+      count[k] += adjusted_weight;
+      accumulator[k] += adjusted_weight * pixel_value;
     }
   }
 }
@@ -205,6 +224,12 @@
   const int mb_height = block_size_high[block_size];
   const int mb_width = block_size_wide[block_size];
   const int mb_pels = mb_height * mb_width;
+  uint16_t luma_sq_error[SSE_STRIDE * BH];
+  uint16_t *chroma_sq_error =
+      (num_planes > 0)
+          ? (uint16_t *)aom_malloc(SSE_STRIDE * BH * sizeof(uint16_t))
+          : NULL;
+
   for (int plane = 0; plane < num_planes; ++plane) {
     const uint32_t plane_h = mb_height >> mbd->plane[plane].subsampling_y;
     const uint32_t plane_w = mb_width >> mbd->plane[plane].subsampling_x;
@@ -212,9 +237,16 @@
     const int frame_offset = mb_row * plane_h * frame_stride + mb_col * plane_w;
 
     const uint8_t *ref = ref_frame->buffers[plane] + frame_offset;
+    const int ss_x_shift =
+        mbd->plane[plane].subsampling_x - mbd->plane[0].subsampling_x;
+    const int ss_y_shift =
+        mbd->plane[plane].subsampling_y - mbd->plane[0].subsampling_y;
+
     apply_temporal_filter_planewise(
         ref, frame_stride, pred + mb_pels * plane, plane_w, plane_w, plane_h,
         noise_levels[plane], decay_control, accum + mb_pels * plane,
-        count + mb_pels * plane);
+        count + mb_pels * plane, luma_sq_error, chroma_sq_error, plane,
+        ss_x_shift, ss_y_shift);
   }
+  if (chroma_sq_error != NULL) aom_free(chroma_sq_error);
 }