Add multi-thread support for loop-filtering

Row based multi-threading of loop-filter is added.

Change-Id: Ib37bd3d63532f196cb780f6cfaf73ae6803ab245
diff --git a/aom_util/aom_thread.h b/aom_util/aom_thread.h
index 0ae8f2f..e22c4cc 100644
--- a/aom_util/aom_thread.h
+++ b/aom_util/aom_thread.h
@@ -173,6 +173,23 @@
   return !ok;
 }
 
+static INLINE int pthread_cond_broadcast(pthread_cond_t *const condition) {
+  int ok = 1;
+#ifdef USE_WINDOWS_CONDITION_VARIABLE
+  WakeAllConditionVariable(condition);
+#else
+  while (WaitForSingleObject(condition->waiting_sem_, 0) == WAIT_OBJECT_0) {
+    // a thread is waiting in pthread_cond_wait: allow it to be notified
+    ok &= SetEvent(condition->signal_event_);
+    // wait until the event is consumed so the signaler cannot consume
+    // the event via its own pthread_cond_wait.
+    ok &= (WaitForSingleObject(condition->received_sem_, INFINITE) !=
+           WAIT_OBJECT_0);
+  }
+#endif
+  return !ok;
+}
+
 static INLINE int pthread_cond_wait(pthread_cond_t *const condition,
                                     pthread_mutex_t *const mutex) {
   int ok;
diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c
index 1a35305..1e285bc 100644
--- a/av1/common/av1_loopfilter.c
+++ b/av1/common/av1_loopfilter.c
@@ -274,8 +274,8 @@
   0x00000000ffffffffULL,  // BLOCK_64X16
 };
 
-static LoopFilterMask *get_loop_filter_mask(AV1_COMMON *const cm, int mi_row,
-                                            int mi_col) {
+LoopFilterMask *av1_get_loop_filter_mask(AV1_COMMON *const cm, int mi_row,
+                                         int mi_col) {
   assert(cm->lf.lfm != NULL);
   const int sb_row = mi_row >> MAX_MIB_SIZE_LOG2;
   const int sb_col = mi_col >> MAX_MIB_SIZE_LOG2;
@@ -364,8 +364,8 @@
 // Update the loop filter for the current frame.
 // This should be called before loop_filter_rows(),
 // av1_loop_filter_frame() calls this function directly.
-static void loop_filter_frame_init(AV1_COMMON *cm, int plane_start,
-                                   int plane_end) {
+void av1_loop_filter_frame_init(AV1_COMMON *cm, int plane_start,
+                                int plane_end) {
   int filt_lvl[MAX_MB_PLANE], filt_lvl_r[MAX_MB_PLANE];
   int plane;
   int seg_id;
@@ -1043,12 +1043,10 @@
   return ts;
 }
 
-static void filter_block_plane_vert(const AV1_COMMON *const cm,
-                                    const MACROBLOCKD *const xd,
-                                    const int plane,
-                                    const MACROBLOCKD_PLANE *const plane_ptr,
-                                    const uint32_t mi_row,
-                                    const uint32_t mi_col) {
+void av1_filter_block_plane_vert(const AV1_COMMON *const cm,
+                                 const MACROBLOCKD *const xd, const int plane,
+                                 const MACROBLOCKD_PLANE *const plane_ptr,
+                                 const uint32_t mi_row, const uint32_t mi_col) {
   const int row_step = MI_SIZE >> MI_SIZE_LOG2;
   const uint32_t scale_horz = plane_ptr->subsampling_x;
   const uint32_t scale_vert = plane_ptr->subsampling_y;
@@ -1130,12 +1128,10 @@
   }
 }
 
-static void filter_block_plane_horz(const AV1_COMMON *const cm,
-                                    const MACROBLOCKD *const xd,
-                                    const int plane,
-                                    const MACROBLOCKD_PLANE *const plane_ptr,
-                                    const uint32_t mi_row,
-                                    const uint32_t mi_col) {
+void av1_filter_block_plane_horz(const AV1_COMMON *const cm,
+                                 const MACROBLOCKD *const xd, const int plane,
+                                 const MACROBLOCKD_PLANE *const plane_ptr,
+                                 const uint32_t mi_row, const uint32_t mi_col) {
   const int col_step = MI_SIZE >> MI_SIZE_LOG2;
   const uint32_t scale_horz = plane_ptr->subsampling_x;
   const uint32_t scale_vert = plane_ptr->subsampling_y;
@@ -1220,7 +1216,7 @@
 }
 
 #if LOOP_FILTER_BITMASK
-static INLINE enum lf_path get_loop_filter_path(
+INLINE enum lf_path av1_get_loop_filter_path(
     int plane, struct macroblockd_plane pd[MAX_MB_PLANE]) {
   if (pd[plane].subsampling_y == 1 && pd[plane].subsampling_x == 1)
     return LF_PATH_420;
@@ -1230,11 +1226,10 @@
     return LF_PATH_SLOW;
 }
 
-static void loop_filter_block_plane_vert(AV1_COMMON *const cm,
-                                         struct macroblockd_plane *pd, int pl,
-                                         int mi_row, int mi_col,
-                                         enum lf_path path,
-                                         LoopFilterMask *lf_mask) {
+void av1_loop_filter_block_plane_vert(AV1_COMMON *const cm,
+                                      struct macroblockd_plane *pd, int pl,
+                                      int mi_row, int mi_col, enum lf_path path,
+                                      LoopFilterMask *lf_mask) {
   MB_MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride + mi_col;
   switch (path) {
     case LF_PATH_420:
@@ -1249,11 +1244,10 @@
   }
 }
 
-static void loop_filter_block_plane_horz(AV1_COMMON *const cm,
-                                         struct macroblockd_plane *pd, int pl,
-                                         int mi_row, int mi_col,
-                                         enum lf_path path,
-                                         LoopFilterMask *lf_mask) {
+void av1_loop_filter_block_plane_horz(AV1_COMMON *const cm,
+                                      struct macroblockd_plane *pd, int pl,
+                                      int mi_row, int mi_col, enum lf_path path,
+                                      LoopFilterMask *lf_mask) {
   MB_MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride + mi_col;
   switch (path) {
     case LF_PATH_420:
@@ -1287,7 +1281,7 @@
       continue;
 
 #if LOOP_FILTER_BITMASK
-    enum lf_path path = get_loop_filter_path(plane, pd);
+    enum lf_path path = av1_get_loop_filter_path(plane, pd);
 
     if (cm->lf.combine_vert_horz_lf) {
       // filter all vertical and horizontal edges in every super block
@@ -1297,7 +1291,8 @@
           av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                                mi_col, plane, plane + 1);
 
-          LoopFilterMask *lf_mask = get_loop_filter_mask(cm, mi_row, mi_col);
+          LoopFilterMask *lf_mask =
+              av1_get_loop_filter_mask(cm, mi_row, mi_col);
           av1_setup_bitmask(cm, mi_row, mi_col, plane, pd[plane].subsampling_x,
                             pd[plane].subsampling_y, lf_mask);
           loop_filter_block_plane_vert(cm, pd, plane, mi_row, mi_col, path,
@@ -1310,7 +1305,7 @@
                                  plane + 1);
 
             LoopFilterMask *lf_mask =
-                get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE);
+                av1_get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE);
             loop_filter_block_plane_horz(cm, pd, plane, mi_row,
                                          mi_col - MAX_MIB_SIZE, path, lf_mask);
           }
@@ -1320,7 +1315,7 @@
                              mi_col - MAX_MIB_SIZE, plane, plane + 1);
 
         LoopFilterMask *lf_mask =
-            get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE);
+            av1_get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE);
         loop_filter_block_plane_horz(cm, pd, plane, mi_row,
                                      mi_col - MAX_MIB_SIZE, path, lf_mask);
       }
@@ -1331,11 +1326,12 @@
           av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                                mi_col, plane, plane + 1);
 
-          LoopFilterMask *lf_mask = get_loop_filter_mask(cm, mi_row, mi_col);
+          LoopFilterMask *lf_mask =
+              av1_get_loop_filter_mask(cm, mi_row, mi_col);
           av1_setup_bitmask(cm, mi_row, mi_col, plane, pd[plane].subsampling_x,
                             pd[plane].subsampling_y, lf_mask);
-          loop_filter_block_plane_vert(cm, pd, plane, mi_row, mi_col, path,
-                                       lf_mask);
+          av1_loop_filter_block_plane_vert(cm, pd, plane, mi_row, mi_col, path,
+                                           lf_mask);
         }
       }
 
@@ -1345,7 +1341,8 @@
           av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                                mi_col, plane, plane + 1);
 
-          LoopFilterMask *lf_mask = get_loop_filter_mask(cm, mi_row, mi_col);
+          LoopFilterMask *lf_mask =
+              av1_get_loop_filter_mask(cm, mi_row, mi_col);
           loop_filter_block_plane_horz(cm, pd, plane, mi_row, mi_col, path,
                                        lf_mask);
         }
@@ -1359,21 +1356,22 @@
           // filter vertical edges
           av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                                mi_col, plane, plane + 1);
-          filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row, mi_col);
+          av1_filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row,
+                                      mi_col);
           // filter horizontal edges
           if (mi_col - MAX_MIB_SIZE >= 0) {
             av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer,
                                  mi_row, mi_col - MAX_MIB_SIZE, plane,
                                  plane + 1);
-            filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row,
-                                    mi_col - MAX_MIB_SIZE);
+            av1_filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row,
+                                        mi_col - MAX_MIB_SIZE);
           }
         }
         // filter horizontal edges
         av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                              mi_col - MAX_MIB_SIZE, plane, plane + 1);
-        filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row,
-                                mi_col - MAX_MIB_SIZE);
+        av1_filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row,
+                                    mi_col - MAX_MIB_SIZE);
       }
     } else {
       // filter all vertical edges in every 128x128 super block
@@ -1381,7 +1379,8 @@
         for (mi_col = col_start; mi_col < col_end; mi_col += MAX_MIB_SIZE) {
           av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                                mi_col, plane, plane + 1);
-          filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row, mi_col);
+          av1_filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row,
+                                      mi_col);
         }
       }
 
@@ -1390,7 +1389,8 @@
         for (mi_col = col_start; mi_col < col_end; mi_col += MAX_MIB_SIZE) {
           av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row,
                                mi_col, plane, plane + 1);
-          filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, mi_col);
+          av1_filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row,
+                                      mi_col);
         }
       }
     }
@@ -1411,7 +1411,7 @@
     mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8);
   }
   end_mi_row = start_mi_row + mi_rows_to_filter;
-  loop_filter_frame_init(cm, plane_start, plane_end);
+  av1_loop_filter_frame_init(cm, plane_start, plane_end);
   loop_filter_rows(frame, cm, xd, start_mi_row, end_mi_row, plane_start,
                    plane_end);
 }
diff --git a/av1/common/av1_loopfilter.h b/av1/common/av1_loopfilter.h
index 712a3da..f39d5b8 100644
--- a/av1/common/av1_loopfilter.h
+++ b/av1/common/av1_loopfilter.h
@@ -161,20 +161,54 @@
 
 void av1_loop_filter_init(struct AV1Common *cm);
 
+void av1_loop_filter_frame_init(struct AV1Common *cm, int plane_start,
+                                int plane_end);
+
 void av1_loop_filter_frame(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm,
                            struct macroblockd *mbd, int plane_start,
                            int plane_end, int partial_frame);
 
+void av1_filter_block_plane_vert(const struct AV1Common *const cm,
+                                 const MACROBLOCKD *const xd, const int plane,
+                                 const MACROBLOCKD_PLANE *const plane_ptr,
+                                 const uint32_t mi_row, const uint32_t mi_col);
+
+void av1_filter_block_plane_horz(const struct AV1Common *const cm,
+                                 const MACROBLOCKD *const xd, const int plane,
+                                 const MACROBLOCKD_PLANE *const plane_ptr,
+                                 const uint32_t mi_row, const uint32_t mi_col);
+
 typedef struct LoopFilterWorkerData {
   YV12_BUFFER_CONFIG *frame_buffer;
   struct AV1Common *cm;
   struct macroblockd_plane planes[MAX_MB_PLANE];
-
-  int start;
-  int stop;
-  int y_only;
+  // TODO(Ranjit): When the filter functions are modified to use xd->lossless
+  // add lossless as a member here.
+  MACROBLOCKD *xd;
 } LFWorkerData;
 
+#if LOOP_FILTER_BITMASK
+INLINE enum lf_path av1_get_loop_filter_path(
+    int plane, struct macroblockd_plane pd[MAX_MB_PLANE]);
+
+LoopFilterMask *av1_get_loop_filter_mask(struct AV1Common *const cm, int mi_row,
+                                         int mi_col);
+
+void av1_setup_bitmask(struct AV1Common *const cm, int mi_row, int mi_col,
+                       int plane, int subsampling_x, int subsampling_y,
+                       LoopFilterMask *lfm);
+
+void av1_loop_filter_block_plane_vert(struct AV1Common *const cm,
+                                      struct macroblockd_plane *pd, int pl,
+                                      int mi_row, int mi_col, enum lf_path path,
+                                      LoopFilterMask *lf_mask);
+
+void av1_loop_filter_block_plane_horz(struct AV1Common *const cm,
+                                      struct macroblockd_plane *pd, int pl,
+                                      int mi_row, int mi_col, enum lf_path path,
+                                      LoopFilterMask *lf_mask);
+#endif
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/av1/common/thread_common.c b/av1/common/thread_common.c
index b816156..d3ce1d4 100644
--- a/av1/common/thread_common.c
+++ b/av1/common/thread_common.c
@@ -12,37 +12,378 @@
 #include "./aom_config.h"
 #include "aom_dsp/aom_dsp_common.h"
 #include "aom_mem/aom_mem.h"
+#include "av1/common/av1_loopfilter.h"
 #include "av1/common/entropymode.h"
 #include "av1/common/thread_common.h"
 #include "av1/common/reconinter.h"
 
+// Set up nsync by width.
+static INLINE int get_sync_range(int width) {
+  // nsync numbers are picked by testing. For example, for 4k
+  // video, using 4 gives best performance.
+  if (width < 640)
+    return 1;
+  else if (width <= 1280)
+    return 2;
+  else if (width <= 4096)
+    return 4;
+  else
+    return 8;
+}
+
+// Allocate memory for lf row synchronization
+static void loop_filter_alloc(AV1LfSync *lf_sync, AV1_COMMON *cm, int rows,
+                              int width, int num_workers) {
+  lf_sync->rows = rows;
+#if CONFIG_MULTITHREAD
+  {
+    int i, j;
+
+    for (j = 0; j < MAX_MB_PLANE; j++) {
+      CHECK_MEM_ERROR(cm, lf_sync->mutex_[j],
+                      aom_malloc(sizeof(*(lf_sync->mutex_[j])) * rows));
+      if (lf_sync->mutex_[j]) {
+        for (i = 0; i < rows; ++i) {
+          pthread_mutex_init(&lf_sync->mutex_[j][i], NULL);
+        }
+      }
+
+      CHECK_MEM_ERROR(cm, lf_sync->cond_[j],
+                      aom_malloc(sizeof(*(lf_sync->cond_[j])) * rows));
+      if (lf_sync->cond_[j]) {
+        for (i = 0; i < rows; ++i) {
+          pthread_cond_init(&lf_sync->cond_[j][i], NULL);
+        }
+      }
+    }
+
+    CHECK_MEM_ERROR(cm, lf_sync->job_mutex,
+                    aom_malloc(sizeof(*(lf_sync->job_mutex))));
+    if (lf_sync->job_mutex) {
+      pthread_mutex_init(lf_sync->job_mutex, NULL);
+    }
+  }
+#endif  // CONFIG_MULTITHREAD
+  CHECK_MEM_ERROR(cm, lf_sync->lfdata,
+                  aom_malloc(num_workers * sizeof(*(lf_sync->lfdata))));
+  lf_sync->num_workers = num_workers;
+
+  for (int j = 0; j < MAX_MB_PLANE; j++) {
+    CHECK_MEM_ERROR(cm, lf_sync->cur_sb_col[j],
+                    aom_malloc(sizeof(*(lf_sync->cur_sb_col[j])) * rows));
+  }
+  CHECK_MEM_ERROR(
+      cm, lf_sync->job_queue,
+      aom_malloc(sizeof(*(lf_sync->job_queue)) * rows * MAX_MB_PLANE * 2));
+  // Set up nsync.
+  lf_sync->sync_range = get_sync_range(width);
+}
+
 // Deallocate lf synchronization related mutex and data
 void av1_loop_filter_dealloc(AV1LfSync *lf_sync) {
   if (lf_sync != NULL) {
+    int j;
 #if CONFIG_MULTITHREAD
     int i;
-
-    if (lf_sync->mutex_ != NULL) {
-      for (i = 0; i < lf_sync->rows; ++i) {
-        pthread_mutex_destroy(&lf_sync->mutex_[i]);
+    for (j = 0; j < MAX_MB_PLANE; j++) {
+      if (lf_sync->mutex_[j] != NULL) {
+        for (i = 0; i < lf_sync->rows; ++i) {
+          pthread_mutex_destroy(&lf_sync->mutex_[j][i]);
+        }
+        aom_free(lf_sync->mutex_[j]);
       }
-      aom_free(lf_sync->mutex_);
+      if (lf_sync->cond_[j] != NULL) {
+        for (i = 0; i < lf_sync->rows; ++i) {
+          pthread_cond_destroy(&lf_sync->cond_[j][i]);
+        }
+        aom_free(lf_sync->cond_[j]);
+      }
     }
-    if (lf_sync->cond_ != NULL) {
-      for (i = 0; i < lf_sync->rows; ++i) {
-        pthread_cond_destroy(&lf_sync->cond_[i]);
-      }
-      aom_free(lf_sync->cond_);
+    if (lf_sync->job_mutex != NULL) {
+      pthread_mutex_destroy(lf_sync->job_mutex);
+      aom_free(lf_sync->job_mutex);
     }
 #endif  // CONFIG_MULTITHREAD
     aom_free(lf_sync->lfdata);
-    aom_free(lf_sync->cur_sb_col);
+    for (j = 0; j < MAX_MB_PLANE; j++) {
+      aom_free(lf_sync->cur_sb_col[j]);
+    }
+
+    aom_free(lf_sync->job_queue);
     // clear the structure as the source of this call may be a resize in which
     // case this call will be followed by an _alloc() which may fail.
     av1_zero(*lf_sync);
   }
 }
 
+static void loop_filter_data_reset(LFWorkerData *lf_data,
+                                   YV12_BUFFER_CONFIG *frame_buffer,
+                                   struct AV1Common *cm, MACROBLOCKD *xd) {
+  struct macroblockd_plane *pd = xd->plane;
+  lf_data->frame_buffer = frame_buffer;
+  lf_data->cm = cm;
+  lf_data->xd = xd;
+  for (int i = 0; i < MAX_MB_PLANE; i++) {
+    memcpy(&lf_data->planes[i].dst, &pd[i].dst, sizeof(lf_data->planes[i].dst));
+    lf_data->planes[i].subsampling_x = pd[i].subsampling_x;
+    lf_data->planes[i].subsampling_y = pd[i].subsampling_y;
+  }
+}
+
+static INLINE void sync_read(AV1LfSync *const lf_sync, int r, int c,
+                             int plane) {
+#if CONFIG_MULTITHREAD
+  const int nsync = lf_sync->sync_range;
+
+  if (r && !(c & (nsync - 1))) {
+    pthread_mutex_t *const mutex = &lf_sync->mutex_[plane][r - 1];
+    pthread_mutex_lock(mutex);
+
+    while (c > lf_sync->cur_sb_col[plane][r - 1] - nsync) {
+      pthread_cond_wait(&lf_sync->cond_[plane][r - 1], mutex);
+    }
+    pthread_mutex_unlock(mutex);
+  }
+#else
+  (void)lf_sync;
+  (void)r;
+  (void)c;
+  (void)plane;
+#endif  // CONFIG_MULTITHREAD
+}
+
+static INLINE void sync_write(AV1LfSync *const lf_sync, int r, int c,
+                              const int sb_cols, int plane) {
+#if CONFIG_MULTITHREAD
+  const int nsync = lf_sync->sync_range;
+  int cur;
+  // Only signal when there are enough filtered SB for next row to run.
+  int sig = 1;
+
+  if (c < sb_cols - 1) {
+    cur = c;
+    if (c % nsync) sig = 0;
+  } else {
+    cur = sb_cols + nsync;
+  }
+
+  if (sig) {
+    pthread_mutex_lock(&lf_sync->mutex_[plane][r]);
+
+    lf_sync->cur_sb_col[plane][r] = cur;
+
+    pthread_cond_broadcast(&lf_sync->cond_[plane][r]);
+    pthread_mutex_unlock(&lf_sync->mutex_[plane][r]);
+  }
+#else
+  (void)lf_sync;
+  (void)r;
+  (void)c;
+  (void)sb_cols;
+  (void)plane;
+#endif  // CONFIG_MULTITHREAD
+}
+
+static void enqueue_lf_jobs(AV1LfSync *lf_sync, AV1_COMMON *cm, int start,
+                            int stop, int plane_start, int plane_end) {
+  int mi_row, plane, dir;
+  AV1LfMTInfo *lf_job_queue = lf_sync->job_queue;
+  lf_sync->jobs_enqueued = 0;
+  lf_sync->jobs_dequeued = 0;
+
+  for (dir = 0; dir < 2; dir++) {
+    for (plane = plane_start; plane < plane_end; plane++) {
+      if (plane == 0 && !(cm->lf.filter_level[0]) && !(cm->lf.filter_level[1]))
+        break;
+      else if (plane == 1 && !(cm->lf.filter_level_u))
+        continue;
+      else if (plane == 2 && !(cm->lf.filter_level_v))
+        continue;
+      for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) {
+        lf_job_queue->mi_row = mi_row;
+        lf_job_queue->plane = plane;
+        lf_job_queue->dir = dir;
+        lf_job_queue++;
+        lf_sync->jobs_enqueued++;
+      }
+    }
+  }
+}
+
+AV1LfMTInfo *get_lf_job_info(AV1LfSync *lf_sync) {
+  AV1LfMTInfo *cur_job_info = NULL;
+
+#if CONFIG_MULTITHREAD
+  pthread_mutex_lock(lf_sync->job_mutex);
+
+  if (lf_sync->jobs_dequeued < lf_sync->jobs_enqueued) {
+    cur_job_info = lf_sync->job_queue + lf_sync->jobs_dequeued;
+    lf_sync->jobs_dequeued++;
+  }
+
+  pthread_mutex_unlock(lf_sync->job_mutex);
+#else
+  (void)lf_sync;
+#endif
+
+  return cur_job_info;
+}
+
+// Implement row loopfiltering for each thread.
+static INLINE void thread_loop_filter_rows(
+    const YV12_BUFFER_CONFIG *const frame_buffer, AV1_COMMON *const cm,
+    struct macroblockd_plane *planes, MACROBLOCKD *xd,
+    AV1LfSync *const lf_sync) {
+  const int sb_cols =
+      ALIGN_POWER_OF_TWO(cm->mi_cols, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2;
+  int mi_row, mi_col, plane, dir;
+  int r, c;
+
+  while (1) {
+    AV1LfMTInfo *cur_job_info = get_lf_job_info(lf_sync);
+
+    if (cur_job_info != NULL) {
+      mi_row = cur_job_info->mi_row;
+      plane = cur_job_info->plane;
+      dir = cur_job_info->dir;
+      r = mi_row >> MAX_MIB_SIZE_LOG2;
+
+#if LOOP_FILTER_BITMASK
+      enum lf_path path = av1_get_loop_filter_path(plane, planes);
+#endif
+
+      if (dir == 0) {
+        for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) {
+          c = mi_col >> MAX_MIB_SIZE_LOG2;
+
+          av1_setup_dst_planes(planes, cm->seq_params.sb_size, frame_buffer,
+                               mi_row, mi_col, plane, plane + 1);
+
+#if LOOP_FILTER_BITMASK
+          LoopFilterMask *lf_mask =
+              av1_get_loop_filter_mask(cm, mi_row, mi_col);
+          av1_setup_bitmask(cm, mi_row, mi_col, plane,
+                            planes[plane].subsampling_x,
+                            planes[plane].subsampling_y, lf_mask);
+          av1_loop_filter_block_plane_vert(cm, planes, plane, mi_row, mi_col,
+                                           path, lf_mask);
+#else
+          av1_filter_block_plane_vert(cm, xd, plane, &planes[plane], mi_row,
+                                      mi_col);
+#endif
+          sync_write(lf_sync, r, c, sb_cols, plane);
+        }
+      } else if (dir == 1) {
+        for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) {
+          c = mi_col >> MAX_MIB_SIZE_LOG2;
+
+          // Wait for vertical edge filtering of the top-right block to be
+          // completed
+          sync_read(lf_sync, r, c, plane);
+
+          // Wait for vertical edge filtering of the right block to be
+          // completed
+          sync_read(lf_sync, r + 1, c, plane);
+
+          av1_setup_dst_planes(planes, cm->seq_params.sb_size, frame_buffer,
+                               mi_row, mi_col, plane, plane + 1);
+#if LOOP_FILTER_BITMASK
+          LoopFilterMask *lf_mask =
+              av1_get_loop_filter_mask(cm, mi_row, mi_col);
+          av1_loop_filter_block_plane_horz(cm, planes, plane, mi_row, mi_col,
+                                           path, lf_mask);
+#else
+          av1_filter_block_plane_horz(cm, xd, plane, &planes[plane], mi_row,
+                                      mi_col);
+#endif
+        }
+      }
+    } else {
+      break;
+    }
+  }
+}
+
+// Row-based multi-threaded loopfilter hook
+static int loop_filter_row_worker(AV1LfSync *const lf_sync,
+                                  LFWorkerData *const lf_data) {
+  thread_loop_filter_rows(lf_data->frame_buffer, lf_data->cm, lf_data->planes,
+                          lf_data->xd, lf_sync);
+  return 1;
+}
+
+static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
+                                MACROBLOCKD *xd, int start, int stop,
+                                int plane_start, int plane_end,
+                                AVxWorker *workers, int nworkers,
+                                AV1LfSync *lf_sync) {
+  const AVxWorkerInterface *const winterface = aom_get_worker_interface();
+  // Number of superblock rows and cols
+  const int sb_rows =
+      ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2;
+  const int num_workers = nworkers;
+  int i;
+
+  if (!lf_sync->sync_range || sb_rows != lf_sync->rows ||
+      num_workers > lf_sync->num_workers) {
+    av1_loop_filter_dealloc(lf_sync);
+    loop_filter_alloc(lf_sync, cm, sb_rows, cm->width, num_workers);
+  }
+
+  // Initialize cur_sb_col to -1 for all SB rows.
+  for (i = 0; i < MAX_MB_PLANE; i++) {
+    memset(lf_sync->cur_sb_col[i], -1,
+           sizeof(*(lf_sync->cur_sb_col[i])) * sb_rows);
+  }
+
+  enqueue_lf_jobs(lf_sync, cm, start, stop, plane_start, plane_end);
+
+  // Set up loopfilter thread data.
+  for (i = 0; i < num_workers; ++i) {
+    AVxWorker *const worker = &workers[i];
+    LFWorkerData *const lf_data = &lf_sync->lfdata[i];
+
+    worker->hook = (AVxWorkerHook)loop_filter_row_worker;
+    worker->data1 = lf_sync;
+    worker->data2 = lf_data;
+
+    // Loopfilter data
+    loop_filter_data_reset(lf_data, frame, cm, xd);
+
+    // Start loopfiltering
+    if (i == num_workers - 1) {
+      winterface->execute(worker);
+    } else {
+      winterface->launch(worker);
+    }
+  }
+
+  // Wait till all rows are finished
+  for (i = 0; i < num_workers; ++i) {
+    winterface->sync(&workers[i]);
+  }
+}
+
+void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
+                              MACROBLOCKD *xd, int plane_start, int plane_end,
+                              int partial_frame, AVxWorker *workers,
+                              int num_workers, AV1LfSync *lf_sync) {
+  int start_mi_row, end_mi_row, mi_rows_to_filter;
+
+  start_mi_row = 0;
+  mi_rows_to_filter = cm->mi_rows;
+  if (partial_frame && cm->mi_rows > 8) {
+    start_mi_row = cm->mi_rows >> 1;
+    start_mi_row &= 0xfffffff8;
+    mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8);
+  }
+  end_mi_row = start_mi_row + mi_rows_to_filter;
+  av1_loop_filter_frame_init(cm, plane_start, plane_end);
+
+  loop_filter_rows_mt(frame, cm, xd, start_mi_row, end_mi_row, plane_start,
+                      plane_end, workers, num_workers, lf_sync);
+}
+
 // Accumulate frame counts. FRAME_COUNTS consist solely of 'unsigned int'
 // members, so we treat it as an array, and sum over the whole length.
 void av1_accumulate_frame_counts(FRAME_COUNTS *acc_counts,
diff --git a/av1/common/thread_common.h b/av1/common/thread_common.h
index c1cbd41..e5011c6 100644
--- a/av1/common/thread_common.h
+++ b/av1/common/thread_common.h
@@ -22,14 +22,20 @@
 struct AV1Common;
 struct FRAME_COUNTS;
 
+typedef struct AV1LfMTInfo {
+  int mi_row;
+  int plane;
+  int dir;
+} AV1LfMTInfo;
+
 // Loopfilter row synchronization
 typedef struct AV1LfSyncData {
 #if CONFIG_MULTITHREAD
-  pthread_mutex_t *mutex_;
-  pthread_cond_t *cond_;
+  pthread_mutex_t *mutex_[MAX_MB_PLANE];
+  pthread_cond_t *cond_[MAX_MB_PLANE];
 #endif
   // Allocate memory to store the loop-filtered superblock index in each row.
-  int *cur_sb_col;
+  int *cur_sb_col[MAX_MB_PLANE];
   // The optimal sync_range for different resolution and platform should be
   // determined by testing. Currently, it is chosen to be a power-of-2 number.
   int sync_range;
@@ -38,11 +44,24 @@
   // Row-based parallel loopfilter data
   LFWorkerData *lfdata;
   int num_workers;
+
+#if CONFIG_MULTITHREAD
+  pthread_mutex_t *job_mutex;
+#endif
+  AV1LfMTInfo *job_queue;
+  int jobs_enqueued;
+  int jobs_dequeued;
 } AV1LfSync;
 
 // Deallocate loopfilter synchronization related mutex and data.
 void av1_loop_filter_dealloc(AV1LfSync *lf_sync);
 
+void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm,
+                              struct macroblockd *mbd, int plane_start,
+                              int plane_end, int partial_frame,
+                              AVxWorker *workers, int num_workers,
+                              AV1LfSync *lf_sync);
+
 void av1_accumulate_frame_counts(struct FRAME_COUNTS *acc_counts,
                                  struct FRAME_COUNTS *counts);
 
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 2544ec2..93b62c53 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -4000,8 +4000,14 @@
 
   if (!cm->allow_intrabc) {
     if (cm->lf.filter_level[0] || cm->lf.filter_level[1]) {
-      av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, 0,
-                            num_planes, 0);
+      if (pbi->num_workers > 1) {
+        av1_loop_filter_frame_mt(get_frame_new_buffer(cm), cm, &pbi->mb, 0,
+                                 num_planes, 0, pbi->tile_workers,
+                                 pbi->num_workers, &pbi->lf_row_sync);
+      } else {
+        av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, 0,
+                              num_planes, 0);
+      }
     }
 
     const int do_loop_restoration =
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c
index cd792a7..33bfb8c 100644
--- a/av1/decoder/decoder.c
+++ b/av1/decoder/decoder.c
@@ -154,6 +154,10 @@
   aom_free(pbi->tile_data);
   aom_free(pbi->tile_workers);
 
+  if (pbi->num_workers > 0) {
+    av1_loop_filter_dealloc(&pbi->lf_row_sync);
+  }
+
 #if CONFIG_ACCOUNTING
   aom_accounting_clear(&pbi->accounting);
 #endif
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h
index 858c998..def7b1e 100644
--- a/av1/decoder/decoder.h
+++ b/av1/decoder/decoder.h
@@ -70,6 +70,7 @@
 
   AVxWorker *frame_worker_owner;  // frame_worker that owns this pbi.
   AVxWorker lf_worker;
+  AV1LfSync lf_row_sync;
   AVxWorker *tile_workers;
   int num_workers;
   DecWorkerData *thread_data;
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index a557380..304091c 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2907,6 +2907,10 @@
   aom_free(cpi->tile_thr_data);
   aom_free(cpi->workers);
 
+  if (cpi->num_workers > 1) {
+    av1_loop_filter_dealloc(&cpi->lf_row_sync);
+  }
+
   dealloc_compressor_data(cpi);
 
   for (i = 0; i < sizeof(cpi->mbgraph_stats) / sizeof(cpi->mbgraph_stats[0]);
@@ -4088,7 +4092,12 @@
   }
 
   if (lf->filter_level[0] || lf->filter_level[1]) {
-    av1_loop_filter_frame(cm->frame_to_show, cm, xd, 0, num_planes, 0);
+    if (cpi->num_workers > 1)
+      av1_loop_filter_frame_mt(cm->frame_to_show, cm, xd, 0, num_planes, 0,
+                               cpi->workers, cpi->num_workers,
+                               &cpi->lf_row_sync);
+    else
+      av1_loop_filter_frame(cm->frame_to_show, cm, xd, 0, num_planes, 0);
   }
 
   if (!no_restoration)
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index e64cb15..031168a 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -604,6 +604,8 @@
   //   A mapping of each reference frame from its encoder side value to the
   //   decoder side value obtained following the short signaling procedure.
   int ref_conv[REF_FRAMES];
+
+  AV1LfSync lf_row_sync;
 } AV1_COMP;
 
 void av1_initialize_enc(void);
diff --git a/av1/encoder/picklpf.c b/av1/encoder/picklpf.c
index 72d4f9b..d790f0b 100644
--- a/av1/encoder/picklpf.c
+++ b/av1/encoder/picklpf.c
@@ -67,8 +67,13 @@
     case 2: cm->lf.filter_level_v = filter_level[0]; break;
   }
 
-  av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane,
-                        plane + 1, partial_frame);
+  if (cpi->num_workers > 1)
+    av1_loop_filter_frame_mt(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane,
+                             plane + 1, partial_frame, cpi->workers,
+                             cpi->num_workers, &cpi->lf_row_sync);
+  else
+    av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane,
+                          plane + 1, partial_frame);
 
   int highbd = 0;
   highbd = cm->use_highbitdepth;