Add multi-thread support for loop-filtering Row based multi-threading of loop-filter is added. Change-Id: Ib37bd3d63532f196cb780f6cfaf73ae6803ab245
diff --git a/aom_util/aom_thread.h b/aom_util/aom_thread.h index 0ae8f2f..e22c4cc 100644 --- a/aom_util/aom_thread.h +++ b/aom_util/aom_thread.h
@@ -173,6 +173,23 @@ return !ok; } +static INLINE int pthread_cond_broadcast(pthread_cond_t *const condition) { + int ok = 1; +#ifdef USE_WINDOWS_CONDITION_VARIABLE + WakeAllConditionVariable(condition); +#else + while (WaitForSingleObject(condition->waiting_sem_, 0) == WAIT_OBJECT_0) { + // a thread is waiting in pthread_cond_wait: allow it to be notified + ok &= SetEvent(condition->signal_event_); + // wait until the event is consumed so the signaler cannot consume + // the event via its own pthread_cond_wait. + ok &= (WaitForSingleObject(condition->received_sem_, INFINITE) != + WAIT_OBJECT_0); + } +#endif + return !ok; +} + static INLINE int pthread_cond_wait(pthread_cond_t *const condition, pthread_mutex_t *const mutex) { int ok;
diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c index 1a35305..1e285bc 100644 --- a/av1/common/av1_loopfilter.c +++ b/av1/common/av1_loopfilter.c
@@ -274,8 +274,8 @@ 0x00000000ffffffffULL, // BLOCK_64X16 }; -static LoopFilterMask *get_loop_filter_mask(AV1_COMMON *const cm, int mi_row, - int mi_col) { +LoopFilterMask *av1_get_loop_filter_mask(AV1_COMMON *const cm, int mi_row, + int mi_col) { assert(cm->lf.lfm != NULL); const int sb_row = mi_row >> MAX_MIB_SIZE_LOG2; const int sb_col = mi_col >> MAX_MIB_SIZE_LOG2; @@ -364,8 +364,8 @@ // Update the loop filter for the current frame. // This should be called before loop_filter_rows(), // av1_loop_filter_frame() calls this function directly. -static void loop_filter_frame_init(AV1_COMMON *cm, int plane_start, - int plane_end) { +void av1_loop_filter_frame_init(AV1_COMMON *cm, int plane_start, + int plane_end) { int filt_lvl[MAX_MB_PLANE], filt_lvl_r[MAX_MB_PLANE]; int plane; int seg_id; @@ -1043,12 +1043,10 @@ return ts; } -static void filter_block_plane_vert(const AV1_COMMON *const cm, - const MACROBLOCKD *const xd, - const int plane, - const MACROBLOCKD_PLANE *const plane_ptr, - const uint32_t mi_row, - const uint32_t mi_col) { +void av1_filter_block_plane_vert(const AV1_COMMON *const cm, + const MACROBLOCKD *const xd, const int plane, + const MACROBLOCKD_PLANE *const plane_ptr, + const uint32_t mi_row, const uint32_t mi_col) { const int row_step = MI_SIZE >> MI_SIZE_LOG2; const uint32_t scale_horz = plane_ptr->subsampling_x; const uint32_t scale_vert = plane_ptr->subsampling_y; @@ -1130,12 +1128,10 @@ } } -static void filter_block_plane_horz(const AV1_COMMON *const cm, - const MACROBLOCKD *const xd, - const int plane, - const MACROBLOCKD_PLANE *const plane_ptr, - const uint32_t mi_row, - const uint32_t mi_col) { +void av1_filter_block_plane_horz(const AV1_COMMON *const cm, + const MACROBLOCKD *const xd, const int plane, + const MACROBLOCKD_PLANE *const plane_ptr, + const uint32_t mi_row, const uint32_t mi_col) { const int col_step = MI_SIZE >> MI_SIZE_LOG2; const uint32_t scale_horz = plane_ptr->subsampling_x; const uint32_t scale_vert = plane_ptr->subsampling_y; @@ -1220,7 +1216,7 @@ } #if LOOP_FILTER_BITMASK -static INLINE enum lf_path get_loop_filter_path( +INLINE enum lf_path av1_get_loop_filter_path( int plane, struct macroblockd_plane pd[MAX_MB_PLANE]) { if (pd[plane].subsampling_y == 1 && pd[plane].subsampling_x == 1) return LF_PATH_420; @@ -1230,11 +1226,10 @@ return LF_PATH_SLOW; } -static void loop_filter_block_plane_vert(AV1_COMMON *const cm, - struct macroblockd_plane *pd, int pl, - int mi_row, int mi_col, - enum lf_path path, - LoopFilterMask *lf_mask) { +void av1_loop_filter_block_plane_vert(AV1_COMMON *const cm, + struct macroblockd_plane *pd, int pl, + int mi_row, int mi_col, enum lf_path path, + LoopFilterMask *lf_mask) { MB_MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride + mi_col; switch (path) { case LF_PATH_420: @@ -1249,11 +1244,10 @@ } } -static void loop_filter_block_plane_horz(AV1_COMMON *const cm, - struct macroblockd_plane *pd, int pl, - int mi_row, int mi_col, - enum lf_path path, - LoopFilterMask *lf_mask) { +void av1_loop_filter_block_plane_horz(AV1_COMMON *const cm, + struct macroblockd_plane *pd, int pl, + int mi_row, int mi_col, enum lf_path path, + LoopFilterMask *lf_mask) { MB_MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride + mi_col; switch (path) { case LF_PATH_420: @@ -1287,7 +1281,7 @@ continue; #if LOOP_FILTER_BITMASK - enum lf_path path = get_loop_filter_path(plane, pd); + enum lf_path path = av1_get_loop_filter_path(plane, pd); if (cm->lf.combine_vert_horz_lf) { // filter all vertical and horizontal edges in every super block @@ -1297,7 +1291,8 @@ av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col, plane, plane + 1); - LoopFilterMask *lf_mask = get_loop_filter_mask(cm, mi_row, mi_col); + LoopFilterMask *lf_mask = + av1_get_loop_filter_mask(cm, mi_row, mi_col); av1_setup_bitmask(cm, mi_row, mi_col, plane, pd[plane].subsampling_x, pd[plane].subsampling_y, lf_mask); loop_filter_block_plane_vert(cm, pd, plane, mi_row, mi_col, path, @@ -1310,7 +1305,7 @@ plane + 1); LoopFilterMask *lf_mask = - get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE); + av1_get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE); loop_filter_block_plane_horz(cm, pd, plane, mi_row, mi_col - MAX_MIB_SIZE, path, lf_mask); } @@ -1320,7 +1315,7 @@ mi_col - MAX_MIB_SIZE, plane, plane + 1); LoopFilterMask *lf_mask = - get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE); + av1_get_loop_filter_mask(cm, mi_row, mi_col - MAX_MIB_SIZE); loop_filter_block_plane_horz(cm, pd, plane, mi_row, mi_col - MAX_MIB_SIZE, path, lf_mask); } @@ -1331,11 +1326,12 @@ av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col, plane, plane + 1); - LoopFilterMask *lf_mask = get_loop_filter_mask(cm, mi_row, mi_col); + LoopFilterMask *lf_mask = + av1_get_loop_filter_mask(cm, mi_row, mi_col); av1_setup_bitmask(cm, mi_row, mi_col, plane, pd[plane].subsampling_x, pd[plane].subsampling_y, lf_mask); - loop_filter_block_plane_vert(cm, pd, plane, mi_row, mi_col, path, - lf_mask); + av1_loop_filter_block_plane_vert(cm, pd, plane, mi_row, mi_col, path, + lf_mask); } } @@ -1345,7 +1341,8 @@ av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col, plane, plane + 1); - LoopFilterMask *lf_mask = get_loop_filter_mask(cm, mi_row, mi_col); + LoopFilterMask *lf_mask = + av1_get_loop_filter_mask(cm, mi_row, mi_col); loop_filter_block_plane_horz(cm, pd, plane, mi_row, mi_col, path, lf_mask); } @@ -1359,21 +1356,22 @@ // filter vertical edges av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col, plane, plane + 1); - filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row, mi_col); + av1_filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row, + mi_col); // filter horizontal edges if (mi_col - MAX_MIB_SIZE >= 0) { av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col - MAX_MIB_SIZE, plane, plane + 1); - filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, - mi_col - MAX_MIB_SIZE); + av1_filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, + mi_col - MAX_MIB_SIZE); } } // filter horizontal edges av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col - MAX_MIB_SIZE, plane, plane + 1); - filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, - mi_col - MAX_MIB_SIZE); + av1_filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, + mi_col - MAX_MIB_SIZE); } } else { // filter all vertical edges in every 128x128 super block @@ -1381,7 +1379,8 @@ for (mi_col = col_start; mi_col < col_end; mi_col += MAX_MIB_SIZE) { av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col, plane, plane + 1); - filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row, mi_col); + av1_filter_block_plane_vert(cm, xd, plane, &pd[plane], mi_row, + mi_col); } } @@ -1390,7 +1389,8 @@ for (mi_col = col_start; mi_col < col_end; mi_col += MAX_MIB_SIZE) { av1_setup_dst_planes(pd, cm->seq_params.sb_size, frame_buffer, mi_row, mi_col, plane, plane + 1); - filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, mi_col); + av1_filter_block_plane_horz(cm, xd, plane, &pd[plane], mi_row, + mi_col); } } } @@ -1411,7 +1411,7 @@ mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8); } end_mi_row = start_mi_row + mi_rows_to_filter; - loop_filter_frame_init(cm, plane_start, plane_end); + av1_loop_filter_frame_init(cm, plane_start, plane_end); loop_filter_rows(frame, cm, xd, start_mi_row, end_mi_row, plane_start, plane_end); }
diff --git a/av1/common/av1_loopfilter.h b/av1/common/av1_loopfilter.h index 712a3da..f39d5b8 100644 --- a/av1/common/av1_loopfilter.h +++ b/av1/common/av1_loopfilter.h
@@ -161,20 +161,54 @@ void av1_loop_filter_init(struct AV1Common *cm); +void av1_loop_filter_frame_init(struct AV1Common *cm, int plane_start, + int plane_end); + void av1_loop_filter_frame(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm, struct macroblockd *mbd, int plane_start, int plane_end, int partial_frame); +void av1_filter_block_plane_vert(const struct AV1Common *const cm, + const MACROBLOCKD *const xd, const int plane, + const MACROBLOCKD_PLANE *const plane_ptr, + const uint32_t mi_row, const uint32_t mi_col); + +void av1_filter_block_plane_horz(const struct AV1Common *const cm, + const MACROBLOCKD *const xd, const int plane, + const MACROBLOCKD_PLANE *const plane_ptr, + const uint32_t mi_row, const uint32_t mi_col); + typedef struct LoopFilterWorkerData { YV12_BUFFER_CONFIG *frame_buffer; struct AV1Common *cm; struct macroblockd_plane planes[MAX_MB_PLANE]; - - int start; - int stop; - int y_only; + // TODO(Ranjit): When the filter functions are modified to use xd->lossless + // add lossless as a member here. + MACROBLOCKD *xd; } LFWorkerData; +#if LOOP_FILTER_BITMASK +INLINE enum lf_path av1_get_loop_filter_path( + int plane, struct macroblockd_plane pd[MAX_MB_PLANE]); + +LoopFilterMask *av1_get_loop_filter_mask(struct AV1Common *const cm, int mi_row, + int mi_col); + +void av1_setup_bitmask(struct AV1Common *const cm, int mi_row, int mi_col, + int plane, int subsampling_x, int subsampling_y, + LoopFilterMask *lfm); + +void av1_loop_filter_block_plane_vert(struct AV1Common *const cm, + struct macroblockd_plane *pd, int pl, + int mi_row, int mi_col, enum lf_path path, + LoopFilterMask *lf_mask); + +void av1_loop_filter_block_plane_horz(struct AV1Common *const cm, + struct macroblockd_plane *pd, int pl, + int mi_row, int mi_col, enum lf_path path, + LoopFilterMask *lf_mask); +#endif + #ifdef __cplusplus } // extern "C" #endif
diff --git a/av1/common/thread_common.c b/av1/common/thread_common.c index b816156..d3ce1d4 100644 --- a/av1/common/thread_common.c +++ b/av1/common/thread_common.c
@@ -12,37 +12,378 @@ #include "./aom_config.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_mem/aom_mem.h" +#include "av1/common/av1_loopfilter.h" #include "av1/common/entropymode.h" #include "av1/common/thread_common.h" #include "av1/common/reconinter.h" +// Set up nsync by width. +static INLINE int get_sync_range(int width) { + // nsync numbers are picked by testing. For example, for 4k + // video, using 4 gives best performance. + if (width < 640) + return 1; + else if (width <= 1280) + return 2; + else if (width <= 4096) + return 4; + else + return 8; +} + +// Allocate memory for lf row synchronization +static void loop_filter_alloc(AV1LfSync *lf_sync, AV1_COMMON *cm, int rows, + int width, int num_workers) { + lf_sync->rows = rows; +#if CONFIG_MULTITHREAD + { + int i, j; + + for (j = 0; j < MAX_MB_PLANE; j++) { + CHECK_MEM_ERROR(cm, lf_sync->mutex_[j], + aom_malloc(sizeof(*(lf_sync->mutex_[j])) * rows)); + if (lf_sync->mutex_[j]) { + for (i = 0; i < rows; ++i) { + pthread_mutex_init(&lf_sync->mutex_[j][i], NULL); + } + } + + CHECK_MEM_ERROR(cm, lf_sync->cond_[j], + aom_malloc(sizeof(*(lf_sync->cond_[j])) * rows)); + if (lf_sync->cond_[j]) { + for (i = 0; i < rows; ++i) { + pthread_cond_init(&lf_sync->cond_[j][i], NULL); + } + } + } + + CHECK_MEM_ERROR(cm, lf_sync->job_mutex, + aom_malloc(sizeof(*(lf_sync->job_mutex)))); + if (lf_sync->job_mutex) { + pthread_mutex_init(lf_sync->job_mutex, NULL); + } + } +#endif // CONFIG_MULTITHREAD + CHECK_MEM_ERROR(cm, lf_sync->lfdata, + aom_malloc(num_workers * sizeof(*(lf_sync->lfdata)))); + lf_sync->num_workers = num_workers; + + for (int j = 0; j < MAX_MB_PLANE; j++) { + CHECK_MEM_ERROR(cm, lf_sync->cur_sb_col[j], + aom_malloc(sizeof(*(lf_sync->cur_sb_col[j])) * rows)); + } + CHECK_MEM_ERROR( + cm, lf_sync->job_queue, + aom_malloc(sizeof(*(lf_sync->job_queue)) * rows * MAX_MB_PLANE * 2)); + // Set up nsync. + lf_sync->sync_range = get_sync_range(width); +} + // Deallocate lf synchronization related mutex and data void av1_loop_filter_dealloc(AV1LfSync *lf_sync) { if (lf_sync != NULL) { + int j; #if CONFIG_MULTITHREAD int i; - - if (lf_sync->mutex_ != NULL) { - for (i = 0; i < lf_sync->rows; ++i) { - pthread_mutex_destroy(&lf_sync->mutex_[i]); + for (j = 0; j < MAX_MB_PLANE; j++) { + if (lf_sync->mutex_[j] != NULL) { + for (i = 0; i < lf_sync->rows; ++i) { + pthread_mutex_destroy(&lf_sync->mutex_[j][i]); + } + aom_free(lf_sync->mutex_[j]); } - aom_free(lf_sync->mutex_); + if (lf_sync->cond_[j] != NULL) { + for (i = 0; i < lf_sync->rows; ++i) { + pthread_cond_destroy(&lf_sync->cond_[j][i]); + } + aom_free(lf_sync->cond_[j]); + } } - if (lf_sync->cond_ != NULL) { - for (i = 0; i < lf_sync->rows; ++i) { - pthread_cond_destroy(&lf_sync->cond_[i]); - } - aom_free(lf_sync->cond_); + if (lf_sync->job_mutex != NULL) { + pthread_mutex_destroy(lf_sync->job_mutex); + aom_free(lf_sync->job_mutex); } #endif // CONFIG_MULTITHREAD aom_free(lf_sync->lfdata); - aom_free(lf_sync->cur_sb_col); + for (j = 0; j < MAX_MB_PLANE; j++) { + aom_free(lf_sync->cur_sb_col[j]); + } + + aom_free(lf_sync->job_queue); // clear the structure as the source of this call may be a resize in which // case this call will be followed by an _alloc() which may fail. av1_zero(*lf_sync); } } +static void loop_filter_data_reset(LFWorkerData *lf_data, + YV12_BUFFER_CONFIG *frame_buffer, + struct AV1Common *cm, MACROBLOCKD *xd) { + struct macroblockd_plane *pd = xd->plane; + lf_data->frame_buffer = frame_buffer; + lf_data->cm = cm; + lf_data->xd = xd; + for (int i = 0; i < MAX_MB_PLANE; i++) { + memcpy(&lf_data->planes[i].dst, &pd[i].dst, sizeof(lf_data->planes[i].dst)); + lf_data->planes[i].subsampling_x = pd[i].subsampling_x; + lf_data->planes[i].subsampling_y = pd[i].subsampling_y; + } +} + +static INLINE void sync_read(AV1LfSync *const lf_sync, int r, int c, + int plane) { +#if CONFIG_MULTITHREAD + const int nsync = lf_sync->sync_range; + + if (r && !(c & (nsync - 1))) { + pthread_mutex_t *const mutex = &lf_sync->mutex_[plane][r - 1]; + pthread_mutex_lock(mutex); + + while (c > lf_sync->cur_sb_col[plane][r - 1] - nsync) { + pthread_cond_wait(&lf_sync->cond_[plane][r - 1], mutex); + } + pthread_mutex_unlock(mutex); + } +#else + (void)lf_sync; + (void)r; + (void)c; + (void)plane; +#endif // CONFIG_MULTITHREAD +} + +static INLINE void sync_write(AV1LfSync *const lf_sync, int r, int c, + const int sb_cols, int plane) { +#if CONFIG_MULTITHREAD + const int nsync = lf_sync->sync_range; + int cur; + // Only signal when there are enough filtered SB for next row to run. + int sig = 1; + + if (c < sb_cols - 1) { + cur = c; + if (c % nsync) sig = 0; + } else { + cur = sb_cols + nsync; + } + + if (sig) { + pthread_mutex_lock(&lf_sync->mutex_[plane][r]); + + lf_sync->cur_sb_col[plane][r] = cur; + + pthread_cond_broadcast(&lf_sync->cond_[plane][r]); + pthread_mutex_unlock(&lf_sync->mutex_[plane][r]); + } +#else + (void)lf_sync; + (void)r; + (void)c; + (void)sb_cols; + (void)plane; +#endif // CONFIG_MULTITHREAD +} + +static void enqueue_lf_jobs(AV1LfSync *lf_sync, AV1_COMMON *cm, int start, + int stop, int plane_start, int plane_end) { + int mi_row, plane, dir; + AV1LfMTInfo *lf_job_queue = lf_sync->job_queue; + lf_sync->jobs_enqueued = 0; + lf_sync->jobs_dequeued = 0; + + for (dir = 0; dir < 2; dir++) { + for (plane = plane_start; plane < plane_end; plane++) { + if (plane == 0 && !(cm->lf.filter_level[0]) && !(cm->lf.filter_level[1])) + break; + else if (plane == 1 && !(cm->lf.filter_level_u)) + continue; + else if (plane == 2 && !(cm->lf.filter_level_v)) + continue; + for (mi_row = start; mi_row < stop; mi_row += MAX_MIB_SIZE) { + lf_job_queue->mi_row = mi_row; + lf_job_queue->plane = plane; + lf_job_queue->dir = dir; + lf_job_queue++; + lf_sync->jobs_enqueued++; + } + } + } +} + +AV1LfMTInfo *get_lf_job_info(AV1LfSync *lf_sync) { + AV1LfMTInfo *cur_job_info = NULL; + +#if CONFIG_MULTITHREAD + pthread_mutex_lock(lf_sync->job_mutex); + + if (lf_sync->jobs_dequeued < lf_sync->jobs_enqueued) { + cur_job_info = lf_sync->job_queue + lf_sync->jobs_dequeued; + lf_sync->jobs_dequeued++; + } + + pthread_mutex_unlock(lf_sync->job_mutex); +#else + (void)lf_sync; +#endif + + return cur_job_info; +} + +// Implement row loopfiltering for each thread. +static INLINE void thread_loop_filter_rows( + const YV12_BUFFER_CONFIG *const frame_buffer, AV1_COMMON *const cm, + struct macroblockd_plane *planes, MACROBLOCKD *xd, + AV1LfSync *const lf_sync) { + const int sb_cols = + ALIGN_POWER_OF_TWO(cm->mi_cols, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2; + int mi_row, mi_col, plane, dir; + int r, c; + + while (1) { + AV1LfMTInfo *cur_job_info = get_lf_job_info(lf_sync); + + if (cur_job_info != NULL) { + mi_row = cur_job_info->mi_row; + plane = cur_job_info->plane; + dir = cur_job_info->dir; + r = mi_row >> MAX_MIB_SIZE_LOG2; + +#if LOOP_FILTER_BITMASK + enum lf_path path = av1_get_loop_filter_path(plane, planes); +#endif + + if (dir == 0) { + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) { + c = mi_col >> MAX_MIB_SIZE_LOG2; + + av1_setup_dst_planes(planes, cm->seq_params.sb_size, frame_buffer, + mi_row, mi_col, plane, plane + 1); + +#if LOOP_FILTER_BITMASK + LoopFilterMask *lf_mask = + av1_get_loop_filter_mask(cm, mi_row, mi_col); + av1_setup_bitmask(cm, mi_row, mi_col, plane, + planes[plane].subsampling_x, + planes[plane].subsampling_y, lf_mask); + av1_loop_filter_block_plane_vert(cm, planes, plane, mi_row, mi_col, + path, lf_mask); +#else + av1_filter_block_plane_vert(cm, xd, plane, &planes[plane], mi_row, + mi_col); +#endif + sync_write(lf_sync, r, c, sb_cols, plane); + } + } else if (dir == 1) { + for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MAX_MIB_SIZE) { + c = mi_col >> MAX_MIB_SIZE_LOG2; + + // Wait for vertical edge filtering of the top-right block to be + // completed + sync_read(lf_sync, r, c, plane); + + // Wait for vertical edge filtering of the right block to be + // completed + sync_read(lf_sync, r + 1, c, plane); + + av1_setup_dst_planes(planes, cm->seq_params.sb_size, frame_buffer, + mi_row, mi_col, plane, plane + 1); +#if LOOP_FILTER_BITMASK + LoopFilterMask *lf_mask = + av1_get_loop_filter_mask(cm, mi_row, mi_col); + av1_loop_filter_block_plane_horz(cm, planes, plane, mi_row, mi_col, + path, lf_mask); +#else + av1_filter_block_plane_horz(cm, xd, plane, &planes[plane], mi_row, + mi_col); +#endif + } + } + } else { + break; + } + } +} + +// Row-based multi-threaded loopfilter hook +static int loop_filter_row_worker(AV1LfSync *const lf_sync, + LFWorkerData *const lf_data) { + thread_loop_filter_rows(lf_data->frame_buffer, lf_data->cm, lf_data->planes, + lf_data->xd, lf_sync); + return 1; +} + +static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + MACROBLOCKD *xd, int start, int stop, + int plane_start, int plane_end, + AVxWorker *workers, int nworkers, + AV1LfSync *lf_sync) { + const AVxWorkerInterface *const winterface = aom_get_worker_interface(); + // Number of superblock rows and cols + const int sb_rows = + ALIGN_POWER_OF_TWO(cm->mi_rows, MAX_MIB_SIZE_LOG2) >> MAX_MIB_SIZE_LOG2; + const int num_workers = nworkers; + int i; + + if (!lf_sync->sync_range || sb_rows != lf_sync->rows || + num_workers > lf_sync->num_workers) { + av1_loop_filter_dealloc(lf_sync); + loop_filter_alloc(lf_sync, cm, sb_rows, cm->width, num_workers); + } + + // Initialize cur_sb_col to -1 for all SB rows. + for (i = 0; i < MAX_MB_PLANE; i++) { + memset(lf_sync->cur_sb_col[i], -1, + sizeof(*(lf_sync->cur_sb_col[i])) * sb_rows); + } + + enqueue_lf_jobs(lf_sync, cm, start, stop, plane_start, plane_end); + + // Set up loopfilter thread data. + for (i = 0; i < num_workers; ++i) { + AVxWorker *const worker = &workers[i]; + LFWorkerData *const lf_data = &lf_sync->lfdata[i]; + + worker->hook = (AVxWorkerHook)loop_filter_row_worker; + worker->data1 = lf_sync; + worker->data2 = lf_data; + + // Loopfilter data + loop_filter_data_reset(lf_data, frame, cm, xd); + + // Start loopfiltering + if (i == num_workers - 1) { + winterface->execute(worker); + } else { + winterface->launch(worker); + } + } + + // Wait till all rows are finished + for (i = 0; i < num_workers; ++i) { + winterface->sync(&workers[i]); + } +} + +void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm, + MACROBLOCKD *xd, int plane_start, int plane_end, + int partial_frame, AVxWorker *workers, + int num_workers, AV1LfSync *lf_sync) { + int start_mi_row, end_mi_row, mi_rows_to_filter; + + start_mi_row = 0; + mi_rows_to_filter = cm->mi_rows; + if (partial_frame && cm->mi_rows > 8) { + start_mi_row = cm->mi_rows >> 1; + start_mi_row &= 0xfffffff8; + mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8); + } + end_mi_row = start_mi_row + mi_rows_to_filter; + av1_loop_filter_frame_init(cm, plane_start, plane_end); + + loop_filter_rows_mt(frame, cm, xd, start_mi_row, end_mi_row, plane_start, + plane_end, workers, num_workers, lf_sync); +} + // Accumulate frame counts. FRAME_COUNTS consist solely of 'unsigned int' // members, so we treat it as an array, and sum over the whole length. void av1_accumulate_frame_counts(FRAME_COUNTS *acc_counts,
diff --git a/av1/common/thread_common.h b/av1/common/thread_common.h index c1cbd41..e5011c6 100644 --- a/av1/common/thread_common.h +++ b/av1/common/thread_common.h
@@ -22,14 +22,20 @@ struct AV1Common; struct FRAME_COUNTS; +typedef struct AV1LfMTInfo { + int mi_row; + int plane; + int dir; +} AV1LfMTInfo; + // Loopfilter row synchronization typedef struct AV1LfSyncData { #if CONFIG_MULTITHREAD - pthread_mutex_t *mutex_; - pthread_cond_t *cond_; + pthread_mutex_t *mutex_[MAX_MB_PLANE]; + pthread_cond_t *cond_[MAX_MB_PLANE]; #endif // Allocate memory to store the loop-filtered superblock index in each row. - int *cur_sb_col; + int *cur_sb_col[MAX_MB_PLANE]; // The optimal sync_range for different resolution and platform should be // determined by testing. Currently, it is chosen to be a power-of-2 number. int sync_range; @@ -38,11 +44,24 @@ // Row-based parallel loopfilter data LFWorkerData *lfdata; int num_workers; + +#if CONFIG_MULTITHREAD + pthread_mutex_t *job_mutex; +#endif + AV1LfMTInfo *job_queue; + int jobs_enqueued; + int jobs_dequeued; } AV1LfSync; // Deallocate loopfilter synchronization related mutex and data. void av1_loop_filter_dealloc(AV1LfSync *lf_sync); +void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm, + struct macroblockd *mbd, int plane_start, + int plane_end, int partial_frame, + AVxWorker *workers, int num_workers, + AV1LfSync *lf_sync); + void av1_accumulate_frame_counts(struct FRAME_COUNTS *acc_counts, struct FRAME_COUNTS *counts);
diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 2544ec2..93b62c53 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c
@@ -4000,8 +4000,14 @@ if (!cm->allow_intrabc) { if (cm->lf.filter_level[0] || cm->lf.filter_level[1]) { - av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, 0, - num_planes, 0); + if (pbi->num_workers > 1) { + av1_loop_filter_frame_mt(get_frame_new_buffer(cm), cm, &pbi->mb, 0, + num_planes, 0, pbi->tile_workers, + pbi->num_workers, &pbi->lf_row_sync); + } else { + av1_loop_filter_frame(get_frame_new_buffer(cm), cm, &pbi->mb, 0, + num_planes, 0); + } } const int do_loop_restoration =
diff --git a/av1/decoder/decoder.c b/av1/decoder/decoder.c index cd792a7..33bfb8c 100644 --- a/av1/decoder/decoder.c +++ b/av1/decoder/decoder.c
@@ -154,6 +154,10 @@ aom_free(pbi->tile_data); aom_free(pbi->tile_workers); + if (pbi->num_workers > 0) { + av1_loop_filter_dealloc(&pbi->lf_row_sync); + } + #if CONFIG_ACCOUNTING aom_accounting_clear(&pbi->accounting); #endif
diff --git a/av1/decoder/decoder.h b/av1/decoder/decoder.h index 858c998..def7b1e 100644 --- a/av1/decoder/decoder.h +++ b/av1/decoder/decoder.h
@@ -70,6 +70,7 @@ AVxWorker *frame_worker_owner; // frame_worker that owns this pbi. AVxWorker lf_worker; + AV1LfSync lf_row_sync; AVxWorker *tile_workers; int num_workers; DecWorkerData *thread_data;
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c index a557380..304091c 100644 --- a/av1/encoder/encoder.c +++ b/av1/encoder/encoder.c
@@ -2907,6 +2907,10 @@ aom_free(cpi->tile_thr_data); aom_free(cpi->workers); + if (cpi->num_workers > 1) { + av1_loop_filter_dealloc(&cpi->lf_row_sync); + } + dealloc_compressor_data(cpi); for (i = 0; i < sizeof(cpi->mbgraph_stats) / sizeof(cpi->mbgraph_stats[0]); @@ -4088,7 +4092,12 @@ } if (lf->filter_level[0] || lf->filter_level[1]) { - av1_loop_filter_frame(cm->frame_to_show, cm, xd, 0, num_planes, 0); + if (cpi->num_workers > 1) + av1_loop_filter_frame_mt(cm->frame_to_show, cm, xd, 0, num_planes, 0, + cpi->workers, cpi->num_workers, + &cpi->lf_row_sync); + else + av1_loop_filter_frame(cm->frame_to_show, cm, xd, 0, num_planes, 0); } if (!no_restoration)
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h index e64cb15..031168a 100644 --- a/av1/encoder/encoder.h +++ b/av1/encoder/encoder.h
@@ -604,6 +604,8 @@ // A mapping of each reference frame from its encoder side value to the // decoder side value obtained following the short signaling procedure. int ref_conv[REF_FRAMES]; + + AV1LfSync lf_row_sync; } AV1_COMP; void av1_initialize_enc(void);
diff --git a/av1/encoder/picklpf.c b/av1/encoder/picklpf.c index 72d4f9b..d790f0b 100644 --- a/av1/encoder/picklpf.c +++ b/av1/encoder/picklpf.c
@@ -67,8 +67,13 @@ case 2: cm->lf.filter_level_v = filter_level[0]; break; } - av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane, - plane + 1, partial_frame); + if (cpi->num_workers > 1) + av1_loop_filter_frame_mt(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane, + plane + 1, partial_frame, cpi->workers, + cpi->num_workers, &cpi->lf_row_sync); + else + av1_loop_filter_frame(cm->frame_to_show, cm, &cpi->td.mb.e_mbd, plane, + plane + 1, partial_frame); int highbd = 0; highbd = cm->use_highbitdepth;