Merge "Use Interlocked calls in win32 once() implementation."
diff --git a/test/resize_test.cc b/test/resize_test.cc
index 16ca8c4..bc91fe2 100644
--- a/test/resize_test.cc
+++ b/test/resize_test.cc
@@ -407,7 +407,7 @@
// Disable dropped frames.
cfg_.rc_dropframe_thresh = 0;
// Starting bitrate low.
- cfg_.rc_target_bitrate = 90;
+ cfg_.rc_target_bitrate = 80;
ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
unsigned int last_w = cfg_.g_w;
@@ -431,10 +431,8 @@
}
}
- // Verify that we get at least 1 resize event in this test.
- // TODO(marpan): We should be able to force 2 resize events
- // with some change in encoder settings/bitrate.
- ASSERT_GE(resize_count, 1) << "Resizing should occur at least once.";
+ // Verify that we get 2 resize events in this test.
+ ASSERT_EQ(resize_count, 2) << "Resizing should occur twice.";
}
vpx_img_fmt_t CspForFrameNumber(int frame) {
diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc
index 5d59e83..d44a64a 100644
--- a/test/test_intra_pred_speed.cc
+++ b/test/test_intra_pred_speed.cc
@@ -191,9 +191,14 @@
INTRA_PRED_TEST(SSE, TestIntraPred4, vpx_dc_predictor_4x4_sse,
vpx_dc_left_predictor_4x4_sse, vpx_dc_top_predictor_4x4_sse,
vpx_dc_128_predictor_4x4_sse, vpx_v_predictor_4x4_sse, NULL,
- NULL, NULL, NULL, NULL, NULL, NULL, vpx_tm_predictor_4x4_sse)
+ NULL, NULL, NULL, NULL, NULL, NULL, NULL)
#endif // HAVE_SSE && CONFIG_USE_X86INC
+#if HAVE_SSE2 && CONFIG_USE_X86INC
+INTRA_PRED_TEST(SSE2, TestIntraPred4, NULL, NULL, NULL, NULL, NULL, NULL,
+ NULL, NULL, NULL, NULL, NULL, NULL, vpx_tm_predictor_4x4_sse2)
+#endif // HAVE_SSE2 && CONFIG_USE_X86INC
+
#if HAVE_SSSE3 && CONFIG_USE_X86INC
INTRA_PRED_TEST(SSSE3, TestIntraPred4, NULL, NULL, NULL, NULL, NULL,
vpx_h_predictor_4x4_ssse3, vpx_d45_predictor_4x4_ssse3, NULL,
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 8ba2110..d87d164 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -3017,9 +3017,9 @@
? (source_variance - recon_variance)
: (recon_variance - source_variance);
- var_error = (200 * source_variance * recon_variance) /
- ((source_variance * source_variance) +
- (recon_variance * recon_variance));
+ var_error = ((int64_t)200 * source_variance * recon_variance) /
+ (((int64_t)source_variance * source_variance) +
+ ((int64_t)recon_variance * recon_variance));
var_error = 100 - var_error;
}
diff --git a/vp9/common/vp9_rtcd_defs.pl b/vp9/common/vp9_rtcd_defs.pl
index f41ee09..890b638 100644
--- a/vp9/common/vp9_rtcd_defs.pl
+++ b/vp9/common/vp9_rtcd_defs.pl
@@ -314,9 +314,6 @@
add_proto qw/int vp9_diamond_search_sad/, "const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv";
specialize qw/vp9_diamond_search_sad avx/;
-add_proto qw/int vp9_full_range_search/, "const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv";
-specialize qw/vp9_full_range_search/;
-
add_proto qw/void vp9_temporal_filter_apply/, "uint8_t *frame1, unsigned int stride, uint8_t *frame2, unsigned int block_width, unsigned int block_height, int strength, int filter_weight, unsigned int *accumulator, uint16_t *count";
specialize qw/vp9_temporal_filter_apply sse2 msa/;
diff --git a/vp9/decoder/vp9_decoder.c b/vp9/decoder/vp9_decoder.c
index 4e88819..f5da07e 100644
--- a/vp9/decoder/vp9_decoder.c
+++ b/vp9/decoder/vp9_decoder.c
@@ -243,7 +243,7 @@
decrease_ref_count(old_idx, frame_bufs, pool);
// Release the reference frame in reference map.
- if ((mask & 1) && old_idx >= 0) {
+ if (mask & 1) {
decrease_ref_count(old_idx, frame_bufs, pool);
}
cm->ref_frame_map[ref_index] = cm->next_ref_frame_map[ref_index];
@@ -350,7 +350,7 @@
decrease_ref_count(old_idx, frame_bufs, pool);
// Release the reference frame in reference map.
- if ((mask & 1) && old_idx >= 0) {
+ if (mask & 1) {
decrease_ref_count(old_idx, frame_bufs, pool);
}
++ref_index;
diff --git a/vp9/decoder/vp9_decoder.h b/vp9/decoder/vp9_decoder.h
index 4a5188f..afa4009 100644
--- a/vp9/decoder/vp9_decoder.h
+++ b/vp9/decoder/vp9_decoder.h
@@ -128,7 +128,7 @@
static INLINE void decrease_ref_count(int idx, RefCntBuffer *const frame_bufs,
BufferPool *const pool) {
- if (idx >= 0) {
+ if (idx >= 0 && frame_bufs[idx].ref_count > 0) {
--frame_bufs[idx].ref_count;
// A worker may only get a free framebuffer index when calling get_free_fb.
// But the private buffer is not set up until finish decoding header.
diff --git a/vp9/encoder/vp9_block.h b/vp9/encoder/vp9_block.h
index fc34786..ca5d746 100644
--- a/vp9/encoder/vp9_block.h
+++ b/vp9/encoder/vp9_block.h
@@ -71,6 +71,8 @@
int rddiv;
int rdmult;
int mb_energy;
+ int * m_search_count_ptr;
+ int * ex_search_count_ptr;
// These are set to their default values at the beginning, and then adjusted
// further in the encoding process.
diff --git a/vp9/encoder/vp9_context_tree.h b/vp9/encoder/vp9_context_tree.h
index 8e365ce..86ba03d 100644
--- a/vp9/encoder/vp9_context_tree.h
+++ b/vp9/encoder/vp9_context_tree.h
@@ -60,6 +60,7 @@
#if CONFIG_VP9_TEMPORAL_DENOISING
unsigned int newmv_sse;
unsigned int zeromv_sse;
+ unsigned int zeromv_lastref_sse;
PREDICTION_MODE best_sse_inter_mode;
int_mv best_sse_mv;
MV_REFERENCE_FRAME best_reference_frame;
diff --git a/vp9/encoder/vp9_denoiser.c b/vp9/encoder/vp9_denoiser.c
index c382b77..e87a12e 100644
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -224,14 +224,19 @@
} else {
// Otherwise, use the zero reference frame.
frame = ctx->best_zeromv_reference_frame;
-
- mbmi->ref_frame[0] = ctx->best_zeromv_reference_frame;
+ ctx->newmv_sse = ctx->zeromv_sse;
+ // Bias to last reference.
+ if (frame != LAST_FRAME &&
+ ((ctx->zeromv_lastref_sse < (5 * ctx->zeromv_sse) >> 2) ||
+ denoiser->denoising_level >= kDenHigh)) {
+ frame = LAST_FRAME;
+ ctx->newmv_sse = ctx->zeromv_lastref_sse;
+ }
+ mbmi->ref_frame[0] = frame;
mbmi->mode = ZEROMV;
mbmi->mv[0].as_int = 0;
-
ctx->best_sse_inter_mode = ZEROMV;
ctx->best_sse_mv.as_int = 0;
- ctx->newmv_sse = ctx->zeromv_sse;
}
if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
@@ -462,6 +467,8 @@
if (mbmi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
ctx->zeromv_sse = sse;
ctx->best_zeromv_reference_frame = mbmi->ref_frame[0];
+ if (mbmi->ref_frame[0] == LAST_FRAME)
+ ctx->zeromv_lastref_sse = sse;
}
if (mbmi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 0475883..f9c28f6 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -3839,6 +3839,10 @@
TOKENEXTRA *tok = cpi->tile_tok[tile_row][tile_col];
int mi_row;
+ // Set up pointers to per thread motion search counters.
+ td->mb.m_search_count_ptr = &td->rd_counts.m_search_count;
+ td->mb.ex_search_count_ptr = &td->rd_counts.ex_search_count;
+
for (mi_row = tile_info->mi_row_start; mi_row < tile_info->mi_row_end;
mi_row += MI_BLOCK_SIZE) {
if (cpi->sf.use_nonrd_pick_mode)
@@ -3895,6 +3899,9 @@
vp9_zero(rdc->coef_counts);
vp9_zero(rdc->comp_pred_diff);
vp9_zero(rdc->filter_diff);
+ rdc->m_search_count = 0; // Count of motion search hits.
+ rdc->ex_search_count = 0; // Exhaustive mesh search hits.
+
xd->lossless = cm->base_qindex == 0 &&
cm->y_dc_delta_q == 0 &&
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index eebd7c5..a57cf87 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -2995,7 +2995,7 @@
recon_err = vp9_get_y_sse(cpi->Source, get_frame_new_buffer(cm));
if (cpi->twopass.total_left_stats.coded_error != 0.0)
- fprintf(f, "%10u %dx%d %d %d %10d %10d %10d %10d"
+ fprintf(f, "%10u %dx%d %10d %10d %d %d %10d %10d %10d %10d"
"%10"PRId64" %10"PRId64" %5d %5d %10"PRId64" "
"%10"PRId64" %10"PRId64" %10d "
"%7.2lf %7.2lf %7.2lf %7.2lf %7.2lf"
@@ -3004,6 +3004,8 @@
"%10lf %8u %10"PRId64" %10d %10d %10d\n",
cpi->common.current_video_frame,
cm->width, cm->height,
+ cpi->td.rd_counts.m_search_count,
+ cpi->td.rd_counts.ex_search_count,
cpi->rc.source_alt_ref_pending,
cpi->rc.source_alt_ref_active,
cpi->rc.this_frame_target,
diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h
index 975d9f4..f6d8931 100644
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -260,6 +260,8 @@
vp9_coeff_count coef_counts[TX_SIZES][PLANE_TYPES];
int64_t comp_pred_diff[REFERENCE_MODES];
int64_t filter_diff[SWITCHABLE_FILTER_CONTEXTS];
+ int m_search_count;
+ int ex_search_count;
} RD_COUNTS;
typedef struct ThreadData {
diff --git a/vp9/encoder/vp9_ethread.c b/vp9/encoder/vp9_ethread.c
index ad25712..1d1926c 100644
--- a/vp9/encoder/vp9_ethread.c
+++ b/vp9/encoder/vp9_ethread.c
@@ -30,6 +30,10 @@
for (n = 0; n < ENTROPY_TOKENS; n++)
td->rd_counts.coef_counts[i][j][k][l][m][n] +=
td_t->rd_counts.coef_counts[i][j][k][l][m][n];
+
+ // Counts of all motion searches and exhuastive mesh searches.
+ td->rd_counts.m_search_count += td_t->rd_counts.m_search_count;
+ td->rd_counts.ex_search_count += td_t->rd_counts.ex_search_count;
}
static int enc_worker_hook(EncWorkerData *const thread_data, void *unused) {
diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c
index b9a104a..327ac19 100644
--- a/vp9/encoder/vp9_mcomp.c
+++ b/vp9/encoder/vp9_mcomp.c
@@ -1517,69 +1517,83 @@
#undef CHECK_BETTER
-int vp9_full_range_search_c(const MACROBLOCK *x,
- const search_site_config *cfg,
- MV *ref_mv, MV *best_mv,
- int search_param, int sad_per_bit, int *num00,
- const vp9_variance_fn_ptr_t *fn_ptr,
- const MV *center_mv) {
+// Exhuastive motion search around a given centre position with a given
+// step size.
+static int exhuastive_mesh_search(const MACROBLOCK *x,
+ MV *ref_mv, MV *best_mv,
+ int range, int step, int sad_per_bit,
+ const vp9_variance_fn_ptr_t *fn_ptr,
+ const MV *center_mv) {
const MACROBLOCKD *const xd = &x->e_mbd;
const struct buf_2d *const what = &x->plane[0].src;
const struct buf_2d *const in_what = &xd->plane[0].pre[0];
- const int range = 64;
- const MV fcenter_mv = {center_mv->row >> 3, center_mv->col >> 3};
+ MV fcenter_mv = {center_mv->row, center_mv->col};
unsigned int best_sad = INT_MAX;
int r, c, i;
int start_col, end_col, start_row, end_row;
+ int col_step = (step > 1) ? step : 4;
- // The cfg and search_param parameters are not used in this search variant
- (void)cfg;
- (void)search_param;
+ assert(step >= 1);
- clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max);
- *best_mv = *ref_mv;
- *num00 = 11;
+ clamp_mv(&fcenter_mv, x->mv_col_min, x->mv_col_max,
+ x->mv_row_min, x->mv_row_max);
+ *best_mv = fcenter_mv;
best_sad = fn_ptr->sdf(what->buf, what->stride,
- get_buf_from_mv(in_what, ref_mv), in_what->stride) +
- mvsad_err_cost(x, ref_mv, &fcenter_mv, sad_per_bit);
- start_row = VPXMAX(-range, x->mv_row_min - ref_mv->row);
- start_col = VPXMAX(-range, x->mv_col_min - ref_mv->col);
- end_row = VPXMIN(range, x->mv_row_max - ref_mv->row);
- end_col = VPXMIN(range, x->mv_col_max - ref_mv->col);
+ get_buf_from_mv(in_what, &fcenter_mv), in_what->stride) +
+ mvsad_err_cost(x, &fcenter_mv, ref_mv, sad_per_bit);
+ start_row = VPXMAX(-range, x->mv_row_min - fcenter_mv.row);
+ start_col = VPXMAX(-range, x->mv_col_min - fcenter_mv.col);
+ end_row = VPXMIN(range, x->mv_row_max - fcenter_mv.row);
+ end_col = VPXMIN(range, x->mv_col_max - fcenter_mv.col);
- for (r = start_row; r <= end_row; ++r) {
- for (c = start_col; c <= end_col; c += 4) {
- if (c + 3 <= end_col) {
- unsigned int sads[4];
- const uint8_t *addrs[4];
- for (i = 0; i < 4; ++i) {
- const MV mv = {ref_mv->row + r, ref_mv->col + c + i};
- addrs[i] = get_buf_from_mv(in_what, &mv);
- }
-
- fn_ptr->sdx4df(what->buf, what->stride, addrs, in_what->stride, sads);
-
- for (i = 0; i < 4; ++i) {
- if (sads[i] < best_sad) {
- const MV mv = {ref_mv->row + r, ref_mv->col + c + i};
- const unsigned int sad = sads[i] +
- mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit);
- if (sad < best_sad) {
- best_sad = sad;
- *best_mv = mv;
- }
+ for (r = start_row; r <= end_row; r += step) {
+ for (c = start_col; c <= end_col; c += col_step) {
+ // Step > 1 means we are not checking every location in this pass.
+ if (step > 1) {
+ const MV mv = {fcenter_mv.row + r, fcenter_mv.col + c};
+ unsigned int sad = fn_ptr->sdf(what->buf, what->stride,
+ get_buf_from_mv(in_what, &mv), in_what->stride);
+ if (sad < best_sad) {
+ sad += mvsad_err_cost(x, &mv, ref_mv, sad_per_bit);
+ if (sad < best_sad) {
+ best_sad = sad;
+ *best_mv = mv;
}
}
} else {
- for (i = 0; i < end_col - c; ++i) {
- const MV mv = {ref_mv->row + r, ref_mv->col + c + i};
- unsigned int sad = fn_ptr->sdf(what->buf, what->stride,
- get_buf_from_mv(in_what, &mv), in_what->stride);
- if (sad < best_sad) {
- sad += mvsad_err_cost(x, &mv, &fcenter_mv, sad_per_bit);
+ // 4 sads in a single call if we are checking every location
+ if (c + 3 <= end_col) {
+ unsigned int sads[4];
+ const uint8_t *addrs[4];
+ for (i = 0; i < 4; ++i) {
+ const MV mv = {fcenter_mv.row + r, fcenter_mv.col + c + i};
+ addrs[i] = get_buf_from_mv(in_what, &mv);
+ }
+ fn_ptr->sdx4df(what->buf, what->stride, addrs,
+ in_what->stride, sads);
+
+ for (i = 0; i < 4; ++i) {
+ if (sads[i] < best_sad) {
+ const MV mv = {fcenter_mv.row + r, fcenter_mv.col + c + i};
+ const unsigned int sad = sads[i] +
+ mvsad_err_cost(x, &mv, ref_mv, sad_per_bit);
+ if (sad < best_sad) {
+ best_sad = sad;
+ *best_mv = mv;
+ }
+ }
+ }
+ } else {
+ for (i = 0; i < end_col - c; ++i) {
+ const MV mv = {fcenter_mv.row + r, fcenter_mv.col + c + i};
+ unsigned int sad = fn_ptr->sdf(what->buf, what->stride,
+ get_buf_from_mv(in_what, &mv), in_what->stride);
if (sad < best_sad) {
- best_sad = sad;
- *best_mv = mv;
+ sad += mvsad_err_cost(x, &mv, ref_mv, sad_per_bit);
+ if (sad < best_sad) {
+ best_sad = sad;
+ *best_mv = mv;
+ }
}
}
}
@@ -2011,6 +2025,70 @@
return bestsme;
}
+#define MIN_RANGE 7
+#define MAX_RANGE 256
+#define MIN_INTERVAL 1
+// Runs an limited range exhaustive mesh search using a pattern set
+// according to the encode speed profile.
+static int full_pixel_exhaustive(VP9_COMP *cpi, MACROBLOCK *x,
+ MV *centre_mv_full, int sadpb, int *cost_list,
+ const vp9_variance_fn_ptr_t *fn_ptr,
+ const MV *ref_mv, MV *dst_mv) {
+ const SPEED_FEATURES *const sf = &cpi->sf;
+ MV temp_mv = {centre_mv_full->row, centre_mv_full->col};
+ MV f_ref_mv = {ref_mv->row >> 3, ref_mv->col >> 3};
+ int bestsme;
+ int i;
+ int interval = sf->mesh_patterns[0].interval;
+ int range = sf->mesh_patterns[0].range;
+ int baseline_interval_divisor;
+
+ // Keep track of number of exhaustive calls (this frame in this thread).
+ ++(*x->ex_search_count_ptr);
+
+ // Trap illegal values for interval and range for this function.
+ if ((range < MIN_RANGE) || (range > MAX_RANGE) ||
+ (interval < MIN_INTERVAL) || (interval > range))
+ return INT_MAX;
+
+ baseline_interval_divisor = range / interval;
+
+ // Check size of proposed first range against magnitude of the centre
+ // value used as a starting point.
+ range = VPXMAX(range, (5 * VPXMAX(abs(temp_mv.row), abs(temp_mv.col))) / 4);
+ range = VPXMIN(range, MAX_RANGE);
+ interval = VPXMAX(interval, range / baseline_interval_divisor);
+
+ // initial search
+ bestsme = exhuastive_mesh_search(x, &f_ref_mv, &temp_mv, range,
+ interval, sadpb, fn_ptr, &temp_mv);
+
+ if ((interval > MIN_INTERVAL) && (range > MIN_RANGE)) {
+ // Progressive searches with range and step size decreasing each time
+ // till we reach a step size of 1. Then break out.
+ for (i = 1; i < MAX_MESH_STEP; ++i) {
+ // First pass with coarser step and longer range
+ bestsme = exhuastive_mesh_search(x, &f_ref_mv, &temp_mv,
+ sf->mesh_patterns[i].range,
+ sf->mesh_patterns[i].interval,
+ sadpb, fn_ptr, &temp_mv);
+
+ if (sf->mesh_patterns[i].interval == 1)
+ break;
+ }
+ }
+
+ if (bestsme < INT_MAX)
+ bestsme = vp9_get_mvpred_var(x, &temp_mv, ref_mv, fn_ptr, 1);
+ *dst_mv = temp_mv;
+
+ // Return cost list.
+ if (cost_list) {
+ calc_int_cost_list(x, ref_mv, sadpb, fn_ptr, dst_mv, cost_list);
+ }
+ return bestsme;
+}
+
int vp9_full_search_sad_c(const MACROBLOCK *x, const MV *ref_mv,
int sad_per_bit, int distance,
const vp9_variance_fn_ptr_t *fn_ptr,
@@ -2324,6 +2402,18 @@
return best_sad;
}
+#define MIN_EX_SEARCH_LIMIT 128
+static int is_exhaustive_allowed(VP9_COMP *cpi, MACROBLOCK *x) {
+ const SPEED_FEATURES *const sf = &cpi->sf;
+ const int max_ex = VPXMAX(MIN_EX_SEARCH_LIMIT,
+ (*x->m_search_count_ptr * sf->max_exaustive_pct) / 100);
+
+ return sf->allow_exhaustive_searches &&
+ (sf->exhaustive_searches_thresh < INT_MAX) &&
+ (*x->ex_search_count_ptr <= max_ex) &&
+ !cpi->rc.is_src_frame_alt_ref;
+}
+
int vp9_full_pixel_search(VP9_COMP *cpi, MACROBLOCK *x,
BLOCK_SIZE bsize, MV *mvp_full,
int step_param, int error_per_bit,
@@ -2342,6 +2432,9 @@
cost_list[4] = INT_MAX;
}
+ // Keep track of number of searches (this frame in this thread).
+ ++(*x->m_search_count_ptr);
+
switch (method) {
case FAST_DIAMOND:
var = fast_dia_search(x, mvp_full, step_param, error_per_bit, 0,
@@ -2367,6 +2460,27 @@
var = full_pixel_diamond(cpi, x, mvp_full, step_param, error_per_bit,
MAX_MVSEARCH_STEPS - 1 - step_param,
1, cost_list, fn_ptr, ref_mv, tmp_mv);
+
+ // Should we allow a follow on exhaustive search?
+ if (is_exhaustive_allowed(cpi, x)) {
+ int64_t exhuastive_thr = sf->exhaustive_searches_thresh;
+ exhuastive_thr >>= 8 - (b_width_log2_lookup[bsize] +
+ b_height_log2_lookup[bsize]);
+
+ // Threshold variance for an exhaustive full search.
+ if (var > exhuastive_thr) {
+ int var_ex;
+ MV tmp_mv_ex;
+ var_ex = full_pixel_exhaustive(cpi, x, tmp_mv,
+ error_per_bit, cost_list, fn_ptr,
+ ref_mv, &tmp_mv_ex);
+
+ if (var_ex < var) {
+ var = var_ex;
+ *tmp_mv = tmp_mv_ex;
+ }
+ }
+ }
break;
default:
assert(0 && "Invalid search method.");
diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c
index 0377cb5..fdff363 100644
--- a/vp9/encoder/vp9_ratectrl.c
+++ b/vp9/encoder/vp9_ratectrl.c
@@ -1558,7 +1558,7 @@
VP9_COMMON *const cm = &cpi->common;
RATE_CONTROL *const rc = &cpi->rc;
int target = rc->avg_frame_bandwidth;
- const int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
+ int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
cpi->svc.temporal_layer_id, cpi->svc.number_temporal_layers);
if ((cm->current_video_frame == 0) ||
@@ -1573,8 +1573,10 @@
cpi->ref_frame_flags &=
(~VP9_LAST_FLAG & ~VP9_GOLD_FLAG & ~VP9_ALT_FLAG);
} else if (is_one_pass_cbr_svc(cpi)) {
- cpi->svc.layer_context[layer].is_key_frame = 1;
reset_temporal_layer_to_zero(cpi);
+ layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
+ cpi->svc.temporal_layer_id, cpi->svc.number_temporal_layers);
+ cpi->svc.layer_context[layer].is_key_frame = 1;
cpi->ref_frame_flags &=
(~VP9_LAST_FLAG & ~VP9_GOLD_FLAG & ~VP9_ALT_FLAG);
// Assumption here is that LAST_FRAME is being updated for a keyframe.
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index a400501..2a6b707 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -1750,8 +1750,9 @@
const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize];
ENTROPY_CONTEXT t_above[2], t_left[2];
int subpelmv = 1, have_ref = 0;
+ SPEED_FEATURES *const sf = &cpi->sf;
const int has_second_rf = has_second_ref(mbmi);
- const int inter_mode_mask = cpi->sf.inter_mode_mask[bsize];
+ const int inter_mode_mask = sf->inter_mode_mask[bsize];
MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
vp9_zero(*bsi);
@@ -1820,7 +1821,7 @@
seg_mvs[i][mbmi->ref_frame[0]].as_int == INVALID_MV) {
MV *const new_mv = &mode_mv[NEWMV][0].as_mv;
int step_param = 0;
- int thissme, bestsme = INT_MAX;
+ int bestsme = INT_MAX;
int sadpb = x->sadperbit4;
MV mvp_full;
int max_mv;
@@ -1845,7 +1846,7 @@
max_mv =
VPXMAX(abs(bsi->mvp.as_mv.row), abs(bsi->mvp.as_mv.col)) >> 3;
- if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) {
+ if (sf->mv.auto_mv_step_size && cm->show_frame) {
// Take wtd average of the step_params based on the last frame's
// max mv magnitude and the best ref mvs of the current block for
// the given reference.
@@ -1858,7 +1859,7 @@
mvp_full.row = bsi->mvp.as_mv.row >> 3;
mvp_full.col = bsi->mvp.as_mv.col >> 3;
- if (cpi->sf.adaptive_motion_search) {
+ if (sf->adaptive_motion_search) {
mvp_full.row = x->pred_mv[mbmi->ref_frame[0]].row >> 3;
mvp_full.col = x->pred_mv[mbmi->ref_frame[0]].col >> 3;
step_param = VPXMAX(step_param, 8);
@@ -1871,31 +1872,10 @@
bestsme = vp9_full_pixel_search(
cpi, x, bsize, &mvp_full, step_param, sadpb,
- cpi->sf.mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL,
+ sf->mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL,
&bsi->ref_mv[0]->as_mv, new_mv,
INT_MAX, 1);
- // Should we do a full search (best quality only)
- if (cpi->oxcf.mode == BEST) {
- int_mv *const best_mv = &mi->bmi[i].as_mv[0];
- /* Check if mvp_full is within the range. */
- clamp_mv(&mvp_full, x->mv_col_min, x->mv_col_max,
- x->mv_row_min, x->mv_row_max);
- thissme = cpi->full_search_sad(x, &mvp_full,
- sadpb, 16, &cpi->fn_ptr[bsize],
- &bsi->ref_mv[0]->as_mv,
- &best_mv->as_mv);
- cost_list[1] = cost_list[2] = cost_list[3] = cost_list[4] = INT_MAX;
- if (thissme < bestsme) {
- bestsme = thissme;
- *new_mv = best_mv->as_mv;
- } else {
- // The full search result is actually worse so re-instate the
- // previous best vector
- best_mv->as_mv = *new_mv;
- }
- }
-
if (bestsme < INT_MAX) {
int distortion;
cpi->find_fractional_mv_step(
@@ -1904,8 +1884,8 @@
&bsi->ref_mv[0]->as_mv,
cm->allow_high_precision_mv,
x->errorperbit, &cpi->fn_ptr[bsize],
- cpi->sf.mv.subpel_force_stop,
- cpi->sf.mv.subpel_iters_per_step,
+ sf->mv.subpel_force_stop,
+ sf->mv.subpel_iters_per_step,
cond_cost_list(cpi, cost_list),
x->nmvjointcost, x->mvcost,
&distortion,
@@ -1916,7 +1896,7 @@
seg_mvs[i][mbmi->ref_frame[0]].as_mv = *new_mv;
}
- if (cpi->sf.adaptive_motion_search)
+ if (sf->adaptive_motion_search)
x->pred_mv[mbmi->ref_frame[0]] = *new_mv;
// restore src pointers
@@ -1933,7 +1913,7 @@
mbmi->interp_filter == EIGHTTAP) {
// adjust src pointers
mi_buf_shift(x, i);
- if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
+ if (sf->comp_inter_joint_search_thresh <= bsize) {
int rate_mv;
joint_motion_search(cpi, x, bsize, frame_mv[this_mode],
mi_row, mi_col, seg_mvs[i],
@@ -2851,9 +2831,9 @@
? (source_variance - recon_variance)
: (recon_variance - source_variance);
- var_error = (200 * source_variance * recon_variance) /
- ((source_variance * source_variance) +
- (recon_variance * recon_variance));
+ var_error = ((int64_t)200 * source_variance * recon_variance) /
+ (((int64_t)source_variance * source_variance) +
+ ((int64_t)recon_variance * recon_variance));
var_error = 100 - var_error;
}
diff --git a/vp9/encoder/vp9_speed_features.c b/vp9/encoder/vp9_speed_features.c
index a539629..318d810 100644
--- a/vp9/encoder/vp9_speed_features.c
+++ b/vp9/encoder/vp9_speed_features.c
@@ -15,6 +15,22 @@
#include "vp9/encoder/vp9_rdopt.h"
#include "vpx_dsp/vpx_dsp_common.h"
+// Mesh search patters for various speed settings
+static MESH_PATTERN best_quality_mesh_pattern[MAX_MESH_STEP] =
+ {{64, 4}, {28, 2}, {15, 1}, {7, 1}};
+
+#define MAX_MESH_SPEED 5 // Max speed setting for mesh motion method
+static MESH_PATTERN good_quality_mesh_patterns[MAX_MESH_SPEED + 1]
+ [MAX_MESH_STEP] =
+ {{{64, 8}, {28, 4}, {15, 1}, {7, 1}},
+ {{64, 8}, {28, 4}, {15, 1}, {7, 1}},
+ {{64, 8}, {14, 2}, {7, 1}, {7, 1}},
+ {{64, 16}, {24, 8}, {12, 4}, {7, 1}},
+ {{64, 16}, {24, 8}, {12, 4}, {7, 1}},
+ {{64, 16}, {24, 8}, {12, 4}, {7, 1}},
+ };
+static unsigned char good_quality_max_mesh_pct[MAX_MESH_SPEED + 1] =
+ {50, 25, 15, 5, 1, 1};
// Intra only frames, golden frames (except alt ref overlays) and
// alt ref frames tend to be coded at a higher than ambient quality
@@ -259,6 +275,8 @@
sf->static_segmentation = 0;
sf->adaptive_rd_thresh = 1;
sf->use_fast_coef_costing = 1;
+ sf->allow_exhaustive_searches = 0;
+ sf->exhaustive_searches_thresh = INT_MAX;
if (speed >= 1) {
sf->use_square_partition_only = !frame_is_intra_only(cm);
@@ -460,7 +478,6 @@
sf->mv.auto_mv_step_size = 0;
sf->mv.fullpel_search_step_param = 6;
sf->comp_inter_joint_search_thresh = BLOCK_4X4;
- sf->adaptive_rd_thresh = 0;
sf->tx_size_search_method = USE_FULL_RD;
sf->use_lp32x32fdct = 0;
sf->adaptive_motion_search = 0;
@@ -516,19 +533,50 @@
// Recode loop tolerance %.
sf->recode_tolerance = 25;
sf->default_interp_filter = SWITCHABLE;
- sf->tx_size_search_breakout = 0;
- sf->partition_search_breakout_dist_thr = 0;
- sf->partition_search_breakout_rate_thr = 0;
sf->simple_model_rd_from_var = 0;
+ // Some speed-up features even for best quality as minimal impact on quality.
+ sf->adaptive_rd_thresh = 1;
+ sf->tx_size_search_breakout = 1;
+ sf->partition_search_breakout_dist_thr = (1 << 19);
+ sf->partition_search_breakout_rate_thr = 80;
+
if (oxcf->mode == REALTIME)
set_rt_speed_feature(cpi, sf, oxcf->speed, oxcf->content);
else if (oxcf->mode == GOOD)
set_good_speed_feature(cpi, cm, sf, oxcf->speed);
cpi->full_search_sad = vp9_full_search_sad;
- cpi->diamond_search_sad = oxcf->mode == BEST ? vp9_full_range_search
- : vp9_diamond_search_sad;
+ cpi->diamond_search_sad = vp9_diamond_search_sad;
+
+ sf->allow_exhaustive_searches = 1;
+ if (oxcf->mode == BEST) {
+ if (cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION)
+ sf->exhaustive_searches_thresh = (1 << 20);
+ else
+ sf->exhaustive_searches_thresh = (1 << 21);
+ sf->max_exaustive_pct = 100;
+ for (i = 0; i < MAX_MESH_STEP; ++i) {
+ sf->mesh_patterns[i].range = best_quality_mesh_pattern[i].range;
+ sf->mesh_patterns[i].interval = best_quality_mesh_pattern[i].interval;
+ }
+ } else {
+ int speed = (oxcf->speed > MAX_MESH_SPEED) ? MAX_MESH_SPEED : oxcf->speed;
+ if (cpi->twopass.fr_content_type == FC_GRAPHICS_ANIMATION)
+ sf->exhaustive_searches_thresh = (1 << 22);
+ else
+ sf->exhaustive_searches_thresh = (1 << 23);
+ sf->max_exaustive_pct = good_quality_max_mesh_pct[speed];
+ if (speed > 0)
+ sf->exhaustive_searches_thresh = sf->exhaustive_searches_thresh << 1;
+
+ for (i = 0; i < MAX_MESH_STEP; ++i) {
+ sf->mesh_patterns[i].range =
+ good_quality_mesh_patterns[speed][i].range;
+ sf->mesh_patterns[i].interval =
+ good_quality_mesh_patterns[speed][i].interval;
+ }
+ }
// Slow quant, dct and trellis not worthwhile for first pass
// so make sure they are always turned off.
diff --git a/vp9/encoder/vp9_speed_features.h b/vp9/encoder/vp9_speed_features.h
index 575e98c..ad7b64a 100644
--- a/vp9/encoder/vp9_speed_features.h
+++ b/vp9/encoder/vp9_speed_features.h
@@ -195,6 +195,13 @@
int fullpel_search_step_param;
} MV_SPEED_FEATURES;
+#define MAX_MESH_STEP 4
+
+typedef struct MESH_PATTERN {
+ int range;
+ int interval;
+} MESH_PATTERN;
+
typedef struct SPEED_FEATURES {
MV_SPEED_FEATURES mv;
@@ -299,6 +306,18 @@
// point for this motion search and limits the search range around it.
int adaptive_motion_search;
+ // Flag for allowing some use of exhaustive searches;
+ int allow_exhaustive_searches;
+
+ // Threshold for allowing exhaistive motion search.
+ int exhaustive_searches_thresh;
+
+ // Maximum number of exhaustive searches for a frame.
+ int max_exaustive_pct;
+
+ // Pattern to be used for any exhaustive mesh searches.
+ MESH_PATTERN mesh_patterns[MAX_MESH_STEP];
+
int schedule_mode_search;
// Allows sub 8x8 modes to use the prediction filter that was determined
diff --git a/vp9/encoder/vp9_svc_layercontext.c b/vp9/encoder/vp9_svc_layercontext.c
index 1dfc45c..13da155 100644
--- a/vp9/encoder/vp9_svc_layercontext.c
+++ b/vp9/encoder/vp9_svc_layercontext.c
@@ -278,7 +278,8 @@
cpi->alt_ref_source = lc->alt_ref_source;
// Reset the frames_since_key and frames_to_key counters to their values
// before the layer restore. Keep these defined for the stream (not layer).
- if (cpi->svc.number_temporal_layers > 1) {
+ if (cpi->svc.number_temporal_layers > 1 ||
+ cpi->svc.number_spatial_layers > 1) {
cpi->rc.frames_since_key = old_frame_since_key;
cpi->rc.frames_to_key = old_frame_to_key;
}
diff --git a/vpx_dsp/bitreader_buffer.c b/vpx_dsp/bitreader_buffer.c
index bb91726..d7b55cf 100644
--- a/vpx_dsp/bitreader_buffer.c
+++ b/vpx_dsp/bitreader_buffer.c
@@ -45,7 +45,7 @@
int bits) {
#if CONFIG_MISC_FIXES
const int nbits = sizeof(unsigned) * 8 - bits - 1;
- const unsigned value = vpx_rb_read_literal(rb, bits + 1) << nbits;
+ const unsigned value = (unsigned)vpx_rb_read_literal(rb, bits + 1) << nbits;
return ((int) value) >> nbits;
#else
return vpx_rb_read_signed_literal(rb, bits);
diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl
index b369b05..4de85a4 100644
--- a/vpx_dsp/vpx_dsp_rtcd_defs.pl
+++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -97,7 +97,7 @@
specialize qw/vpx_ve_predictor_4x4/;
add_proto qw/void vpx_tm_predictor_4x4/, "uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left";
-specialize qw/vpx_tm_predictor_4x4 neon dspr2 msa/, "$sse_x86inc";
+specialize qw/vpx_tm_predictor_4x4 neon dspr2 msa/, "$sse2_x86inc";
add_proto qw/void vpx_dc_predictor_4x4/, "uint8_t *dst, ptrdiff_t y_stride, const uint8_t *above, const uint8_t *left";
specialize qw/vpx_dc_predictor_4x4 dspr2 msa neon/, "$sse_x86inc";
@@ -893,7 +893,7 @@
specialize qw/vpx_idct32x32_1024_add sse2 neon dspr2 msa/;
add_proto qw/void vpx_idct32x32_34_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
- specialize qw/vpx_idct32x32_34_add sse2 neon_asm dspr2 msa/;
+ specialize qw/vpx_idct32x32_34_add sse2 neon_asm dspr2 msa/, "$ssse3_x86_64_x86inc";
# Need to add 34 eob idct32x32 neon implementation.
$vpx_idct32x32_34_add_neon_asm=vpx_idct32x32_1024_add_neon;
diff --git a/vpx_dsp/x86/intrapred_sse2.asm b/vpx_dsp/x86/intrapred_sse2.asm
index 22b5731..04b39a5 100644
--- a/vpx_dsp/x86/intrapred_sse2.asm
+++ b/vpx_dsp/x86/intrapred_sse2.asm
@@ -515,35 +515,34 @@
jnz .loop
REP_RET
-INIT_MMX sse
-cglobal tm_predictor_4x4, 4, 4, 4, dst, stride, above, left
+INIT_XMM sse2
+cglobal tm_predictor_4x4, 4, 4, 5, dst, stride, above, left
pxor m1, m1
- movd m2, [aboveq-1]
- movd m0, [aboveq]
- punpcklbw m2, m1
+ movq m0, [aboveq-1]; [63:0] tl t1 t2 t3 t4 x x x
punpcklbw m0, m1
- pshufw m2, m2, 0x0
- DEFINE_ARGS dst, stride, line, left
- mov lineq, -2
- add leftq, 4
- psubw m0, m2
-.loop:
- movd m2, [leftq+lineq*2]
- movd m3, [leftq+lineq*2+1]
+ pshuflw m2, m0, 0x0 ; [63:0] tl tl tl tl [word]
+ psrldq m0, 2
+ psubw m0, m2 ; [63:0] t1-tl t2-tl t3-tl t4-tl [word]
+ movd m2, [leftq]
punpcklbw m2, m1
- punpcklbw m3, m1
- pshufw m2, m2, 0x0
- pshufw m3, m3, 0x0
- paddw m2, m0
+ pshuflw m4, m2, 0x0 ; [63:0] l1 l1 l1 l1 [word]
+ pshuflw m3, m2, 0x55 ; [63:0] l2 l2 l2 l2 [word]
+ paddw m4, m0
paddw m3, m0
- packuswb m2, m2
+ packuswb m4, m4
packuswb m3, m3
- movd [dstq ], m2
+ movd [dstq ], m4
movd [dstq+strideq], m3
lea dstq, [dstq+strideq*2]
- inc lineq
- jnz .loop
- REP_RET
+ pshuflw m4, m2, 0xaa
+ pshuflw m3, m2, 0xff
+ paddw m4, m0
+ paddw m3, m0
+ packuswb m4, m4
+ packuswb m3, m3
+ movd [dstq ], m4
+ movd [dstq+strideq], m3
+ RET
INIT_XMM sse2
cglobal tm_predictor_8x8, 4, 4, 4, dst, stride, above, left
diff --git a/vpx_dsp/x86/intrapred_ssse3.asm b/vpx_dsp/x86/intrapred_ssse3.asm
index 88df9b2..9010c74 100644
--- a/vpx_dsp/x86/intrapred_ssse3.asm
+++ b/vpx_dsp/x86/intrapred_ssse3.asm
@@ -33,23 +33,20 @@
SECTION .text
-INIT_MMX ssse3
+INIT_XMM ssse3
cglobal h_predictor_4x4, 2, 4, 3, dst, stride, line, left
- movifnidn leftq, leftmp
- add leftq, 4
- mov lineq, -2
- pxor m0, m0
-.loop:
- movd m1, [leftq+lineq*2 ]
- movd m2, [leftq+lineq*2+1]
- pshufb m1, m0
- pshufb m2, m0
- movd [dstq ], m1
- movd [dstq+strideq], m2
+ movd m0, [leftq]
+ punpcklbw m0, m0
+ punpcklbw m0, m0
+ movd [dstq ], m0
+ psrldq m0, 4
+ movd [dstq+strideq], m0
lea dstq, [dstq+strideq*2]
- inc lineq
- jnz .loop
- REP_RET
+ psrldq m0, 4
+ movd [dstq ], m0
+ psrldq m0, 4
+ movd [dstq+strideq], m0
+ RET
INIT_MMX ssse3
cglobal h_predictor_8x8, 2, 4, 3, dst, stride, line, left
diff --git a/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm b/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
index 68e7fa4..d77dc51 100644
--- a/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
+++ b/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
@@ -17,18 +17,46 @@
SECTION_RODATA
pw_11585x2: times 8 dw 23170
+
+pw_m2404x2: times 8 dw -2404*2
+pw_m4756x2: times 8 dw -4756*2
+pw_m5520x2: times 8 dw -5520*2
+
+pw_16364x2: times 8 dw 16364*2
+pw_16305x2: times 8 dw 16305*2
+pw_16207x2: times 8 dw 16207*2
+pw_16069x2: times 8 dw 16069*2
+pw_15893x2: times 8 dw 15893*2
+pw_15679x2: times 8 dw 15679*2
+pw_15426x2: times 8 dw 15426*2
+pw__3981x2: times 8 dw 3981*2
+pw__3196x2: times 8 dw 3196*2
+pw__1606x2: times 8 dw 1606*2
+pw___804x2: times 8 dw 804*2
+
pd_8192: times 4 dd 8192
+pw_32: times 8 dw 32
pw_16: times 8 dw 16
%macro TRANSFORM_COEFFS 2
pw_%1_%2: dw %1, %2, %1, %2, %1, %2, %1, %2
pw_m%2_%1: dw -%2, %1, -%2, %1, -%2, %1, -%2, %1
+pw_m%1_m%2: dw -%1, -%2, -%1, -%2, -%1, -%2, -%1, -%2
%endmacro
TRANSFORM_COEFFS 6270, 15137
TRANSFORM_COEFFS 3196, 16069
TRANSFORM_COEFFS 13623, 9102
+; constants for 32x32_34
+TRANSFORM_COEFFS 804, 16364
+TRANSFORM_COEFFS 15426, 5520
+TRANSFORM_COEFFS 3981, 15893
+TRANSFORM_COEFFS 16207, 2404
+TRANSFORM_COEFFS 1606, 16305
+TRANSFORM_COEFFS 15679, 4756
+TRANSFORM_COEFFS 11585, 11585
+
%macro PAIR_PP_COEFFS 2
dpw_%1_%2: dw %1, %1, %1, %1, %2, %2, %2, %2
%endmacro
@@ -80,6 +108,15 @@
packssdw m%2, m%6
%endmacro
+%macro BUTTERFLY_4Xmm 7 ; dst1, dst2, coef1, coef2, round, tmp1, tmp2
+ punpckhwd m%6, m%2, m%1
+ MUL_ADD_2X %7, %6, %6, %5, [pw_m%4_%3], [pw_m%3_m%4]
+ punpcklwd m%2, m%1
+ MUL_ADD_2X %1, %2, %2, %5, [pw_m%4_%3], [pw_m%3_m%4]
+ packssdw m%1, m%7
+ packssdw m%2, m%6
+%endmacro
+
; matrix transpose
%macro INTERLEAVE_2X 4
punpckh%1 m%4, m%2, m%3
@@ -298,4 +335,453 @@
RET
+%define idx0 16 * 0
+%define idx1 16 * 1
+%define idx2 16 * 2
+%define idx3 16 * 3
+%define idx4 16 * 4
+%define idx5 16 * 5
+%define idx6 16 * 6
+%define idx7 16 * 7
+%define idx8 16 * 0
+%define idx9 16 * 1
+%define idx10 16 * 2
+%define idx11 16 * 3
+%define idx12 16 * 4
+%define idx13 16 * 5
+%define idx14 16 * 6
+%define idx15 16 * 7
+%define idx16 16 * 0
+%define idx17 16 * 1
+%define idx18 16 * 2
+%define idx19 16 * 3
+%define idx20 16 * 4
+%define idx21 16 * 5
+%define idx22 16 * 6
+%define idx23 16 * 7
+%define idx24 16 * 0
+%define idx25 16 * 1
+%define idx26 16 * 2
+%define idx27 16 * 3
+%define idx28 16 * 4
+%define idx29 16 * 5
+%define idx30 16 * 6
+%define idx31 16 * 7
+
+%macro IDCT32X32_34x 4
+ ; FROM idct32x32_add_neon.asm
+ ;
+ ; Instead of doing the transforms stage by stage, it is done by loading
+ ; some input values and doing as many stages as possible to minimize the
+ ; storing/loading of intermediate results. To fit within registers, the
+ ; final coefficients are cut into four blocks:
+ ; BLOCK A: 16-19,28-31
+ ; BLOCK B: 20-23,24-27
+ ; BLOCK C: 8-11,12-15
+ ; BLOCK D: 0-3,4-7
+ ; Blocks A and C are straight calculation through the various stages. In
+ ; block B, further calculations are performed using the results from
+ ; block A. In block D, further calculations are performed using the results
+ ; from block C and then the final calculations are done using results from
+ ; block A and B which have been combined at the end of block B.
+ ;
+ ; BLOCK A STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m11, m1
+ pmulhrsw m1, [pw___804x2] ; stp1_16
+ mova [r4 + 0], m0
+ pmulhrsw m11, [pw_16364x2] ; stp2_31
+ mova [r4 + 16 * 2], m2
+ mova m12, m7
+ pmulhrsw m7, [pw_15426x2] ; stp1_28
+ mova [r4 + 16 * 4], m4
+ pmulhrsw m12, [pw_m5520x2] ; stp2_19
+ mova [r4 + 16 * 6], m6
+
+ ; BLOCK A STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m2, m1 ; stp1_16
+ mova m0, m11 ; stp1_31
+ mova m4, m7 ; stp1_28
+ mova m15, m12 ; stp1_19
+
+ ; BLOCK A STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 0, 2, 3196, 16069, m8, 9, 10 ; stp1_17, stp1_30
+ BUTTERFLY_4Xmm 4, 15, 3196, 16069, m8, 9, 10 ; stp1_29, stp1_18
+
+ ; BLOCK A STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 1, 12, 9 ; stp2_16, stp2_19
+ SUM_SUB 0, 15, 9 ; stp2_17, stp2_18
+ SUM_SUB 11, 7, 9 ; stp2_31, stp2_28
+ SUM_SUB 2, 4, 9 ; stp2_30, stp2_29
+
+ ; BLOCK A STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 4, 15, 6270, 15137, m8, 9, 10 ; stp1_18, stp1_29
+ BUTTERFLY_4X 7, 12, 6270, 15137, m8, 9, 10 ; stp1_19, stp1_28
+
+ ; BLOCK B STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m6, m5
+ pmulhrsw m5, [pw__3981x2] ; stp1_20
+ mova [stp + %4 + idx28], m12
+ mova [stp + %4 + idx29], m15
+ pmulhrsw m6, [pw_15893x2] ; stp2_27
+ mova [stp + %4 + idx30], m2
+ mova m2, m3
+ pmulhrsw m3, [pw_m2404x2] ; stp1_23
+ mova [stp + %4 + idx31], m11
+ pmulhrsw m2, [pw_16207x2] ; stp2_24
+
+ ; BLOCK B STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m13, m5 ; stp1_20
+ mova m14, m6 ; stp1_27
+ mova m15, m3 ; stp1_23
+ mova m11, m2 ; stp1_24
+
+ ; BLOCK B STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 14, 13, 13623, 9102, m8, 9, 10 ; stp1_21, stp1_26
+ BUTTERFLY_4Xmm 11, 15, 13623, 9102, m8, 9, 10 ; stp1_25, stp1_22
+
+ ; BLOCK B STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 3, 5, 9 ; stp2_23, stp2_20
+ SUM_SUB 15, 14, 9 ; stp2_22, stp2_21
+ SUM_SUB 2, 6, 9 ; stp2_24, stp2_27
+ SUM_SUB 11, 13, 9 ; stp2_25, stp2_26
+
+ ; BLOCK B STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4Xmm 6, 5, 6270, 15137, m8, 9, 10 ; stp1_27, stp1_20
+ BUTTERFLY_4Xmm 13, 14, 6270, 15137, m8, 9, 10 ; stp1_26, stp1_21
+
+ ; BLOCK B STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 1, 3, 9 ; stp2_16, stp2_23
+ SUM_SUB 0, 15, 9 ; stp2_17, stp2_22
+ SUM_SUB 4, 14, 9 ; stp2_18, stp2_21
+ SUM_SUB 7, 5, 9 ; stp2_19, stp2_20
+ mova [stp + %3 + idx16], m1
+ mova [stp + %3 + idx17], m0
+ mova [stp + %3 + idx18], m4
+ mova [stp + %3 + idx19], m7
+
+ mova m4, [stp + %4 + idx28]
+ mova m7, [stp + %4 + idx29]
+ mova m10, [stp + %4 + idx30]
+ mova m12, [stp + %4 + idx31]
+ SUM_SUB 4, 6, 9 ; stp2_28, stp2_27
+ SUM_SUB 7, 13, 9 ; stp2_29, stp2_26
+ SUM_SUB 10, 11, 9 ; stp2_30, stp2_25
+ SUM_SUB 12, 2, 9 ; stp2_31, stp2_24
+ mova [stp + %4 + idx28], m4
+ mova [stp + %4 + idx29], m7
+ mova [stp + %4 + idx30], m10
+ mova [stp + %4 + idx31], m12
+
+ ; BLOCK B STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%if 0 ; overflow occurs in SUM_SUB when using test streams
+ mova m10, [pw_11585x2]
+ SUM_SUB 6, 5, 9
+ pmulhrsw m6, m10 ; stp1_27
+ pmulhrsw m5, m10 ; stp1_20
+ SUM_SUB 13, 14, 9
+ pmulhrsw m13, m10 ; stp1_26
+ pmulhrsw m14, m10 ; stp1_21
+ SUM_SUB 11, 15, 9
+ pmulhrsw m11, m10 ; stp1_25
+ pmulhrsw m15, m10 ; stp1_22
+ SUM_SUB 2, 3, 9
+ pmulhrsw m2, m10 ; stp1_24
+ pmulhrsw m3, m10 ; stp1_23
+%else
+ BUTTERFLY_4X 6, 5, 11585, 11585, m8, 9, 10 ; stp1_20, stp1_27
+ SWAP 6, 5
+ BUTTERFLY_4X 13, 14, 11585, 11585, m8, 9, 10 ; stp1_21, stp1_26
+ SWAP 13, 14
+ BUTTERFLY_4X 11, 15, 11585, 11585, m8, 9, 10 ; stp1_22, stp1_25
+ SWAP 11, 15
+ BUTTERFLY_4X 2, 3, 11585, 11585, m8, 9, 10 ; stp1_23, stp1_24
+ SWAP 2, 3
+%endif
+
+ mova [stp + %4 + idx24], m2
+ mova [stp + %4 + idx25], m11
+ mova [stp + %4 + idx26], m13
+ mova [stp + %4 + idx27], m6
+
+ ; BLOCK C STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ ;
+ ; BLOCK C STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m0, [rsp + transposed_in + 16 * 2]
+ mova m6, [rsp + transposed_in + 16 * 6]
+
+ mova m1, m0
+ pmulhrsw m0, [pw__1606x2] ; stp1_8
+ mova [stp + %3 + idx20], m5
+ mova [stp + %3 + idx21], m14
+ pmulhrsw m1, [pw_16305x2] ; stp2_15
+ mova [stp + %3 + idx22], m15
+ mova m7, m6
+ pmulhrsw m7, [pw_m4756x2] ; stp2_11
+ mova [stp + %3 + idx23], m3
+ pmulhrsw m6, [pw_15679x2] ; stp1_12
+
+ ; BLOCK C STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m3, m0 ; stp1_8
+ mova m2, m1 ; stp1_15
+
+ ; BLOCK C STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 2, 3, 6270, 15137, m8, 9, 10 ; stp1_9, stp1_14
+ mova m4, m7 ; stp1_11
+ mova m5, m6 ; stp1_12
+ BUTTERFLY_4Xmm 5, 4, 6270, 15137, m8, 9, 10 ; stp1_13, stp1_10
+
+ ; BLOCK C STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 0, 7, 9 ; stp1_8, stp1_11
+ SUM_SUB 2, 4, 9 ; stp1_9, stp1_10
+ SUM_SUB 1, 6, 9 ; stp1_15, stp1_12
+ SUM_SUB 3, 5, 9 ; stp1_14, stp1_13
+
+ ; BLOCK C STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%if 0 ; overflow occurs in SUM_SUB when using test streams
+ mova m10, [pw_11585x2]
+ SUM_SUB 5, 4, 9
+ pmulhrsw m5, m10 ; stp1_13
+ pmulhrsw m4, m10 ; stp1_10
+ SUM_SUB 6, 7, 9
+ pmulhrsw m6, m10 ; stp1_12
+ pmulhrsw m7, m10 ; stp1_11
+%else
+ BUTTERFLY_4X 5, 4, 11585, 11585, m8, 9, 10 ; stp1_10, stp1_13
+ SWAP 5, 4
+ BUTTERFLY_4X 6, 7, 11585, 11585, m8, 9, 10 ; stp1_11, stp1_12
+ SWAP 6, 7
+%endif
+
+ ; BLOCK C STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova [stp + %2 + idx8], m0
+ mova [stp + %2 + idx9], m2
+ mova [stp + %2 + idx10], m4
+ mova [stp + %2 + idx11], m7
+
+ ; BLOCK D STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ ;
+ ; BLOCK D STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ ;
+ ; BLOCK D STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m11, [rsp + transposed_in + 16 * 4]
+ mova m12, m11
+ pmulhrsw m11, [pw__3196x2] ; stp1_4
+ pmulhrsw m12, [pw_16069x2] ; stp1_7
+
+ ; BLOCK D STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m0, [rsp + transposed_in + 16 * 0]
+ mova m10, [pw_11585x2]
+ mova m7, m0
+ pmulhrsw m0, m10 ; stp1_1
+ pmulhrsw m7, m10 ; stp1_0
+
+ mova m14, m11 ; stp1_4
+ mova m13, m12 ; stp1_7
+
+ ; BLOCK D STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%if 0 ; overflow occurs in SUM_SUB when using test streams
+ SUM_SUB 13, 14, 9
+ pmulhrsw m13, m10 ; stp1_6
+ pmulhrsw m14, m10 ; stp1_5
+%else
+ BUTTERFLY_4X 13, 14, 11585, 11585, m8, 9, 10 ; stp1_5, stp1_6
+ SWAP 13, 14
+%endif
+ mova m4, m0 ; stp1_1
+ mova m2, m7 ; stp1_0
+
+ ; BLOCK D STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 0, 12, 9 ; stp1_0, stp1_7
+ SUM_SUB 7, 13, 9 ; stp1_1, stp1_6
+ SUM_SUB 2, 14, 9 ; stp1_2, stp1_5
+ SUM_SUB 4, 11, 9 ; stp1_3, stp1_4
+
+ ; BLOCK D STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 0, 1, 9 ; stp1_0, stp1_15
+ SUM_SUB 7, 3, 9 ; stp1_1, stp1_14
+ SUM_SUB 2, 5, 9 ; stp1_2, stp1_13
+ SUM_SUB 4, 6, 9 ; stp1_3, stp1_12
+
+ ; 0-3, 28-31 final stage
+ mova m15, [stp + %4 + idx30]
+ mova m10, [stp + %4 + idx31]
+ SUM_SUB 0, 10, 9 ; stp1_0, stp1_31
+ SUM_SUB 7, 15, 9 ; stp1_1, stp1_30
+ mova [stp + %1 + idx0], m0
+ mova [stp + %1 + idx1], m7
+ mova [stp + %4 + idx30], m15
+ mova [stp + %4 + idx31], m10
+ mova m7, [stp + %4 + idx28]
+ mova m0, [stp + %4 + idx29]
+ SUM_SUB 2, 0, 9 ; stp1_2, stp1_29
+ SUM_SUB 4, 7, 9 ; stp1_3, stp1_28
+ mova [stp + %1 + idx2], m2
+ mova [stp + %1 + idx3], m4
+ mova [stp + %4 + idx28], m7
+ mova [stp + %4 + idx29], m0
+
+ ; 12-15, 16-19 final stage
+ mova m0, [stp + %3 + idx16]
+ mova m7, [stp + %3 + idx17]
+ mova m2, [stp + %3 + idx18]
+ mova m4, [stp + %3 + idx19]
+ SUM_SUB 1, 0, 9 ; stp1_15, stp1_16
+ SUM_SUB 3, 7, 9 ; stp1_14, stp1_17
+ SUM_SUB 5, 2, 9 ; stp1_13, stp1_18
+ SUM_SUB 6, 4, 9 ; stp1_12, stp1_19
+ mova [stp + %2 + idx12], m6
+ mova [stp + %2 + idx13], m5
+ mova [stp + %2 + idx14], m3
+ mova [stp + %2 + idx15], m1
+ mova [stp + %3 + idx16], m0
+ mova [stp + %3 + idx17], m7
+ mova [stp + %3 + idx18], m2
+ mova [stp + %3 + idx19], m4
+
+ mova m4, [stp + %2 + idx8]
+ mova m5, [stp + %2 + idx9]
+ mova m6, [stp + %2 + idx10]
+ mova m7, [stp + %2 + idx11]
+ SUM_SUB 11, 7, 9 ; stp1_4, stp1_11
+ SUM_SUB 14, 6, 9 ; stp1_5, stp1_10
+ SUM_SUB 13, 5, 9 ; stp1_6, stp1_9
+ SUM_SUB 12, 4, 9 ; stp1_7, stp1_8
+
+ ; 4-7, 24-27 final stage
+ mova m0, [stp + %4 + idx27]
+ mova m1, [stp + %4 + idx26]
+ mova m2, [stp + %4 + idx25]
+ mova m3, [stp + %4 + idx24]
+ SUM_SUB 11, 0, 9 ; stp1_4, stp1_27
+ SUM_SUB 14, 1, 9 ; stp1_5, stp1_26
+ SUM_SUB 13, 2, 9 ; stp1_6, stp1_25
+ SUM_SUB 12, 3, 9 ; stp1_7, stp1_24
+ mova [stp + %4 + idx27], m0
+ mova [stp + %4 + idx26], m1
+ mova [stp + %4 + idx25], m2
+ mova [stp + %4 + idx24], m3
+ mova [stp + %1 + idx4], m11
+ mova [stp + %1 + idx5], m14
+ mova [stp + %1 + idx6], m13
+ mova [stp + %1 + idx7], m12
+
+ ; 8-11, 20-23 final stage
+ mova m0, [stp + %3 + idx20]
+ mova m1, [stp + %3 + idx21]
+ mova m2, [stp + %3 + idx22]
+ mova m3, [stp + %3 + idx23]
+ SUM_SUB 7, 0, 9 ; stp1_11, stp_20
+ SUM_SUB 6, 1, 9 ; stp1_10, stp_21
+ SUM_SUB 5, 2, 9 ; stp1_9, stp_22
+ SUM_SUB 4, 3, 9 ; stp1_8, stp_23
+ mova [stp + %2 + idx8], m4
+ mova [stp + %2 + idx9], m5
+ mova [stp + %2 + idx10], m6
+ mova [stp + %2 + idx11], m7
+ mova [stp + %3 + idx20], m0
+ mova [stp + %3 + idx21], m1
+ mova [stp + %3 + idx22], m2
+ mova [stp + %3 + idx23], m3
+%endmacro
+
+%macro RECON_AND_STORE 1
+ mova m11, [pw_32]
+ lea stp, [rsp + %1]
+ mov r6, 32
+ pxor m8, m8
+%%recon_and_store:
+ mova m0, [stp + 16 * 32 * 0]
+ mova m1, [stp + 16 * 32 * 1]
+ mova m2, [stp + 16 * 32 * 2]
+ mova m3, [stp + 16 * 32 * 3]
+ add stp, 16
+
+ paddw m0, m11
+ paddw m1, m11
+ paddw m2, m11
+ paddw m3, m11
+ psraw m0, 6
+ psraw m1, 6
+ psraw m2, 6
+ psraw m3, 6
+ movh m4, [outputq + 0]
+ movh m5, [outputq + 8]
+ movh m6, [outputq + 16]
+ movh m7, [outputq + 24]
+ punpcklbw m4, m8
+ punpcklbw m5, m8
+ punpcklbw m6, m8
+ punpcklbw m7, m8
+ paddw m0, m4
+ paddw m1, m5
+ paddw m2, m6
+ paddw m3, m7
+ packuswb m0, m1
+ packuswb m2, m3
+ mova [outputq + 0], m0
+ mova [outputq + 16], m2
+ lea outputq, [outputq + strideq]
+ dec r6
+ jnz %%recon_and_store
+%endmacro
+
+%define i32x32_size 16*32*5
+%define pass_two_start 16*32*0
+%define transposed_in 16*32*4
+%define pass_one_start 16*32*0
+%define stp r8
+
+INIT_XMM ssse3
+cglobal idct32x32_34_add, 3, 11, 16, i32x32_size, input, output, stride
+ mova m8, [pd_8192]
+ lea stp, [rsp + pass_one_start]
+
+idct32x32_34:
+ mov r3, inputq
+ lea r4, [rsp + transposed_in]
+
+idct32x32_34_transpose:
+ mova m0, [r3 + 0]
+ mova m1, [r3 + 16 * 4]
+ mova m2, [r3 + 16 * 8]
+ mova m3, [r3 + 16 * 12]
+ mova m4, [r3 + 16 * 16]
+ mova m5, [r3 + 16 * 20]
+ mova m6, [r3 + 16 * 24]
+ mova m7, [r3 + 16 * 28]
+
+ TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9
+
+ IDCT32X32_34x 16*0, 16*32, 16*64, 16*96
+ lea stp, [stp + 16 * 8]
+ mov r6, 4
+ lea stp, [rsp + pass_one_start]
+ lea r9, [rsp + pass_one_start]
+
+idct32x32_34_2:
+ lea r4, [rsp + transposed_in]
+ mov r3, r9
+
+idct32x32_34_transpose_2:
+ mova m0, [r3 + 0]
+ mova m1, [r3 + 16 * 1]
+ mova m2, [r3 + 16 * 2]
+ mova m3, [r3 + 16 * 3]
+ mova m4, [r3 + 16 * 4]
+ mova m5, [r3 + 16 * 5]
+ mova m6, [r3 + 16 * 6]
+ mova m7, [r3 + 16 * 7]
+
+ TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9
+
+ IDCT32X32_34x 16*0, 16*8, 16*16, 16*24
+
+ lea stp, [stp + 16 * 32]
+ add r9, 16 * 32
+ dec r6
+ jnz idct32x32_34_2
+
+ RECON_AND_STORE pass_two_start
+
+ RET
%endif
