Merge "bitreader/writer: Change shift to signed"
diff --git a/test/datarate_test.cc b/test/datarate_test.cc
index 7d8a0ab..cec6d09 100644
--- a/test/datarate_test.cc
+++ b/test/datarate_test.cc
@@ -770,7 +770,7 @@
::libvpx_test::Encoder *encoder) {
if (video->frame() == 0) {
int i;
- for (i = 0; i < 2; ++i) {
+ for (i = 0; i < VPX_MAX_LAYERS; ++i) {
svc_params_.max_quantizers[i] = 63;
svc_params_.min_quantizers[i] = 0;
}
diff --git a/vp9/common/vp9_entropy.c b/vp9/common/vp9_entropy.c
index 1c81581..fc02209 100644
--- a/vp9/common/vp9_entropy.c
+++ b/vp9/common/vp9_entropy.c
@@ -36,20 +36,6 @@
254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129
};
#if CONFIG_VP9_HIGHBITDEPTH
-const vpx_prob vp9_cat1_prob_high10[] = { 159 };
-const vpx_prob vp9_cat2_prob_high10[] = { 165, 145 };
-const vpx_prob vp9_cat3_prob_high10[] = { 173, 148, 140 };
-const vpx_prob vp9_cat4_prob_high10[] = { 176, 155, 140, 135 };
-const vpx_prob vp9_cat5_prob_high10[] = { 180, 157, 141, 134, 130 };
-const vpx_prob vp9_cat6_prob_high10[] = {
- 255, 255, 254, 254, 254, 252, 249, 243,
- 230, 196, 177, 153, 140, 133, 130, 129
-};
-const vpx_prob vp9_cat1_prob_high12[] = { 159 };
-const vpx_prob vp9_cat2_prob_high12[] = { 165, 145 };
-const vpx_prob vp9_cat3_prob_high12[] = { 173, 148, 140 };
-const vpx_prob vp9_cat4_prob_high12[] = { 176, 155, 140, 135 };
-const vpx_prob vp9_cat5_prob_high12[] = { 180, 157, 141, 134, 130 };
const vpx_prob vp9_cat6_prob_high12[] = {
255, 255, 255, 255, 254, 254, 254, 252, 249,
243, 230, 196, 177, 153, 140, 133, 130, 129
diff --git a/vp9/decoder/vp9_decodeframe.c b/vp9/decoder/vp9_decodeframe.c
index f191663..39e4dcf 100644
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -1979,6 +1979,8 @@
if (!cm->error_resilient_mode) {
cm->refresh_frame_context = vpx_rb_read_bit(rb);
cm->frame_parallel_decoding_mode = vpx_rb_read_bit(rb);
+ if (!cm->frame_parallel_decoding_mode)
+ vp9_zero(cm->counts);
} else {
cm->refresh_frame_context = 0;
cm->frame_parallel_decoding_mode = 1;
@@ -2082,43 +2084,6 @@
return vpx_reader_has_error(&r);
}
-#ifdef NDEBUG
-#define debug_check_frame_counts(cm) (void)0
-#else // !NDEBUG
-// Counts should only be incremented when frame_parallel_decoding_mode and
-// error_resilient_mode are disabled.
-static void debug_check_frame_counts(const VP9_COMMON *const cm) {
- FRAME_COUNTS zero_counts;
- vp9_zero(zero_counts);
- assert(cm->frame_parallel_decoding_mode || cm->error_resilient_mode);
- assert(!memcmp(cm->counts.y_mode, zero_counts.y_mode,
- sizeof(cm->counts.y_mode)));
- assert(!memcmp(cm->counts.uv_mode, zero_counts.uv_mode,
- sizeof(cm->counts.uv_mode)));
- assert(!memcmp(cm->counts.partition, zero_counts.partition,
- sizeof(cm->counts.partition)));
- assert(!memcmp(cm->counts.coef, zero_counts.coef,
- sizeof(cm->counts.coef)));
- assert(!memcmp(cm->counts.eob_branch, zero_counts.eob_branch,
- sizeof(cm->counts.eob_branch)));
- assert(!memcmp(cm->counts.switchable_interp, zero_counts.switchable_interp,
- sizeof(cm->counts.switchable_interp)));
- assert(!memcmp(cm->counts.inter_mode, zero_counts.inter_mode,
- sizeof(cm->counts.inter_mode)));
- assert(!memcmp(cm->counts.intra_inter, zero_counts.intra_inter,
- sizeof(cm->counts.intra_inter)));
- assert(!memcmp(cm->counts.comp_inter, zero_counts.comp_inter,
- sizeof(cm->counts.comp_inter)));
- assert(!memcmp(cm->counts.single_ref, zero_counts.single_ref,
- sizeof(cm->counts.single_ref)));
- assert(!memcmp(cm->counts.comp_ref, zero_counts.comp_ref,
- sizeof(cm->counts.comp_ref)));
- assert(!memcmp(&cm->counts.tx, &zero_counts.tx, sizeof(cm->counts.tx)));
- assert(!memcmp(cm->counts.skip, zero_counts.skip, sizeof(cm->counts.skip)));
- assert(!memcmp(&cm->counts.mv, &zero_counts.mv, sizeof(cm->counts.mv)));
-}
-#endif // NDEBUG
-
static struct vpx_read_bit_buffer *init_read_bit_buffer(
VP9Decoder *pbi,
struct vpx_read_bit_buffer *rb,
@@ -2202,8 +2167,6 @@
vpx_internal_error(&cm->error, VPX_CODEC_CORRUPT_FRAME,
"Uninitialized entropy context.");
- vp9_zero(cm->counts);
-
xd->corrupted = 0;
new_fb->corrupted = read_compressed_header(pbi, data, first_partition_size);
if (new_fb->corrupted)
@@ -2259,8 +2222,6 @@
vp9_adapt_mode_probs(cm);
vp9_adapt_mv_probs(cm, cm->allow_high_precision_mv);
}
- } else {
- debug_check_frame_counts(cm);
}
} else {
vpx_internal_error(&cm->error, VPX_CODEC_CORRUPT_FRAME,
diff --git a/vp9/decoder/vp9_detokenize.c b/vp9/decoder/vp9_detokenize.c
index 5912365..073cbae 100644
--- a/vp9/decoder/vp9_detokenize.c
+++ b/vp9/decoder/vp9_detokenize.c
@@ -65,52 +65,24 @@
const int dq_shift = (tx_size == TX_32X32);
int v, token;
int16_t dqv = dq[0];
- const uint8_t *cat1_prob;
- const uint8_t *cat2_prob;
- const uint8_t *cat3_prob;
- const uint8_t *cat4_prob;
- const uint8_t *cat5_prob;
- const uint8_t *cat6_prob;
+ const uint8_t *const cat6_prob =
+#if CONFIG_VP9_HIGHBITDEPTH
+ (xd->bd == VPX_BITS_12) ? vp9_cat6_prob_high12 :
+ (xd->bd == VPX_BITS_10) ? vp9_cat6_prob_high12 + 2 :
+#endif // CONFIG_VP9_HIGHBITDEPTH
+ vp9_cat6_prob;
+ const int cat6_bits =
+#if CONFIG_VP9_HIGHBITDEPTH
+ (xd->bd == VPX_BITS_12) ? 18 :
+ (xd->bd == VPX_BITS_10) ? 16 :
+#endif // CONFIG_VP9_HIGHBITDEPTH
+ 14;
if (counts) {
coef_counts = counts->coef[tx_size][type][ref];
eob_branch_count = counts->eob_branch[tx_size][type][ref];
}
-#if CONFIG_VP9_HIGHBITDEPTH
- if (xd->bd > VPX_BITS_8) {
- if (xd->bd == VPX_BITS_10) {
- cat1_prob = vp9_cat1_prob_high10;
- cat2_prob = vp9_cat2_prob_high10;
- cat3_prob = vp9_cat3_prob_high10;
- cat4_prob = vp9_cat4_prob_high10;
- cat5_prob = vp9_cat5_prob_high10;
- cat6_prob = vp9_cat6_prob_high10;
- } else {
- cat1_prob = vp9_cat1_prob_high12;
- cat2_prob = vp9_cat2_prob_high12;
- cat3_prob = vp9_cat3_prob_high12;
- cat4_prob = vp9_cat4_prob_high12;
- cat5_prob = vp9_cat5_prob_high12;
- cat6_prob = vp9_cat6_prob_high12;
- }
- } else {
- cat1_prob = vp9_cat1_prob;
- cat2_prob = vp9_cat2_prob;
- cat3_prob = vp9_cat3_prob;
- cat4_prob = vp9_cat4_prob;
- cat5_prob = vp9_cat5_prob;
- cat6_prob = vp9_cat6_prob;
- }
-#else
- cat1_prob = vp9_cat1_prob;
- cat2_prob = vp9_cat2_prob;
- cat3_prob = vp9_cat3_prob;
- cat4_prob = vp9_cat4_prob;
- cat5_prob = vp9_cat5_prob;
- cat6_prob = vp9_cat6_prob;
-#endif
-
while (c < max_eob) {
int val = -1;
band = *band_translate++;
@@ -149,39 +121,22 @@
val = token;
break;
case CATEGORY1_TOKEN:
- val = CAT1_MIN_VAL + read_coeff(cat1_prob, 1, r);
+ val = CAT1_MIN_VAL + read_coeff(vp9_cat1_prob, 1, r);
break;
case CATEGORY2_TOKEN:
- val = CAT2_MIN_VAL + read_coeff(cat2_prob, 2, r);
+ val = CAT2_MIN_VAL + read_coeff(vp9_cat2_prob, 2, r);
break;
case CATEGORY3_TOKEN:
- val = CAT3_MIN_VAL + read_coeff(cat3_prob, 3, r);
+ val = CAT3_MIN_VAL + read_coeff(vp9_cat3_prob, 3, r);
break;
case CATEGORY4_TOKEN:
- val = CAT4_MIN_VAL + read_coeff(cat4_prob, 4, r);
+ val = CAT4_MIN_VAL + read_coeff(vp9_cat4_prob, 4, r);
break;
case CATEGORY5_TOKEN:
- val = CAT5_MIN_VAL + read_coeff(cat5_prob, 5, r);
+ val = CAT5_MIN_VAL + read_coeff(vp9_cat5_prob, 5, r);
break;
case CATEGORY6_TOKEN:
-#if CONFIG_VP9_HIGHBITDEPTH
- switch (xd->bd) {
- case VPX_BITS_8:
- val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
- break;
- case VPX_BITS_10:
- val = CAT6_MIN_VAL + read_coeff(cat6_prob, 16, r);
- break;
- case VPX_BITS_12:
- val = CAT6_MIN_VAL + read_coeff(cat6_prob, 18, r);
- break;
- default:
- assert(0);
- return -1;
- }
-#else
- val = CAT6_MIN_VAL + read_coeff(cat6_prob, 14, r);
-#endif
+ val = CAT6_MIN_VAL + read_coeff(cat6_prob, cat6_bits, r);
break;
}
}
diff --git a/vp9/encoder/vp9_context_tree.h b/vp9/encoder/vp9_context_tree.h
index 8e365ce..86ba03d 100644
--- a/vp9/encoder/vp9_context_tree.h
+++ b/vp9/encoder/vp9_context_tree.h
@@ -60,6 +60,7 @@
#if CONFIG_VP9_TEMPORAL_DENOISING
unsigned int newmv_sse;
unsigned int zeromv_sse;
+ unsigned int zeromv_lastref_sse;
PREDICTION_MODE best_sse_inter_mode;
int_mv best_sse_mv;
MV_REFERENCE_FRAME best_reference_frame;
diff --git a/vp9/encoder/vp9_denoiser.c b/vp9/encoder/vp9_denoiser.c
index c382b77..e87a12e 100644
--- a/vp9/encoder/vp9_denoiser.c
+++ b/vp9/encoder/vp9_denoiser.c
@@ -224,14 +224,19 @@
} else {
// Otherwise, use the zero reference frame.
frame = ctx->best_zeromv_reference_frame;
-
- mbmi->ref_frame[0] = ctx->best_zeromv_reference_frame;
+ ctx->newmv_sse = ctx->zeromv_sse;
+ // Bias to last reference.
+ if (frame != LAST_FRAME &&
+ ((ctx->zeromv_lastref_sse < (5 * ctx->zeromv_sse) >> 2) ||
+ denoiser->denoising_level >= kDenHigh)) {
+ frame = LAST_FRAME;
+ ctx->newmv_sse = ctx->zeromv_lastref_sse;
+ }
+ mbmi->ref_frame[0] = frame;
mbmi->mode = ZEROMV;
mbmi->mv[0].as_int = 0;
-
ctx->best_sse_inter_mode = ZEROMV;
ctx->best_sse_mv.as_int = 0;
- ctx->newmv_sse = ctx->zeromv_sse;
}
if (ctx->newmv_sse > sse_thresh(bs, increase_denoising)) {
@@ -462,6 +467,8 @@
if (mbmi->mv[0].as_int == 0 && sse < ctx->zeromv_sse) {
ctx->zeromv_sse = sse;
ctx->best_zeromv_reference_frame = mbmi->ref_frame[0];
+ if (mbmi->ref_frame[0] == LAST_FRAME)
+ ctx->zeromv_lastref_sse = sse;
}
if (mbmi->mv[0].as_int != 0 && sse < ctx->newmv_sse) {
diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c
index 0377cb5..fdff363 100644
--- a/vp9/encoder/vp9_ratectrl.c
+++ b/vp9/encoder/vp9_ratectrl.c
@@ -1558,7 +1558,7 @@
VP9_COMMON *const cm = &cpi->common;
RATE_CONTROL *const rc = &cpi->rc;
int target = rc->avg_frame_bandwidth;
- const int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
+ int layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
cpi->svc.temporal_layer_id, cpi->svc.number_temporal_layers);
if ((cm->current_video_frame == 0) ||
@@ -1573,8 +1573,10 @@
cpi->ref_frame_flags &=
(~VP9_LAST_FLAG & ~VP9_GOLD_FLAG & ~VP9_ALT_FLAG);
} else if (is_one_pass_cbr_svc(cpi)) {
- cpi->svc.layer_context[layer].is_key_frame = 1;
reset_temporal_layer_to_zero(cpi);
+ layer = LAYER_IDS_TO_IDX(cpi->svc.spatial_layer_id,
+ cpi->svc.temporal_layer_id, cpi->svc.number_temporal_layers);
+ cpi->svc.layer_context[layer].is_key_frame = 1;
cpi->ref_frame_flags &=
(~VP9_LAST_FLAG & ~VP9_GOLD_FLAG & ~VP9_ALT_FLAG);
// Assumption here is that LAST_FRAME is being updated for a keyframe.
diff --git a/vp9/encoder/vp9_tokenize.c b/vp9/encoder/vp9_tokenize.c
index 6076e2a..010903a 100644
--- a/vp9/encoder/vp9_tokenize.c
+++ b/vp9/encoder/vp9_tokenize.c
@@ -375,32 +375,32 @@
#if CONFIG_VP9_HIGHBITDEPTH
const vp9_extra_bit vp9_extra_bits_high10[ENTROPY_TOKENS] = {
- {0, 0, 0, zero_cost}, // ZERO
- {0, 0, 1, one_cost}, // ONE
- {0, 0, 2, two_cost}, // TWO
- {0, 0, 3, three_cost}, // THREE
- {0, 0, 4, four_cost}, // FOUR
- {vp9_cat1_prob_high10, 1, CAT1_MIN_VAL, cat1_cost}, // CAT1
- {vp9_cat2_prob_high10, 2, CAT2_MIN_VAL, cat2_cost}, // CAT2
- {vp9_cat3_prob_high10, 3, CAT3_MIN_VAL, cat3_cost}, // CAT3
- {vp9_cat4_prob_high10, 4, CAT4_MIN_VAL, cat4_cost}, // CAT4
- {vp9_cat5_prob_high10, 5, CAT5_MIN_VAL, cat5_cost}, // CAT5
- {vp9_cat6_prob_high10, 16, CAT6_MIN_VAL, 0}, // CAT6
- {0, 0, 0, zero_cost} // EOB
+ {0, 0, 0, zero_cost}, // ZERO
+ {0, 0, 1, one_cost}, // ONE
+ {0, 0, 2, two_cost}, // TWO
+ {0, 0, 3, three_cost}, // THREE
+ {0, 0, 4, four_cost}, // FOUR
+ {vp9_cat1_prob, 1, CAT1_MIN_VAL, cat1_cost}, // CAT1
+ {vp9_cat2_prob, 2, CAT2_MIN_VAL, cat2_cost}, // CAT2
+ {vp9_cat3_prob, 3, CAT3_MIN_VAL, cat3_cost}, // CAT3
+ {vp9_cat4_prob, 4, CAT4_MIN_VAL, cat4_cost}, // CAT4
+ {vp9_cat5_prob, 5, CAT5_MIN_VAL, cat5_cost}, // CAT5
+ {vp9_cat6_prob_high12 + 2, 16, CAT6_MIN_VAL, 0}, // CAT6
+ {0, 0, 0, zero_cost} // EOB
};
const vp9_extra_bit vp9_extra_bits_high12[ENTROPY_TOKENS] = {
- {0, 0, 0, zero_cost}, // ZERO
- {0, 0, 1, one_cost}, // ONE
- {0, 0, 2, two_cost}, // TWO
- {0, 0, 3, three_cost}, // THREE
- {0, 0, 4, four_cost}, // FOUR
- {vp9_cat1_prob_high12, 1, CAT1_MIN_VAL, cat1_cost}, // CAT1
- {vp9_cat2_prob_high12, 2, CAT2_MIN_VAL, cat2_cost}, // CAT2
- {vp9_cat3_prob_high12, 3, CAT3_MIN_VAL, cat3_cost}, // CAT3
- {vp9_cat4_prob_high12, 4, CAT4_MIN_VAL, cat4_cost}, // CAT4
- {vp9_cat5_prob_high12, 5, CAT5_MIN_VAL, cat5_cost}, // CAT5
- {vp9_cat6_prob_high12, 18, CAT6_MIN_VAL, 0}, // CAT6
- {0, 0, 0, zero_cost} // EOB
+ {0, 0, 0, zero_cost}, // ZERO
+ {0, 0, 1, one_cost}, // ONE
+ {0, 0, 2, two_cost}, // TWO
+ {0, 0, 3, three_cost}, // THREE
+ {0, 0, 4, four_cost}, // FOUR
+ {vp9_cat1_prob, 1, CAT1_MIN_VAL, cat1_cost}, // CAT1
+ {vp9_cat2_prob, 2, CAT2_MIN_VAL, cat2_cost}, // CAT2
+ {vp9_cat3_prob, 3, CAT3_MIN_VAL, cat3_cost}, // CAT3
+ {vp9_cat4_prob, 4, CAT4_MIN_VAL, cat4_cost}, // CAT4
+ {vp9_cat5_prob, 5, CAT5_MIN_VAL, cat5_cost}, // CAT5
+ {vp9_cat6_prob_high12, 18, CAT6_MIN_VAL, 0}, // CAT6
+ {0, 0, 0, zero_cost} // EOB
};
#endif
diff --git a/vp9/vp9_cx_iface.c b/vp9/vp9_cx_iface.c
index 6ccba0f..db7f537 100644
--- a/vp9/vp9_cx_iface.c
+++ b/vp9/vp9_cx_iface.c
@@ -1393,8 +1393,8 @@
LAYER_IDS_TO_IDX(sl, tl, cpi->svc.number_temporal_layers);
LAYER_CONTEXT *lc =
&cpi->svc.layer_context[layer];
- lc->max_q = params->max_quantizers[sl];
- lc->min_q = params->min_quantizers[sl];
+ lc->max_q = params->max_quantizers[layer];
+ lc->min_q = params->min_quantizers[layer];
lc->scaling_factor_num = params->scaling_factor_num[sl];
lc->scaling_factor_den = params->scaling_factor_den[sl];
}
diff --git a/vpx/src/svc_encodeframe.c b/vpx/src/svc_encodeframe.c
index ff7c10a..68d1d8d 100644
--- a/vpx/src/svc_encodeframe.c
+++ b/vpx/src/svc_encodeframe.c
@@ -385,7 +385,7 @@
vpx_codec_iface_t *iface,
vpx_codec_enc_cfg_t *enc_cfg) {
vpx_codec_err_t res;
- int i;
+ int i, sl , tl;
SvcInternal_t *const si = get_svc_internal(svc_ctx);
if (svc_ctx == NULL || codec_ctx == NULL || iface == NULL ||
enc_cfg == NULL) {
@@ -423,11 +423,14 @@
svc_ctx->temporal_layers = 2;
}
- for (i = 0; i < VPX_SS_MAX_LAYERS; ++i) {
- si->svc_params.max_quantizers[i] = MAX_QUANTIZER;
- si->svc_params.min_quantizers[i] = 0;
- si->svc_params.scaling_factor_num[i] = DEFAULT_SCALE_FACTORS_NUM[i];
- si->svc_params.scaling_factor_den[i] = DEFAULT_SCALE_FACTORS_DEN[i];
+ for (tl = 0; tl < svc_ctx->temporal_layers; ++tl) {
+ for (sl = 0; sl < svc_ctx->spatial_layers; ++sl) {
+ i = sl * svc_ctx->temporal_layers + tl;
+ si->svc_params.max_quantizers[i] = MAX_QUANTIZER;
+ si->svc_params.min_quantizers[i] = 0;
+ si->svc_params.scaling_factor_num[sl] = DEFAULT_SCALE_FACTORS_NUM[sl];
+ si->svc_params.scaling_factor_den[sl] = DEFAULT_SCALE_FACTORS_DEN[sl];
+ }
}
// Parse aggregate command line options. Options must start with
diff --git a/vpx_dsp/vpx_dsp_rtcd_defs.pl b/vpx_dsp/vpx_dsp_rtcd_defs.pl
index 03dcd4e..4de85a4 100644
--- a/vpx_dsp/vpx_dsp_rtcd_defs.pl
+++ b/vpx_dsp/vpx_dsp_rtcd_defs.pl
@@ -893,7 +893,7 @@
specialize qw/vpx_idct32x32_1024_add sse2 neon dspr2 msa/;
add_proto qw/void vpx_idct32x32_34_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
- specialize qw/vpx_idct32x32_34_add sse2 neon_asm dspr2 msa/;
+ specialize qw/vpx_idct32x32_34_add sse2 neon_asm dspr2 msa/, "$ssse3_x86_64_x86inc";
# Need to add 34 eob idct32x32 neon implementation.
$vpx_idct32x32_34_add_neon_asm=vpx_idct32x32_1024_add_neon;
diff --git a/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm b/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
index 68e7fa4..2675eab 100644
--- a/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
+++ b/vpx_dsp/x86/inv_txfm_ssse3_x86_64.asm
@@ -17,18 +17,46 @@
SECTION_RODATA
pw_11585x2: times 8 dw 23170
+
+pw_m2404x2: times 8 dw -2404*2
+pw_m4756x2: times 8 dw -4756*2
+pw_m5520x2: times 8 dw -5520*2
+
+pw_16364x2: times 8 dw 16364*2
+pw_16305x2: times 8 dw 16305*2
+pw_16207x2: times 8 dw 16207*2
+pw_16069x2: times 8 dw 16069*2
+pw_15893x2: times 8 dw 15893*2
+pw_15679x2: times 8 dw 15679*2
+pw_15426x2: times 8 dw 15426*2
+pw__3981x2: times 8 dw 3981*2
+pw__3196x2: times 8 dw 3196*2
+pw__1606x2: times 8 dw 1606*2
+pw___804x2: times 8 dw 804*2
+
pd_8192: times 4 dd 8192
+pw_32: times 8 dw 32
pw_16: times 8 dw 16
%macro TRANSFORM_COEFFS 2
pw_%1_%2: dw %1, %2, %1, %2, %1, %2, %1, %2
pw_m%2_%1: dw -%2, %1, -%2, %1, -%2, %1, -%2, %1
+pw_m%1_m%2: dw -%1, -%2, -%1, -%2, -%1, -%2, -%1, -%2
%endmacro
TRANSFORM_COEFFS 6270, 15137
TRANSFORM_COEFFS 3196, 16069
TRANSFORM_COEFFS 13623, 9102
+; constants for 32x32_34
+TRANSFORM_COEFFS 804, 16364
+TRANSFORM_COEFFS 15426, 5520
+TRANSFORM_COEFFS 3981, 15893
+TRANSFORM_COEFFS 16207, 2404
+TRANSFORM_COEFFS 1606, 16305
+TRANSFORM_COEFFS 15679, 4756
+TRANSFORM_COEFFS 11585, 11585
+
%macro PAIR_PP_COEFFS 2
dpw_%1_%2: dw %1, %1, %1, %1, %2, %2, %2, %2
%endmacro
@@ -80,6 +108,15 @@
packssdw m%2, m%6
%endmacro
+%macro BUTTERFLY_4Xmm 7 ; dst1, dst2, coef1, coef2, round, tmp1, tmp2
+ punpckhwd m%6, m%2, m%1
+ MUL_ADD_2X %7, %6, %6, %5, [pw_m%4_%3], [pw_m%3_m%4]
+ punpcklwd m%2, m%1
+ MUL_ADD_2X %1, %2, %2, %5, [pw_m%4_%3], [pw_m%3_m%4]
+ packssdw m%1, m%7
+ packssdw m%2, m%6
+%endmacro
+
; matrix transpose
%macro INTERLEAVE_2X 4
punpckh%1 m%4, m%2, m%3
@@ -298,4 +335,452 @@
RET
+%define idx0 16 * 0
+%define idx1 16 * 1
+%define idx2 16 * 2
+%define idx3 16 * 3
+%define idx4 16 * 4
+%define idx5 16 * 5
+%define idx6 16 * 6
+%define idx7 16 * 7
+%define idx8 16 * 0
+%define idx9 16 * 1
+%define idx10 16 * 2
+%define idx11 16 * 3
+%define idx12 16 * 4
+%define idx13 16 * 5
+%define idx14 16 * 6
+%define idx15 16 * 7
+%define idx16 16 * 0
+%define idx17 16 * 1
+%define idx18 16 * 2
+%define idx19 16 * 3
+%define idx20 16 * 4
+%define idx21 16 * 5
+%define idx22 16 * 6
+%define idx23 16 * 7
+%define idx24 16 * 0
+%define idx25 16 * 1
+%define idx26 16 * 2
+%define idx27 16 * 3
+%define idx28 16 * 4
+%define idx29 16 * 5
+%define idx30 16 * 6
+%define idx31 16 * 7
+
+%macro IDCT32X32_34 4
+ ; FROM idct32x32_add_neon.asm
+ ;
+ ; Instead of doing the transforms stage by stage, it is done by loading
+ ; some input values and doing as many stages as possible to minimize the
+ ; storing/loading of intermediate results. To fit within registers, the
+ ; final coefficients are cut into four blocks:
+ ; BLOCK A: 16-19,28-31
+ ; BLOCK B: 20-23,24-27
+ ; BLOCK C: 8-11,12-15
+ ; BLOCK D: 0-3,4-7
+ ; Blocks A and C are straight calculation through the various stages. In
+ ; block B, further calculations are performed using the results from
+ ; block A. In block D, further calculations are performed using the results
+ ; from block C and then the final calculations are done using results from
+ ; block A and B which have been combined at the end of block B.
+ ;
+ ; BLOCK A STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m11, m1
+ pmulhrsw m1, [pw___804x2] ; stp1_16
+ mova [r4 + 0], m0
+ pmulhrsw m11, [pw_16364x2] ; stp2_31
+ mova [r4 + 16 * 2], m2
+ mova m12, m7
+ pmulhrsw m7, [pw_15426x2] ; stp1_28
+ mova [r4 + 16 * 4], m4
+ pmulhrsw m12, [pw_m5520x2] ; stp2_19
+ mova [r4 + 16 * 6], m6
+
+ ; BLOCK A STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m2, m1 ; stp1_16
+ mova m0, m11 ; stp1_31
+ mova m4, m7 ; stp1_28
+ mova m15, m12 ; stp1_19
+
+ ; BLOCK A STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 0, 2, 3196, 16069, m8, 9, 10 ; stp1_17, stp1_30
+ BUTTERFLY_4Xmm 4, 15, 3196, 16069, m8, 9, 10 ; stp1_29, stp1_18
+
+ ; BLOCK A STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 1, 12, 9 ; stp2_16, stp2_19
+ SUM_SUB 0, 15, 9 ; stp2_17, stp2_18
+ SUM_SUB 11, 7, 9 ; stp2_31, stp2_28
+ SUM_SUB 2, 4, 9 ; stp2_30, stp2_29
+
+ ; BLOCK A STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 4, 15, 6270, 15137, m8, 9, 10 ; stp1_18, stp1_29
+ BUTTERFLY_4X 7, 12, 6270, 15137, m8, 9, 10 ; stp1_19, stp1_28
+
+ ; BLOCK B STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m6, m5
+ pmulhrsw m5, [pw__3981x2] ; stp1_20
+ mova [stp + %4 + idx28], m12
+ mova [stp + %4 + idx29], m15
+ pmulhrsw m6, [pw_15893x2] ; stp2_27
+ mova [stp + %4 + idx30], m2
+ mova m2, m3
+ pmulhrsw m3, [pw_m2404x2] ; stp1_23
+ mova [stp + %4 + idx31], m11
+ pmulhrsw m2, [pw_16207x2] ; stp2_24
+
+ ; BLOCK B STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m13, m5 ; stp1_20
+ mova m14, m6 ; stp1_27
+ mova m15, m3 ; stp1_23
+ mova m11, m2 ; stp1_24
+
+ ; BLOCK B STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 14, 13, 13623, 9102, m8, 9, 10 ; stp1_21, stp1_26
+ BUTTERFLY_4Xmm 11, 15, 13623, 9102, m8, 9, 10 ; stp1_25, stp1_22
+
+ ; BLOCK B STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 3, 5, 9 ; stp2_23, stp2_20
+ SUM_SUB 15, 14, 9 ; stp2_22, stp2_21
+ SUM_SUB 2, 6, 9 ; stp2_24, stp2_27
+ SUM_SUB 11, 13, 9 ; stp2_25, stp2_26
+
+ ; BLOCK B STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4Xmm 6, 5, 6270, 15137, m8, 9, 10 ; stp1_27, stp1_20
+ BUTTERFLY_4Xmm 13, 14, 6270, 15137, m8, 9, 10 ; stp1_26, stp1_21
+
+ ; BLOCK B STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 1, 3, 9 ; stp2_16, stp2_23
+ SUM_SUB 0, 15, 9 ; stp2_17, stp2_22
+ SUM_SUB 4, 14, 9 ; stp2_18, stp2_21
+ SUM_SUB 7, 5, 9 ; stp2_19, stp2_20
+ mova [stp + %3 + idx16], m1
+ mova [stp + %3 + idx17], m0
+ mova [stp + %3 + idx18], m4
+ mova [stp + %3 + idx19], m7
+
+ mova m4, [stp + %4 + idx28]
+ mova m7, [stp + %4 + idx29]
+ mova m10, [stp + %4 + idx30]
+ mova m12, [stp + %4 + idx31]
+ SUM_SUB 4, 6, 9 ; stp2_28, stp2_27
+ SUM_SUB 7, 13, 9 ; stp2_29, stp2_26
+ SUM_SUB 10, 11, 9 ; stp2_30, stp2_25
+ SUM_SUB 12, 2, 9 ; stp2_31, stp2_24
+ mova [stp + %4 + idx28], m4
+ mova [stp + %4 + idx29], m7
+ mova [stp + %4 + idx30], m10
+ mova [stp + %4 + idx31], m12
+
+ ; BLOCK B STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%if 0 ; overflow occurs in SUM_SUB when using test streams
+ mova m10, [pw_11585x2]
+ SUM_SUB 6, 5, 9
+ pmulhrsw m6, m10 ; stp1_27
+ pmulhrsw m5, m10 ; stp1_20
+ SUM_SUB 13, 14, 9
+ pmulhrsw m13, m10 ; stp1_26
+ pmulhrsw m14, m10 ; stp1_21
+ SUM_SUB 11, 15, 9
+ pmulhrsw m11, m10 ; stp1_25
+ pmulhrsw m15, m10 ; stp1_22
+ SUM_SUB 2, 3, 9
+ pmulhrsw m2, m10 ; stp1_24
+ pmulhrsw m3, m10 ; stp1_23
+%else
+ BUTTERFLY_4X 6, 5, 11585, 11585, m8, 9, 10 ; stp1_20, stp1_27
+ SWAP 6, 5
+ BUTTERFLY_4X 13, 14, 11585, 11585, m8, 9, 10 ; stp1_21, stp1_26
+ SWAP 13, 14
+ BUTTERFLY_4X 11, 15, 11585, 11585, m8, 9, 10 ; stp1_22, stp1_25
+ SWAP 11, 15
+ BUTTERFLY_4X 2, 3, 11585, 11585, m8, 9, 10 ; stp1_23, stp1_24
+ SWAP 2, 3
+%endif
+
+ mova [stp + %4 + idx24], m2
+ mova [stp + %4 + idx25], m11
+ mova [stp + %4 + idx26], m13
+ mova [stp + %4 + idx27], m6
+
+ ; BLOCK C STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ ;
+ ; BLOCK C STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m0, [rsp + transposed_in + 16 * 2]
+ mova m6, [rsp + transposed_in + 16 * 6]
+
+ mova m1, m0
+ pmulhrsw m0, [pw__1606x2] ; stp1_8
+ mova [stp + %3 + idx20], m5
+ mova [stp + %3 + idx21], m14
+ pmulhrsw m1, [pw_16305x2] ; stp2_15
+ mova [stp + %3 + idx22], m15
+ mova m7, m6
+ pmulhrsw m7, [pw_m4756x2] ; stp2_11
+ mova [stp + %3 + idx23], m3
+ pmulhrsw m6, [pw_15679x2] ; stp1_12
+
+ ; BLOCK C STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m3, m0 ; stp1_8
+ mova m2, m1 ; stp1_15
+
+ ; BLOCK C STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ BUTTERFLY_4X 2, 3, 6270, 15137, m8, 9, 10 ; stp1_9, stp1_14
+ mova m4, m7 ; stp1_11
+ mova m5, m6 ; stp1_12
+ BUTTERFLY_4Xmm 5, 4, 6270, 15137, m8, 9, 10 ; stp1_13, stp1_10
+
+ ; BLOCK C STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 0, 7, 9 ; stp1_8, stp1_11
+ SUM_SUB 2, 4, 9 ; stp1_9, stp1_10
+ SUM_SUB 1, 6, 9 ; stp1_15, stp1_12
+ SUM_SUB 3, 5, 9 ; stp1_14, stp1_13
+
+ ; BLOCK C STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%if 0 ; overflow occurs in SUM_SUB when using test streams
+ mova m10, [pw_11585x2]
+ SUM_SUB 5, 4, 9
+ pmulhrsw m5, m10 ; stp1_13
+ pmulhrsw m4, m10 ; stp1_10
+ SUM_SUB 6, 7, 9
+ pmulhrsw m6, m10 ; stp1_12
+ pmulhrsw m7, m10 ; stp1_11
+%else
+ BUTTERFLY_4X 5, 4, 11585, 11585, m8, 9, 10 ; stp1_10, stp1_13
+ SWAP 5, 4
+ BUTTERFLY_4X 6, 7, 11585, 11585, m8, 9, 10 ; stp1_11, stp1_12
+ SWAP 6, 7
+%endif
+
+ ; BLOCK C STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova [stp + %2 + idx8], m0
+ mova [stp + %2 + idx9], m2
+ mova [stp + %2 + idx10], m4
+ mova [stp + %2 + idx11], m7
+
+ ; BLOCK D STAGE 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ ;
+ ; BLOCK D STAGE 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ ;
+ ; BLOCK D STAGE 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m11, [rsp + transposed_in + 16 * 4]
+ mova m12, m11
+ pmulhrsw m11, [pw__3196x2] ; stp1_4
+ pmulhrsw m12, [pw_16069x2] ; stp1_7
+
+ ; BLOCK D STAGE 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ mova m0, [rsp + transposed_in + 16 * 0]
+ mova m10, [pw_11585x2]
+ pmulhrsw m0, m10 ; stp1_1
+
+ mova m14, m11 ; stp1_4
+ mova m13, m12 ; stp1_7
+
+ ; BLOCK D STAGE 5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+%if 0 ; overflow occurs in SUM_SUB when using test streams
+ SUM_SUB 13, 14, 9
+ pmulhrsw m13, m10 ; stp1_6
+ pmulhrsw m14, m10 ; stp1_5
+%else
+ BUTTERFLY_4X 13, 14, 11585, 11585, m8, 9, 10 ; stp1_5, stp1_6
+ SWAP 13, 14
+%endif
+ mova m7, m0 ; stp1_0 = stp1_1
+ mova m4, m0 ; stp1_1
+ mova m2, m7 ; stp1_0
+
+ ; BLOCK D STAGE 6 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 0, 12, 9 ; stp1_0, stp1_7
+ SUM_SUB 7, 13, 9 ; stp1_1, stp1_6
+ SUM_SUB 2, 14, 9 ; stp1_2, stp1_5
+ SUM_SUB 4, 11, 9 ; stp1_3, stp1_4
+
+ ; BLOCK D STAGE 7 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ SUM_SUB 0, 1, 9 ; stp1_0, stp1_15
+ SUM_SUB 7, 3, 9 ; stp1_1, stp1_14
+ SUM_SUB 2, 5, 9 ; stp1_2, stp1_13
+ SUM_SUB 4, 6, 9 ; stp1_3, stp1_12
+
+ ; 0-3, 28-31 final stage
+ mova m15, [stp + %4 + idx30]
+ mova m10, [stp + %4 + idx31]
+ SUM_SUB 0, 10, 9 ; stp1_0, stp1_31
+ SUM_SUB 7, 15, 9 ; stp1_1, stp1_30
+ mova [stp + %1 + idx0], m0
+ mova [stp + %1 + idx1], m7
+ mova [stp + %4 + idx30], m15
+ mova [stp + %4 + idx31], m10
+ mova m7, [stp + %4 + idx28]
+ mova m0, [stp + %4 + idx29]
+ SUM_SUB 2, 0, 9 ; stp1_2, stp1_29
+ SUM_SUB 4, 7, 9 ; stp1_3, stp1_28
+ mova [stp + %1 + idx2], m2
+ mova [stp + %1 + idx3], m4
+ mova [stp + %4 + idx28], m7
+ mova [stp + %4 + idx29], m0
+
+ ; 12-15, 16-19 final stage
+ mova m0, [stp + %3 + idx16]
+ mova m7, [stp + %3 + idx17]
+ mova m2, [stp + %3 + idx18]
+ mova m4, [stp + %3 + idx19]
+ SUM_SUB 1, 0, 9 ; stp1_15, stp1_16
+ SUM_SUB 3, 7, 9 ; stp1_14, stp1_17
+ SUM_SUB 5, 2, 9 ; stp1_13, stp1_18
+ SUM_SUB 6, 4, 9 ; stp1_12, stp1_19
+ mova [stp + %2 + idx12], m6
+ mova [stp + %2 + idx13], m5
+ mova [stp + %2 + idx14], m3
+ mova [stp + %2 + idx15], m1
+ mova [stp + %3 + idx16], m0
+ mova [stp + %3 + idx17], m7
+ mova [stp + %3 + idx18], m2
+ mova [stp + %3 + idx19], m4
+
+ mova m4, [stp + %2 + idx8]
+ mova m5, [stp + %2 + idx9]
+ mova m6, [stp + %2 + idx10]
+ mova m7, [stp + %2 + idx11]
+ SUM_SUB 11, 7, 9 ; stp1_4, stp1_11
+ SUM_SUB 14, 6, 9 ; stp1_5, stp1_10
+ SUM_SUB 13, 5, 9 ; stp1_6, stp1_9
+ SUM_SUB 12, 4, 9 ; stp1_7, stp1_8
+
+ ; 4-7, 24-27 final stage
+ mova m0, [stp + %4 + idx27]
+ mova m1, [stp + %4 + idx26]
+ mova m2, [stp + %4 + idx25]
+ mova m3, [stp + %4 + idx24]
+ SUM_SUB 11, 0, 9 ; stp1_4, stp1_27
+ SUM_SUB 14, 1, 9 ; stp1_5, stp1_26
+ SUM_SUB 13, 2, 9 ; stp1_6, stp1_25
+ SUM_SUB 12, 3, 9 ; stp1_7, stp1_24
+ mova [stp + %4 + idx27], m0
+ mova [stp + %4 + idx26], m1
+ mova [stp + %4 + idx25], m2
+ mova [stp + %4 + idx24], m3
+ mova [stp + %1 + idx4], m11
+ mova [stp + %1 + idx5], m14
+ mova [stp + %1 + idx6], m13
+ mova [stp + %1 + idx7], m12
+
+ ; 8-11, 20-23 final stage
+ mova m0, [stp + %3 + idx20]
+ mova m1, [stp + %3 + idx21]
+ mova m2, [stp + %3 + idx22]
+ mova m3, [stp + %3 + idx23]
+ SUM_SUB 7, 0, 9 ; stp1_11, stp_20
+ SUM_SUB 6, 1, 9 ; stp1_10, stp_21
+ SUM_SUB 5, 2, 9 ; stp1_9, stp_22
+ SUM_SUB 4, 3, 9 ; stp1_8, stp_23
+ mova [stp + %2 + idx8], m4
+ mova [stp + %2 + idx9], m5
+ mova [stp + %2 + idx10], m6
+ mova [stp + %2 + idx11], m7
+ mova [stp + %3 + idx20], m0
+ mova [stp + %3 + idx21], m1
+ mova [stp + %3 + idx22], m2
+ mova [stp + %3 + idx23], m3
+%endmacro
+
+%macro RECON_AND_STORE 1
+ mova m11, [pw_32]
+ lea stp, [rsp + %1]
+ mov r6, 32
+ pxor m8, m8
+%%recon_and_store:
+ mova m0, [stp + 16 * 32 * 0]
+ mova m1, [stp + 16 * 32 * 1]
+ mova m2, [stp + 16 * 32 * 2]
+ mova m3, [stp + 16 * 32 * 3]
+ add stp, 16
+
+ paddw m0, m11
+ paddw m1, m11
+ paddw m2, m11
+ paddw m3, m11
+ psraw m0, 6
+ psraw m1, 6
+ psraw m2, 6
+ psraw m3, 6
+ movh m4, [outputq + 0]
+ movh m5, [outputq + 8]
+ movh m6, [outputq + 16]
+ movh m7, [outputq + 24]
+ punpcklbw m4, m8
+ punpcklbw m5, m8
+ punpcklbw m6, m8
+ punpcklbw m7, m8
+ paddw m0, m4
+ paddw m1, m5
+ paddw m2, m6
+ paddw m3, m7
+ packuswb m0, m1
+ packuswb m2, m3
+ mova [outputq + 0], m0
+ mova [outputq + 16], m2
+ lea outputq, [outputq + strideq]
+ dec r6
+ jnz %%recon_and_store
+%endmacro
+
+%define i32x32_size 16*32*5
+%define pass_two_start 16*32*0
+%define transposed_in 16*32*4
+%define pass_one_start 16*32*0
+%define stp r8
+
+INIT_XMM ssse3
+cglobal idct32x32_34_add, 3, 11, 16, i32x32_size, input, output, stride
+ mova m8, [pd_8192]
+ lea stp, [rsp + pass_one_start]
+
+idct32x32_34:
+ mov r3, inputq
+ lea r4, [rsp + transposed_in]
+
+idct32x32_34_transpose:
+ mova m0, [r3 + 0]
+ mova m1, [r3 + 16 * 4]
+ mova m2, [r3 + 16 * 8]
+ mova m3, [r3 + 16 * 12]
+ mova m4, [r3 + 16 * 16]
+ mova m5, [r3 + 16 * 20]
+ mova m6, [r3 + 16 * 24]
+ mova m7, [r3 + 16 * 28]
+
+ TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9
+
+ IDCT32X32_34 16*0, 16*32, 16*64, 16*96
+ lea stp, [stp + 16 * 8]
+ mov r6, 4
+ lea stp, [rsp + pass_one_start]
+ lea r9, [rsp + pass_one_start]
+
+idct32x32_34_2:
+ lea r4, [rsp + transposed_in]
+ mov r3, r9
+
+idct32x32_34_transpose_2:
+ mova m0, [r3 + 0]
+ mova m1, [r3 + 16 * 1]
+ mova m2, [r3 + 16 * 2]
+ mova m3, [r3 + 16 * 3]
+ mova m4, [r3 + 16 * 4]
+ mova m5, [r3 + 16 * 5]
+ mova m6, [r3 + 16 * 6]
+ mova m7, [r3 + 16 * 7]
+
+ TRANSPOSE8X8 0, 1, 2, 3, 4, 5, 6, 7, 9
+
+ IDCT32X32_34 16*0, 16*8, 16*16, 16*24
+
+ lea stp, [stp + 16 * 32]
+ add r9, 16 * 32
+ dec r6
+ jnz idct32x32_34_2
+
+ RECON_AND_STORE pass_two_start
+
+ RET
%endif
diff --git a/vpx_ports/vpx_once.h b/vpx_ports/vpx_once.h
index f1df394..da04db4 100644
--- a/vpx_ports/vpx_once.h
+++ b/vpx_ports/vpx_once.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2011 The WebM project authors. All Rights Reserved.
+ * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@@ -13,63 +13,83 @@
#include "vpx_config.h"
+/* Implement a function wrapper to guarantee initialization
+ * thread-safety for library singletons.
+ *
+ * NOTE: These functions use static locks, and can only be
+ * used with one common argument per compilation unit. So
+ *
+ * file1.c:
+ * vpx_once(foo);
+ * ...
+ * vpx_once(foo);
+ *
+ * file2.c:
+ * vpx_once(bar);
+ *
+ * will ensure foo() and bar() are each called only once, but in
+ *
+ * file1.c:
+ * vpx_once(foo);
+ * vpx_once(bar):
+ *
+ * bar() will never be called because the lock is used up
+ * by the call to foo().
+ */
+
#if CONFIG_MULTITHREAD && defined(_WIN32)
#include <windows.h>
#include <stdlib.h>
+/* Declare a per-compilation-unit state variable to track the progress
+ * of calling func() only once. This must be at global scope because
+ * local initializers are not thread-safe in MSVC prior to Visual
+ * Studio 2015.
+ *
+ * As a static, once_state will be zero-initialized as program start.
+ */
+static LONG once_state;
static void once(void (*func)(void))
{
- static CRITICAL_SECTION *lock;
- static LONG waiters;
- static int done;
- void *lock_ptr = &lock;
-
- /* If the initialization is complete, return early. This isn't just an
- * optimization, it prevents races on the destruction of the global
- * lock.
+ /* Try to advance once_state from its initial value of 0 to 1.
+ * Only one thread can succeed in doing so.
*/
- if(done)
- return;
-
- InterlockedIncrement(&waiters);
-
- /* Get a lock. We create one and try to make it the one-true-lock,
- * throwing it away if we lost the race.
- */
-
- {
- /* Scope to protect access to new_lock */
- CRITICAL_SECTION *new_lock = malloc(sizeof(CRITICAL_SECTION));
- InitializeCriticalSection(new_lock);
- if (InterlockedCompareExchangePointer(lock_ptr, new_lock, NULL) != NULL)
- {
- DeleteCriticalSection(new_lock);
- free(new_lock);
- }
- }
-
- /* At this point, we have a lock that can be synchronized on. We don't
- * care which thread actually performed the allocation.
- */
-
- EnterCriticalSection(lock);
-
- if (!done)
- {
+ if (InterlockedCompareExchange(&once_state, 1, 0) == 0) {
+ /* We're the winning thread, having set once_state to 1.
+ * Call our function. */
func();
- done = 1;
+ /* Now advance once_state to 2, unblocking any other threads. */
+ InterlockedIncrement(&once_state);
+ return;
}
- LeaveCriticalSection(lock);
-
- /* Last one out should free resources. The destructed objects are
- * protected by checking if(done) above.
+ /* We weren't the winning thread, but we want to block on
+ * the state variable so we don't return before func()
+ * has finished executing elsewhere.
+ *
+ * Try to advance once_state from 2 to 2, which is only possible
+ * after the winning thead advances it from 1 to 2.
*/
- if(!InterlockedDecrement(&waiters))
- {
- DeleteCriticalSection(lock);
- free(lock);
- lock = NULL;
+ while (InterlockedCompareExchange(&once_state, 2, 2) != 2) {
+ /* State isn't yet 2. Try again.
+ *
+ * We are used for singleton initialization functions,
+ * which should complete quickly. Contention will likewise
+ * be rare, so it's worthwhile to use a simple but cpu-
+ * intensive busy-wait instead of successive backoff,
+ * waiting on a kernel object, or another heavier-weight scheme.
+ *
+ * We can at least yield our timeslice.
+ */
+ Sleep(0);
}
+
+ /* We've seen once_state advance to 2, so we know func()
+ * has been called. And we've left once_state as we found it,
+ * so other threads will have the same experience.
+ *
+ * It's safe to return now.
+ */
+ return;
}
