Merge "Fix bilateral filter asan error for highbitdepth" into nextgenv2
diff --git a/av1/common/entropy.c b/av1/common/entropy.c
index 80eaccc..95368c6 100644
--- a/av1/common/entropy.c
+++ b/av1/common/entropy.c
@@ -2924,7 +2924,7 @@
#if CONFIG_ADAPT_SCAN
for (tx_size = TX_4X4; tx_size < TX_SIZES; ++tx_size)
- for (tx_type = TX_4X4; tx_type < TX_TYPES; ++tx_type) {
+ for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
av1_update_scan_prob(cm, tx_size, tx_type, ADAPT_SCAN_UPDATE_RATE_16);
av1_update_scan_order_facade(cm, tx_size, tx_type);
}
diff --git a/av1/common/warped_motion.c b/av1/common/warped_motion.c
index f73e777..89534de 100644
--- a/av1/common/warped_motion.c
+++ b/av1/common/warped_motion.c
@@ -215,7 +215,7 @@
} else if (x == (1 << WARPEDPIXEL_PREC_BITS)) {
return p[1];
} else {
- const int64_t v1 = x * x * x * (3 * (p[0] - p[1]) + p[2] - p[-1]);
+ const int64_t v1 = (int64_t)x * x * x * (3 * (p[0] - p[1]) + p[2] - p[-1]);
const int64_t v2 = x * x * (2 * p[-1] - 5 * p[0] + 4 * p[1] - p[2]);
const int64_t v3 = x * (p[1] - p[-1]);
const int64_t v4 = 2 * p[0];
diff --git a/av1/encoder/global_motion.c b/av1/encoder/global_motion.c
index 5d88dbf..d8abea9 100644
--- a/av1/encoder/global_motion.c
+++ b/av1/encoder/global_motion.c
@@ -25,7 +25,7 @@
#define MAX_CORNERS 4096
#define MIN_INLIER_PROB 0.1
-INLINE RansacFunc get_ransac_type(TransformationType type) {
+static INLINE RansacFunc get_ransac_type(TransformationType type) {
switch (type) {
case HOMOGRAPHY: return ransac_homography;
case AFFINE: return ransac_affine;
diff --git a/av1/encoder/ransac.c b/av1/encoder/ransac.c
index 714d567..2699c4f 100644
--- a/av1/encoder/ransac.c
+++ b/av1/encoder/ransac.c
@@ -81,10 +81,12 @@
const int stride_points,
const int stride_proj) {
int i;
- double x, y, Z;
+ double x, y, Z, Z_inv;
for (i = 0; i < n; ++i) {
x = *(points++), y = *(points++);
- Z = 1. / (mat[7] * x + mat[6] * y + 1);
+ Z_inv = mat[7] * x + mat[6] * y + 1;
+ assert(fabs(Z_inv) > 0.00001);
+ Z = 1. / Z_inv;
*(proj++) = (mat[1] * x + mat[0] * y + mat[3]) * Z;
*(proj++) = (mat[2] * x + mat[4] * y + mat[4]) * Z;
points += stride_points - 2;
@@ -155,7 +157,7 @@
double T1[9], T2[9];
*number_of_inliers = 0;
- if (npoints < minpts * MINPTS_MULTIPLIER) {
+ if (npoints < minpts * MINPTS_MULTIPLIER || npoints == 0) {
printf("Cannot find motion with %d matches\n", npoints);
return 1;
}
@@ -245,11 +247,15 @@
}
}
- if (num_inliers >= max_inliers) {
- double mean_distance = sum_distance / ((double)num_inliers);
- double variance = sum_distance_squared / ((double)num_inliers - 1.0) -
- mean_distance * mean_distance * ((double)num_inliers) /
- ((double)num_inliers - 1.0);
+ if (num_inliers >= max_inliers && num_inliers > 1) {
+ int temp;
+ double fracinliers, pNoOutliers, mean_distance, variance;
+
+ assert(num_inliers > 1);
+ mean_distance = sum_distance / ((double)num_inliers);
+ variance = sum_distance_squared / ((double)num_inliers - 1.0) -
+ mean_distance * mean_distance * ((double)num_inliers) /
+ ((double)num_inliers - 1.0);
if ((num_inliers > max_inliers) ||
(num_inliers == max_inliers && variance < best_variance)) {
best_variance = variance;
@@ -262,16 +268,15 @@
memcpy(best_inlier_mask, inlier_mask,
npoints * sizeof(*best_inlier_mask));
- if (num_inliers > 0) {
- double fracinliers = (double)num_inliers / (double)npoints;
- double pNoOutliers = 1 - pow(fracinliers, minpts);
- int temp;
- pNoOutliers = fmax(EPS, pNoOutliers);
- pNoOutliers = fmin(1 - EPS, pNoOutliers);
- temp = (int)(log(1.0 - PROBABILITY_REQUIRED) / log(pNoOutliers));
- if (temp > 0 && temp < N) {
- N = AOMMAX(temp, MIN_TRIALS);
- }
+ assert(npoints > 0);
+ fracinliers = (double)num_inliers / (double)npoints;
+ pNoOutliers = 1 - pow(fracinliers, minpts);
+ pNoOutliers = fmax(EPS, pNoOutliers);
+ pNoOutliers = fmin(1 - EPS, pNoOutliers);
+ assert(fabs(1.0 - pNoOutliers) > 0.00001);
+ temp = (int)(log(1.0 - PROBABILITY_REQUIRED) / log(pNoOutliers));
+ if (temp > 0 && temp < N) {
+ N = AOMMAX(temp, MIN_TRIALS);
}
}
}
@@ -356,6 +361,7 @@
// normalize so that H33 = 1
int i;
const double m = 1.0 / best_params[8];
+ assert(fabs(best_params[8]) > 0.00001);
for (i = 0; i < 8; ++i) best_params[i] *= m;
best_params[8] = 1.0;
}
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 3dfa2ee..1169fa4 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -9424,7 +9424,10 @@
#endif // CONFIG_REF_MV
#if CONFIG_GLOBAL_MOTION
zeromv[0].as_int = cm->global_motion[refs[0]].motion_params.wmmat[0].as_int;
- zeromv[1].as_int = cm->global_motion[refs[1]].motion_params.wmmat[0].as_int;
+ if (comp_pred_mode) {
+ zeromv[1].as_int =
+ cm->global_motion[refs[1]].motion_params.wmmat[0].as_int;
+ }
#else
zeromv[0].as_int = 0;
zeromv[1].as_int = 0;
