hash_motion: add missing alloc checks
in av1_add_to_hash_map_by_row_with_precal_data() and
av1_hash_table_create(); these functions now return bool
Bug: aomedia:3276
Change-Id: Ie7711f3585de9a58567bc1efdcc727007a8cad36
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index cfee892..73ef574 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1444,7 +1444,10 @@
}
av1_hash_table_init(intrabc_hash_info);
- av1_hash_table_create(&intrabc_hash_info->intrabc_hash_table);
+ if (!av1_hash_table_create(&intrabc_hash_info->intrabc_hash_table)) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Error allocating intrabc_hash_table");
+ }
hash_table_created = 1;
av1_generate_block_2x2_hash_value(intrabc_hash_info, cpi->source,
block_hash_values[0], is_block_same[0]);
@@ -1453,6 +1456,7 @@
const int max_sb_size =
(1 << (cm->seq_params->mib_size_log2 + MI_SIZE_LOG2));
int src_idx = 0;
+ bool error = false;
for (int size = 4; size <= max_sb_size; size *= 2, src_idx = !src_idx) {
const int dst_idx = !src_idx;
av1_generate_block_hash_value(
@@ -1460,9 +1464,13 @@
block_hash_values[dst_idx], is_block_same[src_idx],
is_block_same[dst_idx]);
if (size >= min_alloc_size) {
- av1_add_to_hash_map_by_row_with_precal_data(
- &intrabc_hash_info->intrabc_hash_table, block_hash_values[dst_idx],
- is_block_same[dst_idx][2], pic_width, pic_height, size);
+ if (!av1_add_to_hash_map_by_row_with_precal_data(
+ &intrabc_hash_info->intrabc_hash_table,
+ block_hash_values[dst_idx], is_block_same[dst_idx][2],
+ pic_width, pic_height, size)) {
+ error = true;
+ break;
+ }
}
}
@@ -1475,6 +1483,11 @@
aom_free(is_block_same[k][j]);
}
}
+
+ if (error) {
+ aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+ "Error adding data to intrabc_hash_table");
+ }
}
const CommonQuantParams *quant_params = &cm->quant_params;
diff --git a/av1/encoder/hash_motion.c b/av1/encoder/hash_motion.c
index 310cde8..164aa09 100644
--- a/av1/encoder/hash_motion.c
+++ b/av1/encoder/hash_motion.c
@@ -10,6 +10,7 @@
*/
#include <assert.h>
+#include <stdbool.h>
#include "config/av1_rtcd.h"
@@ -120,23 +121,26 @@
p_hash_table->p_lookup_table = NULL;
}
-void av1_hash_table_create(hash_table *p_hash_table) {
+bool av1_hash_table_create(hash_table *p_hash_table) {
if (p_hash_table->p_lookup_table != NULL) {
av1_hash_table_clear_all(p_hash_table);
- return;
+ return true;
}
p_hash_table->p_lookup_table =
- (Vector **)aom_malloc(sizeof(p_hash_table->p_lookup_table[0]) * kMaxAddr);
- memset(p_hash_table->p_lookup_table, 0,
- sizeof(p_hash_table->p_lookup_table[0]) * kMaxAddr);
+ (Vector **)aom_calloc(kMaxAddr, sizeof(p_hash_table->p_lookup_table[0]));
+ if (!p_hash_table) return false;
+ return true;
}
-static void hash_table_add_to_table(hash_table *p_hash_table,
+static bool hash_table_add_to_table(hash_table *p_hash_table,
uint32_t hash_value,
block_hash *curr_block_hash) {
if (p_hash_table->p_lookup_table[hash_value] == NULL) {
p_hash_table->p_lookup_table[hash_value] =
aom_malloc(sizeof(p_hash_table->p_lookup_table[0][0]));
+ if (p_hash_table->p_lookup_table[hash_value] == NULL) {
+ return false;
+ }
aom_vector_setup(p_hash_table->p_lookup_table[hash_value], 10,
sizeof(curr_block_hash[0]));
aom_vector_push_back(p_hash_table->p_lookup_table[hash_value],
@@ -145,6 +149,7 @@
aom_vector_push_back(p_hash_table->p_lookup_table[hash_value],
curr_block_hash);
}
+ return true;
}
int32_t av1_hash_table_count(const hash_table *p_hash_table,
@@ -307,7 +312,7 @@
}
}
-void av1_add_to_hash_map_by_row_with_precal_data(hash_table *p_hash_table,
+bool av1_add_to_hash_map_by_row_with_precal_data(hash_table *p_hash_table,
uint32_t *pic_hash[2],
int8_t *pic_is_same,
int pic_width, int pic_height,
@@ -335,10 +340,14 @@
const uint32_t hash_value1 = (src_hash[0][pos] & crc_mask) + add_value;
curr_block_hash.hash_value2 = src_hash[1][pos];
- hash_table_add_to_table(p_hash_table, hash_value1, &curr_block_hash);
+ if (!hash_table_add_to_table(p_hash_table, hash_value1,
+ &curr_block_hash)) {
+ return false;
+ }
}
}
}
+ return true;
}
int av1_hash_is_horizontal_perfect(const YV12_BUFFER_CONFIG *picture,
diff --git a/av1/encoder/hash_motion.h b/av1/encoder/hash_motion.h
index e4ea1f3..8974ba2 100644
--- a/av1/encoder/hash_motion.h
+++ b/av1/encoder/hash_motion.h
@@ -12,6 +12,8 @@
#ifndef AOM_AV1_ENCODER_HASH_MOTION_H_
#define AOM_AV1_ENCODER_HASH_MOTION_H_
+#include <stdbool.h>
+
#include "config/aom_config.h"
#include "aom/aom_integer.h"
@@ -56,7 +58,7 @@
void av1_hash_table_init(IntraBCHashInfo *intra_bc_hash_info);
void av1_hash_table_clear_all(hash_table *p_hash_table);
void av1_hash_table_destroy(hash_table *p_hash_table);
-void av1_hash_table_create(hash_table *p_hash_table);
+bool av1_hash_table_create(hash_table *p_hash_table);
int32_t av1_hash_table_count(const hash_table *p_hash_table,
uint32_t hash_value);
Iterator av1_hash_get_first_iterator(hash_table *p_hash_table,
@@ -74,7 +76,7 @@
uint32_t *dst_pic_block_hash[2],
int8_t *src_pic_block_same_info[3],
int8_t *dst_pic_block_same_info[3]);
-void av1_add_to_hash_map_by_row_with_precal_data(hash_table *p_hash_table,
+bool av1_add_to_hash_map_by_row_with_precal_data(hash_table *p_hash_table,
uint32_t *pic_hash[2],
int8_t *pic_is_same,
int pic_width, int pic_height,
