commit 0d0ee66268a69e0d89d8a6c6ed0357f156978e95
author Hui Su <huisu@google.com> Mon Jul 29 14:38:36 2019 -0700
committer Hui Su <huisu@google.com> Wed Jul 31 23:58:33 2019 +0000
tree 62409f2d76dbe243c2039ae38b8f62281c5a2e94
parent ff57adccd8ea276d1c6fbd614e98db135d517253

Fix potential infinite loop in encoder_encode()

Before this patch, an infinite loop may occur if av1_get_compressed_data()
returns with error.

Change-Id: Iee945dbb71d181f3d272b99c4adfa66816ac893c

av1/av1_cx_iface.c

diff --git a/av1/av1_cx_iface.c b/av1/av1_cx_iface.c
index 33f252f..a13a143 100644
--- a/av1/av1_cx_iface.c
+++ b/av1/av1_cx_iface.c
@@ -1836,10 +1836,15 @@
     int has_fwd_keyframe = 0;
     // invisible frames get packed with the next visible frame
     while (cx_data_sz - index_size >= ctx->cx_data_sz / 2 &&
-           !is_frame_visible &&
-           -1 != av1_get_compressed_data(cpi, &lib_flags, &frame_size, cx_data,
-                                         &dst_time_stamp, &dst_end_time_stamp,
-                                         !img, timestamp_ratio)) {
+           !is_frame_visible) {
+      const int status = av1_get_compressed_data(
+          cpi, &lib_flags, &frame_size, cx_data, &dst_time_stamp,
+          &dst_end_time_stamp, !img, timestamp_ratio);
+      if (status == -1) break;
+      if (status != AOM_CODEC_OK) {
+        aom_internal_error(&cpi->common.error, AOM_CODEC_ERROR, NULL);
+      }
+
       cpi->seq_params_locked = 1;
       if (frame_size) {
         if (ctx->pending_cx_data == 0) ctx->pending_cx_data = cx_data;

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 9e23c55..4f55317 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -5845,7 +5845,7 @@
 
   aom_clear_system_state();
 
-  return 0;
+  return AOM_CODEC_OK;
 }
 
 int av1_get_preview_raw_frame(AV1_COMP *cpi, YV12_BUFFER_CONFIG *dest) {