commit d064cf03ba29fe3726010dc1a31992a9fa932e36
author Jingning Han <jingning@google.com> Thu Jun 01 10:00:39 2017 -0700
committer Jingning Han <jingning@google.com> Fri Jun 02 15:04:56 2017 +0000
tree 20b574263ce84eef673094c9e436672cc0a066ad
parent c77799bea785013563088f3575c14904d6982b6d

Resolve extremely large stack alloc in rdopt

Move the large stack allocation from stack initialization to
dedicated mem space. This resolves the extremely large stack issue
when ext-partition, motion-var, and high bit-depth are all turned
on.

BUG=aomedia:415

Change-Id: I85b77bbc6429093fcb0152176d9e237087d6bbd8

av1/encoder/encoder.c

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index af89c0b..4782ce2 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -453,6 +453,20 @@
   aom_free(cpi->active_map.map);
   cpi->active_map.map = NULL;
 
+#if CONFIG_MOTION_VAR
+  aom_free(cpi->td.mb.above_pred_buf);
+  cpi->td.mb.above_pred_buf = NULL;
+
+  aom_free(cpi->td.mb.left_pred_buf);
+  cpi->td.mb.left_pred_buf = NULL;
+
+  aom_free(cpi->td.mb.wsrc_buf);
+  cpi->td.mb.wsrc_buf = NULL;
+
+  aom_free(cpi->td.mb.mask_buf);
+  cpi->td.mb.mask_buf = NULL;
+#endif
+
   // Free up-sampled reference buffers.
   for (i = 0; i < (REF_FRAMES + 1); i++)
     aom_free_frame_buffer(&cpi->upsampled_ref_bufs[i].buf);
@@ -2239,6 +2253,31 @@
   }
 #endif
 
+#if CONFIG_MOTION_VAR
+#if CONFIG_HIGHBITDEPTH
+  int buf_scaler = 2;
+#else
+  int buf_scaler = 1;
+#endif
+  CHECK_MEM_ERROR(
+      cm, cpi->td.mb.above_pred_buf,
+      (uint8_t *)aom_memalign(16, buf_scaler * MAX_MB_PLANE * MAX_SB_SQUARE *
+                                      sizeof(*cpi->td.mb.above_pred_buf)));
+  CHECK_MEM_ERROR(
+      cm, cpi->td.mb.left_pred_buf,
+      (uint8_t *)aom_memalign(16, buf_scaler * MAX_MB_PLANE * MAX_SB_SQUARE *
+                                      sizeof(*cpi->td.mb.left_pred_buf)));
+
+  CHECK_MEM_ERROR(cm, cpi->td.mb.wsrc_buf,
+                  (int32_t *)aom_memalign(
+                      16, MAX_SB_SQUARE * sizeof(*cpi->td.mb.wsrc_buf)));
+
+  CHECK_MEM_ERROR(cm, cpi->td.mb.mask_buf,
+                  (int32_t *)aom_memalign(
+                      16, MAX_SB_SQUARE * sizeof(*cpi->td.mb.mask_buf)));
+
+#endif
+
   init_upsampled_ref_frame_bufs(cpi);
 
   av1_set_speed_features_framesize_independent(cpi);
@@ -2530,6 +2569,12 @@
       if (cpi->common.allow_screen_content_tools)
         aom_free(thread_data->td->palette_buffer);
 #endif  // CONFIG_PALETTE
+#if CONFIG_MOTION_VAR
+      aom_free(thread_data->td->above_pred_buf);
+      aom_free(thread_data->td->left_pred_buf);
+      aom_free(thread_data->td->wsrc_buf);
+      aom_free(thread_data->td->mask_buf);
+#endif  // CONFIG_MOTION_VAR
       aom_free(thread_data->td->counts);
       av1_free_pc_tree(thread_data->td);
       aom_free(thread_data->td);