diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 2580b3d..cbcc24d 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -990,7 +990,6 @@
     ($w, $h) = @$_;
     add_proto qw/void/, "aom_sad${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, uint32_t sad_array[4]";
     add_proto qw/void/, "aom_sad${w}x${h}x3d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, uint32_t sad_array[4]";
-    add_proto qw/void/, "aom_sad${w}x${h}x4d_avg", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, const uint8_t *second_pred, uint32_t sad_array[4]";
     add_proto qw/void/, "aom_sad_skip_${w}x${h}x4d", "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_ptr[4], int ref_stride, uint32_t sad_array[4]";
     add_proto qw/void/, "aom_masked_sad${w}x${h}x4d", "const uint8_t *src, int src_stride, const uint8_t *ref[4], int ref_stride, const uint8_t *second_pred, const uint8_t *msk, int msk_stride, int invert_mask, unsigned sads[4]";
   }
@@ -1059,37 +1058,6 @@
   specialize qw/aom_sad32x8x3d    avx2/;
   specialize qw/aom_sad16x64x3d   avx2/;
 
-  if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
-    specialize qw/aom_sad128x128x4d_avg sse2/;
-    specialize qw/aom_sad128x64x4d_avg  sse2/;
-    specialize qw/aom_sad64x128x4d_avg  sse2/;
-    specialize qw/aom_sad64x64x4d_avg   sse2/;
-    specialize qw/aom_sad64x32x4d_avg   sse2/;
-    specialize qw/aom_sad64x16x4d_avg   sse2/;
-    specialize qw/aom_sad32x64x4d_avg   sse2/;
-    specialize qw/aom_sad32x32x4d_avg   sse2/;
-    specialize qw/aom_sad32x16x4d_avg   sse2/;
-    specialize qw/aom_sad32x8x4d_avg    sse2/;
-    specialize qw/aom_sad16x64x4d_avg   sse2/;
-    specialize qw/aom_sad16x32x4d_avg   sse2/;
-    specialize qw/aom_sad16x16x4d_avg   sse2/;
-    specialize qw/aom_sad16x8x4d_avg    sse2/;
-
-    specialize qw/aom_sad8x16x4d_avg    sse2/;
-    specialize qw/aom_sad8x8x4d_avg     sse2/;
-    specialize qw/aom_sad8x4x4d_avg     sse2/;
-    specialize qw/aom_sad4x16x4d_avg    sse2/;
-    specialize qw/aom_sad4x8x4d_avg     sse2/;
-    specialize qw/aom_sad4x4x4d_avg     sse2/;
-
-    specialize qw/aom_sad4x32x4d_avg    sse2/;
-    specialize qw/aom_sad4x16x4d_avg    sse2/;
-    specialize qw/aom_sad16x4x4d_avg    sse2/;
-    specialize qw/aom_sad8x32x4d_avg    sse2/;
-    specialize qw/aom_sad32x8x4d_avg    sse2/;
-    specialize qw/aom_sad64x16x4d_avg   sse2/;
-  }
-
   specialize qw/aom_masked_sad128x128x4d  ssse3/;
   specialize qw/aom_masked_sad128x64x4d   ssse3/;
   specialize qw/aom_masked_sad64x128x4d   ssse3/;
diff --git a/aom_dsp/sad.c b/aom_dsp/sad.c
index 723f9f3..5138fde 100644
--- a/aom_dsp/sad.c
+++ b/aom_dsp/sad.c
@@ -94,15 +94,6 @@
           aom_sad##m##x##n##_c(src, src_stride, ref_array[i], ref_stride);    \
     }                                                                         \
   }                                                                           \
-  void aom_sad##m##x##n##x4d_avg_c(                                           \
-      const uint8_t *src, int src_stride, const uint8_t *const ref_array[4],  \
-      int ref_stride, const uint8_t *second_pred, uint32_t sad_array[4]) {    \
-    int i;                                                                    \
-    for (i = 0; i < 4; ++i) {                                                 \
-      sad_array[i] = aom_sad##m##x##n##_avg_c(src, src_stride, ref_array[i],  \
-                                              ref_stride, second_pred);       \
-    }                                                                         \
-  }                                                                           \
   void aom_sad_skip_##m##x##n##x4d_c(const uint8_t *src, int src_stride,      \
                                      const uint8_t *const ref_array[4],       \
                                      int ref_stride, uint32_t sad_array[4]) { \
diff --git a/aom_dsp/x86/sad4d_sse2.asm b/aom_dsp/x86/sad4d_sse2.asm
index 6de708b..1e89521 100644
--- a/aom_dsp/x86/sad4d_sse2.asm
+++ b/aom_dsp/x86/sad4d_sse2.asm
@@ -235,6 +235,7 @@
 ;   2: Height
 ;   3: If 0, then normal sad, else avg
 ;   4: If 0, then normal sad, else skip rows
+;   TODO(jzern): remove the remnants of avg and merge %3 & %4
 %macro SADNXN4D 2-4 0,0
 
 %define spill_src_stride 0
@@ -276,7 +277,8 @@
                                     ref3, ref4
 %endif
 %endif
-%elif %3 == 0  ; normal sad
+%else ; normal sad
+ASSERT %3 == 0
 %if ARCH_X86_64
 %if use_ref_offset
 cglobal sad%1x%2x4d, 5, 10, 8, src, src_stride, ref1, ref_stride, res, ref2, \
@@ -301,34 +303,6 @@
                               ref4
 %endif
 %endif
-%else ; avg
-%if ARCH_X86_64
-%if use_ref_offset
-cglobal sad%1x%2x4d_avg, 6, 11, 8, src, src_stride, ref1, ref_stride, \
-                                   second_pred, res, ref2, ref3, ref4, cnt, \
-                                   ref_offset
-%elif use_loop
-cglobal sad%1x%2x4d_avg, 6, 10, 8, src, src_stride, ref1, ref_stride, \
-                                   second_pred, res, ref2, ref3, ref4, cnt
-%else
-cglobal sad%1x%2x4d_avg, 6, 9, 8, src, src_stride, ref1, ref_stride, \
-                                   second_pred, res, ref2, ref3, ref4
-%endif
-%else
-%if use_ref_offset
-cglobal sad%1x%2x4d_avg, 5, 7, 8, src, ref4, ref1, ref_offset, second_pred, ref2, ref3
-  %define spill_src_stride 1
-  %define spill_ref_stride 1
-  %define spill_cnt 1
-%elif use_loop
-cglobal sad%1x%2x4d_avg, 5, 7, 8, src, ref4, ref1, ref_stride, second_pred, ref2, ref3
-  %define spill_src_stride 1
-  %define spill_cnt 1
-%else
-cglobal sad%1x%2x4d_avg, 5, 7, 8, src, ref4, ref1, ref_stride, second_pred, ref2, ref3
-  %define spill_src_stride 1
-%endif
-%endif
 %endif
 
 %if spill_src_stride
@@ -421,9 +395,6 @@
 %if %3 == 0
   %define resultq r4
   %define resultmp r4mp
-%else
-  %define resultq r5
-  %define resultmp r5mp
 %endif
 
 ; Undo modifications on parameters on the stack
@@ -492,30 +463,6 @@
 SADNXN4D  16,  64
 SADNXN4D  64,  16
 %endif
-%if CONFIG_REALTIME_ONLY==0
-SADNXN4D 128, 128, 1
-SADNXN4D 128,  64, 1
-SADNXN4D  64, 128, 1
-SADNXN4D  64,  64, 1
-SADNXN4D  64,  32, 1
-SADNXN4D  32,  64, 1
-SADNXN4D  32,  32, 1
-SADNXN4D  32,  16, 1
-SADNXN4D  16,  32, 1
-SADNXN4D  16,  16, 1
-SADNXN4D  16,   8, 1
-SADNXN4D   8,  16, 1
-SADNXN4D   8,   8, 1
-SADNXN4D   8,   4, 1
-SADNXN4D   4,   8, 1
-SADNXN4D   4,   4, 1
-SADNXN4D   4,  16, 1
-SADNXN4D  16,   4, 1
-SADNXN4D   8,  32, 1
-SADNXN4D  32,   8, 1
-SADNXN4D  16,  64, 1
-SADNXN4D  64,  16, 1
-%endif
 SADNXN4D 128, 128, 0, 1
 SADNXN4D 128,  64, 0, 1
 SADNXN4D  64, 128, 0, 1
diff --git a/test/sad_test.cc b/test/sad_test.cc
index ca3e34b..707a3ec 100644
--- a/test/sad_test.cc
+++ b/test/sad_test.cc
@@ -481,42 +481,6 @@
   }
 };
 
-#if !CONFIG_REALTIME_ONLY
-class SADx4AvgTest : public ::testing::WithParamInterface<SadMxNx4AvgParam>,
-                     public SADTestBase {
- public:
-  SADx4AvgTest() : SADTestBase(GET_PARAM(0), GET_PARAM(1), GET_PARAM(3)) {}
-
- protected:
-  void SADs(unsigned int *results) {
-    const uint8_t *references[] = { GetReference(0), GetReference(1),
-                                    GetReference(2), GetReference(3) };
-
-    API_REGISTER_STATE_CHECK(GET_PARAM(2)(source_data_, source_stride_,
-                                          references, reference_stride_,
-                                          second_pred_, results));
-  }
-
-  void CheckSADs() {
-    unsigned int reference_sad, exp_sad[4];
-
-    SADs(exp_sad);
-    for (int block = 0; block < 4; ++block) {
-      reference_sad = ReferenceSADavg(block);
-
-      EXPECT_EQ(reference_sad, exp_sad[block]) << "block " << block;
-    }
-  }
-
-  void SADForSpeedTest(unsigned int *results,
-                       const uint8_t *const *references) {
-    GET_PARAM(2)
-    (source_data_, source_stride_, references, reference_stride_, second_pred_,
-     results);
-  }
-};
-#endif  // !CONFIG_REALTIME_ONLY
-
 class SADTest : public ::testing::WithParamInterface<SadMxNParam>,
                 public SADTestBase {
  public:
@@ -1173,69 +1137,6 @@
 
 using std::make_tuple;
 
-#if !CONFIG_REALTIME_ONLY
-TEST_P(SADx4AvgTest, DISABLED_Speed) {
-  int tmp_stride = reference_stride_;
-  reference_stride_ >>= 1;
-  FillRandom(source_data_, source_stride_);
-  FillRandom(GetReference(0), reference_stride_);
-  FillRandom(GetReference(1), reference_stride_);
-  FillRandom(GetReference(2), reference_stride_);
-  FillRandom(GetReference(3), reference_stride_);
-  FillRandom(second_pred_, width_);
-  SpeedSAD();
-  reference_stride_ = tmp_stride;
-}
-
-TEST_P(SADx4AvgTest, MaxRef) {
-  FillConstant(source_data_, source_stride_, 0);
-  FillConstant(GetReference(0), reference_stride_, mask_);
-  FillConstant(GetReference(1), reference_stride_, mask_);
-  FillConstant(GetReference(2), reference_stride_, mask_);
-  FillConstant(GetReference(3), reference_stride_, mask_);
-  FillConstant(second_pred_, width_, 0);
-  CheckSADs();
-}
-
-TEST_P(SADx4AvgTest, MaxSrc) {
-  FillConstant(source_data_, source_stride_, mask_);
-  FillConstant(GetReference(0), reference_stride_, 0);
-  FillConstant(GetReference(1), reference_stride_, 0);
-  FillConstant(GetReference(2), reference_stride_, 0);
-  FillConstant(GetReference(3), reference_stride_, 0);
-  FillConstant(second_pred_, width_, 0);
-  CheckSADs();
-}
-
-TEST_P(SADx4AvgTest, ShortRef) {
-  int tmp_stride = reference_stride_;
-  reference_stride_ >>= 1;
-  FillRandom(source_data_, source_stride_);
-  FillRandom(GetReference(0), reference_stride_);
-  FillRandom(GetReference(1), reference_stride_);
-  FillRandom(GetReference(2), reference_stride_);
-  FillRandom(GetReference(3), reference_stride_);
-  FillRandom(second_pred_, width_);
-  CheckSADs();
-  reference_stride_ = tmp_stride;
-}
-
-TEST_P(SADx4AvgTest, UnalignedRef) {
-  // The reference frame, but not the source frame, may be unaligned for
-  // certain types of searches.
-  int tmp_stride = reference_stride_;
-  reference_stride_ -= 1;
-  FillRandom(source_data_, source_stride_);
-  FillRandom(GetReference(0), reference_stride_);
-  FillRandom(GetReference(1), reference_stride_);
-  FillRandom(GetReference(2), reference_stride_);
-  FillRandom(GetReference(3), reference_stride_);
-  FillRandom(second_pred_, width_);
-  CheckSADs();
-  reference_stride_ = tmp_stride;
-}
-#endif  // !CONFIG_REALTIME_ONLY
-
 //------------------------------------------------------------------------------
 // C functions
 const SadMxNParam c_tests[] = {
@@ -1913,34 +1814,6 @@
 INSTANTIATE_TEST_SUITE_P(C, SADSkipx4Test,
                          ::testing::ValuesIn(skip_x4d_c_tests));
 
-#if !CONFIG_REALTIME_ONLY
-const SadMxNx4AvgParam x4d_avg_c_tests[] = {
-  make_tuple(128, 128, &aom_sad128x128x4d_avg_c, -1),
-  make_tuple(128, 64, &aom_sad128x64x4d_avg_c, -1),
-  make_tuple(64, 128, &aom_sad64x128x4d_avg_c, -1),
-  make_tuple(64, 64, &aom_sad64x64x4d_avg_c, -1),
-  make_tuple(64, 32, &aom_sad64x32x4d_avg_c, -1),
-  make_tuple(32, 64, &aom_sad32x64x4d_avg_c, -1),
-  make_tuple(32, 32, &aom_sad32x32x4d_avg_c, -1),
-  make_tuple(32, 16, &aom_sad32x16x4d_avg_c, -1),
-  make_tuple(16, 32, &aom_sad16x32x4d_avg_c, -1),
-  make_tuple(16, 16, &aom_sad16x16x4d_avg_c, -1),
-  make_tuple(16, 8, &aom_sad16x8x4d_avg_c, -1),
-  make_tuple(8, 16, &aom_sad8x16x4d_avg_c, -1),
-  make_tuple(8, 8, &aom_sad8x8x4d_avg_c, -1),
-  make_tuple(8, 4, &aom_sad8x4x4d_avg_c, -1),
-  make_tuple(4, 8, &aom_sad4x8x4d_avg_c, -1),
-  make_tuple(4, 4, &aom_sad4x4x4d_avg_c, -1),
-  make_tuple(64, 16, &aom_sad64x16x4d_avg_c, -1),
-  make_tuple(16, 64, &aom_sad16x64x4d_avg_c, -1),
-  make_tuple(32, 8, &aom_sad32x8x4d_avg_c, -1),
-  make_tuple(8, 32, &aom_sad8x32x4d_avg_c, -1),
-  make_tuple(16, 4, &aom_sad16x4x4d_avg_c, -1),
-  make_tuple(4, 16, &aom_sad4x16x4d_avg_c, -1),
-};
-INSTANTIATE_TEST_SUITE_P(C, SADx4AvgTest, ::testing::ValuesIn(x4d_avg_c_tests));
-#endif  // !CONFIG_REALTIME_ONLY
-
 //------------------------------------------------------------------------------
 // ARM functions
 #if HAVE_NEON
@@ -2529,35 +2402,6 @@
 INSTANTIATE_TEST_SUITE_P(SSE2, SADSkipx4Test,
                          ::testing::ValuesIn(skip_x4d_sse2_tests));
 
-#if !CONFIG_REALTIME_ONLY
-const SadMxNx4AvgParam x4d_avg_sse2_tests[] = {
-  make_tuple(128, 128, &aom_sad128x128x4d_avg_sse2, -1),
-  make_tuple(128, 64, &aom_sad128x64x4d_avg_sse2, -1),
-  make_tuple(64, 128, &aom_sad64x128x4d_avg_sse2, -1),
-  make_tuple(64, 64, &aom_sad64x64x4d_avg_sse2, -1),
-  make_tuple(64, 32, &aom_sad64x32x4d_avg_sse2, -1),
-  make_tuple(32, 64, &aom_sad32x64x4d_avg_sse2, -1),
-  make_tuple(32, 32, &aom_sad32x32x4d_avg_sse2, -1),
-  make_tuple(32, 16, &aom_sad32x16x4d_avg_sse2, -1),
-  make_tuple(16, 32, &aom_sad16x32x4d_avg_sse2, -1),
-  make_tuple(16, 16, &aom_sad16x16x4d_avg_sse2, -1),
-  make_tuple(16, 8, &aom_sad16x8x4d_avg_sse2, -1),
-  make_tuple(8, 16, &aom_sad8x16x4d_avg_sse2, -1),
-  make_tuple(8, 8, &aom_sad8x8x4d_avg_sse2, -1),
-  make_tuple(8, 4, &aom_sad8x4x4d_avg_sse2, -1),
-  make_tuple(4, 8, &aom_sad4x8x4d_avg_sse2, -1),
-  make_tuple(4, 4, &aom_sad4x4x4d_avg_sse2, -1),
-  make_tuple(64, 16, &aom_sad64x16x4d_avg_sse2, -1),
-  make_tuple(16, 64, &aom_sad16x64x4d_avg_sse2, -1),
-  make_tuple(32, 8, &aom_sad32x8x4d_avg_sse2, -1),
-  make_tuple(8, 32, &aom_sad8x32x4d_avg_sse2, -1),
-  make_tuple(16, 4, &aom_sad16x4x4d_avg_sse2, -1),
-  make_tuple(4, 16, &aom_sad4x16x4d_avg_sse2, -1),
-};
-INSTANTIATE_TEST_SUITE_P(SSE2, SADx4AvgTest,
-                         ::testing::ValuesIn(x4d_avg_sse2_tests));
-#endif  // !CONFIG_REALTIME_ONLY
-
 const DistWtdSadMxNAvgParam dist_wtd_avg_sse2_tests[] = {
   make_tuple(128, 128, &aom_dist_wtd_sad128x128_avg_sse2, -1),
   make_tuple(128, 64, &aom_dist_wtd_sad128x64_avg_sse2, -1),