Free restoration buffers in decoder with RT build

Bug: chromium:1321141
Bug: aomedia:3248
Change-Id: I1125f40afe780befa5c82b0057f73a4c9b24c99e
diff --git a/av1/av1_dx_iface.c b/av1/av1_dx_iface.c
index 45e8cb9..d6d147b 100644
--- a/av1/av1_dx_iface.c
+++ b/av1/av1_dx_iface.c
@@ -124,9 +124,7 @@
     av1_free_cdef_buffers(&pbi->common, &pbi->cdef_worker, &pbi->cdef_sync,
                           pbi->num_workers);
     av1_free_cdef_sync(&pbi->cdef_sync);
-#if !CONFIG_REALTIME_ONLY
     av1_free_restoration_buffers(&pbi->common);
-#endif
     av1_decoder_remove(pbi);
     aom_free(frame_worker_data);
 #if CONFIG_MULTITHREAD