diff --git a/av1/encoder/intra_mode_search.c b/av1/encoder/intra_mode_search.c
index d4f413f..c99841c 100644
--- a/av1/encoder/intra_mode_search.c
+++ b/av1/encoder/intra_mode_search.c
@@ -1765,9 +1765,6 @@
   const int skip_ctx = av1_get_skip_txfm_context(xd);
 
   int known_rate = mode_cost;
-  // TODO(chiyotsai@google.com): known rate seems to be counting ref_frame_cost
-  // twice here. Try to remove this.
-  known_rate += ref_frame_cost;
   if (mode != DC_PRED && mode != PAETH_PRED) known_rate += intra_cost_penalty;
   known_rate += AOMMIN(mode_costs->skip_txfm_cost[skip_ctx][0],
                        mode_costs->skip_txfm_cost[skip_ctx][1]);