commit 1354c6942cbf4e87b71f829e6386b3e92442e941
author Geza Lore <gezalore@gmail.com> Thu May 12 15:51:34 2016 +0100
committer Debargha Mukherjee <debargha@google.com> Fri Jun 03 06:43:34 2016 -0700
tree 4ebd0372ca832bca9bc14f49bf09fb67b506efe2
parent 353930d212bc06704f9aa253d1d812802dd75ed5

Compute rate of partition type accurately for edge blocks.

This patch factors in the different partition coding syntax used for
right and bottom edge blocks when doing RD search.

Change-Id: I2f31650512b6a4a7a2c03352414693aff6fbf87b

vp10/encoder/encodeframe.c

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 78d7067..b1cfa14 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -3362,6 +3362,8 @@
   PICK_MODE_CONTEXT *ctx = &pc_tree->none;
   int i;
   const int pl = partition_plane_context(xd, mi_row, mi_col, bsize);
+  int *partition_cost = cpi->partition_cost[pl];
+  int tmp_partition_cost[PARTITION_TYPES];
   BLOCK_SIZE subsize;
   RD_COST this_rdc, sum_rdc, best_rdc;
 #if CONFIG_SUPERTX
@@ -3399,6 +3401,30 @@
                                bsize >= BLOCK_8X8;
   (void) *tp_orig;
 
+  if (force_horz_split || force_vert_split) {
+    tmp_partition_cost[PARTITION_NONE] = INT_MAX;
+
+    if (!force_vert_split) {  // force_horz_split only
+      tmp_partition_cost[PARTITION_VERT] = INT_MAX;
+      tmp_partition_cost[PARTITION_HORZ] =
+          vp10_cost_bit(cm->fc->partition_prob[pl][PARTITION_HORZ], 0);
+      tmp_partition_cost[PARTITION_SPLIT] =
+          vp10_cost_bit(cm->fc->partition_prob[pl][PARTITION_HORZ], 1);
+    } else if (!force_horz_split) {  // force_vert_split only
+      tmp_partition_cost[PARTITION_HORZ] = INT_MAX;
+      tmp_partition_cost[PARTITION_VERT] =
+          vp10_cost_bit(cm->fc->partition_prob[pl][PARTITION_VERT], 0);
+      tmp_partition_cost[PARTITION_SPLIT] =
+          vp10_cost_bit(cm->fc->partition_prob[pl][PARTITION_VERT], 1);
+    } else {  // force_ horz_split && force_vert_split horz_split
+      tmp_partition_cost[PARTITION_HORZ] = INT_MAX;
+      tmp_partition_cost[PARTITION_VERT] = INT_MAX;
+      tmp_partition_cost[PARTITION_SPLIT] = 0;
+    }
+
+    partition_cost = tmp_partition_cost;
+  }
+
 #if CONFIG_VAR_TX
 #ifndef NDEBUG
   // Nothing should rely on the default value of this array (which is just
@@ -3523,11 +3549,11 @@
                      bsize, ctx, best_rdc.rdcost);
     if (this_rdc.rate != INT_MAX) {
       if (bsize >= BLOCK_8X8) {
-        this_rdc.rate += cpi->partition_cost[pl][PARTITION_NONE];
+        this_rdc.rate += partition_cost[PARTITION_NONE];
         this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                                  this_rdc.rate, this_rdc.dist);
 #if CONFIG_SUPERTX
-        this_rate_nocoef += cpi->partition_cost[pl][PARTITION_NONE];
+        this_rate_nocoef += partition_cost[PARTITION_NONE];
 #endif
       }
 
@@ -3782,11 +3808,11 @@
     }
 
     if (sum_rdc.rdcost < best_rdc.rdcost && i == 4) {
-      sum_rdc.rate += cpi->partition_cost[pl][PARTITION_SPLIT];
+      sum_rdc.rate += partition_cost[PARTITION_SPLIT];
       sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                               sum_rdc.rate, sum_rdc.dist);
 #if CONFIG_SUPERTX
-      sum_rate_nocoef += cpi->partition_cost[pl][PARTITION_SPLIT];
+      sum_rate_nocoef += partition_cost[PARTITION_SPLIT];
 #endif  // CONFIG_SUPERTX
 
       if (sum_rdc.rdcost < best_rdc.rdcost) {
@@ -3934,10 +3960,10 @@
 #endif  // CONFIG_SUPERTX
 
     if (sum_rdc.rdcost < best_rdc.rdcost) {
-      sum_rdc.rate += cpi->partition_cost[pl][PARTITION_HORZ];
+      sum_rdc.rate += partition_cost[PARTITION_HORZ];
       sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
 #if CONFIG_SUPERTX
-      sum_rate_nocoef += cpi->partition_cost[pl][PARTITION_HORZ];
+      sum_rate_nocoef += partition_cost[PARTITION_HORZ];
 #endif  // CONFIG_SUPERTX
       if (sum_rdc.rdcost < best_rdc.rdcost) {
         best_rdc = sum_rdc;
@@ -4079,11 +4105,11 @@
 #endif  // CONFIG_SUPERTX
 
     if (sum_rdc.rdcost < best_rdc.rdcost) {
-      sum_rdc.rate += cpi->partition_cost[pl][PARTITION_VERT];
+      sum_rdc.rate += partition_cost[PARTITION_VERT];
       sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                               sum_rdc.rate, sum_rdc.dist);
 #if CONFIG_SUPERTX
-      sum_rate_nocoef += cpi->partition_cost[pl][PARTITION_VERT];
+      sum_rate_nocoef += partition_cost[PARTITION_VERT];
 #endif  // CONFIG_SUPERTX
       if (sum_rdc.rdcost < best_rdc.rdcost) {
         best_rdc = sum_rdc;