Bitmask: add helper functions

Helper functions locate 4x4 tx block position in bitmask.
Initialization of bitmask structures.

Change-Id: I332b3394615c1cda4d2e5ebc450ce8a37282e9d3
diff --git a/av1/common/av1_loopfilter.c b/av1/common/av1_loopfilter.c
index 821583e..141bb6e 100644
--- a/av1/common/av1_loopfilter.c
+++ b/av1/common/av1_loopfilter.c
@@ -644,8 +644,111 @@
       }
     }
   }
+
+#if LOOP_FILTER_BITMASK
+  memset(lf->neighbor_sb_lpf_info.tx_size_y_above, TX_64X64,
+         sizeof(TX_SIZE) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.tx_size_y_left, TX_64X64,
+         sizeof(TX_SIZE) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.tx_size_uv_above, TX_64X64,
+         sizeof(TX_SIZE) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.tx_size_uv_left, TX_64X64,
+         sizeof(TX_SIZE) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.y_level_above, 0,
+         sizeof(uint8_t) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.y_level_left, 0,
+         sizeof(uint8_t) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.u_level_above, 0,
+         sizeof(uint8_t) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.u_level_left, 0,
+         sizeof(uint8_t) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.v_level_above, 0,
+         sizeof(uint8_t) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.v_level_left, 0,
+         sizeof(uint8_t) * MI_SIZE_64X64);
+  memset(lf->neighbor_sb_lpf_info.skip, 0, sizeof(uint8_t) * MI_SIZE_64X64);
+#endif  // LOOP_FILTER_BITMASK
 }
 
+#if LOOP_FILTER_BITMASK
+// A 64x64 tx block requires 256 bits to represent each 4x4 tx block.
+// Every 4 rows is represented by one uint64_t mask. Hence,
+// there are 4 uint64_t bitmask[4] to represent the 64x64 block.
+//
+// Given a location by (idx, idy), This function returns the index
+// 0, 1, 2, 3 to select which bitmask[] to use.
+// Then the pointer y_shift contains the shift value in the bit mask.
+// Function returns y_shift; y_index contains the index.
+//
+// For example, idy is the offset of pixels,
+// (idy >> MI_SIZE_LOG2) converts to 4x4 unit.
+// ((idy >> MI_SIZE_LOG2) / 4) returns which uint64_t.
+// After locating which uint64_t, (idy >> MI_SIZE_LOG2) % 4 is the
+// row offset, and each row has 16 = 1 << stride_log2 4x4 units.
+// Therefore, shift = (row << stride_log2) + (idx >> MI_SIZE_LOG2);
+static int get_y_index_shift(int idx, int idy, int *y_index) {
+  // idy_unit = idy >> MI_SIZE_LOG2;
+  // idx_unit = idx >> MI_SIZE_LOG2;
+  // *y_index = idy_unit >> 2;
+  // rows = idy_unit % 4;
+  // stride_log2 = 4;
+  // shift = (rows << stride_log2) + idx_unit;
+
+  *y_index = idy >> 4;
+  return ((idy & 12) << 2) | (idx >> 2);
+}
+
+// Largest tx size of U/V plane is 32x32.
+// We need one uint64_t bitmask to present all 4x4 tx block.
+// ss_x, ss_y: subsampling. for 420 format, ss_x = 1, ss_y = 1.
+// Each row has 8 = (1 << stride_log2) 4x4 units.
+static int get_uv_index_shift(int idx, int idy) {
+  // stride_log2 = 3;
+  // idy_unit = (idy >> (MI_SIZE_LOG2 + ss_y));
+  // idx_unit = (idx >> (MI_SIZE_LOG2 + ss_x));
+  // shift = (idy_unit << stride_log2) + idx_unit;
+  return (idy & ~7) | (idx >> 3);
+}
+
+static void check_mask_y(const FilterMaskY *lfm) {
+#ifndef NDEBUG
+  int i;
+  for (i = 0; i < 4; ++i) {
+    assert(!(lfm[TX_4X4].bits[i] & lfm[TX_8X8].bits[i]));
+    assert(!(lfm[TX_4X4].bits[i] & lfm[TX_16X16].bits[i]));
+    assert(!(lfm[TX_4X4].bits[i] & lfm[TX_32X32].bits[i]));
+    assert(!(lfm[TX_8X8].bits[i] & lfm[TX_16X16].bits[i]));
+    assert(!(lfm[TX_8X8].bits[i] & lfm[TX_32X32].bits[i]));
+    assert(!(lfm[TX_16X16].bits[i] & lfm[TX_32X32].bits[i]));
+  }
+#else
+  (void)lfm;
+#endif
+}
+
+static void check_mask_uv(const FilterMaskUV *lfm) {
+#ifndef NDEBUG
+  int i;
+  for (i = 0; i < 4; ++i) {
+    assert(!(lfm[TX_4X4] & lfm[TX_8X8]));
+    assert(!(lfm[TX_4X4] & lfm[TX_16X16]));
+    assert(!(lfm[TX_4X4] & lfm[TX_32X32]));
+    assert(!(lfm[TX_8X8] & lfm[TX_16X16]));
+    assert(!(lfm[TX_8X8] & lfm[TX_32X32]));
+    assert(!(lfm[TX_16X16] & lfm[TX_32X32]));
+  }
+#else
+  (void)lfm;
+#endif
+}
+
+// mi_row, mi_col represent the starting postion of the coding block for
+// which the mask is built. idx, idy represet the offset from the startint
+// point, in the unit of actual distance. For example (idx >> MI_SIZE_LOG2)
+// is in the unit of MI.
+// static void setup_masks()
+#endif  // LOOP_FILTER_BITMASK
+
 static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                          int pitch, unsigned int mask_16x16_l,
                                          unsigned int mask_8x8_l,
diff --git a/av1/common/av1_loopfilter.h b/av1/common/av1_loopfilter.h
index 0c49d51..767cdf1 100644
--- a/av1/common/av1_loopfilter.h
+++ b/av1/common/av1_loopfilter.h
@@ -61,12 +61,12 @@
   FilterMaskUV above_v[TX_SIZES];
 
   // Y plane vertical edge and horizontal edge filter level
-  uint8_t lfl_y_hor[MAX_MIB_SIZE / 2][MAX_MIB_SIZE / 2];
-  uint8_t lfl_y_ver[MAX_MIB_SIZE / 2][MAX_MIB_SIZE / 2];
+  uint8_t lfl_y_hor[MI_SIZE_64X64][MI_SIZE_64X64];
+  uint8_t lfl_y_ver[MI_SIZE_64X64][MI_SIZE_64X64];
 
   // UV plane vertical edge and horizontal edge shares the same level
-  uint8_t lfl_u[MAX_MIB_SIZE / 4][MAX_MIB_SIZE / 4];
-  uint8_t lfl_v[MAX_MIB_SIZE / 4][MAX_MIB_SIZE / 4];
+  uint8_t lfl_u[MI_SIZE_64X64 / 2][MI_SIZE_64X64 / 2];
+  uint8_t lfl_v[MI_SIZE_64X64 / 2][MI_SIZE_64X64 / 2];
 } LoopFilterMaskInfo;
 // TODO(chengchen): remove old version of bitmask construction code once
 // new bitmask is complete.
@@ -88,7 +88,7 @@
 // superblock bitmask building.
 // Information includes:
 // ------------------------------------------------------------
-//                    MAX_MIB_SIZE
+//                    MI_SIZE_64X64
 // Y  tx_size above |--------------|
 // Y  tx_size left  |--------------|
 // UV tx_size above |--------------|
@@ -102,17 +102,17 @@
 // skip             |--------------|
 // ------------------------------------------------------------
 typedef struct {
-  TX_SIZE tx_size_y_above[MAX_MIB_SIZE];
-  TX_SIZE tx_size_y_left[MAX_MIB_SIZE];
-  TX_SIZE tx_size_uv_above[MAX_MIB_SIZE];
-  TX_SIZE tx_size_uv_left[MAX_MIB_SIZE];
-  uint8_t y_level_above[MAX_MIB_SIZE];
-  uint8_t y_level_left[MAX_MIB_SIZE];
-  uint8_t u_level_above[MAX_MIB_SIZE];
-  uint8_t u_level_left[MAX_MIB_SIZE];
-  uint8_t v_level_above[MAX_MIB_SIZE];
-  uint8_t v_level_left[MAX_MIB_SIZE];
-  uint8_t skip[MAX_MIB_SIZE];
+  TX_SIZE tx_size_y_above[MI_SIZE_64X64];
+  TX_SIZE tx_size_y_left[MI_SIZE_64X64];
+  TX_SIZE tx_size_uv_above[MI_SIZE_64X64];
+  TX_SIZE tx_size_uv_left[MI_SIZE_64X64];
+  uint8_t y_level_above[MI_SIZE_64X64];
+  uint8_t y_level_left[MI_SIZE_64X64];
+  uint8_t u_level_above[MI_SIZE_64X64];
+  uint8_t u_level_left[MI_SIZE_64X64];
+  uint8_t v_level_above[MI_SIZE_64X64];
+  uint8_t v_level_left[MI_SIZE_64X64];
+  uint8_t skip[MI_SIZE_64X64];
 } LpfSuperblockInfo;
 #endif  // LOOP_FILTER_BITMASK
 
@@ -141,7 +141,7 @@
   size_t lfm_num;
   int lfm_stride;
   LpfSuperblockInfo neighbor_sb_lpf_info;
-#endif
+#endif  // LOOP_FILTER_BITMASK
 };
 
 // Need to align this structure so when it is declared and