Support rectangular tx_size in level-map optimize_b

Validate the provided coefficient location with respect to the
height and width of the transform block size.

Change-Id: Id4f10052141fd914f5aea5ae4202cf35d3e63867

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 1fd462a..bfa8f3d 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -868,7 +868,7 @@
 
       const int nb_scan_idx = iscan[nb_coeff_idx];
       if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
-          nb_row < txb_info->stride && nb_col < txb_info->stride) {
+          nb_row < txb_info->height && nb_col < txb_info->stride) {
         const int cost_diff = try_neighbor_level_down_nz(
             nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info);
         if (cost_map)
@@ -886,7 +886,7 @@
       const int nb_coeff_idx = nb_row * txb_info->stride + nb_col;
       const int nb_scan_idx = iscan[nb_coeff_idx];
       if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
-          nb_row < txb_info->stride && nb_col < txb_info->stride) {
+          nb_row < txb_info->height && nb_col < txb_info->stride) {
         const int cost_diff = try_neighbor_level_down_base(
             nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info);
         if (cost_map)
@@ -904,7 +904,7 @@
       const int nb_coeff_idx = nb_row * txb_info->stride + nb_col;
       const int nb_scan_idx = iscan[nb_coeff_idx];
       if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
-          nb_row < txb_info->stride && nb_col < txb_info->stride) {
+          nb_row < txb_info->height && nb_col < txb_info->stride) {
         const int cost_diff = try_neighbor_level_down_br(
             nb_coeff_idx, coeff_idx, txb_cache, txb_probs, txb_info);
         if (cost_map)
@@ -1032,7 +1032,7 @@
     const int nb_coeff_idx = nb_row * txb_info->stride + nb_col;
     const int nb_scan_idx = iscan[nb_coeff_idx];
     if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
-        nb_row < txb_info->stride && nb_col < txb_info->stride) {
+        nb_row < txb_info->height && nb_col < txb_info->stride) {
       const int scan_idx = iscan[coeff_idx];
       if (scan_idx < nb_scan_idx) {
         const int level = 1;
@@ -1060,7 +1060,7 @@
     if (!has_base(nb_coeff, 0)) continue;
     const int nb_scan_idx = iscan[nb_coeff_idx];
     if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
-        nb_row < txb_info->stride && nb_col < txb_info->stride) {
+        nb_row < txb_info->height && nb_col < txb_info->stride) {
       if (row >= nb_row && col >= nb_col)
         update_mag_arr(txb_cache->base_mag_arr[nb_coeff_idx], abs_qc);
       const int mag =
@@ -1093,7 +1093,7 @@
     const tran_low_t nb_coeff = txb_info->qcoeff[nb_coeff_idx];
     if (!has_br(nb_coeff)) continue;
     if (nb_scan_idx < eob && nb_row >= 0 && nb_col >= 0 &&
-        nb_row < txb_info->stride && nb_col < txb_info->stride) {
+        nb_row < txb_info->height && nb_col < txb_info->stride) {
       const int level = 1 + NUM_BASE_LEVELS;
       if (abs_qc == level) {
         txb_cache->br_count_arr[nb_coeff_idx] -= 1;