[FasterRCNN] use numpy iou implementation to avoid running TF in multiple processes (#546)

examples/FasterRCNN/data.py

@@ -8,13 +8,13 @@ import copy
 
 from tensorpack.utils.argtools import memoized, log_once
 from tensorpack.dataflow import (
-    MapData, imgaug, TestDataSpeed,
+    MapData, imgaug, TestDataSpeed, PrefetchDataZMQ,
     MapDataComponent, DataFromList)
 # import tensorpack.utils.viz as tpviz
 
 from coco import COCODetection
 from utils.generate_anchors import generate_anchors
-from utils.box_ops import get_iou_callable
+from utils.np_box_ops import iou as np_iou
 from common import (
     DataFromListOfDict, CustomResize,
     box_to_point8, point8_to_box, segmentation_to_mask)
@@ -91,10 +91,9 @@ def get_anchor_labels(anchors, gt_boxes, crowd_boxes):
             curr_inds = np.where(labels == value)[0]
         return curr_inds
 
-    bbox_iou_float = get_iou_callable()
     NA, NB = len(anchors), len(gt_boxes)
     assert NB > 0  # empty images should have been filtered already
-    box_ious = bbox_iou_float(anchors, gt_boxes)  # NA x NB
+    box_ious = np_iou(anchors, gt_boxes)  # NA x NB
     ious_argmax_per_anchor = box_ious.argmax(axis=1)  # NA,
     ious_max_per_anchor = box_ious.max(axis=1)
     ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True)  # 1xNB
@@ -113,7 +112,7 @@ def get_anchor_labels(anchors, gt_boxes, crowd_boxes):
     if crowd_boxes.size > 0:
         cand_inds = np.where(anchor_labels >= 0)[0]
         cand_anchors = anchors[cand_inds]
-        ious = bbox_iou_float(cand_anchors, crowd_boxes)
+        ious = np_iou(cand_anchors, crowd_boxes)
         overlap_with_crowd = cand_inds[ious.max(axis=1) > config.CROWD_OVERLAP_THRES]
         anchor_labels[overlap_with_crowd] = -1
 
@@ -231,7 +230,6 @@ def get_train_dataflow(add_mask=False):
 
         ret = [im, fm_labels, fm_boxes, boxes, klass]
 
-        # masks
         if add_mask:
             # augmentation will modify the polys in-place
             segmentation = copy.deepcopy(img.get('segmentation', None))
@@ -254,7 +252,7 @@ def get_train_dataflow(add_mask=False):
         return ret
 
     ds = MapData(ds, preprocess)
-    # ds = PrefetchDataZMQ(ds, 1)
+    ds = PrefetchDataZMQ(ds, 1)
     return ds
 
 
@@ -268,7 +266,7 @@ def get_eval_dataflow():
         assert im is not None, fname
         return im
     ds = MapDataComponent(ds, f, 0)
-    # ds = PrefetchDataZMQ(ds, 1)
+    ds = PrefetchDataZMQ(ds, 1)
     return ds
 
 

examples/FasterRCNN/utils/README.md

@@ -3,4 +3,5 @@
 
 + generate_anchors.py: copied from [py-faster-rcnn](https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/rpn/generate_anchors.py).
 + box_ops.py: modified from [TF object detection API](https://github.com/tensorflow/models/blob/master/research/object_detection/core/box_list_ops.py).
++ np_box_ops.py: copied from [TF object detection API](https://github.com/tensorflow/models/blob/master/research/object_detection/utils/np_box_ops.py).
 

examples/FasterRCNN/utils/box_ops.py

@@ -4,8 +4,6 @@
 
 import tensorflow as tf
 from tensorpack.tfutils.scope_utils import under_name_scope
-from tensorpack.tfutils import get_default_sess_config
-from tensorpack.utils.argtools import memoized
 
 """
 This file is modified from
@@ -67,19 +65,3 @@ def pairwise_iou(boxlist1, boxlist2):
     return tf.where(
         tf.equal(intersections, 0.0),
         tf.zeros_like(intersections), tf.truediv(intersections, unions))
-
-
-@memoized
-def get_iou_callable():
-    """
-    Get a pairwise box iou callable.
-    """
-    # We don't want the dataflow process to touch CUDA
-    # Data needs tensorflow. As a result, the training cannot run on GPUs with
-    # EXCLUSIVE_PROCESS mode, unless you disable multiprocessing prefetch.
-    with tf.Graph().as_default(), tf.device('/cpu:0'):
-        A = tf.placeholder(tf.float32, shape=[None, 4])
-        B = tf.placeholder(tf.float32, shape=[None, 4])
-        iou = pairwise_iou(A, B)
-        sess = tf.Session(config=get_default_sess_config())
-        return sess.make_callable(iou, [A, B])

examples/FasterRCNN/utils/np_box_ops.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Operations for [N, 4] numpy arrays representing bounding boxes.
+
+Example box operations that are supported:
+  * Areas: compute bounding box areas
+  * IOU: pairwise intersection-over-union scores
+"""
+import numpy as np
+
+
+def area(boxes):
+  """Computes area of boxes.
+
+  Args:
+    boxes: Numpy array with shape [N, 4] holding N boxes
+
+  Returns:
+    a numpy array with shape [N*1] representing box areas
+  """
+  return (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
+
+
+def intersection(boxes1, boxes2):
+  """Compute pairwise intersection areas between boxes.
+
+  Args:
+    boxes1: a numpy array with shape [N, 4] holding N boxes
+    boxes2: a numpy array with shape [M, 4] holding M boxes
+
+  Returns:
+    a numpy array with shape [N*M] representing pairwise intersection area
+  """
+  [y_min1, x_min1, y_max1, x_max1] = np.split(boxes1, 4, axis=1)
+  [y_min2, x_min2, y_max2, x_max2] = np.split(boxes2, 4, axis=1)
+
+  all_pairs_min_ymax = np.minimum(y_max1, np.transpose(y_max2))
+  all_pairs_max_ymin = np.maximum(y_min1, np.transpose(y_min2))
+  intersect_heights = np.maximum(
+      np.zeros(all_pairs_max_ymin.shape),
+      all_pairs_min_ymax - all_pairs_max_ymin)
+  all_pairs_min_xmax = np.minimum(x_max1, np.transpose(x_max2))
+  all_pairs_max_xmin = np.maximum(x_min1, np.transpose(x_min2))
+  intersect_widths = np.maximum(
+      np.zeros(all_pairs_max_xmin.shape),
+      all_pairs_min_xmax - all_pairs_max_xmin)
+  return intersect_heights * intersect_widths
+
+
+def iou(boxes1, boxes2):
+  """Computes pairwise intersection-over-union between box collections.
+
+  Args:
+    boxes1: a numpy array with shape [N, 4] holding N boxes.
+    boxes2: a numpy array with shape [M, 4] holding M boxes.
+
+  Returns:
+    a numpy array with shape [N, M] representing pairwise iou scores.
+  """
+  intersect = intersection(boxes1, boxes2)
+  area1 = area(boxes1)
+  area2 = area(boxes2)
+  union = np.expand_dims(area1, axis=1) + np.expand_dims(
+      area2, axis=0) - intersect
+  return intersect / union
+
+
+def ioa(boxes1, boxes2):
+  """Computes pairwise intersection-over-area between box collections.
+
+  Intersection-over-area (ioa) between two boxes box1 and box2 is defined as
+  their intersection area over box2's area. Note that ioa is not symmetric,
+  that is, IOA(box1, box2) != IOA(box2, box1).
+
+  Args:
+    boxes1: a numpy array with shape [N, 4] holding N boxes.
+    boxes2: a numpy array with shape [M, 4] holding N boxes.
+
+  Returns:
+    a numpy array with shape [N, M] representing pairwise ioa scores.
+  """
+  intersect = intersection(boxes1, boxes2)
+  areas = np.expand_dims(area(boxes2), axis=0)
+  return intersect / areas

examples/FasterRCNN/viz.py

@@ -8,7 +8,7 @@ import numpy as np
 from tensorpack.utils import viz
 from tensorpack.utils.palette import PALETTE_RGB
 
-from utils.box_ops import get_iou_callable
+from utils.np_box_ops import iou as np_iou
 import config
 
 
@@ -37,8 +37,7 @@ def draw_proposal_recall(img, proposals, proposal_scores, gt_boxes):
         proposal_scores: NP
         gt_boxes: NG
     """
-    bbox_iou_float = get_iou_callable()
-    box_ious = bbox_iou_float(gt_boxes, proposals)    # ng x np
+    box_ious = np_iou(gt_boxes, proposals)    # ng x np
     box_ious_argsort = np.argsort(-box_ious, axis=1)
     good_proposals_ind = box_ious_argsort[:, :3]   # for each gt, find 3 best proposals
     good_proposals_ind = np.unique(good_proposals_ind.ravel())