[MaskRCNN] move some functions around

docs/modules/utils.rst

 tensorpack.utils package
 ========================
 
+.. automodule:: tensorpack.utils
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 tensorpack.utils.argtools module
 --------------------------------
 

examples/FasterRCNN/common.py

@@ -141,3 +141,24 @@ def filter_boxes_inside_shape(boxes, shape):
         (boxes[:, 2] <= w) &
         (boxes[:, 3] <= h))[0]
     return indices, boxes[indices, :]
+
+
+try:
+    import pycocotools.mask as cocomask
+
+    # Much faster than utils/np_box_ops
+    def np_iou(A, B):
+        def to_xywh(box):
+            box = box.copy()
+            box[:, 2] -= box[:, 0]
+            box[:, 3] -= box[:, 1]
+            return box
+
+        ret = cocomask.iou(
+            to_xywh(A), to_xywh(B),
+            np.zeros((len(B),), dtype=np.bool))
+        # can accelerate even more, if using float32
+        return ret.astype('float32')
+
+except ImportError:
+    from utils.np_box_ops import iou as np_iou  # noqa

examples/FasterRCNN/config.py

@@ -31,7 +31,7 @@ class AttrDict():
         super().__setattr__(name, value)
 
     def __str__(self):
-        return pprint.pformat(self.to_dict(), indent=1)
+        return pprint.pformat(self.to_dict(), indent=1, width=100, compact=True)
 
     __repr__ = __str__
 

examples/FasterRCNN/data.py

@@ -13,37 +13,16 @@ from tensorpack.utils import logger
 from tensorpack.utils.argtools import log_once, memoized
 
 from common import (
-    CustomResize, DataFromListOfDict, box_to_point8, filter_boxes_inside_shape, point8_to_box, segmentation_to_mask)
+    CustomResize, DataFromListOfDict, box_to_point8,
+    filter_boxes_inside_shape, point8_to_box, segmentation_to_mask, np_iou)
 from config import config as cfg
 from dataset import DetectionDataset
 from utils.generate_anchors import generate_anchors
-from utils.np_box_ops import area as np_area
-from utils.np_box_ops import ioa as np_ioa
+from utils.np_box_ops import area as np_area, ioa as np_ioa
 
 # import tensorpack.utils.viz as tpviz
 
 
-try:
-    import pycocotools.mask as cocomask
-
-    # Much faster than utils/np_box_ops
-    def np_iou(A, B):
-        def to_xywh(box):
-            box = box.copy()
-            box[:, 2] -= box[:, 0]
-            box[:, 3] -= box[:, 1]
-            return box
-
-        ret = cocomask.iou(
-            to_xywh(A), to_xywh(B),
-            np.zeros((len(B),), dtype=np.bool))
-        # can accelerate even more, if using float32
-        return ret.astype('float32')
-
-except ImportError:
-    from utils.np_box_ops import iou as np_iou
-
-
 class MalformedData(BaseException):
     pass
 
@@ -143,7 +122,7 @@ def get_anchor_labels(anchors, gt_boxes, crowd_boxes):
     Label each anchor as fg/bg/ignore.
     Args:
         anchors: Ax4 float
-        gt_boxes: Bx4 float
+        gt_boxes: Bx4 float, non-crowd
         crowd_boxes: Cx4 float
 
     Returns:

examples/FasterRCNN/eval.py

@@ -2,6 +2,8 @@
 # File: eval.py
 
 import itertools
+import os
+import json
 import numpy as np
 from collections import namedtuple
 from concurrent.futures import ThreadPoolExecutor
@@ -9,12 +11,24 @@ from contextlib import ExitStack
 import cv2
 import pycocotools.mask as cocomask
 import tqdm
+import tensorflow as tf
 
-from tensorpack.utils.utils import get_tqdm_kwargs
+from tensorpack.callbacks import Callback
+from tensorpack.tfutils import get_tf_version_tuple
+from tensorpack.utils import logger
+from tensorpack.utils.utils import get_tqdm
 
 from common import CustomResize, clip_boxes
+from data import get_eval_dataflow
+from dataset import DetectionDataset
 from config import config as cfg
 
+try:
+    import horovod.tensorflow as hvd
+except ImportError:
+    pass
+
+
 DetectionResult = namedtuple(
     'DetectionResult',
     ['box', 'score', 'class_id', 'mask'])
@@ -26,7 +40,7 @@ mask: None, or a binary image of the original image shape
 """
 
 
-def paste_mask(box, mask, shape):
+def _paste_mask(box, mask, shape):
     """
     Args:
         box: 4 float
@@ -79,7 +93,7 @@ def predict_image(img, model_func):
 
     if masks:
         # has mask
-        full_masks = [paste_mask(box, mask, orig_shape)
+        full_masks = [_paste_mask(box, mask, orig_shape)
                       for box, mask in zip(boxes, masks[0])]
         masks = full_masks
     else:
@@ -105,11 +119,10 @@ def predict_dataflow(df, model_func, tqdm_bar=None):
     """
     df.reset_state()
     all_results = []
-    # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323
     with ExitStack() as stack:
+        # tqdm is not quite thread-safe: https://github.com/tqdm/tqdm/issues/323
         if tqdm_bar is None:
-            tqdm_bar = stack.enter_context(
-                tqdm.tqdm(total=df.size(), **get_tqdm_kwargs()))
+            tqdm_bar = stack.enter_context(get_tqdm(total=df.size()))
         for img, img_id in df:
             results = predict_image(img, model_func)
             for r in results:
@@ -143,8 +156,10 @@ def multithread_predict_dataflow(dataflows, model_funcs):
         list of dict, in the format used by
         `DetectionDataset.eval_or_save_inference_results`
     """
-    num_worker = len(dataflows)
-    assert len(dataflows) == len(model_funcs)
+    num_worker = len(model_funcs)
+    assert len(dataflows) == num_worker
+    if num_worker == 1:
+        return predict_dataflow(dataflows[0], model_funcs[0])
     with ThreadPoolExecutor(max_workers=num_worker, thread_name_prefix='EvalWorker') as executor, \
             tqdm.tqdm(total=sum([df.size() for df in dataflows])) as pbar:
         futures = []
@@ -152,3 +167,90 @@ def multithread_predict_dataflow(dataflows, model_funcs):
             futures.append(executor.submit(predict_dataflow, dataflow, pred, pbar))
         all_results = list(itertools.chain(*[fut.result() for fut in futures]))
         return all_results
+
+
+class EvalCallback(Callback):
+    """
+    A callback that runs evaluation once a while.
+    It supports multi-gpu evaluation.
+    """
+
+    _chief_only = False
+
+    def __init__(self, eval_dataset, in_names, out_names, output_dir):
+        self._eval_dataset = eval_dataset
+        self._in_names, self._out_names = in_names, out_names
+        self._output_dir = output_dir
+
+    def _setup_graph(self):
+        num_gpu = cfg.TRAIN.NUM_GPUS
+        if cfg.TRAINER == 'replicated':
+            # TF bug in version 1.11, 1.12: https://github.com/tensorflow/tensorflow/issues/22750
+            buggy_tf = get_tf_version_tuple() in [(1, 11), (1, 12)]
+
+            # Use two predictor threads per GPU to get better throughput
+            self.num_predictor = num_gpu if buggy_tf else num_gpu * 2
+            self.predictors = [self._build_predictor(k % num_gpu) for k in range(self.num_predictor)]
+            self.dataflows = [get_eval_dataflow(self._eval_dataset,
+                                                shard=k, num_shards=self.num_predictor)
+                              for k in range(self.num_predictor)]
+        else:
+            # Only eval on the first machine.
+            # Alternatively, can eval on all ranks and use allgather, but allgather sometimes hangs
+            self._horovod_run_eval = hvd.rank() == hvd.local_rank()
+            if self._horovod_run_eval:
+                self.predictor = self._build_predictor(0)
+                self.dataflow = get_eval_dataflow(self._eval_dataset,
+                                                  shard=hvd.local_rank(), num_shards=hvd.local_size())
+
+            self.barrier = hvd.allreduce(tf.random_normal(shape=[1]))
+
+    def _build_predictor(self, idx):
+        return self.trainer.get_predictor(self._in_names, self._out_names, device=idx)
+
+    def _before_train(self):
+        eval_period = cfg.TRAIN.EVAL_PERIOD
+        self.epochs_to_eval = set()
+        for k in itertools.count(1):
+            if k * eval_period > self.trainer.max_epoch:
+                break
+            self.epochs_to_eval.add(k * eval_period)
+        self.epochs_to_eval.add(self.trainer.max_epoch)
+        logger.info("[EvalCallback] Will evaluate every {} epochs".format(eval_period))
+
+    def _eval(self):
+        logdir = self._output_dir
+        if cfg.TRAINER == 'replicated':
+            all_results = multithread_predict_dataflow(self.dataflows, self.predictors)
+        else:
+            filenames = [os.path.join(
+                logdir, 'outputs{}-part{}.json'.format(self.global_step, rank)
+            ) for rank in range(hvd.local_size())]
+
+            if self._horovod_run_eval:
+                local_results = predict_dataflow(self.dataflow, self.predictor)
+                fname = filenames[hvd.local_rank()]
+                with open(fname, 'w') as f:
+                    json.dump(local_results, f)
+            self.barrier.eval()
+            if hvd.rank() > 0:
+                return
+            all_results = []
+            for fname in filenames:
+                with open(fname, 'r') as f:
+                    obj = json.load(f)
+                all_results.extend(obj)
+                os.unlink(fname)
+
+        output_file = os.path.join(
+            logdir, '{}-outputs{}.json'.format(self._eval_dataset, self.global_step))
+
+        scores = DetectionDataset().eval_or_save_inference_results(
+            all_results, self._eval_dataset, output_file)
+        for k, v in scores.items():
+            self.trainer.monitors.put_scalar(k, v)
+
+    def _trigger_epoch(self):
+        if self.epoch_num in self.epochs_to_eval:
+            logger.info("Running evaluation ...")
+            self._eval()

examples/FasterRCNN/train.py

@@ -4,12 +4,12 @@
 
 import argparse
 import itertools
-import json
 import numpy as np
 import os
 import shutil
 import cv2
 import six
+assert six.PY3, "FasterRCNN requires Python 3!"
 import tensorflow as tf
 import tqdm
 
@@ -25,7 +25,7 @@ from basemodel import image_preprocess, resnet_c4_backbone, resnet_conv5, resnet
 from dataset import DetectionDataset
 from config import finalize_configs, config as cfg
 from data import get_all_anchors, get_all_anchors_fpn, get_eval_dataflow, get_train_dataflow
-from eval import DetectionResult, predict_image, predict_dataflow, multithread_predict_dataflow
+from eval import DetectionResult, predict_image, multithread_predict_dataflow, EvalCallback
 from model_box import RPNAnchors, clip_boxes, crop_and_resize, roi_align
 from model_cascade import CascadeRCNNHead
 from model_fpn import fpn_model, generate_fpn_proposals, multilevel_roi_align, multilevel_rpn_losses
@@ -39,8 +39,6 @@ try:
 except ImportError:
     pass
 
-assert six.PY3, "FasterRCNN requires Python 3!"
-
 
 class DetectionModel(ModelDesc):
     def preprocess(self, image):
@@ -56,7 +54,7 @@ class DetectionModel(ModelDesc):
         lr = tf.get_variable('learning_rate', initializer=0.003, trainable=False)
         tf.summary.scalar('learning_rate-summary', lr)
 
-        # The learning rate is set for 8 GPUs, and we use trainers with average=False.
+        # The learning rate in the config is set for 8 GPUs, and we use trainers with average=False.
         lr = lr / 8.
         opt = tf.train.MomentumOptimizer(lr, 0.9)
         if cfg.TRAIN.NUM_GPUS < 8:
@@ -384,10 +382,7 @@ def do_evaluate(pred_config, output_file):
         dataflows = [
             get_eval_dataflow(dataset, shard=k, num_shards=num_gpu)
             for k in range(num_gpu)]
-        if num_gpu > 1:
-            all_results = multithread_predict_dataflow(dataflows, graph_funcs)
-        else:
-            all_results = predict_dataflow(dataflows[0], graph_funcs[0])
+        all_results = multithread_predict_dataflow(dataflows, graph_funcs)
         output = output_file + '-' + dataset
         DetectionDataset().eval_or_save_inference_results(all_results, dataset, output)
 
@@ -402,92 +397,6 @@ def do_predict(pred_func, input_file):
     tpviz.interactive_imshow(viz)
 
 
-class EvalCallback(Callback):
-    """
-    A callback that runs COCO evaluation once a while.
-    It supports multi-gpu evaluation.
-    """
-
-    _chief_only = False
-
-    def __init__(self, eval_dataset, in_names, out_names):
-        self._eval_dataset = eval_dataset
-        self._in_names, self._out_names = in_names, out_names
-
-    def _setup_graph(self):
-        num_gpu = cfg.TRAIN.NUM_GPUS
-        if cfg.TRAINER == 'replicated':
-            # TF bug in version 1.11, 1.12: https://github.com/tensorflow/tensorflow/issues/22750
-            buggy_tf = get_tf_version_tuple() in [(1, 11), (1, 12)]
-
-            # Use two predictor threads per GPU to get better throughput
-            self.num_predictor = num_gpu if buggy_tf else num_gpu * 2
-            self.predictors = [self._build_predictor(k % num_gpu) for k in range(self.num_predictor)]
-            self.dataflows = [get_eval_dataflow(self._eval_dataset,
-                                                shard=k, num_shards=self.num_predictor)
-                              for k in range(self.num_predictor)]
-        else:
-            # Only eval on the first machine.
-            # Alternatively, can eval on all ranks and use allgather, but allgather sometimes hangs
-            self._horovod_run_eval = hvd.rank() == hvd.local_rank()
-            if self._horovod_run_eval:
-                self.predictor = self._build_predictor(0)
-                self.dataflow = get_eval_dataflow(self._eval_dataset,
-                                                  shard=hvd.local_rank(), num_shards=hvd.local_size())
-
-            self.barrier = hvd.allreduce(tf.random_normal(shape=[1]))
-
-    def _build_predictor(self, idx):
-        return self.trainer.get_predictor(self._in_names, self._out_names, device=idx)
-
-    def _before_train(self):
-        eval_period = cfg.TRAIN.EVAL_PERIOD
-        self.epochs_to_eval = set()
-        for k in itertools.count(1):
-            if k * eval_period > self.trainer.max_epoch:
-                break
-            self.epochs_to_eval.add(k * eval_period)
-        self.epochs_to_eval.add(self.trainer.max_epoch)
-        logger.info("[EvalCallback] Will evaluate every {} epochs".format(eval_period))
-
-    def _eval(self):
-        logdir = args.logdir
-        if cfg.TRAINER == 'replicated':
-            all_results = multithread_predict_dataflow(self.dataflows, self.predictors)
-        else:
-            filenames = [os.path.join(
-                logdir, 'outputs{}-part{}.json'.format(self.global_step, rank)
-            ) for rank in range(hvd.local_size())]
-
-            if self._horovod_run_eval:
-                local_results = predict_dataflow(self.dataflow, self.predictor)
-                fname = filenames[hvd.local_rank()]
-                with open(fname, 'w') as f:
-                    json.dump(local_results, f)
-            self.barrier.eval()
-            if hvd.rank() > 0:
-                return
-            all_results = []
-            for fname in filenames:
-                with open(fname, 'r') as f:
-                    obj = json.load(f)
-                all_results.extend(obj)
-                os.unlink(fname)
-
-        output_file = os.path.join(
-            logdir, '{}-outputs{}.json'.format(self._eval_dataset, self.global_step))
-
-        scores = DetectionDataset().eval_or_save_inference_results(
-            all_results, self._eval_dataset, output_file)
-        for k, v in scores.items():
-            self.trainer.monitors.put_scalar(k, v)
-
-    def _trigger_epoch(self):
-        if self.epoch_num in self.epochs_to_eval:
-            logger.info("Running evaluation ...")
-            self._eval()
-
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--load', help='load a model for evaluation or training. Can overwrite BACKBONE.WEIGHTS')
@@ -574,7 +483,7 @@ if __name__ == '__main__':
             EstimatedTimeLeft(median=True),
             SessionRunTimeout(60000).set_chief_only(True),   # 1 minute timeout
         ] + [
-            EvalCallback(dataset, *MODEL.get_inference_tensor_names())
+            EvalCallback(dataset, *MODEL.get_inference_tensor_names(), args.logdir)
             for dataset in cfg.DATA.VAL
         ]
         if not is_horovod:

tensorpack/predict/config.py

@@ -41,8 +41,10 @@ class PredictConfig(object):
                 the list of inputs it takes.
 
             input_names (list): a list of input tensor names. Defaults to match inputs_desc.
+                The name can be either the name of a tensor, or the name of one input defined
+                by `inputs_desc` or by `model`.
             output_names (list): a list of names of the output tensors to predict, the
-                tensors can be any computable tensor in the graph.
+                tensors can be any tensor in the graph that's computable from the tensors correponding to `input_names`.
 
             session_creator (tf.train.SessionCreator): how to create the
                 session. Defaults to :class:`tf.train.ChiefSessionCreator()`.

tensorpack/tfutils/common.py

@@ -11,6 +11,7 @@ from ..utils.develop import deprecated
 __all__ = ['get_default_sess_config',
            'get_global_step_value',
            'get_global_step_var',
+           'get_tf_version_tuple'
            # 'get_op_tensor_name',
            # 'get_tensors_by_names',
            # 'get_op_or_tensor_by_name',

tensorpack/utils/__init__.py

 #  -*- coding: utf-8 -*-
 #  File: __init__.py
 
-
-from pkgutil import iter_modules
-import os
-
 """
 Common utils.
 These utils should be irrelevant to tensorflow.
 """
 
-__all__ = []
+# https://github.com/celery/kombu/blob/7d13f9b95d0b50c94393b962e6def928511bfda6/kombu/__init__.py#L34-L36
+STATICA_HACK = True
+globals()['kcah_acitats'[::-1].upper()] = False
+if STATICA_HACK:
+    from .utils import *
 
 
-# this two functions for back-compat only
-def get_nr_gpu():
-    from .gpu import get_nr_gpu as gg
-    logger.warn(    # noqa
-        "Please use `from tensorpack.utils.gpu import get_num_gpu`")
-    return gg()
+__all__ = []
 
 
-def change_gpu(val):
-    from .gpu import change_gpu as cg
-    logger.warn(    # noqa
-        "change_gpu will not be automatically imported any more! "
-        "Please do `from tensorpack.utils.gpu import change_gpu`")
-    return cg(val)
+def _global_import(name):
+    p = __import__(name, globals(), None, level=1)
+    lst = p.__all__ if '__all__' in dir(p) else dir(p)
+    for k in lst:
+        if not k.startswith('__'):
+            globals()[k] = p.__dict__[k]
+            __all__.append(k)
 
 
-def get_rng(obj=None):
-    from .utils import get_rng as gr
-    logger.warn(    # noqa
-        "get_rng will not be automatically imported any more! "
-        "Please do `from tensorpack.utils.utils import get_rng`")
-    return gr(obj)
+_global_import('utils')
 
-# Import no submodules. they are supposed to be explicitly imported by users.
-__all__.extend(['logger', 'get_nr_gpu', 'change_gpu', 'get_rng'])
+# Import no other submodules. they are supposed to be explicitly imported by users.
+__all__.extend(['logger'])