lint & TF2

.github/workflows/workflow.yml

@@ -22,6 +22,7 @@ jobs:
       - name: Lint
         run: |
           flake8 .
+          cd examples && flake8 .
 
   unittest:
     runs-on: ubuntu-latest

examples/A3C-Gym/train-atari.py

@@ -208,7 +208,7 @@ class MySimulatorMaster(SimulatorMaster, Callback):
 
         mem.reverse()
         R = float(init_r)
-        for idx, k in enumerate(mem):
+        for k in mem:
             R = np.clip(k.reward, -1, 1) + GAMMA * R
             self.queue.put([k.state, k.action, R, k.prob])
 

examples/CTC-TIMIT/timitdata.py

@@ -11,7 +11,7 @@ __all__ = ['TIMITBatch']
 
 def batch_feature(feats):
     # pad to the longest in the batch
-    maxlen = max([k.shape[0] for k in feats])
+    maxlen = max(k.shape[0] for k in feats)
     bsize = len(feats)
     ret = np.zeros((bsize, maxlen, feats[0].shape[1]))
     for idx, feat in enumerate(feats):
@@ -20,7 +20,7 @@ def batch_feature(feats):
 
 
 def sparse_label(labels):
-    maxlen = max([k.shape[0] for k in labels])
+    maxlen = max(k.shape[0] for k in labels)
     shape = [len(labels), maxlen]   # bxt
     indices = []
     values = []
@@ -47,7 +47,7 @@ class TIMITBatch(ProxyDataFlow):
         for _ in range(self.__len__()):
             feats = []
             labs = []
-            for b in range(self.batch):
+            for _ in range(self.batch):
                 feat, lab = next(itr)
                 feats.append(feat)
                 labs.append(lab)

examples/CaffeModels/load-alexnet.py

 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # File: load-alexnet.py
 # Author: Yuxin Wu
 

examples/Char-RNN/char-rnn.py

@@ -155,7 +155,7 @@ def sample(path, start, length):
     # generate more
     ret = start
     c = start[-1]
-    for k in range(length):
+    for _ in range(length):
         x = np.array([[ds.char2idx[c]]], dtype='int32')
         prob, state = pred(x, state[0, 0], state[0, 1], state[1, 0], state[1, 1])
         c = ds.chars[pick(prob[0])]

examples/DynamicFilterNetwork/steering-filter.py

@@ -211,7 +211,7 @@ class ThetaImages(ProxyDataFlow, RNGDataFlow):
         RNGDataFlow.reset_state(self)
 
     def __iter__(self):
-        for image, label in self.ds:
+        for image, _ in self.ds:
             theta = self.rng.uniform(0, 2 * np.pi)
             filtered_image, gt_filter = ThetaImages.filter_with_theta(image, theta)
             yield [theta, image, filtered_image, gt_filter]

examples/FasterRCNN/NOTES.md

@@ -44,7 +44,7 @@ This is a minimal implementation that simply contains these files:
 
 <p align="center"> <img src="https://user-images.githubusercontent.com/1381301/31527740-2f1b38ce-af84-11e7-8de1-628e90089826.png"> </p>
 
-2. We use ROIAlign, and `tf.image.crop_and_resize` is __NOT__ ROIAlign.
+2. We use ROIAlign, and `tf.image.crop_and_resize` is [__NOT__](https://github.com/tensorflow/tensorflow/issues/26278) ROIAlign.
 
 3. We currently only support single image per GPU in this example.
 
@@ -60,10 +60,10 @@ Training throughput (larger is better) of standard R50-FPN Mask R-CNN, on 8 V100
 
 | Implementation                                                                                    | Throughput (img/s) |
 |---------------------------------------------------------------------------------------------------|:------------------:|
-| [Detectron2](https://github.com/facebookresearch/detectron2)                                      | 60                 |
-| [maskrcnn-benchmark](https://github.com/facebookresearch/maskrcnn-benchmark/)                     | 51                 |
+| [Detectron2](https://github.com/facebookresearch/detectron2)                                      | 62                 |
+| [mmdetection](https://github.com/open-mmlab/mmdetection/blob/master/docs/MODEL_ZOO.md#mask-r-cnn) | 53                 |
+| [maskrcnn-benchmark](https://github.com/facebookresearch/maskrcnn-benchmark/)                     | 53                 |
 | tensorpack                                                                                        | 50                 |
-| [mmdetection](https://github.com/open-mmlab/mmdetection/blob/master/docs/MODEL_ZOO.md#mask-r-cnn) | 41                 |
 | [Detectron](https://github.com/facebookresearch/Detectron)                                        | 19                 |
 | [matterport/Mask_RCNN](https://github.com/matterport/Mask_RCNN/)                                  | 14                 |
 
@@ -92,14 +92,13 @@ Training throughput (larger is better) of standard R50-FPN Mask R-CNN, on 8 V100
 	 in standard format but it does not help you on optimized inference.
 	 In fact, the current implementation uses some slow numpy operations in inference (in `eval.py:_paste_mask`).
 
-Possible Future Enhancements:
+Possible Future Speed Enhancements:
 
 1. Support batch>1 per GPU. Batching with inconsistent shapes is
    non-trivial to implement in TensorFlow.
 
-1. Use dedicated ops to improve speed. (e.g. a TF implementation of ROIAlign op
-   can be found in [light-head RCNN](https://github.com/zengarden/light_head_rcnn/tree/master/lib/lib_kernel))
-
+1. Use dedicated CUDA ops. (e.g. [ROIAlign](https://github.com/zengarden/light_head_rcnn/tree/master/lib/lib_kernel) or
+   `tf.image.generate_bounding_box_proposals`)
 
 ### TensorFlow version notes
 
@@ -118,6 +117,8 @@ Therefore, not every version of TF ≥ 1.6 supports every feature in this implem
 1. TF 1.13: MKL inference will fail ([issue](https://github.com/tensorflow/tensorflow/issues/24650)).
 1. TF > 1.12: Horovod training will fail ([issue](https://github.com/tensorflow/tensorflow/issues/25946)).
    Latest tensorpack will apply a workaround.
+1. TF > 1.14: NCCL produce wrong gradients ([issue](https://github.com/tensorflow/tensorflow/issues/41539)).
+   Latest tensorpack will avoid using NCCL.
 
 This implementation contains workaround for some of these TF bugs.
 However, note that the workaround needs to check your TF version by `tf.VERSION`,

examples/FasterRCNN/config.py

@@ -59,7 +59,7 @@ class AttrDict():
             keylist = keys.split('.')
 
             dic = self
-            for i, k in enumerate(keylist[:-1]):
+            for k in keylist[:-1]:
                 assert k in dir(dic), "Unknown config key: {}".format(keys)
                 dic = getattr(dic, k)
             key = keylist[-1]

examples/FasterRCNN/data.py

@@ -346,7 +346,7 @@ def get_train_dataflow():
     # The model does support training with empty images, but it is not useful for COCO.
     num = len(roidbs)
     if cfg.DATA.FILTER_EMPTY_ANNOTATIONS:
-        roidbs = list(filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, roidbs))       
+        roidbs = list(filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, roidbs))
     logger.info(
         "Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}".format(
             num - len(roidbs), len(roidbs)
@@ -406,5 +406,5 @@ if __name__ == "__main__":
     ds = get_train_dataflow()
     ds = PrintData(ds, 10)
     TestDataSpeed(ds, 50000).start()
-    for k in ds:
+    for _ in ds:
         pass

examples/FasterRCNN/eval.py

@@ -200,7 +200,7 @@ def multithread_predict_dataflow(dataflows, model_funcs):
         return predict_dataflow(dataflows[0], model_funcs[0])
     kwargs = {'thread_name_prefix': 'EvalWorker'} if sys.version_info.minor >= 6 else {}
     with ThreadPoolExecutor(max_workers=num_worker, **kwargs) as executor, \
-            tqdm.tqdm(total=sum([df.size() for df in dataflows])) as pbar:
+            tqdm.tqdm(total=sum(df.size() for df in dataflows)) as pbar:
         futures = []
         for dataflow, pred in zip(dataflows, model_funcs):
             futures.append(executor.submit(predict_dataflow, dataflow, pred, pbar))

examples/FasterRCNN/modeling/generalized_rcnn.py

@@ -218,7 +218,7 @@ class ResNetFPNModel(GeneralizedRCNN):
         return ret
 
     def slice_feature_and_anchors(self, p23456, anchors):
-        for i, stride in enumerate(cfg.FPN.ANCHOR_STRIDES):
+        for i in range(len(cfg.FPN.ANCHOR_STRIDES)):
             with tf.name_scope('FPN_slice_lvl{}'.format(i)):
                 anchors[i] = anchors[i].narrow_to(p23456[i])
 

examples/README.md

@@ -23,7 +23,7 @@ so you're confident that they are correct.
 ## Getting Started:
 These are the only toy examples in tensorpack. They are supposed to be just demos.
 + [An illustrative MNIST example with explanation of the framework](basics/mnist-convnet.py)
-+ Tensorpack supports any symbolic libraries. See the same MNIST example written with [tf.layers](basics/mnist-tflayers.py), [tf-slim](basics/mnist-tfslim.py), and [with weights visualizations](basics/mnist-visualizations.py)
++ Tensorpack supports any symbolic libraries. See the same MNIST example written with [tf.layers](basics/mnist-tflayers.py), and [with weights visualizations](basics/mnist-visualizations.py)
 + A tiny [Cifar ConvNet](basics/cifar-convnet.py) and [SVHN ConvNet](basics/svhn-digit-convnet.py)
 + If you've used Keras, check out [Keras+Tensorpack examples](keras)
 + [A boilerplate file to start with, for your own tasks](boilerplate.py)

examples/SpatialTransformer/mnist-addition.py

@@ -211,9 +211,9 @@ def view_warp(modelpath):
                     [WARP_TARGET_SIZE, WARP_TARGET_SIZE, 1],
                     [0, WARP_TARGET_SIZE, 1]], dtype='float32')
 
-    def draw_rect(img, affine, c, offset=[0, 0]):
+    def draw_rect(img, affine, c, offset=(0, 0)):
         a = np.transpose(affine)  # 3x2
-        a = (np.matmul(xys, a) + offset).astype('int32')
+        a = (np.matmul(xys, a) + list(offset)).astype('int32')
         cv2.line(img, tuple(a[0][::-1]), tuple(a[1][::-1]), c)
         cv2.line(img, tuple(a[1][::-1]), tuple(a[2][::-1]), c)
         cv2.line(img, tuple(a[2][::-1]), tuple(a[3][::-1]), c)

examples/basics/cifar-convnet.py

 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # File: cifar-convnet.py
 # Author: Yuxin Wu
 import argparse
 import os
-import tensorflow as tf
+from tensorpack.compat import tfv1 as tf
 
 from tensorpack import *
 from tensorpack.dataflow import dataset

examples/basics/export-model.py

@@ -3,8 +3,7 @@
 
 import argparse
 import cv2
-import tensorflow as tf
-
+from tensorpack.compat import tfv1 as tf
 from tensorpack import *
 from tensorpack.tfutils.export import ModelExporter
 

examples/basics/mnist-convnet.py

@@ -2,8 +2,7 @@
 # -*- coding: utf-8 -*-
 # File: mnist-convnet.py
 
-import tensorflow as tf
-
+from tensorpack.compat import tfv1 as tf
 from tensorpack import *
 from tensorpack.dataflow import dataset
 from tensorpack.tfutils import summary

examples/basics/mnist-tflayers.py

@@ -2,8 +2,7 @@
 # -*- coding: utf-8 -*-
 # File: mnist-tflayers.py
 
-import tensorflow as tf
-
+from tensorpack.compat import tfv1 as tf
 from tensorpack import *
 from tensorpack.dataflow import dataset
 from tensorpack.tfutils import summary

examples/basics/mnist-tfslim.py

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# File: mnist-tfslim.py
-
-"""
-MNIST ConvNet example using TensorFlow-slim.
-Mostly the same as 'mnist-convnet.py',
-the only differences are:
-    1. use slim.layers, slim.arg_scope, etc
-    2. use slim names to summarize weights
-"""
-
-
-import tensorflow as tf
-import tensorflow.contrib.slim as slim
-
-from tensorpack import *
-from tensorpack.dataflow import dataset
-
-IMAGE_SIZE = 28
-
-
-class Model(ModelDesc):
-    def inputs(self):
-        return [tf.TensorSpec((None, IMAGE_SIZE, IMAGE_SIZE), tf.float32, 'input'),
-                tf.TensorSpec((None,), tf.int32, 'label')]
-
-    def build_graph(self, image, label):
-        image = tf.expand_dims(image, 3)
-
-        image = image * 2 - 1
-
-        with slim.arg_scope([slim.layers.fully_connected],
-                            weights_regularizer=slim.l2_regularizer(1e-5)):
-            l = slim.layers.conv2d(image, 32, [3, 3], scope='conv0')
-            l = slim.layers.max_pool2d(l, [2, 2], scope='pool0')
-            l = slim.layers.conv2d(l, 32, [3, 3], padding='SAME', scope='conv1')
-            l = slim.layers.conv2d(l, 32, [3, 3], scope='conv2')
-            l = slim.layers.max_pool2d(l, [2, 2], scope='pool1')
-            l = slim.layers.conv2d(l, 32, [3, 3], scope='conv3')
-            l = slim.layers.flatten(l, scope='flatten')
-            l = slim.layers.fully_connected(l, 512, scope='fc0')
-            l = slim.layers.dropout(l, is_training=self.training)
-            logits = slim.layers.fully_connected(l, 10, activation_fn=None, scope='fc1')
-
-        cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label)
-        cost = tf.reduce_mean(cost, name='cross_entropy_loss')
-
-        acc = tf.cast(tf.nn.in_top_k(logits, label, 1), tf.float32)
-
-        acc = tf.reduce_mean(acc, name='accuracy')
-        summary.add_moving_summary(acc)
-
-        summary.add_moving_summary(cost)
-        summary.add_param_summary(('.*/weights', ['histogram', 'rms']))  # slim uses different variable names
-        return cost + regularize_cost_from_collection()
-
-    def optimizer(self):
-        lr = tf.train.exponential_decay(
-            learning_rate=1e-3,
-            global_step=get_global_step_var(),
-            decay_steps=468 * 10,
-            decay_rate=0.3, staircase=True, name='learning_rate')
-        tf.summary.scalar('lr', lr)
-        return tf.train.AdamOptimizer(lr)
-
-
-def get_data():
-    train = BatchData(dataset.Mnist('train'), 128)
-    test = BatchData(dataset.Mnist('test'), 256, remainder=True)
-    return train, test
-
-
-if __name__ == '__main__':
-    logger.auto_set_dir()
-    dataset_train, dataset_test = get_data()
-
-    config = TrainConfig(
-        model=Model(),
-        dataflow=dataset_train,
-        callbacks=[
-            ModelSaver(),
-            InferenceRunner(
-                dataset_test,
-                ScalarStats(['cross_entropy_loss', 'accuracy'])),
-        ],
-        max_epoch=100,
-    )
-    launch_train_with_config(config, SimpleTrainer())

examples/basics/mnist-visualizations.py

@@ -6,8 +6,7 @@
 The same MNIST ConvNet example, but with weights/activations visualization.
 """
 
-import tensorflow as tf
-
+from tensorpack.compat import tfv1 as tf
 from tensorpack import *
 from tensorpack.dataflow import dataset
 

examples/basics/svhn-digit-convnet.py

 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# File: svhn-digit-convnet.py
 # Author: Yuxin Wu
 
 import argparse
 import os
-import tensorflow as tf
-
+from tensorpack.compat import tfv1 as tf
 from tensorpack import *
 from tensorpack.dataflow import dataset
 from tensorpack.tfutils.summary import *

examples/tox.ini

 [flake8]
 max-line-length = 120
-ignore = F403,F405,E402,E741,E742,E743,W504,W605
+ignore = F403,F405,E402,E741,E742,E743,W504,W605,B008
 exclude = private,
           FasterRCNN/utils,
 					__init__.py

tensorpack/compat/__init__.py

-#!/usr/bin/env python
-
 import tensorflow as tf
 
 

tensorpack/predict/base.py

@@ -146,10 +146,11 @@ class OfflinePredictor(OnlinePredictor):
 
             config = PredictConfig(model=my_model,
                                    inputs_names=['image'],
+                                   # use names of tensors defined in the model
                                    output_names=['linear/output', 'prediction'])
             predictor = OfflinePredictor(config)
-            batch_image = np.random.rand(1, 100, 100, 3)
-            batch_output, batch_prediction = predictor(batch_image)
+            image = np.random.rand(1, 100, 100, 3)  # the shape of "image" defined in the model
+            linear_output, prediction = predictor(image)
     """
 
     def __init__(self, config):