Freeze BN use nonfuse implementation

.github/ISSUE_TEMPLATE.md

@@ -28,8 +28,8 @@ About efficiency issues, PLEASE first read http://tensorpack.readthedocs.io/en/l
   (See http://tensorpack.readthedocs.io/en/latest/tutorial/index.html#extend-tensorpack).
   It does not have to be added to Tensorpack unless you have a good reason.
 + "Could you improve/implement an example/paper ?" 
-  -- the answer is: we have no plans to do so and we don't take feature requests for
-  examples. If you don't know how to do it, you may ask a usage question.
+  -- the answer is: we have no plans to do so. We don't take feature requests for
+  examples or implement a paper for you. If you don't know how to do it, you may ask a usage question.
 
 ## Usage Questions:
 

examples/FasterRCNN/README.md

@@ -73,8 +73,8 @@ MaskRCNN results contain both box and mask mAP.
 
  | Backbone | mAP<br/>(box;mask)                                                                                    | Detectron mAP <sup>[1](#ft1)</sup><br/> (box;mask) | Time on 8 V100s | Configurations <br/> (click to expand)                                                                                                                                                                          |
  | -        | -                                                                                                     | -                                                  | -               | -                                                                                                                                                                                                               |
- | R50-C4   | 33.1                                                                                                  |                                                    | 18h             | <details><summary>super quick</summary>`MODE_MASK=False FRCNN.BATCH_PER_IM=64`<br/>`PREPROC.SHORT_EDGE_SIZE=600 PREPROC.MAX_SIZE=1024`<br/>`TRAIN.LR_SCHEDULE=[150000,230000,280000]` </details>                |
- | R50-C4   | 36.6                                                                                                  | 36.5                                               | 44h             | <details><summary>standard</summary>`MODE_MASK=False` </details>                                                                                                                                                |
+ | R50-C4   | 33.8                                                                                                  |                                                    | 18h             | <details><summary>super quick</summary>`MODE_MASK=False FRCNN.BATCH_PER_IM=64`<br/>`PREPROC.SHORT_EDGE_SIZE=600 PREPROC.MAX_SIZE=1024`<br/>`TRAIN.LR_SCHEDULE=[150000,230000,280000]` </details>                |
+ | R50-C4   | 37.1                                                                                                  | 36.5                                               | 44h             | <details><summary>standard</summary>`MODE_MASK=False` </details>                                                                                                                                                |
  | R50-FPN  | 37.5                                                                                                  | 37.9                                               | 30h             | <details><summary>standard</summary>`MODE_MASK=False MODE_FPN=True` </details>                                                                                                                                  |
  | R50-C4   | 38.5;33.7 [:arrow_down:](http://models.tensorpack.com/FasterRCNN/COCO-R50C4-MaskRCNN-Standard.npz)    | 37.8;32.8                                          | 49h             | <details><summary>standard</summary>`MODE_MASK=True` </details>                                                                                                                                                 |
  | R50-FPN  | 38.8;35.4 [:arrow_down:](http://models.tensorpack.com/FasterRCNN/COCO-R50FPN-MaskRCNN-Standard.npz)   | 38.6;34.5                                          | 32h             | <details><summary>standard</summary>`MODE_MASK=True MODE_FPN=True` </details>                                                                                                                                   |

examples/FasterRCNN/model_frcnn.py

@@ -226,7 +226,7 @@ def fastrcnn_2fc_head(feature, num_classes):
         num_classes(int): num_category + 1
 
     Returns:
-        cls_logits (Nxnum_class), reg_logits (Nx num_class-1 x 4)
+        outputs of `fastrcnn_outputs()`
     """
     dim = cfg.FPN.FRCNN_FC_HEAD_DIM
     init = tf.variance_scaling_initializer()
@@ -245,7 +245,7 @@ def fastrcnn_Xconv1fc_head(feature, num_classes, num_convs, norm=None):
         norm (str or None): either None or 'GN'
 
     Returns:
-        cls_logits (Nxnum_class), reg_logits (Nx num_class-1 x 4)
+        outputs of `fastrcnn_outputs()`
     """
     assert norm in [None, 'GN'], norm
     l = feature

tensorpack/models/batch_norm.py

@@ -156,10 +156,10 @@ def BatchNorm(inputs, axis=None, training=None, momentum=0.9, epsilon=1e-5,
         training = ctx.is_training
     training = bool(training)
     TF_version = get_tf_version_tuple()
-    if not training and ctx.is_training:
+    freeze_bn_backward = not training and ctx.is_training
+    if freeze_bn_backward:
         assert TF_version >= (1, 4), \
-            "Fine tuning a BatchNorm model with fixed statistics is only " \
-            "supported after https://github.com/tensorflow/tensorflow/pull/12580 "
+            "Fine tuning a BatchNorm model with fixed statistics needs TF>=1.4!"
         if ctx.is_main_training_tower:  # only warn in first tower
             logger.warn("[BatchNorm] Using moving_mean/moving_variance in training.")
         # Using moving_mean/moving_variance in training, which means we
@@ -176,7 +176,8 @@ def BatchNorm(inputs, axis=None, training=None, momentum=0.9, epsilon=1e-5,
                 center=center, scale=scale,
                 beta_initializer=beta_initializer,
                 gamma_initializer=gamma_initializer,
-                fused=(ndims == 4 and axis in [1, 3]),
+                # https://github.com/tensorflow/tensorflow/issues/10857#issuecomment-410185429
+                fused=(ndims == 4 and axis in [1, 3] and not freeze_bn_backward),
                 _reuse=tf.get_variable_scope().reuse)
             if TF_version >= (1, 5):
                 tf_args['virtual_batch_size'] = virtual_batch_size