name scope in symbf & reuse name scope in InputSource (fix #340)

examples/cifar-convnet.py

@@ -155,4 +155,4 @@ if __name__ == '__main__':
         if config.nr_tower <= 1:
             QueueInputTrainer(config).train()
         else:
-            AsyncMultiGPUTrainer(config).train()
+            SyncMultiGPUTrainerParameterServer(config).train()

tensorpack/graph_builder/input_source.py

@@ -228,8 +228,7 @@ class QueueInput(FeedfreeInput):
         self._input_placehdrs = [v.build_placeholder_reuse() for v in inputs]
         assert len(self._input_placehdrs) > 0, \
             "QueueInput has to be used with some inputs!"
-        with tf.name_scope('QueueInput') as ns:
-            self._name_scope = ns
+        with self.cached_name_scope():
             if self.queue is None:
                 self.queue = tf.FIFOQueue(
                     50, [x.dtype for x in self._input_placehdrs],
@@ -243,7 +242,7 @@ class QueueInput(FeedfreeInput):
         return [cb]
 
     def _get_input_tensors(self):
-        with tf.device('/cpu:0'), tf.name_scope(self._name_scope):
+        with tf.device('/cpu:0'), self.cached_name_scope():
             ret = self.queue.dequeue(name='input_deque')
             if isinstance(ret, tf.Tensor):  # only one input
                 ret = [ret]
@@ -294,8 +293,7 @@ class BatchQueueInput(QueueInput):
             assert p.get_shape().is_fully_defined(), shape_err
             shapes.append(p.get_shape())
 
-        with tf.name_scope('BatchQueueInput') as ns:
-            self._name_scope = ns
+        with self.cached_name_scope():
             if self.queue is None:
                 self.queue = tf.FIFOQueue(
                     3000, [x.dtype for x in self.input_placehdrs],
@@ -307,7 +305,7 @@ class BatchQueueInput(QueueInput):
             self.thread = EnqueueThread(self.queue, self.ds, placehdrs_nobatch)
 
     def _get_input_tensors(self):
-        with tf.device('/cpu:0'), tf.name_scope(self._name_scope):
+        with tf.device('/cpu:0'), self.cached_name_scope():
             ret = self.queue.dequeue_many(self.batch_size, name='input_deque')
             if isinstance(ret, tf.Tensor):  # only one input
                 ret = [ret]
@@ -345,7 +343,8 @@ class TensorInput(FeedfreeInput):
         return self._fixed_size
 
     def _get_input_tensors(self):
-        ret = self.get_tensor_fn()
+        with self.cached_name_scope():
+            ret = self.get_tensor_fn()
         assert len(ret) == len(self._desc), "{} != {}".format(len(ret), len(self._desc))
         return ret
 
@@ -452,8 +451,7 @@ class StagingInputWrapper(FeedfreeInput):
 
     def _setup_staging_areas(self):
         logger.info("Setting up StagingArea for GPU prefetching ...")
-        with tf.name_scope('StagingInputWrapper') as ns:
-            self._name_scope = ns
+        with self.cached_name_scope():
             for idx, device in enumerate(self._devices):
                 with tf.device(device):
                     inputs = self._input.get_input_tensors()
@@ -477,10 +475,10 @@ class StagingInputWrapper(FeedfreeInput):
         return ret
 
     def _get_stage_op(self):
-        with tf.name_scope(self._name_scope):
+        with self.cached_name_scope():
             return tf.group(*self._stage_ops)
 
     def _get_unstage_op(self):
-        with tf.name_scope(self._name_scope):
+        with self.cached_name_scope():
             all_outputs = list(chain.from_iterable(self._unstage_ops))
             return tf.group(*all_outputs)

tensorpack/graph_builder/input_source_base.py

@@ -4,6 +4,8 @@
 
 from abc import ABCMeta, abstractmethod
 import six
+from contextlib import contextmanager
+import tensorflow as tf
 
 from ..utils.argtools import memoized
 from ._utils import get_sublist_by_names, get_tensors_inputs
@@ -15,6 +17,8 @@ __all__ = ['InputSource', 'remap_input_source']
 class InputSource(object):
     """ Base class for the abstract InputSource. """
 
+    _name_scope = None
+
     def get_input_tensors(self):
         """
         Returns:
@@ -76,6 +80,21 @@ class InputSource(object):
     def _size(self):
         raise NotImplementedError()
 
+    @contextmanager
+    def cached_name_scope(self):
+        """
+        Yield a context under a cached name scope, whose name is the name of
+        this InputSource class.
+        """
+        if self._name_scope:
+            with tf.name_scope(self._name_scope):
+                yield self._name_scope
+        else:
+            name = type(self).__name__
+            with tf.name_scope(name) as ns:
+                self._name_scope = ns
+                yield ns
+
 
 class ProxyInputSource(InputSource):
     """

tensorpack/tfutils/summary.py

@@ -156,16 +156,14 @@ def _enter_vs_reuse_ns(name):
             yield vs
 
 
-def add_moving_summary(v, *args, **kwargs):
+def add_moving_summary(*args, **kwargs):
     """
     Enable moving average summary for some tensors.
     It's only effective in the main training tower, otherwise calling this
     function is a no-op.
 
     Args:
-        v (tf.Tensor or list): tensor or list of tensors to summary. Must have
-            scalar type.
-        args: tensors to summary (to support positional arguments)
+        args: tensors to summary
         decay (float): the decay rate. Defaults to 0.95.
         collection (str): the name of the collection to add EMA-maintaining ops.
             The default will work together with the default
@@ -178,9 +176,12 @@ def add_moving_summary(v, *args, **kwargs):
     ctx = get_current_tower_context()
     if ctx is not None and not ctx.is_main_training_tower:
         return
-    if not isinstance(v, list):
-        v = [v]
-    v.extend(args)
+
+    if not isinstance(args[0], list):
+        v = args
+    else:
+        log_deprecated("Call add_moving_summary with positional args instead of a list!")
+        v = args[0]
     for x in v:
         assert isinstance(x, tf.Tensor), x
         assert x.get_shape().ndims == 0, x.get_shape()
@@ -195,8 +196,7 @@ def add_moving_summary(v, *args, **kwargs):
                 ema_var = tf.get_variable(name, shape=c.shape, dtype=c.dtype,
                                           initializer=tf.constant_initializer(), trainable=False)
                 ns = vs.original_name_scope
-            # first clear NS to avoid duplicated name in variables
-            with tf.name_scope(ns):
+            with tf.name_scope(ns):     # reuse VS&NS so that no EMA_1 will appear
                 ema_op = moving_averages.assign_moving_average(
                     ema_var, c, decay,
                     zero_debias=True, name=name + '_EMA_apply')

tensorpack/tfutils/symbolic_functions.py

@@ -65,16 +65,17 @@ def class_balanced_cross_entropy(pred, label, name='cross_entropy_loss'):
     Returns:
         class-balanced cross entropy loss.
     """
-    z = batch_flatten(pred)
-    y = tf.cast(batch_flatten(label), tf.float32)
+    with tf.name_scope('class_balanced_cross_entropy'):
+        z = batch_flatten(pred)
+        y = tf.cast(batch_flatten(label), tf.float32)
 
-    count_neg = tf.reduce_sum(1. - y)
-    count_pos = tf.reduce_sum(y)
-    beta = count_neg / (count_neg + count_pos)
+        count_neg = tf.reduce_sum(1. - y)
+        count_pos = tf.reduce_sum(y)
+        beta = count_neg / (count_neg + count_pos)
 
-    eps = 1e-12
-    loss_pos = -beta * tf.reduce_mean(y * tf.log(z + eps))
-    loss_neg = (1. - beta) * tf.reduce_mean((1. - y) * tf.log(1. - z + eps))
+        eps = 1e-12
+        loss_pos = -beta * tf.reduce_mean(y * tf.log(z + eps))
+        loss_neg = (1. - beta) * tf.reduce_mean((1. - y) * tf.log(1. - z + eps))
     cost = tf.subtract(loss_pos, loss_neg, name=name)
     return cost
 
@@ -84,16 +85,18 @@ def class_balanced_sigmoid_cross_entropy(logits, label, name='cross_entropy_loss
     This function accepts logits rather than predictions, and is more numerically stable than
     :func:`class_balanced_cross_entropy`.
     """
-    y = tf.cast(label, tf.float32)
+    with tf.name_scope('class_balanced_sigmoid_cross_entropy'):
+        y = tf.cast(label, tf.float32)
 
-    count_neg = tf.reduce_sum(1. - y)
-    count_pos = tf.reduce_sum(y)
-    beta = count_neg / (count_neg + count_pos)
+        count_neg = tf.reduce_sum(1. - y)
+        count_pos = tf.reduce_sum(y)
+        beta = count_neg / (count_neg + count_pos)
 
-    pos_weight = beta / (1 - beta)
-    cost = tf.nn.weighted_cross_entropy_with_logits(logits=logits, targets=y, pos_weight=pos_weight)
-    cost = tf.reduce_mean(cost * (1 - beta))
-    return tf.where(tf.equal(count_pos, 0.0), 0.0, cost, name=name)
+        pos_weight = beta / (1 - beta)
+        cost = tf.nn.weighted_cross_entropy_with_logits(logits=logits, targets=y, pos_weight=pos_weight)
+        cost = tf.reduce_mean(cost * (1 - beta))
+        zero = tf.equal(count_pos, 0.0)
+    return tf.where(zero, 0.0, cost, name=name)
 
 
 def print_stat(x, message=None):
@@ -135,12 +138,14 @@ def huber_loss(x, delta=1, name='huber_loss'):
     Returns:
         a tensor of the same shape of x.
     """
-    sqrcost = tf.square(x)
-    abscost = tf.abs(x)
-    return tf.where(abscost < delta,
-                    sqrcost * 0.5,
-                    abscost * delta - 0.5 * delta ** 2,
-                    name=name)
+    with tf.name_scope('huber_loss'):
+        sqrcost = tf.square(x)
+        abscost = tf.abs(x)
+
+        cond = abscost < delta
+        l2 = sqrcost * 0.5
+        l1 = abscost * delta - 0.5 * delta ** 2
+    return tf.where(cond, l2, l1, name=name)
 
 
 def get_scalar_var(name, init_value, summary=False, trainable=False):