deprecate more from symbolic_functions

.github/ISSUE_TEMPLATE.md

@@ -2,10 +2,11 @@ Bug Reports/Feature Requests/Usage Questions Only:
 
 Any unexpected problems: PLEASE always include
 1. What you did:
-	+ Are you using any examples?
-	+ If yes, post the command you run and changes you made (if any).
+	+ If you're using examples:
+		+ What's the command you run:
+		+ Have you made and changes to code? Post them if any:
 	+ If not, describe what you did that may be relevant.
-		But we may not be able to resolve it if there is no reproducible code.
+		But we may not be able to resolve it since there is no reproducible code.
 2. What you observed, e.g. as much as logs possible.
 3. What you expected, if not obvious.
 4. Your environment (TF version, tensorpack version, cudnn version, number & type of GPUs), if it matters.

docs/conf.py

@@ -375,7 +375,7 @@ def autodoc_skip_member(app, what, name, obj, skip, options):
         'LeakyReLU',
         'PrefetchOnGPUs',
 
-        'guided_relu', 'saliency_map', 'get_scalar_var',
+        'guided_relu', 'saliency_map', 'get_scalar_var', 'psnr',
         'prediction_incorrect', 'huber_loss',
         ]:
         return True

docs/modules/tfutils.rst

@@ -71,14 +71,6 @@ tensorpack.tfutils.summary module
     :undoc-members:
     :show-inheritance:
 
-tensorpack.tfutils.symbolic_functions module
---------------------------------------------
-
-.. automodule:: tensorpack.tfutils.symbolic_functions
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 tensorpack.tfutils.varmanip module
 ----------------------------------
 

docs/tutorial/efficient-dataflow.md

@@ -229,9 +229,9 @@ Let me summarize what this DataFlow does:
 	 send them through ZMQ IPC pipe.
 3. The main process takes data from the pipe, makes batches.
 
-The two DataFlow mentioned in this tutorial (both random read and sequential read) can run at a speed of 1k ~ 2k images per second if you have good CPUs, RAM, disks.
+The two DataFlow mentioned in this tutorial (both random read and sequential read) can run at a speed of 1k ~ 2.5k images per second if you have good CPUs, RAM, disks.
 As a reference, tensorpack can train ResNet-18 at 1.2k images/s on 4 old TitanX.
-A DGX-1 (8 P100) can train ResNet-50 at 1.7k images/s according to the [official benchmark](https://www.tensorflow.org/performance/benchmarks).
+A DGX-1 (8 P100s) can train ResNet-50 at 1.7k images/s according to the [official benchmark](https://www.tensorflow.org/performance/benchmarks).
 So DataFlow will not be a serious bottleneck if configured properly.
 
 ## Distributed DataFlow

docs/tutorial/extend/dataflow.md

 
 ### Write a DataFlow
 
+#### Write a Source DataFlow
+
 There are several existing DataFlow, e.g. [ImageFromFile](../../modules/dataflow.html#tensorpack.dataflow.ImageFromFile),
 [DataFromList](../../modules/dataflow.html#tensorpack.dataflow.DataFromList),
 which you can use if your data format is simple.
@@ -23,14 +25,27 @@ Optionally, you can implement the following two methods:
 
 + `reset_state()`. It is guaranteed that the actual process which runs a DataFlow will invoke this method before using it.
 	So if this DataFlow needs to do something after a `fork()`, you should put it here.
+	The convention is that, `reset_state()` must be called once and usually only once for each DataFlow instance.
 
 	A typical situation is when your DataFlow uses random number generator (RNG). Then you would need to reset the RNG here.
 	Otherwise, child processes will have the same random seed. The `RNGDataFlow` base class does this for you.
 	You can subclass `RNGDataFlow` to access `self.rng` whose seed has been taken care of.
 
-The convention is that, `reset_state()` must be called once and usually only once for each DataFlow instance.
-To reinitialize the dataflow (i.e. get a new iterator from the beginning), simply call `get_data()` again.
-
 DataFlow implementations for several well-known datasets are provided in the
 [dataflow.dataset](../../modules/dataflow.dataset.html)
 module, you can take them as a reference.
+
+#### More Data Processing
+
+You can put any data processing you need in the source DataFlow, or write a new DataFlow for data
+processing on top of the source DataFlow, e.g.:
+
+```python
+class ProcessingDataFlow(DataFlow):
+  def __init__(self, ds):
+	  self.ds = ds
+  def get_data(self):
+    for datapoint in self.ds.get_data():
+      # do something
+			yield new_datapoint
+```

examples/A3C-Gym/train-atari.py

@@ -17,7 +17,6 @@ from six.moves import queue
 from tensorpack import *
 from tensorpack.utils.concurrency import ensure_proc_terminate, start_proc_mask_signal
 from tensorpack.utils.serialize import dumps
-from tensorpack.tfutils import symbolic_functions as symbf
 from tensorpack.tfutils.gradproc import MapGradient, SummaryGradient
 from tensorpack.utils.gpu import get_nr_gpu
 
@@ -117,7 +116,7 @@ class Model(ModelDesc):
         value_loss = tf.nn.l2_loss(value - futurereward, name='value_loss')
 
         pred_reward = tf.reduce_mean(value, name='predict_reward')
-        advantage = symbf.rms(advantage, name='rms_advantage')
+        advantage = tf.sqrt(tf.reduce_mean(tf.square(advantage)), name='rms_advantage')
         entropy_beta = tf.get_variable('entropy_beta', shape=[],
                                        initializer=tf.constant_initializer(0.01), trainable=False)
         self.cost = tf.add_n([policy_loss, xentropy_loss * entropy_beta, value_loss])

examples/Char-RNN/char-rnn.py

@@ -14,7 +14,7 @@ from six.moves import range
 
 
 from tensorpack import *
-from tensorpack.tfutils import symbolic_functions, summary, optimizer
+from tensorpack.tfutils import summary, optimizer
 from tensorpack.tfutils.gradproc import GlobalNormClip
 from tensorpack.utils.globvars import globalns as param
 
@@ -80,7 +80,7 @@ class Model(ModelDesc):
             ret = tf.get_variable(n + '_unused', [param.batch_size, param.rnn_size],
                                   trainable=False,
                                   initializer=tf.constant_initializer())
-            ret = symbolic_functions.shapeless_placeholder(ret, 0, name=n)
+            ret = tf.placeholder_with_default(ret, shape=[None, param.rnn_size], name=n)
             return ret
         initial = (rnn.LSTMStateTuple(get_v('c0'), get_v('h0')),
                    rnn.LSTMStateTuple(get_v('c1'), get_v('h1')))

examples/GAN/ConditionalGAN-mnist.py

@@ -12,7 +12,6 @@ import argparse
 
 from tensorpack import *
 from tensorpack.utils.viz import interactive_imshow, stack_patches
-import tensorpack.tfutils.symbolic_functions as symbf
 from tensorpack.tfutils.scope_utils import auto_reuse_variable_scope
 from tensorpack.dataflow import dataset
 from GAN import GANTrainer, RandomZData, GANModelDesc
@@ -30,6 +29,16 @@ A pretrained model is at http://models.tensorpack.com/GAN/
 BATCH = 128
 
 
+def batch_flatten(x):
+    """
+    Flatten the tensor except the first dimension.
+    """
+    shape = x.get_shape().as_list()[1:]
+    if None not in shape:
+        return tf.reshape(x, [-1, int(np.prod(shape))])
+    return tf.reshape(x, tf.stack([tf.shape(x)[0], -1]))
+
+
 class Model(GANModelDesc):
     def _get_inputs(self):
         return [InputDesc(tf.float32, (None, 28, 28), 'input'),
@@ -65,7 +74,7 @@ class Model(GANModelDesc):
                  .BatchNorm('bn1')
                  .tf.nn.leaky_relu()
 
-                 .apply(symbf.batch_flatten)
+                 .apply(batch_flatten)
                  .ConcatWith(yv, 1)
                  .FullyConnected('fc1', 1024, nl=tf.identity)
                  .BatchNorm('bn2')
@@ -81,7 +90,7 @@ class Model(GANModelDesc):
         y = tf.one_hot(y, 10, name='label_onehot')
 
         z = tf.random_uniform([BATCH, 100], -1, 1, name='z_train')
-        z = symbf.shapeless_placeholder(z, [0], name='z')
+        z = tf.placeholder_with_default(z, [None, 100], name='z')   # clear the static shape
 
         with argscope([Conv2D, Deconv2D, FullyConnected],
                       W_init=tf.truncated_normal_initializer(stddev=0.02)):

examples/GAN/InfoGAN-mnist.py

@@ -14,7 +14,6 @@ from tensorpack import *
 from tensorpack.utils import viz
 from tensorpack.tfutils.scope_utils import auto_reuse_variable_scope, under_name_scope
 from tensorpack.tfutils import optimizer, summary
-import tensorpack.tfutils.symbolic_functions as symbf
 from tensorpack.dataflow import dataset
 from GAN import GANTrainer, GANModelDesc
 
@@ -38,6 +37,35 @@ NOISE_DIM = 62
 DIST_PRIOR_PARAM = [1.] * NUM_CLASS + [0.] * NUM_UNIFORM
 
 
+def shapeless_placeholder(x, axis, name):
+    """
+    Make the static shape of a tensor less specific.
+
+    If you want to feed to a tensor, the shape of the feed value must match
+    the tensor's static shape. This function creates a placeholder which
+    defaults to x if not fed, but has a less specific static shape than x.
+    See also `tensorflow#5680 <https://github.com/tensorflow/tensorflow/issues/5680>`_.
+
+    Args:
+        x: a tensor
+        axis(int or list of ints): these axes of ``x.get_shape()`` will become
+            None in the output.
+        name(str): name of the output tensor
+
+    Returns:
+        a tensor equal to x, but shape information is partially cleared.
+    """
+    shp = x.get_shape().as_list()
+    if not isinstance(axis, list):
+        axis = [axis]
+    for a in axis:
+        if shp[a] is None:
+            raise ValueError("Axis {} of shape {} is already unknown!".format(a, shp))
+        shp[a] = None
+    x = tf.placeholder_with_default(x, shape=shp, name=name)
+    return x
+
+
 def get_distributions(vec_cat, vec_uniform):
     cat = tf.distributions.Categorical(logits=vec_cat, validate_args=True, name='cat')
     uni = tf.distributions.Normal(vec_uniform, scale=1., validate_args=True, allow_nan_stats=False, name='uni_a')
@@ -115,8 +143,8 @@ class Model(GANModelDesc):
         real_sample = tf.expand_dims(real_sample, -1)
 
         # sample the latent code:
-        zc = symbf.shapeless_placeholder(sample_prior(BATCH), 0, name='z_code')
-        z_noise = symbf.shapeless_placeholder(
+        zc = shapeless_placeholder(sample_prior(BATCH), 0, name='z_code')
+        z_noise = shapeless_placeholder(
             tf.random_uniform([BATCH, NOISE_DIM], -1, 1), 0, name='z_noise')
         z = tf.concat([zc, z_noise], 1, name='z')
 

examples/ResNet/imagenet_utils.py

@@ -17,6 +17,7 @@ from tensorpack.predict import PredictConfig, SimpleDatasetPredictor
 from tensorpack.utils.stats import RatioCounter
 from tensorpack.models import regularize_cost
 from tensorpack.tfutils.summary import add_moving_summary
+from tensorpack.utils import logger
 
 
 class GoogleNetResize(imgaug.ImageAugmentor):
@@ -97,6 +98,8 @@ def get_imagenet_dataflow(
     if isTrain:
         ds = dataset.ILSVRC12(datadir, name, shuffle=True)
         ds = AugmentImageComponent(ds, augmentors, copy=False)
+        if parallel < 16:
+            logger.warn("DataFlow may become the bottleneck when too few processes are used.")
         ds = PrefetchDataZMQ(ds, parallel)
         ds = BatchData(ds, batch_size, remainder=False)
     else:

examples/Saliency/saliency-maps.py

@@ -38,6 +38,21 @@ def guided_relu():
         yield
 
 
+def saliency_map(output, input, name="saliency_map"):
+    """
+    Produce a saliency map as described in the paper:
+    `Deep Inside Convolutional Networks: Visualising Image Classification Models and Saliency Maps
+    <https://arxiv.org/abs/1312.6034>`_.
+    The saliency map is the gradient of the max element in output w.r.t input.
+
+    Returns:
+        tf.Tensor: the saliency map. Has the same shape as input.
+    """
+    max_outp = tf.reduce_max(output, 1)
+    saliency_op = tf.gradients(max_outp, input)[:][0]
+    return tf.identity(saliency_op, name=name)
+
+
 class Model(tp.ModelDesc):
     def _get_inputs(self):
         return [tp.InputDesc(tf.float32, (IMAGE_SIZE, IMAGE_SIZE, 3), 'image')]
@@ -49,7 +64,7 @@ class Model(tp.ModelDesc):
             with slim.arg_scope(resnet_v1.resnet_arg_scope(is_training=False)):
                 image = tf.expand_dims(orig_image - mean, 0)
                 logits, _ = resnet_v1.resnet_v1_50(image, 1000)
-            tp.symbolic_functions.saliency_map(logits, orig_image, name="saliency")
+            saliency_map(logits, orig_image, name="saliency")
 
 
 def run(model_path, image_path):

tensorpack/dataflow/common.py

@@ -84,6 +84,7 @@ class BatchData(ProxyDataFlow):
                 enough to form a batch, whether or not to also produce the remaining
                 data as a smaller batch.
                 If set to False, all produced datapoints are guranteed to have the same batch size.
+                If set to True, `ds.size()` must be accurate.
             use_list (bool): if True, each component will contain a list
                 of datapoints instead of an numpy array of an extra dimension.
         """

tensorpack/tfutils/symbolic_functions.py

@@ -171,6 +171,7 @@ def get_scalar_var(name, init_value, summary=False, trainable=False):
     return ret
 
 
+@deprecated("Please implement it by yourself.", "2018-04-28")
 def psnr(prediction, ground_truth, maxp=None, name='psnr'):
     """`Peek Signal to Noise Ratio <https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio>`_.
 
@@ -228,6 +229,7 @@ def guided_relu():
         yield
 
 
+@deprecated("Please implement it by yourself.", "2018-04-28")
 def saliency_map(output, input, name="saliency_map"):
     """
     Produce a saliency map as described in the paper:
@@ -244,6 +246,7 @@ def saliency_map(output, input, name="saliency_map"):
     return saliency_op
 
 
+@deprecated("Please implement it by yourself.", "2018-04-28")
 def shapeless_placeholder(x, axis, name):
     """
     Make the static shape of a tensor less specific.