Simply simple examples (#258)

the mnist-convnet.py examples was overloaded. This commit splits the
file into the most minimal mnist-convnet and mnist-tfslim. Further
it adds an example about the visualization of filters which
illustrates the usage of `VariableHolder`.

In addition, this also adds a boilerplate template for projects,
which might facilitate the process of starting a new project.

examples/boilerplate.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Author: Your Name <your@email.com>
+
+import os
+import argparse
+from tensorpack import *
+import tensorflow as tf
+
+"""
+This is a boiler-plate template.
+All code is in this file is the most minimalistic way to solve a deep-learning problem with cross-validation.
+"""
+
+BATCH_SIZE = 16
+
+
+class Model(ModelDesc):
+    def _get_inputs(self):
+        return [InputDesc(tf.float32, (None, 28, 28, 1), 'input'),
+                InputDesc(tf.int32, (None,), 'label')]
+
+    def _build_graph(self, inputs):
+        image, label = inputs
+        image = image * 2 - 1
+
+        self.cost = tf.identity(0, name='total_costs')
+        summary.add_moving_summary(cost)
+
+    def _get_optimizer(self):
+        lr = symbolic_functions.get_scalar_var('learning_rate', 5e-3, summary=True)
+        return tf.train.AdamOptimizer(lr)
+
+
+def get_data(subset):
+    ds = None  # something.get_data() that yields [[28, 28, 1], [1]]
+    # ...
+
+    ds = PrefetchDataZMQ(ds, 2)
+    ds = BatchData(ds, BATCH_SIZE)
+    return ds
+
+
+def get_config():
+    logger.auto_set_dir()
+
+    ds_train = get_data('train')
+    ds_test = get_data('test')
+
+    return TrainConfig(
+        model=Model(),
+        dataflow=ds_train,
+        callbacks=[
+            ModelSaver(),
+            InferenceRunner(ds_test, [ScalarStats('total_costs')]),
+        ],
+        steps_per_epoch=ds_train.size(),
+        max_epoch=100,
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu', help='comma separated list of GPU(s) to use.', required=True)
+    parser.add_argument('--load', help='load model')
+    args = parser.parse_args()
+
+    NR_GPU = len(args.gpu.split(','))
+    with change_gpu(args.gpu):
+        config = get_config()
+        config.nr_tower = NR_GPU
+
+        if args.load:
+            config.session_init = SaverRestore(args.load)
+
+        SyncMultiGPUTrainer(config).train()

examples/mnist-convnet.py

@@ -7,7 +7,6 @@ import numpy as np
 import os
 import sys
 import argparse
-
 """
 MNIST ConvNet example.
 about 0.6% validation error after 30 epochs.
@@ -16,10 +15,9 @@ about 0.6% validation error after 30 epochs.
 # Just import everything into current namespace
 from tensorpack import *
 import tensorflow as tf
-import tensorflow.contrib.slim as slim
+import tensorpack.tfutils.symbolic_functions as symbf
 
 IMAGE_SIZE = 28
-USE_SLIM = False
 
 
 class Model(ModelDesc):
@@ -37,47 +35,26 @@ class Model(ModelDesc):
 
         # inputs contains a list of input variables defined above
         image, label = inputs
+
         # In tensorflow, inputs to convolution function are assumed to be
         # NHWC. Add a single channel here.
         image = tf.expand_dims(image, 3)
 
         image = image * 2 - 1   # center the pixels values at zero
 
-        if USE_SLIM:
-            is_training = get_current_tower_context().is_training
-            with slim.arg_scope([slim.layers.fully_connected],
-                                weights_regularizer=slim.l2_regularizer(1e-5)):
-                l = slim.layers.conv2d(image, 32, [3, 3], scope='conv0')
-                l = slim.layers.max_pool2d(l, [2, 2], scope='pool0')
-                l = slim.layers.conv2d(l, 32, [3, 3], padding='SAME', scope='conv1')
-                l = slim.layers.conv2d(l, 32, [3, 3], scope='conv2')
-                l = slim.layers.max_pool2d(l, [2, 2], scope='pool1')
-                l = slim.layers.conv2d(l, 32, [3, 3], scope='conv3')
-                l = slim.layers.flatten(l, scope='flatten')
-                l = slim.layers.fully_connected(l, 512, scope='fc0')
-                l = slim.layers.dropout(l, is_training=is_training)
-                logits = slim.layers.fully_connected(l, 10, activation_fn=None, scope='fc1')
-        else:
-            # The context manager `argscope` sets the default option for all the layers under
-            # this context. Here we use 32 channel convolution with shape 3x3
-            with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32):
-                """
-                LinearWrap is just a convenient way to compose a linear symbolic graph.
-                You can also do the equivalent in tensorflow style:
-                l = Conv2D('conv0', image)
-                l = MaxPooling('pool0', l, 2)
-                ...  """
-
-                logits = (LinearWrap(image)  # the starting brace is only for line-breaking
-                          .Conv2D('conv0')
-                          .MaxPooling('pool0', 2)
-                          .Conv2D('conv1')
-                          .Conv2D('conv2')
-                          .MaxPooling('pool1', 2)
-                          .Conv2D('conv3')
-                          .FullyConnected('fc0', 512, nl=tf.nn.relu)
-                          .Dropout('dropout', 0.5)
-                          .FullyConnected('fc1', out_dim=10, nl=tf.identity)())
+        # The context manager `argscope` sets the default option for all the layers under
+        # this context. Here we use 32 channel convolution with shape 3x3
+        with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32):
+            logits = (LinearWrap(image)
+                      .Conv2D('conv0')
+                      .MaxPooling('pool0', 2)
+                      .Conv2D('conv1')
+                      .Conv2D('conv2')
+                      .MaxPooling('pool1', 2)
+                      .Conv2D('conv3')
+                      .FullyConnected('fc0', 512, nl=tf.nn.relu)
+                      .Dropout('dropout', 0.5)
+                      .FullyConnected('fc1', out_dim=10, nl=tf.identity)())
 
         prob = tf.nn.softmax(logits, name='prob')   # a Bx10 with probabilities
 
@@ -86,27 +63,23 @@ class Model(ModelDesc):
         cost = tf.reduce_mean(cost, name='cross_entropy_loss')  # the average cross-entropy loss
 
         # compute the "incorrect vector", for the callback ClassificationError to use at validation time
-        wrong = symbolic_functions.prediction_incorrect(logits, label, name='incorrect')
+        wrong = symbf.prediction_incorrect(logits, label, name='incorrect')
+        accuracy = symbf.accuracy(logits, label, name='accuracy')
 
         # This will monitor training error (in a moving_average fashion):
         # 1. write the value to tensosrboard
         # 2. write the value to stat.json
         # 3. print the value after each epoch
         train_error = tf.reduce_mean(wrong, name='train_error')
-        summary.add_moving_summary(train_error)
-
-        if not USE_SLIM:
-            # Use a regex to find parameters to apply weight decay.
-            # Here we apply a weight decay on all W (weight matrix) of all fc layers
-            wd_cost = tf.multiply(1e-5,
-                                  regularize_cost('fc.*/W', tf.nn.l2_loss),
-                                  name='regularize_loss')
-            self.cost = tf.add_n([wd_cost, cost], name='total_cost')
-            summary.add_moving_summary(cost, wd_cost, self.cost)
-        else:
-            # slim already adds regularization to a collection, no extra handling
-            self.cost = cost
-            summary.add_moving_summary(cost)
+        summary.add_moving_summary(train_error, accuracy)
+
+        # Use a regex to find parameters to apply weight decay.
+        # Here we apply a weight decay on all W (weight matrix) of all fc layers
+        wd_cost = tf.multiply(1e-5,
+                              regularize_cost('fc.*/W', tf.nn.l2_loss),
+                              name='regularize_loss')
+        self.cost = tf.add_n([wd_cost, cost], name='total_cost')
+        summary.add_moving_summary(cost, wd_cost, self.cost)
 
         # monitor histogram of all weight (of conv and fc layers) in tensorboard
         summary.add_param_summary(('.*/W', ['histogram', 'rms']),
@@ -149,7 +122,8 @@ def get_config():
             InferenceRunner(    # run inference(for validation) after every epoch
                 dataset_test,   # the DataFlow instance used for validation
                 # Calculate both the cost and the error for this DataFlow
-                [ScalarStats('cross_entropy_loss'), ClassificationError('incorrect')]),
+                [ScalarStats('cross_entropy_loss'), ScalarStats('accuracy'),
+                 ClassificationError('incorrect')]),
         ],
         steps_per_epoch=steps_per_epoch,
         max_epoch=100,

examples/mnist-tfslim.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# File: mnist-tfslim.py
+
+import numpy as np
+import os
+import sys
+import argparse
+
+"""
+MNIST ConvNet example.
+about 0.6% validation error after 30 epochs.
+"""
+
+# Just import everything into current namespace
+from tensorpack import *
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+IMAGE_SIZE = 28
+
+
+class Model(ModelDesc):
+    def _get_inputs(self):
+        """
+        Define all the inputs (with type, shape, name) that
+        the graph will need.
+        """
+        return [InputDesc(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE), 'input'),
+                InputDesc(tf.int32, (None,), 'label')]
+
+    def _build_graph(self, inputs):
+        """This function should build the model which takes the input variables
+        and define self.cost at the end"""
+
+        # inputs contains a list of input variables defined above
+        image, label = inputs
+        # In tensorflow, inputs to convolution function are assumed to be
+        # NHWC. Add a single channel here.
+        image = tf.expand_dims(image, 3)
+
+        image = image * 2 - 1   # center the pixels values at zero
+
+        is_training = get_current_tower_context().is_training
+        with slim.arg_scope([slim.layers.fully_connected],
+                            weights_regularizer=slim.l2_regularizer(1e-5)):
+            l = slim.layers.conv2d(image, 32, [3, 3], scope='conv0')
+            l = slim.layers.max_pool2d(l, [2, 2], scope='pool0')
+            l = slim.layers.conv2d(l, 32, [3, 3], padding='SAME', scope='conv1')
+            l = slim.layers.conv2d(l, 32, [3, 3], scope='conv2')
+            l = slim.layers.max_pool2d(l, [2, 2], scope='pool1')
+            l = slim.layers.conv2d(l, 32, [3, 3], scope='conv3')
+            l = slim.layers.flatten(l, scope='flatten')
+            l = slim.layers.fully_connected(l, 512, scope='fc0')
+            l = slim.layers.dropout(l, is_training=is_training)
+            logits = slim.layers.fully_connected(l, 10, activation_fn=None, scope='fc1')
+
+        prob = tf.nn.softmax(logits, name='prob')   # a Bx10 with probabilities
+
+        # a vector of length B with loss of each sample
+        cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label)
+        cost = tf.reduce_mean(cost, name='cross_entropy_loss')  # the average cross-entropy loss
+
+        # compute the "incorrect vector", for the callback ClassificationError to use at validation time
+        wrong = symbolic_functions.prediction_incorrect(logits, label, name='incorrect')
+
+        # This will monitor training error (in a moving_average fashion):
+        # 1. write the value to tensosrboard
+        # 2. write the value to stat.json
+        # 3. print the value after each epoch
+        train_error = tf.reduce_mean(wrong, name='train_error')
+        summary.add_moving_summary(train_error)
+
+        # slim already adds regularization to a collection, no extra handling
+        self.cost = cost
+        summary.add_moving_summary(cost)
+
+        # monitor histogram of all weight (of conv and fc layers) in tensorboard
+        summary.add_param_summary(('.*/W', ['histogram', 'rms']),
+                                  ('.*/weights', ['histogram', 'rms'])  # to also work with slim
+                                  )
+
+    def _get_optimizer(self):
+        lr = tf.train.exponential_decay(
+            learning_rate=1e-3,
+            global_step=get_global_step_var(),
+            decay_steps=468 * 10,
+            decay_rate=0.3, staircase=True, name='learning_rate')
+        # This will also put the summary in tensorboard, stat.json and print in terminal
+        # but this time without moving average
+        tf.summary.scalar('lr', lr)
+        return tf.train.AdamOptimizer(lr)
+
+
+def get_data():
+    train = BatchData(dataset.Mnist('train'), 128)
+    test = BatchData(dataset.Mnist('test'), 256, remainder=True)
+    return train, test
+
+
+def get_config():
+    # automatically setup the directory train_log/mnist-convnet for logging
+    logger.auto_set_dir()
+
+    dataset_train, dataset_test = get_data()
+    # How many iterations you want in each epoch.
+    # This is the default value, don't actually need to set it in the config
+    steps_per_epoch = dataset_train.size()
+
+    # get the config which contains everything necessary in a training
+    return TrainConfig(
+        model=Model(),
+        dataflow=dataset_train,  # the DataFlow instance for training
+        callbacks=[
+            ModelSaver(),   # save the model after every epoch
+            InferenceRunner(    # run inference(for validation) after every epoch
+                dataset_test,   # the DataFlow instance used for validation
+                # Calculate both the cost and the error for this DataFlow
+                [ScalarStats('cross_entropy_loss'), ClassificationError('incorrect')]),
+        ],
+        steps_per_epoch=steps_per_epoch,
+        max_epoch=100,
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu', help='comma separated list of GPU(s) to use.')
+    parser.add_argument('--load', help='load model')
+    args = parser.parse_args()
+    if args.gpu:
+        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
+
+    config = get_config()
+    if args.load:
+        config.session_init = SaverRestore(args.load)
+    SimpleTrainer(config).train()

examples/mnist-visualizations.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# File: mnist-convnet.py
+# Author: Yuxin Wu <ppwwyyxxc@gmail.com>
+
+import numpy as np
+import os
+import sys
+import argparse
+
+"""
+MNIST ConvNet example.
+about 0.6% validation error after 30 epochs.
+"""
+
+# Just import everything into current namespace
+from tensorpack import *
+import tensorflow as tf
+import tensorpack.tfutils.symbolic_functions as symbf
+
+IMAGE_SIZE = 28
+
+
+def visualize_conv_weights(filters, name):
+    """Visualize use weights in convolution filters.
+
+    Args:
+        filters: tensor containing the weights [H,W,Cin,Cout]
+        name: label for tensorboard
+
+    Returns:
+        image of all weight
+    """
+    with tf.name_scope('visualize_w_' + name):
+        filters = tf.transpose(filters, (3, 2, 0, 1))   # [h, w, cin, cout] -> [cout, cin, h, w]
+        filters = tf.unstack(filters)                   # --> cout * [cin, h, w]
+        filters = tf.concat(filters, 1)                 # --> [cin, cout * h, w]
+        filters = tf.unstack(filters)                   # --> cin * [cout * h, w]
+        filters = tf.concat(filters, 1)                 # --> [cout * h, cin * w]
+        filters = tf.expand_dims(filters, 0)
+        filters = tf.expand_dims(filters, -1)
+
+    tf.summary.image('visualize_w_' + name, filters)
+
+
+def visualize_conv_activations(activation, name):
+    """Visualize activations for convolution layers.
+
+    Remarks:
+        This tries to place all activations into a square.
+
+    Args:
+        activation: tensor with the activation [B,H,W,C]
+        name: label for tensorboard
+
+    Returns:
+        image of almost all activations
+    """
+    import math
+    with tf.name_scope('visualize_act_' + name):
+        _, h, w, c = activation.get_shape().as_list()
+        rows = []
+        c_per_row = int(math.sqrt(c))
+        for y in range(0, c - c_per_row, c_per_row):
+            row = activation[:, :, :, y:y + c_per_row]  # [?, H, W, 32] --> [?, H, W, 5]
+            cols = tf.unstack(row, axis=3)              # [?, H, W, 5] --> 5 * [?, H, W]
+            row = tf.concat(cols, 1)
+            rows.append(row)
+
+        viz = tf.concat(rows, 2)
+    tf.summary.image('visualize_act_' + name, tf.expand_dims(viz, -1))
+
+
+class Model(ModelDesc):
+    def _get_inputs(self):
+        """
+        Define all the inputs (with type, shape, name) that
+        the graph will need.
+        """
+        return [InputDesc(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE), 'input'),
+                InputDesc(tf.int32, (None,), 'label')]
+
+    def _build_graph(self, inputs):
+
+        image, label = inputs
+        image = tf.expand_dims(image * 2 - 1, 3)
+
+        with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32):
+            c0 = Conv2D('conv0', image)
+            p0 = MaxPooling('pool0', c0, 2)
+            c1 = Conv2D('conv1', p0)
+            c2 = Conv2D('conv2', c1)
+            p1 = MaxPooling('pool1', c2, 2)
+            c3 = Conv2D('conv3', p1)
+            fc1 = FullyConnected('fc0', c3, 512, nl=tf.nn.relu)
+            fc1 = Dropout('dropout', fc1, 0.5)
+            logits = FullyConnected('fc1', fc1, out_dim=10, nl=tf.identity)
+
+        with tf.name_scope('visualizations'):
+            visualize_conv_weights(c0.variables.W, 'conv0')
+            visualize_conv_activations(c0, 'conv0')
+            visualize_conv_weights(c1.variables.W, 'conv1')
+            visualize_conv_activations(c1, 'conv1')
+            visualize_conv_weights(c2.variables.W, 'conv2')
+            visualize_conv_activations(c2, 'conv2')
+            visualize_conv_weights(c3.variables.W, 'conv3')
+            visualize_conv_activations(c3, 'conv3')
+
+            tf.summary.image('input', (image + 1.0) * 128., 3)
+
+        cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label)
+        cost = tf.reduce_mean(cost, name='cross_entropy_loss')
+
+        # compute the "incorrect vector", for the callback ClassificationError to use at validation time
+        wrong = symbf.prediction_incorrect(logits, label, name='incorrect')
+        accuracy = symbf.accuracy(logits, label)
+
+        wd_cost = tf.multiply(1e-5,
+                              regularize_cost('fc.*/W', tf.nn.l2_loss),
+                              name='regularize_loss')
+        self.cost = tf.add_n([wd_cost, cost], name='total_cost')
+        summary.add_moving_summary(cost, wd_cost, self.cost, accuracy)
+
+        summary.add_param_summary(('.*/W', ['histogram', 'rms']),
+                                  ('.*/weights', ['histogram', 'rms'])  # to also work with slim
+                                  )
+
+    def _get_optimizer(self):
+        lr = tf.train.exponential_decay(
+            learning_rate=1e-3,
+            global_step=get_global_step_var(),
+            decay_steps=468 * 10,
+            decay_rate=0.3, staircase=True, name='learning_rate')
+        tf.summary.scalar('lr', lr)
+        return tf.train.AdamOptimizer(lr)
+
+
+def get_data():
+    train = BatchData(dataset.Mnist('train'), 128)
+    test = BatchData(dataset.Mnist('test'), 256, remainder=True)
+    return train, test
+
+
+def get_config():
+
+    logger.auto_set_dir()
+    dataset_train, dataset_test = get_data()
+
+    return TrainConfig(
+        model=Model(),
+        dataflow=dataset_train,
+        callbacks=[
+            ModelSaver(),
+            InferenceRunner(
+                dataset_test, [ScalarStats('cross_entropy_loss'), ClassificationError('incorrect')]),
+        ],
+        steps_per_epoch=dataset_train.size(),
+        max_epoch=100,
+    )
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--gpu', help='comma separated list of GPU(s) to use.')
+    parser.add_argument('--load', help='load model')
+    args = parser.parse_args()
+    if args.gpu:
+        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
+
+    config = get_config()
+    if args.load:
+        config.session_init = SaverRestore(args.load)
+    SimpleTrainer(config).train()

tensorpack/tfutils/symbolic_functions.py

@@ -22,6 +22,18 @@ def prediction_incorrect(logits, label, topk=1, name='incorrect_vector'):
                    tf.float32, name=name)
 
 
+def accuracy(logits, label, topk=1, name='accuracy'):
+    """
+    Args:
+        logits: shape [B,C].
+        label: shape [B].
+        topk(int): topk
+    Returns:
+        a single scalar
+    """
+    return tf.reduce_mean(tf.cast(tf.nn.in_top_k(logits, label, topk), tf.float32), name=name)
+
+
 def flatten(x):
     """
     Flatten the tensor.