model docs

tensorpack/models/__init__.py

@@ -6,7 +6,7 @@ from pkgutil import walk_packages
 import os
 import os.path
 
-def global_import(name):
+def _global_import(name):
     p = __import__(name, globals(), locals(), level=1)
     lst = p.__all__ if '__all__' in dir(p) else dir(p)
     for k in lst:
@@ -15,5 +15,5 @@ def global_import(name):
 for _, module_name, _ in walk_packages(
         [os.path.dirname(__file__)]):
     if not module_name.startswith('_'):
-        global_import(module_name)
+        _global_import(module_name)
 

tensorpack/models/batch_norm.py

@@ -10,7 +10,6 @@ from ._common import layer_register
 
 __all__ = ['BatchNorm']
 
-
 # http://stackoverflow.com/questions/33949786/how-could-i-use-batch-normalization-in-tensorflow
 # TF batch_norm only works for 4D tensor right now: #804
 # decay: being too close to 1 leads to slow start-up, but ends up better
@@ -19,15 +18,20 @@ __all__ = ['BatchNorm']
 def BatchNorm(x, use_local_stat=True, decay=0.999, epsilon=1e-5):
     """
     Batch normalization layer as described in:
-    Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift
-    http://arxiv.org/abs/1502.03167
+
+    `Batch Normalization: Accelerating Deep Network Training by
+    Reducing Internal Covariate Shift <http://arxiv.org/abs/1502.03167>`_.
+
     Notes:
-    Whole-population mean/variance is calculated by a running-average mean/variance, with decay rate 0.999
-    Epsilon for variance is set to 1e-5, as is torch/nn: https://github.com/torch/nn/blob/master/BatchNormalization.lua
 
-    x: BHWC tensor or a vector
-    use_local_stat: bool. whether to use mean/var of this batch or the running
-    average. Usually set to True in training and False in testing
+    * Whole-population mean/variance is calculated by a running-average mean/variance, with decay rate 0.999
+    * Epsilon for variance is set to 1e-5, as is `torch/nn <https://github.com/torch/nn/blob/master/BatchNormalization.lua>`_.
+
+    :param input: a NHWC tensor or a NC vector
+    :param use_local_stat: bool. whether to use mean/var of this batch or the running average.
+        Usually set to True in training and False in testing
+    :param decay: decay rate. default to 0.999.
+    :param epsilon: default to 1e-5.
     """
 
     shape = x.get_shape().as_list()

tensorpack/models/conv2d.py

@@ -16,11 +16,18 @@ def Conv2D(x, out_channel, kernel_shape,
            W_init=None, b_init=None,
            nl=tf.nn.relu, split=1, use_bias=True):
     """
-    kernel_shape: (h, w) or a int
-    stride: (h, w) or a int
-    padding: 'valid' or 'same'
-    split: split channels. used in Alexnet
-    use_bias: whether to use bias
+    2D convolution on 4D inputs.
+
+    :param input: a tensor of shape NHWC
+    :param kernel_shape: (h, w) or a int
+    :param stride: (h, w) or a int. default to 1
+    :param padding: 'valid' or 'same'. default to 'same'
+    :param split: split channels as used in Alexnet. a int default to 1
+    :param W_init: initializer for W. default to `xavier_initializer_conv2d`.
+    :param b_init: initializer for b. default to zero initializer.
+    :param nl: nonlinearity. default to `relu`.
+    :param use_bias: whether to use bias. a boolean default to True
+    :returns: a NHWC tensor
     """
     in_shape = x.get_shape().as_list()
     num_in = np.prod(in_shape[1:])

tensorpack/models/fc.py

@@ -16,7 +16,15 @@ def FullyConnected(x, out_dim,
                    W_init=None, b_init=None,
                    nl=tf.nn.relu, use_bias=True):
     """
-    Fully-Connected layer
+    Fully-Connected layer.
+
+    :param input: a tensor to be flattened except the first dimension.
+    :param out_dim: output dimension
+    :param W_init: initializer for W. default to `xavier_initializer_conv2d`.
+    :param b_init: initializer for b. default to zero initializer.
+    :param nl: nonlinearity. default to `relu`.
+    :param use_bias: whether to use bias. a boolean default to True
+    :returns: a 2D tensor
     """
     x = batch_flatten(x)
     in_dim = x.get_shape().as_list()[1]

tensorpack/models/image_sample.py

@@ -43,12 +43,12 @@ def ImageSample(inputs):
     """
     Sample the template image, using the given coordinate, by bilinear interpolation.
     It mimics the same behavior described in:
-    Spatial Transformer Networks, http://arxiv.org/abs/1506.02025
+    `Spatial Transformer Networks <http://arxiv.org/abs/1506.02025>`_.
 
-    inputs: list of [template, mapping]
-    template: bxhxwxc
-    mapping: bxh2xw2x2  (y, x) real-value coordinates
-    Return: bxh2xw2xc
+    :param input: [template, mapping]. template of shape NHWC. mapping of
+        shape NHW2, where each pair of the last dimension is a (y, x) real-value
+        coordinate.
+    :returns: a NHWC output tensor.
     """
     template, mapping = inputs
     assert template.get_shape().ndims == 4 and mapping.get_shape().ndims == 4

tensorpack/models/model_desc.py

@@ -14,16 +14,14 @@ __all__ = ['ModelDesc', 'InputVar']
 InputVar = namedtuple('InputVar', ['type', 'shape', 'name'])
 
 class ModelDesc(object):
+    """ Base class for a model description """
     __metaclass__ = ABCMeta
 
-
-    def __init__(self):
-        pass
-
     def get_input_vars(self):
         """
-        return the list of raw input vars in the graph
-        if reuse=True, results will be cached, to avoid creating the same variable
+        Create and return raw input vars in the graph.
+
+        :returns: the list of raw input vars in the graph
         """
         input_vars = self._get_input_vars()
         ret = []
@@ -32,16 +30,13 @@ class ModelDesc(object):
         return ret
 
     def reuse_input_vars(self):
-        """ find input_vars in default graph"""
+        """ Find and return already-defined input_vars in default graph"""
         input_var_names = [k.name for k in self._get_input_vars()]
         g = tf.get_default_graph()
         return [g.get_tensor_by_name(name + ":0") for name in input_var_names]
 
     @abstractmethod
     def _get_input_vars(self):
-        """
-        return the list of input vars in the graph
-        """
         pass
 
     # TODO move this to QueueInputTrainer
@@ -55,21 +50,21 @@ class ModelDesc(object):
         return tf.FIFOQueue(100, [x.dtype for x in input_vars], name='input_queue')
 
     def get_cost(self, input_vars, is_training):
+        """
+        :param input_vars: a list of input variable in the graph
+            e.g.: [image_var, label_var] with:
+
+            * image_var: bx28x28
+            * label_var: bx1 integer
+        :param is_training: a boolean
+        :returns: the cost to minimize. a scalar variable
+        """
         assert type(is_training) == bool
         return self._get_cost(input_vars, is_training)
 
     @abstractmethod
     def _get_cost(self, input_vars, is_training):
-        """
-        Args:
-            input_vars: a list of input variable in the graph
-            e.g.: [image_var, label_var] with:
-                image_var: bx28x28
-                label_var: bx1 integer
-            is_training: a python bool variable
-        Returns:
-            the cost to minimize. scalar variable
-        """
+        pass
 
     def get_gradient_processor(self):
         """ Return a list of GradientProcessor. They will be executed in order"""

tensorpack/models/nonlin.py

@@ -12,6 +12,13 @@ __all__ = ['Maxout', 'PReLU', 'LeakyReLU']
 
 @layer_register()
 def Maxout(x, num_unit):
+    """
+    Maxout networks as in `Maxout Networks <http://arxiv.org/abs/1302.4389>`_.
+
+    :param input: a NHWC tensor.
+    :param num_unit: a int. must be divisible by C.
+    :returns: a NHW(C/num_unit) tensor
+    """
     input_shape = x.get_shape().as_list()
     assert len(input_shape) == 4
     ch = input_shape[3]
@@ -21,7 +28,14 @@ def Maxout(x, num_unit):
 
 @layer_register()
 def PReLU(x, init=tf.constant_initializer(0.001), name=None):
-    """ allow name to be compatible to other builtin nonlinearity function"""
+    """
+    Parameterized relu as in `Delving Deep into Rectifiers: Surpassing
+    Human-Level Performance on ImageNet Classification
+    <http://arxiv.org/abs/1502.01852>`_.
+
+    :param input: any tensor.
+    :param init: initializer for the p. default to 0.001.
+    """
     alpha = tf.get_variable('alpha', [], initializer=init)
     x = ((1 + alpha) * x + (1 - alpha) * tf.abs(x))
     if name is None:
@@ -31,6 +45,14 @@ def PReLU(x, init=tf.constant_initializer(0.001), name=None):
 
 @layer_register()
 def LeakyReLU(x, alpha, name=None):
+    """
+    Leaky relu as in `Rectifier Nonlinearities Improve Neural Network Acoustic
+    Models
+    <http://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`_.
+
+    :param input: any tensor.
+    :param alpha: the negative slope.
+    """
     alpha = float(alpha)
     x = ((1 + alpha) * x + (1 - alpha) * tf.abs(x))
     if name is None:

tensorpack/models/pool.py

@@ -13,9 +13,13 @@ __all__ = ['MaxPooling', 'FixedUnPooling', 'AvgPooling', 'GlobalAvgPooling']
 @layer_register()
 def MaxPooling(x, shape, stride=None, padding='VALID'):
     """
-        shape, stride: int or list/tuple of length 2
-        if stride is None, use shape by default
-        padding: 'VALID' or 'SAME'
+    MaxPooling on images.
+
+    :param input: NHWC tensor.
+    :param shape: int or [h, w]
+    :param stride: int or [h, w]. default to be shape.
+    :param padding: 'valid' or 'same'. default to 'valid'
+    :returns: NHWC tensor.
     """
     padding = padding.upper()
     shape = shape4d(shape)
@@ -29,9 +33,13 @@ def MaxPooling(x, shape, stride=None, padding='VALID'):
 @layer_register()
 def AvgPooling(x, shape, stride=None, padding='VALID'):
     """
-        shape, stride: int or list/tuple of length 2
-        if stride is None, use shape by default
-        padding: 'VALID' or 'SAME'
+    Average pooling on images.
+
+    :param input: NHWC tensor.
+    :param shape: int or [h, w]
+    :param stride: int or [h, w]. default to be shape.
+    :param padding: 'valid' or 'same'. default to 'valid'
+    :returns: NHWC tensor.
     """
     padding = padding.upper()
     shape = shape4d(shape)
@@ -44,16 +52,26 @@ def AvgPooling(x, shape, stride=None, padding='VALID'):
 
 @layer_register()
 def GlobalAvgPooling(x):
+    """
+    Global average pooling as in `Network In Network
+    <http://arxiv.org/abs/1312.4400>`_.
+
+    :param input: NHWC tensor.
+    :returns: NC tensor.
+    """
     assert x.get_shape().ndims == 4
     return tf.reduce_mean(x, [1, 2])
 
 @layer_register()
 def FixedUnPooling(x, shape, unpool_mat=None):
     """
-    Unpool the input with a fixed mat to perform kronecker product with
-    x: 4D tensor of (b, h, w, c)
-    shape: int or list/tuple of length 2
-    unpool_mat: a tf matrix with size=shape. if None, will use a mat with 1 at top-left corner
+    Unpool the input with a fixed mat to perform kronecker product with.
+
+    :param input: NHWC tensor
+    :param shape: int or [h, w]
+    :param unpool_mat: a tf matrix with size=shape. If None, will use a mat
+        with 1 at top-left corner.
+    :returns: NHWC tensor
     """
     shape = shape2d(shape)
     input_shape = x.get_shape().as_list()

tensorpack/models/regularize.py

@@ -19,7 +19,9 @@ l1_regularizer = tf.contrib.layers.l1_regularizer
 
 def regularize_cost(regex, func, name=None):
     """
-    Apply a regularizer on every trainable variable matching the regex
+    Apply a regularizer on every trainable variable matching the regex.
+
+    :param func: a function that takes a tensor and return a scalar.
     """
     G = tf.get_default_graph()
     params = G.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)