Add centerloss and refactor symbf (#400)

* add centerloss and refactor symbf

symbolic functions should be slim as for most projects there is no
need for the entire collection of embedding losses.

* update readme

* make the image smaller

examples/SimilarityLearning/README.md

@@ -7,14 +7,15 @@ Verification](http://yann.lecun.com/exdb/publis/pdf/chopra-05.pdf).
 * Cosine loss
 * Triplet loss in the paper [FaceNet: A Unified Embedding for Face Recognition and Clustering](https://arxiv.org/abs/1503.03832)
 * Softmax triplet loss in the paper [Deep Metric Learning using Triplet Network](https://arxiv.org/abs/1412.6622)
+* Center loss in the paper [A Discriminative Feature Learning Approach for Deep Face Recognition](http://ydwen.github.io/papers/WenECCV16.pdf)
 
 
 ## Usage:
 ```
 # to train:
-./mnist-embeddings.py --algorithm [siamese/cosine/triplet/softtriplet]
+./mnist-embeddings.py --algorithm [siamese/cosine/triplet/softtriplet/center]
 # to visualize:
-./mnist-embeddings.py --algorithm [siamese/cosine/triplet/softtriplet] --visualize --load train_log/mnist-embeddings/checkpoint
+./mnist-embeddings.py --algorithm [siamese/cosine/triplet/softtriplet/center] --visualize --load train_log/mnist-embeddings/checkpoint
 ```
 
 <p align="center"> <img src="./results.jpg"> </p>

examples/SimilarityLearning/mnist-embeddings.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # File: mnist-embeddings.py
-# Author: PatWie <mail@patwie.com>
 
 import numpy as np
 import os
@@ -28,6 +27,180 @@ except ImportError:
     MATPLOTLIB_AVAIBLABLE = False
 
 
+def contrastive_loss(left, right, y, margin, extra=False, scope="constrastive_loss"):
+    r"""Loss for Siamese networks as described in the paper:
+    `Learning a Similarity Metric Discriminatively, with Application to Face
+    Verification <http://yann.lecun.com/exdb/publis/pdf/chopra-05.pdf>`_ by Chopra et al.
+
+    .. math::
+        \frac{1}{2} [y \cdot d^2 + (1-y) \cdot \max(0, m - d)^2], d = \Vert l - r \Vert_2
+
+    Args:
+        left (tf.Tensor): left feature vectors of shape [Batch, N].
+        right (tf.Tensor): right feature vectors of shape [Batch, N].
+        y (tf.Tensor): binary labels of shape [Batch]. 1: similar, 0: not similar.
+        margin (float): horizon for negative examples (y==0).
+        extra (bool): also return distances for pos and neg.
+
+    Returns:
+        tf.Tensor: constrastive_loss (averaged over the batch), (and optionally average_pos_dist, average_neg_dist)
+    """
+    with tf.name_scope(scope):
+        y = tf.cast(y, tf.float32)
+
+        delta = tf.reduce_sum(tf.square(left - right), 1)
+        delta_sqrt = tf.sqrt(delta + 1e-10)
+
+        match_loss = delta
+        missmatch_loss = tf.square(tf.nn.relu(margin - delta_sqrt))
+
+        loss = tf.reduce_mean(0.5 * (y * match_loss + (1 - y) * missmatch_loss))
+
+        if extra:
+            num_pos = tf.count_nonzero(y)
+            num_neg = tf.count_nonzero(1 - y)
+            pos_dist = tf.where(tf.equal(num_pos, 0), 0.,
+                                tf.reduce_sum(y * delta_sqrt) / tf.cast(num_pos, tf.float32),
+                                name="pos-dist")
+            neg_dist = tf.where(tf.equal(num_neg, 0), 0.,
+                                tf.reduce_sum((1 - y) * delta_sqrt) / tf.cast(num_neg, tf.float32),
+                                name="neg-dist")
+            return loss, pos_dist, neg_dist
+        else:
+            return loss
+
+
+def siamese_cosine_loss(left, right, y, scope="cosine_loss"):
+    r"""Loss for Siamese networks (cosine version).
+    Same as :func:`contrastive_loss` but with different similarity measurement.
+
+    .. math::
+        [\frac{l \cdot r}{\lVert l\rVert \lVert r\rVert} - (2y-1)]^2
+
+    Args:
+        left (tf.Tensor): left feature vectors of shape [Batch, N].
+        right (tf.Tensor): right feature vectors of shape [Batch, N].
+        y (tf.Tensor): binary labels of shape [Batch]. 1: similar, 0: not similar.
+
+    Returns:
+        tf.Tensor: cosine-loss as a scalar tensor.
+    """
+
+    def l2_norm(t, eps=1e-12):
+        """
+        Returns:
+            tf.Tensor: norm of 2D input tensor on axis 1
+        """
+        with tf.name_scope("l2_norm"):
+            return tf.sqrt(tf.reduce_sum(tf.square(t), 1) + eps)
+
+    with tf.name_scope(scope):
+        y = 2 * tf.cast(y, tf.float32) - 1
+        pred = tf.reduce_sum(left * right, 1) / (l2_norm(left) * l2_norm(right) + 1e-10)
+
+        return tf.nn.l2_loss(y - pred) / tf.cast(tf.shape(left)[0], tf.float32)
+
+
+def triplet_loss(anchor, positive, negative, margin, extra=False, scope="triplet_loss"):
+    r"""Loss for Triplet networks as described in the paper:
+    `FaceNet: A Unified Embedding for Face Recognition and Clustering
+    <https://arxiv.org/abs/1503.03832>`_
+    by Schroff et al.
+
+    Learn embeddings from an anchor point and a similar input (positive) as
+    well as a not-similar input (negative).
+    Intuitively, a matching pair (anchor, positive) should have a smaller relative distance
+    than a non-matching pair (anchor, negative).
+
+    .. math::
+        \max(0, m + \Vert a-p\Vert^2 - \Vert a-n\Vert^2)
+
+    Args:
+        anchor (tf.Tensor): anchor feature vectors of shape [Batch, N].
+        positive (tf.Tensor): features of positive match of the same shape.
+        negative (tf.Tensor): features of negative match of the same shape.
+        margin (float): horizon for negative examples
+        extra (bool): also return distances for pos and neg.
+
+    Returns:
+        tf.Tensor: triplet-loss as scalar (and optionally average_pos_dist, average_neg_dist)
+    """
+
+    with tf.name_scope(scope):
+        d_pos = tf.reduce_sum(tf.square(anchor - positive), 1)
+        d_neg = tf.reduce_sum(tf.square(anchor - negative), 1)
+
+        loss = tf.reduce_mean(tf.maximum(0., margin + d_pos - d_neg))
+
+        if extra:
+            pos_dist = tf.reduce_mean(tf.sqrt(d_pos + 1e-10), name='pos-dist')
+            neg_dist = tf.reduce_mean(tf.sqrt(d_neg + 1e-10), name='neg-dist')
+            return loss, pos_dist, neg_dist
+        else:
+            return loss
+
+
+def soft_triplet_loss(anchor, positive, negative, extra=True, scope="soft_triplet_loss"):
+    r"""Loss for triplet networks as described in the paper:
+    `Deep Metric Learning using Triplet Network
+    <https://arxiv.org/abs/1412.6622>`_ by Hoffer et al.
+
+    It is a softmax loss using :math:`(anchor-positive)^2` and
+    :math:`(anchor-negative)^2` as logits.
+
+    Args:
+        anchor (tf.Tensor): anchor feature vectors of shape [Batch, N].
+        positive (tf.Tensor): features of positive match of the same shape.
+        negative (tf.Tensor): features of negative match of the same shape.
+        extra (bool): also return distances for pos and neg.
+
+    Returns:
+        tf.Tensor: triplet-loss as scalar (and optionally average_pos_dist, average_neg_dist)
+    """
+
+    eps = 1e-10
+    with tf.name_scope(scope):
+        d_pos = tf.sqrt(tf.reduce_sum(tf.square(anchor - positive), 1) + eps)
+        d_neg = tf.sqrt(tf.reduce_sum(tf.square(anchor - negative), 1) + eps)
+
+        logits = tf.stack([d_pos, d_neg], axis=1)
+        ones = tf.ones_like(tf.squeeze(d_pos), dtype="int32")
+
+        loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=ones))
+
+        if extra:
+            pos_dist = tf.reduce_mean(d_pos, name='pos-dist')
+            neg_dist = tf.reduce_mean(d_neg, name='neg-dist')
+            return loss, pos_dist, neg_dist
+        else:
+            return loss
+
+
+def center_loss(embedding, label, num_classes, alpha=0.1, scope="center_loss"):
+    r"""Center-Loss as described in the paper
+    `A Discriminative Feature Learning Approach for Deep Face Recognition`
+    <http://ydwen.github.io/papers/WenECCV16.pdf> by Wen et al.
+
+    Args:
+        embedding (tf.Tensor): features produced by the network
+        label (tf.Tensor): ground-truth label for each feature
+        num_classes (int): number of different classes
+        alpha (float): learning rate for updating the centers
+
+    Returns:
+        tf.Tensor: center loss
+    """
+    nrof_features = embedding.get_shape()[1]
+    centers = tf.get_variable('centers', [num_classes, nrof_features], dtype=tf.float32,
+                              initializer=tf.constant_initializer(0), trainable=False)
+    label = tf.reshape(label, [-1])
+    centers_batch = tf.gather(centers, label)
+    diff = (1 - alpha) * (centers_batch - embedding)
+    centers = tf.scatter_sub(centers, label, diff)
+    loss = tf.reduce_mean(tf.square(embedding - centers_batch), name=scope)
+    return loss
+
+
 class EmbeddingModel(ModelDesc):
     def embed(self, x, nfeatures=2):
         """Embed all given tensors into an nfeatures-dim space.  """
@@ -83,7 +256,7 @@ class SiameseModel(EmbeddingModel):
             tf.identity(self.embed(inputs[0]), name="emb")
 
         # compute the actual loss
-        cost, pos_dist, neg_dist = symbf.contrastive_loss(x, y, label, 5., extra=True, scope="loss")
+        cost, pos_dist, neg_dist = contrastive_loss(x, y, label, 5., extra=True, scope="loss")
         self.cost = tf.identity(cost, name="cost")
 
         # track these values during training
@@ -98,7 +271,7 @@ class CosineModel(SiameseModel):
         with tf.variable_scope(tf.get_variable_scope(), reuse=True):
             tf.identity(self.embed(inputs[0]), name="emb")
 
-        cost = symbf.siamese_cosine_loss(x, y, label, scope="loss")
+        cost = siamese_cosine_loss(x, y, label, scope="loss")
         self.cost = tf.identity(cost, name="cost")
         add_moving_summary(self.cost)
 
@@ -116,7 +289,7 @@ class TripletModel(EmbeddingModel):
                 InputDesc(tf.float32, (None, 28, 28), 'input_n')]
 
     def loss(self, a, p, n):
-        return symbf.triplet_loss(a, p, n, 5., extra=True, scope="loss")
+        return triplet_loss(a, p, n, 5., extra=True, scope="loss")
 
     def _build_graph(self, inputs):
         a, p, n = inputs
@@ -133,13 +306,47 @@ class TripletModel(EmbeddingModel):
 
 class SoftTripletModel(TripletModel):
     def loss(self, a, p, n):
-        return symbf.soft_triplet_loss(a, p, n, scope="loss")
+        return soft_triplet_loss(a, p, n, scope="loss")
+
+
+class CenterModel(EmbeddingModel):
+    @staticmethod
+    def get_data():
+        ds = dataset.Mnist('train')
+        ds = BatchData(ds, 128)
+        return ds
+
+    def _get_inputs(self):
+        return [InputDesc(tf.float32, (None, 28, 28), 'input'),
+                InputDesc(tf.int32, (None,), 'label')]
+
+    def _build_graph(self, inputs):
+        # get inputs
+        x, label = inputs
+        # embed them
+        x = self.embed(x)
+
+        # tag the embedding of 'input' with name 'emb', just for inference later on
+        with tf.variable_scope(tf.get_variable_scope(), reuse=True):
+            tf.identity(self.embed(inputs[0]), name="emb")
+
+        # compute the embedding loss
+        emb_cost = center_loss(x, label, 10, 0.01)
+        # compute the classification loss
+        logits = slim.layers.fully_connected(tf.nn.relu(x), 10, activation_fn=None, scope='logits')
+
+        cls_cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label),
+                                  name='classification_costs')
+        self.cost = tf.add(emb_cost, 100 * cls_cost, name="cost")
+
+        # track these values during training
+        add_moving_summary(self.cost, cls_cost, emb_cost)
 
 
 def get_config(model, algorithm_name):
 
     extra_display = ["cost"]
-    if not algorithm_name == "cosine":
+    if not algorithm_name == "cosine" and not algorithm_name == "center":
         extra_display = extra_display + ["loss/pos-dist", "loss/neg-dist"]
 
     return TrainConfig(
@@ -215,14 +422,15 @@ if __name__ == '__main__':
     parser.add_argument('--gpu', help='comma separated list of GPU(s) to use.')
     parser.add_argument('--load', help='load model')
     parser.add_argument('-a', '--algorithm', help='used algorithm', required=True,
-                        choices=["siamese", "cosine", "triplet", "softtriplet"])
+                        choices=["siamese", "cosine", "triplet", "softtriplet", "center"])
     parser.add_argument('--visualize', help='export embeddings into an image', action='store_true')
     args = parser.parse_args()
 
     ALGO_CONFIGS = {"siamese": SiameseModel,
                     "cosine": CosineModel,
                     "triplet": TripletModel,
-                    "softtriplet": SoftTripletModel}
+                    "softtriplet": SoftTripletModel,
+                    "center": CenterModel}
 
     logger.auto_set_dir(name=args.algorithm)
 

tensorpack/tfutils/symbolic_functions.py

@@ -243,155 +243,6 @@ def saliency_map(output, input, name="saliency_map"):
     return saliency_op
 
 
-def contrastive_loss(left, right, y, margin, extra=False, scope="constrastive_loss"):
-    r"""Loss for Siamese networks as described in the paper:
-    `Learning a Similarity Metric Discriminatively, with Application to Face
-    Verification <http://yann.lecun.com/exdb/publis/pdf/chopra-05.pdf>`_ by Chopra et al.
-
-    .. math::
-        \frac{1}{2} [y \cdot d^2 + (1-y) \cdot \max(0, m - d)^2], d = \Vert l - r \Vert_2
-
-    Args:
-        left (tf.Tensor): left feature vectors of shape [Batch, N].
-        right (tf.Tensor): right feature vectors of shape [Batch, N].
-        y (tf.Tensor): binary labels of shape [Batch]. 1: similar, 0: not similar.
-        margin (float): horizon for negative examples (y==0).
-        extra (bool): also return distances for pos and neg.
-
-    Returns:
-        tf.Tensor: constrastive_loss (averaged over the batch), (and optionally average_pos_dist, average_neg_dist)
-    """
-    with tf.name_scope(scope):
-        y = tf.cast(y, tf.float32)
-
-        delta = tf.reduce_sum(tf.square(left - right), 1)
-        delta_sqrt = tf.sqrt(delta + 1e-10)
-
-        match_loss = delta
-        missmatch_loss = tf.square(tf.nn.relu(margin - delta_sqrt))
-
-        loss = tf.reduce_mean(0.5 * (y * match_loss + (1 - y) * missmatch_loss))
-
-        if extra:
-            num_pos = tf.count_nonzero(y)
-            num_neg = tf.count_nonzero(1 - y)
-            pos_dist = tf.where(tf.equal(num_pos, 0), 0.,
-                                tf.reduce_sum(y * delta_sqrt) / tf.cast(num_pos, tf.float32),
-                                name="pos-dist")
-            neg_dist = tf.where(tf.equal(num_neg, 0), 0.,
-                                tf.reduce_sum((1 - y) * delta_sqrt) / tf.cast(num_neg, tf.float32),
-                                name="neg-dist")
-            return loss, pos_dist, neg_dist
-        else:
-            return loss
-
-
-def siamese_cosine_loss(left, right, y, scope="cosine_loss"):
-    r"""Loss for Siamese networks (cosine version).
-    Same as :func:`contrastive_loss` but with different similarity measurement.
-
-    .. math::
-        [\frac{l \cdot r}{\lVert l\rVert \lVert r\rVert} - (2y-1)]^2
-
-    Args:
-        left (tf.Tensor): left feature vectors of shape [Batch, N].
-        right (tf.Tensor): right feature vectors of shape [Batch, N].
-        y (tf.Tensor): binary labels of shape [Batch]. 1: similar, 0: not similar.
-
-    Returns:
-        tf.Tensor: cosine-loss as a scalar tensor.
-    """
-
-    def l2_norm(t, eps=1e-12):
-        """
-        Returns:
-            tf.Tensor: norm of 2D input tensor on axis 1
-        """
-        with tf.name_scope("l2_norm"):
-            return tf.sqrt(tf.reduce_sum(tf.square(t), 1) + eps)
-
-    with tf.name_scope(scope):
-        y = 2 * tf.cast(y, tf.float32) - 1
-        pred = tf.reduce_sum(left * right, 1) / (l2_norm(left) * l2_norm(right) + 1e-10)
-
-        return tf.nn.l2_loss(y - pred) / tf.cast(tf.shape(left)[0], tf.float32)
-
-
-def triplet_loss(anchor, positive, negative, margin, extra=False, scope="triplet_loss"):
-    r"""Loss for Triplet networks as described in the paper:
-    `FaceNet: A Unified Embedding for Face Recognition and Clustering
-    <https://arxiv.org/abs/1503.03832>`_
-    by Schroff et al.
-
-    Learn embeddings from an anchor point and a similar input (positive) as
-    well as a not-similar input (negative).
-    Intuitively, a matching pair (anchor, positive) should have a smaller relative distance
-    than a non-matching pair (anchor, negative).
-
-    .. math::
-        \max(0, m + \Vert a-p\Vert^2 - \Vert a-n\Vert^2)
-
-    Args:
-        anchor (tf.Tensor): anchor feature vectors of shape [Batch, N].
-        positive (tf.Tensor): features of positive match of the same shape.
-        negative (tf.Tensor): features of negative match of the same shape.
-        margin (float): horizon for negative examples
-        extra (bool): also return distances for pos and neg.
-
-    Returns:
-        tf.Tensor: triplet-loss as scalar (and optionally average_pos_dist, average_neg_dist)
-    """
-
-    with tf.name_scope(scope):
-        d_pos = tf.reduce_sum(tf.square(anchor - positive), 1)
-        d_neg = tf.reduce_sum(tf.square(anchor - negative), 1)
-
-        loss = tf.reduce_mean(tf.maximum(0., margin + d_pos - d_neg))
-
-        if extra:
-            pos_dist = tf.reduce_mean(tf.sqrt(d_pos + 1e-10), name='pos-dist')
-            neg_dist = tf.reduce_mean(tf.sqrt(d_neg + 1e-10), name='neg-dist')
-            return loss, pos_dist, neg_dist
-        else:
-            return loss
-
-
-def soft_triplet_loss(anchor, positive, negative, extra=True, scope="soft_triplet_loss"):
-    """Loss for triplet networks as described in the paper:
-    `Deep Metric Learning using Triplet Network
-    <https://arxiv.org/abs/1412.6622>`_ by Hoffer et al.
-
-    It is a softmax loss using :math:`(anchor-positive)^2` and
-    :math:`(anchor-negative)^2` as logits.
-
-    Args:
-        anchor (tf.Tensor): anchor feature vectors of shape [Batch, N].
-        positive (tf.Tensor): features of positive match of the same shape.
-        negative (tf.Tensor): features of negative match of the same shape.
-        extra (bool): also return distances for pos and neg.
-
-    Returns:
-        tf.Tensor: triplet-loss as scalar (and optionally average_pos_dist, average_neg_dist)
-    """
-
-    eps = 1e-10
-    with tf.name_scope(scope):
-        d_pos = tf.sqrt(tf.reduce_sum(tf.square(anchor - positive), 1) + eps)
-        d_neg = tf.sqrt(tf.reduce_sum(tf.square(anchor - negative), 1) + eps)
-
-        logits = tf.stack([d_pos, d_neg], axis=1)
-        ones = tf.ones_like(tf.squeeze(d_pos), dtype="int32")
-
-        loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=ones))
-
-        if extra:
-            pos_dist = tf.reduce_mean(d_pos, name='pos-dist')
-            neg_dist = tf.reduce_mean(d_neg, name='neg-dist')
-            return loss, pos_dist, neg_dist
-        else:
-            return loss
-
-
 def shapeless_placeholder(x, axis, name):
     """
     Make the static shape of a tensor less specific.