Skip to content

C
CapsuleNet_vs_CNN
Commit 8c63ba68 authored by PANKAJ KUMAR's avatar PANKAJ KUMAR
Browse files
Options

submission files

parents
Hide whitespace changes
Inline Side-by-side
Showing with 494 additions and 0 deletions
Google Colab links.txt 0 → 100644
View file @ 8c63ba68
CapsuleNet: https://colab.research.google.com/drive/1pDQ4xTUhj3MzL2jsi9eGm2VGrXsds7jY
CNN: https://colab.research.google.com/drive/1bvo0Ab0oloxMfZdMhSWh3OTbp8WWqGog
src/CNN.ipynb 0 → 100644
View file @ 8c63ba68
This source diff could not be displayed because it is too large. You can view the blob instead.
src/CapsuleNet.ipynb 0 → 100644
View file @ 8c63ba68
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "CapsuleNet.ipynb",
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "code",
"metadata": {
"colab_type": "code",
"id": "Vn9Z_CNTPktk",
"colab": {}
},
"source": [
"from __future__ import division, print_function, unicode_literals\n",
"\n",
"%matplotlib inline\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"import numpy as np\n",
"import tensorflow as tf\n",
"from tensorflow.examples.tutorials.mnist import input_data\n",
"tf.reset_default_graph()\n",
"np.random.seed(42)\n",
"tf.set_random_seed(42)\n",
"import keras.backend as K"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "lfXJcMAJ970c",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 88
},
"outputId": "616f2a98-1bb5-408c-e072-bdaa4f2017e9"
},
"source": [
"\n",
"\n",
"mnist = input_data.read_data_sets(\"/tmp/data/\")\n",
"\n",
"\n",
"X = tf.placeholder(shape=[None, 28, 28, 1], dtype=tf.float32, name=\"X\")\n",
"caps1_n_maps = 32\n",
"caps1_n_caps = caps1_n_maps * 6 * 6 \n",
"caps1_n_dims = 8\n",
"caps2_n_caps = 10\n",
"caps2_n_dims = 16\n",
"init_sigma = 0.1\n",
"\n",
"\n",
"def initializelayer(input1):\n",
" conv1 = tf.layers.conv2d(input1, name=\"conv1\", filters=256,kernel_size=9,strides=1,padding='valid',activation=tf.nn.relu)\n",
" conv2 = tf.layers.conv2d(conv1, name=\"conv2\", filters=256,kernel_size=9,strides=2,padding='valid',activation=tf.nn.relu)\n",
" caps1_raw = tf.reshape(conv2, [-1, caps1_n_caps, caps1_n_dims],\n",
" name=\"caps1_raw\")\n",
" \n",
" W_init = tf.random_normal(\n",
" shape=(1, caps1_n_caps, caps2_n_caps, caps2_n_dims, caps1_n_dims),\n",
" stddev=init_sigma, dtype=tf.float32, name=\"W_init\")\n",
" W = tf.Variable(W_init, name=\"W\") \n",
" return conv1,conv2,caps1_raw,W\n",
"\n",
"conv1,conv2,primecapsraw,initialW=initializelayer(X) \n",
"\n"
],
"execution_count": 32,
"outputs": [
{
"output_type": "stream",
"text": [
"Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
"Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
"Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
"Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "9ve5Y6MBnZhJ",
"colab_type": "code",
"colab": {}
},
"source": [
"\n",
"\n",
"def squash(x, axis=-1, epsilon=1e-7, name=None):\n",
" s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n",
" scale = K.sqrt(s_squared_norm) / (1 + s_squared_norm)\n",
" return scale * x\n",
"\n"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "n4f86FUgndp1",
"colab_type": "code",
"colab": {}
},
"source": [
"primecaps1 = squash(primecapsraw, name=\"primecaps1\")\n",
"\n",
"\n",
"\n",
"batch_size = tf.shape(X)[0]\n",
"#############################################################################\n",
"W_tiled = tf.tile(initialW, [batch_size, 1, 1, 1, 1], name=\"W_tiled\")\n",
"caps1_output_expanded = tf.expand_dims(primecaps1, -1,\n",
" name=\"caps1_output_expanded\")\n",
"caps1_output_tile = tf.expand_dims(caps1_output_expanded, 2,\n",
" name=\"caps1_output_tile\")\n",
"caps1_output_tiled = tf.tile(caps1_output_tile, [1, 1, caps2_n_caps, 1, 1],\n",
" name=\"caps1_output_tiled\")\n",
"caps2_predicted = tf.matmul(W_tiled, caps1_output_tiled,\n",
" name=\"caps2_predicted\")\n",
"raw_weights = tf.zeros([batch_size, caps1_n_caps, caps2_n_caps, 1, 1],\n",
" dtype=np.float32, name=\"raw_weights\")\n",
"routing_weights = tf.nn.softmax(raw_weights, dim=2, name=\"routing_weights\")\n",
"weighted_predictions = tf.multiply(routing_weights, caps2_predicted,\n",
" name=\"weighted_predictions\")\n",
"weighted_sum = tf.reduce_sum(weighted_predictions, axis=1, keep_dims=True,\n",
" name=\"weighted_sum\")\n",
"\n",
"caps2_output_round_1 = squash(weighted_sum, axis=-2,\n",
" name=\"caps2_output_round_1\")\n",
"##########################################################################\n",
"\n",
"caps2_output_round_1_tiled = tf.tile(\n",
" caps2_output_round_1, [1, caps1_n_caps, 1, 1, 1],\n",
" name=\"caps2_output_round_1_tiled\")\n",
"\n",
"agreement = tf.matmul(caps2_predicted, caps2_output_round_1_tiled,\n",
" transpose_a=True, name=\"agreement\")\n",
"raw_weights_round_2 = tf.add(raw_weights, agreement,\n",
" name=\"raw_weights_round_2\")\n",
"routing_weights_round_2 = tf.nn.softmax(raw_weights_round_2,\n",
" dim=2,\n",
" name=\"routing_weights_round_2\")\n",
"weighted_predictions_round_2 = tf.multiply(routing_weights_round_2,\n",
" caps2_predicted,\n",
" name=\"weighted_predictions_round_2\")\n",
"weighted_sum_round_2 = tf.reduce_sum(weighted_predictions_round_2,\n",
" axis=1, keep_dims=True,\n",
" name=\"weighted_sum_round_2\")\n",
"caps2_output_round_2 = squash(weighted_sum_round_2,\n",
" axis=-2,\n",
" name=\"caps2_output_round_2\")\n",
"\n",
"##############################################################################\n",
"\n",
"\n",
"caps2_output = caps2_output_round_2"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "GrwKXEuS-6DA",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"outputId": "d9103bef-a6fc-4b87-9cba-a904b6f3d03b"
},
"source": [
"\n",
"\n",
"\n",
"\n",
"def condition(input, counter):\n",
" return tf.less(counter, 100)\n",
"\n",
"def loop_body(input, counter):\n",
" output = tf.add(input, tf.square(counter))\n",
" return output, tf.add(counter, 1)\n",
"\n",
"with tf.name_scope(\"compute_sum_of_squares\"):\n",
" counter = tf.constant(1)\n",
" sum_of_squares = tf.constant(0)\n",
"\n",
" result = tf.while_loop(condition, loop_body, [sum_of_squares, counter])\n",
" \n",
"with tf.Session() as sess:\n",
" print(sess.run(result))\n",
" sum([i**2 for i in range(1, 100 + 1)])\n",
" \n"
],
"execution_count": 35,
"outputs": [
{
"output_type": "stream",
"text": [
"(328350, 100)\n"
],
"name": "stdout"
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "z8Dwh3U1ukkj",
"colab_type": "code",
"colab": {}
},
"source": [
"def safe_norm(s, axis=-1, epsilon=1e-7, keep_dims=False, name=None):\n",
" with tf.name_scope(name, default_name=\"safe_norm\"):\n",
" squared_norm = tf.reduce_sum(tf.square(s), axis=axis,\n",
" keep_dims=keep_dims)\n",
" return tf.sqrt(squared_norm + epsilon)\n",
"y_proba = safe_norm(caps2_output, axis=-2, name=\"y_proba\")\n",
"y_proba_argmax = tf.argmax(y_proba, axis=2, name=\"y_proba\")\n",
"y_proba_argmax\n",
"\n",
"y_pred = tf.squeeze(y_proba_argmax, axis=[1,2], name=\"y_pred\")\n",
"y = tf.placeholder(shape=[None], dtype=tf.int64, name=\"y\")\n",
"m_plus = 0.9\n",
"m_minus = 0.1\n",
"lambda_ = 0.5\n",
"T = tf.one_hot(y, depth=caps2_n_caps, name=\"T\")\n",
"\n",
"with tf.Session():\n",
" \n",
" caps2_output_norm = safe_norm(caps2_output, axis=-2, keep_dims=True,\n",
" name=\"caps2_output_norm\")\n",
" \n",
"present_error_raw = tf.square(tf.maximum(0., m_plus - caps2_output_norm),\n",
" name=\"present_error_raw\")\n",
"present_error = tf.reshape(present_error_raw, shape=(-1, 10),\n",
" name=\"present_error\")\n",
"absent_error_raw = tf.square(tf.maximum(0., caps2_output_norm - m_minus),\n",
" name=\"absent_error_raw\")\n",
"absent_error = tf.reshape(absent_error_raw, shape=(-1, 10),\n",
" name=\"absent_error\")\n",
"L = tf.add(T * present_error, lambda_ * (1.0 - T) * absent_error,\n",
" name=\"L\")\n",
"\n",
"margin_loss = tf.reduce_mean(tf.reduce_sum(L, axis=1), name=\"margin_loss\")\n",
"mask_with_labels = tf.placeholder_with_default(False, shape=(),\n",
" name=\"mask_with_labels\")\n"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "nQtTjhKZvHx8",
"colab_type": "code",
"colab": {}
},
"source": [
"\n",
"\n",
"reconstruction_targets = tf.cond(mask_with_labels,\n",
" lambda: y, \n",
" lambda: y_pred, \n",
" name=\"reconstruction_targets\")\n",
"reconstruction_mask = tf.one_hot(reconstruction_targets,\n",
" depth=caps2_n_caps,\n",
" name=\"reconstruction_mask\")\n",
"reconstruction_mask_reshaped = tf.reshape(\n",
" reconstruction_mask, [-1, 1, caps2_n_caps, 1, 1],\n",
" name=\"reconstruction_mask_reshaped\")\n",
"caps2_output_masked = tf.multiply(\n",
" caps2_output, reconstruction_mask_reshaped,\n",
" name=\"caps2_output_masked\")\n",
"decoder_input = tf.reshape(caps2_output_masked,\n",
" [-1, caps2_n_caps * caps2_n_dims],\n",
" name=\"decoder_input\")"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "vlYwRKv0uZxJ",
"colab_type": "code",
"colab": {}
},
"source": [
"\n",
"\n",
"\n",
"n_hidden1 = 512\n",
"n_hidden2 = 1024\n",
"n_output = 28 * 28\n",
"with tf.name_scope(\"decoder\"):\n",
" hidden1 = tf.layers.dense(decoder_input, n_hidden1,\n",
" activation=tf.nn.relu,\n",
" name=\"hidden1\")\n",
" hidden2 = tf.layers.dense(hidden1, n_hidden2,\n",
" activation=tf.nn.relu,\n",
" name=\"hidden2\")\n",
" decoder_output = tf.layers.dense(hidden2, n_output,\n",
" activation=tf.nn.sigmoid,\n",
" name=\"decoder_output\")\n",
" X_flat = tf.reshape(X, [-1, n_output], name=\"X_flat\")\n",
"squared_difference = tf.square(X_flat - decoder_output,\n",
" name=\"squared_difference\")\n",
"reconstruction_loss = tf.reduce_mean(squared_difference,\n",
" name=\"reconstruction_loss\")\n",
"alpha = 0.0005\n",
"\n",
"loss = tf.add(margin_loss, alpha * reconstruction_loss, name=\"loss\")\n",
"correct = tf.equal(y, y_pred, name=\"correct\")\n",
"accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name=\"accuracy\")\n",
"optimizer = tf.train.AdamOptimizer()\n",
"training_op = optimizer.minimize(loss, name=\"training_op\")\n",
"init = tf.global_variables_initializer()\n",
"saver = tf.train.Saver()"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab_type": "code",
"id": "B_pL1JDwPiBH",
"colab": {}
},
"source": [
"\n",
"n_epochs = 10\n",
"batch_size = 128\n",
"restore_checkpoint = True\n",
"\n",
"n_iterations_per_epoch = mnist.train.num_examples // batch_size\n",
"n_iterations_validation = mnist.validation.num_examples // batch_size\n",
"best_loss_val = np.infty\n",
"checkpoint_path = \"./my_capsule_network\"\n",
"\n",
"with tf.Session() as sess:\n",
" if restore_checkpoint and tf.train.checkpoint_exists(checkpoint_path):\n",
" saver.restore(sess, checkpoint_path)\n",
" else:\n",
" init.run()\n",
"\n",
" for epoch in range(n_epochs):\n",
" for iteration in range(1, n_iterations_per_epoch + 1):\n",
" X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
" \n",
" _, loss_train = sess.run(\n",
" [training_op, loss],\n",
" feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
" y: y_batch,\n",
" mask_with_labels: True})\n",
" print(\"\\rIteration: {}/{} ({:.1f}%) Loss: {:.5f}\".format(\n",
" iteration, n_iterations_per_epoch,\n",
" iteration * 100 / n_iterations_per_epoch,\n",
" loss_train),\n",
" end=\"\")\n",
"\n",
" \n",
" loss_vals = []\n",
" acc_vals = []\n",
" for iteration in range(1, n_iterations_validation + 1):\n",
" X_batch, y_batch = mnist.validation.next_batch(batch_size)\n",
" loss_val, acc_val = sess.run(\n",
" [loss, accuracy],\n",
" feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
" y: y_batch})\n",
" loss_vals.append(loss_val)\n",
" acc_vals.append(acc_val)\n",
" print(\"\\rEvaluating the model: {}/{} ({:.1f}%)\".format(\n",
" iteration, n_iterations_validation,\n",
" iteration * 100 / n_iterations_validation),\n",
" end=\" \" * 10)\n",
" loss_val = np.mean(loss_vals)\n",
" acc_val = np.mean(acc_vals)\n",
" print(\"\\rEpoch: {} Val accuracy: {:.4f}% Loss: {:.6f}{}\".format(\n",
" epoch + 1, acc_val * 100, loss_val,\n",
" \" (improved)\" if loss_val < best_loss_val else \"\"))\n",
"\n",
" \n",
" if loss_val < best_loss_val:\n",
" save_path = saver.save(sess, checkpoint_path)\n",
" best_loss_val = loss_val"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab_type": "code",
"id": "mDokl--wPdez",
"colab": {}
},
"source": [
"n_iterations_test = mnist.test.num_examples // batch_size\n",
"\n",
"with tf.Session() as sess:\n",
" saver.restore(sess, checkpoint_path)\n",
"\n",
" loss_tests = []\n",
" acc_tests = []\n",
" for iteration in range(1, n_iterations_test + 1):\n",
" X_batch, y_batch = mnist.test.next_batch(batch_size)\n",
" loss_test, acc_test = sess.run(\n",
" [loss, accuracy],\n",
" feed_dict={X: X_batch.reshape([-1, 28, 28, 1]),\n",
" y: y_batch})\n",
" loss_tests.append(loss_test)\n",
" acc_tests.append(acc_test)\n",
" print(\"\\rEvaluating the model: {}/{} ({:.1f}%)\".format(\n",
" iteration, n_iterations_test,\n",
" iteration * 100 / n_iterations_test),\n",
" end=\" \" * 10)\n",
" loss_test = np.mean(loss_tests)\n",
" acc_test = np.mean(acc_tests)\n",
" print(\"\\rFinal test accuracy: {:.4f}% Loss: {:.6f}\".format(\n",
" acc_test * 100, loss_test))"
],
"execution_count": 0,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "PuBEQloOEgk2",
"colab_type": "code",
"colab": {}
},
"source": [
"n_samples = 5\n",
"\n",
"sample_images = mnist.test.images[:n_samples].reshape([-1, 28, 28, 1])\n",
"\n",
"with tf.Session() as sess:\n",
" saver.restore(sess, checkpoint_path)\n",
" caps2_output_value, decoder_output_value, y_pred_value = sess.run(\n",
" [caps2_output, decoder_output, y_pred],\n",
" feed_dict={X: sample_images,\n",
" y: np.array([], dtype=np.int64)})\n",
" sample_images = sample_images.reshape(-1, 28, 28)\n",
"reconstructions = decoder_output_value.reshape([-1, 28, 28])\n",
"\n",
"plt.figure(figsize=(n_samples * 2, 3))\n",
"for index in range(n_samples):\n",
" plt.subplot(1, n_samples, index + 1)\n",
" plt.imshow(sample_images[index], cmap=\"binary\")\n",
" plt.title(\"Label:\" + str(mnist.test.labels[index]))\n",
" plt.axis(\"off\")\n",
"\n",
"plt.show()\n",
"\n",
"plt.figure(figsize=(n_samples * 2, 3))\n",
"for index in range(n_samples):\n",
" plt.subplot(1, n_samples, index + 1)\n",
" plt.title(\"Predicted:\" + str(y_pred_value[index]))\n",
" plt.imshow(reconstructions[index], cmap=\"binary\")\n",
" plt.axis(\"off\")\n",
" \n",
"plt.show()"
],
"execution_count": 0,
"outputs": []
}
]
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment