added 18 marks

Assignment2/autograder.py

@@ -286,7 +286,7 @@ def check_crossEntropyDelta(task_number):
 
 if __name__ == "__main__":
 
-	np.random.seed(42)
+	np.random.seed(7)
 	print()
 	correct_status = False
 	total_marks = 0

Assignment2/main.py

@@ -11,8 +11,19 @@ def taskXor():
 	# Eg. nn1.addLayer(FullyConnectedLayer(x,y))
 	###############################################
 	# TASK 3a (Marks 7) - YOUR CODE HERE
-	raise NotImplementedError
+	# raise NotImplementedError
 	###############################################
+	YTrain , YVal, YTest = np.array([int(i[1]==1) for i in YTrain]).reshape((-1,1)),  np.array([int(i[1]==1) for i in YVal]).reshape((-1,1)),  np.array([int(i[1]==1) for i in YTest]).reshape((-1,1))
+	lr,batchSize,epochs = 0.2,50,10
+	nn1 = nn.NeuralNetwork(lr, batchSize, epochs)
+	# Add layers to neural network corresponding to inputs and outputs of given data
+	input_layer = XTrain.shape[1]
+	hidden_layer = 2
+	output_layer = 1
+# 	activation_fn = 'softmax' # 'relu'
+	nn1.addLayer(nn.FullyConnectedLayer(input_layer,hidden_layer,'relu'))
+	nn1.addLayer(nn.FullyConnectedLayer(hidden_layer,output_layer,'relu'))
+
 	nn1.train(XTrain, YTrain, XVal, YVal)
 	pred, acc = nn1.validate(XTest, YTest)
 	with open("predictionsXor.csv", 'w') as file:
@@ -24,24 +35,31 @@ def taskXor():
 	return nn1
 
 def preprocessMnist(X):
-	# Perform any data preprocessing that you wish to do here
-	# Input: A 2-d numpy array containing an entire train, val or test split | Shape: n x 28*28
-	# Output: A 2-d numpy array of the same shape as the input (If the size is changed, you will get downstream errors)
-	###############################################
-	# TASK 3c (Marks 0) - YOUR CODE HERE
-	raise NotImplementedError
-	###############################################
+# Perform any data preprocessing that you wish to do here
+# Input: A 2-d numpy array containing an entire train, val or test split | Shape: n x 28*28
+# Output: A 2-d numpy array of the same shape as the input (If the size is changed, you will get downstream errors)
+###############################################
+# TASK 3c (Marks 0) - YOUR CODE HERE
+# raise NotImplementedError
+###############################################
+	return X
 
 
 def taskMnist():
 	XTrain, YTrain, XVal, YVal, XTest, _ = loadMnist()
-	# Create a NeuralNetwork object 'nn1' as follows with optimal parameters. For parameter definition, refer to nn.py file.
-	# nn1 = nn.NeuralNetwork(lr, batchSize, epochs)
+	# Create a NeuralNetwork object 'nn1' as follows with optimal parameters. For parameter definition, refer to py file.
+	lr,batchSize,epochs = 0.2,256,50
+	nn1 = nn.NeuralNetwork(lr, batchSize, epochs)
 	# Add layers to neural network corresponding to inputs and outputs of given data
-	# Eg. nn1.addLayer(FullyConnectedLayer(x,y))
+	input_layer = XTrain.shape[1]
+	hidden_layer = 256
+	output_layer = 10
+# 	activation_fn = 'softmax' # 'relu'
+	nn1.addLayer(nn.FullyConnectedLayer(input_layer,hidden_layer,'relu'))
+	nn1.addLayer(nn.FullyConnectedLayer(hidden_layer,output_layer,'softmax'))
 	###############################################
 	# TASK 3b (Marks 13) - YOUR CODE HERE
-	raise NotImplementedError
+	# raise NotImplementedError
 	###############################################
 	nn1.train(XTrain, YTrain, XVal, YVal)
 	pred, _ = nn1.validate(XTest, None)
@@ -80,11 +98,11 @@ def loadMnist():
 	testY = None # For MNIST the test labels have not been provided
 	trainX, trainY = preprocessMnist(np.array(train[0][:50000])), np.array(oneHotEncodeY(train[1][:50000],10))
 	valX, valY = preprocessMnist(np.array(train[0][50000:])), np.array(oneHotEncodeY(train[1][50000:],10))
-
+	
 	return trainX, trainY, valX, valY, testX, testY
 #################################################################################################
 
 if __name__ == "__main__":
 	np.random.seed(7)
 	taskXor()
-	taskMnist()
+	# taskMnist()

Assignment2/nn.py

@@ -17,19 +17,37 @@ class NeuralNetwork:
 		# Method to add layers to the Neural Network
 		self.layers.append(layer)
 
-	def train(self, trainX, trainY, validX=None, validY=None):
+	def train(self, trainX, trainY, validX=None, validY=None): 
 		# Method for training the Neural Network
 		# Input
 		# trainX - A list of training input data to the neural network
 		# trainY - Corresponding list of training data labels
 		# validX - A list of validation input data to the neural network
 		# validY - Corresponding list of validation data labels
-		
+		for i in range(self.epochs):
+
+			dataSize = trainX.shape[0]
+			rndIndex = np.random.permutation(dataSize)
+			trainX,trainY = trainX[rndIndex],trainY[rndIndex]
+			print('Epoch',i+1)
+			batchDataX = trainX[:self.batchSize]
+			batchDataY = trainY[:self.batchSize]
+#       prediction = predict(batchDataX)
+			activations = []
+			activations.append(batchDataX)
+			for l in self.layers:
+				activations.append(l.forwardpass(activations[-1]))
+			L = self.crossEntropyLoss(batchDataY,activations[-1])
+			delta = self.crossEntropyDelta(batchDataY,activations[-1])
+			for l in reversed(self.layers):
+				prev_activation = activations[self.layers.index(l)]
+				delta = l.backwardpass(prev_activation,delta)
+				l.updateWeights(0.01)  
 		# The methods trains the weights and baises using the training data(trainX, trainY)
 		# Feel free to print accuracy at different points using the validate() or computerAccuracy() functions of this class
 		###############################################
 		# TASK 2c (Marks 0) - YOUR CODE HERE
-		raise NotImplementedError
+		# raise NotImplementedError
 		###############################################
 		
 	def crossEntropyLoss(self, Y, predictions):
@@ -38,6 +56,7 @@ class NeuralNetwork:
 		# predictions : Predictions of the model | shape = batchSize x number of output labels
 		# Returns the cross-entropy loss between the predictions and the ground truth labels | shape = scalar
 		###############################################
+		return np.sum((-1)*Y*np.log(predictions))
 		# TASK 2a (Marks 3) - YOUR CODE HERE
 		raise NotImplementedError
 		###############################################
@@ -50,6 +69,8 @@ class NeuralNetwork:
 		#		output of the last layer of the network | shape = batchSize x number of output labels
 		###############################################
 		# TASK 2b (Marks 3) - YOUR CODE HERE
+		predictions[predictions==0] = 1e-8
+		return -Y/predictions
 		raise NotImplementedError
 		###############################################
 		
@@ -108,9 +129,9 @@ class FullyConnectedLayer:
 		# Create np arrays of appropriate sizes for weights and biases and initialise them as you see fit
 		###############################################
 		# TASK 1a (Marks 0) - YOUR CODE HERE
-		raise NotImplementedError
-		self.weights = None
-		self.biases = None
+		# raise NotImplementedError
+		self.weights = np.random.randn(in_nodes,out_nodes)/np.sqrt(in_nodes)
+		self.biases = np.zeros((out_nodes,1))
 		###############################################
 		# NOTE: You must NOT change the above code but you can add extra variables if necessary
 		
@@ -124,6 +145,8 @@ class FullyConnectedLayer:
 		# Returns: Activations after one forward pass through this relu layer | shape: batchSize x self.out_nodes
 		# This will only be called for layers with activation relu
 		###############################################
+		X[X<0] = 0
+		return X
 		# TASK 1b (Marks 1) - YOUR CODE HERE
 		raise NotImplementedError
 		###############################################
@@ -136,6 +159,9 @@ class FullyConnectedLayer:
 		# This will only be called for layers with activation relu amd during backwardpass
 		###############################################
 		# TASK 1e (Marks 1) - YOUR CODE HERE
+		X[X<0] = 0
+		X[X>0] = 1
+		return X*delta
 		raise NotImplementedError
 		###############################################
 
@@ -146,6 +172,8 @@ class FullyConnectedLayer:
 		# This will only be called for layers with activation softmax
 		###############################################
 		# TASK 1c (Marks 3) - YOUR CODE HERE
+		exps = np.exp(X)
+		return exps / np.sum(exps)
 		raise NotImplementedError
 		###############################################
 
@@ -158,9 +186,19 @@ class FullyConnectedLayer:
 		# Hint: You might need to compute Jacobian first
 		###############################################
 		# TASK 1f (Marks 7) - YOUR CODE HERE
-		raise NotImplementedError
+		nonZero = delta[delta!=0]
+		indexList = []
+		for i in range(len(delta)):
+			temp = list(delta[i])
+			indexList.append(temp.index(nonZero[i]))
+		for i in range(len(indexList)):
+			temp = X[i][indexList[i]]
+			X[i] = -temp*X[i]
+			X[i][indexList[i]] = temp*(1-temp)
+		return X
+		# raise NotImplementedError
 		###############################################
-
+		
 	def forwardpass(self, X):
 		# Input
 		# activations : Activations from previous layer/input | shape: batchSize x self.in_nodes
@@ -169,8 +207,12 @@ class FullyConnectedLayer:
 		###############################################
 		# TASK 1d (Marks 4) - YOUR CODE HERE
 		if self.activation == 'relu':
+			self.data = X @ self.weights + self.biases.T
+			return self.relu_of_X(self.data)
 			raise NotImplementedError
 		elif self.activation == 'softmax':
+			self.data = X @ self.weights + self.biases.T
+			return self.softmax_of_X(self.data)
 			raise NotImplementedError
 		else:
 			print("ERROR: Incorrect activation specified: " + self.activation)
@@ -188,6 +230,7 @@ class FullyConnectedLayer:
 		# Just compute and store the gradients here - do not make the actual updates
 		###############################################
 		# TASK 1g (Marks 6) - YOUR CODE HERE
+		# print(activation_prev.shape,delta.shape)
 		if self.activation == 'relu':
 			inp_delta = self.gradient_relu_of_X(self.data, delta)
 		elif self.activation == 'softmax':
@@ -195,6 +238,10 @@ class FullyConnectedLayer:
 		else:
 			print("ERROR: Incorrect activation specified: " + self.activation)
 			exit()
+		self.weightsGrad = (activation_prev.T @ inp_delta)/delta.shape[0]
+		self.biasesGrad = np.average(inp_delta,axis=0).reshape((delta.shape[1],-1))
+		new_delta = inp_delta @ self.weights.T
+		return new_delta
 		###############################################
 		
 	def updateWeights(self, lr):
@@ -204,6 +251,8 @@ class FullyConnectedLayer:
 		# This function should actually update the weights using the gradients computed in the backwardpass
 		###############################################
 		# TASK 1h (Marks 2) - YOUR CODE HERE
-		raise NotImplementedError
+		self.weights = self.weights - lr*(self.weightsGrad)
+		self.biases = self.biases - lr*(self.biasesGrad)
+    
+# 		raise NotImplementedError
 		###############################################
-		
\ No newline at end of file