added assignment2 ML fodler

Assignment1/data/plot1/plot.py

@@ -13,7 +13,7 @@ with open('logcosh.log','r') as csvfile:
 
 
 
-plt.plot(x,y, label='LOGCOSH')
+plt.plot(x,y,'--', label='LOGCOSH')
 plt.xlabel('epoch')
 plt.ylabel('mean_squared_loss')
 

Assignment1/main.py

-import numpy as np
-import argparse
-import csv
-import warnings
-
-'''
-Commented portion may not help much in 
-optimization but will help in visualization !!
-'''
-
-def mean_squared_loss(xdata, ydata, weights):
-
-	guess = np.dot(xdata,weights)
-	samples = np.shape(guess)[0]
-	err = (1/samples)*np.sum(np.square(ydata-guess))
-	return err
-	raise NotImplementedError
-
-def mean_squared_gradient(xdata, ydata, weights):
-
-	samples = np.shape(xdata)[0]
-	guess = np.dot(xdata,weights)
-	gradient = (2/samples)*np.dot(xdata.T,(guess-ydata))
-	return gradient
-
-	raise NotImplementedError
-
-def mean_absolute_loss(xdata, ydata, weights):
-
-	guess = np.dot(xdata,weights)
-	samples = np.shape(guess)[0]
-	err = (1/samples)*np.sum(np.absolute(ydata-guess))
-	return err
-	raise NotImplementedError
-
-def mean_absolute_gradient(xdata, ydata, weights):
-
-	guess = np.dot(xdata,weights)
-	samples = np.shape(guess)[0]
-	signInfo = np.sign(guess-ydata)
-	gradient = np.dot(xdata.T,signInfo)/samples
-	return gradient
-	raise NotImplementedError
-
-def mean_log_cosh_loss(xdata, ydata, weights):
-
-	guess = np.dot(xdata,weights)
-	samples = np.shape(guess)[0]
-	warnings.filterwarnings("error")
-	try:
-		err = np.sum(np.log(np.cosh(guess-ydata)))/samples
-	except Exception as e:
-		err = np.sum(np.absolute(guess-ydata)+np.log(2))/samples
-	return err
-	raise NotImplementedError
-
-def mean_log_cosh_gradient(xdata, ydata, weights):
-
-	guess = np.dot(xdata,weights)
-	samples = np.shape(guess)[0]
-	gradient = np.dot(xdata.T,np.tanh(guess-ydata))/samples
-	return gradient
-
-	raise NotImplementedError
-
-def root_mean_squared_loss(xdata, ydata, weights):
-
-	guess = np.dot(xdata,weights)
-	samples = np.shape(guess)[0]
-	err = np.sqrt(np.divide(np.sum(np.square(ydata.T-guess)),samples))
-	return err
-	raise NotImplementedError
-
-def root_mean_squared_gradient(xdata, ydata, weights):
-
-	samples = np.shape(xdata)[0]
-	guess = np.dot(xdata,weights)
-	gradient = mean_squared_gradient(xdata, ydata, weights)/(2*root_mean_squared_loss(xdata, ydata, weights))
-	return gradient
-
-	raise NotImplementedError
-
-class LinearRegressor:
-
-	def __init__(self, dims):
-		
-		self.dims = dims
-		self.W = np.random.rand(dims)
-		#self.W = np.random.uniform(low=0.0, high=1.0, size=dims)
-		return
-
-		raise NotImplementedError
-
-	def train(self, xtrain, ytrain, loss_function, gradient_function, epoch=100, lr=1):
-		errlog = []
-		samples = np.shape(xtrain)[0]
-		for iterations in range(epoch):
-			self.W = self.W - lr*gradient_function(xtrain,ytrain,self.W)
-			errlog.append(loss_function(xtrain,ytrain,self.W))
-			# errlog.append(mean_squared_loss(xtrain,ytrain,self.W))
-		return errlog
-		raise NotImplementedError
-
-	def predict(self, xtest):
-		
-		return np.dot(xtest,self.W)
-		raise NotImplementedError
-
-
-def read_dataset(trainfile, testfile):
-	
-	xtrain = []
-	ytrain = []
-	xtest = []
-
-	with open(trainfile,'r') as f:
-		reader = csv.reader(f,delimiter=',')
-		next(reader, None)
-		for row in reader:
-			xtrain.append(row[:-1])
-			ytrain.append(row[-1])
-
-	with open(testfile,'r') as f:
-		reader = csv.reader(f,delimiter=',')
-		next(reader, None)
-		for row in reader:
-			xtest.append(row)
-
-	return np.array(xtrain), np.array(ytrain), np.array(xtest)
-
-def one_hot_encoding(value_list, classes):
-    res = np.eye(classes)[value_list.reshape(-1)]
-    return res.reshape(list(value_list.shape)+[classes])
-
-norm_dict = {}
-
-dictionary_of_classes_for_features = {
-	2 : 5,
-	3 : 25,
-	5: 8,
-	7: 5
-}
-
-dictionary_of_days = {
-	'Monday' : 1,
-	'Tuesday': 2,
-	'Wednesday': 3,
-	'Thursday' : 4,
-	'Friday' : 5,
-	'Saturday': 6,
-	'Sunday' : 7
-}
-
-def slicer(arr, beg, end):
-	return np.array([i[beg:end] for i in arr]).reshape(-1, 1)
-"""	
-#for normalization of parametes 'wind speed' and 'humidity' uncoment
-def normalize(arr):
-	arr = arr
-	if not norm_dict: # make dictionary once at training to be used later during test
-		# for i in range(arr.shape[1]):
-		norm_dict['init'] = [np.min(arr), np.max(arr)]
-		#norm_dict['init'] = [np.mean(arr), np.std(arr)]
-	# for i in range(arr.shape[1]):
-	arr = np.array([(x - norm_dict['init'][0])/(norm_dict['init'][1] - norm_dict['init'][0]) for x in arr]) # min-max
-	#arr = np.array([(x - norm_dict['init'][0])/(norm_dict['init'][1]) for x in arr]) # standardization
-		
-	return arr
-"""
-
-def preprocess_dataset(xdata, ydata=None):
-	
-	# converting weekdays to numeric for one_hot_encoding
-    """
-
-	#for normalization of parametes 'wind speed' and 'humidity' uncoment
-	xdata[:, 10] = normalize(xdata[:, 10].astype('float'))# normalized
-	xdata[:, 11] = normalize(xdata[:, 10].astype('float'))"""
-    xdata[:, 5] = [dictionary_of_days[i] for i in xdata[:, 5]]
-
-    cat_cols = [2, 3, 5, 7]
-
-	
-    for i in cat_cols:
-		# dropping 2 columns for C-1 encoding and removing additional 0 column
-        t = one_hot_encoding(xdata[:, i].astype('int'), dictionary_of_classes_for_features[i])[:, 2:]
-        xdata = np.concatenate((xdata, t),axis=1)
-	
-    xdata = np.delete(xdata, cat_cols, 1) # removing useless columns
-    xdata = np.delete(xdata, 6, 1)
-    xdata = np.delete(xdata, 8, 1)
-	
-    # extracting features from date
-    month = slicer(xdata[:, 1], 5,7)
-    t = one_hot_encoding(month[:,0].astype('int'), 13)[:, 2:]
-    xdata = np.concatenate((xdata, t), axis=1)
-    date = slicer(xdata[:, 1], 8, 10)
-    week = np.ceil(date.astype('int') / 7)  # week of month
-    t = one_hot_encoding(week[:,0].astype('int'), 6)[:, 2:]
-    xdata = np.concatenate((xdata, t), axis=1)
-
-
-    xdata = xdata[:,2:] # dropping first 2 unnecessary columns
-	
-    xdata = xdata.astype('float32')
-    bias = np.ones((np.shape(xdata)[0],1)) # adding Bias in feature Matrix
-    xdata = np.concatenate((bias,xdata),axis=1)
-
-    if ydata is None:
-        return xdata
-    ydata = ydata.astype('float32')
-    return xdata,ydata
-    raise NotImplementedError
-
-dictionary_of_losses = {
-	'mse':(mean_squared_loss, mean_squared_gradient),
-	'mae':(mean_absolute_loss, mean_absolute_gradient),
-	'rmse':(root_mean_squared_loss, root_mean_squared_gradient),
-	'logcosh':(mean_log_cosh_loss, mean_log_cosh_gradient),
-}
-
-"""
-#For outliers removal from wind speed column uncomment
-def out(x, std, mean):
-    if ((x < mean + 2 * std)and (x > mean - 2 * std)):
-        return 0
-    else:
-        return 1
-
-
-def outlier(xtrain, ytrain, std, mean):
-    a =[]
-    for i in xtrain[:, 11].astype('float32'):
-        a.append(out(i,std, mean))
-    a = np.array(a)
-    xdata = np.concatenate((xtrain, a.reshape(-1, 1)), axis=1)
-    ytrain = np.delete(ytrain, np.argwhere(xdata[:, -1].astype('int') > 0), 0)
-    xdata = np.delete(xdata, np.argwhere(xdata[:, -1].astype('int') > 0), 0)
-    xdata = np.delete(xdata, -1, 1)
-    return (xdata, ytrain)"""
-
-def main():
-    # You are free to modify the main function as per your requirements.
-	# Uncomment the below lines and pass the appropriate value
-
-    xtrain, ytrain, xtest = read_dataset(args.train_file, args.test_file)
-
-    """
-    #For outliers removal from wind speed column uncomment
-    std = np.std(xtrain[:, 11].astype('float32'))
-    mean = np.mean(xtrain[:, 11].astype('float32'))
-    xtrain, ytrain =outlier(xtrain, ytrain, std, mean)"""
-    xtrainprocessed, ytrainprocessed = preprocess_dataset(xtrain, ytrain)
-    xtestprocessed = preprocess_dataset(xtest)
-	
-    model = LinearRegressor(np.shape(xtrainprocessed)[1])
-
-    # The loss function is provided by command line argument
-    loss_fn, loss_grad = dictionary_of_losses[args.loss]
-
-    errlog = model.train(xtrainprocessed, ytrainprocessed, loss_fn, loss_grad, args.epoch, args.lr)
-    ytest = model.predict(xtestprocessed)
-    ytest = ytest.astype('int')
-    ytest[ytest<0] = 0
-    print(ytest)
-    output = [(i,ytest[i]) for i in range(len(ytest))]
-    np.savetxt("prediction.csv",output,delimiter=',',fmt="%d",header="instance (id),count",comments='')
-    #np.savetxt("error.log",errlog,delimiter='\n',fmt="%f")
-
-if __name__ == '__main__':
-
-	parser = argparse.ArgumentParser()
-
-	parser.add_argument('--loss', default='mse', choices=['mse','mae','rmse','logcosh'], help='loss function')
-	parser.add_argument('--lr', default=1.0, type=float, help='learning rate')
-	parser.add_argument('--epoch', default=100, type=int, help='number of epochs')
-	parser.add_argument('--train_file', type=str, help='location of the training file')
-	parser.add_argument('--test_file', type=str, help='location of the test file')
-
-	args = parser.parse_args()
-
-	main()

Assignment2/main.py

+import numpy as np
+import nn
+import csv
+import pickle
+
+def taskXor():
+	XTrain, YTrain, XVal, YVal, XTest, YTest = loadXor()
+	# Create a NeuralNetwork object 'nn1' as follows with optimal parameters. For parameter definition, refer to nn.py file.
+	# nn1 = nn.NeuralNetwork(lr, batchSize, epochs)
+	# Add layers to neural network corresponding to inputs and outputs of given data
+	# Eg. nn1.addLayer(FullyConnectedLayer(x,y))
+	###############################################
+	# TASK 3a (Marks 7) - YOUR CODE HERE
+	raise NotImplementedError
+	###############################################
+	nn1.train(XTrain, YTrain, XVal, YVal)
+	pred, acc = nn1.validate(XTest, YTest)
+	with open("predictionsXor.csv", 'w') as file:
+		writer = csv.writer(file)
+		writer.writerow(["id", "prediction"])
+		for i, p in enumerate(pred):
+			writer.writerow([i, p])
+	print('Test Accuracy',acc)
+	return nn1
+
+def preprocessMnist(X):
+	# Perform any data preprocessing that you wish to do here
+	# Input: A 2-d numpy array containing an entire train, val or test split | Shape: n x 28*28
+	# Output: A 2-d numpy array of the same shape as the input (If the size is changed, you will get downstream errors)
+	###############################################
+	# TASK 3c (Marks 0) - YOUR CODE HERE
+	raise NotImplementedError
+	###############################################
+
+
+def taskMnist():
+	XTrain, YTrain, XVal, YVal, XTest, _ = loadMnist()
+	# Create a NeuralNetwork object 'nn1' as follows with optimal parameters. For parameter definition, refer to nn.py file.
+	# nn1 = nn.NeuralNetwork(lr, batchSize, epochs)
+	# Add layers to neural network corresponding to inputs and outputs of given data
+	# Eg. nn1.addLayer(FullyConnectedLayer(x,y))
+	###############################################
+	# TASK 3b (Marks 13) - YOUR CODE HERE
+	raise NotImplementedError
+	###############################################
+	nn1.train(XTrain, YTrain, XVal, YVal)
+	pred, _ = nn1.validate(XTest, None)
+	with open("predictionsMnist.csv", 'w') as file:
+		writer = csv.writer(file)
+		writer.writerow(["id", "prediction"])
+		for i, p in enumerate(pred):
+			writer.writerow([i, p])
+	return nn1
+
+################################# UTILITY FUNCTIONS ############################################
+def oneHotEncodeY(Y, nb_classes):
+	# Calculates one-hot encoding for a given list of labels
+	# Input :- Y : An integer or a list of labels
+	# Output :- Coreesponding one hot encoded vector or the list of one-hot encoded vectors
+	return (np.eye(nb_classes)[Y]).astype(int)
+
+def loadXor():
+	# This is a toy dataset with 10k points and 2 labels.
+	# The output can represented as the XOR of the input as described in the problem statement
+	# There are 7k training points, 1k validation points and 2k test points
+	train = pickle.load(open("data/xor/train.pkl", 'rb'))
+	test = pickle.load(open("data/xor/test.pkl", 'rb'))
+	testX, testY = np.array(test[0]), np.array(oneHotEncodeY(test[1],2))
+	trainX, trainY = np.array(train[0][:7000]), np.array(oneHotEncodeY(train[1][:7000],2))
+	valX, valY = np.array(train[0][7000:]), np.array(oneHotEncodeY(train[1][7000:],2))
+
+	return trainX, trainY, valX, valY, testX, testY
+
+def loadMnist():
+	# MNIST dataset has 50k train, 10k val, 10k test
+	# The test labels have not been provided for this task
+	train = pickle.load(open("data/mnist/train.pkl", 'rb'))
+	test = pickle.load(open("data/mnist/test.pkl", 'rb'))
+	testX = preprocessMnist(np.array(test[0]))
+	testY = None # For MNIST the test labels have not been provided
+	trainX, trainY = preprocessMnist(np.array(train[0][:50000])), np.array(oneHotEncodeY(train[1][:50000],10))
+	valX, valY = preprocessMnist(np.array(train[0][50000:])), np.array(oneHotEncodeY(train[1][50000:],10))
+
+	return trainX, trainY, valX, valY, testX, testY
+#################################################################################################
+
+if __name__ == "__main__":
+	np.random.seed(7)
+	taskXor()
+	taskMnist()

Assignment2/nn.py

+import numpy as np
+
+class NeuralNetwork:
+
+	def __init__(self, lr, batchSize, epochs):
+		# Method to initialize a Neural Network Object
+		# Parameters
+		# lr - learning rate
+		# batchSize - Mini batch size
+		# epochs - Number of epochs for training
+		self.lr = lr
+		self.batchSize = batchSize
+		self.epochs = epochs
+		self.layers = []
+
+	def addLayer(self, layer):
+		# Method to add layers to the Neural Network
+		self.layers.append(layer)
+
+	def train(self, trainX, trainY, validX=None, validY=None):
+		# Method for training the Neural Network
+		# Input
+		# trainX - A list of training input data to the neural network
+		# trainY - Corresponding list of training data labels
+		# validX - A list of validation input data to the neural network
+		# validY - Corresponding list of validation data labels
+		
+		# The methods trains the weights and baises using the training data(trainX, trainY)
+		# Feel free to print accuracy at different points using the validate() or computerAccuracy() functions of this class
+		###############################################
+		# TASK 2c (Marks 0) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+		
+	def crossEntropyLoss(self, Y, predictions):
+		# Input 
+		# Y : Ground truth labels (encoded as 1-hot vectors) | shape = batchSize x number of output labels
+		# predictions : Predictions of the model | shape = batchSize x number of output labels
+		# Returns the cross-entropy loss between the predictions and the ground truth labels | shape = scalar
+		###############################################
+		# TASK 2a (Marks 3) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+
+	def crossEntropyDelta(self, Y, predictions):
+		# Input 
+		# Y : Ground truth labels (encoded as 1-hot vectors) | shape = batchSize x number of output labels
+		# predictions : Predictions of the model | shape = batchSize x number of output labels
+		# Returns the derivative of the loss with respect to the last layer outputs, ie dL/dp_i where p_i is the ith 
+		#		output of the last layer of the network | shape = batchSize x number of output labels
+		###############################################
+		# TASK 2b (Marks 3) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+		
+	def computeAccuracy(self, Y, predictions):
+		# Returns the accuracy given the true labels Y and final output of the model
+		correct = 0
+		for i in range(len(Y)):
+			if np.argmax(Y[i]) == np.argmax(predictions[i]):
+				correct += 1
+		accuracy = (float(correct) / len(Y)) * 100
+		return accuracy
+
+	def validate(self, validX, validY):
+		# Input 
+		# validX : Validation Input Data
+		# validY : Validation Labels
+		# Returns the predictions and validation accuracy evaluated over the current neural network model
+		valActivations = self.predict(validX)
+		pred = np.argmax(valActivations, axis=1)
+		if validY is not None:
+			valAcc = self.computeAccuracy(validY, valActivations)
+			return pred, valAcc
+		else:
+			return pred, None
+
+	def predict(self, X):
+		# Input
+		# X : Current Batch of Input Data as an nparray
+		# Output
+		# Returns the predictions made by the model (which are the activations output by the last layer)
+		# Note: Activations at the first layer(input layer) is X itself		
+		activations = X
+		for l in self.layers:
+			activations = l.forwardpass(activations)
+		return activations
+
+
+
+
+
+
+
+class FullyConnectedLayer:
+	def __init__(self, in_nodes, out_nodes, activation):
+		# Method to initialize a Fully Connected Layer
+		# Parameters
+		# in_nodes - number of input nodes of this layer
+		# out_nodes - number of output nodes of this layer
+		self.in_nodes = in_nodes
+		self.out_nodes = out_nodes
+		self.activation = activation
+		# Stores a quantity that is computed in the forward pass but actually used in the backward pass. Try to identify
+		# this quantity to avoid recomputing it in the backward pass and hence, speed up computation
+		self.data = None
+
+		# Create np arrays of appropriate sizes for weights and biases and initialise them as you see fit
+		###############################################
+		# TASK 1a (Marks 0) - YOUR CODE HERE
+		raise NotImplementedError
+		self.weights = None
+		self.biases = None
+		###############################################
+		# NOTE: You must NOT change the above code but you can add extra variables if necessary
+		
+		# Store the gradients with respect to the weights and biases in these variables during the backward pass
+		self.weightsGrad = None
+		self.biasesGrad = None
+
+	def relu_of_X(self, X):
+		# Input
+		# data : Output from current layer/input for Activation | shape: batchSize x self.out_nodes
+		# Returns: Activations after one forward pass through this relu layer | shape: batchSize x self.out_nodes
+		# This will only be called for layers with activation relu
+		###############################################
+		# TASK 1b (Marks 1) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+
+	def gradient_relu_of_X(self, X, delta):
+		# Input
+		# data : Output from next layer/input | shape: batchSize x self.out_nodes
+		# delta : del_Error/ del_activation_curr | shape: batchSize x self.out_nodes
+		# Returns: Current del_Error to pass to current layer in backward pass through relu layer | shape: batchSize x self.out_nodes
+		# This will only be called for layers with activation relu amd during backwardpass
+		###############################################
+		# TASK 1e (Marks 1) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+
+	def softmax_of_X(self, X):
+		# Input
+		# data : Output from current layer/input for Activation | shape: batchSize x self.out_nodes
+		# Returns: Activations after one forward pass through this softmax layer | shape: batchSize x self.out_nodes
+		# This will only be called for layers with activation softmax
+		###############################################
+		# TASK 1c (Marks 3) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+
+	def gradient_softmax_of_X(self, X, delta):
+		# Input
+		# data : Output from next layer/input | shape: batchSize x self.out_nodes
+		# delta : del_Error/ del_activation_curr | shape: batchSize x self.out_nodes
+		# Returns: Current del_Error to pass to current layer in backward pass through softmax layer | shape: batchSize x self.out_nodes
+		# This will only be called for layers with activation softmax amd during backwardpass
+		# Hint: You might need to compute Jacobian first
+		###############################################
+		# TASK 1f (Marks 7) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+
+	def forwardpass(self, X):
+		# Input
+		# activations : Activations from previous layer/input | shape: batchSize x self.in_nodes
+		# Returns: Activations after one forward pass through this layer | shape: batchSize x self.out_nodes
+		# You may need to write different code for different activation layers
+		###############################################
+		# TASK 1d (Marks 4) - YOUR CODE HERE
+		if self.activation == 'relu':
+			raise NotImplementedError
+		elif self.activation == 'softmax':
+			raise NotImplementedError
+		else:
+			print("ERROR: Incorrect activation specified: " + self.activation)
+			exit()
+		###############################################
+
+	def backwardpass(self, activation_prev, delta):
+		# Input
+		# activation_prev : Output from next layer/input | shape: batchSize x self.out_nodes]
+		# delta : del_Error/ del_activation_curr | shape: self.out_nodes
+		# Output
+		# new_delta : del_Error/ del_activation_prev | shape: self.in_nodes
+		# You may need to write different code for different activation layers
+
+		# Just compute and store the gradients here - do not make the actual updates
+		###############################################
+		# TASK 1g (Marks 6) - YOUR CODE HERE
+		if self.activation == 'relu':
+			inp_delta = self.gradient_relu_of_X(self.data, delta)
+		elif self.activation == 'softmax':
+			inp_delta = self.gradient_softmax_of_X(self.data, delta)
+		else:
+			print("ERROR: Incorrect activation specified: " + self.activation)
+			exit()
+		###############################################
+		
+	def updateWeights(self, lr):
+		# Input
+		# lr: Learning rate being used
+		# Output: None
+		# This function should actually update the weights using the gradients computed in the backwardpass
+		###############################################
+		# TASK 1h (Marks 2) - YOUR CODE HERE
+		raise NotImplementedError
+		###############################################
+		
\ No newline at end of file

cs725-2019a/sampleSubmission.csv

-instance (id),count
-0,8
-1,466
-2,176
-3,58
-4,288
-5,795
-6,1
-7,292
-8,427
-9,73
-10,216
-11,11
-12,171
-13,238