## Documentation on How to run the code.
#
# To make the code run in your system, python and django should be installed to your system.
#
# To install django: 
#
#	$ sudo apt-get install django-python
#           
#      OR 
#
#    $ sudo apt install python3-pip
#   
#     AND
#
#     $ pip3 install django
#
## To install python:
#
#     $ sudo apt-get install python
#
# Create a directory named 'papers'
#
# start a project in terminal by using the following:
#
#      $ django -admin startproject research_project
#
# A manage.py file will be created.
#
# From paper folder to host the server:
#
#      $ python3 manage.py runserver
#
# The code is in views.py and templates are being created in 'template' folder which is in research_paper.
#
# Settings for the setting.py  and manage.py  file:
#
#        It has to be done as per specified in the setting.py file and as per requirement.
#
# Server is setup. Now you can easily access!!

##@mainpage
# @author Learners

 
 
import requests 
from bs4 import BeautifulSoup 
import os
import glob

from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
from collections import Counter 
import PyPDF2
import csv
from pattern.text.en import singularize, pluralize
import numpy as np

from io import StringIO
from wordcloud import WordCloud, STOPWORDS
from PIL import Image
import urllib
import requests
import matplotlib.pyplot as plt
import csv
import numpy as np
from django.shortcuts import render
from research_papers.forms import MyForm
from django.template import loader
from django.http import HttpResponse

##Documentation for wc() function.
# @brief This function opens a csv file inside it which contains a list of word along with its frequencies.
# @details It then saves a image of word cloud generated at specified location with different word size. This function is not used by us in current development but if image word cloud is needed, function code can be uncommented.
def wc():
	'''reader = csv.reader(open('counts.csv', 'r'))
	d = {}
	for k,v in reader:
		#insert the frequency values in d array 
		d[k] = float(v)

	#creates a dummy mask(shape) for wordcloud 
	#mask = np.array(Image.open("images.jpeg"))
	#word cloud corresponding to the word whose frequencies are in d array will be generated
	wordcloud = WordCloud(width=5000, height=6000,max_words=400,stopwords=STOPWORDS,
random_state=42).generate_from_frequencies(d)
	wordcloud.to_file('papers/static/image/images.png') #saves the wordcloud to image.png file'''

##Documentation for convertpdf function.
# @param path It takes the path of file which has to converted to string . the input file is of pdf type.
# @brief This function opens the pdf file inside it and convert the file into string
# @return str which is a string storing the whole file content.

def convert_pdf(path):
 
	rsrcmgr = PDFResourceManager()
	retstr = StringIO()
	codec = 'utf-8'
	laparams = LAParams()
	device = TextConverter(rsrcmgr, retstr,codec=codec,  laparams=laparams)
	fp = open(path, 'rb')
	interpreter = PDFPageInterpreter(rsrcmgr, device)
	password = ""
	maxpages = 0
	caching = True
	pagenos=set()
	for page in PDFPage.get_pages(fp, pagenos, maxpages=maxpages, password=password,caching=caching, check_extractable=True):
		interpreter.process_page(page)
	fp.close()
	device.close()
	str = retstr.getvalue()
	retstr.close()
	return str
	
##Documentation for extractpdf_pb function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. Pushpak Bhattacharya).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_pb(soup, add_url):
	#print(soup.prettify()) 
	table = soup.find('ul', attrs = {'class':'sub-list'})
	#print(table)
	for row in table.findAll('li'):
		if((row.get_text())=="Yearwise Papers"):
			#pdfurl['url'] = row.a['href']
			#pdfurl['type'] = row.a.text 
			url_significant= row.a['href']	
	url_significant=add_url + url_significant	
	print(url_significant)
	list_pdf=[]
	req_pdf= requests.get(url_significant)
	soup = BeautifulSoup(req_pdf.content, 'html5lib') 
	#print(soup.prettify()) 
	table = soup.find('ol')
	url_pdf=[]
	count=0
	c=0
	for row in table.findAll('li'):
		pdf={}
		if(row.a==None):
			c=c+1
			continue
		else: 
			pdf['url'] = add_url + row.a['href']
			pdf['name'] = row.a.get_text()
			url_pdf.append(pdf)
			count=count+1
	i=0
	'''for i in range(len(url_pdf)):
		print(url_pdf[i])
	(len(url_pdf))
	print(c)
	print(count)'''
	return url_pdf
	
##Documentation for extractpdf_varsha function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. Varsha Apte).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_varsha(soup, add_url):
	#print(soup.prettify())
	table = soup.find('div', attrs = {'id':'menu'}) #to go to research page
	#print(table)
	for row in table.findAll('li'):
		if((row.get_text())=="Research"):
			url_significant= row.a['href']
	url_significant= add_url + url_significant
	#print(url_significant)
	
	# to go to publications page
	req_pdf= requests.get(url_significant) 
	soup = BeautifulSoup(req_pdf.content, 'html5lib')
	#table = soup.find('div') 
	for row in soup.findAll('a'):
		if((row.get_text())=="Here"):
			url_significant= row.get('href')
	url_significant= add_url + url_significant
	#print(url_significant)
	
	#to get the list of pdf
	list_pdf=[]
	req_pdf= requests.get(url_significant)
	soup = BeautifulSoup(req_pdf.content, 'html5lib') 
	#print(soup.prettify()) 
	table = soup.find('ol', attrs = {'type':'1'})
	url_pdf=[]
	count=0
	c=0
	for row in table.findAll('li'):
		pdf={} #to store the url corresponding to name of pdf
		if(row.a==None or row.a.get_text()== " "):
			c=c+1
			continue
		else: 
			pdf['url'] = add_url + row.a['href']
			pdf['name'] = row.a.get_text()
			url_pdf.append(pdf)
			count=count+1
	i=0
	return url_pdf

	
##Documentation for extractpdf_varsha function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof Ganesh Ramakrishnan).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_ganesh(soup, add_url):
	#print(soup.prettify()) 
	table = soup.find('center')
	#print(table)
	for row in table.findAll('a'):
		if((row.b.get_text())=="Publications"):
			#pdfurl['url'] = row.a['href']
			#pdfurl['type'] = row.a.text 
			url_significant= row['href']	
	url_significant=add_url + url_significant	
	print(url_significant)
	list_pdf=[]
	req_pdf= requests.get(url_significant)
	soup = BeautifulSoup(req_pdf.content, 'html5lib') 
	#print(soup.prettify())
	#exit() 
	url_pdf=[]
	count=0
	c=0
	for row in soup.findAll('ol'):
		for x in row.findAll('li'):
			pdf={}
			#print(x.i)
			#print(x.i.a)
			if(x.a==None or ("Handbook" in x.a.get_text())):
				c=c+1
				continue
			else:
				if ("http" in x.a['href']):
					pdf['url']= x.a['href']
				else:
					pdf['url'] = add_url + x.a['href']
				pdf['name'] = x.a.get_text()
				url_pdf.append(pdf)
				count=count+1
	#for i in range(len(url_pdf)):
		#print(url_pdf[i])
	#(len(url_pdf))
	#print(c)
	#print(count)
	return url_pdf	

##Documentation for extractpdf_supratik function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. supratik Bhattacharya).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_supratik(soup, add_url):
	#print(soup.prettify()) 
	#table = soup.find('h3')
	#print(table)
	publications="\n"+" Publications"
	for row in soup.findAll('a'):
		if("Publications" in (row.get_text())):
			#pdfurl['url'] = row.a['href']
			#pdfurl['type'] = row.a.text 
			url_significant= row.get('href')	
	url_significant=add_url + url_significant	
	#print(url_significant)
	list_pdf=[]
	req_pdf= requests.get(url_significant)
	soup = BeautifulSoup(req_pdf.content, 'html5lib') 
	#print(soup.prettify()) 
	table = soup.find('ol')
	url_pdf=[]
	count=0
	c=0
	for row in table.findAll('li'):
		pdf={}
		if(row.a==None):
			c=c+1
			continue
		else: 
			pdf['url'] = add_url + row.a['href']
			pdf['name'] = row.a.get_text()
			url_pdf.append(pdf)
			count=count+1
	i=0
	#for i in range(len(url_pdf)):
		#print(url_pdf[i])
	#(len(url_pdf))
	#print(c)
	#print(count)
	return url_pdf	


##Documentation for extractpdf_mythili function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. Mythili Vutukuru).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_mythili(soup, add_url):
	table = soup.find('table', attrs = {'cellpadding':'10', 'cellspacing':'0','width':'100%'})
	url_pdf=[]
	count=0
	c=0
	for row in soup.findAll('ul'):
		for x in row.findAll('li'):
			pdf={}
			if(x.a==None or x.em==None):
				c=c+1
				continue
			else: 
				pdf['url'] = add_url + x.a['href']
				#print(pdf['url'])
				pdf['name'] = x.em.get_text()
				#print(pdf['name'])
				url_pdf.append(pdf)
				count=count+1
	i=0
	#for i in range(len(url_pdf)):
		#print(url_pdf[i])
	#(len(url_pdf))
	#print(c)
	#print(count)
	return url_pdf	
	
##Documentation for extractpdf_rohit function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. Rohit Gurjar).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_rohit(soup, add_url):
	url_pdf=[]
	count=0
	c=0
	for row in soup.findAll('ul'):
		for x in row.findAll('li'):
			pdf={}
			if(x.a==None or x.i==None):
				c=c+1
				continue
			else: 
				pdf['url'] = add_url + x.a['href']
				#print(pdf['url'])
				pdf['name'] = x.a.get_text()
				#print(pdf['name'])
				url_pdf.append(pdf)
				count=count+1
	i=0
	#for i in range(len(url_pdf)):
		#print(url_pdf[i])
	#(len(url_pdf))
	#print(c)
	#print(count)
	return url_pdf	
	
##Documentation for extractpdf_om function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. Om Damani).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.	
def extractpdf_om(soup, add_url):
	table = soup.find('ul')
	url_pdf=[]
	count=0
	c=0
	for row in table.findAll('li'):
			pdf={}
			if(row.a==None):
				c=c+1
				continue
			else: 
				if("http" in row.a['href']):
					continue
				else:
					pdf['url'] = add_url + row.a['href']
				#print(pdf['url'])
				pdf['name'] = row.a.get_text()
				#print(pdf['name'])
				url_pdf.append(pdf)
				count=count+1
	#for i in range(len(url_pdf)):
	#	print(url_pdf[i])
	#print(c, count)
	return url_pdf	
	
##Documentation for extractpdf_ajit function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param add_url It is the base url of the Professors page
# @brief This function process the content of html page to extract the name and links of the pdf of research paper publish by the faculty( Prof. Ajit Diwan).
# @details This function visit to the actual page where papers list has been displayed with the help of beautifulsoup and fetch the paper name along with the url to download the paper. The details for a single paper is stored in a dictionary with key as 'name' and 'url' and form a list of such dictionaries
# @return url_pdf it is the list of dictionary containing the paper name along with its URL.
def extractpdf_ajit(soup, add_url):
	table = soup.find('ol')
	url_pdf=[]
	count=0
	c=0
	for row in table.findAll('li'):
			pdf={}
			if(row.a==None):
				c=c+1
				continue
			else: 
				if("http" in row.a['href']):
					continue
				else:
					pdf['url'] = add_url + row.a['href']
				#print(pdf['url'])
				pdf['name'] = row.a.get_text()
				#print(pdf['name'])
				url_pdf.append(pdf)
				count=count+1
	#for i in range(len(url_pdf)):
	#	print(url_pdf[i])
	#print(c, count)
	return url_pdf	
	
##Documentation for extractpdf_ajit function.
# @param soup It contains the html page fetched by the request function which is converted to html5lib format using beautiful soup
# @param URL It is the base url of the Professors page
# @brief This function figures out the function to be called on the basis of given input URL.
# @details This function calls the extractpdf function as per professor URL which returns the list of pdf mapped to URLs. If any URL for which no function mapping is yet done is passed, it print error message.
# @return url_pdf it is the list of dictionary containing the paper name along with its URL	
def extractpdf_prof(soup, URL):
	list_pdf=[]
	if(URL=="https://www.cse.iitb.ac.in/~pb/"):
		list_pdf=extractpdf_pb(soup, URL)
	elif(URL=="https://www.cse.iitb.ac.in/~ganesh/"):
		list_pdf=extractpdf_ganesh(soup, URL)
	elif(URL=="https://www.cse.iitb.ac.in/~aad/"):
		list_pdf=extractpdf_ajit(soup, URL)
	elif(URL=="https://www.cse.iitb.ac.in/~damani/"):
		list_pdf=extractpdf_om(soup, URL)
	elif(URL=="https://www.cse.iitb.ac.in/~mythili/"):
		list_pdf=extractpdf_mythili(soup, URL)
	elif(URL=="https://www.cse.iitk.ac.in/users/rgurjar/"):
		list_pdf=extractpdf_rohit(soup, URL)
	elif(URL=="https://www.cse.iitb.ac.in/~varsha/"):
		list_pdf=extractpdf_varsha(soup, URL)
	else:
		print("You need to define function yet:)")
		exit()
	return list_pdf

##Documentation for savepdf function.
# @param list_pdf It contains the list of dictionary storing he pdf name mapped to its URL
# @brief This function will store all the pdf into system for further processing.
# @details It creates a directory named pdfstore where all the pdf are being stored as pdf0.pdf, pdf1.pdf and do on. 
# @return It return nothing. display a message saved when all pdf has been downloaded.
def savepdf(list_pdf):
	x=len(list_pdf)
	os.mkdir("pdfstore")
	for i in range(0,x):
		file_url=list_pdf[i]['url']
		r = requests.get(file_url, stream = True)
		filename= "pdf"+str(i)+".pdf"
		print(filename)
		path="pdfstore/"+filename  
		print(path)
		with open(path,"wb") as pdf: 
			for chunk in r.iter_content(chunk_size=1024): 
				if chunk: 
					pdf.write(chunk)
			print("saved")
			
##Documentation for research_paper function.
# @param request this function is passed with request which is used to get the requested posted on page
# @brief This function process the URL and return the processed words to be displayed inthe form of word cloud.
# @details On recieving the URL from User it calls the functions to extract the list of pdf links and download them using savpdf function. Afterwards, It will call convertpdf to convert pdf to string and then store it as text file to make word count and the whole procedure is repeated for all the files. then the top 50 occuring words are choosen to be displayed.
# @return  response It return the response to the page after processing the request in form of word cloud.		
def research_papers(request):
 #if form is submitted
	if request.method == 'POST':
		myForm = MyForm(request.POST)
		if myForm.is_valid():
			URL = myForm.cleaned_data['URL']
			#URL = "https://www.cse.iitb.ac.in/~pb/"
			#context={}
			req_home = requests.get(URL) 
			soup = BeautifulSoup(req_home.content, 'html5lib')
			list_pdf= extractpdf_prof(soup, URL)
			pdf_name_list1=[]
			pdf_name_list=[]
			pdf_url_list=[]
			for i in list_pdf:
				pdf_name_list1.append(i['name'])
				pdf_url_list.append(i['url'])
			for i in pdf_name_list1:
				z = i.replace(","," ")
				z2 = z.rstrip()
				pdf_name_list.append(z2)
			#print(pdf_name_list)	
			#print(pdf_url_list)
			savepdf(list_pdf) # to store pdf
			#pdflist = glob.glob("/home/monika/CS699/PROJECT/Learners_ResearchPapersEasyAccess/source/papers/pdfstore/*.pdf")
			pdflist = glob.glob("pdfstore/*.pdf")
			print(pdflist)
			wordcount = {}
			filemap={}
			for pdf in pdflist:
				print("Working on: " + pdf + '\n')
				xlist= list(pdf)
				ind= xlist[::-1].index("/")
				ind=len(xlist)-ind
				pdfname = xlist[ind +3:-4]
				pdfname = "".join(pdfname)
				pdfname=int(pdfname)	
				fout = open('pdfs.txt','w+',encoding="utf-8")
				try:
					PyPDF2.PdfFileReader(open(pdf,'rb'))
				except PyPDF2.utils.PdfReadError:
					continue
				else:
					fout.write(convert_pdf(pdf))
					fout.close()
					fin = open('pdfs.txt','r',encoding="utf-8")
					words = fin.read().lower()
#print(words)
					split_it = words.split() 
# split() returns list of all the words in the string 
					stopwords = set(line.strip() for line in open('stopwords.txt',encoding="utf-8"))
# Instantiate a dictionary, and for every word in the file, 
# Add to the dictionary if it doesn't exist. If it does, increase the count.

# To eliminate duplicates, remember to split by punctuation, and use case demiliters.
				for word in split_it:
					word = word.replace(".","")
					word = word.replace(",","")
					word = word.replace(":","")
					word = word.replace("\"","")
					word = word.replace("!","")
					word = word.replace("â€œ","")
					word = word.replace("â€˜","")
					word = word.replace("*","")
					word = word.replace("(","")
					word = word.replace(")","")
					word = word.replace("\'","")
					word = word.replace("-","")
		
					if word not in stopwords:
						if word in wordcount:
							wordcount[word] += 1
							if pdfname not in filemap[word]:
                                                		filemap[word].append(pdfname)
						elif singularize(word) in wordcount:
							wordcount[singularize(word)] += 1
							if pdfname not in filemap[singularize(word)]:
                                                		filemap[singularize(word)].append(pdfname)
						elif pluralize(word) in wordcount:
							wordcount[pluralize(word)] += 1
							if pdfname not in filemap[pluralize(word)]:
                                                		filemap[pluralize(word)].append(pdfname)		
						else:
							wordcount[word] = 1
							x=[]
							x.append(pdfname)
							filemap[word]=x
			print("done")   
			n_print = 50
			print(n_print)
			print("\nOK. The {} most common words are as follows\n".format(n_print))
			cnt = Counter(wordcount)
#print(cnt)
# most_common() produces k frequently encountered 
# input values and their respective counts. 
			most_occur = cnt.most_common(n_print) 
# with open("counts.csv", 'w+',encoding="utf-8") as file:
# 		writer = csv.writer(file)
# 		#writer.writerow(["word", "count"])

# 		for i in enumerate(most_occur):
# 			writer.writerow([i[1][0],i[1][1]])		
			lst = list()
			for i in enumerate(most_occur):
				tmp2=list()
				tmp2.append(i[1][0])
				tmp2.append(i[1][1])
				lst.append(tmp2)
			np.savetxt("counts.csv",lst, delimiter=',',fmt='%s',encoding="utf-8")	
			#print(most_occur)
			#print(most_occur)
			#print(filemap)
			most_occur_file_map={}
			most_occur_file_map_len={}
			most_occur_dict={}
			for i in enumerate(most_occur):
        			most_occur_dict[i[1][0]]= i[1][1]
        			most_occur_file_map[i[1][0]] = filemap[i[1][0]]
        			most_occur_file_map_len[i[1][0]] = len(filemap[i[1][0]])

			#print(most_occur_dict)
			#most_occur_dict= { 'a':1, 'b':2}
			word_list_display=[]
			word_count_display=[]
			for i in most_occur_dict:
				word_list_display.append(i)
				word_count_display.append(most_occur_dict[i])
			#print(most_occur_file_map)
			#print(word_list_display)
			#print(word_count_display)
			#wc()
			x  = len(word_list_display)
			y = len(pdf_name_list)
			
			context= {'URL': URL, 'word_list_display': word_list_display,'word_count_display': word_count_display, 'x' : x, 'most_occur_file_map':most_occur_file_map, 'pdf_name_list':pdf_name_list, 'pdf_url_list':pdf_url_list , 'most_occur_file_map_len': most_occur_file_map_len, 'y':y }
			
			template = loader.get_template('thankyou.html')
			
			#URL = "https://www.cse.iitb.ac.in/~pb/"
			return HttpResponse(template.render(context, request))
	    
	else:
		form = MyForm()

	return render(request, 'research_papers.html', {'form':form});

