Skip to content

P
POS_taging_using_HMM
Commit 1bd1b8b1 authored by Kunal's avatar Kunal
Browse files
Options

Removed redundant testing code blocks

parent 96ee918b
Hide whitespace changes
Inline Side-by-side
Showing with 18 additions and 81 deletions
POS tagging using HMM.ipynb
View file @ 1bd1b8b1
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 28, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
...@@ -36,34 +36,6 @@ ...@@ -36,34 +36,6 @@
"Brown_Dataset_sentences = brown.tagged_sents(tagset='universal')" "Brown_Dataset_sentences = brown.tagged_sents(tagset='universal')"
] ]
}, },
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"words_with_tags = []\n",
"for sent in TrainingSet:\n",
" for tup in sent:\n",
" words_with_tags.append(tup)\n",
"\n",
"word_given_tag_dict = {}\n",
"TagCounter = Counter(tag for word,tag in words_with_tags)\n",
"for sent in TrainingSet:\n",
" for word,tag in sent:\n",
" try:\n",
" word_given_tag_dict[(word.lower(),tag)]+=1\n",
" except:\n",
" word_given_tag_dict[(word.lower(),tag)]=1\n",
" \n",
"uniquetags = list({tup[1] for tup in words_with_tags})\n",
"transition_matrix = [[0]*len(uniquetags) for _ in range(len(uniquetags))]\n",
"for i in range(len(uniquetags)):\n",
" for j in range(len(uniquetags)):\n",
" transition_matrix[i][j] = prob_tag2_given_tag1(uniquetags[j],uniquetags[i],words_with_tags)\n",
"transition_matrix_df = pd.DataFrame(transition_matrix,columns = uniquetags, index=uniquetags)"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 15,
...@@ -86,7 +58,7 @@ ...@@ -86,7 +58,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 54, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
...@@ -100,7 +72,7 @@ ...@@ -100,7 +72,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 55, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
...@@ -117,7 +89,7 @@ ...@@ -117,7 +89,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 56, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
...@@ -148,7 +120,7 @@ ...@@ -148,7 +120,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 76, "execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
...@@ -188,54 +160,19 @@ ...@@ -188,54 +160,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 73, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "ename": "NameError",
"text/plain": [ "evalue": "name 'TestSet' is not defined",
"(0.9166767664525513,\n", "output_type": "error",
" array([[12982, 0, 7, 230, 0, 0, 0, 0, 3,\n", "traceback": [
" 0, 0, 561],\n", "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
" [ 124, 15, 1, 1, 0, 6, 3, 0, 0,\n", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
" 0, 31, 0],\n", "\u001b[0;32m<ipython-input-7-cbd1e5a0ff04>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mCheckAccuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTestSet\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtransition_matrix_df\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muniquetags\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mword_given_tag_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mTagCounter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
" [ 131, 0, 5318, 881, 0, 17, 38, 0, 84,\n", "\u001b[0;31mNameError\u001b[0m: name 'TestSet' is not defined"
" 0, 42, 2],\n", ]
" [ 16, 0, 1349, 16628, 0, 46, 32, 0, 309,\n",
" 23, 6, 21],\n",
" [ 14, 0, 0, 0, 1058, 0, 0, 0, 0,\n",
" 0, 0, 0],\n",
" [ 907, 0, 2, 41, 0, 30476, 90, 0, 38,\n",
" 0, 734, 0],\n",
" [ 724, 1, 10, 8, 0, 73, 8441, 0, 366,\n",
" 0, 228, 0],\n",
" [ 0, 0, 0, 0, 0, 0, 0, 29080, 0,\n",
" 0, 0, 0],\n",
" [ 210, 0, 294, 404, 0, 12, 532, 0, 8654,\n",
" 61, 33, 99],\n",
" [ 8, 0, 0, 0, 0, 0, 0, 0, 1,\n",
" 5946, 0, 19],\n",
" [ 4301, 9, 17, 15, 152, 768, 522, 0, 30,\n",
" 2, 28207, 2],\n",
" [ 23, 0, 0, 388, 2, 0, 0, 0, 14,\n",
" 7, 0, 19592]]),\n",
" {'PRON': 0.9418849307117464,\n",
" 'X': 0.08287292817679558,\n",
" 'PRT': 0.8165208045447566,\n",
" 'ADP': 0.9022246337493217,\n",
" 'NUM': 0.9869402985074627,\n",
" 'VERB': 0.9438800792864221,\n",
" 'ADJ': 0.8568673231144046,\n",
" '.': 1.0,\n",
" 'ADV': 0.8402757549276629,\n",
" 'CONJ': 0.9953130231001004,\n",
" 'NOUN': 0.8290080822924321,\n",
" 'DET': 0.978328173374613})"
]
},
"execution_count": 73,
"metadata": {},
"output_type": "execute_result"
} }
], ],
"source": [ "source": [
...@@ -251,7 +188,7 @@ ...@@ -251,7 +188,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 91, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
...@@ -262,7 +199,7 @@ ...@@ -262,7 +199,7 @@
"Training started\n", "Training started\n",
"Training Complete\n", "Training Complete\n",
"Calculating accuracy over sentences in TestSet\n", "Calculating accuracy over sentences in TestSet\n",
"Accuracy in 1 epoch is 0.9110812851270866\n", "Accuracy in 1 epoch is 0.9109215030698128\n",
"############################### EPOCH NUMBER- 2 #############################\n", "############################### EPOCH NUMBER- 2 #############################\n",
"Training started\n", "Training started\n",
"Training Complete\n", "Training Complete\n",
...@@ -282,7 +219,7 @@ ...@@ -282,7 +219,7 @@
"Training started\n", "Training started\n",
"Training Complete\n", "Training Complete\n",
"Calculating accuracy over sentences in TestSet\n", "Calculating accuracy over sentences in TestSet\n",
"Accuracy in 5 epoch is 0.9166767664525513\n" "Accuracy in 5 epoch is 0.9174590407774265\n"
] ]
} }
], ],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment