From 08411cb176ae365c097e6eedd54d802b0a0d2f64 Mon Sep 17 00:00:00 2001
From: Siddharth Aravindan <siddharth.aravindan@gmail.com>
Date: Fri, 11 Nov 2016 17:02:38 +0530
Subject: [PATCH] Refactored SARSA libraries, added hand coded agent
---
example/hand_coded_defense_agent | Bin 0 -> 37304 bytes
example/hand_coded_defense_agent.cpp | 257 ++++++++++++++++++
...ake_hand_coded_defense_agent_executable.sh | 3 +
example/sarsa/Makefile | 58 ----
example/sarsa/funcapprox/Makefile | 28 --
example/sarsa/policy/Makefile | 34 ---
.../high_level_sarsa_defense_agent.cpp | 257 ++++++++++++++++++
.../funcapprox/CMAC.cpp | 0
.../funcapprox/CMAC.h | 0
.../funcapprox/FuncApprox.cpp | 0
.../funcapprox/FuncApprox.h | 0
.../funcapprox/tiles2.cpp | 0
.../funcapprox/tiles2.h | 0
.../policy/PolicyAgent.cpp | 0
.../policy/PolicyAgent.h | 0
.../policy/SarsaAgent.cpp | 13 +-
.../policy/SarsaAgent.h | 3 +-
.../high_level_sarsa_agent.cpp | 6 +-
18 files changed, 529 insertions(+), 130 deletions(-)
create mode 100755 example/hand_coded_defense_agent
create mode 100644 example/hand_coded_defense_agent.cpp
create mode 100755 example/make_hand_coded_defense_agent_executable.sh
delete mode 100644 example/sarsa/Makefile
delete mode 100644 example/sarsa/funcapprox/Makefile
delete mode 100644 example/sarsa/policy/Makefile
create mode 100644 example/sarsa_defense/high_level_sarsa_defense_agent.cpp
rename example/{sarsa => sarsa_libraries}/funcapprox/CMAC.cpp (100%)
rename example/{sarsa => sarsa_libraries}/funcapprox/CMAC.h (100%)
rename example/{sarsa => sarsa_libraries}/funcapprox/FuncApprox.cpp (100%)
rename example/{sarsa => sarsa_libraries}/funcapprox/FuncApprox.h (100%)
rename example/{sarsa => sarsa_libraries}/funcapprox/tiles2.cpp (100%)
rename example/{sarsa => sarsa_libraries}/funcapprox/tiles2.h (100%)
rename example/{sarsa => sarsa_libraries}/policy/PolicyAgent.cpp (100%)
rename example/{sarsa => sarsa_libraries}/policy/PolicyAgent.h (100%)
rename example/{sarsa => sarsa_libraries}/policy/SarsaAgent.cpp (84%)
rename example/{sarsa => sarsa_libraries}/policy/SarsaAgent.h (81%)
rename example/{sarsa => sarsa_offense}/high_level_sarsa_agent.cpp (97%)
diff --git a/example/hand_coded_defense_agent b/example/hand_coded_defense_agent
new file mode 100755
index 0000000000000000000000000000000000000000..376dafcf2a52924e5ddec7fc5f64bdbeca7e47d8
GIT binary patch
literal 37304
zcmeHwdwf*Ywf~-(41@s5pr{mlj3{1E2?+!YYGo2KFo6Uk2{b-%GMP-sNRr8&Oaf?W
z;|)=!F*UWKXlt83?9bcQmexn9y@v3pplvl;wP>|XZL2e4Eh-nI74!S9{g~Njm;(my
z?|y!Ncp$URdhWH?ex7soIs41*vL#lFg{i~Ju3&_@oGwlhzl`mOa#Yd^**NB6PIeX>
z1BxAgF3uq+&J-M$rWMSTayGyW{1sXSq0s8$Ckmb^$V@@yAwfzndAQ`MfFG|CsDcb$
z(m{IU>jP^DBUodAz2poICkgSx$$!Z`BJG!Y`=y?OdD4FcRsP5v{Z&bRmDL4|a2Ekq
z!he%cCeJGV9;u^q=L#X<T;bv;3Z5wgp`g;c3VI3Gh3avC9g@3Jre_wHYU#g%GI=(q
zF<85xU`}J*?8abo+lJX23Kz^?P>|ai%AL#eP4-E4=?YOGDvnOj$mhxUqdGwR*ozbD
z##KCVXw&MCYT}!Eex13zy@PlQ@ke@N@z>Gg7&p#RG;!SI49k`=%tE{?@V6L$#qJm0
z%G$K-)`o#I_iwvu^44Gd^^Pw(j~*WP+k)kXvNxag`i`D=vS0f5>NB&x(Q^3B-~Tgv
z^0o<!H<Ye>be8*tGN`9Lu7{JSq~ZJtK~6`nH=z&1MmqT)Wu>Rrj89J=8ke5_f{FdV
znE1aFjVGNy2TlBc4#7_+Z$Sa2qkj$!ARYZ~6F*B${Q1ZvK7}Uq_e}D;z{H<YlQ{nl
zl{TIK!zSsjH?e;YDsDPEx0=K)Vj_RKNx6T?ByNQ!`2v|a@t1mh)5QKm5KQNPpGlmH
zP2yH-Qh)ZE(6^e<vrOvebQAhQllc72q#Vydex0eo0iI(L&!Z;xt4-qhLzD9TvWY(d
zlXTB8iSuP9{=aNuf0>E=StjM1Z{q(wCh=bn{~c^9E3B<SO37mp>i=|>%~mgUablZf
z{S@tp=$k!)J|Fp{;|Bb3`eiOc*%G8j$5*)hX>7m0hS0?EvGm_{mMcM{lbNz!)uhl@
z%Q&~B(CeUc3LDQF&UBG_;`owGH#J``nB?mjXr97y*z?O=2^(~hi*lL7rZJa{k3;77
zEg9#EbByvgOS;o(q%(}e4fP=myv)0--MccdHrN^ogcmpZT3Z9H%v)Mj?yU=iCAq46
zabu`CQ01#_3^1>EZBwY(+ZyqOBVMl%O)nU14n~;wnicbqQhq^^KN1WzSA{Ag;b8OH
z;`#m!8}jqrD=NK(KF;w*+(}{2k}KWK?ZI%Uxhc>bDam)YoADNNUOmz>lPFvph*U;=
zk$}080%X3$oIMX78a=(yEV2Cg{!nvsz#l<I1L5{SnDp}J)dzf$ws64P8i*jDD=H(G
zc)dIT`EzT1twBG+Nl_{FS4Q&l{SCgbHxl**Bdw)=ccl+MZg;Tq5-<MO;*SciHB#rl
z=pvdpEr2VPkw%D`>8=E0o=7HybOOGn(kgFNp4SaqE0>!)Q(54xobRp5_qut>jNa)5
zzLu6ibDewTa+sos@w}+gtc(-}L#<vo7APo1RY}l8jjisAB`YgKm0p=}et~d5STFrg
zlUFxtLd9}o%=4DQ?v+ToDOATR?1l~AmO!{Q)a+{vMmBoe^T?Uyqt8%5Yw$){jYhA$
zcy4KhzuoPwqH;t?=82Gux+>()^%gZYhWy@!Aet!U+W#r6EzbX+@k*67ir^%JTj=+-
z`1~jz(GX7}z`O#;SfF|NytU}Sn!SEs#NWWHUohP2URj6J%18k!W@EZG<2G+D3s7R*
zE_5}k5mh}y$E%)~1Y5l!)Jc)Mlc;=`c+n^OBEC-%9%DlkT_;*)N<&Q2N83v+2sXC{
z!Vz~<GND9pj@x$U`x`^8C*890^AVav?P=mgO}%Ex+)oj!D^;I0>IRzV$P#%!ZpAR>
z{S(zCkpi!Z;XtGz99qxwJZewTRhIKHsWH&JHqzjx(4hle==E}=-p0^cj7@=XI24xc
zdi1R_QEe-zW8fpzu>)1QM(3}>2sPh((g~xVM6C!lW5{UpN6^c;-R->Zt&Gggs}F_O
z`@(hJAZ9(jNC@MtZ>@;oCn^Bbv4&dHXj{!f@)N@{W+*G0MC~25jr=?qsSCEYe41u7
z&s&LrwQTg#3<P7aAK}M;UTg*aP#d4kjXr<8mSMp1HER7AI>3c;P!FY@ySV2CLiMWs
zszNz#Cz5D)G=ZRAenlmn1!=z2B&JF&Xeq7#9*z)ctsm1aBIh&js^;}U^yw=Dt!+&K
zVz!_G1blVG&Li9}r&=`c^{+#K)_LoF!A3bApy)PWuG`+)+=6Lbq@IO+&2Yd=igKph
z<O?=4Z-6>*Flt*{1sx<v=H}9kF}8H^V(&bpx3sLZc(HeG?%dq@iSq)sxVpNixYV1U
zJ5La+t07#VpXbia$2Ec#cgS#AfxirXAIy$l@lR04a0x5tj^VOG!^-cN;evr{*>Jy<
z>{vNuER&xT1(!<PI?2&LOX43%XJ{B%5C@mx*FfTWx#EJ2z%tog_@#sRZ{ymA?*BRQ
z*EThXE8;QiTM53E<x2fL@Fxd@6X_1$4<${=XK3;^){ko=b-(iWKV63_Z##Qf%1371
zfh$KF+bQvXAHEwIx3b>=r(@V#!y#QZ&f{m|M_qTp7;|g%DK#}5!{s7!l02Lmog%A_
znHpWih<JG#UBp>v6l!!zTOBTqE-!I8%cIfBCv{Y4bOa!ARBLnuK5^7&bQDkGXwc|%
zKUEzq8eN8xvo>h-MM@a3L!-Mi`WB5oL8JF-bX0)Eu}!0AC#a0|Y4jY8{(weTb%fa4
zHTon?euqZa?{5ug^iwtY{TiL_S*v4Eqo1aTfCn`C=^A}Vqo1MCV;X&mMn9y{Kc~@$
zHTqPI&U`f-qdGE8qi1S#^(=+h4vl`6CO<`^pRLiI8r`YUXKM6wHF};#pRUmhHTroP
z-KEjb*XSONK0~8dX!Hv-dbLLXH;rDS(PwJ(28}*Tqqk`EB^rH$M!!&_cWCsBHTo8f
zK3k*rYV=%<zD=Xg(dc~|o%&66JfP8O%uvU6jXqZq0e5Kh0*yYP(HCg+{TlrejXtQ+
z3pM%yjlNW)dph6B^mN<17CV`zYan8eJ!E6d)44Zuk0ylEk@;bWjLeAOHwX8?MrIJ+
zKtl1M5gapaAe_49_#nq?3AYkH!0~GchjA9)&heFmQx_cX<M=Ydsq2mRa=e&u>T=^9
z9KV!sYDV!Ej?X2Wy4ZLP$1fzDy4H9F$EOocU25FL@u`GUml@CF_$h=_*B5tkJd1GZ
z^5PDTTM4JGF3vdqVKH#(;^K!s24Tj#gj3fRAL96%gj1ImALRIJgi}`*AK>`Qgi{w5
z-_G&p2&b+q-pBE$2&XP9-pldbgi}`)@8I}jgi{w4Z{hgQ38$_pUc>Pp5>8!Gyn^H3
zC7imVxQpZW5>8!EJdfjd5KdiB+{y8;5KdiA+`;kxAe_3IIOF(@gr7tBp`(=lFyT(Z
zhd6!%;ndZ{2RU9#_;kVtIDQS`=Mlc0<0}cLE+yW_@nwWlR}$~#croGBg~U5JektM9
zb;MgZK9_LnGU7EHzmRb1D&iF!pH4V+1#uV0rxJb<;dvZCg>dQ;;!cid5l&q}+`(}x
z;nW4h8OJ{?0!~doe&{1!{)AJLj}LMDO~R?E#|Jt78sXH$;{zOjnQ)rw#J6+&Il`%_
z$NM<`6yfs;@8$Sz!l}u}J2?Ir;ndXQEgb(j;e~|P2>C@$R^JPpANl_%jXrz5C;FPF
z^KXYLs!I0_3@&pq&%RwX9PHaP&&6V2Kt2xF=bTf5{%28b4O``zP4~||ox|Cl=-=C?
z?I2UsElgUolNcjI5Z=t~uh~PrKz**l`-#8GbMxEWz^;#NFyM(E^6ZK&@>rhmJo|Cv
zw1l<s%397jY|S2#e#hspM!Z?uq*YLiHN&wrPuQ1$WO<LLv~yq$Ikt$k*$;jldI@9i
zkiy8|n)r>-p>y<Tj@X0$8X1Z1jt=e`K4(|#Y|HM>*KD(Zb`Io3-5+(kKYDTxiN2dN
z$vymTj-#A@>^Gxwyqk0Nt~HzoGkKCfsr(SmR>knMW(bm`bDbNgW8y04#Hf5o^Z;2|
z=W@b|V<N%&oSBfn{x5*PB|VtA&JD`dyGY`5q+7yutKA1;FT>?}PtKPHJW+1ReLy&Q
z97k4>H|6Ar`;e#GJ&bf-#^1B}+n=x!%SB*_0x|o2R7rAjcjeqYU<d)vqQicRV7rIg
z$MOIlwC}K>xJ5*YRuxrMdTxGeKUIq8;~tnruDhsr1?(y0(4nBc7;R5w^$LDoO)hK|
z@(%l4!QcIEPBp@D@j$feI#}rDzaIYqPj?rk;EC>w?Klq!F5Hu|bzt*AB)@-fxr^<j
z>x9@D2w54bIQJF8Bc5{yJ)Qd}mPYp$Jx<j$?nH+!E%g(D%-s%i@8vk~cMak<>HZp0
z6sge_4VPJ$ltFkWJ=mbhSf$L;q|89<LG%%_^txz#LQWHrC`7NMB2mK6DaVe9dGZ_u
zVlTjX>)3Za2*km&iFqDpUbl;rJdcs{I66G}gP!iKA~H{SqJN0(LKiD@L$&mg4JnEL
zYGyL=Kk##r_(SrX68{Cx4T=ANGf$j&NiuP?iAkF}R2*&am12EfQh;I|*rEhz;3olV
zQh);8)T;+>1l_#~+<49RGc@1-1sQ|yZ*rv*`@ZE|L*UxB=_z+<DSwThi>L+UIYsSe
zoEuVZ;mi}KoSjUWN3Bm!c|12FdWe6MsZo8f$#Y71EY1xnAHg_@zU73K<xXcJ&%8Ju
z&{O^zHzHDgke`b@N98%C+=FvN%3tKn6Q?{inKCcu?MQhQWxSl??(RkRwZ+(db<s2p
zJPu>Q4-@Uny~Ci}CEF%7<KtQQH_gI;!NQ4Ck3e@zSG91+-3Mp4i9Xio<d3-6)k16)
zWwD$hd}90EA0%_PpL|Q`e$w2PY8D0!7EYY{UC`aqb&bs3cI57Xbh-N-7rR!7*>76t
zM3obRo_R#>fet$Dizy1Gh9g2~EFWty<e+O{n+I|>-)E&-J*sN*n_-^bmq)2K9%{&p
zD$NsA%$XttT?Au;?^CtQ<*FDpX#h!NA>vFkIyo>pm323D#5a!-*-Mo5iA(A&B!+OF
zXKMCA>i**wO3R~YU*lj|i@;WDR0v`ZwJIvSbPd!Hhy6#m;N#S45s%@VTcQXBmBT3V
z*U60}%3<_g6bmozmtG7VQz93n3Ee3uky1WiVC>MM_*)p8(n#ePrJjJ)C~H|Roa;Pf
zK>-$xGA@VzTWO}$u$;{`sBxq8<1ASFRAv3jVP4h`$|%Q@Wj%@-cQ-fkDav}a@Zu5a
z#i3D`HFL!oCMqUnEoYE_O6`O`X^;Gi_xN~3-MM{5ftYr$Uv4P)Pim@BGi8&ej>+xx
zH2KI(&V)}F`7rOQ-$o*$cFus4=Dqb3klQ@aR(@Qe?<9=zuHSUT+{leZQfhT1U0$gD
z$ue0c>b!YgrYDl0b6ht+d{^Z;#e<y9G~=ei8byu%^*e@kbplmG@A(y|s!;^jX6Pc$
zN>iL_=r~wr%H#vU!HfH+mZ^)RcmHvMi=?vXP+9cNvglvRq6a<E`=~h~<9tB#+`M-`
z;&5%zbwz9BMIgTftU+a}^K@s-z(gF=cy}y+z|+$;glnM3GC}FT7^&0j{>8UPM)r06
z7<5n>M7HC!dn*w<{^vZqUbA^DyQQWl`XZ^k_ZE1<md8$ydWiTf*g>5u?&h{U+dSQy
ze@&)5(XMR(<Ss#RRrnnU_tQE<(#$~aJGqu6{zc@hSJ?)y`XE>BAxkh}AHdx)aW(3g
z_+R{-Cc~5L4S7zg3SB$6+$4LA#0Ml^E%E&luaNkl#61G<Ilys;y;@7BN=s)Z<UHMH
zoeNObJ>%UuPS$@(q8KkiT*{*N>)ECJJ&gSIFOlIMOR<aImnzc>89m<}ZxY_^j@5bQ
zi6V@gfAB0%bdMIB>mcKa9-t)t0;{nJWcud4RPd+`qAt+QfSJ5onbbvjOQ{k4PL%mh
z2YzCGs1;FF(EXLt-ED}!mgJL25*5Dy*SBOX!PzR}=;$Iy4h@hmvWfuR1v7f~p5^%Q
zv-cn+EwX2&?9C?p=Dn1gYxRmghzb~e{Bqm?^+X?``=%**gd4q_xi6Ll6Z^U<fFZ}Y
z$@0$u8e^-0Q{9T-&Me_{uNF!jQfVut+q0FUJzY*z(EjUWbe=*uavrrmG{?zGavmLm
z5+in>zlK-r@o{cw2!{GQB<=keTGv^R%tkfp=^7wTjpV#aocVWyQzq<qdLAcLq)l=v
z8$CVb!c|38<<VWG(c$Pz6z;32Ds=93NSj3JhdH3!0??ldl>8%ZpFqhJp<fed2c-k_
z4miHA%L5Eyl5x6_<A_sF4e00y_vhSlHiqVXU1d_P^KpKb)75|?Qtraq-`*grT@4WH
ze4GbAr|VhjHPjsw7w$L^7se<48Mn_p(XBHD+C`8&ItT3|x|`p<iHf3#c5MN#n-^2(
zup=^Fom$&aJ$pD)6`Cmj9^EMK$@rx}ECazi*e=w&2HGm9F0{Z6stYaB56xV64-sYl
z#6?&d%LoCB{Sa|HcxJcmhDevf8oQ0ENB34T?dfrMq{;QSFo8{IyHHl5{~e1*Fi5|K
z3a_V!=#Q=8HS%c+RV$h^?DctiR&{{QrJ|hU!J`py20T6GeUEvQ8bwM2+PA*W^Cq<R
z#Wo0((jePGj*_qYVr#_t0G(fj^Mlv)`(RUa@J^I@Y!U(+?W#ilIv=NBN82<Rwtjku
z14Vuv8S6ytqcD+-6wtyH5Oq_O;l9}Fzmrj_Y4L7!Hlos5+Rmh!sao>Ar>Rjwv~J3D
z_g2yo0i2M90dQ@?*;dL)bT{rVk_5kH*8%_gTX=}}#VQ4#hfc811tl?ll@Oc4ia>ba
zT_?9=iSOn2jUBRoxt{kcRHrs1v)P}m$KfVwD9PTc47PZHJl*9q4ou)fdCun7p%Z_F
zE7EYlI|n5FyT2idxTnvD2ucF9UurZa%kX}U=9ILrX*AUd^dR|zcsv1ns7x@{s0pe1
zQLWKbB`HbLL`|7Tgvr7ZkJ9v{M51rL9yHnApNfFSu0Zmhp1>h(DC>M{W^~gl(Y6CA
zJ$&b;S1dVQMd(s)-gJQatema_AUk*YaRa&?b{>oQYFIy^JLx5W*h)&P$NdK7>QSB!
zo$v17K~;kK<JcihVl&Q#1!C=_*C1jJjUUl0{Fy9tkcFRU62%D%#1a;s{%g{QxtfJI
zS?D7R6mR7vOIRS5uy7kJ99$v7%CquEGATyPLDJ<#$2q%_oJWnE|C8i=lQ@Pk*&f?L
z?9RP2<%ru)(cOvIvRDvRObo;H^urU~ME4BcLweB+=MU_2ANEjJ53`=`RYRWWssm`K
z?gPJ!FLSZSh}BQ8kDwq#U;GMYAwsx(I8o*C$U6^yh^HaZO&>+uhVgu8yn9&9aZoYf
zz=3Gn^Lp}_@pikP?;JiQ=hode%n=4K8~$~5IUbA;sj&W%ve&cf4J3J>|1G#gHRAAV
zL=?4S3(n*HyuTHMTXl_hq{ePdV>M~qrfICwHI7J)UQOeC(zr&`I8WC&DmAXxG~WFS
zG-hfV?{KEfhjqEg$Hkh)0BIb-aH2A{Q`Z<LH9mxt68I`<?A0`SbPb2p*sp0^M;do%
z8rSL?Q>DgTn#P5sQKM<h(lySP8g-h+@DMa|HI4T<Q|5e@)R?PjJV6>Cq5M_GeywZF
zks3_XxRW%V(loxVYZOS0ry-?G){@3HO~a>aTrM@fscGbqM!lvnN7pEp8aHSfAO9H|
z1)9cD&XhU#NR2{G<F}+?)ii#iYm`fkF`CA`q%mks#_AqjqgHA>4=I)N2GaPJrm<Gn
zXqFn^)if56Mx&-NU)Q)%YP4t?8KiNkreV`Hx}?Sxn#OZ~f<`83B)h<V&}8TM5Q!!^
z-%oPxF>=0=<b2u4X-RSdM$YOaXStDcd6JWF<V;U;&M<N^lboacltyFTrN8}ZB0qmM
za-L6eo-}eEOLBf{<n$#uTaBDslAN22oW>-_XXLC*a+Vr7g-OnABj@ZS=TsvnBgy%I
zpC)4P<|~O9{MpDEOmcor9Q5yoSrmqpS4ku}v{hmlJKiW_&$+b_!<86C4jD#~R!TvQ
zB0aJ`{)EiSvMR=T{52W}C80>yxKnC;QPXgc#&%63N7uMVYAn+<{`5y^d`Z*b`BPr~
zNNP;cG=5APWtzs1I8*j0zmOWQLrS@QBWX<4G&brQyQId$nudooVwf)|6H9ds)pq|w
z(>R?p9wm)r3$+qoOl;ogr->R6dpW^*)yR1|$=PG%Je=hGr;&4Sl5@L}(~;z~89DVy
z&b7ozQ`1^WM6YQ#zm-Wrj6R+@Lhe7pIM_P;F=0K20Lr5L@~$k}(*PblPs*9a7cJz>
zD(}ddwW>E~)|$SYS%K|2vzi8SX0;9G%-W>Sy}AmahC6=UUEF9-Ma~Yo%j4;cSv;1(
zU(q?g`u4p4o@d>F!!v8ICpzT8+8e9}%JA$O%D@Ht6qwk@Z+JK%zOU<i;CW=E$L#=n
zCQwfgUt0o<Zb{;5(@CIvD`mmc{RmIH`*AV^PbilogVC<Zg!XiuN&wTX`G`)pAo>R(
z#-)2Mx8lAB7wLXP>Wg4K>xq6yYR~g(n6UEa13V2b(|undY-9R(Cck1z-Z@iA{+xtG
zefmEy;acN4uDoVAv6=_@+cb=#9Hx}^zKEMwk|oK>iDw~AJU!92K}>D;Lj!PMEc^$u
zzXvGR`0R`M3AI3qCc#3Y1~xsHxnr%LQ$i~wC2Y9Oy~X`T$EG(}WIATg2#$MyVu{R8
zcpmn}0z8bn51tJP=>We;dp$BniMs_Kx^RGV?xd6m?c7_4d3`Tmk)`t6?aIWJ5ZK5*
z^UT`qi4J;p4O`_*pQNH~hZMVe6;_YEW<`xkTnRXN<VYsEil+ROW1RDZe<fZ1Ij`p;
z5y7ztmrwxQ8`T%bkk}AiGkBuqlzZUFFv&)Dag>50?rl87EkLrnd`L+0TOQ}5z8;(^
z%7SZ8<7va-zSxhcSVXfFF_70WDpCDf0+%w#Owu}*<Q7%Pq-lXB>^ORnE?u<9`?dDi
zP4G$PNL?fSkJ&yaX`7-|s8>VXak}j<>zo}e20h6nj$h{9f6ODDtesnM@fq9uJk<|F
zWfFHA+!R^#62m#|pT|8U!VZXDM3e#LS5zLvmzVfU!W5U3q!)QXQDxj0TlRu1N|hMy
zy*v~9{_lDEy3!=rq3~+A!{;5mm`vQwg@w6kFc+)mHuuE7#UW2-57FNyS453ZRQS$K
z!z|~{fn;l*slu2V!XA+Vt@Wf_BRp8e_Eu)4%+yR-1ybi28+rZ{9SfRIYC}Q`qP8}R
z8X6W_CKhXA{bbP>idGjzM`%r7S<j3Gr#i8CvuE}dfR&}uBc;(7%4X3@%1qC$qqf)|
zo<;V${t<EJ|52^?<WG!_l)%Es<X1eMdn}%X2ix963^K1>Lr;r}t}pWL=}jj2-X5yQ
zum=g8;=vAP&~(hGXC=PY23FM;Ze~?@^$TA>SRVAR<3Cj)Rvu~(0F;D-wfHy!&Pto{
zMxj5@65%XX*$@gvSOq?-pk&x${yOrC(CknPyQ&SZjjasSwfO_yqUN=Y0dG~vyENo$
zWaYl_I&VdzZ(|_LN&@xt#xbWa4UtH(ud$J>YF^hITHlN$m-_J58fP!|wa|wK>WVjV
zrPctuvMq9DeQ~I*xvo{5R|O#~K2A_V>U5Mgx8v3LI_DDcodV}-e8C_TcFu7wZV33-
zIU@}LXLDOqZ6NFn)jK2jf<O~qd25{$YH0~I;{y$?&X#bf9j~L;DW{x^L(O$T`r-p?
z@r8X&t+|m65w;GIqmM{9@up(L$>>!&WVtcegeds&Z2~UDoGgp2#0zvy%-s@f#Vh<y
z=0yGjb<Xv{NP`n^;6>V6FLjdN;tfK);faqYTzaYgz9;-`Y;-n<B2Jz&62^fS?P`O~
zb0`O`bLh(hbMS&_eQ@m@ybp>j!QIx`b$C0FURG>?LzF<k8xF2*h;aQWvrDS}b?!J9
zM;McEiiGD&m?`0O35OOle$15kp+(O<cJca6xo<Aw<Sh?L`qi#q$?ui?L&9?|7p$u!
z?3C~k3Ez<LbDtM-%O%_-;ZG&}i-f0MA>`Z=u9xsXCH$j=6Bh}&MG{(y1wK#0fP{BT
z_@spYlF%~Eb{TzZY&v7C<>`?Tdh^2h+{g%BMx2gr%Z27u1V|6;)&O<@z7E(2Sb~mY
z05AkN1Q@}MkxUyFm*O5>9^e|>F{}n$j)zmdxDoUWo{;ST{2O2|?61W061se-13U!y
z!}mr;rohhRBUncUy@%fiAMhE#LBLl54*@<f3_09hc^a?~@NK|qzy(<UwE^(QfPH|O
zAHqMtI=~^od;SIgkp7PV>BiK=k48o+0M7ty0dxT#0Hk&G>N|1tVg((R8&@*R28U(Z
zgt3{umaz_^pNYSr!I2RxKu#R#<rzC45+u)l2j7qfmhD)QJ!M(W`1P3`Y|+%qE}VD%
zbYhcC3I56uGv4BhvK_Zr7f&2x4I*X1uLJ*H@ad%xrugNYo&28OjTL3w<FPjS)QnB|
zE&L!~8lg|$+R3=C%9XI;?zGu1AHxliOd0-YLBS0ulZ;nW<Z7%5e-1<LX~Y5R(3Aeu
zC;gd>vV94Bo`2=<7m|Fs5B>u9QC;7<IbpvF{3|eYTyNlCo#02n|04M5(x-gf3I46%
z+p{(Q%6e?KWpvtYhbMA7BzHOF>i#@3GJ*2zSe1;+vfFH(8MdPqNRnI~<UF_)_A1UZ
z&eP;Poi^(rH^(jp?>5LCh1}byQyKF$Id0E7lIyVEm&<I6Go;=Bg8XKzB<TapxI>eN
zUE9|wby@!K?<2GoJn~P*d$mZHV!Rpr_ra&^Cd#%{@X7!0ga3E%pVIj#d+UIZAu`$d
zE#$Tw9vQhnm-8jlp>G*|6?~`8zc$IAfO<~+r8?hQ%mv7gA4C69@Y`^n@dGWt<jcCG
zohsPL`0tUCYYlcphRM$e_%0JZr861)FM?0soXXf?OviRnGOqsxxhVRcbnS}lyaN7B
z;MeMYkgwIkS0a-?85xW{0=X=MTrxlBga2*tNj6bFvOQ2bdy&pV=sP!)ze6eh-k9{a
z5q5I$I1UdIlXfaZIwc_827V3F2@t<H+i|<KID5)hZSHJmPeyU}%-igq?7UmXc(MyS
z$ChWi!r6sI*?C3TGmEpGkb``2b|(Lh<MXhy1$IzehGS&JvZ>Pk?M&8XvHvoY{la4Z
zeI|=r?ElDQ->}+m9mjrWwSQ+E8?oAdHIDtvW`AKE`>W0V&v9&PhCQ0a_GQ?=m&HEF
zus@!~eq^`*A&b3gx4)CcZXaXs9?y1-vEM(Qy*I{=H^1*6Ykz4x`@>lKd*fL&)7~?I
zJ)HR-f&>TVRNA-N$bp~Q$blzq<iKBT<iJrIIdFFdIq<U#_Cwou5KpM&<fA>-9kBk%
z#*Wx<3R3oBz?Gl>nT0)R5tOS705@Cg&u6jkS?mK@Wc<fj>{*L_TNdlF+Bavx;7;qF
z@$6rA`*+5(d&by%#<PdU*gwc(zZqkXWwCFK{hkHv-(;5jd_4R8I6Kb&I9?ccbXXC~
zC$08jE9;dmOmLz(m)omY5HfJeHp4Qt($CoA7W+|)<#lrFHfz>hR<vH6A}8X2^HVPc
zp~e0UEBm>{{)m+ww%DJsvhCJ|O6D{8Sp+_dz-JNoECQcJ;Ijz){}}<txi0S5KQ9&V
zkW)ZT#E%;AQz3uTRO8dPdFV*UjW^<Fr{MIh9y+k<jvu#6e1gQ)_fcsr93ARAr=JsO
z;ydg7S_CwDXhX+T0oaZ;HT;avb#aDE7=HZs$0H#+*=BVSz!!hSF)+@>PjG1}4x5WZ
znJxCR^iKjdMT+q`uskP#*`YXUYB*GL*J_!Nx=tS}#m<n>afXYZDf-aq0#|<1vn4vl
z36Mx1ckFq7@}xW-XmA`;XMP-#_`RpOB;|hxsPEXPruKs~g&&Vf_=JQnO8B~jM<g6`
zmXJSP!V4r^AmI`TS4&tgVY`I4NO+foKalWI37?SgMG0S*@Q8$C&X(y*c!7iqBwQll
zY6<HlY?tsB3Gb5d2NFIi;S&<RDB<f89+7YiZlcg}x`Y=<xIn@s60Vl8Ucz<>Z;|jW
z34b8rqY^$L;foT!F5wXg$H@NRbO|p==^war_@Q=B?*+6UJv{dw$MB+(?RWm<gZLYZ
z*7;An`r^9ZE&Auhch^4?eR1dD8{a$dfNl4p8O>jL_7{U6EE4)xxj1w!bKwhTmpW&z
zs%>kIv^nSJ7Ubs7p4-N;{4eD%$jvL5C8*B1dHD<S7UmTQ1IPWf!Mj4M>lhjO8d+6Z
zx{XEB(lZkMhHi|$gvMllVWf{?s$VhE$Fg+o(S}}1wPU0FIHvjoBRz|$e#1yV_WRPr
zx3LMVSGFso{6v<Qrd)0KZni5e{n+nL3;PZ>w9KWO=I1tiky`CDp)+mxg0<_)bn>S%
zx;>bB*l<@s?XRKpZ0xi|e{ZCp&gionsfUf7fqgg9(Wfx=T{m6F#xUy0{@h5P%7%RD
z>`Y61f7>X3rjGJ+YBw3|osm=qje{0;tp7CrSy3)6Ddj@rAknv{&~qg{bzG<WG!}f(
ztHNJ{v*fs4$<L3C=Z&C~eKlVo9Xf8opCwJX+z)!XeC_7)4i=O3gz`_vtN0^-cKK^?
zmh|T>oKw2%7P^RxTO%AWdUil`HE&Vluvt9mUatl9tvaP%(f$JI;!LTe^M3^;kfuKS
zxSeC`)eT&J@-g-ECP`1N&tK(sj;(Ls2c6#^q~x{Z2<X!hhuJcoYQEyOrkD4c(7z4(
zB+Q-;`)as#HGiOcwq!qhsf*M3Jyo2&Yhq^-CO~AzSzE*9)%=CthDrC`{u!XB%a@Pa
znZ`Q&!j6+$V{C(o{8zaAR5mju-EWx4e*ik=D|Nqs+c6<^!vEBEU4(#4LJm~_rs7k<
z>C;$Eil6nIj@f96KR1D%F5Nw#r;EeuQa*M6f^rNw>E!7vmSq2zr7j*ZC3B~V{A-+^
z&FFhdbkH*cI^K}-^d6#q%)tbCl65Mhmj?9Xa?npfy1sK=oUD4fm7K0iit}%i9klm8
ze+?%%`7iE2OL=;cKtGO3dg^`(lQCgddDP|Cnb3b}LjNo1<j;V=hMTcSV{d{^@qDVt
z#VPbmijMy{mC`+S96SRRKAk`FK&N&wC8a);n#gyX(C;&$9|oQLykwb+`={dn1za$s
z^Ctv4$#1?)$gBM?%=T^hhm=3+7V=N=RIry4IvmPZUP``}gPtzkfC>F2(8-_Q$a-5V
z?N7r6S2{ZvgHH0#OMldLU_{dYDe3%K6l{GR?V9SR%PIW7P};}+{j~NkLIt3F9g_7N
z&5$2wV_;6_&*dicdqF2ZGi02brJaW*J=On@K&NywQ{p)Z7sBb%y@=D%er3ChO8ZMq
z<nJ<}zhOe}o<!*$JD;>6u+$IG=hE~e2Xv=Skmu)tp3eXEWC!^jmT^Wk=0_K&C+ATP
zlm{K(1fAmf+odj!U(0{7j*Z*TW(#B-E|T;tS?+byU7VumPEMCX8o1nq-UB*~U#a6l
zJN$B>-`MZ3(X8M(IpoRDnI0FXD?g7)`rAtc-O0IF3xFFK>EiQs6Z-EtJ)7N`C+yI(
zV><o@I>pl=%SFZWLlZlbdHD)m{kH;i%2#UnUS}eIGpA>>8rlB@q_M9_`k3i1PPtyv
z?>DjYft2r?E97ZjNXIw?CtZAINP6nNHcLRK_!pG8xPG6kKRZqAya>9JOB#RPHIdKa
z{l8I`u=7pm8#q0i(N_%Vkj2HefKGmfW&eQW`0-s6JMWm#FPtpw?3eZG0%>O{=*p%f
zaZqnU{~G8{T~MC)naKabME+S5`RSObQ$KwBbQibG*GAxM1*db8;paOh^dpk)bh-@k
z%pZ=lM%wD@bFrC#ciG~V-m=olDr`fP+^onOY4Tz>iDufy$Xgfku5Ap}(wFD4o~_mE
zYukW@W=$=Pfk>b(_mcUAh3SR7w4spK7Y_S2;sfoG@J3dTMQj0YU0YMrMyP0*7m^W!
zXam-+;XC?Ru!cR9uyPM8$b4&ICD-55!oQ_omcOti81M(y2f<40P#{=^UT-i&CV>{@
zd+96j5$wT0lpvPq-N+ZIc`1du0M@->X9Fx+^X3Jy{n660D~pTDpwd?BE%nMRhbRh5
zRu+}Jz3vqyv|kgCnmWfmie9%z3V2FZGH=P%D~ifX7emgFFZ!CjMu1oNh`vhk$|XxG
zv8_#2QE{1@Z)$TAyPIgczqnld1Nh?m_Vi7AwJAknvj*(w5?C7wZ=_8fu;D`f`~*jC
zn}n?+@-Imcl{jrgG?zOlHX_3IAcnM%5Lx>qn+>T=JB<1*A&Lq%xoAndwT#%x2}zjj
z5W#n4;`=RKSy_mUFT~ypa{nyjmRM@XD#H>=wY|V6&IRq5V9bH{n*4lh;ew5;yuP}+
zu-vAL?`I$d<-P=-qOv94C8h4N67LsEOR78~b|x81vt5&j1-6DryXAzzDNzN?!)9td
z-QGr}SE5PPVOOBGR;*S<cJssvP;8!&$SrNtgBBg|dD~jV>QwKBlgnTFJ)gveXvP9f
z*`>_1j+kclB(^(A-6P56()O4@TfjarYn$63wILq`i~WmeO|7>D-4oV0)2?K-6;+}+
znbl}hZ?S<2-y`Ulf)@K^ob(1w27BnU>RLly>W^vTx@+<mU@JH;HgjvkfZ(I$)?$~C
zlkuLm2QnlkYUx5+TI_AcSfQE@?^8zGE{<uKWIeIHh)h?u#FI*Z@5w^D(W#w((&Xh+
z?2wkmn)$v$24S`qtKWlETC(4(<jcL&?Hec-f}d#Lz+_`?46T=~8AG_<M;>>3rsTd|
zMn7pA8A<un1)*<_V@M@2`sJZ2W6iv;*&jIWpf|rRSYMBg8zJY7Y-|a*G2qev$lL|A
zCrzjc)!!e$?lXzOQEhxA_C_<Lo-Bk<xOwB~do68uC_~aJ<Np%_Y2`vM(}t@u_ll)7
zUh&>f?A()Dj{nd1Q%sxJ!cSAe3|kweX5%D=OylN)az`cXme+*$TuN7-gh99aN@(RT
z2nWc7H;f4}ubtR}(p|u7D9v?H(@tt<Fhd-9qt-W%JeE7^p57P_+V)FqwrDUdDytTp
zg^>_u8T3+$`y@vgW4e4JyJSa}n3afrjBj<9ZYH;ohC8ZM-e!8e1y>J&Iyy$t`<ZNX
zX2?F7JBb;8qUjr_pgfp#*`sZ<C6hxxTYz?p?P%4-OuD^msb?SUcx&7~&5%3tN4>Yi
zgURbE-ww?nCxSZagv~aogSol$sr+QO-GFzHyp8QNd%*=CZHJ4i9(3c)d{aq2TNMSH
zZgPoI<QwRYJ}{#EF;$zFX12q(bEBf-S3M^a_67PYTXc19*rpGS0K2R;;v$(k3JNf)
z*@AZdZdz$Qt|02MFYX4uYhl`pr2m!8ct80}BK<WZU2)M?b6)YDTsup)^W=mDmmm6W
zkEs*&<VEXor!vRPDq&UIiH;hMaZ6sRHz}1|>}lt3ZR58Cs2b*@8mb18>|I9N5R6-g
z8WI)VYkuDPFeWXyUxMp58IHt0tj2(N*B-OsuTj8oX@g3J?s|jhuS-v&7Q1mJROPK$
z60E?c%1yXz$F)1=*WPv?f`!f(Gc0#gWwDoU8ZEXYH01C2+ie@RCWM~cBao^cb^K|H
z=wv!T;~ta-_e6)xw=$$+ms@RD;(i*copG<_N_jQ-iEg2YuH_gThO}jy|3o`Qrc+Gm
zEE4-Si(1TkE-zhiO5Nl{K3y-Gb{p`N-Ov$xzWZc~Gqj(2`c2YLrm(cVmJR91hC;Uz
zl6M~Yc!3T)wc}LNfKfM8eN*o=$)7|6O4+|Ry-{P&bZl3~a$7exMSQh@k+6Ub>Wts8
zXkoc{9XODSO*CiIMUTL6gCn=LE!bE$JBWQa32UJDy>sg}Hp8BPk+9(4u2?I*(agmp
z;sq}pX!MbSJZou;uv~r%Hy5Y5Yw-ejE;b0oFK;`!;Sk>iG&j&7?;|(V!HB|O9UGGd
zHqbld=sNJWv@j-aF$){&{02W1k+82R=!ae?LJ5LGn~dh7XJ@&%+1-R&&&HeRCy&u{
zSbEf=o^PmicZHImo=d9~<YO6yvh#2@R@1Mtx_~jP)8pWnXA|nVtl%=1j}u)#Vs&v=
zMCzZ<vE)yxehFB3<t(K9SL=kgNkW5E#qtP#sOM`EWh-!kN6Lx(Gn-4WYyu2m9RnQ%
zDDO&NttVa$8i}g()%w8!Jb$M{twR==u%XTg((^T11FZDbIz#&ECmp>4B-TAEIR)2&
zPiu-5uGS;woh}r&OMUX6VyFC<WJc@6h_3Y2y2T->pC<)V<EQe!S?U){IkmpgAs0%k
zb=0JfRZ@xcmHt;jqqxd2V~02wFeY}2161q7IT1hPtA5b)b6sDpm+X}lw@;R%(s9TW
z1@BDJSL-VCoI)@)Ua9zfDf%v_iz~XE9I!jo!Ucii8Ljd79#GwXwQm0TWkOC?ql8jw
z{{K_zC(3V`i^~p4eOF5UQ~m#0ioRO+>7<5%gVycoM{4>z!6XrtzFHsZx>9JJsVa^%
zlA8XLDf((1>2*~?;kzeD|Bn#M#-GZjTA%m88lj&W@6`0)fEXU}C-l{O2s@;Hs(dQ`
z9(2h?mA+cPI`9eghoMV#U+JrLuKT5ao(!yNA1Zy7uWc#mtM#zMUNCSdd4VNyMYq6t
z-T%b8*&3lSloEfXr(jl!zFKcF<P-YUDq#tgo+8sa5u&O5(>`l-6xMRUwt%l6P94Gd
zY2cGcUJ705&rP8#a|&YlLLzNl^y^y$cD53f%oLQTt?It6-y;3blm>m<9M#`B(iP
VwQrSoU4KJZVAEtjtaz#E|6iD<N>%^>
literal 0
HcmV?d00001
diff --git a/example/hand_coded_defense_agent.cpp b/example/hand_coded_defense_agent.cpp
new file mode 100644
index 0000000..94fe16c
--- /dev/null
+++ b/example/hand_coded_defense_agent.cpp
@@ -0,0 +1,257 @@
+#include <vector>
+#include <HFO.hpp>
+#include <cstdlib>
+#include <math.h>
+#include <fstream>
+
+using namespace std;
+using namespace hfo;
+
+/* Before running this program, first Start HFO server:
+ ../bin/HFO --offense-npcs 2 --defense-agents 1 --defense-npcs 1
+This is a hand coded defense agent, which can play a 2v2 game againt 2 offense npcs when paired up with a goal keeper
+Server Connection Options. See printouts from bin/HFO.*/
+
+feature_set_t features = HIGH_LEVEL_FEATURE_SET;
+string config_dir = "../bin/teams/base/config/formations-dt";
+int port = 7000;
+string server_addr = "localhost";
+string team_name = "base_right";
+bool goalie = false;
+
+double kickable_dist = 1.504052352;
+double open_area_up_limit_x = 0.747311440447;
+double open_area_up_limit_y = 0.229619544504;
+double open_area_low_limit_x = -0.352161264597;
+double open_area_low_limit_y = 0.140736680776;
+double tackle_limit = 1.613456553;
+
+
+double HALF_FIELD_WIDTH = 68 ; // y coordinate -34 to 34 (-34 = bottom 34 = top)
+double HALF_FIELD_LENGTH = 52.5; // x coordinate 0 to 52.5 (0 = goalline 52.5 = center)
+
+struct action_with_params {
+ action_t action;
+ double param;
+};
+
+// Returns a random high-level action
+action_t get_random_high_lv_action() {
+ action_t action_indx = (action_t) ((rand() % 4) + REDUCE_ANGLE_TO_GOAL);
+ return action_indx;
+}
+
+double get_actual_angle( double normalized_angle) {
+ return normalized_angle * M_PI;
+}
+
+
+
+double get_dist_normalized (double ref_x, double ref_y, double src_x, double src_y) {
+ return sqrt(pow(ref_x - src_x,2) + pow((HALF_FIELD_WIDTH/HALF_FIELD_LENGTH)*(ref_y - src_y),2));
+}
+
+
+
+
+bool is_kickable(double ball_pos_x, double ball_pos_y, double opp_pos_x, double opp_pos_y) {
+ return get_dist_normalized(ball_pos_x, ball_pos_y, opp_pos_x, opp_pos_y) < kickable_dist; //#param
+}
+
+bool is_in_open_area(double pos_x, double pos_y) {
+ if (pos_x < open_area_up_limit_x ) {//&& pos_x > open_area_low_limit_x && pos_y < open_area_up_limit_y && pos_y > open_area_low_limit_y ) { //#param
+ return false;
+ } else {
+ return true;
+ }
+}
+
+action_with_params get_defense_action(const std::vector<float>& state_vec, double no_of_opponents, double numTMates) {
+ int size_of_vec = 10 + 6*numTMates + 3*no_of_opponents;
+ if (size_of_vec != state_vec.size()) {
+ std :: cout <<"Invalid Feature Vector / Check the number of teammates/opponents provided";
+ return {NOOP,0};
+ }
+
+ double agent_posx = state_vec[0];
+ double agent_posy = state_vec[1];
+ double agent_orientation = get_actual_angle(state_vec[2]);
+
+ double opp1_unum = state_vec[9+6*numTMates+3];
+ double opp2_unum = state_vec[9+(1*3)+6*numTMates+3];
+
+ double ball_pos_x = state_vec[3];
+ double ball_pos_y = state_vec[4];
+ double opp1_pos_x = state_vec[9+6*numTMates+1];
+ double opp1_pos_y = state_vec[9+6*numTMates+2];
+ double opp2_pos_x = state_vec[9+(1*3)+6*numTMates+3];
+ double opp2_pos_y = state_vec[9+(1*3)+6*numTMates+3];
+
+
+ double opp1_dist_to_ball = get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y);
+ double opp1_dist_to_agent = get_dist_normalized(agent_posx, agent_posy, opp1_pos_x, opp1_pos_y);
+ bool is_kickable_opp1 = is_kickable(ball_pos_x, ball_pos_y,opp1_pos_x, opp1_pos_y);
+ bool is_in_open_area_opp1 = is_in_open_area(opp1_pos_x, opp1_pos_y);
+
+ double opp2_dist_to_ball = get_dist_normalized(ball_pos_x, ball_pos_y, opp2_pos_x, opp2_pos_y);
+ double opp2_dist_to_agent = get_dist_normalized(agent_posx, agent_posy, opp2_pos_x, opp2_pos_y);
+ bool is_kickable_opp2 = is_kickable(ball_pos_x, ball_pos_y,opp2_pos_x, opp2_pos_y);
+ bool is_in_open_area_opp2 = is_in_open_area(opp2_pos_x, opp2_pos_y);
+
+ double tackle_limit_nn = tackle_limit;
+
+ if (is_in_open_area(opp1_pos_x, opp1_pos_y) && is_in_open_area(opp2_pos_x, opp2_pos_y)) {
+ //std:: cout << "In open Area" << "\n";
+ if (is_kickable(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) &&
+ get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) <
+ get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y)) {
+ return {MARK_PLAYER, opp2_unum};
+ // return {REDUCE_ANGLE_TO_GOAL, 1};
+
+ } else if (is_kickable(ball_pos_x, ball_pos_y, opp2_pos_x, opp2_pos_y)) {
+ return {MARK_PLAYER, opp1_unum};
+ // return {REDUCE_ANGLE_TO_GOAL, 1};
+
+ } else if (get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) >
+ get_dist_normalized(ball_pos_x, ball_pos_y, agent_posx, agent_posy) &&
+ get_dist_normalized(ball_pos_x, ball_pos_y, opp2_pos_x, opp2_pos_y) >
+ get_dist_normalized(ball_pos_x, ball_pos_y, agent_posx, agent_posy)) {
+ return {GO_TO_BALL,1};
+ } else {
+ return {REDUCE_ANGLE_TO_GOAL, 1};
+ }
+ } else {
+ //std:: cout << "In Penalty Area" << "\n";
+ if (! is_kickable(ball_pos_x,ball_pos_y,opp1_pos_x, opp1_pos_y) && ! is_kickable(ball_pos_x,ball_pos_y,opp2_pos_x,opp2_pos_y))
+ {
+ //std :: cout <<"IN AREA BUT GOTO\n";
+ if (get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) > get_dist_normalized(ball_pos_x, ball_pos_y, agent_posx, agent_posy) &&
+ get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) > get_dist_normalized(ball_pos_x, ball_pos_y, agent_posx, agent_posy)) {
+ return {GO_TO_BALL,2};
+ } else {
+ return {REDUCE_ANGLE_TO_GOAL, 0};
+ }
+
+ } else if ( get_dist_normalized (agent_posx, agent_posy, opp1_pos_x, opp1_pos_y) < tackle_limit
+ || get_dist_normalized (agent_posx, agent_posy, opp2_pos_x, opp2_pos_y) < tackle_limit ) { //#param
+ /*double turn_angle;
+ //REVISIT TURN ANGLE.. CONSIDER ACTUAL DIRECTION ALSO..
+ if (get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) < get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y)) {
+ turn_angle = atan2((HALF_FIELD_WIDTH/HALF_FIELD_LENGTH)*(opp1_pos_y), opp1_pos_x) - (agent_orientation*M_PI);
+ } else {
+ turn_angle = atan2((HALF_FIELD_WIDTH/HALF_FIELD_LENGTH)*(opp2_pos_y), opp2_pos_x) - (agent_orientation*M_PI);
+ }
+ turn_angle = atan2(tan(turn_angle),1);
+ if (turn_angle > M_PI ) {
+ turn_angle -= 2*M_PI;
+ } else if (turn_angle < -M_PI) {
+ turn_angle += 2*M_PI;
+ }*/
+ /*string s = "TACKLE " + std ::to_string (turn_angle) + "\n";
+ cout << s; */
+ //return {DASH, turn_angle*180/M_PI};
+ //return {TACKLE, turn_angle*180/M_PI}; //TACKLE needs power od dir
+ return {MOVE,0};
+ } else if ((!is_in_open_area(opp1_pos_x, opp1_pos_y) && is_in_open_area(opp2_pos_x, opp2_pos_y)) ||
+ (!is_in_open_area(opp2_pos_x, opp2_pos_y) && is_in_open_area(opp1_pos_x, opp1_pos_y))) {
+ return {REDUCE_ANGLE_TO_GOAL,0};
+ } else if (!is_in_open_area(opp1_pos_x, opp1_pos_y) && !is_in_open_area(opp2_pos_x, opp2_pos_y)) {
+ if (get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y) < get_dist_normalized(ball_pos_x, ball_pos_y, opp1_pos_x, opp1_pos_y)) {
+ return {MARK_PLAYER,opp2_unum};
+ } else {
+ return {MARK_PLAYER,opp1_unum};
+ }
+ } else {
+ std :: cout <<"Unknown Condition";
+ return {NOOP,0};
+ }
+ }
+
+}
+void read_params() {
+ std::ifstream fin("params.txt");
+ double d[6];
+ int i = 0;
+ while (true) {
+ fin >> d[i];
+ if( fin.eof() ) break;
+ std::cout << d[i] << std::endl;
+ i++;
+ //if (i >=6 ) {
+ // std::cout << "invalid params" << d[5];
+ // exit(0);
+ //}
+ }
+ fin.close();
+ kickable_dist = (d[0]+1)* 0.818175061;
+ open_area_up_limit_x = d[1];
+ open_area_up_limit_y = d[2];
+ open_area_low_limit_x = d[3];
+ open_area_low_limit_y = d[4];
+ tackle_limit = (d[5]+1)* 0.818175061;
+ std :: cout << "kickable dist " << kickable_dist << "tackle_limit" <<tackle_limit;
+ return;
+}
+
+void write_cost(double cost) {
+ std::ofstream myfile ("cost.txt");
+ if (myfile.is_open()) {
+ myfile << cost;
+ myfile.close();
+ }
+ return;
+}
+
+int main(int argc, char** argv) {
+ // Create the HFO environment
+ //read_params();
+ HFOEnvironment hfo;
+ int random = 0;
+ double numGoals = 0;
+ double numEpisodes = 5000;
+ // Connect to the server and request high-level feature set. See
+ // manual for more information on feature sets.
+ hfo.connectToServer(features, config_dir, port, server_addr,
+ team_name, goalie);
+ for (int episode=0; episode<numEpisodes; episode++) {
+ status_t status = IN_GAME;
+ while (status == IN_GAME) {
+
+ // Get the vector of state features for the current state
+ const vector<float>& feature_vec = hfo.getState();
+ if (random == 0) {
+ action_with_params a = get_defense_action(feature_vec, 2,1);
+ // std::cout << a.action << a.param;
+ if (a.action == hfo :: MARK_PLAYER || a.action == hfo::TACKLE) {
+ hfo.act(a.action, a.param);
+ } else if (a.action == hfo :: DASH) {
+ double power = 100;
+ hfo.act(a.action, power, a.param);
+ } else {
+ hfo.act(a.action);
+ }
+ string s = hfo::ActionToString(a.action) + " " +to_string(a.param) + "\n";
+ // std::cout << s;
+ } else {
+ std::cout <<"Randm";
+ action_t a = get_random_high_lv_action();
+ if (a == hfo :: MARK_PLAYER) {
+ hfo.act(NOOP);
+ } else {
+ hfo.act(a);
+ }
+ }
+ //hfo.act(hfo::INTERCEPT);
+ status = hfo.step();
+ }
+ if (status==GOAL)
+ numGoals = numGoals+1;
+ // Check what the outcome of the episode was
+ cout << "Episode " << episode << " ended with status: "
+ << StatusToString(status) << std::endl;
+ }
+ double cost = numGoals/numEpisodes;
+ hfo.act(QUIT);
+ //write_cost(cost);
+};
+
diff --git a/example/make_hand_coded_defense_agent_executable.sh b/example/make_hand_coded_defense_agent_executable.sh
new file mode 100755
index 0000000..d28b8d2
--- /dev/null
+++ b/example/make_hand_coded_defense_agent_executable.sh
@@ -0,0 +1,3 @@
+# Run this file to create an executable of hand_coded_defense_agent.cpp
+g++ -c hand_coded_defense_agent.cpp -I ../src/ -std=c++0x -pthread
+g++ -L ../lib/ hand_coded_defense_agent.o -lhfo -pthread -o hand_coded_defense_agent -Wl,-rpath,../lib
diff --git a/example/sarsa/Makefile b/example/sarsa/Makefile
deleted file mode 100644
index a9ce73f..0000000
--- a/example/sarsa/Makefile
+++ /dev/null
@@ -1,58 +0,0 @@
-#Directories
-FA_DIR = ./funcapprox
-POLICY_DIR = ./policy
-HFO_SRC_DIR = ../../src
-HFO_LIB_DIR = ../../lib
-
-#Includes
-INCLUDES = -I$(FA_DIR) -I$(POLICY_DIR) -I$(HFO_SRC_DIR)
-
-#Libs
-FA_LIB = funcapprox
-POLICY_LIB = policyagent
-
-#Flags
-CXXFLAGS = -g -Wall -std=c++11 -pthread
-LDFLAGS = -l$(FA_LIB) -l$(POLICY_LIB) -lhfo -pthread
-LDLIBS = -L$(FA_DIR) -L$(POLICY_DIR) -L$(HFO_LIB_DIR)
-LINKEROPTIONS = -Wl,-rpath,$(HFO_LIB_DIR)
-
-#Compiler
-CXX = g++
-
-#Sources
-SRC = high_level_sarsa_agent.cpp
-
-#Objects
-OBJ = $(SRC:.cpp=.o)
-
-#Target
-TARGET = high_level_sarsa_agent
-
-#Rules
-
-.PHONY: $(FA_LIB)
-
-all: $(TARGET)
-
-.cpp.o:
- $(CXX) $(CXXFLAGS) $(INCLUDES) -c -o $@ $(@F:%.o=%.cpp)
-
-$(FA_LIB):
- $(MAKE) -C $(FA_DIR)
-
-$(POLICY_LIB):
- $(MAKE) -C $(POLICY_DIR)
-
-$(TARGET): $(FA_LIB) $(POLICY_LIB) $(OBJ)
- $(CXX) $(OBJ) $(CXXFLAGS) $(LDLIBS) $(LDFLAGS) -o $(TARGET) $(LINKEROPTIONS)
-
-cleanfa:
- $(MAKE) clean -C $(FA_DIR)
-
-cleanpolicy:
- $(MAKE) clean -C $(POLICY_DIR)
-
-clean: cleanfa cleanpolicy
- rm -f $(TARGET) $(OBJ) *~
-
diff --git a/example/sarsa/funcapprox/Makefile b/example/sarsa/funcapprox/Makefile
deleted file mode 100644
index e0e66c9..0000000
--- a/example/sarsa/funcapprox/Makefile
+++ /dev/null
@@ -1,28 +0,0 @@
-#Flags
-CXXFLAGS = -g -O3 -Wall
-
-#Compiler
-CXX = g++
-
-#Sources
-SRCS = FuncApprox.cpp tiles2.cpp CMAC.cpp
-
-#Objects
-OBJS = $(SRCS:.cpp=.o)
-
-#Target
-TARGET = libfuncapprox.a
-
-#Rules
-
-all: $(TARGET)
-
-.cpp.o:
- $(CXX) $(CXXFLAGS) -c -o $@ $(@F:%.o=%.cpp)
-
-$(TARGET): $(OBJS)
- ar cq $@ $(OBJS)
-
-clean:
- rm -f $(TARGET) $(OBJS) *~
-
diff --git a/example/sarsa/policy/Makefile b/example/sarsa/policy/Makefile
deleted file mode 100644
index c74757a..0000000
--- a/example/sarsa/policy/Makefile
+++ /dev/null
@@ -1,34 +0,0 @@
-#Directories
-FA_DIR = ../funcapprox
-
-#Includes
-INCLUDES = -I$(FA_DIR)
-
-#Flags
-CXXFLAGS = -g -O3 -Wall
-
-#Compiler
-CXX = g++
-
-#Sources
-SRCS = PolicyAgent.cpp SarsaAgent.cpp
-
-#Objects
-OBJS = $(SRCS:.cpp=.o)
-
-#Target
-TARGET = libpolicyagent.a
-
-#Rules
-
-all: $(TARGET)
-
-.cpp.o:
- $(CXX) $(CXXFLAGS) $(INCLUDES) -c -o $@ $(@F:%.o=%.cpp)
-
-$(TARGET): $(OBJS)
- ar cq $@ $(OBJS)
-
-clean:
- rm -f $(TARGET) $(OBJS) *~
-
diff --git a/example/sarsa_defense/high_level_sarsa_defense_agent.cpp b/example/sarsa_defense/high_level_sarsa_defense_agent.cpp
new file mode 100644
index 0000000..69059f1
--- /dev/null
+++ b/example/sarsa_defense/high_level_sarsa_defense_agent.cpp
@@ -0,0 +1,257 @@
+#include <iostream>
+#include <vector>
+#include <HFO.hpp>
+#include <cstdlib>
+#include <thread>
+#include "SarsaAgent.h"
+#include "CMAC.h"
+#include <unistd.h>
+
+// Before running this program, first Start HFO server:
+// $./bin/HFO --offense-agents numAgents
+
+void printUsage() {
+ std::cout<<"Usage:123 ./high_level_sarsa_agent [Options]"<<std::endl;
+ std::cout<<"Options:"<<std::endl;
+ std::cout<<" --numAgents <int> Number of SARSA agents"<<std::endl;
+ std::cout<<" Default: 0"<<std::endl;
+ std::cout<<" --numEpisodes <int> Number of episodes to run"<<std::endl;
+ std::cout<<" Default: 10"<<std::endl;
+ std::cout<<" --basePort <int> SARSA agent base port"<<std::endl;
+ std::cout<<" Default: 6000"<<std::endl;
+ std::cout<<" --learnRate <float> Learning rate of SARSA agents"<<std::endl;
+ std::cout<<" Range: [0.0, 1.0]"<<std::endl;
+ std::cout<<" Default: 0.1"<<std::endl;
+ std::cout<<" --suffix <int> Suffix for weights files"<<std::endl;
+ std::cout<<" Default: 0"<<std::endl;
+ std::cout<<" --noOpponent Sets opponent present flag to false"<<std::endl;
+ std::cout<<" --step Sets the persistent step size"<<std::endl;
+ std::cout<<" --eps Sets the exploration rate"<<std::endl;
+ std::cout<<" --numOpponents Sets the number of opponents"<<std::endl;
+ std::cout<<" --weightId Sets the given Id for weight File"<<std::endl;
+ std::cout<<" --help Displays this help and exit"<<std::endl;
+}
+
+// Returns the reward for SARSA based on current state
+double getReward(hfo::status_t status) {
+ double reward;
+ if (status==hfo::GOAL) reward = -1;
+ else if (status==hfo::CAPTURED_BY_DEFENSE) reward = 1;
+ else if (status==hfo::OUT_OF_BOUNDS) reward = 1;
+ else reward = 0;
+ return reward;
+}
+
+// Fill state with only the required features from state_vec
+void purgeFeatures(double *state, const std::vector<float>& state_vec,
+ int numTMates, int numOpponents, bool oppPres) {
+
+ int stateIndex = 0;
+
+ // If no opponents ignore features Distance to Opponent
+ // and Distance from Teammate i to Opponent are absent
+ int tmpIndex = oppPres ? (9 + 3 * numTMates) : (9 + 2 * numTMates);
+
+ for(int i = 0; i < state_vec.size(); i++) {
+
+ // Ignore first six featues
+ if(i == 5 || i==8) continue;
+ if(i>9 && i<= 9+numTMates) continue; // Ignore Goal Opening angles, as invalid
+ if(i<= 9+3*numTMates && i > 9+2*numTMates) continue; // Ignore Pass Opening angles, as invalid
+ // Ignore Uniform Number of Teammates and opponents
+ int temp = i-tmpIndex;
+ if(temp > 0 && (temp % 3 == 0) )continue;
+ //if (i > 9+6*numTMates) continue;
+ state[stateIndex] = state_vec[i];
+ stateIndex++;
+ }
+}
+
+// Convert int to hfo::Action
+hfo::action_t toAction(int action, const std::vector<float>& state_vec) {
+hfo::action_t a;
+ switch (action) {
+ case 0: a = hfo::MOVE; break;
+ case 1: a = hfo::REDUCE_ANGLE_TO_GOAL; break;
+ case 2: a = hfo::GO_TO_BALL; break;
+ case 3: a = hfo::NOOP; break;
+ case 4: a = hfo::DEFEND_GOAL; break;
+ default : a = hfo::MARK_PLAYER; break;
+ }
+ return a;
+}
+
+void offenseAgent(int port, int numTMates, int numOpponents, int numEpi, double learnR,
+ int suffix, bool oppPres, double eps, int step, std::string weightid) {
+
+ // Number of features
+ int numF = oppPres ? (8 + 3 * numTMates + 2*numOpponents) : (3 + 3 * numTMates);
+ // Number of actions
+ int numA = 5 + numOpponents; //DEF_GOAL+MOVE+GTB+NOOP+RATG+MP(unum)
+
+ // Other SARSA parameters
+ eps = 0.01;
+ double discFac = 1;
+ double lambda=0.9375;
+ // Tile coding parameter
+ double resolution = 0.1;
+
+ double range[numF];
+ double min[numF];
+ double res[numF];
+ for(int i = 0; i < numF; i++) {
+ min[i] = -1;
+ range[i] = 2;
+ res[i] = resolution;
+ }
+
+ // Weights file
+ char *wtFile;
+ std::string s = "weights_" + std::to_string(port) +
+ "_" + std::to_string(numTMates + 1) +
+ "_" + std::to_string(suffix) +
+ "_" + std::to_string(step) +
+ "_" + weightid;
+ wtFile = &s[0u];
+
+ CMAC *fa = new CMAC(numF, numA, range, min, res);
+ SarsaAgent *sa = new SarsaAgent(numF, numA, learnR, eps, lambda, fa, wtFile, wtFile);
+
+ hfo::HFOEnvironment hfo;
+ hfo::status_t status;
+ hfo::action_t a;
+ double state[numF];
+ int action = -1;
+ double reward;
+ int no_of_offense = numTMates + 1;
+ hfo.connectToServer(hfo::HIGH_LEVEL_FEATURE_SET,"../../bin/teams/base/config/formations-dt",port,"localhost","base_right",false,"");
+
+
+ for (int episode=0; episode < numEpi; episode++) {
+ int count = 0;
+ status = hfo::IN_GAME;
+ action = -1;
+ int count_steps = 0;
+ double unum = -1;
+ const std::vector<float>& state_vec = hfo.getState();
+ int num_steps_per_epi = 0;
+ while (status == hfo::IN_GAME) {
+ num_steps_per_epi++;
+ //std::cout << "::::::"<< hfo::ActionToString(a) <<" "<<count_steps <<std::endl;
+ if (count_steps != step && action >=0 && (a != hfo :: MARK_PLAYER || unum>0)) {
+ count_steps ++;
+ if (a == hfo::MARK_PLAYER) {
+ hfo.act(a,unum);
+ //std::cout << "MARKING" << unum <<"\n";
+ } else {
+ hfo.act(a);
+ }
+ status = hfo.step();
+ continue;
+
+ } else {
+ count_steps = 0;
+ }
+
+ if(action != -1) {
+ reward = getReward(status);
+ sa->update(state, action, reward, discFac);
+ }
+
+ // Fill up state array
+ purgeFeatures(state, state_vec, numTMates, numOpponents, oppPres);
+
+ // Get raw action
+ action = sa->selectAction(state);
+
+ // Get hfo::Action
+ a = toAction(action, state_vec);
+ if (a== hfo::MARK_PLAYER) {
+ unum = state_vec[(state_vec.size()-1 - (action-5)*3)];
+ hfo.act(a,unum);
+ } else {
+ hfo.act(a);
+ }
+ std::string s = std::to_string(action);
+ for (int state_vec_fc=0; state_vec_fc < state_vec.size(); state_vec_fc++) {
+ s+=std::to_string(state_vec[state_vec_fc]) + ",";
+ }
+ s+="UNUM" +std::to_string(unum) +"\n";;
+ status = hfo.step();
+ // std::cout <<s;
+ }
+ //std :: cout <<":::::::::::::" << num_steps_per_epi<< " "<<step << " "<<"\n";
+ // End of episode
+ if(action != -1) {
+ reward = getReward(status);
+ sa->update(state, action, reward, discFac);
+ sa->endEpisode();
+ }
+ }
+
+ delete sa;
+ delete fa;
+}
+
+int main(int argc, char **argv) {
+
+ int numAgents = 0;
+ int numEpisodes = 10;
+ int basePort = 6000;
+ double learnR = 0.1;
+ int suffix = 0;
+ bool opponentPresent = true;
+ int numOpponents = 0;
+ double eps = 0.01;
+ int step = 10;
+ std::string weightid;
+ for (int i = 0; i<argc; i++) {
+ std::string param = std::string(argv[i]);
+ std::cout<<param<<"\n";
+ }
+ for(int i = 1; i < argc; i++) {
+ std::string param = std::string(argv[i]);
+ if(param == "--numAgents") {
+ numAgents = atoi(argv[++i]);
+ }else if(param == "--numEpisodes") {
+ numEpisodes = atoi(argv[++i]);
+ }else if(param == "--basePort") {
+ basePort = atoi(argv[++i]);
+ }else if(param == "--learnRate") {
+ learnR = atof(argv[++i]);
+ if(learnR < 0 || learnR > 1) {
+ printUsage();
+ return 0;
+ }
+ }else if(param == "--suffix") {
+ suffix = atoi(argv[++i]);
+ }else if(param == "--noOpponent") {
+ opponentPresent = false;
+ }else if(param=="--eps"){
+ eps=atoi(argv[++i]);
+ }else if(param=="--numOpponents"){
+ numOpponents=atoi(argv[++i]);
+ }else if(param=="--step"){
+ step=atoi(argv[++i]);
+ }else if(param=="--weightId"){
+ weightid=std::string(argv[++i]);
+ }else {
+ printUsage();
+ return 0;
+ }
+ }
+ int numTeammates = numAgents; //using goalie npc
+ std::thread agentThreads[numAgents];
+ for (int agent = 0; agent < numAgents; agent++) {
+ agentThreads[agent] = std::thread(offenseAgent, basePort + agent,
+ numTeammates, numOpponents, numEpisodes, learnR,
+ suffix, opponentPresent, eps, step, weightid);
+ usleep(500000L);
+ }
+ for (int agent = 0; agent < numAgents; agent++) {
+ agentThreads[agent].join();
+ }
+ return 0;
+}
+
+
diff --git a/example/sarsa/funcapprox/CMAC.cpp b/example/sarsa_libraries/funcapprox/CMAC.cpp
similarity index 100%
rename from example/sarsa/funcapprox/CMAC.cpp
rename to example/sarsa_libraries/funcapprox/CMAC.cpp
diff --git a/example/sarsa/funcapprox/CMAC.h b/example/sarsa_libraries/funcapprox/CMAC.h
similarity index 100%
rename from example/sarsa/funcapprox/CMAC.h
rename to example/sarsa_libraries/funcapprox/CMAC.h
diff --git a/example/sarsa/funcapprox/FuncApprox.cpp b/example/sarsa_libraries/funcapprox/FuncApprox.cpp
similarity index 100%
rename from example/sarsa/funcapprox/FuncApprox.cpp
rename to example/sarsa_libraries/funcapprox/FuncApprox.cpp
diff --git a/example/sarsa/funcapprox/FuncApprox.h b/example/sarsa_libraries/funcapprox/FuncApprox.h
similarity index 100%
rename from example/sarsa/funcapprox/FuncApprox.h
rename to example/sarsa_libraries/funcapprox/FuncApprox.h
diff --git a/example/sarsa/funcapprox/tiles2.cpp b/example/sarsa_libraries/funcapprox/tiles2.cpp
similarity index 100%
rename from example/sarsa/funcapprox/tiles2.cpp
rename to example/sarsa_libraries/funcapprox/tiles2.cpp
diff --git a/example/sarsa/funcapprox/tiles2.h b/example/sarsa_libraries/funcapprox/tiles2.h
similarity index 100%
rename from example/sarsa/funcapprox/tiles2.h
rename to example/sarsa_libraries/funcapprox/tiles2.h
diff --git a/example/sarsa/policy/PolicyAgent.cpp b/example/sarsa_libraries/policy/PolicyAgent.cpp
similarity index 100%
rename from example/sarsa/policy/PolicyAgent.cpp
rename to example/sarsa_libraries/policy/PolicyAgent.cpp
diff --git a/example/sarsa/policy/PolicyAgent.h b/example/sarsa_libraries/policy/PolicyAgent.h
similarity index 100%
rename from example/sarsa/policy/PolicyAgent.h
rename to example/sarsa_libraries/policy/PolicyAgent.h
diff --git a/example/sarsa/policy/SarsaAgent.cpp b/example/sarsa_libraries/policy/SarsaAgent.cpp
similarity index 84%
rename from example/sarsa/policy/SarsaAgent.cpp
rename to example/sarsa_libraries/policy/SarsaAgent.cpp
index 0e5ec60..2dc0dc6 100644
--- a/example/sarsa/policy/SarsaAgent.cpp
+++ b/example/sarsa_libraries/policy/SarsaAgent.cpp
@@ -1,10 +1,11 @@
#include "SarsaAgent.h"
-SarsaAgent::SarsaAgent(int numFeatures, int numActions, double learningRate, double epsilon, FunctionApproximator *FA, char *loadWeightsFile, char *saveWeightsFile):PolicyAgent(numFeatures, numActions, learningRate, epsilon, FA, loadWeightsFile, saveWeightsFile){
-
+//add lambda as parameter to sarsaagent
+SarsaAgent::SarsaAgent(int numFeatures, int numActions, double learningRate, double epsilon, double lambda, FunctionApproximator *FA, char *loadWeightsFile, char *saveWeightsFile):PolicyAgent(numFeatures, numActions, learningRate, epsilon, FA, loadWeightsFile, saveWeightsFile){
+ this->lambda = lambda;
episodeNumber = 0;
lastAction = -1;
-
+ //have memory for lambda
}
void SarsaAgent::update(double state[], int action, double reward, double discountFactor){
@@ -34,7 +35,7 @@ void SarsaAgent::update(double state[], int action, double reward, double discou
FA->updateWeights(delta, learningRate);
//Assume gamma, lambda are 0.
- FA->decayTraces(0);
+ FA->decayTraces(discountFactor*lambda);//replace 0 with gamma*lambda
for(int i = 0; i < getNumFeatures(); i++){
lastState[i] = state[i];
@@ -59,8 +60,8 @@ void SarsaAgent::endEpisode(){
double delta = lastReward - oldQ;
FA->updateWeights(delta, learningRate);
- //Assume lambda is 0.
- FA->decayTraces(0);
+ //Assume lambda is 0. this comment looks wrong.
+ FA->decayTraces(0);//remains 0
}
if(toSaveWeights && (episodeNumber + 1) % 5 == 0){
diff --git a/example/sarsa/policy/SarsaAgent.h b/example/sarsa_libraries/policy/SarsaAgent.h
similarity index 81%
rename from example/sarsa/policy/SarsaAgent.h
rename to example/sarsa_libraries/policy/SarsaAgent.h
index b6bd6f4..7197037 100644
--- a/example/sarsa/policy/SarsaAgent.h
+++ b/example/sarsa_libraries/policy/SarsaAgent.h
@@ -12,10 +12,11 @@ class SarsaAgent:public PolicyAgent{
double lastState[MAX_STATE_VARS];
int lastAction;
double lastReward;
+ double lambda;
public:
- SarsaAgent(int numFeatures, int numActions, double learningRate, double epsilon, FunctionApproximator *FA, char *loadWeightsFile, char *saveWeightsFile);
+ SarsaAgent(int numFeatures, int numActions, double learningRate, double epsilon, double lambda, FunctionApproximator *FA, char *loadWeightsFile, char *saveWeightsFile);
int argmaxQ(double state[]);
double computeQ(double state[], int action);
diff --git a/example/sarsa/high_level_sarsa_agent.cpp b/example/sarsa_offense/high_level_sarsa_agent.cpp
similarity index 97%
rename from example/sarsa/high_level_sarsa_agent.cpp
rename to example/sarsa_offense/high_level_sarsa_agent.cpp
index 5e3df93..4bfe4f7 100644
--- a/example/sarsa/high_level_sarsa_agent.cpp
+++ b/example/sarsa_offense/high_level_sarsa_agent.cpp
@@ -56,7 +56,7 @@ void purgeFeatures(double *state, const std::vector<float>& state_vec,
// Ignore Angle and Uniform Number of Teammates
int temp = i-tmpIndex;
if(temp > 0 && (temp % 3 == 2 || temp % 3 == 0)) continue;
-
+ if (i > 9+6*numTMates) continue;
state[stateIndex] = state_vec[i];
stateIndex++;
}
@@ -107,9 +107,9 @@ void offenseAgent(int port, int numTMates, int numEpi, double learnR,
"_" + std::to_string(numTMates + 1) +
"_" + std::to_string(suffix);
wtFile = &s[0u];
-
+ double lambda = 0;
CMAC *fa = new CMAC(numF, numA, range, min, res);
- SarsaAgent *sa = new SarsaAgent(numF, numA, learnR, eps, fa, wtFile, wtFile);
+ SarsaAgent *sa = new SarsaAgent(numF, numA, learnR, eps, lambda, fa, wtFile, wtFile);
hfo::HFOEnvironment hfo;
hfo::status_t status;
--
2.24.1