Commit 19460175 authored by Matthew Hausknecht's avatar Matthew Hausknecht

Refactored high-level state features, adding opponent features.

parent 388df450
No preview for this file type
...@@ -273,9 +273,9 @@ are encoded a floating point values and normalized to the range of ...@@ -273,9 +273,9 @@ are encoded a floating point values and normalized to the range of
follows: follows:
\subsubsection{High Level State Feature List} \subsubsection{High Level State Feature List}
Let $T$ denote the number of teammates in the HFO game. Then there are Let $T$ denote the number of teammates in the HFO game and $O$ the
a total of $9 + 5T$ high-level features with an additional $T+1$ number of opponents. There are a total of $10 + 6T + 3O$ high-level
features if at least one opponent is present. features.
\begin{enumerate}[noitemsep] \begin{enumerate}[noitemsep]
\setcounter{enumi}{-1} \setcounter{enumi}{-1}
...@@ -292,17 +292,16 @@ features if at least one opponent is present. ...@@ -292,17 +292,16 @@ features if at least one opponent is present.
\item{\textbf{Goal Opening Angle} - The size of the largest open angle \item{\textbf{Goal Opening Angle} - The size of the largest open angle
of the agent to the goal, shown as $\theta_g$ in Figure of the agent to the goal, shown as $\theta_g$ in Figure
\ref{fig:openAngle}. Invalid if agent is not playing offense.} \ref{fig:openAngle}. Invalid if agent is not playing offense.}
\item [$T$] {\textbf{Teammate i's Goal Opening Angle} - For each \item{\textbf{Proximity to Opponent} - If an opponent is present,
teammate i: i’s goal opening angle. Invalid if agent is not playing proximity to the closest opponent. Invalid if there are no
offense.} opponents.}
\item [$1$] {\textbf{Proximity to Opponent} - If an opponent is \item [$T$] {\textbf{Teammate's Goal Opening Angle} - For each
present, proximity to the closest opponent. This feature is absent teammate $i$: $i$’s goal opening angle. Invalid if agent is not
if there are no opponents.} playing offense.}
\item [$T$] {\textbf{Proximity from Teammate i to Opponent} - For each \item [$T$] {\textbf{Proximity from Teammate i to Opponent} - For each
teammate i: the proximity from the teammate to the closest teammate i: the proximity from the teammate to the closest
opponent. This feature is absent if there are no opponents. If opponent. This feature is invalid if there are no opponents or if
teammates are present but not detected, this feature is considered teammates are present but not detected.}
invalid and given the value of -2.}
\item [$T$] {\textbf{Pass Opening Angle} - For each teammate i: the open \item [$T$] {\textbf{Pass Opening Angle} - For each teammate i: the open
angle available to pass to teammate i. Shown as $\theta_p$ in Figure angle available to pass to teammate i. Shown as $\theta_p$ in Figure
\ref{fig:openAngle}. If teammates are present but not detected, this \ref{fig:openAngle}. If teammates are present but not detected, this
...@@ -310,6 +309,9 @@ features if at least one opponent is present. ...@@ -310,6 +309,9 @@ features if at least one opponent is present.
\item [$3T$] {\textbf{Proximity, Angle, and Uniform Number of \item [$3T$] {\textbf{Proximity, Angle, and Uniform Number of
Teammates} - For each teammate i: the proximity, angle, and Teammates} - For each teammate i: the proximity, angle, and
uniform number of that teammate.} uniform number of that teammate.}
\item [$3O$] {\textbf{Proximity, Angle, and Uniform Number of
Opponents} - For each opponent: the proximity, angle, and
uniform number of that opponent.}
\end{enumerate} \end{enumerate}
\begin{figure}[htp] \begin{figure}[htp]
......
...@@ -137,19 +137,17 @@ status_t HFOEnvironment::step() { ...@@ -137,19 +137,17 @@ status_t HFOEnvironment::step() {
// Update the state features // Update the state features
agent->preAction(); agent->preAction();
assert(agent->currentTime().cycle() == (current_cycle + 1)); assert(agent->currentTime().cycle() == (current_cycle + 1));
current_cycle = agent->currentTime().cycle(); current_cycle = agent->currentTime().cycle();
status_t status = agent->getGameStatus(); status_t status = agent->getGameStatus();
// If the episode is over, take three NOOPs to refresh state features // If the episode is over, take three NOOPs to refresh state features
if (status != IN_GAME && status != SERVER_DOWN) { if (status != IN_GAME && status != SERVER_DOWN) {
for (int i = 0; i < 3; ++i) { for (int i = 0; i < 3; ++i) {
act(NOOP); act(NOOP);
step(); if (step() == SERVER_DOWN) {
return SERVER_DOWN;
}
} }
} }
return status; return status;
} }
...@@ -14,11 +14,8 @@ HighLevelFeatureExtractor::HighLevelFeatureExtractor(int num_teammates, ...@@ -14,11 +14,8 @@ HighLevelFeatureExtractor::HighLevelFeatureExtractor(int num_teammates,
{ {
assert(numTeammates >= 0); assert(numTeammates >= 0);
assert(numOpponents >= 0); assert(numOpponents >= 0);
numFeatures = num_basic_features + features_per_teammate * numTeammates; numFeatures = num_basic_features + features_per_teammate * numTeammates
if (numOpponents > 0) { + features_per_opponent * numOpponents;
// One extra basic feature and one feature per teammate
numFeatures += 1 + numTeammates;
}
feature_vec.resize(numFeatures); feature_vec.resize(numFeatures);
} }
...@@ -32,6 +29,7 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -32,6 +29,7 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
const Vector2D& self_pos = self.pos(); const Vector2D& self_pos = self.pos();
const float self_ang = self.body().radian(); const float self_ang = self.body().radian();
const PlayerCont& teammates = wm.teammates(); const PlayerCont& teammates = wm.teammates();
const PlayerCont& opponents = wm.opponents();
float maxR = sqrtf(SP.pitchHalfLength() * SP.pitchHalfLength() float maxR = sqrtf(SP.pitchHalfLength() * SP.pitchHalfLength()
+ SP.pitchHalfWidth() * SP.pitchHalfWidth()); + SP.pitchHalfWidth() * SP.pitchHalfWidth());
// features about self pos // features about self pos
...@@ -44,49 +42,52 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -44,49 +42,52 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
} else { } else {
addNormFeature(self_pos.x, -SP.pitchHalfLength()-tolerance_x, tolerance_x); addNormFeature(self_pos.x, -SP.pitchHalfLength()-tolerance_x, tolerance_x);
} }
// addFeature(self_pos.x);
// Feature[1]: Y-Position // Feature[1]: Y-Position
addNormFeature(self_pos.y, -SP.pitchHalfWidth() - tolerance_y, addNormFeature(self_pos.y, -SP.pitchHalfWidth() - tolerance_y,
SP.pitchHalfWidth() + tolerance_y); SP.pitchHalfWidth() + tolerance_y);
// addFeature(self_pos.y);
// Feature[2]: Self Angle // Feature[2]: Self Angle
addNormFeature(self_ang, -M_PI, M_PI); // addFeature(self_ang); addNormFeature(self_ang, -M_PI, M_PI);
float r; float r;
float th; float th;
// features about the ball // Features about the ball
Vector2D ball_pos = wm.ball().pos(); Vector2D ball_pos = wm.ball().pos();
angleDistToPoint(self_pos, ball_pos, th, r); angleDistToPoint(self_pos, ball_pos, th, r);
// Feature[3]: Dist to ball // Feature[3]: Dist to ball
addNormFeature(r, 0, maxR); // addFeature(r); addNormFeature(r, 0, maxR);
// Feature[4]: Ang to ball // Feature[4]: Ang to ball
addNormFeature(th, -M_PI, M_PI); // addFeature(th); addNormFeature(th, -M_PI, M_PI);
// Feature[5]: Able to kick // Feature[5]: Able to kick
addNormFeature(self.isKickable(), false, true); // addFeature(self.isKickable()); addNormFeature(self.isKickable(), false, true);
// features about distance to goal center // Features about distance to goal center
Vector2D goalCenter(SP.pitchHalfLength(), 0); Vector2D goalCenter(SP.pitchHalfLength(), 0);
if (!playingOffense) { if (!playingOffense) {
goalCenter.assign(-SP.pitchHalfLength(), 0); goalCenter.assign(-SP.pitchHalfLength(), 0);
} }
angleDistToPoint(self_pos, goalCenter, th, r); angleDistToPoint(self_pos, goalCenter, th, r);
// Feature[6]: Goal Center Distance // Feature[6]: Goal Center Distance
addNormFeature(r, 0, maxR); // addFeature(r); addNormFeature(r, 0, maxR);
// Feature[7]: Angle to goal center // Feature[7]: Angle to goal center
addNormFeature(th, -M_PI, M_PI); // addFeature(th); addNormFeature(th, -M_PI, M_PI);
// Feature[8]: largest open goal angle // Feature[8]: largest open goal angle
addNormFeature(calcLargestGoalAngle(wm, self_pos), 0, M_PI); addNormFeature(calcLargestGoalAngle(wm, self_pos), 0, M_PI);
// addFeature(calcLargestGoalAngle(wm, self_pos)); // Feature[9]: Dist to our closest opp
if (numOpponents > 0) {
calcClosestOpp(wm, self_pos, th, r);
addNormFeature(r, 0, maxR);
} else {
addFeature(FEAT_INVALID);
}
// Features [9+T - 9 + 2T]: teammate's open angle to goal // Features[9 - 9+T]: teammate's open angle to goal
int detected_teammates = 0; int detected_teammates = 0;
for (PlayerCont::const_iterator it=teammates.begin(); it != teammates.end(); ++it) { for (PlayerCont::const_iterator it=teammates.begin(); it != teammates.end(); ++it) {
const PlayerObject& teammate = *it; const PlayerObject& teammate = *it;
if (valid(teammate) && detected_teammates < numTeammates) { if (valid(teammate) && detected_teammates < numTeammates) {
addNormFeature(calcLargestGoalAngle(wm, teammate.pos()), 0, M_PI); addNormFeature(calcLargestGoalAngle(wm, teammate.pos()), 0, M_PI);
// addFeature(calcLargestGoalAngle(wm, teammate.pos()));
detected_teammates++; detected_teammates++;
} }
} }
...@@ -95,18 +96,14 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -95,18 +96,14 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
addFeature(FEAT_INVALID); addFeature(FEAT_INVALID);
} }
// dist to our closest opp // Features[9+T - 9+2T]: teammates' dists to closest opps
if (numOpponents > 0) { if (numOpponents > 0) {
calcClosestOpp(wm, self_pos, th, r);
addNormFeature(r, 0, maxR); // addFeature(r);
// teammates dists to closest opps
detected_teammates = 0; detected_teammates = 0;
for (PlayerCont::const_iterator it=teammates.begin(); it != teammates.end(); ++it) { for (PlayerCont::const_iterator it=teammates.begin(); it != teammates.end(); ++it) {
const PlayerObject& teammate = *it; const PlayerObject& teammate = *it;
if (valid(teammate) && detected_teammates < numTeammates) { if (valid(teammate) && detected_teammates < numTeammates) {
calcClosestOpp(wm, teammate.pos(), th, r); calcClosestOpp(wm, teammate.pos(), th, r);
addNormFeature(r, 0, maxR); // addFeature(r); addNormFeature(r, 0, maxR);
detected_teammates++; detected_teammates++;
} }
} }
...@@ -114,6 +111,10 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -114,6 +111,10 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
for (int i=detected_teammates; i<numTeammates; ++i) { for (int i=detected_teammates; i<numTeammates; ++i) {
addFeature(FEAT_INVALID); addFeature(FEAT_INVALID);
} }
} else { // If no opponents, add invalid features
for (int i=0; i<numTeammates; ++i) {
addFeature(FEAT_INVALID);
}
} }
// Features [9+2T - 9+3T]: open angle to teammates // Features [9+2T - 9+3T]: open angle to teammates
...@@ -122,7 +123,6 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -122,7 +123,6 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
const PlayerObject& teammate = *it; const PlayerObject& teammate = *it;
if (valid(teammate) && detected_teammates < numTeammates) { if (valid(teammate) && detected_teammates < numTeammates) {
addNormFeature(calcLargestTeammateAngle(wm, self_pos, teammate.pos()),0,M_PI); addNormFeature(calcLargestTeammateAngle(wm, self_pos, teammate.pos()),0,M_PI);
// addFeature(calcLargestTeammateAngle(wm, self_pos, teammate.pos()));
detected_teammates++; detected_teammates++;
} }
} }
...@@ -131,14 +131,14 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -131,14 +131,14 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
addFeature(FEAT_INVALID); addFeature(FEAT_INVALID);
} }
// Features [9+3T - end]: dist, angle, unum of teammates // Features [9+3T - 9+6T]: dist, angle, unum of teammates
detected_teammates = 0; detected_teammates = 0;
for (PlayerCont::const_iterator it=teammates.begin(); it != teammates.end(); ++it) { for (PlayerCont::const_iterator it=teammates.begin(); it != teammates.end(); ++it) {
const PlayerObject& teammate = *it; const PlayerObject& teammate = *it;
if (valid(teammate) && detected_teammates < numTeammates) { if (valid(teammate) && detected_teammates < numTeammates) {
angleDistToPoint(self_pos, teammate.pos(), th, r); angleDistToPoint(self_pos, teammate.pos(), th, r);
addNormFeature(r,0,maxR); // addFeature(r); addNormFeature(r,0,maxR);
addNormFeature(th,-M_PI,M_PI); // addFeature(th); addNormFeature(th,-M_PI,M_PI);
addFeature(teammate.unum()); addFeature(teammate.unum());
detected_teammates++; detected_teammates++;
} }
...@@ -150,8 +150,26 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures( ...@@ -150,8 +150,26 @@ const std::vector<float>& HighLevelFeatureExtractor::ExtractFeatures(
addFeature(FEAT_INVALID); addFeature(FEAT_INVALID);
} }
// Features [9+6T - 9+6T+3O]: dist, angle, unum of opponents
int detected_opponents = 0;
for (PlayerCont::const_iterator it = opponents.begin(); it != opponents.end(); ++it) {
const PlayerObject& opponent = *it;
if (valid(opponent) && detected_opponents < numOpponents) {
angleDistToPoint(self_pos, opponent.pos(), th, r);
addNormFeature(r,0,maxR);
addNormFeature(th,-M_PI,M_PI);
addFeature(opponent.unum());
detected_opponents++;
}
}
// Add zero features for any missing opponents
for (int i=detected_opponents; i<numOpponents; ++i) {
addFeature(FEAT_INVALID);
addFeature(FEAT_INVALID);
addFeature(FEAT_INVALID);
}
assert(featIndx == numFeatures); assert(featIndx == numFeatures);
// checkFeatures(); // checkFeatures();
//std::cout << "features: " << features.rows(0,ind-1).t();
return feature_vec; return feature_vec;
} }
...@@ -22,9 +22,10 @@ public: ...@@ -22,9 +22,10 @@ public:
protected: protected:
// Number of features for non-player objects. // Number of features for non-player objects.
const static int num_basic_features = 9; const static int num_basic_features = 10;
// Number of features for each player or opponent in game. // Number of features for each teammate and opponent in game.
const static int features_per_teammate = 5; const static int features_per_teammate = 6;
const static int features_per_opponent = 3;
}; };
#endif // HIGHLEVEL_FEATURE_EXTRACTOR_H #endif // HIGHLEVEL_FEATURE_EXTRACTOR_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment