Refactored agent-environment interaction.

example/communication_agent.cpp

@@ -31,9 +31,10 @@ int main(int argc, char** argv) {
   hfo.connectToServer(features, config_dir, port, server_addr,
                       team_name, goalie);
   int unum = hfo.getUnum();
-  for (int episode=0; episode<10; episode++) {
-    int agent_on_ball = 7;
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
+    int agent_on_ball = 7;
     while (status == IN_GAME) {
       // Get the vector of state features for the current state
       const vector<float>& feature_vec = hfo.getState();
@@ -70,7 +71,7 @@ int main(int argc, char** argv) {
     }
     // Check what the outcome of the episode was
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;;
+         << StatusToString(status) << endl;;
   }
   hfo.act(QUIT);
 };

example/hfo_example_agent.cpp

@@ -24,9 +24,9 @@ int main() {
   // manual for more information on feature sets.
   hfo.connectToServer(features, config_dir, port, server_addr,
                       team_name, goalie);
-  // Play 5 episodes
-  for (int episode=0; episode<5; episode++) {
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
     while (status == IN_GAME) {
       // Get the vector of state features for the current state
       const std::vector<float>& feature_vec = hfo.getState();
@@ -37,7 +37,7 @@ int main() {
     }
     // Check what the outcome of the episode was
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;;
+         << StatusToString(status) << endl;
   }
   hfo.act(QUIT);
 };

example/high_level_random_agent.cpp

@@ -17,11 +17,8 @@ string server_addr = "localhost";
 string team_name = "base_left";
 bool goalie = false;
 
-// Returns a random high-level action
-action_t get_random_high_lv_action() {
-  action_t action_indx = (action_t) ((rand() % 5) + MOVE);
-  return action_indx;
-}
+// We omit PASS & CATCH actions here
+action_t HIGH_LEVEL_ACTIONS[3] = { MOVE, SHOOT, DRIBBLE };
 
 int main(int argc, char** argv) {
   // Create the HFO environment
@@ -30,19 +27,20 @@ int main(int argc, char** argv) {
   // manual for more information on feature sets.
   hfo.connectToServer(features, config_dir, port, server_addr,
                       team_name, goalie);
-  for (int episode=0; episode<10; episode++) {
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
     while (status == IN_GAME) {
       // Get the vector of state features for the current state
       const vector<float>& feature_vec = hfo.getState();
       // Perform the action
-      hfo.act(get_random_high_lv_action());
+      hfo.act(HIGH_LEVEL_ACTIONS[rand() % 3]);
       // Advance the environment and get the game status
       status = hfo.step();
     }
     // Check what the outcome of the episode was
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;
+         << StatusToString(status) << endl;
   }
   hfo.act(QUIT);
 };

example/low_level_random_agent.cpp

@@ -53,8 +53,9 @@ int main(int argc, char** argv) {
   // manual for more information on feature sets.
   hfo.connectToServer(features, config_dir, port, server_addr,
                       team_name, goalie);
-  for (int episode=0; episode<10; episode++) {
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
     while (status == IN_GAME) {
       // Get the vector of state features for the current state
       const vector<float>& feature_vec = hfo.getState();
@@ -64,7 +65,7 @@ int main(int argc, char** argv) {
       status = hfo.step();
     }
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;;
+         << StatusToString(status) << endl;;
   }
   hfo.act(QUIT);
 };

example/mid_level_dribble_agent.cpp

@@ -28,8 +28,9 @@ int main(int argc, char** argv) {
   // manual for more information on feature sets.
   hfo.connectToServer(features, config_dir, port, server_addr,
                            team_name, goalie);
-  for (int episode=0; episode<10; episode++) {
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
     int step = 0;
     while (status == IN_GAME) {
       // Get the vector of state features for the current state
@@ -44,7 +45,7 @@ int main(int argc, char** argv) {
     }
     // Check what the outcome of the episode was
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;;
+         << StatusToString(status) << endl;;
   }
   hfo.act(QUIT);
 };

example/mid_level_kick_agent.cpp

@@ -26,8 +26,9 @@ int main(int argc, char** argv) {
   // manual for more information on feature sets.
   hfo.connectToServer(features, config_dir, port, server_addr,
                            team_name, goalie);
-  for (int episode=0; episode < 10; episode++) {
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
     while (status == IN_GAME) {
       // Get the vector of state features for the current state
       const vector<float>& feature_vec = hfo.getState();
@@ -53,7 +54,7 @@ int main(int argc, char** argv) {
     }
     // Check what the outcome of the episode was
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;;
+         << StatusToString(status) << endl;;
   }
   hfo.act(QUIT);
 };

example/mid_level_move_agent.cpp

@@ -27,8 +27,9 @@ int main(int argc, char** argv) {
                            team_name, goalie);
   float target_x = .82;
   float target_y = .9;
-  for (int episode = 0; episode < 10; episode++) {
   status_t status = IN_GAME;
+  for (int episode = 0; status != SERVER_DOWN; episode++) {
+    status = IN_GAME;
     if (episode % 2 != 0) {
       target_x *= -1;
     } else {
@@ -45,7 +46,7 @@ int main(int argc, char** argv) {
     }
     // Check what the outcome of the episode was
     cout << "Episode " << episode << " ended with status: "
-         << StatusToString(status) << std::endl;;
+         << StatusToString(status) << endl;;
   }
   hfo.act(QUIT);
 };

example/random_2v1.sh

+#!/bin/bash
+
+./bin/HFO --offense-agents=2 --defense-npcs=1 --trials 100 --headless &
+sleep 5
+./example/high_level_random_agent 6000 &
+sleep 5
+./example/high_level_random_agent 6000 &
+
+# The magic line
+#   $$ holds the PID for this script
+#   Negation means kill by process group id instead of PID
+trap "kill -TERM -$$" SIGINT
+wait

src/HFO.cpp

@@ -56,15 +56,16 @@ void HFOEnvironment::connectToServer(feature_set_t feature_set,
     std::cerr << "Unable to start agent" << std::endl;
     exit(1);
   }
-  assert(client->isServerAlive() == true);
   // Do nothing until the agent begins getting state features
-  while (agent->getState().empty()) {
   act(NOOP);
-    agent->executeAction();
-    do {
+  while (agent->getState().empty()) {
+    if (!client->isServerAlive()) {
+      std::cerr << "Server Down!" << std::endl;
+      exit(1);
+    }
     client->runStep(agent);
-    } while (agent->lastPreActionTime() < agent->currentTime());
   }
+  current_cycle = agent->currentTime().cycle();
 }
 
 const std::vector<float>& HFOEnvironment::getState() {
@@ -109,36 +110,14 @@ Player HFOEnvironment::playerOnBall() {
 }
 
 status_t HFOEnvironment::step() {
-  // Agent sends action to server
-  agent->executeAction();
-  // Wait until server replies with new game state
-  long start_cycle = agent->currentTime().cycle();
-  bool cycle_advanced = false;
-  bool end_of_trial = agent->getGameStatus() != IN_GAME;
-  bool still_eot = false;
-  int steps = 0;
-  do {
-    client->runStep(agent);
-    if (steps++ > MAX_STEPS) {
-      break;
-    }
+  assert(agent->currentTime().cycle() == current_cycle);
+  while (agent->statusUpdateTime() <= current_cycle) {
     if (!client->isServerAlive()) {
       return SERVER_DOWN;
     }
-    cycle_advanced = agent->currentTime().cycle() > start_cycle;
-  } while (!cycle_advanced || agent->lastPreActionTime() < agent->currentTime());
-  // If the trial is over, wait until the next episode starts
-  if (end_of_trial) {
-    while (agent->getGameStatus() != IN_GAME) {
-      act(NOOP);
-      agent->executeAction();
-      do {
     client->runStep(agent);
-        if (!client->isServerAlive()) {
-          return SERVER_DOWN;
-        }
-      } while (agent->lastPreActionTime() < agent->currentTime());
-    }
   }
+  assert(agent->currentTime().cycle() == (current_cycle + 1));
+  current_cycle = agent->currentTime().cycle();
   return agent->getGameStatus();
 }

src/HFO.hpp

@@ -59,6 +59,7 @@ class HFOEnvironment {
  private:
   rcsc::BasicClient* client;
   Agent* agent;
+  long current_cycle;
 };
 
 } // namespace hfo

src/agent.cpp

@@ -99,6 +99,7 @@ Agent::Agent()
       feature_extractor(NULL),
       lastTrainerMessageTime(-1),
       lastTeammateMessageTime(-1),
+      lastStatusUpdateTime(-1),
       game_status(IN_GAME),
       requested_action(NOOP)
 {
@@ -227,13 +228,6 @@ FeatureExtractor* Agent::getFeatureExtractor(feature_set_t feature_set_indx,
   }
 }
 
-// Instead of calling PlayerAgent::action() we instead call preAction
-// which does everything up to actionImpl(). actionImpl() is then
-// called in hfo::step(), PlayerAgent::executeAction().
-void Agent::action() {
-  preAction();
-}
-
 /*!
   main decision
   virtual method in super class
@@ -312,9 +306,6 @@ void Agent::actionImpl() {
                 << requested_action << std::endl;
       exit(1);
   }
-  // Clear the action
-  requested_action = (hfo::action_t) -1;
-  params.clear();
   // For now let's not worry about turning the neck or setting the vision.
   this->setViewAction(new View_Tactical());
   this->setNeckAction(new Neck_TurnToBallOrScan());
@@ -342,7 +333,9 @@ Agent::handleActionStart()
             feature_set, num_teammates, num_opponents, playing_offense);
       }
     }
-    hfo::ParseGameStatus(message, game_status);
+    if (hfo::ParseGameStatus(message, game_status)) {
+      lastStatusUpdateTime = audioSensor().trainerMessageTime().cycle();
+    }
     hfo::ParsePlayerOnBall(message, player_on_ball);
     lastTrainerMessageTime = audioSensor().trainerMessageTime().cycle();
   }

src/agent.h

@@ -22,14 +22,13 @@ public:
                                                int num_opponents,
                                                bool playing_offense);
 
+  inline long statusUpdateTime() { return lastStatusUpdateTime; }
+
 protected:
   // You can override this method. But you must call
   // PlayerAgent::initImpl() in this method.
   virtual bool initImpl(rcsc::CmdLineParser& cmd_parser);
 
-  // We override PlayerAgent's Action function
-  virtual void action();
-
   // main decision
   virtual void actionImpl();
 
@@ -48,6 +47,7 @@ protected:
   FeatureExtractor* feature_extractor; // Extracts the features
   long lastTrainerMessageTime;         // Last time the trainer sent a message
   long lastTeammateMessageTime;        // Last time a teammate sent a message
+  long lastStatusUpdateTime;           // Last time we got a status update
   hfo::status_t game_status;           // Current status of the game
   hfo::Player player_on_ball;          // Player in posession of the ball
   std::vector<float> state;            // Vector of current state features