Fix hyperparam and optimizer issue in distributed trainer (#431)

tensorpack/callbacks/param.py

@@ -66,7 +66,7 @@ class GraphVarParam(HyperParam):
 
     def setup_graph(self):
         """ Will setup the assign operator for that variable. """
-        all_vars = tf.global_variables()
+        all_vars = tf.all_variables()
         for v in all_vars:
             if v.name == self.var_name:
                 self.var = v

tensorpack/train/distributed.py

@@ -204,6 +204,8 @@ class DistributedTrainerReplicated(MultiGPUTrainerBase):
         cbs = self._input_source.setup(self.model.get_inputs_desc())
         self.config.callbacks.extend(cbs)
 
+        # build the optimizer first, before entering any tower
+        self.model.get_optimizer()
         # Ngpu * Nvar * 2
         grad_list = MultiGPUTrainerBase.build_on_multi_tower(
             self.config.tower,