add gpu_compat option, improve globalstepcounter

tensorpack/callbacks/steps.py

@@ -55,13 +55,14 @@ class MaintainStepCounter(Callback):
         # ensure it exists
         gs_var = get_global_step_var()
         with tf.name_scope(None):
-            self.gs_incr_var = tf.assign_add(
-                gs_var, 1,
-                name=GLOBAL_STEP_INCR_OP_NAME)
+            with tf.device(gs_var.device):
+                self.gs_incr_op = tf.assign_add(
+                    gs_var, 1,
+                    name=GLOBAL_STEP_INCR_OP_NAME).op
             # tf.mod(
             #     self.gs_incr_var, self.trainer.config.steps_per_epoch,
             #     name=LOCAL_STEP_OP_NAME)
-        self._fetches = tf.train.SessionRunArgs(self.gs_incr_var)
+        self._fetches = tf.train.SessionRunArgs(self.gs_incr_op)
 
     def _before_train(self):
         gs_val = get_global_step_value()

tensorpack/tfutils/common.py

@@ -39,9 +39,11 @@ def get_default_sess_config(mem_fraction=0.99):
     conf.inter_op_parallelism_threads = 0
 
     conf.gpu_options.per_process_gpu_memory_fraction = mem_fraction
+    # TODO check version
+    conf.gpu_options.force_gpu_compatible = True
+
     conf.gpu_options.allocator_type = 'BFC'
     conf.gpu_options.allow_growth = True
-    # force gpu compatible?
 
     conf.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
     return conf