make dataflow.dataset a lazy module (#363)

examples/DisturbLabel/mnist-disturb.py

@@ -9,6 +9,7 @@ import sys
 import argparse
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 import tensorflow as tf
 from disturb import DisturbLabel
 

examples/DisturbLabel/svhn-disturb.py

@@ -9,9 +9,7 @@ import os
 import imp
 
 from tensorpack import *
-from tensorpack.tfutils.symbolic_functions import *
-from tensorpack.tfutils.summary import *
-import tensorflow as tf
+from tensorpack.dataflow import dataset
 
 from disturb import DisturbLabel
 

examples/DoReFa-Net/alexnet-dorefa.py

@@ -15,6 +15,7 @@ from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
 from tensorpack.tfutils.varreplace import remap_variables
+from tensorpack.dataflow import dataset
 from dorefa import get_dorefa
 
 """

examples/DoReFa-Net/resnet-dorefa.py

@@ -10,8 +10,8 @@ import os
 import sys
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 from tensorpack.tfutils.symbolic_functions import *
-from tensorpack.tfutils.summary import *
 from tensorpack.utils.stats import RatioCounter
 from tensorpack.tfutils.varreplace import remap_variables
 from dorefa import get_dorefa

examples/DoReFa-Net/svhn-digit-dorefa.py

@@ -10,6 +10,7 @@ import os
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 from tensorpack.tfutils.varreplace import remap_variables
 import tensorflow as tf
 

examples/DynamicFilterNetwork/steering-filter.py

@@ -14,6 +14,7 @@ from tensorpack import *
 from tensorpack.utils import logger
 from tensorpack.utils.viz import *
 from tensorpack.utils.argtools import shape2d, shape4d
+from tensorpack.dataflow import dataset
 
 BATCH = 32
 SHAPE = 64

examples/GAN/ConditionalGAN-mnist.py

@@ -14,6 +14,7 @@ from tensorpack import *
 from tensorpack.utils.viz import *
 import tensorpack.tfutils.symbolic_functions as symbf
 from tensorpack.tfutils.scope_utils import auto_reuse_variable_scope
+from tensorpack.dataflow import dataset
 from GAN import GANTrainer, RandomZData, GANModelDesc
 
 """

examples/GAN/InfoGAN-mnist.py

@@ -17,6 +17,7 @@ from tensorpack.tfutils.scope_utils import auto_reuse_variable_scope
 from tensorpack.tfutils import optimizer, summary
 import tensorpack.tfutils.symbolic_functions as symbf
 from tensorpack.tfutils.gradproc import ScaleGradient, CheckGradient
+from tensorpack.dataflow import dataset
 from GAN import GANTrainer, GANModelDesc
 
 """

examples/HED/hed.py

@@ -13,6 +13,7 @@ import sys
 
 from tensorpack import *
 import tensorpack.tfutils.symbolic_functions as symbf
+from tensorpack.dataflow import dataset
 from tensorpack.tfutils import optimizer
 from tensorpack.tfutils.summary import *
 

examples/Inception/inception-bn.py

@@ -12,6 +12,7 @@ import tensorflow as tf
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 
 
 TOTAL_BATCH_SIZE = 64 * 6

examples/Inception/inceptionv3.py

@@ -13,6 +13,7 @@ import multiprocessing
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 
 """
 InceptionV3 on ILSVRC12.

examples/ResNet/cifar10-resnet.py

@@ -10,6 +10,7 @@ import os
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 
 import tensorflow as tf
 from tensorflow.contrib.layers import variance_scaling_initializer

examples/ResNet/imagenet-resnet-se.py

@@ -11,6 +11,7 @@ import multiprocessing
 import tensorflow as tf
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
 

examples/ResNet/imagenet-resnet.py

@@ -13,6 +13,7 @@ import tensorflow as tf
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 
 from imagenet_resnet_utils import (
     fbresnet_augmentor, resnet_basicblock, resnet_bottleneck, resnet_backbone,

examples/ResNet/load-resnet.py

@@ -19,7 +19,7 @@ from tensorpack.utils import logger
 from tensorpack.utils.stats import RatioCounter
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
-from tensorpack.dataflow.dataset import ILSVRCMeta
+from tensorpack.dataflow.dataset import ILSVRCMeta, ILSVRC12
 
 MODEL_DEPTH = None
 
@@ -134,7 +134,7 @@ def run_test(params, input):
 
 
 def eval_on_ILSVRC12(params, data_dir):
-    ds = dataset.ILSVRC12(data_dir, 'val', shuffle=False, dir_structure='train')
+    ds = ILSVRC12(data_dir, 'val', shuffle=False, dir_structure='train')
     ds = AugmentImageComponent(ds, get_inference_augmentor())
     ds = BatchData(ds, 128, remainder=True)
     pred_config = PredictConfig(

examples/ResNet/svhn-resnet.py

@@ -10,6 +10,7 @@ import os
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import *
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 import tensorflow as tf
 
 """

examples/Saliency/README.md

@@ -36,14 +36,14 @@ Left to right:
 Usage:
 1. Fine tune or retrain the ResNet:
 ```bash
-./CAM-resnet.py --data /path/to/imagenet [--load ImageNet-ResNet18.npy] [--gpu 0,1,2,3]
+./CAM-resnet.py --data /path/to/imagenet [--load ImageNet-ResNet18.npz] [--gpu 0,1,2,3]
 ```
 Pretrained and fine-tuned ResNet can be downloaded
 [here](https://drive.google.com/open?id=0B9IPQTvr2BBkTXBlZmh1cmlnQ0k) and [here](https://drive.google.com/open?id=0B9IPQTvr2BBkQk9qcmtGSERlNUk).
 
 2. Generate CAM on ImageNet validation set:
 ```bash
-./CAM-resnet.py --data /path/to/imagenet --load ImageNet-ResNet18-2xGAP.npy --cam
+./CAM-resnet.py --data /path/to/imagenet --load ImageNet-ResNet18-2xGAP.npz --cam
 ```
 
 <p align="center"> <img src="./CAM-demo.jpg" width="900"> </p>

examples/SpatialTransformer/mnist-addition.py

@@ -11,6 +11,7 @@ import sys
 import argparse
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 from tensorpack.tfutils import sesscreate, optimizer, summary
 import tensorpack.tfutils.symbolic_functions as symbf
 

examples/cifar-convnet.py

@@ -10,6 +10,7 @@ import os
 
 import tensorpack.tfutils.symbolic_functions as symbf
 from tensorpack.tfutils.summary import *
+from tensorpack.dataflow import dataset
 from tensorpack.utils.gpu import get_nr_gpu
 
 """

examples/mnist-convnet.py

@@ -6,6 +6,7 @@ import numpy as np
 import os
 import sys
 import argparse
+import tensorflow as tf
 """
 MNIST ConvNet example.
 about 0.6% validation error after 30 epochs.
@@ -14,7 +15,7 @@ about 0.6% validation error after 30 epochs.
 # Just import everything into current namespace
 from tensorpack import *
 from tensorpack.tfutils import summary
-import tensorflow as tf
+from tensorpack.dataflow import dataset
 import tensorpack.tfutils.symbolic_functions as symbf
 
 IMAGE_SIZE = 28

examples/mnist-keras.py

@@ -21,6 +21,7 @@ This way you can define models in Keras-style, and benefit from the more efficei
 """
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 from tensorpack.utils.argtools import memoized
 
 IMAGE_SIZE = 28

examples/mnist-tfslim.py

@@ -15,6 +15,7 @@ the only differences are:
 """
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 import tensorflow as tf
 import tensorflow.contrib.slim as slim
 

examples/mnist-visualizations.py

@@ -12,6 +12,7 @@ MNIST ConvNet example with weights/activations visualization.
 """
 
 from tensorpack import *
+from tensorpack.dataflow import dataset
 import tensorflow as tf
 import tensorpack.tfutils.symbolic_functions as symbf
 

examples/svhn-digit-convnet.py

@@ -9,6 +9,7 @@ import os
 
 from tensorpack import *
 from tensorpack.tfutils.symbolic_functions import prediction_incorrect
+from tensorpack.dataflow import dataset
 from tensorpack.tfutils.summary import *
 import tensorflow as tf
 

tensorpack/dataflow/__init__.py

@@ -6,10 +6,7 @@ from pkgutil import iter_modules
 import os
 import os.path
 
-from . import dataset
-from . import imgaug
-
-__all__ = ['dataset', 'imgaug', 'dftools']
+__all__ = []
 
 
 def _global_import(name):
@@ -21,9 +18,22 @@ def _global_import(name):
         __all__.append(k)
 
 
-__SKIP = ['dftools', 'dataset', 'imgaug']
+__SKIP = set(['dftools', 'dataset', 'imgaug'])
 for _, module_name, __ in iter_modules(
         [os.path.dirname(__file__)]):
     if not module_name.startswith('_') and \
             module_name not in __SKIP:
         _global_import(module_name)
+
+
+class _LazyModule(object):
+    def __init__(self, modname):
+        self._modname = modname
+
+    def __getattr__(self, name):
+        dataset = __import__(self._modname, globals(), locals(), [name], 1)
+        return getattr(dataset, name)
+
+
+dataset = _LazyModule('dataset')
+__all__.extend(['imgaug', 'dftools', 'dataset'])