some simple docs

docs/Makefile

+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: singlehtml
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+.PHONY: pickle
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+.PHONY: json
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+.PHONY: htmlhelp
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+.PHONY: qthelp
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/tensorpack.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/tensorpack.qhc"
+
+.PHONY: applehelp
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+.PHONY: devhelp
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/tensorpack"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/tensorpack"
+	@echo "# devhelp"
+
+.PHONY: epub
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+.PHONY: latex
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+.PHONY: latexpdf
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: latexpdfja
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: text
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+.PHONY: man
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+.PHONY: texinfo
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+.PHONY: info
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+.PHONY: gettext
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+.PHONY: changes
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: xml
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+.PHONY: pseudoxml
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

docs/conf.py

@@ -61,6 +61,10 @@ templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
+from recommonmark.parser import CommonMarkParser
+source_parsers = {
+    '.md': CommonMarkParser,
+}
 source_suffix = ['.rst', '.md']
 
 # The encoding of source files.
@@ -308,7 +312,9 @@ texinfo_documents = [
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
 def skip(app, what, name, obj, skip, options):
+    # keep __init__
     if name == "__init__":
         return False
     return skip
@@ -316,10 +322,21 @@ def skip(app, what, name, obj, skip, options):
 def get_rst(app, what, name, obj, options, signature,
             return_annotation):
     if signature:
+        # replace Mock function names
         signature = re.sub('<Mock name=\'([^\']+)\'.*>', '\g<1>', signature)
         signature = re.sub('tensorflow', 'tf', signature)
     return signature, return_annotation
 
 def setup(app):
+    from recommonmark.transform import AutoStructify
     app.connect('autodoc-process-signature', get_rst)
     app.connect("autodoc-skip-member", skip)
+    app.add_config_value(
+        'recommonmark_config',
+        {'url_resolver': lambda url: \
+         "https://github.com/ppwwyyxx/tensorpack/blob/master/tensorpack/" + url,
+         'auto_toc_tree_section': 'Contents',
+         'enable_math': True,
+         'enable_inline_math': True,
+        }, True)
+    app.add_transform(AutoStructify)

docs/index.rst

@@ -3,18 +3,32 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to tensorpack's documentation!
+Welcome to tensorpack
 ======================================
 
+Tensorpack is in early development.
+
 Contents:
 
 .. toctree::
    :maxdepth: 2
 
+   user/dataflow
+   user/models
 
 .. toctree::
   :maxdepth: 2
 
+  modules/tensorpack
+
+..
+  modules/tensorpack.models
+  modules/tensorpack.tfutils
+  modules/tensorpack.train
+  modules/tensorpack.callbacks
+  modules/tensorpack.utils
+
+
 Indices and tables
 ==================
 

docs/requirements.txt

@@ -2,3 +2,4 @@ termcolor
 tqdm
 nltk
 decorator
+recommonmark

docs/user/dataflow.md

+
+## Dataflow
+
+Dataflow uses Python generator to produce data in an efficient way.
+
+A Dataflow has to implement the `get_data()` generator method, which generates `datapoints` when called.
+A datapoint must be a list of picklable Python object.
+
+Then, multiple Dataflows can be composed together to build a complex data-preprocessing pipeline,
+including __reading from disk, batching, augmentations, prefetching__, etc. These components written in Python
+can provide a much more flexible data pipeline than with TensorFlow operators.
+Take a look at [common Dataflow](../../tensorpack/dataflow/common.py) and a [example of use](../../examples/ResNet/cifar10-resnet.py#L125).
+
+Optionally, Dataflow can implement the following two methods:
+
++ `size()`. Return the number of elements. Some components in the pipeline might require this to be
+	implemented. For example, only Dataflows with the same number of elements can be [joined](../../tensorpack/dataflow/common.py#L276).
+
++ `reset_state()`. It's necessary if your Dataflow uses RNG. This
+	method should reset the state of RNG and will be called after a fork, so that different child
+	processes won't produce identical data.
+
+
+NOTE: Dataflow aims to be independent of tensorflow.
+It should be useful for other python-based learning libraries as well.
+
+Common public datasets are also a kind of Dataflow. Some are defined in [dataflow.dataset](../../tensorpack/dataflow/dataset).

docs/user/models.md

+
+## Model Primitives
+
+`models` in tensorpack contains a collection of common model primitives.
+Take a look at the [Convolutional Layer](../../tensorpack/models/conv2d.py#L14) implementation for an example of how to define a
+model primitive:
+
+```python
+@layer_register()
+def Conv2D(x, out_channel, kernel_shape,
+           padding='SAME', stride=1,
+           W_init=None, b_init=None,
+           nl=tf.nn.relu, split=1, use_bias=True):
+```
+
+A primitive has the same interface as a tensorflow symbolic function: it takes a symbolic input `x` with
+some parameters, and return some symbolic outputs.
+
+`@layer_register()` will make this symbolic function become a `layer`, with the following benefits:
+
++ A variable scope for everything happening in this function.
++ Auto-inferred input/output shapes can be logged to terminal.
++ Work with `argscope` to define default arguments in a simple way.
+
+Some convention when working with a primitive defined under `@layer_register()`:
+
++ The input must be the first argument in the signature so that logging will know. It can be either a Tensor or a list of Tensor.
++ When called, the first argument should be the name scope and the second be the input.
+
+Take a look at the [Inception example](../../examples/Inception/inception-bn.py#L36) to see how a complicated model can be described with these primitives.
+

tensorpack/dataflow/README.md

-
-## Dataflow
-
-Dataflow: use `yield` to build a simple and efficient data processing pipeline.
-
-NOTE: Dataflow aims to be independent of tensorflow.
-It should be useful for other python-based learning libraries.
-
-Documents to be done.

tensorpack/dataflow/dataset/ilsvrc.py

@@ -79,7 +79,7 @@ class ILSVRCMeta(object):
         mean_file = os.path.join(self.dir, 'imagenet_mean.binaryproto')
         with open(mean_file) as f:
             obj.ParseFromString(f.read())
-        arr = np.array(obj.data).reshape((3, 256, 256))
+        arr = np.array(obj.data).reshape((3, 256, 256)).astype('float32')
         arr = np.transpose(arr, [1,2,0])
         if size is not None:
             arr = cv2.resize(arr, size[::-1])

tensorpack/models/nonlin.py

@@ -61,7 +61,7 @@ def LeakyReLU(x, alpha, name=None):
     else:
         return tf.mul(x, 0.5, name=name)
 
-
+# I'm not a layer, but I return a layer.
 def BNReLU(is_training, **kwargs):
     """
     :param is_traning: boolean