Commit ae7b0774 authored by Yuxin Wu's avatar Yuxin Wu

update dataflow docs about windows (#1219)

parent afba8dee
......@@ -272,6 +272,17 @@ df = RemoteDataZMQ('ipc://@my-socket', 'tcp://0.0.0.0:8877')
TestDataSpeed(df).start()
```
## Common Issues on Windows:
1. Windows does not support ZMQ. You can only use `MultiProcessRunner`,
`MultiThreadRunner`, and `MultiThreadMapData`. But you cannot use
`MultiProcessRunnerZMQ` or `MultiProcessMapData` (which is an alias of `MultiProcessMapDataZMQ`).
2. Windows needs to pickle your dataflow to run it in multiple processes.
As a result you cannot use lambda functions for mappings, like the examples above.
You need to write a new function in global scope that does the mapping.
This issue also exist on Linux if you do not use the 'fork' start method.
[1]: #ref
<div id=ref> </div>
......
......@@ -178,7 +178,8 @@ class MultiProcessRunner(ProxyDataFlow):
Args:
ds (DataFlow): input DataFlow.
num_prefetch (int): size of the queue to hold prefetched datapoints.
num_proc (int): number of processes to use.
Required.
num_proc (int): number of processes to use. Required.
nr_prefetch, nr_proc: deprecated argument names
"""
if nr_prefetch is not None:
......@@ -187,6 +188,8 @@ class MultiProcessRunner(ProxyDataFlow):
if nr_proc is not None:
log_deprecated("MultiProcessRunner(nr_proc)", "Renamed to 'num_proc'", "2020-01-01")
num_proc = nr_proc
if num_prefetch is None or num_proc is None:
raise TypeError("Missing argument num_prefetch or num_proc in MultiProcessRunner!")
# https://docs.python.org/3.6/library/multiprocessing.html?highlight=process#the-spawn-and-forkserver-start-methods
if os.name == 'nt':
......@@ -424,6 +427,9 @@ class MultiThreadRunner(DataFlow):
if nr_thread is not None:
log_deprecated("MultiThreadRunner(nr_thread)", "Renamed to 'num_thread'", "2020-01-01")
num_thread = nr_thread
if num_prefetch is None or num_thread is None:
raise TypeError("Missing argument num_prefetch or num_thread in MultiThreadRunner!")
assert num_thread > 0, num_thread
assert num_prefetch > 0, num_prefetch
self.num_thread = num_thread
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment