gluon里面终于有了多线程迭代器,试了下,自定义程度高。
https://mxnet.incubator.apache.org/api/python/gluon/data.html?highlight=dataloader#mxnet.gluon.data.DataLoader
https://github.com/apache/incubator-mxnet/pull/8413
import mxnet as mx
import numpy as np
def mybatchify_fn(data):
print len(data),data[0]
return None
class tonyIter(mx.gluon.data.Dataset):
def __init__(self, N):
¦ self.N = N
def __getitem__(self, key):
¦ return [(np.random.randint(0,10,(3,4)), np.random.uniform(size=(4,5)) )]
def __len__(self):
¦ return self.N
BS = 10
N = 40
it_ = tonyIter(N)
num_workers = 2
it = mx.gluon.data.DataLoader(it_, BS,batchify_fn=mybatchify_fn,
¦ ¦ num_workers=num_workers)
for d in it:
print d