实际上关于pytorch加载自己的数据之前有写过一篇博客，但是最近接触了mxnet，发现关于这方面的教程很少
如果要加载自己定义的数据的话，看mxnet关于mnist基本上能够推测12
看pytorch与mxnet他们加载数据方式的对比
上图左边是pytorch的，右图是mxnet
实际上，mxnet与pytorch他们的datalayer有着相似之处，为什么这样说呢？直接看上面的代码，基本上都是输入图像的路径，然后输出一个可以供loader调用的可以迭代的对象，所以无论是pytorch或者是mxnet，如果要有自己的数据，只需要在自己的数据那一部分继承与修改ImageFolderDataset这个函数就行，就是直接继承dataset.Dataset类即可
对于pytorch而言，它使用了find_class这样一个函数，而对于mxnet而言，实际上它在类内部定义了一个_list_images的函数，事实上我并没有发现这有没有用，只需要get_item这个函数中返回list，list中是一个tuple，一个是文件的名字，另外一个是文件所对应的label即可。
只需要继承这一个类即可
直接撸代码
这个是我参加kaggle比赛的一段代码，尽管并不收敛，但请不要在意这些细节
# -*-coding:utf-8-*-
from mxnet import autograd
from mxnet import gluon
from mxnet import image
from mxnet import init
from mxnet import nd
from mxnet.gluon.data import vision
import numpy as np
from mxnet.gluon.data import dataset
import os
import warnings
import random
from mxnet import gpu
from mxnet.gluon.data.vision import datasets class MyImageFolderDataset(dataset.Dataset):
def __init__(self, root, label, flag=1, transform=None):
self._root = os.path.expanduser(root)
self._flag = flag
self._label = label
self._transform = transform
self._exts = ['.jpg', '.jpeg', '.png']
self._list_images(self._root, self._label) def _list_images(self, root, label): # label是一个list
self.synsets = []
self.synsets.append(root)
self.items = []
#file = open(label)
#lines = file.readlines()
#random.shuffle(lines)
c = 0
for line in label:
cls = line.split()
fn = cls.pop(0)
fn = fn + '.jpg'
# print(os.path.join(root, fn))
if os.path.isfile(os.path.join(root, fn)):
self.items.append((os.path.join(root, fn), float(cls[0])))
# print((os.path.join(root, fn), float(cls[0])))
else:
print('what')
c = c + 1
print('the total image is ', c) def __getitem__(self, idx):
img = image.imread(self.items[idx][0], self._flag)
label = self.items[idx][1]
if self._transform is not None:
return self._transform(img, label)
return img, label def __len__(self):
return len(self.items) def _get_batch(batch, ctx): # 可以在循环中直接for i, data, label,函数主要把data放在ctx上
"""return data and label on ctx"""
if isinstance(batch, mx.io.DataBatch):
data = batch.data[0]
label = batch.label[0]
else:
data, label = batch
return (gluon.utils.split_and_load(data, ctx),
gluon.utils.split_and_load(label, ctx),
data.shape[0]) def transform_train(data, label):
im = image.imresize(data.astype('float32') / 255, 256, 256)
auglist = image.CreateAugmenter(data_shape=(3, 256, 256), resize=0,
rand_crop=False, rand_resize=False, rand_mirror=True,
mean=None, std=None,
brightness=0, contrast=0,
saturation=0, hue=0,
pca_noise=0, rand_gray=0, inter_method=2)
for aug in auglist:
im = aug(im)
# 将数据格式从"高*宽*通道"改为"通道*高*宽"。
im = nd.transpose(im, (2, 0, 1))
return (im, nd.array([label]).asscalar().astype('float32')) def transform_test(data, label):
im = image.imresize(data.astype('float32') / 255, 256, 256)
im = nd.transpose(im, (2, 0, 1)) # 之前没有运行此变换
return (im, nd.array([label]).asscalar().astype('float32')) batch_size = 16
root = '/home/ying/data2/shiyongjie/landmark_recognition/data/image'
def random_choose_data(label_path):
f = open(label_path)
lines = f.readlins()
random.shuffle(lines)
total_number = len(lines)
train_number = total_number/10*7
train_list = lines[:train_number]
test_list = lines[train_number:]
return (train_list, test_list) label_path = '/home/ying/data2/shiyongjie/landmark_recognition/data/train.txt'
train_list, test_list = random_choose_data(label_path)
loader = gluon.data.DataLoader
train_ds = MyImageFolderDataset(os.path.join(root, 'image'), train_list, flag=1, transform=transform_train)
test_ds = MyImageFolderDataset(os.path.join(root, 'Testing'), test_list, flag=1, transform=transform_test)
train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep')
test_data = loader(test_ds, batch_size, shuffle=False, last_batch='keep')
softmax_cross_entropy = gluon.loss.L2Loss() # 定义L2 loss from mxnet.gluon import nn net = nn.Sequential()
with net.name_scope():
net.add(
# 第一阶段
nn.Conv2D(channels=96, kernel_size=11,
strides=4, activation='relu'),
nn.MaxPool2D(pool_size=3, strides=2),
# 第二阶段
nn.Conv2D(channels=256, kernel_size=5,
padding=2, activation='relu'),
nn.MaxPool2D(pool_size=3, strides=2),
# 第三阶段
nn.Conv2D(channels=384, kernel_size=3,
padding=1, activation='relu'),
nn.Conv2D(channels=384, kernel_size=3,
padding=1, activation='relu'),
nn.Conv2D(channels=256, kernel_size=3,
padding=1, activation='relu'),
nn.MaxPool2D(pool_size=3, strides=2),
# 第四阶段
nn.Flatten(),
nn.Dense(4096, activation="relu"),
nn.Dropout(.5),
# 第五阶段
nn.Dense(4096, activation="relu"),
nn.Dropout(.5),
# 第六阶段
nn.Dense(14950) # 输出为１个值
) from mxnet import init
from mxnet import gluon
import mxnet as mx
import utils
import datetime
from time import time ctx = utils.try_gpu()
net.initialize(ctx=ctx, init=init.Xavier()) mse_loss = gluon.loss.L2Loss() # utils.train(train_data, test_data, net, loss,
# trainer, ctx, num_epochs=10)
#def train(train_data, test_data, net, loss, trainer, ctx, num_epochs, print_batches=None):
num_epochs = 10
print_batches = 100
"""Train a network"""
print("Start training on ", ctx)
if isinstance(ctx, mx.Context):
ctx = [ctx]
def train(net, train_data, valid_data, num_epochs, lr, wd, ctx, lr_period, lr_decay):
trainer = gluon.Trainer(net.collect_params(), 'sgd',
{'learning_rate': lr, 'momentum': 0.9, 'wd': wd})
prev_time = datetime.datetime.now()
for epoch in range(num_epochs):
train_loss = 0.0
if epoch > 0 and epoch % lr_period == 0:
trainer.set_learning_rate(trainer.learning_rate*lr_decay)
for data, label in train_data:
label = label.as_in_context(ctx)
with autograd.record():
output = net(data.as_in_context(ctx))
loss = mse_loss(output, label)
loss.backward()
trainer.step(batch_size) # do the update, Trainer needs to know the batch size of the data to normalize
# the gradient by 1/batch_size
train_loss += nd.mean(loss).asscalar()
print(nd.mean(loss).asscalar())
cur_time = datetime.datetime.now()
h, remainder = divmod((cur_time - prev_time).seconds, 3600)
m, s = divmod(remainder, 60)
time_str = "Time %02d:%02d:%02d" % (h, m, s)
epoch_str = ('Epoch %d. Train loss: %f, ' % (epoch, train_loss / len(train_data)))
prev_time = cur_time
print(epoch_str + time_str + ', lr' + str(trainer.learning_rate))
net.collect_params().save('./model/alexnet.params')
ctx = utils.try_gpu()
num_epochs = 100
learning_rate = 0.001
weight_decay = 5e-4
lr_period = 10
lr_decay = 0.1 train(net, train_data, test_data, num_epochs, learning_rate,
weight_decay, ctx, lr_period, lr_decay)
请看这一段
class MyImageFolderDataset(dataset.Dataset):
def __init__(self, root, label, flag=1, transform=None):
self._root = os.path.expanduser(root)
self._flag = flag
self._label = label
self._transform = transform
self._exts = ['.jpg', '.jpeg', '.png']
self._list_images(self._root, self._label) def _list_images(self, root, label): # label是一个list
self.synsets = []
self.synsets.append(root)
self.items = []
#file = open(label)
#lines = file.readlines()
#random.shuffle(lines)
c = 0
for line in label:
cls = line.split()
fn = cls.pop(0)
fn = fn + '.jpg'
# print(os.path.join(root, fn))
if os.path.isfile(os.path.join(root, fn)):
self.items.append((os.path.join(root, fn), float(cls[0])))
# print((os.path.join(root, fn), float(cls[0])))
else:
print('what')
c = c + 1
print('the total image is ', c) def __getitem__(self, idx):
img = image.imread(self.items[idx][0], self._flag)
label = self.items[idx][1]
if self._transform is not None:
return self._transform(img, label)
return img, label def __len__(self):
return len(self.items)
batch_size = 16
root = '/home/ying/data2/shiyongjie/landmark_recognition/data/image'
def random_choose_data(label_path):
f = open(label_path)
lines = f.readlins()
random.shuffle(lines)
total_number = len(lines)
train_number = total_number/10*7
train_list = lines[:train_number]
test_list = lines[train_number:]
return (train_list, test_list) label_path = '/home/ying/data2/shiyongjie/landmark_recognition/data/train.txt'
train_list, test_list = random_choose_data(label_path) loader = gluon.data.DataLoader
train_ds = MyImageFolderDataset(os.path.join(root, 'image'), train_list, flag=1, transform=transform_train)
test_ds = MyImageFolderDataset(os.path.join(root, 'Testing'), test_list, flag=1, transform=transform_test)
train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep')
test_data = loader(test_ds, batch_size, shuffle=False, last_batch='keep')
MyImageFolderDataset是dataset.Dataset的子类，主要是是重载索引运算__getitem__，并且返回image以及其对应的label即可，前面的的_list_image函数只要是能够返回item这个list就行，关于运算符重载给自己挖个坑
可以说和pytorch非常像了，就连沐神在讲课的时候还在说，其实在写mxnet的时候，借鉴了很多pytorch的内容