【吐槽】
啊，代码，你这个大猪蹄子
自己写了cifar10的数据接口，跟官方接口load的数据一样，
沾沾自喜，以为自己会写数据接口了
几天之后，突然想，自己的代码为啥有点慢呢，这数据集不大啊
用了官方接口，真快啊。。。
啊啊啊啊啊啊啊啊
但这是好事，至少我明白了一点知识对吧
【lesson】
看了cifar10的接口，发现自己在数据集初始化的地方写的太少了，应该在初始化的时候就把所有数据读进来，这样的话在__getitem__的时候才能快。
人家的初始化：
if self.train:
self.train_data = []
self.train_labels = []
for fentry in self.train_list:
f = fentry[0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
if sys.version_info[0] == 2:
entry = pickle.load(fo)
else:
entry = pickle.load(fo, encoding='latin1')
self.train_data.append(entry['data'])
if 'labels' in entry:
self.train_labels += entry['labels']
else:
self.train_labels += entry['fine_labels']
fo.close() self.train_data = np.concatenate(self.train_data)
self.train_data = self.train_data.reshape((50000, 3, 32, 32))
self.train_data = self.train_data.transpose((0, 2, 3, 1)) # convert to HWC
人家的getitem
def __getitem__(self, index):
"""
Args:
index (int): Index Returns:
tuple: (image, target) where target is index of the target class.
"""
if self.train:
img, target = self.train_data[index], self.train_labels[index]
else:
img, target = self.test_data[index], self.test_labels[index] # doing this so that it is consistent with all other datasets
# to return a PIL Image
img = Image.fromarray(img) if self.transform is not None:
img = self.transform(img) if self.target_transform is not None:
target = self.target_transform(target) return img, target
自己：（都写到getitem里面了）
def __init__(self, root, transforms=transform(), train=True, test=False):
self.root = root
self.transform = transforms
self.train = train
self.test = test
if self.test:
self.train = False def __getitem__(self, item):
x = math.floor(item / 10000) + 1
y = item % 10000
if not self.train and not self.test:
x = 5
y = 5000+item imgpath = os.path.join(self.root, "data_batch_"+str(x))
with open(imgpath, 'rb') as fo:
dict = pickle.load(fo, encoding='bytes')
d_decode = {}
for k,v in dict.items():
d_decode[k.decode('utf8')] = v
dict = d_decode
data = dict['data'][y] # 3*32*32==3072
data = np.reshape(data,(3,32,32))
data = data.transpose(1,2,0)
data = self.transform(data)
label = dict['labels'][y]
# label = torch.from_numpy(label) return data, label
附自己的代码和人家的代码全部
人家：
base_folder = 'cifar-10-batches-py'
url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
filename = "cifar-10-python.tar.gz"
tgz_md5 = 'c58f30108f718f92721af3b95e74349a'
train_list = [
['data_batch_1', 'c99cafc152244af753f735de768cd75f'],
['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'],
['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'],
['data_batch_4', '634d18415352ddfa80567beed471001a'],
['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'],
] test_list = [
['test_batch', '40351d587109b95175f43aff81a1287e'],
] def __init__(self, root, train=True,
transform=None, target_transform=None,
download=False):
self.root = os.path.expanduser(root)
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set if download:
self.download() if not self._check_integrity():
raise RuntimeError('Dataset not found or corrupted.' +
' You can use download=True to download it') # now load the picked numpy arrays
if self.train:
self.train_data = []
self.train_labels = []
for fentry in self.train_list:
f = fentry[0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
if sys.version_info[0] == 2:
entry = pickle.load(fo)
else:
entry = pickle.load(fo, encoding='latin1')
self.train_data.append(entry['data'])
if 'labels' in entry:
self.train_labels += entry['labels']
else:
self.train_labels += entry['fine_labels']
fo.close() self.train_data = np.concatenate(self.train_data)
self.train_data = self.train_data.reshape((50000, 3, 32, 32))
self.train_data = self.train_data.transpose((0, 2, 3, 1)) # convert to HWC
else:
f = self.test_list[0][0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
if sys.version_info[0] == 2:
entry = pickle.load(fo)
else:
entry = pickle.load(fo, encoding='latin1')
self.test_data = entry['data']
if 'labels' in entry:
self.test_labels = entry['labels']
else:
self.test_labels = entry['fine_labels']
fo.close()
self.test_data = self.test_data.reshape((10000, 3, 32, 32))
self.test_data = self.test_data.transpose((0, 2, 3, 1)) # convert to HWC def __getitem__(self, index):
"""
Args:
index (int): Index Returns:
tuple: (image, target) where target is index of the target class.
"""
if self.train:
img, target = self.train_data[index], self.train_labels[index]
else:
img, target = self.test_data[index], self.test_labels[index] # doing this so that it is consistent with all other datasets
# to return a PIL Image
img = Image.fromarray(img) if self.transform is not None:
img = self.transform(img) if self.target_transform is not None:
target = self.target_transform(target) return img, target