Python torch.utils.data.read() Examples
The following are 30
code examples of torch.utils.data.read().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch.utils.data
, or try the search function
.
Example #1
Source File: norb.py From Matrix-Capsules-EM-PyTorch with MIT License | 6 votes |
def parse_info_file(path): """ -info file stores the additional info for each image. The specific meaning of each dimension is: (:, 0): 10 instances (:, 1): 9 elevation (:, 2): 18 azimuth (:, 3): 6 lighting conditions Return: ByteTensor of shape (N, 4) """ with open(path, 'rb') as f: header = parse_header(f) num, num_info = header['dim'] struct.unpack('<BBBB', f.read(4)) info = np.zeros(shape=(num, num_info), dtype=np.int32) for r in range(num): for c in range(num_info): info[r, c], = struct.unpack('<i', f.read(4)) return torch.from_numpy(info)
Example #2
Source File: norb.py From Matrix-Capsules-EM-PyTorch with MIT License | 6 votes |
def parse_dat_file(path): """ -dat file stores N image pairs. Each image pair, [i, :, :] and [i+1, :, :], includes two images taken from two cameras. They share the category and additional information. Return: ByteTensor of shape (2*N, 96, 96) """ with open(path, 'rb') as f: header = parse_header(f) num, c, h, w = header['dim'] imgs = np.zeros(shape=(num * c, h, w), dtype=np.uint8) for i in range(num * c): img = struct.unpack('<' + h * w * 'B', f.read(h * w)) imgs[i] = np.uint8(np.reshape(img, newshape=(h, w))) return torch.from_numpy(imgs)
Example #3
Source File: norb.py From Matrix-Capsules-EM-PyTorch with MIT License | 6 votes |
def parse_cat_file(path): """ -cat file stores corresponding category of images Return: ByteTensor of shape (N,) """ with open(path, 'rb') as f: header = parse_header(f) num, = header['dim'] struct.unpack('<BBBB', f.read(4)) struct.unpack('<BBBB', f.read(4)) labels = np.zeros(shape=num, dtype=np.int32) for i in range(num): labels[i], = struct.unpack('<i', f.read(4)) return torch.from_numpy(labels).long()
Example #4
Source File: fashionmnist.py From awesome_cnn with Apache License 2.0 | 6 votes |
def read_image_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2051 length = get_int(data[4:8]) num_rows = get_int(data[8:12]) num_cols = get_int(data[12:16]) images = [] idx = 16 for l in range(length): img = [] images.append(img) for r in range(num_rows): row = [] img.append(row) for c in range(num_cols): row.append(parse_byte(data[idx])) idx += 1 assert len(images) == length return torch.ByteTensor(images).view(-1, 28, 28)
Example #5
Source File: fashion.py From Generative_Continual_Learning with MIT License | 6 votes |
def read_image_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2051 length = get_int(data[4:8]) num_rows = get_int(data[8:12]) num_cols = get_int(data[12:16]) images = [] idx = 16 for l in range(length): img = [] images.append(img) for r in range(num_rows): row = [] img.append(row) for c in range(num_cols): row.append(parse_byte(data[idx])) idx += 1 assert len(images) == length return torch.ByteTensor(images).view(-1, 28, 28)
Example #6
Source File: fashion.py From MagnetLoss-PyTorch with MIT License | 6 votes |
def read_image_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2051 length = get_int(data[4:8]) num_rows = get_int(data[8:12]) num_cols = get_int(data[12:16]) images = [] idx = 16 for l in range(length): img = [] images.append(img) for r in range(num_rows): row = [] img.append(row) for c in range(num_cols): row.append(parse_byte(data[idx])) idx += 1 assert len(images) == length return torch.ByteTensor(images).view(-1, 28, 28)
Example #7
Source File: datasets.py From bgd with MIT License | 5 votes |
def read_label_file(self, path): with open(path, 'rb') as f: data = f.read() assert self.get_int(data[:4]) == 2049 length = self.get_int(data[4:8]) parsed = np.frombuffer(data, dtype=np.uint8, offset=8) return torch.from_numpy(parsed).view(length).long()
Example #8
Source File: norb.py From Matrix-Capsules-EM-PyTorch with MIT License | 5 votes |
def parse_header(fd): magic = struct.unpack('<BBBB', fd.read(4)) ndim, = struct.unpack('<i', fd.read(4)) dim = [] for _ in range(ndim): dim += struct.unpack('<i', fd.read(4)) header = {'magic': magic, 'type': magic2type(magic), 'dim': dim} return header
Example #9
Source File: fashionmnist.py From awesome_cnn with Apache License 2.0 | 5 votes |
def read_label_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2049 length = get_int(data[4:8]) labels = [parse_byte(b) for b in data[8:]] assert len(labels) == length return torch.LongTensor(labels)
Example #10
Source File: mnist.py From torch_DCEC with MIT License | 5 votes |
def read_image_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2051 length = get_int(data[4:8]) num_rows = get_int(data[8:12]) num_cols = get_int(data[12:16]) images = [] parsed = np.frombuffer(data, dtype=np.uint8, offset=16) return torch.from_numpy(parsed).view(length, num_rows, num_cols)
Example #11
Source File: mnist.py From torch_DCEC with MIT License | 5 votes |
def read_label_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2049 length = get_int(data[4:8]) parsed = np.frombuffer(data, dtype=np.uint8, offset=8) return torch.from_numpy(parsed).view(length).long()
Example #12
Source File: fashion.py From Generative_Continual_Learning with MIT License | 5 votes |
def read_label_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2049 length = get_int(data[4:8]) labels = [parse_byte(b) for b in data[8:]] assert len(labels) == length return torch.LongTensor(labels)
Example #13
Source File: multi_mnist_loader.py From MultiObjectiveOptimization with MIT License | 5 votes |
def read_image_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2051 length = get_int(data[4:8]) num_rows = get_int(data[8:12]) num_cols = get_int(data[12:16]) images = [] parsed = np.frombuffer(data, dtype=np.uint8, offset=16) pv = parsed.reshape(length, num_rows, num_cols) multi_length = length * 1 multi_data = np.zeros((1*length, num_rows, num_cols)) extension = np.zeros(1*length, dtype=np.int32) for left in range(length): chosen_ones = np.random.permutation(length)[:1] extension[left*1:(left+1)*1] = chosen_ones for j, right in enumerate(chosen_ones): lim = pv[left,:,:] rim = pv[right,:,:] new_im = np.zeros((36,36)) new_im[0:28,0:28] = lim new_im[6:34,6:34] = rim new_im[6:28,6:28] = np.maximum(lim[6:28,6:28], rim[0:22,0:22]) multi_data_im = m.imresize(new_im, (28, 28), interp='nearest') multi_data[left*1 + j,:,:] = multi_data_im return torch.from_numpy(parsed).view(length, num_rows, num_cols), torch.from_numpy(multi_data).view(length,num_rows, num_cols), extension
Example #14
Source File: multi_mnist_loader.py From MultiObjectiveOptimization with MIT License | 5 votes |
def read_label_file(path, extension): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2049 length = get_int(data[4:8]) parsed = np.frombuffer(data, dtype=np.uint8, offset=8) multi_labels_l = np.zeros((1*length),dtype=np.long) multi_labels_r = np.zeros((1*length),dtype=np.long) for im_id in range(length): for rim in range(1): multi_labels_l[1*im_id+rim] = parsed[im_id] multi_labels_r[1*im_id+rim] = parsed[extension[1*im_id+rim]] return torch.from_numpy(parsed).view(length).long(), torch.from_numpy(multi_labels_l).view(length*1).long(), torch.from_numpy(multi_labels_r).view(length*1).long()
Example #15
Source File: datasets.py From bgd with MIT License | 5 votes |
def read_image_file(self, path): with open(path, 'rb') as f: data = f.read() assert self.get_int(data[:4]) == 2051 length = self.get_int(data[4:8]) num_rows = self.get_int(data[8:12]) num_cols = self.get_int(data[12:16]) images = [] parsed = np.frombuffer(data, dtype=np.uint8, offset=16) return torch.from_numpy(parsed).view(length, num_rows, num_cols) ########################################################################### # Callable datasets ###########################################################################
Example #16
Source File: fashion.py From MagnetLoss-PyTorch with MIT License | 5 votes |
def read_label_file(path): with open(path, 'rb') as f: data = f.read() assert get_int(data[:4]) == 2049 length = get_int(data[4:8]) labels = [parse_byte(b) for b in data[8:]] assert len(labels) == length return torch.LongTensor(labels)
Example #17
Source File: toysets.py From cortex with BSD 3-Clause "New" or "Revised" License | 5 votes |
def download(self): """ Download, and unzip in the correct location. Returns: """ import urllib import zipfile if self.check_exists(): return # download files try: os.makedirs(self.root) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, filename) ext = os.path.splitext(file_path)[1] with open(file_path, 'wb') as f: f.write(data.read()) if ext == '.zip': with zipfile.ZipFile(file_path) as zip_f: zip_f.extractall(self.root) os.unlink(file_path) print('Done!')
Example #18
Source File: omniglot.py From MAML-Pytorch with MIT License | 5 votes |
def download(self): from six.moves import urllib import zipfile if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('== Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) file_processed = os.path.join(self.root, self.processed_folder) print("== Unzip from " + file_path + " to " + file_processed) zip_ref = zipfile.ZipFile(file_path, 'r') zip_ref.extractall(file_processed) zip_ref.close() print("Download finished.")
Example #19
Source File: omniglot_loaders.py From higher with Apache License 2.0 | 5 votes |
def download(self): from six.moves import urllib import zipfile if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('== Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) file_processed = os.path.join(self.root, self.processed_folder) print("== Unzip from " + file_path + " to " + file_processed) zip_ref = zipfile.ZipFile(file_path, 'r') zip_ref.extractall(file_processed) zip_ref.close() print("Download finished.")
Example #20
Source File: omniglot_dataset.py From Prototypical-Networks-for-Few-shot-Learning-PyTorch with MIT License | 5 votes |
def get_current_classes(fname): with open(fname) as f: classes = f.read().replace('/', os.sep).splitlines() return classes
Example #21
Source File: datasets.py From bgd with MIT License | 4 votes |
def download(self): """Download the MNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) with open(file_path.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print('Processing...') training_set = ( self.read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')), self.read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte')) ) test_set = ( self.read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')), self.read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte')) ) with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: torch.save(test_set, f) print('Done!')
Example #22
Source File: omniglot_dataset.py From Prototypical-Networks-for-Few-shot-Learning-PyTorch with MIT License | 4 votes |
def download(self): from six.moves import urllib import zipfile if self._check_exists(): return try: os.makedirs(os.path.join(self.root, self.splits_folder)) os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for k, url in self.vinyals_split_sizes.items(): print('== Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition(os.sep)[-1] file_path = os.path.join(self.root, self.splits_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) for url in self.urls: print('== Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition(os.sep)[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) orig_root = os.path.join(self.root, self.raw_folder) print("== Unzip from " + file_path + " to " + orig_root) zip_ref = zipfile.ZipFile(file_path, 'r') zip_ref.extractall(orig_root) zip_ref.close() file_processed = os.path.join(self.root, self.processed_folder) for p in ['images_background', 'images_evaluation']: for f in os.listdir(os.path.join(orig_root, p)): shutil.move(os.path.join(orig_root, p, f), file_processed) os.rmdir(os.path.join(orig_root, p)) print("Download finished.")
Example #23
Source File: fashionmnist.py From awesome_cnn with Apache License 2.0 | 4 votes |
def download(self): """Download the MNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) with open(file_path.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print('Processing...') training_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte')) ) test_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte')) ) with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: torch.save(test_set, f) print('Done!')
Example #24
Source File: mnistm.py From PyTorch with MIT License | 4 votes |
def download(self): """Download the MNIST data.""" # import essential packages from six.moves import urllib import gzip import pickle from torchvision import datasets # check if dataset already exists if self._check_exists(): return # make data dirs try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise # download pkl files print("Downloading " + self.url) filename = self.url.rpartition("/")[2] file_path = os.path.join(self.root, self.raw_folder, filename) if not os.path.exists(file_path.replace(".gz", "")): data = urllib.request.urlopen(self.url) with open(file_path, "wb") as f: f.write(data.read()) with open(file_path.replace(".gz", ""), "wb") as out_f, gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print("Processing...") # load MNIST-M images from pkl file with open(file_path.replace(".gz", ""), "rb") as f: mnist_m_data = pickle.load(f, encoding="bytes") mnist_m_train_data = torch.ByteTensor(mnist_m_data[b"train"]) mnist_m_test_data = torch.ByteTensor(mnist_m_data[b"test"]) # get MNIST labels mnist_train_labels = datasets.MNIST(root=self.mnist_root, train=True, download=True).train_labels mnist_test_labels = datasets.MNIST(root=self.mnist_root, train=False, download=True).test_labels # save MNIST-M dataset training_set = (mnist_m_train_data, mnist_train_labels) test_set = (mnist_m_test_data, mnist_test_labels) with open(os.path.join(self.root, self.processed_folder, self.training_file), "wb") as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), "wb") as f: torch.save(test_set, f) print("Done!")
Example #25
Source File: mnistm.py From PyTorch-GAN with MIT License | 4 votes |
def download(self): """Download the MNIST data.""" # import essential packages from six.moves import urllib import gzip import pickle from torchvision import datasets # check if dataset already exists if self._check_exists(): return # make data dirs try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise # download pkl files print("Downloading " + self.url) filename = self.url.rpartition("/")[2] file_path = os.path.join(self.root, self.raw_folder, filename) if not os.path.exists(file_path.replace(".gz", "")): data = urllib.request.urlopen(self.url) with open(file_path, "wb") as f: f.write(data.read()) with open(file_path.replace(".gz", ""), "wb") as out_f, gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print("Processing...") # load MNIST-M images from pkl file with open(file_path.replace(".gz", ""), "rb") as f: mnist_m_data = pickle.load(f, encoding="bytes") mnist_m_train_data = torch.ByteTensor(mnist_m_data[b"train"]) mnist_m_test_data = torch.ByteTensor(mnist_m_data[b"test"]) # get MNIST labels mnist_train_labels = datasets.MNIST(root=self.mnist_root, train=True, download=True).train_labels mnist_test_labels = datasets.MNIST(root=self.mnist_root, train=False, download=True).test_labels # save MNIST-M dataset training_set = (mnist_m_train_data, mnist_train_labels) test_set = (mnist_m_test_data, mnist_test_labels) with open(os.path.join(self.root, self.processed_folder, self.training_file), "wb") as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), "wb") as f: torch.save(test_set, f) print("Done!")
Example #26
Source File: mnist.py From torch_DCEC with MIT License | 4 votes |
def download(self): """Download the EMNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip import shutil import zipfile if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise print('Downloading ' + self.url) data = urllib.request.urlopen(self.url) filename = self.url.rpartition('/')[2] raw_folder = os.path.join(self.root, self.raw_folder) file_path = os.path.join(raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) print('Extracting zip archive') with zipfile.ZipFile(file_path) as zip_f: zip_f.extractall(raw_folder) os.unlink(file_path) gzip_folder = os.path.join(raw_folder, 'gzip') for gzip_file in os.listdir(gzip_folder): if gzip_file.endswith('.gz'): print('Extracting ' + gzip_file) with open(os.path.join(raw_folder, gzip_file.replace('.gz', '')), 'wb') as out_f, \ gzip.GzipFile(os.path.join(gzip_folder, gzip_file)) as zip_f: out_f.write(zip_f.read()) shutil.rmtree(gzip_folder) # process and save as torch files for split in self.splits: print('Processing ' + split) training_set = ( read_image_file(os.path.join(raw_folder, 'emnist-{}-train-images-idx3-ubyte'.format(split))), read_label_file(os.path.join(raw_folder, 'emnist-{}-train-labels-idx1-ubyte'.format(split))) ) test_set = ( read_image_file(os.path.join(raw_folder, 'emnist-{}-test-images-idx3-ubyte'.format(split))), read_label_file(os.path.join(raw_folder, 'emnist-{}-test-labels-idx1-ubyte'.format(split))) ) with open(os.path.join(self.root, self.processed_folder, self._training_file(split)), 'wb') as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self._test_file(split)), 'wb') as f: torch.save(test_set, f) print('Done!')
Example #27
Source File: mnist.py From torch_DCEC with MIT License | 4 votes |
def download(self): """Download the MNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) with open(file_path.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print('Processing...') training_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte')) ) test_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte')) ) with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: torch.save(test_set, f) print('Done!')
Example #28
Source File: fashion.py From MagnetLoss-PyTorch with MIT License | 4 votes |
def download(self): """Download the MNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) with open(file_path.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print('Processing...') training_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte')) ) test_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte')) ) with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: torch.save(test_set, f) print('Done!')
Example #29
Source File: fashion.py From Generative_Continual_Learning with MIT License | 4 votes |
def download(self): """Download the MNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip if self._check_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) with open(file_path.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print('Processing...') training_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte')) ) test_set = ( read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')), read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte')) ) with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: torch.save(training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: torch.save(test_set, f) print('Done!')
Example #30
Source File: multi_mnist_loader.py From MultiObjectiveOptimization with MIT License | 4 votes |
def download(self): """Download the MNIST data if it doesn't exist in processed_folder already.""" from six.moves import urllib import gzip if self._check_exists() and self._check_multi_exists(): return # download files try: os.makedirs(os.path.join(self.root, self.raw_folder)) os.makedirs(os.path.join(self.root, self.processed_folder)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise for url in self.urls: print('Downloading ' + url) data = urllib.request.urlopen(url) filename = url.rpartition('/')[2] file_path = os.path.join(self.root, self.raw_folder, filename) with open(file_path, 'wb') as f: f.write(data.read()) with open(file_path.replace('.gz', ''), 'wb') as out_f, \ gzip.GzipFile(file_path) as zip_f: out_f.write(zip_f.read()) os.unlink(file_path) # process and save as torch files print('Processing...') mnist_ims, multi_mnist_ims, extension = read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')) mnist_labels, multi_mnist_labels_l, multi_mnist_labels_r = read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte'), extension) tmnist_ims, tmulti_mnist_ims, textension = read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')) tmnist_labels, tmulti_mnist_labels_l, tmulti_mnist_labels_r = read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte'), textension) mnist_training_set = (mnist_ims, mnist_labels) multi_mnist_training_set = (multi_mnist_ims, multi_mnist_labels_l, multi_mnist_labels_r) mnist_test_set = (tmnist_ims, tmnist_labels) multi_mnist_test_set = (tmulti_mnist_ims, tmulti_mnist_labels_l, tmulti_mnist_labels_r) with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: torch.save(mnist_training_set, f) with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: torch.save(mnist_test_set, f) with open(os.path.join(self.root, self.processed_folder, self.multi_training_file), 'wb') as f: torch.save(multi_mnist_training_set, f) with open(os.path.join(self.root, self.processed_folder, self.multi_test_file), 'wb') as f: torch.save(multi_mnist_test_set, f) print('Done!')