Python torch.utils.data.Subset() Examples
The following are 30
code examples of torch.utils.data.Subset().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
torch.utils.data
, or try the search function
.
Example #1
Source File: datautils.py From obman_train with GNU General Public License v3.0 | 7 votes |
def __next__(self): loader_idx = next(self.idx_cycle) loader = self.iters[loader_idx] batch = next(loader) if isinstance(loader.dataset, Subset): dataset = loader.dataset.dataset else: dataset = loader.dataset dat_name = dataset.pose_dataset.name batch["dataset"] = dat_name if dat_name == "stereohands" or dat_name == "zimsynth": batch["root"] = "palm" else: batch["root"] = "wrist" if dat_name == "stereohands": batch["use_stereohands"] = True else: batch["use_stereohands"] = False batch["split"] = dataset.pose_dataset.split return batch
Example #2
Source File: data_loading.py From modular-metalearning with MIT License | 6 votes |
def get_data_loaders(dataset, batch_size=32, shuffle=True): # Creating data indices for training and validation splits: dataset_size = len(dataset) indices = list(range(dataset_size)) train_indices, val_indices, test_indices \ = indices[:dataset.mtrain], indices[dataset.mtrain: dataset.mtrain + dataset.mval], \ indices[dataset.mtrain + dataset.mval:] train_dataset = Subset(dataset, train_indices) val_dataset = Subset(dataset, val_indices) test_dataset = Subset(dataset, test_indices) def collate(batch): return batch custom_collate = collate train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=custom_collate) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=custom_collate) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=custom_collate) if len(test_dataset.indices) else None return train_loader, val_loader, test_loader
Example #3
Source File: dataset.py From rising with MIT License | 6 votes |
def get_subset(self, indices: Sequence[int]) -> Subset: """ Returns a :class:`torch.utils.data.Subset` of the current dataset based on given indices Args: indices: valid indices to extract subset from current dataset Returns: Subset: the subset of the current dataset """ subset = Subset(self, indices) subset.__iter__ = self.__iter__ return subset
Example #4
Source File: dataset_utils.py From htmpapers with GNU Affero General Public License v3.0 | 6 votes |
def splitDataset(dataset, groupby): """ Split the given dataset into multiple datasets grouped by the given groupby function. For example:: # Split mnist dataset into 10 datasets, one dataset for each label splitDataset(mnist, groupby=lambda x: x[1]) # Split mnist dataset into 5 datasets, one dataset for each label pair: [0,1], [2,3],... splitDataset(mnist, groupby=lambda x: x[1] // 2) :param dataset: Source dataset to split :param groupby: Group by function. See :func:`itertools.groupby` :return: List of datasets """ # Split dataset based on the group by function and keep track of indices indicesByGroup = collections.defaultdict(list) for k, g in itertools.groupby(enumerate(dataset), key=lambda x: groupby(x[1])): indicesByGroup[k].extend([i[0] for i in g]) # Sort by group and create a Subset dataset for each of the group indices _, indices = zip(*(sorted(indicesByGroup.items(), key=lambda x: x[0]))) return [Subset(dataset, indices=i) for i in indices]
Example #5
Source File: datasets.py From CROWN-IBP with BSD 2-Clause "Simplified" License | 6 votes |
def mnist_loaders(dataset, batch_size, shuffle_train = True, shuffle_test = False, normalize_input = False, num_examples = None, test_batch_size=None): mnist_train = dataset("./data", train=True, download=True, transform=transforms.ToTensor()) mnist_test = dataset("./data", train=False, download=True, transform=transforms.ToTensor()) if num_examples: indices = list(range(num_examples)) mnist_train = data.Subset(mnist_train, indices) mnist_test = data.Subset(mnist_test, indices) train_loader = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=shuffle_train, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),2)) if test_batch_size: batch_size = test_batch_size test_loader = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=shuffle_test, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),2)) std = [1.0] mean = [0.0] train_loader.std = std test_loader.std = std train_loader.mean = mean test_loader.mean = mean return train_loader, test_loader
Example #6
Source File: data.py From convNet.pytorch with MIT License | 6 votes |
def get_loader(self, force_update=False, override_settings=None, subset_indices=None): if force_update or self.regime.update(self.epoch, self.steps): setting = self.get_setting() if override_settings is not None: setting.update(override_settings) self._transform = get_transform(**setting['transform']) setting['data'].setdefault('transform', self._transform) self._data = get_dataset(**setting['data']) if subset_indices is not None: self._data = Subset(self._data, subset_indices) if setting['other'].get('distributed', False): setting['loader']['sampler'] = DistributedSampler(self._data) setting['loader']['shuffle'] = None # pin-memory currently broken for distributed setting['loader']['pin_memory'] = False self._sampler = setting['loader'].get('sampler', None) self._loader = torch.utils.data.DataLoader( self._data, **setting['loader']) return self._loader
Example #7
Source File: dataset.py From baal with Apache License 2.0 | 6 votes |
def pool(self) -> torchdata.Dataset: """Returns a new Dataset made from unlabelled samples. Raises: ValueError if a pool specific attribute cannot be set. """ pool_dataset = copy(self._dataset) for attr, new_val in self.pool_specifics.items(): if hasattr(pool_dataset, attr): setattr(pool_dataset, attr, new_val) else: raise ValueError(f"{pool_dataset} doesn't have {attr}") pool_dataset = torchdata.Subset(pool_dataset, (~self._labelled).nonzero()[0].reshape([-1])) ald = ActiveLearningPool(pool_dataset, make_unlabelled=self.make_unlabelled) return ald
Example #8
Source File: dataset_enum.py From BatchBALD with GNU General Public License v3.0 | 6 votes |
def get_targets(dataset): """Get the targets of a dataset without any target target transforms(!).""" if isinstance(dataset, TransformedDataset): return get_targets(dataset.dataset) if isinstance(dataset, data.Subset): targets = get_targets(dataset.dataset) return torch.as_tensor(targets)[dataset.indices] if isinstance(dataset, data.ConcatDataset): return torch.cat([get_targets(sub_dataset) for sub_dataset in dataset.datasets]) if isinstance( dataset, (datasets.MNIST, datasets.ImageFolder,) ): return torch.as_tensor(dataset.targets) if isinstance(dataset, datasets.SVHN): return dataset.labels raise NotImplementedError(f"Unknown dataset {dataset}!")
Example #9
Source File: active_loop.py From baal with Apache License 2.0 | 6 votes |
def step(self) -> bool: """Perform an active learning step. Returns: boolean, Flag indicating if we continue training. """ # High to low pool = self.dataset.pool if len(pool) > 0: # Limit number of samples if self.max_sample != -1 and self.max_sample < len(pool): indices = np.random.choice(len(pool), self.max_sample, replace=False) pool = torchdata.Subset(pool, indices) else: indices = np.arange(len(pool)) probs = self.get_probabilities(pool, **self.kwargs) if probs is not None and (isinstance(probs, types.GeneratorType) or len(probs) > 0): to_label = self.heuristic(probs) to_label = indices[np.array(to_label)] if len(to_label) > 0: self.dataset.label(to_label[: self.ndata_to_label]) return True return False
Example #10
Source File: torch_utils.py From BatchBALD with GNU General Public License v3.0 | 5 votes |
def get_subset_base_indices(dataset: Subset, indices: typing.List[int]): return [int(dataset.indices[index]) for index in indices]
Example #11
Source File: pytorch_pbt_failure.py From ray with Apache License 2.0 | 5 votes |
def cifar_creator(config): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) # meanstd transformation transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) train_dataset = CIFAR10( root="~/data", train=True, download=True, transform=transform_train) validation_dataset = CIFAR10( root="~/data", train=False, download=False, transform=transform_test) if config.get("test_mode"): train_dataset = Subset(train_dataset, list(range(64))) validation_dataset = Subset(validation_dataset, list(range(64))) train_loader = DataLoader( train_dataset, batch_size=config[BATCH_SIZE], num_workers=2) validation_loader = DataLoader( validation_dataset, batch_size=config[BATCH_SIZE], num_workers=2) return train_loader, validation_loader
Example #12
Source File: torchutils.py From irn with MIT License | 5 votes |
def split_dataset(dataset, n_splits): return [Subset(dataset, np.arange(i, len(dataset), n_splits)) for i in range(n_splits)]
Example #13
Source File: subrange_dataset.py From BatchBALD with GNU General Public License v3.0 | 5 votes |
def SubrangeDataset(dataset, begin, end): if end > len(dataset): end = len(dataset) return Subset(dataset, range(begin, end))
Example #14
Source File: cifar_pytorch_pbt.py From ray with Apache License 2.0 | 5 votes |
def cifar_creator(config): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) # meanstd transformation transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) train_dataset = CIFAR10( root="~/data", train=True, download=True, transform=transform_train) validation_dataset = CIFAR10( root="~/data", train=False, download=False, transform=transform_test) if config.get("test_mode"): train_dataset = Subset(train_dataset, list(range(64))) validation_dataset = Subset(validation_dataset, list(range(64))) train_loader = DataLoader( train_dataset, batch_size=config[BATCH_SIZE], num_workers=2) validation_loader = DataLoader( validation_dataset, batch_size=config[BATCH_SIZE], num_workers=2) return train_loader, validation_loader
Example #15
Source File: torch_utils.py From BatchBALD with GNU General Public License v3.0 | 5 votes |
def get_base_indices(dataset: Dataset, indices: typing.List[int]): if isinstance(dataset, Subset): return get_base_indices(dataset.dataset, get_subset_base_indices(dataset, indices)) return indices #### ADDED FOR HEURISTIC
Example #16
Source File: active_learning_data.py From BatchBALD with GNU General Public License v3.0 | 5 votes |
def __init__(self, dataset: data.Dataset): super().__init__() self.dataset = dataset self.active_mask = np.full((len(dataset),), False) self.available_mask = np.full((len(dataset),), True) self.active_dataset = data.Subset(self.dataset, None) self.available_dataset = data.Subset(self.dataset, None) self._update_indices()
Example #17
Source File: experiment.py From catalyst with Apache License 2.0 | 5 votes |
def get_datasets(self, stage: str, n_samples: int = 320, **kwargs): """ @TODO: Docs. Contribution is welcome """ datasets = OrderedDict() if stage != "infer": trainset = MNIST( "./data", train=False, download=True, transform=Experiment.get_transforms(stage=stage, mode="train"), ) testset = MNIST( "./data", train=False, download=True, transform=Experiment.get_transforms(stage=stage, mode="valid"), ) if n_samples > 0: trainset = Subset(trainset, list(range(n_samples))) testset = Subset(testset, list(range(n_samples))) datasets["train"] = trainset datasets["valid"] = testset else: testset = MNIST( "./data", train=False, download=True, transform=Experiment.get_transforms(stage=stage, mode="valid"), ) if n_samples > 0: testset = Subset(testset, list(range(n_samples))) datasets["infer"] = testset return datasets
Example #18
Source File: base_dataset.py From deepsaber with GNU General Public License v3.0 | 5 votes |
def make_subset(self, indices): return data.Subset(self, indices)
Example #19
Source File: cifar_pytorch_example.py From ray with Apache License 2.0 | 5 votes |
def cifar_creator(config): transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) # meanstd transformation transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) train_dataset = CIFAR10( root="~/data", train=True, download=True, transform=transform_train) validation_dataset = CIFAR10( root="~/data", train=False, download=False, transform=transform_test) if config["test_mode"]: train_dataset = Subset(train_dataset, list(range(64))) validation_dataset = Subset(validation_dataset, list(range(64))) train_loader = DataLoader( train_dataset, batch_size=config[BATCH_SIZE], num_workers=2) validation_loader = DataLoader( validation_dataset, batch_size=config[BATCH_SIZE], num_workers=2) return train_loader, validation_loader
Example #20
Source File: cifar10.py From Deep-SAD-PyTorch with MIT License | 5 votes |
def __init__(self, root: str, normal_class: int = 5, known_outlier_class: int = 3, n_known_outlier_classes: int = 0, ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0): super().__init__(root) # Define normal and outlier classes self.n_classes = 2 # 0: normal, 1: outlier self.normal_classes = tuple([normal_class]) self.outlier_classes = list(range(0, 10)) self.outlier_classes.remove(normal_class) self.outlier_classes = tuple(self.outlier_classes) if n_known_outlier_classes == 0: self.known_outlier_classes = () elif n_known_outlier_classes == 1: self.known_outlier_classes = tuple([known_outlier_class]) else: self.known_outlier_classes = tuple(random.sample(self.outlier_classes, n_known_outlier_classes)) # CIFAR-10 preprocessing: feature scaling to [0, 1] transform = transforms.ToTensor() target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes)) # Get train set train_set = MyCIFAR10(root=self.root, train=True, transform=transform, target_transform=target_transform, download=True) # Create semi-supervised setting idx, _, semi_targets = create_semisupervised_setting(np.array(train_set.targets), self.normal_classes, self.outlier_classes, self.known_outlier_classes, ratio_known_normal, ratio_known_outlier, ratio_pollution) train_set.semi_targets[idx] = torch.tensor(semi_targets) # set respective semi-supervised labels # Subset train_set to semi-supervised setup self.train_set = Subset(train_set, idx) # Get test set self.test_set = MyCIFAR10(root=self.root, train=False, transform=transform, target_transform=target_transform, download=True)
Example #21
Source File: mnist.py From Deep-SVDD-PyTorch with MIT License | 5 votes |
def __init__(self, root: str, normal_class=0): super().__init__(root) self.n_classes = 2 # 0: normal, 1: outlier self.normal_classes = tuple([normal_class]) self.outlier_classes = list(range(0, 10)) self.outlier_classes.remove(normal_class) # Pre-computed min and max values (after applying GCN) from train data per class min_max = [(-0.8826567065619495, 9.001545489292527), (-0.6661464580883915, 20.108062262467364), (-0.7820454743183202, 11.665100841080346), (-0.7645772083211267, 12.895051191467457), (-0.7253923114302238, 12.683235701611533), (-0.7698501867861425, 13.103278415430502), (-0.778418217980696, 10.457837397569108), (-0.7129780970522351, 12.057777597673047), (-0.8280402650205075, 10.581538445782988), (-0.7369959242164307, 10.697039838804978)] # MNIST preprocessing: GCN (with L1 norm) and min-max feature scaling to [0,1] transform = transforms.Compose([transforms.ToTensor(), transforms.Lambda(lambda x: global_contrast_normalization(x, scale='l1')), transforms.Normalize([min_max[normal_class][0]], [min_max[normal_class][1] - min_max[normal_class][0]])]) target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes)) train_set = MyMNIST(root=self.root, train=True, download=True, transform=transform, target_transform=target_transform) # Subset train_set to normal class train_idx_normal = get_target_label_idx(train_set.train_labels.clone().data.cpu().numpy(), self.normal_classes) self.train_set = Subset(train_set, train_idx_normal) self.test_set = MyMNIST(root=self.root, train=False, download=True, transform=transform, target_transform=target_transform)
Example #22
Source File: cifar10.py From Deep-SVDD-PyTorch with MIT License | 5 votes |
def __init__(self, root: str, normal_class=5): super().__init__(root) self.n_classes = 2 # 0: normal, 1: outlier self.normal_classes = tuple([normal_class]) self.outlier_classes = list(range(0, 10)) self.outlier_classes.remove(normal_class) # Pre-computed min and max values (after applying GCN) from train data per class min_max = [(-28.94083453598571, 13.802961825439636), (-6.681770233365245, 9.158067708230273), (-34.924463588638204, 14.419298165027628), (-10.599172931391799, 11.093187820377565), (-11.945022995801637, 10.628045447867583), (-9.691969487694928, 8.948326776180823), (-9.174940012342555, 13.847014686472365), (-6.876682005899029, 12.282371383343161), (-15.603507135507172, 15.2464923804279), (-6.132882973622672, 8.046098172351265)] # CIFAR-10 preprocessing: GCN (with L1 norm) and min-max feature scaling to [0,1] transform = transforms.Compose([transforms.ToTensor(), transforms.Lambda(lambda x: global_contrast_normalization(x, scale='l1')), transforms.Normalize([min_max[normal_class][0]] * 3, [min_max[normal_class][1] - min_max[normal_class][0]] * 3)]) target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes)) train_set = MyCIFAR10(root=self.root, train=True, download=True, transform=transform, target_transform=target_transform) # Subset train set to normal class train_idx_normal = get_target_label_idx(train_set.train_labels, self.normal_classes) self.train_set = Subset(train_set, train_idx_normal) self.test_set = MyCIFAR10(root=self.root, train=False, download=True, transform=transform, target_transform=target_transform)
Example #23
Source File: generate.py From Human-Pose-Transfer with MIT License | 5 votes |
def get_data_loader(config): cfg = config["dataset"]["path"]["test"] image_dataset = dataset.PairBoneDataset(cfg["pair"], cfg["image"], cfg["bone"], cfg["mask"], cfg["annotation"]) if "generated_limit" in config: image_dataset = Subset(image_dataset, range(config["generated_limit"])) image_loader = DataLoader(image_dataset, batch_size=config["train"]["batch_size"], num_workers=8, pin_memory=True, drop_last=True) print(image_dataset) return image_loader
Example #24
Source File: mnist.py From Deep-SAD-PyTorch with MIT License | 5 votes |
def __init__(self, root: str, normal_class: int = 0, known_outlier_class: int = 1, n_known_outlier_classes: int = 0, ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0): super().__init__(root) # Define normal and outlier classes self.n_classes = 2 # 0: normal, 1: outlier self.normal_classes = tuple([normal_class]) self.outlier_classes = list(range(0, 10)) self.outlier_classes.remove(normal_class) self.outlier_classes = tuple(self.outlier_classes) if n_known_outlier_classes == 0: self.known_outlier_classes = () elif n_known_outlier_classes == 1: self.known_outlier_classes = tuple([known_outlier_class]) else: self.known_outlier_classes = tuple(random.sample(self.outlier_classes, n_known_outlier_classes)) # MNIST preprocessing: feature scaling to [0, 1] transform = transforms.ToTensor() target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes)) # Get train set train_set = MyMNIST(root=self.root, train=True, transform=transform, target_transform=target_transform, download=True) # Create semi-supervised setting idx, _, semi_targets = create_semisupervised_setting(train_set.targets.cpu().data.numpy(), self.normal_classes, self.outlier_classes, self.known_outlier_classes, ratio_known_normal, ratio_known_outlier, ratio_pollution) train_set.semi_targets[idx] = torch.tensor(semi_targets) # set respective semi-supervised labels # Subset train_set to semi-supervised setup self.train_set = Subset(train_set, idx) # Get test set self.test_set = MyMNIST(root=self.root, train=False, transform=transform, target_transform=target_transform, download=True)
Example #25
Source File: fmnist.py From Deep-SAD-PyTorch with MIT License | 5 votes |
def __init__(self, root: str, normal_class: int = 0, known_outlier_class: int = 1, n_known_outlier_classes: int = 0, ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0): super().__init__(root) # Define normal and outlier classes self.n_classes = 2 # 0: normal, 1: outlier self.normal_classes = tuple([normal_class]) self.outlier_classes = list(range(0, 10)) self.outlier_classes.remove(normal_class) self.outlier_classes = tuple(self.outlier_classes) if n_known_outlier_classes == 0: self.known_outlier_classes = () elif n_known_outlier_classes == 1: self.known_outlier_classes = tuple([known_outlier_class]) else: self.known_outlier_classes = tuple(random.sample(self.outlier_classes, n_known_outlier_classes)) # FashionMNIST preprocessing: feature scaling to [0, 1] transform = transforms.ToTensor() target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes)) # Get train set train_set = MyFashionMNIST(root=self.root, train=True, transform=transform, target_transform=target_transform, download=True) # Create semi-supervised setting idx, _, semi_targets = create_semisupervised_setting(train_set.targets.cpu().data.numpy(), self.normal_classes, self.outlier_classes, self.known_outlier_classes, ratio_known_normal, ratio_known_outlier, ratio_pollution) train_set.semi_targets[idx] = torch.tensor(semi_targets) # set respective semi-supervised labels # Subset train_set to semi-supervised setup self.train_set = Subset(train_set, idx) # Get test set self.test_set = MyFashionMNIST(root=self.root, train=False, transform=transform, target_transform=target_transform, download=True)
Example #26
Source File: odds.py From Deep-SAD-PyTorch with MIT License | 5 votes |
def __init__(self, root: str, dataset_name: str, n_known_outlier_classes: int = 0, ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0, random_state=None): super().__init__(root) # Define normal and outlier classes self.n_classes = 2 # 0: normal, 1: outlier self.normal_classes = (0,) self.outlier_classes = (1,) if n_known_outlier_classes == 0: self.known_outlier_classes = () else: self.known_outlier_classes = (1,) # Get train set train_set = ODDSDataset(root=self.root, dataset_name=dataset_name, train=True, random_state=random_state, download=True) # Create semi-supervised setting idx, _, semi_targets = create_semisupervised_setting(train_set.targets.cpu().data.numpy(), self.normal_classes, self.outlier_classes, self.known_outlier_classes, ratio_known_normal, ratio_known_outlier, ratio_pollution) train_set.semi_targets[idx] = torch.tensor(semi_targets) # set respective semi-supervised labels # Subset train_set to semi-supervised setup self.train_set = Subset(train_set, idx) # Get test set self.test_set = ODDSDataset(root=self.root, dataset_name=dataset_name, train=False, random_state=random_state)
Example #27
Source File: datasets.py From CROWN-IBP with BSD 2-Clause "Simplified" License | 4 votes |
def svhn_loaders(batch_size, shuffle_train = True, shuffle_test = False, train_random_transform = False, normalize_input = False, num_examples = None, test_batch_size=None): if normalize_input: mean = [0.43768206, 0.44376972, 0.47280434] std = [0.19803014, 0.20101564, 0.19703615] normalize = transforms.Normalize(mean = mean, std = std) else: std = [1.0, 1.0, 1.0] mean = [0, 0, 0] normalize = transforms.Normalize(mean = mean, std = std) if train_random_transform: if normalize_input: train = datasets.SVHN('./data', split='train', download=True, transform=transforms.Compose([ transforms.RandomCrop(32, 4), transforms.ToTensor(), normalize, ])) else: train = datasets.SVHN('./data', split='train', download=True, transform=transforms.Compose([ transforms.RandomCrop(32, 4), transforms.ToTensor(), ])) else: train = datasets.SVHN('./data', split='train', download=True, transform=transforms.Compose([transforms.ToTensor(),normalize])) test = datasets.SVHN('./data', split='test', download=True, transform=transforms.Compose([transforms.ToTensor(), normalize])) if num_examples: indices = list(range(num_examples)) train = data.Subset(train, indices) test = data.Subset(test, indices) train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=shuffle_train, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),6)) if test_batch_size: batch_size = test_batch_size test_loader = torch.utils.data.DataLoader(test, batch_size=max(batch_size, 1), shuffle=shuffle_test, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),6)) train_loader.std = std test_loader.std = std train_loader.mean = mean test_loader.mean = mean mean, std = get_stats(train_loader) print('dataset mean = ', mean.numpy(), 'std = ', std.numpy()) return train_loader, test_loader # when new loaders is added, they must be registered here
Example #28
Source File: datasets.py From CROWN-IBP with BSD 2-Clause "Simplified" License | 4 votes |
def cifar_loaders(batch_size, shuffle_train = True, shuffle_test = False, train_random_transform = False, normalize_input = False, num_examples = None, test_batch_size=None): if normalize_input: std = [0.2023, 0.1994, 0.2010] mean = [0.4914, 0.4822, 0.4465] normalize = transforms.Normalize(mean = mean, std = std) else: std = [1.0, 1.0, 1.0] mean = [0, 0, 0] normalize = transforms.Normalize(mean = mean, std = std) if train_random_transform: if normalize_input: train = datasets.CIFAR10('./data', train=True, download=True, transform=transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, 4), transforms.ToTensor(), normalize, ])) else: train = datasets.CIFAR10('./data', train=True, download=True, transform=transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, 4), transforms.ToTensor(), ])) else: train = datasets.CIFAR10('./data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor(),normalize])) test = datasets.CIFAR10('./data', train=False, transform=transforms.Compose([transforms.ToTensor(), normalize])) if num_examples: indices = list(range(num_examples)) train = data.Subset(train, indices) test = data.Subset(test, indices) train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=shuffle_train, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),6)) if test_batch_size: batch_size = test_batch_size test_loader = torch.utils.data.DataLoader(test, batch_size=max(batch_size, 1), shuffle=shuffle_test, pin_memory=True, num_workers=min(multiprocessing.cpu_count(),6)) train_loader.std = std test_loader.std = std train_loader.mean = mean test_loader.mean = mean return train_loader, test_loader
Example #29
Source File: cgnn.py From cgnn with Apache License 2.0 | 4 votes |
def main(device, model_param, optimizer_param, scheduler_param, dataset_param, dataloader_param, num_epochs, seed, load_model): print("Seed:", seed) print() torch.manual_seed(seed) dataloader_param["collate_fn"] = graph_collate # Create dataset dataset = GraphDataset(dataset_param["dataset_path"], dataset_param["target_name"]) # split the dataset into training, validation, and test sets. split_file_path = dataset_param["split_file"] if split_file_path is not None and os.path.isfile(split_file_path): with open(split_file_path) as f: split = json.load(f) else: print("No split file. Default split: 256 (train), 32 (val), 32 (test)") split = {"train": range(256), "val": range(256, 288), "test": range(288, 320)} print(" ".join(["{}: {}".format(k, len(x)) for k, x in split.items()])) # Create a CGNN model model = create_model(device, model_param, optimizer_param, scheduler_param) if load_model: print("Loading weights from model.pth") model.load() #print("Model:", model.device) # Train train_sampler = SubsetRandomSampler(split["train"]) val_sampler = SubsetRandomSampler(split["val"]) train_dl = DataLoader(dataset, sampler=train_sampler, **dataloader_param) val_dl = DataLoader(dataset, sampler=val_sampler, **dataloader_param) model.train(train_dl, val_dl, num_epochs) if num_epochs > 0: model.save() # Test test_set = Subset(dataset, split["test"]) test_dl = DataLoader(test_set, **dataloader_param) outputs, targets = model.evaluate(test_dl) names = [dataset.graph_names[i] for i in split["test"]] df_predictions = pd.DataFrame({"name": names, "prediction": outputs, "target": targets}) df_predictions.to_csv("test_predictions.csv", index=False) print("\nEND")
Example #30
Source File: train_setops_stripped.py From LaSO with BSD 3-Clause "New" or "Revised" License | 4 votes |
def setup_datasets(self): """Load the training datasets.""" train_transform = transforms.Compose( [ transforms.Resize(self.crop_size), transforms.RandomRotation(degrees=self.random_angle, resample=Image.BILINEAR), transforms.RandomResizedCrop( size=self.crop_size, scale=(1-self.random_scale, 1+self.random_scale), ratio=(1, 1)), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) ] ) val_transform = transforms.Compose( [ transforms.Resize(self.crop_size), transforms.CenterCrop(self.crop_size), transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ) ] ) train_dataset = CocoDatasetPairs( root_dir=self.coco_path, set_name='train2014', transform=train_transform, dataset_size_ratio=self.dataset_size_ratio ) train_subset_dataset = Subset(train_dataset, range(0, len(train_dataset), 5*self.dataset_size_ratio)) val_dataset = CocoDatasetPairs( root_dir=self.coco_path, set_name='val2014', transform=val_transform, ) train_loader = DataLoader( train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=self.num_workers ) train_subset_loader = DataLoader( train_subset_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers ) val_loader = DataLoader( val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self.num_workers ) return train_loader, train_subset_loader, val_loader