Python tensor2tensor.data_generators.generator_utils.generate_dataset_and_shuffle() Examples
The following are 30
code examples of tensor2tensor.data_generators.generator_utils.generate_dataset_and_shuffle().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
tensor2tensor.data_generators.generator_utils
, or try the search function
.

Example #1
Source File: algorithmic.py From fine-lm with MIT License | 6 votes |
def generate_data(self, data_dir, _, task_id=-1): def generator_eos(nbr_symbols, max_length, nbr_cases): """Shift by NUM_RESERVED_IDS and append EOS token.""" for case in self.generator(nbr_symbols, max_length, nbr_cases): new_case = {} for feature in case: new_case[feature] = [ i + text_encoder.NUM_RESERVED_TOKENS for i in case[feature] ] + [text_encoder.EOS_ID] yield new_case utils.generate_dataset_and_shuffle( generator_eos(self.num_symbols, self.train_length, self.train_size), self.training_filepaths(data_dir, self.num_shards, shuffled=True), generator_eos(self.num_symbols, self.dev_length, self.dev_size), self.dev_filepaths(data_dir, 1, shuffled=True), shuffle=False)
Example #2
Source File: librispeech_specaugment.py From specAugment with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #3
Source File: librispeech.py From training_results_v0.5 with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #4
Source File: common_voice.py From training_results_v0.5 with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #5
Source File: algorithmic.py From training_results_v0.5 with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, _, task_id=-1): def generator_eos(nbr_symbols, max_length, nbr_cases): """Shift by NUM_RESERVED_IDS and append EOS token.""" for case in self.generator(nbr_symbols, max_length, nbr_cases): new_case = {} for feature in case: new_case[feature] = [ i + text_encoder.NUM_RESERVED_TOKENS for i in case[feature] ] + [text_encoder.EOS_ID] yield new_case utils.generate_dataset_and_shuffle( generator_eos(self.num_symbols, self.train_length, self.train_size), self.training_filepaths(data_dir, self.num_shards, shuffled=True), generator_eos(self.num_symbols, self.dev_length, self.dev_size), self.dev_filepaths(data_dir, 1, shuffled=True), shuffle=False)
Example #6
Source File: librispeech.py From BERT with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #7
Source File: common_voice.py From BERT with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #8
Source File: algorithmic.py From BERT with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, _, task_id=-1): def generator_eos(nbr_symbols, max_length, nbr_cases): """Shift by NUM_RESERVED_IDS and append EOS token.""" for case in self.generator(nbr_symbols, max_length, nbr_cases): new_case = {} for feature in case: new_case[feature] = [ i + text_encoder.NUM_RESERVED_TOKENS for i in case[feature] ] + [text_encoder.EOS_ID] yield new_case utils.generate_dataset_and_shuffle( generator_eos(self.num_symbols, self.train_length, self.train_size), self.training_filepaths(data_dir, self.num_shards, shuffled=True), generator_eos(self.num_symbols, self.dev_length, self.dev_size), self.dev_filepaths(data_dir, 1, shuffled=True), shuffle=False)
Example #9
Source File: librispeech.py From tensor2tensor with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #10
Source File: common_voice.py From tensor2tensor with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #11
Source File: algorithmic.py From tensor2tensor with Apache License 2.0 | 6 votes |
def generate_data(self, data_dir, _, task_id=-1): def generator_eos(nbr_symbols, max_length, nbr_cases): """Shift by NUM_RESERVED_IDS and append EOS token.""" for case in self.generator(nbr_symbols, max_length, nbr_cases): new_case = {} for feature in case: new_case[feature] = [ i + text_encoder.NUM_RESERVED_TOKENS for i in case[feature] ] + [text_encoder.EOS_ID] yield new_case utils.generate_dataset_and_shuffle( generator_eos(self.num_symbols, self.train_length, self.train_size), self.training_filepaths(data_dir, self.num_shards, shuffled=True), generator_eos(self.num_symbols, self.dev_length, self.dev_size), self.dev_filepaths(data_dir, 1, shuffled=True), shuffle=False)
Example #12
Source File: common_voice.py From fine-lm with MIT License | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #13
Source File: librispeech.py From fine-lm with MIT License | 6 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): train_paths = self.training_filepaths( data_dir, self.num_shards, shuffled=False) dev_paths = self.dev_filepaths( data_dir, self.num_dev_shards, shuffled=False) test_paths = self.test_filepaths( data_dir, self.num_test_shards, shuffled=True) generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TEST_DATASETS), test_paths) if self.use_train_shards_for_dev: all_paths = train_paths + dev_paths generator_utils.generate_files( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), all_paths) generator_utils.shuffle_dataset(all_paths) else: generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, self.TRAIN_DATASETS), train_paths, self.generator(data_dir, tmp_dir, self.DEV_DATASETS), dev_paths)
Example #14
Source File: image_lsun.py From BERT with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): """Generates LSUN bedrooms dataset and writes it in data_dir.""" generator_utils.generate_dataset_and_shuffle( self.read_and_convert_to_png(tmp_dir, "train"), self.training_filepaths(data_dir, 100, shuffled=False), self.read_and_convert_to_png(tmp_dir, "val"), self.dev_filepaths(data_dir, 1, shuffled=False))
Example #15
Source File: video_utils.py From fine-lm with MIT License | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #16
Source File: vqa.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, problem.DatasetSplit.TRAIN), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, problem.DatasetSplit.EVAL), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #17
Source File: imagenet.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=True), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=True))
Example #18
Source File: imagenet.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=True), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=True))
Example #19
Source File: image_lsun.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): """Generates LSUN bedrooms dataset and writes it in data_dir.""" generator_utils.generate_dataset_and_shuffle( self.read_and_convert_to_png(tmp_dir, "train"), self.training_filepaths(data_dir, 100, shuffled=False), self.read_and_convert_to_png(tmp_dir, "val"), self.dev_filepaths(data_dir, 1, shuffled=False))
Example #20
Source File: image_utils.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #21
Source File: ice_parsing.py From fine-lm with MIT License | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( tabbed_parsing_token_generator(data_dir, tmp_dir, True, "ice", self.source_vocab_size, self.targeted_vocab_size), self.training_filepaths(data_dir, self.num_shards, shuffled=False), tabbed_parsing_token_generator(data_dir, tmp_dir, False, "ice", self.source_vocab_size, self.targeted_vocab_size), self.dev_filepaths(data_dir, 1, shuffled=False))
Example #22
Source File: ice_parsing.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( tabbed_parsing_token_generator(data_dir, tmp_dir, True, "ice", self.source_vocab_size, self.targeted_vocab_size), self.training_filepaths(data_dir, self.num_shards, shuffled=False), tabbed_parsing_token_generator(data_dir, tmp_dir, False, "ice", self.source_vocab_size, self.targeted_vocab_size), self.dev_filepaths(data_dir, 1, shuffled=False))
Example #23
Source File: video_utils.py From training_results_v0.5 with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #24
Source File: vqa.py From BERT with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, problem.DatasetSplit.TRAIN), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, problem.DatasetSplit.EVAL), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #25
Source File: imagenet.py From BERT with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=True), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=True))
Example #26
Source File: imagenet.py From BERT with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=True), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=True))
Example #27
Source File: video_utils.py From tensor2tensor with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #28
Source File: image_utils.py From BERT with Apache License 2.0 | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #29
Source File: image_utils.py From fine-lm with MIT License | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))
Example #30
Source File: image_utils.py From fine-lm with MIT License | 5 votes |
def generate_data(self, data_dir, tmp_dir, task_id=-1): generator_utils.generate_dataset_and_shuffle( self.generator(data_dir, tmp_dir, True), self.training_filepaths(data_dir, self.train_shards, shuffled=False), self.generator(data_dir, tmp_dir, False), self.dev_filepaths(data_dir, self.dev_shards, shuffled=False))