Python tensorflow.logging.set_verbosity() Examples
The following are 30
code examples of tensorflow.logging.set_verbosity().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
tensorflow.logging
, or try the search function
.
.
Example #1
| Source File: inference-pre-ensemble-get-input.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.model_checkpoint_path, FLAGS.input_data_pattern,
FLAGS.output_dir, FLAGS.batch_size, FLAGS.top_k)
Example #2
| Source File: inference.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #3
| Source File: inference-layer.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #4
| Source File: inference_autoencoder.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #5
| Source File: utils.py From dvt with GNU General Public License v2.0 | 6 votes |
|
def setup_tensorflow():
"""Setup options for TensorFlow.
These options should allow most users to run TensorFlow with either a
GPU or CPU. It sets several options to avoid keras taking up too much
memory space and ignore a common warnings about library conflicts that
can occur on macOS. It also silences verbose warnings from TensorFlow
that most users can safely ignore.
"""
from keras.backend.tensorflow_backend import set_session
from tensorflow import logging, ConfigProto, Session
from os import environ
# supress warnings
logging.set_verbosity(logging.ERROR)
environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
# ensure that keras does not use all of the available memory
config = ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.3
config.gpu_options.visible_device_list = "0"
set_session(Session(config=config))
# fix a common local bug
environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
Example #6
| Source File: inference.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #7
| Source File: inference_test.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #8
| Source File: inference-pre-ensemble.py From youtube-8m with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.model_checkpoint_path, FLAGS.input_data_pattern,
FLAGS.output_dir, FLAGS.batch_size, FLAGS.top_k)
Example #9
| Source File: inference.py From Y8M with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.checkpoint_file, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #10
| Source File: inference.py From Youtube-8M-WILLOW with Apache License 2.0 | 6 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #11
| Source File: inference.py From youtube8mchallenge with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(open(flags_dict_file).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #12
| Source File: run_preprocessing.py From professional-services with Apache License 2.0 | 5 votes |
|
def main(): logging.set_verbosity(getattr(logging, FLAGS.logging_verbosity)) run(FLAGS)
Example #13
| Source File: convert_prediction_from_json_to_csv.py From Y8M with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if not FLAGS.json_prediction_files_pattern:
raise ValueError(
"The flag --json_prediction_files_pattern must be specified.")
if not FLAGS.csv_output_file:
raise ValueError("The flag --csv_output_file must be specified.")
logging.info("Looking for prediction files with pattern: %s",
FLAGS.json_prediction_files_pattern)
file_paths = gfile.Glob(FLAGS.json_prediction_files_pattern)
logging.info("Found files: %s", file_paths)
logging.info("Writing submission file to: %s", FLAGS.csv_output_file)
with gfile.Open(FLAGS.csv_output_file, "w+") as output_file:
output_file.write(get_csv_header())
for file_path in file_paths:
logging.info("processing file: %s", file_path)
with gfile.Open(file_path) as input_file:
for line in input_file:
json_data = json.loads(line)
output_file.write(to_csv_row(json_data))
output_file.flush()
logging.info("done")
Example #14
| Source File: split_video.py From Y8M with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.ERROR)
print("tensorflow version: %s" % tf.__version__)
all_frame_files = gfile.Glob(FLAGS.input_data_pattern)
f_fullpath = all_frame_files[FLAGS.file_from : FLAGS.file_to]
f_fns = [x.split('/')[-1] for x in f_fullpath]
exist_files = gfile.Glob(FLAGS.output_path + "E*tfrecord")
exist_fn = [x.split('/')[-1].replace('Etr', 'tr') for x in exist_files]
exist_fn = [x.split('/')[-1].replace('Eval', 'val') for x in exist_fn]
exist_fn = [x.split('/')[-1].replace('Etes', 'tes') for x in exist_fn]
yet_2_split = [x for x,y in zip(f_fullpath, f_fns) if y not in exist_fn]
vf0 = [FLAGS.output_path + 'O' + x.split('/')[-1] for x in yet_2_split]
vf1 = [FLAGS.output_path + 'E' + x.split('/')[-1] for x in yet_2_split]
vf2 = [FLAGS.output_path + 'F' + x.split('/')[-1] for x in yet_2_split]
mylog('number of files suggested: %d'%len(f_fullpath))
mylog('number of files yet to process: %d'%len(yet_2_split))
#with tf.device("/gpu:0"):
with tf.Graph().as_default():
build_graph()
split_files(zip(yet_2_split, vf0, vf1, vf2))
mylog("done")
Example #15
| Source File: eval.py From Y8M with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
Example #16
| Source File: inference.py From Y8M with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("START FLAGS===========================")
print("train_dir: " +str(FLAGS.train_dir))
print("output_file: " +str(FLAGS.output_file))
print("input_data_pattern: " + str(FLAGS.input_data_pattern))
print("frame_features: " + str(FLAGS.frame_features))
print("batch_size: " +str(FLAGS.batch_size))
print("feature_names: " + str(FLAGS.feature_names))
print("feature_sizes: " + str(FLAGS.feature_sizes))
print("c_vars: " + str(FLAGS.c_vars))
print("num_readers: " + str(FLAGS.num_readers))
print("top_k: " + str(FLAGS.top_k))
print("layers_keep_probs: " + str(FLAGS.layers_keep_probs))
print("gpu_only: " + str(FLAGS.gpu_only))
print("END FLAGS ============================")
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes,
num_classes=FLAGS.truncated_num_classes,
decode_zlib=FLAGS.decode_zlib)
else:
reader = readers.YT8MAggregatedFeatureReader(
num_classes = FLAGS.truncated_num_classes,
decode_zlib = FLAGS.decode_zlib,
feature_names=feature_names, feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #17
| Source File: convert_prediction_from_json_to_csv.py From Y8M with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if not FLAGS.json_prediction_files_pattern:
raise ValueError(
"The flag --json_prediction_files_pattern must be specified.")
if not FLAGS.csv_output_file:
raise ValueError("The flag --csv_output_file must be specified.")
logging.info("Looking for prediction files with pattern: %s",
FLAGS.json_prediction_files_pattern)
file_paths = gfile.Glob(FLAGS.json_prediction_files_pattern)
logging.info("Found files: %s", file_paths)
logging.info("Writing submission file to: %s", FLAGS.csv_output_file)
with gfile.Open(FLAGS.csv_output_file, "w+") as output_file:
output_file.write(get_csv_header())
for file_path in file_paths:
logging.info("processing file: %s", file_path)
with gfile.Open(file_path) as input_file:
for line in input_file:
json_data = json.loads(line)
output_file.write(to_csv_row(json_data))
output_file.flush()
logging.info("done")
Example #18
| Source File: train.py From AttentionCluster with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features,
model=model,
reader=reader)
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
Example #19
| Source File: eval.py From AttentionCluster with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
evaluate()
Example #20
| Source File: inference.py From AttentionCluster with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not file_io.file_exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(file_io.FileIO(flags_dict_file, "r").read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #21
| Source File: eval_custom.py From youtube8mchallenge with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
evaluate()
Example #22
| Source File: convert_prediction_from_json_to_csv.py From youtube8mchallenge with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if not FLAGS.json_prediction_files_pattern:
raise ValueError(
"The flag --json_prediction_files_pattern must be specified.")
if not FLAGS.csv_output_file:
raise ValueError("The flag --csv_output_file must be specified.")
logging.info("Looking for prediction files with pattern: %s",
FLAGS.json_prediction_files_pattern)
file_paths = gfile.Glob(FLAGS.json_prediction_files_pattern)
logging.info("Found files: %s", file_paths)
logging.info("Writing submission file to: %s", FLAGS.csv_output_file)
with gfile.Open(FLAGS.csv_output_file, "w+") as output_file:
output_file.write(get_csv_header())
for file_path in file_paths:
logging.info("processing file: %s", file_path)
with gfile.Open(file_path) as input_file:
for line in input_file:
json_data = json.loads(line)
output_file.write(to_csv_row(json_data))
output_file.flush()
logging.info("done")
Example #23
| Source File: inference_gpu.py From youtube8mchallenge with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(open(flags_dict_file).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #24
| Source File: train.py From youtube-8m with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features, model=model, reader=reader)
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
Example #25
| Source File: inference.py From youtube-8m with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(file_io.FileIO(flags_dict_file, "r").read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if not FLAGS.output_file:
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if not FLAGS.input_data_pattern:
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
Example #26
| Source File: convert_prediction_from_json_to_csv.py From youtube-8m with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if not FLAGS.json_prediction_files_pattern:
raise ValueError(
"The flag --json_prediction_files_pattern must be specified.")
if not FLAGS.csv_output_file:
raise ValueError("The flag --csv_output_file must be specified.")
logging.info("Looking for prediction files with pattern: %s",
FLAGS.json_prediction_files_pattern)
file_paths = gfile.Glob(FLAGS.json_prediction_files_pattern)
logging.info("Found files: %s", file_paths)
logging.info("Writing submission file to: %s", FLAGS.csv_output_file)
with gfile.Open(FLAGS.csv_output_file, "w+") as output_file:
output_file.write(get_csv_header())
for file_path in file_paths:
logging.info("processing file: %s", file_path)
with gfile.Open(file_path) as input_file:
for line in input_file:
json_data = json.loads(line)
output_file.write(to_csv_row(json_data))
output_file.flush()
logging.info("done")
Example #27
| Source File: train.py From youtube-8m with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
Example #28
| Source File: train-with-predictions.py From youtube-8m with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
Trainer(cluster, task, FLAGS.train_dir, FLAGS.log_device_placement).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
Example #29
| Source File: tensorboard_handle.py From dataiku-contrib with Apache License 2.0 | 5 votes |
|
def __init__(self, folder_name, host="127.0.0.1", verbosity=logging.WARN):
Thread.__init__(self)
self.project_key = os.environ["DKU_CURRENT_PROJECT_KEY"]
self.folder_name = folder_name
self.client = dataiku.api_client()
logging.set_verbosity(verbosity)
# Getting app
logs_path = self.__get_logs_path()
app = self.__get_tb_app(logs_path)
# Setting server
self.srv = make_server(host, 0, app)
Example #30
| Source File: prepare_distill_dataset.py From youtube8mchallenge with Apache License 2.0 | 5 votes |
|
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
inference_loop()
