Python tensorflow.Event() Examples

def _setup_graph(self):
        # special heuristics for Horovod
        from ..train import HorovodTrainer
        if isinstance(self.trainer, HorovodTrainer):
            if self.trainer.mpi_enabled():
                logger.warn("GPUUtilizationTracker is disabled under MPI.")
                self._enabled = False
                return
            else:
                self._devices = [self.trainer.hvd.local_rank()]

        if self._devices is None:
            self._devices = self._guess_devices()
        assert len(self._devices), "[GPUUtilizationTracker] No GPU device given!"

        self._evt = mp.Event()
        self._stop_evt = mp.Event()
        self._queue = mp.Queue()
        self._proc = mp.Process(target=self.worker, args=(
            self._evt, self._queue, self._stop_evt, self._devices))
        ensure_proc_terminate(self._proc)
        start_proc_mask_signal(self._proc) 

def on_graph_def(self, graph_def, device_name, wall_time):
    """Implementation of the GraphDef-carrying Event proto callback.

    Args:
      graph_def: A GraphDef proto. N.B.: The GraphDef is from
        the core runtime of a debugged Session::Run() call, after graph
        partition. Therefore it may differ from the GraphDef available to
        the general TensorBoard. For example, the GraphDef in general
        TensorBoard may get partitioned for multiple devices (CPUs and GPUs),
        each of which will generate a GraphDef event proto sent to this
        method.
      device_name: Name of the device on which the graph was created.
      wall_time: An epoch timestamp (in microseconds) for the graph.
    """
    # For now, we do nothing with the graph def. However, we must define this
    # method to satisfy the handler's interface. Furthermore, we may use the
    # graph in the future (for instance to provide a graph if there is no graph
    # provided otherwise).
    del device_name
    del wall_time
    del graph_def 

def _setup_graph(self):
        # special heuristics for Horovod
        from ..train import HorovodTrainer
        if isinstance(self.trainer, HorovodTrainer):
            if self.trainer.mpi_enabled():
                logger.warn("GPUUtilizationTracker is disabled under MPI.")
                self._enabled = False
                return
            else:
                self._devices = [self.trainer.hvd.local_rank()]

        if self._devices is None:
            self._devices = self._guess_devices()
        assert len(self._devices), "[GPUUtilizationTracker] No GPU device given!"

        self._evt = mp.Event()
        self._stop_evt = mp.Event()
        self._queue = mp.Queue()
        self._proc = mp.Process(target=self.worker, args=(
            self._evt, self._queue, self._stop_evt, self._devices))
        ensure_proc_terminate(self._proc)
        start_proc_mask_signal(self._proc) 

def Load(self):
    """Loads all new values from disk.

    Calling Load multiple times in a row will not 'drop' events as long as the
    return value is not iterated over.

    Yields:
      All values that were written to disk that have not been yielded yet.
    """
    tf.logging.debug('Loading events from %s', self._file_path)
    while True:
      try:
        with tf.errors.raise_exception_on_not_ok_status() as status:
          self._reader.GetNext(status)
      except (tf.errors.DataLossError, tf.errors.OutOfRangeError):
        # We ignore partial read exceptions, because a record may be truncated.
        # PyRecordReader holds the offset prior to the failed read, so retrying
        # will succeed.
        break
      event = tf.Event()
      event.ParseFromString(self._reader.record())
      yield event
    tf.logging.debug('No more events in %s', self._file_path) 

def tb_add_histogram(experiment, name, wall_time, step, histo):
  # Tensorflow does not support key being unicode
  histo_string = {}
  for k,v in histo.items():
    histo_string[str(k)] = v
  histo = histo_string

  writer = tb_get_xp_writer(experiment)
  summary = tf.Summary(value=[
      tf.Summary.Value(tag=name, histo=histo),
  ])
  event = tf.Event(wall_time=wall_time, step=step, summary=summary)
  writer.add_event(event)
  writer.flush()
  tb_modified_xp(experiment, modified_type="histograms", wall_time=wall_time)

# Perform requests to tensorboard http api 

def on_graph_def(self, graph_def, device_name, wall_time):
        """Implementation of the GraphDef-carrying Event proto callback.

        Args:
          graph_def: A GraphDef proto. N.B.: The GraphDef is from
            the core runtime of a debugged Session::Run() call, after graph
            partition. Therefore it may differ from the GraphDef available to
            the general TensorBoard. For example, the GraphDef in general
            TensorBoard may get partitioned for multiple devices (CPUs and GPUs),
            each of which will generate a GraphDef event proto sent to this
            method.
          device_name: Name of the device on which the graph was created.
          wall_time: An epoch timestamp (in microseconds) for the graph.
        """
        # For now, we do nothing with the graph def. However, we must define this
        # method to satisfy the handler's interface. Furthermore, we may use the
        # graph in the future (for instance to provide a graph if there is no graph
        # provided otherwise).
        del wall_time
        self._graph_defs[device_name] = graph_def

        if not self._graph_defs_arrive_first:
            self._add_graph_def(device_name, graph_def)
            self._incoming_channel.get() 

def _write_event(self, metadata):
        evt = tf.Event()
        evt.tagged_run_metadata.tag = 'trace-{}'.format(self.global_step)
        evt.tagged_run_metadata.run_metadata = metadata.SerializeToString()
        self.trainer.monitors.put_event(evt) 

def WriteScalarSeries(writer, tag, f, n=5):
  """Write a series of scalar events to writer, using f to create values."""
  step = 0
  wall_time = _start_time
  for i in xrange(n):
    v = f(i)
    value = tf.Summary.Value(tag=tag, simple_value=v)
    summary = tf.Summary(value=[value])
    event = tf.Event(wall_time=wall_time, step=step, summary=summary)
    writer.add_event(event)
    step += 1
    wall_time += 10 

def on_core_metadata_event(self, event):
    """Implementation of the core metadata-carrying Event proto callback.

    Args:
      event: An Event proto that contains core metadata about the debugged
        Session::Run() in its log_message.message field, as a JSON string.
        See the doc string of debug_data.DebugDumpDir.core_metadata for details.
    """
    self._session_run_index = self._parse_session_run_index(event) 

def tb_add_scalar(experiment, name, wall_time, step, value):
  writer = tb_get_xp_writer(experiment)
  summary = tf.Summary(value=[
      tf.Summary.Value(tag=name, simple_value=value),
  ])
  event = tf.Event(wall_time=wall_time, step=step, summary=summary)
  writer.add_event(event)
  writer.flush()
  tb_modified_xp(experiment, modified_type="scalars", wall_time=wall_time) 

def _CreateEventWithDebugNumericSummary(
        self, device_name, op_name, output_slot, wall_time, step, list_of_values
    ):
        """Creates event with a health pill summary.

        Note the debugger plugin only works with TensorFlow and, thus, uses TF
        protos and TF EventsWriter.

        Args:
          device_name: The name of the op's device.
          op_name: The name of the op to which a DebugNumericSummary was attached.
          output_slot: The numeric output slot for the tensor.
          wall_time: The numeric wall time of the event.
          step: The step of the event.
          list_of_values: A python list of values within the tensor.

        Returns:
          A `tf.Event` with a health pill summary.
        """
        event = tf.compat.v1.Event(step=step, wall_time=wall_time)
        tensor = tf.compat.v1.make_tensor_proto(
            list_of_values, dtype=tf.float64, shape=[len(list_of_values)]
        )
        value = event.summary.value.add(
            tag=op_name,
            node_name="%s:%d:DebugNumericSummary" % (op_name, output_slot),
            tensor=tensor,
        )
        content_proto = debugger_event_metadata_pb2.DebuggerEventMetadata(
            device=device_name, output_slot=output_slot
        )
        value.metadata.plugin_data.plugin_name = constants.DEBUGGER_PLUGIN_NAME
        value.metadata.plugin_data.content = tf.compat.as_bytes(
            json_format.MessageToJson(
                content_proto, including_default_value_fields=True
            )
        )
        return event 

def on_core_metadata_event(self, event):
        """Implementation of the core metadata-carrying Event proto callback.

        Args:
          event: An Event proto that contains core metadata about the debugged
            Session::Run() in its log_message.message field, as a JSON string.
            See the doc string of debug_data.DebugDumpDir.core_metadata for details.
        """
        core_metadata = json.loads(event.log_message.message)
        input_names = ",".join(core_metadata["input_names"])
        output_names = ",".join(core_metadata["output_names"])
        target_nodes = ",".join(core_metadata["target_nodes"])

        self._run_key = RunKey(input_names, output_names, target_nodes)
        if not self._graph_defs:
            self._graph_defs_arrive_first = False
        else:
            for device_name in self._graph_defs:
                self._add_graph_def(device_name, self._graph_defs[device_name])

        self._outgoing_channel.put(
            _comm_metadata(self._run_key, event.wall_time)
        )

        # Wait for acknowledgement from client. Blocks until an item is got.
        logger.info("on_core_metadata_event() waiting for client ack (meta)...")
        self._incoming_channel.get()
        logger.info("on_core_metadata_event() client ack received (meta).")

        # TODO(cais): If eager mode, this should return something to yield. 

def _extract_device_name_from_event(event):
    """Extract device name from a tf.Event proto carrying tensor value."""
    plugin_data_content = json.loads(
        tf.compat.as_str(event.summary.value[0].metadata.plugin_data.content)
    )
    return plugin_data_content["device"] 

def WriteHistogramSeries(writer, tag, mu_sigma_tuples, n=20):
    """Write a sequence of normally distributed histograms to writer."""
    step = 0
    wall_time = _start_time
    for [mean, stddev] in mu_sigma_tuples:
        data = [random.normalvariate(mean, stddev) for _ in xrange(n)]
        histo = _MakeHistogram(data)
        summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=histo)])
        event = tf.Event(wall_time=wall_time, step=step, summary=summary)
        writer.add_event(event)
        step += 10
        wall_time += 100 

def WriteScalarSeries(writer, tag, f, n=5):
    """Write a series of scalar events to writer, using f to create values."""
    step = 0
    wall_time = _start_time
    for i in xrange(n):
        v = f(i)
        value = tf.Summary.Value(tag=tag, simple_value=v)
        summary = tf.Summary(value=[value])
        event = tf.Event(wall_time=wall_time, step=step, summary=summary)
        writer.add_event(event)
        step += 1
        wall_time += 10 

def _write_event(self, metadata):
        evt = tf.Event()
        evt.tagged_run_metadata.tag = 'trace-{}'.format(self.global_step)
        evt.tagged_run_metadata.run_metadata = metadata.SerializeToString()
        self.trainer.monitors.put_event(evt) 

def _write_event(self, metadata):
        evt = tf.Event()
        evt.tagged_run_metadata.tag = 'trace-{}'.format(self.global_step)
        evt.tagged_run_metadata.run_metadata = metadata.SerializeToString()
        self.trainer.monitors.put_event(evt) 

def _before_train(self):
        assert gpu_available_in_session(), "[GPUUtilizationTracker] needs GPU!"
        self._evt = mp.Event()
        self._stop_evt = mp.Event()
        self._queue = mp.Queue()
        self._proc = mp.Process(target=self.worker, args=(
            self._evt, self._queue, self._stop_evt))
        ensure_proc_terminate(self._proc)
        start_proc_mask_signal(self._proc) 

def WriteHistogramSeries(writer, tag, mu_sigma_tuples, n=20):
  """Write a sequence of normally distributed histograms to writer."""
  step = 0
  wall_time = _start_time
  for [mean, stddev] in mu_sigma_tuples:
    data = [random.normalvariate(mean, stddev) for _ in xrange(n)]
    histo = _MakeHistogram(data)
    summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=histo)])
    event = tf.Event(wall_time=wall_time, step=step, summary=summary)
    writer.add_event(event)
    step += 10
    wall_time += 100 

def on_value_event(self, event):
    """Records the summary values based on an updated message from the debugger.

    Logs an error message if writing the event to disk fails.

    Args:
      event: The Event proto to be processed.
    """
    if not event.summary.value:
      tf.logging.warn("The summary of the event lacks a value.")
      return

    # The node name property is actually a watch key, which is a concatenation
    # of several pieces of data.
    watch_key = event.summary.value[0].node_name
    if not watch_key.endswith(constants.DEBUG_NUMERIC_SUMMARY_SUFFIX):
      # Ignore events that lack a DebugNumericSummary.
      # NOTE(@chihuahua): We may later handle other types of debug ops.
      return

    # We remove the constants.DEBUG_NUMERIC_SUMMARY_SUFFIX from the end of the
    # watch name because it is not distinguishing: every health pill entry ends
    # with it.
    node_name_and_output_slot = watch_key[
        :-len(constants.DEBUG_NUMERIC_SUMMARY_SUFFIX)]

    shape = tf.make_ndarray(event.summary.value[0].tensor).shape
    if (len(shape) != 1 or
        shape[0] < constants.MIN_DEBUG_NUMERIC_SUMMARY_TENSOR_LENGTH):
      tf.logging.warning("Health-pill tensor either lacks a dimension or is "
                         "shaped incorrectly: %s" % shape)
      return

    match = re.match(r"^(.*):(\d+)$", node_name_and_output_slot)
    if not match:
      tf.logging.warning(
          ("A event with a health pill has an invalid node name and output "
           "slot combination, (i.e., an unexpected debug op): %r"),
          node_name_and_output_slot)
      return

    if self._session_run_index >= 0:
      event.step = self._session_run_index
    else:
      # Data from parameter servers (or any graphs without a master) do not
      # contain core metadata. So the session run count is missing. Set its
      # value to a microsecond epoch timestamp.
      event.step = int(time.time() * 1e6)

    # Write this event to the events file designated for data from the
    # debugger.
    self._events_writer_manager.write_event(event)

    alert = numerics_alert.extract_numerics_alert(event)
    if self._numerics_alert_callback and alert:
      self._numerics_alert_callback(alert)