Python object_detection.utils.metrics.classes() Examples

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
    """
    detection_classes = detections_dict[
        standard_fields.DetectionResultFields.detection_classes]
    detection_classes -= self._label_id_offset
    self._evaluation.add_single_detected_image_info(
        image_id,
        detections_dict[standard_fields.DetectionResultFields.detection_boxes],
        detections_dict[standard_fields.DetectionResultFields.detection_scores],
        detection_classes) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    if image_id not in self._image_ids:
      # Since for the correct work of evaluator it is assumed that groundtruth
      # is inserted first we make sure to break the code if is it not the case.
      self._image_ids.update([image_id])
      self._evaluatable_labels[image_id] = np.array([])

    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    allowed_classes = np.where(
        np.isin(detection_classes, self._evaluatable_labels[image_id]))
    detection_classes = detection_classes[allowed_classes]
    detected_boxes = detections_dict[
        standard_fields.DetectionResultFields.detection_boxes][allowed_classes]
    detected_scores = detections_dict[
        standard_fields.DetectionResultFields.detection_scores][allowed_classes]

    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detected_boxes,
        detected_scores=detected_scores,
        detected_class_labels=detection_classes) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy array
          of shape [num_boxes, height, width] containing `num_boxes` masks of
          values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    if image_id not in self._image_ids:
      # Since for the correct work of evaluator it is assumed that groundtruth
      # is inserted first we make sure to break the code if is it not the case.
      self._image_ids.update([image_id])
      self._evaluatable_labels[image_id] = np.array([])

    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    allowed_classes = np.where(
        np.isin(detection_classes, self._evaluatable_labels[image_id]))
    detection_classes = detection_classes[allowed_classes]
    detected_boxes = detections_dict[
        standard_fields.DetectionResultFields.detection_boxes][allowed_classes]
    detected_scores = detections_dict[
        standard_fields.DetectionResultFields.detection_scores][allowed_classes]

    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detected_boxes,
        detected_scores=detected_scores,
        detected_class_labels=detection_classes) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
    """Adds groundtruth for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      groundtruth_dict: A dictionary containing -
        standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array
          of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of
          the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.InputDataFields.groundtruth_classes: integer numpy array
          of shape [num_boxes] containing 1-indexed groundtruth classes for the
          boxes.
        standard_fields.InputDataFields.groundtruth_image_classes: integer 1D
          numpy array containing all classes for which labels are verified.
        standard_fields.InputDataFields.groundtruth_group_of: Optional length M
          numpy boolean array denoting whether a groundtruth box contains a
          group of instances.

    Raises:
      ValueError: On adding groundtruth for an image more than once.
    """
    super(OpenImagesChallengeEvaluator,
          self).add_single_ground_truth_image_info(image_id, groundtruth_dict)
    groundtruth_classes = (
        groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes] -
        self._label_id_offset)
    self._evaluatable_labels[image_id] = np.unique(
        np.concatenate(((groundtruth_dict.get(
            standard_fields.InputDataFields.groundtruth_image_classes,
            np.array([], dtype=int)) - self._label_id_offset),
                        groundtruth_classes))) 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    if image_id not in self._image_ids:
      # Since for the correct work of evaluator it is assumed that groundtruth
      # is inserted first we make sure to break the code if is it not the case.
      self._image_ids.update([image_id])
      self._evaluatable_labels[image_id] = np.array([])

    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    allowed_classes = np.where(
        np.isin(detection_classes, self._evaluatable_labels[image_id]))
    detection_classes = detection_classes[allowed_classes]
    detected_boxes = detections_dict[
        standard_fields.DetectionResultFields.detection_boxes][allowed_classes]
    detected_scores = detections_dict[
        standard_fields.DetectionResultFields.detection_scores][allowed_classes]

    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detected_boxes,
        detected_scores=detected_scores,
        detected_class_labels=detection_classes) 

def __init__(self,
               categories,
               matching_iou_threshold=0.5,
               evaluate_corlocs=False,
               metric_prefix=None,
               use_weighted_mean_ap=False):
    """Constructor.

    Args:
      categories: A list of dicts, each of which has the following keys -
        'id': (required) an integer id uniquely identifying this category.
        'name': (required) string representing category name e.g., 'cat', 'dog'.
      matching_iou_threshold: IOU threshold to use for matching groundtruth
        boxes to detection boxes.
      evaluate_corlocs: (optional) boolean which determines if corloc scores
        are to be returned or not.
      metric_prefix: (optional) string prefix for metric name; if None, no
        prefix is used.
      use_weighted_mean_ap: (optional) boolean which determines if the mean
        average precision is computed directly from the scores and tp_fp_labels
        of all classes.
    """
    super(ObjectDetectionEvaluator, self).__init__(categories)
    self._num_classes = max([cat['id'] for cat in categories])
    self._matching_iou_threshold = matching_iou_threshold
    self._use_weighted_mean_ap = use_weighted_mean_ap
    self._label_id_offset = 1
    self._evaluation = ObjectDetectionEvaluation(
        self._num_classes,
        matching_iou_threshold=self._matching_iou_threshold,
        use_weighted_mean_ap=self._use_weighted_mean_ap,
        label_id_offset=self._label_id_offset)
    self._image_ids = set([])
    self._evaluate_corlocs = evaluate_corlocs
    self._metric_prefix = (metric_prefix + '/') if metric_prefix else '' 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    if image_id not in self._image_ids:
      # Since for the correct work of evaluator it is assumed that groundtruth
      # is inserted first we make sure to break the code if is it not the case.
      self._image_ids.update([image_id])
      self._evaluatable_labels[image_id] = np.array([])

    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    allowed_classes = np.where(
        np.isin(detection_classes, self._evaluatable_labels[image_id]))
    detection_classes = detection_classes[allowed_classes]
    detected_boxes = detections_dict[
        standard_fields.DetectionResultFields.detection_boxes][allowed_classes]
    detected_scores = detections_dict[
        standard_fields.DetectionResultFields.detection_scores][allowed_classes]

    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detected_boxes,
        detected_scores=detected_scores,
        detected_class_labels=detection_classes) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    if image_id not in self._image_ids:
      # Since for the correct work of evaluator it is assumed that groundtruth
      # is inserted first we make sure to break the code if is it not the case.
      self._image_ids.update([image_id])
      self._evaluatable_labels[image_id] = np.array([])

    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    allowed_classes = np.where(
        np.isin(detection_classes, self._evaluatable_labels[image_id]))
    detection_classes = detection_classes[allowed_classes]
    detected_boxes = detections_dict[
        standard_fields.DetectionResultFields.detection_boxes][allowed_classes]
    detected_scores = detections_dict[
        standard_fields.DetectionResultFields.detection_scores][allowed_classes]

    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detected_boxes,
        detected_scores=detected_scores,
        detected_class_labels=detection_classes) 

def add_single_detected_image_info(self, image_id, detections_dict):
    """Adds detections for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      detections_dict: A dictionary containing -
        standard_fields.DetectionResultFields.detection_boxes: float32 numpy
          array of shape [num_boxes, 4] containing `num_boxes` detection boxes
          of the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.DetectionResultFields.detection_scores: float32 numpy
          array of shape [num_boxes] containing detection scores for the boxes.
        standard_fields.DetectionResultFields.detection_classes: integer numpy
          array of shape [num_boxes] containing 1-indexed detection classes for
          the boxes.
        standard_fields.DetectionResultFields.detection_masks: uint8 numpy
          array of shape [num_boxes, height, width] containing `num_boxes` masks
          of values ranging between 0 and 1.

    Raises:
      ValueError: If detection masks are not in detections dictionary.
    """
    detection_classes = (
        detections_dict[standard_fields.DetectionResultFields.detection_classes]
        - self._label_id_offset)
    detection_masks = None
    if self._evaluate_masks:
      if (standard_fields.DetectionResultFields.detection_masks not in
          detections_dict):
        raise ValueError('Detection masks not in detections dictionary.')
      detection_masks = detections_dict[
          standard_fields.DetectionResultFields.detection_masks]
    self._evaluation.add_single_detected_image_info(
        image_key=image_id,
        detected_boxes=detections_dict[
            standard_fields.DetectionResultFields.detection_boxes],
        detected_scores=detections_dict[
            standard_fields.DetectionResultFields.detection_scores],
        detected_class_labels=detection_classes,
        detected_masks=detection_masks) 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold,
                category_index[idx + self._label_id_offset]['name']))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold,
                      category_index[idx + self._label_id_offset]['name']))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
    """Adds groundtruth for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      groundtruth_dict: A dictionary containing -
        standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array
          of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of
          the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.InputDataFields.groundtruth_classes: integer numpy array
          of shape [num_boxes] containing 1-indexed groundtruth classes for the
          boxes.
        standard_fields.InputDataFields.groundtruth_image_classes: integer 1D
          numpy array containing all classes for which labels are verified.
        standard_fields.InputDataFields.groundtruth_group_of: Optional length
          M numpy boolean array denoting whether a groundtruth box contains a
          group of instances.

    Raises:
      ValueError: On adding groundtruth for an image more than once.
    """
    super(OpenImagesDetectionChallengeEvaluator,
          self).add_single_ground_truth_image_info(image_id, groundtruth_dict)
    groundtruth_classes = (
        groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes] -
        self._label_id_offset)
    self._evaluatable_labels[image_id] = np.unique(
        np.concatenate(((groundtruth_dict.get(
            standard_fields.InputDataFields.groundtruth_image_classes,
            np.array([], dtype=int)) - self._label_id_offset),
                        groundtruth_classes))) 

def evaluate(self):
    """Compute evaluation result.

    Returns:
      A dictionary of metrics with the following fields -

      1. summary_metrics:
        'Precision/mAP@<matching_iou_threshold>IOU': mean average precision at
        the specified IOU threshold.

      2. per_category_ap: category specific results with keys of the form
        'PerformanceByCategory/mAP@<matching_iou_threshold>IOU/category'.
    """
    (per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc) = (
        self._evaluation.evaluate())
    pascal_metrics = {
        self._metric_prefix +
        'Precision/mAP@{}IOU'.format(self._matching_iou_threshold):
            mean_ap
    }
    if self._evaluate_corlocs:
      pascal_metrics[self._metric_prefix + 'Precision/meanCorLoc@{}IOU'.format(
          self._matching_iou_threshold)] = mean_corloc
    category_index = label_map_util.create_category_index(self._categories)
    for idx in range(per_class_ap.size):
      if idx + self._label_id_offset in category_index:
        category_name = category_index[idx + self._label_id_offset]['name']
        try:
          category_name = unicode(category_name, 'utf-8')
        except TypeError:
          pass
        category_name = unicodedata.normalize(
            'NFKD', category_name).encode('ascii', 'ignore')
        display_name = (
            self._metric_prefix + 'PerformanceByCategory/AP@{}IOU/{}'.format(
                self._matching_iou_threshold, category_name))
        pascal_metrics[display_name] = per_class_ap[idx]

        # Optionally add CorLoc metrics.classes
        if self._evaluate_corlocs:
          display_name = (
              self._metric_prefix + 'PerformanceByCategory/CorLoc@{}IOU/{}'
              .format(self._matching_iou_threshold, category_name))
          pascal_metrics[display_name] = per_class_corloc[idx]

    return pascal_metrics 

def add_single_ground_truth_image_info(self,
                                         image_key,
                                         groundtruth_boxes,
                                         groundtruth_class_labels,
                                         groundtruth_is_difficult_list=None,
                                         groundtruth_is_group_of_list=None,
                                         groundtruth_masks=None):
    """Adds groundtruth for a single image to be used for evaluation.

    Args:
      image_key: A unique string/integer identifier for the image.
      groundtruth_boxes: float32 numpy array of shape [num_boxes, 4] containing
        `num_boxes` groundtruth boxes of the format [ymin, xmin, ymax, xmax] in
        absolute image coordinates.
      groundtruth_class_labels: integer numpy array of shape [num_boxes]
        containing 0-indexed groundtruth classes for the boxes.
      groundtruth_is_difficult_list: A length M numpy boolean array denoting
        whether a ground truth box is a difficult instance or not. To support
        the case that no boxes are difficult, it is by default set as None.
      groundtruth_is_group_of_list: A length M numpy boolean array denoting
        whether a ground truth box is a group-of box or not. To support the case
        that no boxes are groups-of, it is by default set as None.
      groundtruth_masks: uint8 numpy array of shape [num_boxes, height, width]
        containing `num_boxes` groundtruth masks. The mask values range from 0
        to 1.
    """
    if image_key in self.groundtruth_boxes:
      logging.warning(
          'image %s has already been added to the ground truth database.',
          image_key)
      return

    self.groundtruth_boxes[image_key] = groundtruth_boxes
    self.groundtruth_class_labels[image_key] = groundtruth_class_labels
    self.groundtruth_masks[image_key] = groundtruth_masks
    if groundtruth_is_difficult_list is None:
      num_boxes = groundtruth_boxes.shape[0]
      groundtruth_is_difficult_list = np.zeros(num_boxes, dtype=bool)
    self.groundtruth_is_difficult_list[
        image_key] = groundtruth_is_difficult_list.astype(dtype=bool)
    if groundtruth_is_group_of_list is None:
      num_boxes = groundtruth_boxes.shape[0]
      groundtruth_is_group_of_list = np.zeros(num_boxes, dtype=bool)
    if groundtruth_masks is None:
      num_boxes = groundtruth_boxes.shape[0]
      mask_presence_indicator = np.zeros(num_boxes, dtype=bool)
    else:
      mask_presence_indicator = (np.sum(groundtruth_masks,
                                        axis=(1, 2)) == 0).astype(dtype=bool)

    self.groundtruth_is_group_of_list[
        image_key] = groundtruth_is_group_of_list.astype(dtype=bool)

    self._update_ground_truth_statistics(
        groundtruth_class_labels,
        groundtruth_is_difficult_list.astype(dtype=bool)
        | mask_presence_indicator,  # ignore boxes without masks
        groundtruth_is_group_of_list.astype(dtype=bool)) 

def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
    """Adds groundtruth for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      groundtruth_dict: A dictionary containing -
        standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array
          of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of
          the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.InputDataFields.groundtruth_classes: integer numpy array
          of shape [num_boxes] containing 1-indexed groundtruth classes for the
          boxes.
        standard_fields.InputDataFields.groundtruth_group_of: Optional length
          M numpy boolean array denoting whether a groundtruth box contains a
          group of instances.

    Raises:
      ValueError: On adding groundtruth for an image more than once.
    """
    if image_id in self._image_ids:
      raise ValueError('Image with id {} already added.'.format(image_id))

    groundtruth_classes = (
        groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes] -
        self._label_id_offset)
    # If the key is not present in the groundtruth_dict or the array is empty
    # (unless there are no annotations for the groundtruth on this image)
    # use values from the dictionary or insert None otherwise.
    if (standard_fields.InputDataFields.groundtruth_group_of in
        groundtruth_dict.keys() and
        (groundtruth_dict[standard_fields.InputDataFields.groundtruth_group_of]
         .size or not groundtruth_classes.size)):
      groundtruth_group_of = groundtruth_dict[
          standard_fields.InputDataFields.groundtruth_group_of]
    else:
      groundtruth_group_of = None
      if not len(self._image_ids) % 1000:
        logging.warn(
            'image %s does not have groundtruth group_of flag specified',
            image_id)
    self._evaluation.add_single_ground_truth_image_info(
        image_id,
        groundtruth_dict[standard_fields.InputDataFields.groundtruth_boxes],
        groundtruth_classes,
        groundtruth_is_difficult_list=None,
        groundtruth_is_group_of_list=groundtruth_group_of)
    self._image_ids.update([image_id]) 

def add_single_ground_truth_image_info(self, image_id, groundtruth_dict):
    """Adds groundtruth for a single image to be used for evaluation.

    Args:
      image_id: A unique string/integer identifier for the image.
      groundtruth_dict: A dictionary containing -
        standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array
          of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of
          the format [ymin, xmin, ymax, xmax] in absolute image coordinates.
        standard_fields.InputDataFields.groundtruth_classes: integer numpy array
          of shape [num_boxes] containing 1-indexed groundtruth classes for the
          boxes.
        standard_fields.InputDataFields.groundtruth_difficult: Optional length
          M numpy boolean array denoting whether a ground truth box is a
          difficult instance or not. This field is optional to support the case
          that no boxes are difficult.
        standard_fields.InputDataFields.groundtruth_instance_masks: Optional
          numpy array of shape [num_boxes, height, width] with values in {0, 1}.

    Raises:
      ValueError: On adding groundtruth for an image more than once. Will also
        raise error if instance masks are not in groundtruth dictionary.
    """
    if image_id in self._image_ids:
      raise ValueError('Image with id {} already added.'.format(image_id))

    groundtruth_classes = (
        groundtruth_dict[standard_fields.InputDataFields.groundtruth_classes] -
        self._label_id_offset)
    # If the key is not present in the groundtruth_dict or the array is empty
    # (unless there are no annotations for the groundtruth on this image)
    # use values from the dictionary or insert None otherwise.
    if (standard_fields.InputDataFields.groundtruth_difficult in
        groundtruth_dict.keys() and
        (groundtruth_dict[standard_fields.InputDataFields.groundtruth_difficult]
         .size or not groundtruth_classes.size)):
      groundtruth_difficult = groundtruth_dict[
          standard_fields.InputDataFields.groundtruth_difficult]
    else:
      groundtruth_difficult = None
      if not len(self._image_ids) % 1000:
        logging.warn(
            'image %s does not have groundtruth difficult flag specified',
            image_id)
    groundtruth_masks = None
    if self._evaluate_masks:
      if (standard_fields.InputDataFields.groundtruth_instance_masks not in
          groundtruth_dict):
        raise ValueError('Instance masks not in groundtruth dictionary.')
      groundtruth_masks = groundtruth_dict[
          standard_fields.InputDataFields.groundtruth_instance_masks]
    self._evaluation.add_single_ground_truth_image_info(
        image_key=image_id,
        groundtruth_boxes=groundtruth_dict[
            standard_fields.InputDataFields.groundtruth_boxes],
        groundtruth_class_labels=groundtruth_classes,
        groundtruth_is_difficult_list=groundtruth_difficult,
        groundtruth_masks=groundtruth_masks)
    self._image_ids.update([image_id])