Python chainer.Reporter() Examples

The following are 30 code examples of chainer.Reporter(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module chainer , or try the search function .
Example #1
Source File: test_regressor.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def test_report_key(self, metrics_fun, compute_metrics):
        repo = chainer.Reporter()

        link = Regressor(predictor=DummyPredictor(),
                         metrics_fun=metrics_fun)
        link.compute_metrics = compute_metrics
        repo.add_observer('target', link)
        with repo:
            observation = {}
            with reporter.report_scope(observation):
                link(self.x, self.t)

        # print('observation ', observation)
        actual_keys = set(observation.keys())
        if compute_metrics:
            if metrics_fun is None:
                assert set(['target/loss']) == actual_keys
            elif isinstance(metrics_fun, dict):
                assert set(['target/loss', 'target/user_key']) == actual_keys
            elif callable(metrics_fun):
                assert set(['target/loss', 'target/metrics']) == actual_keys
            else:
                raise TypeError()
        else:
            assert set(['target/loss']) == actual_keys
Example #2
Source File: test_roc_auc_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_roc_auc_evaluator_raise_error(data, raise_value_error=True):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = ROCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None,
        raise_value_error=raise_value_error
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    return observation['target/roc_auc']
Example #3
Source File: test_roc_auc_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_roc_auc_evaluator_default_args(data0):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data0)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = ROCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected_roc_auc = 0.75
    # print('observation ', observation)
    assert observation['target/roc_auc'] == expected_roc_auc

    # --- test __call__ ---
    result = evaluator()
    # print('result ', result)
    assert result['train/main/roc_auc'] == expected_roc_auc
Example #4
Source File: test_prc_auc_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_prc_auc_evaluator_raise_error(data, raise_value_error=True):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = PRCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None,
        raise_value_error=raise_value_error
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    return observation['target/prc_auc']
Example #5
Source File: test_prc_auc_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_prc_auc_evaluator_default_args(data0):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data0)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = PRCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected_prc_auc = 0.7916
    pytest.approx(observation['target/prc_auc'], expected_prc_auc)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/prc_auc'], expected_prc_auc)
Example #6
Source File: test_r2_score_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_r2_score_evaluator_raw_values(inputs):
    predictor = DummyPredictor()
    x0, x1, _ = inputs
    dataset = NumpyTupleDataset(x0, x1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(
        iterator, predictor, name='train', multioutput='raw_values')
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x1, multioutput='raw_values')
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected)
Example #7
Source File: test_r2_score_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_r2_score_evaluator_ignore_nan_with_nonnan_value(inputs):
    predictor = DummyPredictor()
    x0, x1, _ = inputs
    dataset = NumpyTupleDataset(x0, x1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(
        iterator, predictor, name='train', ignore_nan=True)
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x1, ignore_nan=True)
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected)
Example #8
Source File: test_r2_score_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_r2_score_evaluator_ignore_nan(inputs):
    predictor = DummyPredictor()
    x0, _, x2 = inputs
    dataset = NumpyTupleDataset(x0, x2)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(
        iterator, predictor, name='train', ignore_nan=True)
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x2, ignore_nan=True)
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected)
Example #9
Source File: test_r2_score_evaluator.py    From chainer-chemistry with MIT License 6 votes vote down vote up 
def _test_r2_score_evaluator(inputs):
    predictor = DummyPredictor()
    x0, x1, _ = inputs
    dataset = NumpyTupleDataset(x0, x1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(iterator, predictor, name='train')
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x1)
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected)
Example #10
Source File: test_detection_coco_evaluator.py    From chainercv with MIT License 6 votes vote down vote up 
def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('target', self.link)
        with reporter:
            mean = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        self.assertEqual(len(reporter.observation), 0)

        key = 'ap/iou=0.50:0.95/area=all/max_dets=100'
        np.testing.assert_equal(
            mean['target/m{}'.format(key)], self.expected_ap)
        np.testing.assert_equal(mean['target/{}/cls0'.format(key)], np.nan)
        np.testing.assert_equal(mean['target/{}/cls1'.format(key)], np.nan)
        np.testing.assert_equal(
            mean['target/{}/cls2'.format(key)], self.expected_ap)
Example #11
Source File: test_instance_segmentation_coco_evaluator.py    From chainercv with MIT License 6 votes vote down vote up 
def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('target', self.link)
        with reporter:
            mean = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        self.assertEqual(len(reporter.observation), 0)

        key = 'ap/iou=0.50:0.95/area=all/max_dets=100'
        np.testing.assert_equal(
            mean['target/m{}'.format(key)], self.expected_ap)
        np.testing.assert_equal(mean['target/{}/cls0'.format(key)], np.nan)
        np.testing.assert_equal(
            mean['target/{}/cls1'.format(key)], self.expected_ap)
        np.testing.assert_equal(mean['target/{}/cls2'.format(key)], np.nan)
Example #12
Source File: test_semantic_segmentation_evaluator.py    From chainercv with MIT License 6 votes vote down vote up 
def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('main', self.link)
        with reporter:
            eval_ = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        np.testing.assert_equal(len(reporter.observation), 0)

        np.testing.assert_equal(eval_['main/miou'], self.miou)
        np.testing.assert_equal(eval_['main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)
Example #13
Source File: test_detection_voc_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('target', self.link)
        with reporter:
            mean = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        self.assertEqual(len(reporter.observation), 0)

        np.testing.assert_equal(mean['target/map'], self.expected_ap)
        np.testing.assert_equal(mean['target/ap/cls0'], np.nan)
        np.testing.assert_equal(mean['target/ap/cls1'], self.expected_ap)
        np.testing.assert_equal(mean['target/ap/cls2'], np.nan)
Example #14
Source File: test_instance_segmentation_voc_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('target', self.link)
        with reporter:
            mean = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        self.assertEqual(len(reporter.observation), 0)

        np.testing.assert_equal(mean['target/map'], self.expected_ap)
        np.testing.assert_equal(mean['target/ap/cls0'], np.nan)
        np.testing.assert_equal(mean['target/ap/cls1'], self.expected_ap)
        np.testing.assert_equal(mean['target/ap/cls2'], np.nan)
Example #15
Source File: test_detection_voc_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_current_report(self):
        reporter = chainer.Reporter()
        with reporter:
            mean = self.evaluator()
        # The result is reported to the current reporter.
        self.assertEqual(reporter.observation, mean)
Example #16
Source File: test_detection_voc_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_consistency(self):
        reporter = chainer.Reporter()

        if self.comm.rank == 0:
            multi_iterator = SerialIterator(
                self.dataset, self.batchsize, repeat=False, shuffle=False)
        else:
            multi_iterator = None
        multi_link = _DetectionStubLink(
            self.bboxes, self.labels, self.initial_count)
        multi_evaluator = DetectionVOCEvaluator(
            multi_iterator, multi_link,
            label_names=('cls0', 'cls1', 'cls2'),
            comm=self.comm)
        reporter.add_observer('target', multi_link)
        with reporter:
            multi_mean = multi_evaluator.evaluate()

        if self.comm.rank != 0:
            self.assertEqual(multi_mean, {})
            return

        single_iterator = SerialIterator(
            self.dataset, self.batchsize, repeat=False, shuffle=False)
        single_link = _DetectionStubLink(
            self.bboxes, self.labels)
        single_evaluator = DetectionVOCEvaluator(
            single_iterator, single_link,
            label_names=('cls0', 'cls1', 'cls2'))
        reporter.add_observer('target', single_link)
        with reporter:
            single_mean = single_evaluator.evaluate()

        self.assertEqual(set(multi_mean.keys()), set(single_mean.keys()))
        for key in multi_mean.keys():
            np.testing.assert_equal(single_mean[key], multi_mean[key])
Example #17
Source File: test_semantic_segmentation_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_current_report(self):
        reporter = chainer.Reporter()
        with reporter:
            eval_ = self.evaluator()
        # The result is reported to the current reporter.
        np.testing.assert_equal(reporter.observation, eval_)
Example #18
Source File: test_instance_segmentation_coco_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_current_report(self):
        reporter = chainer.Reporter()
        with reporter:
            mean = self.evaluator()
        # The result is reported to the current reporter.
        self.assertEqual(reporter.observation, mean)
Example #19
Source File: test_instance_segmentation_coco_evaluator.py    From chainercv with MIT License 5 votes vote down vote up 
def test_consistency(self):
        reporter = chainer.Reporter()

        if self.comm.rank == 0:
            multi_iterator = SerialIterator(
                self.dataset, self.batchsize, repeat=False, shuffle=False)
        else:
            multi_iterator = None
        multi_link = _InstanceSegmentationStubLink(
            self.masks, self.labels, self.initial_count)
        multi_evaluator = InstanceSegmentationCOCOEvaluator(
            multi_iterator, multi_link,
            label_names=('cls0', 'cls1', 'cls2'),
            comm=self.comm)
        reporter.add_observer('target', multi_link)
        with reporter:
            multi_mean = multi_evaluator.evaluate()

        if self.comm.rank != 0:
            self.assertEqual(multi_mean, {})
            return

        single_iterator = SerialIterator(
            self.dataset, self.batchsize, repeat=False, shuffle=False)
        single_link = _InstanceSegmentationStubLink(
            self.masks, self.labels)
        single_evaluator = InstanceSegmentationCOCOEvaluator(
            single_iterator, single_link,
            label_names=('cls0', 'cls1', 'cls2'))
        reporter.add_observer('target', single_link)
        with reporter:
            single_mean = single_evaluator.evaluate()

        self.assertEqual(set(multi_mean.keys()), set(single_mean.keys()))
        for key in multi_mean.keys():
            np.testing.assert_equal(single_mean[key], multi_mean[key])
Example #20
Source File: test_roc_auc_evaluator.py    From chainer-chemistry with MIT License 5 votes vote down vote up 
def _test_roc_auc_evaluator_with_labels(data1):
    """test `pos_labels` and `ignore_labels` behavior"""

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = ROCAUCEvaluator(
        iterator, predictor, name='val',
        pos_labels=[1, 2], ignore_labels=-1,
    )

    # --- test evaluate ---
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected_roc_auc = 0.75
    # print('observation ', observation)
    assert observation['target/roc_auc'] == expected_roc_auc

    # --- test __call__ ---
    result = evaluator()
    # print('result ', result)
    assert result['val/main/roc_auc'] == expected_roc_auc
Example #21
Source File: test.py    From mesh_reconstruction with MIT License 5 votes vote down vote up 
def run():
    # arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-eid', '--experiment_id', type=str)
    parser.add_argument('-d', '--model_directory', type=str, default=MODEL_DIRECTORY)
    parser.add_argument('-dd', '--dataset_directory', type=str, default=DATASET_DIRECTORY)
    parser.add_argument('-cls', '--class_ids', type=str, default=CLASS_IDS_ALL)
    parser.add_argument('-s', '--seed', type=int, default=RANDOM_SEED)
    parser.add_argument('-g', '--gpu', type=int, default=GPU)
    args = parser.parse_args()
    directory_output = os.path.join(args.model_directory, args.experiment_id)

    # set random seed, gpu
    random.seed(args.seed)
    np.random.seed(args.seed)
    cp.random.seed(args.seed)
    chainer.cuda.get_device(args.gpu).use()

    # load dataset
    dataset_test = datasets.ShapeNet(args.dataset_directory, args.class_ids.split(','), 'test')

    # setup model & optimizer
    model = models.Model()
    model.to_gpu()
    chainer.serializers.load_npz(os.path.join(directory_output, 'model.npz'), model)

    # evaluate
    reporter = chainer.Reporter()
    observation = {}
    with reporter.scope(observation):
        training.validation(None, model, dataset_test)
    for key in sorted(observation.keys()):
        key_display = key
        for class_id in CLASS_NAMES.keys():
            key_display = key_display.replace(class_id, CLASS_NAMES[class_id])
        print '%s: %.4f' % (key_display, observation[key])
Example #22
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_report_with_unregistered_observer(self):
        reporter = chainer.Reporter()
        observer = object()
        with reporter:
            with self.assertRaises(KeyError):
                chainer.report({'x': 1}, observer)
Example #23
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_enter_exit(self):
        reporter1 = chainer.Reporter()
        reporter2 = chainer.Reporter()
        with reporter1:
            self.assertIs(chainer.get_current_reporter(), reporter1)
            with reporter2:
                self.assertIs(chainer.get_current_reporter(), reporter2)
            self.assertIs(chainer.get_current_reporter(), reporter1)
Example #24
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_enter_exit_threadsafe(self):
        # This test ensures reporter.__enter__ correctly stores the reporter
        # in the thread-local storage.

        def thread_func(reporter, record):
            with reporter:
                # Sleep for a tiny moment to cause an overlap of the context
                # managers.
                time.sleep(0.01)
                record.append(chainer.get_current_reporter())

        record1 = []  # The current repoter in each thread is stored here.
        record2 = []
        reporter1 = chainer.Reporter()
        reporter2 = chainer.Reporter()
        thread1 = threading.Thread(
            target=thread_func,
            args=(reporter1, record1))
        thread2 = threading.Thread(
            target=thread_func,
            args=(reporter2, record2))
        thread1.daemon = True
        thread2.daemon = True
        thread1.start()
        thread2.start()
        thread1.join()
        thread2.join()
        self.assertIs(record1[0], reporter1)
        self.assertIs(record2[0], reporter2)
Example #25
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_scope(self):
        reporter1 = chainer.Reporter()
        reporter2 = chainer.Reporter()
        with reporter1:
            observation = {}
            with reporter2.scope(observation):
                self.assertIs(chainer.get_current_reporter(), reporter2)
                self.assertIs(reporter2.observation, observation)
            self.assertIs(chainer.get_current_reporter(), reporter1)
            self.assertIsNot(reporter2.observation, observation)
Example #26
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_add_observer(self):
        reporter = chainer.Reporter()
        observer = object()
        reporter.add_observer('o', observer)

        reporter.report({'x': 1}, observer)

        observation = reporter.observation
        self.assertIn('o/x', observation)
        self.assertEqual(observation['o/x'], 1)
        self.assertNotIn('x', observation)
Example #27
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_report_without_observer(self):
        reporter = chainer.Reporter()
        reporter.report({'x': 1})

        observation = reporter.observation
        self.assertIn('x', observation)
        self.assertEqual(observation['x'], 1)
Example #28
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_keep_graph_default(self):
        x = chainer.Variable(numpy.array([1], numpy.float32))
        y = functions.sigmoid(x)
        reporter = chainer.Reporter()
        with self._scope(None):
            reporter.report({'y': y})
        self.assertIsNone(reporter.observation['y'].creator)
Example #29
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_keep_graph(self):
        x = chainer.Variable(numpy.array([1], numpy.float32))
        y = functions.sigmoid(x)
        reporter = chainer.Reporter()
        with self._scope(True):
            reporter.report({'y': y})
        assert reporter.observation['y'].creator is not None
Example #30
Source File: test_reporter.py    From chainer with MIT License 5 votes vote down vote up 
def test_not_keep_graph(self):
        x = chainer.Variable(numpy.array([1], numpy.float32))
        y = functions.sigmoid(x)
        reporter = chainer.Reporter()
        with self._scope(False):
            reporter.report({'y': y})
        self.assertIsNone(reporter.observation['y'].creator)