|
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Tests for eval_util."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
from absl.testing import parameterized
|
|
|
|
import tensorflow as tf
|
|
|
|
from object_detection import eval_util
|
|
from object_detection.core import standard_fields as fields
|
|
from object_detection.protos import eval_pb2
|
|
from object_detection.utils import test_case
|
|
|
|
|
|
class EvalUtilTest(test_case.TestCase, parameterized.TestCase):
|
|
|
|
def _get_categories_list(self):
|
|
return [{'id': 0, 'name': 'person'},
|
|
{'id': 1, 'name': 'dog'},
|
|
{'id': 2, 'name': 'cat'}]
|
|
|
|
def _make_evaluation_dict(self,
|
|
resized_groundtruth_masks=False,
|
|
batch_size=1,
|
|
max_gt_boxes=None,
|
|
scale_to_absolute=False):
|
|
input_data_fields = fields.InputDataFields
|
|
detection_fields = fields.DetectionResultFields
|
|
|
|
image = tf.zeros(shape=[batch_size, 20, 20, 3], dtype=tf.uint8)
|
|
if batch_size == 1:
|
|
key = tf.constant('image1')
|
|
else:
|
|
key = tf.constant([str(i) for i in range(batch_size)])
|
|
detection_boxes = tf.tile(tf.constant([[[0., 0., 1., 1.]]]),
|
|
multiples=[batch_size, 1, 1])
|
|
detection_scores = tf.tile(tf.constant([[0.8]]), multiples=[batch_size, 1])
|
|
detection_classes = tf.tile(tf.constant([[0]]), multiples=[batch_size, 1])
|
|
detection_masks = tf.tile(tf.ones(shape=[1, 1, 20, 20], dtype=tf.float32),
|
|
multiples=[batch_size, 1, 1, 1])
|
|
num_detections = tf.ones([batch_size])
|
|
groundtruth_boxes = tf.constant([[0., 0., 1., 1.]])
|
|
groundtruth_classes = tf.constant([1])
|
|
groundtruth_instance_masks = tf.ones(shape=[1, 20, 20], dtype=tf.uint8)
|
|
if resized_groundtruth_masks:
|
|
groundtruth_instance_masks = tf.ones(shape=[1, 10, 10], dtype=tf.uint8)
|
|
|
|
if batch_size > 1:
|
|
groundtruth_boxes = tf.tile(tf.expand_dims(groundtruth_boxes, 0),
|
|
multiples=[batch_size, 1, 1])
|
|
groundtruth_classes = tf.tile(tf.expand_dims(groundtruth_classes, 0),
|
|
multiples=[batch_size, 1])
|
|
groundtruth_instance_masks = tf.tile(
|
|
tf.expand_dims(groundtruth_instance_masks, 0),
|
|
multiples=[batch_size, 1, 1, 1])
|
|
|
|
detections = {
|
|
detection_fields.detection_boxes: detection_boxes,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes,
|
|
detection_fields.detection_masks: detection_masks,
|
|
detection_fields.num_detections: num_detections
|
|
}
|
|
groundtruth = {
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
input_data_fields.groundtruth_instance_masks: groundtruth_instance_masks
|
|
}
|
|
if batch_size > 1:
|
|
return eval_util.result_dict_for_batched_example(
|
|
image, key, detections, groundtruth,
|
|
scale_to_absolute=scale_to_absolute,
|
|
max_gt_boxes=max_gt_boxes)
|
|
else:
|
|
return eval_util.result_dict_for_single_example(
|
|
image, key, detections, groundtruth,
|
|
scale_to_absolute=scale_to_absolute)
|
|
|
|
@parameterized.parameters(
|
|
{'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': True},
|
|
{'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': True},
|
|
{'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': False},
|
|
{'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': False}
|
|
)
|
|
def test_get_eval_metric_ops_for_coco_detections(self, batch_size=1,
|
|
max_gt_boxes=None,
|
|
scale_to_absolute=False):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(['coco_detection_metrics'])
|
|
categories = self._get_categories_list()
|
|
eval_dict = self._make_evaluation_dict(batch_size=batch_size,
|
|
max_gt_boxes=max_gt_boxes,
|
|
scale_to_absolute=scale_to_absolute)
|
|
metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
|
|
eval_config, categories, eval_dict)
|
|
_, update_op = metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
metrics = {}
|
|
for key, (value_op, _) in metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
sess.run(update_op)
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(1.0, metrics['DetectionBoxes_Precision/mAP'])
|
|
self.assertNotIn('DetectionMasks_Precision/mAP', metrics)
|
|
|
|
@parameterized.parameters(
|
|
{'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': True},
|
|
{'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': True},
|
|
{'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': False},
|
|
{'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': False}
|
|
)
|
|
def test_get_eval_metric_ops_for_coco_detections_and_masks(
|
|
self, batch_size=1, max_gt_boxes=None, scale_to_absolute=False):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(
|
|
['coco_detection_metrics', 'coco_mask_metrics'])
|
|
categories = self._get_categories_list()
|
|
eval_dict = self._make_evaluation_dict(batch_size=batch_size,
|
|
max_gt_boxes=max_gt_boxes,
|
|
scale_to_absolute=scale_to_absolute)
|
|
metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
|
|
eval_config, categories, eval_dict)
|
|
_, update_op_boxes = metric_ops['DetectionBoxes_Precision/mAP']
|
|
_, update_op_masks = metric_ops['DetectionMasks_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
metrics = {}
|
|
for key, (value_op, _) in metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
sess.run(update_op_boxes)
|
|
sess.run(update_op_masks)
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(1.0, metrics['DetectionBoxes_Precision/mAP'])
|
|
self.assertAlmostEqual(1.0, metrics['DetectionMasks_Precision/mAP'])
|
|
|
|
@parameterized.parameters(
|
|
{'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': True},
|
|
{'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': True},
|
|
{'batch_size': 1, 'max_gt_boxes': None, 'scale_to_absolute': False},
|
|
{'batch_size': 8, 'max_gt_boxes': [1], 'scale_to_absolute': False}
|
|
)
|
|
def test_get_eval_metric_ops_for_coco_detections_and_resized_masks(
|
|
self, batch_size=1, max_gt_boxes=None, scale_to_absolute=False):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(
|
|
['coco_detection_metrics', 'coco_mask_metrics'])
|
|
categories = self._get_categories_list()
|
|
eval_dict = self._make_evaluation_dict(batch_size=batch_size,
|
|
max_gt_boxes=max_gt_boxes,
|
|
scale_to_absolute=scale_to_absolute,
|
|
resized_groundtruth_masks=True)
|
|
metric_ops = eval_util.get_eval_metric_ops_for_evaluators(
|
|
eval_config, categories, eval_dict)
|
|
_, update_op_boxes = metric_ops['DetectionBoxes_Precision/mAP']
|
|
_, update_op_masks = metric_ops['DetectionMasks_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
metrics = {}
|
|
for key, (value_op, _) in metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
sess.run(update_op_boxes)
|
|
sess.run(update_op_masks)
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(1.0, metrics['DetectionBoxes_Precision/mAP'])
|
|
self.assertAlmostEqual(1.0, metrics['DetectionMasks_Precision/mAP'])
|
|
|
|
def test_get_eval_metric_ops_raises_error_with_unsupported_metric(self):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(['unsupported_metric'])
|
|
categories = self._get_categories_list()
|
|
eval_dict = self._make_evaluation_dict()
|
|
with self.assertRaises(ValueError):
|
|
eval_util.get_eval_metric_ops_for_evaluators(
|
|
eval_config, categories, eval_dict)
|
|
|
|
def test_get_eval_metric_ops_for_evaluators(self):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(
|
|
['coco_detection_metrics', 'coco_mask_metrics'])
|
|
eval_config.include_metrics_per_category = True
|
|
|
|
evaluator_options = eval_util.evaluator_options_from_eval_config(
|
|
eval_config)
|
|
self.assertTrue(evaluator_options['coco_detection_metrics'][
|
|
'include_metrics_per_category'])
|
|
self.assertTrue(evaluator_options['coco_mask_metrics'][
|
|
'include_metrics_per_category'])
|
|
|
|
def test_get_evaluator_with_evaluator_options(self):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(['coco_detection_metrics'])
|
|
eval_config.include_metrics_per_category = True
|
|
categories = self._get_categories_list()
|
|
|
|
evaluator_options = eval_util.evaluator_options_from_eval_config(
|
|
eval_config)
|
|
evaluator = eval_util.get_evaluators(
|
|
eval_config, categories, evaluator_options)
|
|
|
|
self.assertTrue(evaluator[0]._include_metrics_per_category)
|
|
|
|
def test_get_evaluator_with_no_evaluator_options(self):
|
|
eval_config = eval_pb2.EvalConfig()
|
|
eval_config.metrics_set.extend(['coco_detection_metrics'])
|
|
eval_config.include_metrics_per_category = True
|
|
categories = self._get_categories_list()
|
|
|
|
evaluator = eval_util.get_evaluators(
|
|
eval_config, categories, evaluator_options=None)
|
|
|
|
# Even though we are setting eval_config.include_metrics_per_category = True
|
|
# this option is never passed into the DetectionEvaluator constructor (via
|
|
# `evaluator_options`).
|
|
self.assertFalse(evaluator[0]._include_metrics_per_category)
|
|
|
|
if __name__ == '__main__':
|
|
tf.test.main()
|