You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
951 lines
44 KiB
951 lines
44 KiB
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Tests for tensorflow_models.object_detection.metrics.coco_evaluation."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import numpy as np
|
|
import tensorflow as tf
|
|
from object_detection.core import standard_fields
|
|
from object_detection.metrics import coco_evaluation
|
|
|
|
|
|
def _get_categories_list():
|
|
return [{
|
|
'id': 1,
|
|
'name': 'person'
|
|
}, {
|
|
'id': 2,
|
|
'name': 'dog'
|
|
}, {
|
|
'id': 3,
|
|
'name': 'cat'
|
|
}]
|
|
|
|
|
|
class CocoDetectionEvaluationTest(tf.test.TestCase):
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
|
|
"""Tests that mAP is calculated correctly on GT and Detections."""
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image1',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1])
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image2',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[50., 50., 100., 100.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1])
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image2',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[50., 50., 100., 100.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image3',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[25., 25., 50., 50.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1])
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image3',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[25., 25., 50., 50.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
metrics = coco_evaluator.evaluate()
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsSkipCrowd(self):
|
|
"""Tests computing mAP with is_crowd GT boxes skipped."""
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image1',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[100., 100., 200., 200.], [99., 99., 200., 200.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes:
|
|
np.array([1, 2]),
|
|
standard_fields.InputDataFields.groundtruth_is_crowd:
|
|
np.array([0, 1])
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
metrics = coco_evaluator.evaluate()
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsEmptyCrowd(self):
|
|
"""Tests computing mAP with empty is_crowd array passed in."""
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image1',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes:
|
|
np.array([1]),
|
|
standard_fields.InputDataFields.groundtruth_is_crowd:
|
|
np.array([])
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
metrics = coco_evaluator.evaluate()
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
|
|
def testRejectionOnDuplicateGroundtruth(self):
|
|
"""Tests that groundtruth cannot be added more than once for an image."""
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
# Add groundtruth
|
|
image_key1 = 'img1'
|
|
groundtruth_boxes1 = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]],
|
|
dtype=float)
|
|
groundtruth_class_labels1 = np.array([1, 3, 1], dtype=int)
|
|
coco_evaluator.add_single_ground_truth_image_info(image_key1, {
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
groundtruth_boxes1,
|
|
standard_fields.InputDataFields.groundtruth_classes:
|
|
groundtruth_class_labels1
|
|
})
|
|
groundtruth_lists_len = len(coco_evaluator._groundtruth_list)
|
|
|
|
# Add groundtruth with the same image id.
|
|
coco_evaluator.add_single_ground_truth_image_info(image_key1, {
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
groundtruth_boxes1,
|
|
standard_fields.InputDataFields.groundtruth_classes:
|
|
groundtruth_class_labels1
|
|
})
|
|
self.assertEqual(groundtruth_lists_len,
|
|
len(coco_evaluator._groundtruth_list))
|
|
|
|
def testRejectionOnDuplicateDetections(self):
|
|
"""Tests that detections cannot be added more than once for an image."""
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
# Add groundtruth
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image1',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[99., 100., 200., 200.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1])
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
detections_lists_len = len(coco_evaluator._detection_boxes_list)
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1', # Note that this image id was previously added.
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
self.assertEqual(detections_lists_len,
|
|
len(coco_evaluator._detection_boxes_list))
|
|
|
|
def testExceptionRaisedWithMissingGroundtruth(self):
|
|
"""Tests that exception is raised for detection with missing groundtruth."""
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
with self.assertRaises(ValueError):
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1])
|
|
})
|
|
|
|
|
|
class CocoEvaluationPyFuncTest(tf.test.TestCase):
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
image_id = tf.placeholder(tf.string, shape=())
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
|
|
detection_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
detection_fields.detection_boxes: detection_boxes,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image1',
|
|
groundtruth_boxes: np.array([[100., 100., 200., 200.]]),
|
|
groundtruth_classes: np.array([1]),
|
|
detection_boxes: np.array([[100., 100., 200., 200.]]),
|
|
detection_scores: np.array([.8]),
|
|
detection_classes: np.array([1])
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image2',
|
|
groundtruth_boxes: np.array([[50., 50., 100., 100.]]),
|
|
groundtruth_classes: np.array([3]),
|
|
detection_boxes: np.array([[50., 50., 100., 100.]]),
|
|
detection_scores: np.array([.7]),
|
|
detection_classes: np.array([3])
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image3',
|
|
groundtruth_boxes: np.array([[25., 25., 50., 50.]]),
|
|
groundtruth_classes: np.array([2]),
|
|
detection_boxes: np.array([[25., 25., 50., 50.]]),
|
|
detection_scores: np.array([.9]),
|
|
detection_classes: np.array([2])
|
|
})
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._detection_boxes_list)
|
|
self.assertFalse(coco_evaluator._image_ids)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsIsAnnotated(self):
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
image_id = tf.placeholder(tf.string, shape=())
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
|
|
is_annotated = tf.placeholder(tf.bool, shape=())
|
|
detection_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
'is_annotated': is_annotated,
|
|
detection_fields.detection_boxes: detection_boxes,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image1',
|
|
groundtruth_boxes: np.array([[100., 100., 200., 200.]]),
|
|
groundtruth_classes: np.array([1]),
|
|
is_annotated: True,
|
|
detection_boxes: np.array([[100., 100., 200., 200.]]),
|
|
detection_scores: np.array([.8]),
|
|
detection_classes: np.array([1])
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image2',
|
|
groundtruth_boxes: np.array([[50., 50., 100., 100.]]),
|
|
groundtruth_classes: np.array([3]),
|
|
is_annotated: True,
|
|
detection_boxes: np.array([[50., 50., 100., 100.]]),
|
|
detection_scores: np.array([.7]),
|
|
detection_classes: np.array([3])
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image3',
|
|
groundtruth_boxes: np.array([[25., 25., 50., 50.]]),
|
|
groundtruth_classes: np.array([2]),
|
|
is_annotated: True,
|
|
detection_boxes: np.array([[25., 25., 50., 50.]]),
|
|
detection_scores: np.array([.9]),
|
|
detection_classes: np.array([2])
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image4',
|
|
groundtruth_boxes: np.zeros((0, 4)),
|
|
groundtruth_classes: np.zeros((0)),
|
|
is_annotated: False, # Note that this image isn't annotated.
|
|
detection_boxes: np.array([[25., 25., 50., 50.],
|
|
[25., 25., 70., 50.],
|
|
[25., 25., 80., 50.],
|
|
[25., 25., 90., 50.]]),
|
|
detection_scores: np.array([0.6, 0.7, 0.8, 0.9]),
|
|
detection_classes: np.array([1, 2, 2, 3])
|
|
})
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._detection_boxes_list)
|
|
self.assertFalse(coco_evaluator._image_ids)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsPadded(self):
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
image_id = tf.placeholder(tf.string, shape=())
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
|
|
detection_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
detection_fields.detection_boxes: detection_boxes,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(
|
|
update_op,
|
|
feed_dict={
|
|
image_id:
|
|
'image1',
|
|
groundtruth_boxes:
|
|
np.array([[100., 100., 200., 200.], [-1, -1, -1, -1]]),
|
|
groundtruth_classes:
|
|
np.array([1, -1]),
|
|
detection_boxes:
|
|
np.array([[100., 100., 200., 200.], [0., 0., 0., 0.]]),
|
|
detection_scores:
|
|
np.array([.8, 0.]),
|
|
detection_classes:
|
|
np.array([1, -1])
|
|
})
|
|
sess.run(
|
|
update_op,
|
|
feed_dict={
|
|
image_id:
|
|
'image2',
|
|
groundtruth_boxes:
|
|
np.array([[50., 50., 100., 100.], [-1, -1, -1, -1]]),
|
|
groundtruth_classes:
|
|
np.array([3, -1]),
|
|
detection_boxes:
|
|
np.array([[50., 50., 100., 100.], [0., 0., 0., 0.]]),
|
|
detection_scores:
|
|
np.array([.7, 0.]),
|
|
detection_classes:
|
|
np.array([3, -1])
|
|
})
|
|
sess.run(
|
|
update_op,
|
|
feed_dict={
|
|
image_id:
|
|
'image3',
|
|
groundtruth_boxes:
|
|
np.array([[25., 25., 50., 50.], [10., 10., 15., 15.]]),
|
|
groundtruth_classes:
|
|
np.array([2, 2]),
|
|
detection_boxes:
|
|
np.array([[25., 25., 50., 50.], [10., 10., 15., 15.]]),
|
|
detection_scores:
|
|
np.array([.95, .9]),
|
|
detection_classes:
|
|
np.array([2, 2])
|
|
})
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 0.83333331)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._detection_boxes_list)
|
|
self.assertFalse(coco_evaluator._image_ids)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsBatched(self):
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
batch_size = 3
|
|
image_id = tf.placeholder(tf.string, shape=(batch_size))
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
detection_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
detection_fields.detection_boxes: detection_boxes,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: ['image1', 'image2', 'image3'],
|
|
groundtruth_boxes: np.array([[[100., 100., 200., 200.]],
|
|
[[50., 50., 100., 100.]],
|
|
[[25., 25., 50., 50.]]]),
|
|
groundtruth_classes: np.array([[1], [3], [2]]),
|
|
detection_boxes: np.array([[[100., 100., 200., 200.]],
|
|
[[50., 50., 100., 100.]],
|
|
[[25., 25., 50., 50.]]]),
|
|
detection_scores: np.array([[.8], [.7], [.9]]),
|
|
detection_classes: np.array([[1], [3], [2]])
|
|
})
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._detection_boxes_list)
|
|
self.assertFalse(coco_evaluator._image_ids)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsPaddedBatches(self):
|
|
coco_evaluator = coco_evaluation.CocoDetectionEvaluator(
|
|
_get_categories_list())
|
|
batch_size = 3
|
|
image_id = tf.placeholder(tf.string, shape=(batch_size))
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
num_gt_boxes_per_image = tf.placeholder(tf.int32, shape=(None))
|
|
detection_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
num_det_boxes_per_image = tf.placeholder(tf.int32, shape=(None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
detection_fields.detection_boxes: detection_boxes,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes,
|
|
'num_groundtruth_boxes_per_image': num_gt_boxes_per_image,
|
|
'num_det_boxes_per_image': num_det_boxes_per_image
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(
|
|
update_op,
|
|
feed_dict={
|
|
image_id: ['image1', 'image2', 'image3'],
|
|
groundtruth_boxes:
|
|
np.array([[[100., 100., 200., 200.], [-1, -1, -1, -1]],
|
|
[[50., 50., 100., 100.], [-1, -1, -1, -1]],
|
|
[[25., 25., 50., 50.], [10., 10., 15., 15.]]]),
|
|
groundtruth_classes:
|
|
np.array([[1, -1], [3, -1], [2, 2]]),
|
|
num_gt_boxes_per_image:
|
|
np.array([1, 1, 2]),
|
|
detection_boxes:
|
|
np.array([[[100., 100., 200., 200.],
|
|
[0., 0., 0., 0.],
|
|
[0., 0., 0., 0.]],
|
|
[[50., 50., 100., 100.],
|
|
[0., 0., 0., 0.],
|
|
[0., 0., 0., 0.]],
|
|
[[25., 25., 50., 50.],
|
|
[10., 10., 15., 15.],
|
|
[10., 10., 15., 15.]]]),
|
|
detection_scores:
|
|
np.array([[.8, 0., 0.], [.7, 0., 0.], [.95, .9, 0.9]]),
|
|
detection_classes:
|
|
np.array([[1, -1, -1], [3, -1, -1], [2, 2, 2]]),
|
|
num_det_boxes_per_image:
|
|
np.array([1, 1, 3]),
|
|
})
|
|
|
|
# Check the number of bounding boxes added.
|
|
self.assertEqual(len(coco_evaluator._groundtruth_list), 4)
|
|
self.assertEqual(len(coco_evaluator._detection_boxes_list), 5)
|
|
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 0.83333331)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._detection_boxes_list)
|
|
self.assertFalse(coco_evaluator._image_ids)
|
|
|
|
|
|
class CocoMaskEvaluationTest(tf.test.TestCase):
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
|
|
coco_evaluator = coco_evaluation.CocoMaskEvaluator(_get_categories_list())
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image1',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1]),
|
|
standard_fields.InputDataFields.groundtruth_instance_masks:
|
|
np.pad(np.ones([1, 100, 100], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)), mode='constant')
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image1',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[100., 100., 200., 200.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1]),
|
|
standard_fields.DetectionResultFields.detection_masks:
|
|
np.pad(np.ones([1, 100, 100], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)), mode='constant')
|
|
})
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image2',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[50., 50., 100., 100.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1]),
|
|
standard_fields.InputDataFields.groundtruth_instance_masks:
|
|
np.pad(np.ones([1, 50, 50], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)), mode='constant')
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image2',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[50., 50., 100., 100.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1]),
|
|
standard_fields.DetectionResultFields.detection_masks:
|
|
np.pad(np.ones([1, 50, 50], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)), mode='constant')
|
|
})
|
|
coco_evaluator.add_single_ground_truth_image_info(
|
|
image_id='image3',
|
|
groundtruth_dict={
|
|
standard_fields.InputDataFields.groundtruth_boxes:
|
|
np.array([[25., 25., 50., 50.]]),
|
|
standard_fields.InputDataFields.groundtruth_classes: np.array([1]),
|
|
standard_fields.InputDataFields.groundtruth_instance_masks:
|
|
np.pad(np.ones([1, 25, 25], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)), mode='constant')
|
|
})
|
|
coco_evaluator.add_single_detected_image_info(
|
|
image_id='image3',
|
|
detections_dict={
|
|
standard_fields.DetectionResultFields.detection_boxes:
|
|
np.array([[25., 25., 50., 50.]]),
|
|
standard_fields.DetectionResultFields.detection_scores:
|
|
np.array([.8]),
|
|
standard_fields.DetectionResultFields.detection_classes:
|
|
np.array([1]),
|
|
standard_fields.DetectionResultFields.detection_masks:
|
|
np.pad(np.ones([1, 25, 25], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)), mode='constant')
|
|
})
|
|
metrics = coco_evaluator.evaluate()
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0)
|
|
coco_evaluator.clear()
|
|
self.assertFalse(coco_evaluator._image_id_to_mask_shape_map)
|
|
self.assertFalse(coco_evaluator._image_ids_with_detections)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._detection_masks_list)
|
|
|
|
|
|
class CocoMaskEvaluationPyFuncTest(tf.test.TestCase):
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetections(self):
|
|
coco_evaluator = coco_evaluation.CocoMaskEvaluator(_get_categories_list())
|
|
image_id = tf.placeholder(tf.string, shape=())
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(None))
|
|
groundtruth_masks = tf.placeholder(tf.uint8, shape=(None, None, None))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(None))
|
|
detection_masks = tf.placeholder(tf.uint8, shape=(None, None, None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
input_data_fields.groundtruth_instance_masks: groundtruth_masks,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes,
|
|
detection_fields.detection_masks: detection_masks,
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionMasks_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(
|
|
update_op,
|
|
feed_dict={
|
|
image_id:
|
|
'image1',
|
|
groundtruth_boxes:
|
|
np.array([[100., 100., 200., 200.], [50., 50., 100., 100.]]),
|
|
groundtruth_classes:
|
|
np.array([1, 2]),
|
|
groundtruth_masks:
|
|
np.stack([
|
|
np.pad(
|
|
np.ones([100, 100], dtype=np.uint8), ((10, 10),
|
|
(10, 10)),
|
|
mode='constant'),
|
|
np.pad(
|
|
np.ones([50, 50], dtype=np.uint8), ((0, 70), (0, 70)),
|
|
mode='constant')
|
|
]),
|
|
detection_scores:
|
|
np.array([.9, .8]),
|
|
detection_classes:
|
|
np.array([2, 1]),
|
|
detection_masks:
|
|
np.stack([
|
|
np.pad(
|
|
np.ones([50, 50], dtype=np.uint8), ((0, 70), (0, 70)),
|
|
mode='constant'),
|
|
np.pad(
|
|
np.ones([100, 100], dtype=np.uint8), ((10, 10),
|
|
(10, 10)),
|
|
mode='constant'),
|
|
])
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image2',
|
|
groundtruth_boxes: np.array([[50., 50., 100., 100.]]),
|
|
groundtruth_classes: np.array([1]),
|
|
groundtruth_masks: np.pad(np.ones([1, 50, 50],
|
|
dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)),
|
|
mode='constant'),
|
|
detection_scores: np.array([.8]),
|
|
detection_classes: np.array([1]),
|
|
detection_masks: np.pad(np.ones([1, 50, 50], dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)),
|
|
mode='constant')
|
|
})
|
|
sess.run(update_op,
|
|
feed_dict={
|
|
image_id: 'image3',
|
|
groundtruth_boxes: np.array([[25., 25., 50., 50.]]),
|
|
groundtruth_classes: np.array([1]),
|
|
groundtruth_masks: np.pad(np.ones([1, 25, 25],
|
|
dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)),
|
|
mode='constant'),
|
|
detection_scores: np.array([.8]),
|
|
detection_classes: np.array([1]),
|
|
detection_masks: np.pad(np.ones([1, 25, 25],
|
|
dtype=np.uint8),
|
|
((0, 0), (10, 10), (10, 10)),
|
|
mode='constant')
|
|
})
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@1'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._image_ids_with_detections)
|
|
self.assertFalse(coco_evaluator._image_id_to_mask_shape_map)
|
|
self.assertFalse(coco_evaluator._detection_masks_list)
|
|
|
|
def testGetOneMAPWithMatchingGroundtruthAndDetectionsBatched(self):
|
|
coco_evaluator = coco_evaluation.CocoMaskEvaluator(_get_categories_list())
|
|
batch_size = 3
|
|
image_id = tf.placeholder(tf.string, shape=(batch_size))
|
|
groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4))
|
|
groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
groundtruth_masks = tf.placeholder(
|
|
tf.uint8, shape=(batch_size, None, None, None))
|
|
detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None))
|
|
detection_masks = tf.placeholder(
|
|
tf.uint8, shape=(batch_size, None, None, None))
|
|
|
|
input_data_fields = standard_fields.InputDataFields
|
|
detection_fields = standard_fields.DetectionResultFields
|
|
eval_dict = {
|
|
input_data_fields.key: image_id,
|
|
input_data_fields.groundtruth_boxes: groundtruth_boxes,
|
|
input_data_fields.groundtruth_classes: groundtruth_classes,
|
|
input_data_fields.groundtruth_instance_masks: groundtruth_masks,
|
|
detection_fields.detection_scores: detection_scores,
|
|
detection_fields.detection_classes: detection_classes,
|
|
detection_fields.detection_masks: detection_masks,
|
|
}
|
|
|
|
eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict)
|
|
|
|
_, update_op = eval_metric_ops['DetectionMasks_Precision/mAP']
|
|
|
|
with self.test_session() as sess:
|
|
sess.run(
|
|
update_op,
|
|
feed_dict={
|
|
image_id: ['image1', 'image2', 'image3'],
|
|
groundtruth_boxes:
|
|
np.array([[[100., 100., 200., 200.]],
|
|
[[50., 50., 100., 100.]],
|
|
[[25., 25., 50., 50.]]]),
|
|
groundtruth_classes:
|
|
np.array([[1], [1], [1]]),
|
|
groundtruth_masks:
|
|
np.stack([
|
|
np.pad(
|
|
np.ones([1, 100, 100], dtype=np.uint8),
|
|
((0, 0), (0, 0), (0, 0)),
|
|
mode='constant'),
|
|
np.pad(
|
|
np.ones([1, 50, 50], dtype=np.uint8),
|
|
((0, 0), (25, 25), (25, 25)),
|
|
mode='constant'),
|
|
np.pad(
|
|
np.ones([1, 25, 25], dtype=np.uint8),
|
|
((0, 0), (37, 38), (37, 38)),
|
|
mode='constant')
|
|
],
|
|
axis=0),
|
|
detection_scores:
|
|
np.array([[.8], [.8], [.8]]),
|
|
detection_classes:
|
|
np.array([[1], [1], [1]]),
|
|
detection_masks:
|
|
np.stack([
|
|
np.pad(
|
|
np.ones([1, 100, 100], dtype=np.uint8),
|
|
((0, 0), (0, 0), (0, 0)),
|
|
mode='constant'),
|
|
np.pad(
|
|
np.ones([1, 50, 50], dtype=np.uint8),
|
|
((0, 0), (25, 25), (25, 25)),
|
|
mode='constant'),
|
|
np.pad(
|
|
np.ones([1, 25, 25], dtype=np.uint8),
|
|
((0, 0), (37, 38), (37, 38)),
|
|
mode='constant')
|
|
],
|
|
axis=0)
|
|
})
|
|
metrics = {}
|
|
for key, (value_op, _) in eval_metric_ops.iteritems():
|
|
metrics[key] = value_op
|
|
metrics = sess.run(metrics)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP@.50IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP@.75IOU'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (small)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@1'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@10'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (large)'], 1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (medium)'],
|
|
1.0)
|
|
self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (small)'], 1.0)
|
|
self.assertFalse(coco_evaluator._groundtruth_list)
|
|
self.assertFalse(coco_evaluator._image_ids_with_detections)
|
|
self.assertFalse(coco_evaluator._image_id_to_mask_shape_map)
|
|
self.assertFalse(coco_evaluator._detection_masks_list)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
tf.test.main()
|