You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
479 lines
21 KiB
479 lines
21 KiB
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Tensorflow Example proto decoder for object detection.
|
|
|
|
A decoder to decode string tensors containing serialized tensorflow.Example
|
|
protos for object detection.
|
|
"""
|
|
import tensorflow as tf
|
|
|
|
from object_detection.core import data_decoder
|
|
from object_detection.core import standard_fields as fields
|
|
from object_detection.protos import input_reader_pb2
|
|
from object_detection.utils import label_map_util
|
|
|
|
slim_example_decoder = tf.contrib.slim.tfexample_decoder
|
|
|
|
|
|
class _ClassTensorHandler(slim_example_decoder.Tensor):
|
|
"""An ItemHandler to fetch class ids from class text."""
|
|
|
|
def __init__(self,
|
|
tensor_key,
|
|
label_map_proto_file,
|
|
shape_keys=None,
|
|
shape=None,
|
|
default_value=''):
|
|
"""Initializes the LookupTensor handler.
|
|
|
|
Simply calls a vocabulary (most often, a label mapping) lookup.
|
|
|
|
Args:
|
|
tensor_key: the name of the `TFExample` feature to read the tensor from.
|
|
label_map_proto_file: File path to a text format LabelMapProto message
|
|
mapping class text to id.
|
|
shape_keys: Optional name or list of names of the TF-Example feature in
|
|
which the tensor shape is stored. If a list, then each corresponds to
|
|
one dimension of the shape.
|
|
shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is
|
|
reshaped accordingly.
|
|
default_value: The value used when the `tensor_key` is not found in a
|
|
particular `TFExample`.
|
|
|
|
Raises:
|
|
ValueError: if both `shape_keys` and `shape` are specified.
|
|
"""
|
|
name_to_id = label_map_util.get_label_map_dict(
|
|
label_map_proto_file, use_display_name=False)
|
|
# We use a default_value of -1, but we expect all labels to be contained
|
|
# in the label map.
|
|
name_to_id_table = tf.contrib.lookup.HashTable(
|
|
initializer=tf.contrib.lookup.KeyValueTensorInitializer(
|
|
keys=tf.constant(list(name_to_id.keys())),
|
|
values=tf.constant(list(name_to_id.values()), dtype=tf.int64)),
|
|
default_value=-1)
|
|
display_name_to_id = label_map_util.get_label_map_dict(
|
|
label_map_proto_file, use_display_name=True)
|
|
# We use a default_value of -1, but we expect all labels to be contained
|
|
# in the label map.
|
|
display_name_to_id_table = tf.contrib.lookup.HashTable(
|
|
initializer=tf.contrib.lookup.KeyValueTensorInitializer(
|
|
keys=tf.constant(list(display_name_to_id.keys())),
|
|
values=tf.constant(
|
|
list(display_name_to_id.values()), dtype=tf.int64)),
|
|
default_value=-1)
|
|
|
|
self._name_to_id_table = name_to_id_table
|
|
self._display_name_to_id_table = display_name_to_id_table
|
|
super(_ClassTensorHandler, self).__init__(tensor_key, shape_keys, shape,
|
|
default_value)
|
|
|
|
def tensors_to_item(self, keys_to_tensors):
|
|
unmapped_tensor = super(_ClassTensorHandler,
|
|
self).tensors_to_item(keys_to_tensors)
|
|
return tf.maximum(self._name_to_id_table.lookup(unmapped_tensor),
|
|
self._display_name_to_id_table.lookup(unmapped_tensor))
|
|
|
|
|
|
class _BackupHandler(slim_example_decoder.ItemHandler):
|
|
"""An ItemHandler that tries two ItemHandlers in order."""
|
|
|
|
def __init__(self, handler, backup):
|
|
"""Initializes the BackupHandler handler.
|
|
|
|
If the first Handler's tensors_to_item returns a Tensor with no elements,
|
|
the second Handler is used.
|
|
|
|
Args:
|
|
handler: The primary ItemHandler.
|
|
backup: The backup ItemHandler.
|
|
|
|
Raises:
|
|
ValueError: if either is not an ItemHandler.
|
|
"""
|
|
if not isinstance(handler, slim_example_decoder.ItemHandler):
|
|
raise ValueError('Primary handler is of type %s instead of ItemHandler' %
|
|
type(handler))
|
|
if not isinstance(backup, slim_example_decoder.ItemHandler):
|
|
raise ValueError(
|
|
'Backup handler is of type %s instead of ItemHandler' % type(backup))
|
|
self._handler = handler
|
|
self._backup = backup
|
|
super(_BackupHandler, self).__init__(handler.keys + backup.keys)
|
|
|
|
def tensors_to_item(self, keys_to_tensors):
|
|
item = self._handler.tensors_to_item(keys_to_tensors)
|
|
return tf.cond(
|
|
pred=tf.equal(tf.reduce_prod(tf.shape(item)), 0),
|
|
true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors),
|
|
false_fn=lambda: item)
|
|
|
|
|
|
class TfExampleDecoder(data_decoder.DataDecoder):
|
|
"""Tensorflow Example proto decoder."""
|
|
|
|
def __init__(self,
|
|
load_instance_masks=False,
|
|
instance_mask_type=input_reader_pb2.NUMERICAL_MASKS,
|
|
label_map_proto_file=None,
|
|
use_display_name=False,
|
|
dct_method='',
|
|
num_keypoints=0,
|
|
num_additional_channels=0,
|
|
load_multiclass_scores=False):
|
|
"""Constructor sets keys_to_features and items_to_handlers.
|
|
|
|
Args:
|
|
load_instance_masks: whether or not to load and handle instance masks.
|
|
instance_mask_type: type of instance masks. Options are provided in
|
|
input_reader.proto. This is only used if `load_instance_masks` is True.
|
|
label_map_proto_file: a file path to a
|
|
object_detection.protos.StringIntLabelMap proto. If provided, then the
|
|
mapped IDs of 'image/object/class/text' will take precedence over the
|
|
existing 'image/object/class/label' ID. Also, if provided, it is
|
|
assumed that 'image/object/class/text' will be in the data.
|
|
use_display_name: whether or not to use the `display_name` for label
|
|
mapping (instead of `name`). Only used if label_map_proto_file is
|
|
provided.
|
|
dct_method: An optional string. Defaults to None. It only takes
|
|
effect when image format is jpeg, used to specify a hint about the
|
|
algorithm used for jpeg decompression. Currently valid values
|
|
are ['INTEGER_FAST', 'INTEGER_ACCURATE']. The hint may be ignored, for
|
|
example, the jpeg library does not have that specific option.
|
|
num_keypoints: the number of keypoints per object.
|
|
num_additional_channels: how many additional channels to use.
|
|
load_multiclass_scores: Whether to load multiclass scores associated with
|
|
boxes.
|
|
|
|
Raises:
|
|
ValueError: If `instance_mask_type` option is not one of
|
|
input_reader_pb2.DEFAULT, input_reader_pb2.NUMERICAL, or
|
|
input_reader_pb2.PNG_MASKS.
|
|
"""
|
|
# TODO(rathodv): delete unused `use_display_name` argument once we change
|
|
# other decoders to handle label maps similarly.
|
|
del use_display_name
|
|
self.keys_to_features = {
|
|
'image/encoded':
|
|
tf.FixedLenFeature((), tf.string, default_value=''),
|
|
'image/format':
|
|
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
|
|
'image/filename':
|
|
tf.FixedLenFeature((), tf.string, default_value=''),
|
|
'image/key/sha256':
|
|
tf.FixedLenFeature((), tf.string, default_value=''),
|
|
'image/source_id':
|
|
tf.FixedLenFeature((), tf.string, default_value=''),
|
|
'image/height':
|
|
tf.FixedLenFeature((), tf.int64, default_value=1),
|
|
'image/width':
|
|
tf.FixedLenFeature((), tf.int64, default_value=1),
|
|
# Image-level labels.
|
|
'image/class/text':
|
|
tf.VarLenFeature(tf.string),
|
|
'image/class/label':
|
|
tf.VarLenFeature(tf.int64),
|
|
# Object boxes and classes.
|
|
'image/object/bbox/xmin':
|
|
tf.VarLenFeature(tf.float32),
|
|
'image/object/bbox/xmax':
|
|
tf.VarLenFeature(tf.float32),
|
|
'image/object/bbox/ymin':
|
|
tf.VarLenFeature(tf.float32),
|
|
'image/object/bbox/ymax':
|
|
tf.VarLenFeature(tf.float32),
|
|
'image/object/class/label':
|
|
tf.VarLenFeature(tf.int64),
|
|
'image/object/class/text':
|
|
tf.VarLenFeature(tf.string),
|
|
'image/object/area':
|
|
tf.VarLenFeature(tf.float32),
|
|
'image/object/is_crowd':
|
|
tf.VarLenFeature(tf.int64),
|
|
'image/object/difficult':
|
|
tf.VarLenFeature(tf.int64),
|
|
'image/object/group_of':
|
|
tf.VarLenFeature(tf.int64),
|
|
'image/object/weight':
|
|
tf.VarLenFeature(tf.float32),
|
|
|
|
}
|
|
# We are checking `dct_method` instead of passing it directly in order to
|
|
# ensure TF version 1.6 compatibility.
|
|
if dct_method:
|
|
image = slim_example_decoder.Image(
|
|
image_key='image/encoded',
|
|
format_key='image/format',
|
|
channels=3,
|
|
dct_method=dct_method)
|
|
additional_channel_image = slim_example_decoder.Image(
|
|
image_key='image/additional_channels/encoded',
|
|
format_key='image/format',
|
|
channels=1,
|
|
repeated=True,
|
|
dct_method=dct_method)
|
|
else:
|
|
image = slim_example_decoder.Image(
|
|
image_key='image/encoded', format_key='image/format', channels=3)
|
|
additional_channel_image = slim_example_decoder.Image(
|
|
image_key='image/additional_channels/encoded',
|
|
format_key='image/format',
|
|
channels=1,
|
|
repeated=True)
|
|
self.items_to_handlers = {
|
|
fields.InputDataFields.image:
|
|
image,
|
|
fields.InputDataFields.source_id: (
|
|
slim_example_decoder.Tensor('image/source_id')),
|
|
fields.InputDataFields.key: (
|
|
slim_example_decoder.Tensor('image/key/sha256')),
|
|
fields.InputDataFields.filename: (
|
|
slim_example_decoder.Tensor('image/filename')),
|
|
# Object boxes and classes.
|
|
fields.InputDataFields.groundtruth_boxes: (
|
|
slim_example_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
|
|
'image/object/bbox/')),
|
|
fields.InputDataFields.groundtruth_area:
|
|
slim_example_decoder.Tensor('image/object/area'),
|
|
fields.InputDataFields.groundtruth_is_crowd: (
|
|
slim_example_decoder.Tensor('image/object/is_crowd')),
|
|
fields.InputDataFields.groundtruth_difficult: (
|
|
slim_example_decoder.Tensor('image/object/difficult')),
|
|
fields.InputDataFields.groundtruth_group_of: (
|
|
slim_example_decoder.Tensor('image/object/group_of')),
|
|
fields.InputDataFields.groundtruth_weights: (
|
|
slim_example_decoder.Tensor('image/object/weight')),
|
|
|
|
}
|
|
if load_multiclass_scores:
|
|
self.keys_to_features[
|
|
'image/object/class/multiclass_scores'] = tf.VarLenFeature(tf.float32)
|
|
self.items_to_handlers[fields.InputDataFields.multiclass_scores] = (
|
|
slim_example_decoder.Tensor('image/object/class/multiclass_scores'))
|
|
if num_additional_channels > 0:
|
|
self.keys_to_features[
|
|
'image/additional_channels/encoded'] = tf.FixedLenFeature(
|
|
(num_additional_channels,), tf.string)
|
|
self.items_to_handlers[
|
|
fields.InputDataFields.
|
|
image_additional_channels] = additional_channel_image
|
|
self._num_keypoints = num_keypoints
|
|
if num_keypoints > 0:
|
|
self.keys_to_features['image/object/keypoint/x'] = (
|
|
tf.VarLenFeature(tf.float32))
|
|
self.keys_to_features['image/object/keypoint/y'] = (
|
|
tf.VarLenFeature(tf.float32))
|
|
self.items_to_handlers[fields.InputDataFields.groundtruth_keypoints] = (
|
|
slim_example_decoder.ItemHandlerCallback(
|
|
['image/object/keypoint/y', 'image/object/keypoint/x'],
|
|
self._reshape_keypoints))
|
|
if load_instance_masks:
|
|
if instance_mask_type in (input_reader_pb2.DEFAULT,
|
|
input_reader_pb2.NUMERICAL_MASKS):
|
|
self.keys_to_features['image/object/mask'] = (
|
|
tf.VarLenFeature(tf.float32))
|
|
self.items_to_handlers[
|
|
fields.InputDataFields.groundtruth_instance_masks] = (
|
|
slim_example_decoder.ItemHandlerCallback(
|
|
['image/object/mask', 'image/height', 'image/width'],
|
|
self._reshape_instance_masks))
|
|
elif instance_mask_type == input_reader_pb2.PNG_MASKS:
|
|
self.keys_to_features['image/object/mask'] = tf.VarLenFeature(tf.string)
|
|
self.items_to_handlers[
|
|
fields.InputDataFields.groundtruth_instance_masks] = (
|
|
slim_example_decoder.ItemHandlerCallback(
|
|
['image/object/mask', 'image/height', 'image/width'],
|
|
self._decode_png_instance_masks))
|
|
else:
|
|
raise ValueError('Did not recognize the `instance_mask_type` option.')
|
|
if label_map_proto_file:
|
|
# If the label_map_proto is provided, try to use it in conjunction with
|
|
# the class text, and fall back to a materialized ID.
|
|
label_handler = _BackupHandler(
|
|
_ClassTensorHandler(
|
|
'image/object/class/text', label_map_proto_file,
|
|
default_value=''),
|
|
slim_example_decoder.Tensor('image/object/class/label'))
|
|
image_label_handler = _BackupHandler(
|
|
_ClassTensorHandler(
|
|
fields.TfExampleFields.image_class_text,
|
|
label_map_proto_file,
|
|
default_value=''),
|
|
slim_example_decoder.Tensor(fields.TfExampleFields.image_class_label))
|
|
else:
|
|
label_handler = slim_example_decoder.Tensor('image/object/class/label')
|
|
image_label_handler = slim_example_decoder.Tensor(
|
|
fields.TfExampleFields.image_class_label)
|
|
self.items_to_handlers[
|
|
fields.InputDataFields.groundtruth_classes] = label_handler
|
|
self.items_to_handlers[
|
|
fields.InputDataFields.groundtruth_image_classes] = image_label_handler
|
|
|
|
def decode(self, tf_example_string_tensor):
|
|
"""Decodes serialized tensorflow example and returns a tensor dictionary.
|
|
|
|
Args:
|
|
tf_example_string_tensor: a string tensor holding a serialized tensorflow
|
|
example proto.
|
|
|
|
Returns:
|
|
A dictionary of the following tensors.
|
|
fields.InputDataFields.image - 3D uint8 tensor of shape [None, None, 3]
|
|
containing image.
|
|
fields.InputDataFields.original_image_spatial_shape - 1D int32 tensor of
|
|
shape [2] containing shape of the image.
|
|
fields.InputDataFields.source_id - string tensor containing original
|
|
image id.
|
|
fields.InputDataFields.key - string tensor with unique sha256 hash key.
|
|
fields.InputDataFields.filename - string tensor with original dataset
|
|
filename.
|
|
fields.InputDataFields.groundtruth_boxes - 2D float32 tensor of shape
|
|
[None, 4] containing box corners.
|
|
fields.InputDataFields.groundtruth_classes - 1D int64 tensor of shape
|
|
[None] containing classes for the boxes.
|
|
fields.InputDataFields.groundtruth_weights - 1D float32 tensor of
|
|
shape [None] indicating the weights of groundtruth boxes.
|
|
fields.InputDataFields.groundtruth_area - 1D float32 tensor of shape
|
|
[None] containing containing object mask area in pixel squared.
|
|
fields.InputDataFields.groundtruth_is_crowd - 1D bool tensor of shape
|
|
[None] indicating if the boxes enclose a crowd.
|
|
|
|
Optional:
|
|
fields.InputDataFields.image_additional_channels - 3D uint8 tensor of
|
|
shape [None, None, num_additional_channels]. 1st dim is height; 2nd dim
|
|
is width; 3rd dim is the number of additional channels.
|
|
fields.InputDataFields.groundtruth_difficult - 1D bool tensor of shape
|
|
[None] indicating if the boxes represent `difficult` instances.
|
|
fields.InputDataFields.groundtruth_group_of - 1D bool tensor of shape
|
|
[None] indicating if the boxes represent `group_of` instances.
|
|
fields.InputDataFields.groundtruth_keypoints - 3D float32 tensor of
|
|
shape [None, None, 2] containing keypoints, where the coordinates of
|
|
the keypoints are ordered (y, x).
|
|
fields.InputDataFields.groundtruth_instance_masks - 3D float32 tensor of
|
|
shape [None, None, None] containing instance masks.
|
|
fields.InputDataFields.groundtruth_image_classes - 1D uint64 of shape
|
|
[None] containing classes for the boxes.
|
|
fields.InputDataFields.multiclass_scores - 1D float32 tensor of shape
|
|
[None * num_classes] containing flattened multiclass scores for
|
|
groundtruth boxes.
|
|
"""
|
|
serialized_example = tf.reshape(tf_example_string_tensor, shape=[])
|
|
decoder = slim_example_decoder.TFExampleDecoder(self.keys_to_features,
|
|
self.items_to_handlers)
|
|
keys = decoder.list_items()
|
|
tensors = decoder.decode(serialized_example, items=keys)
|
|
tensor_dict = dict(zip(keys, tensors))
|
|
is_crowd = fields.InputDataFields.groundtruth_is_crowd
|
|
tensor_dict[is_crowd] = tf.cast(tensor_dict[is_crowd], dtype=tf.bool)
|
|
tensor_dict[fields.InputDataFields.image].set_shape([None, None, 3])
|
|
tensor_dict[fields.InputDataFields.original_image_spatial_shape] = tf.shape(
|
|
tensor_dict[fields.InputDataFields.image])[:2]
|
|
|
|
if fields.InputDataFields.image_additional_channels in tensor_dict:
|
|
channels = tensor_dict[fields.InputDataFields.image_additional_channels]
|
|
channels = tf.squeeze(channels, axis=3)
|
|
channels = tf.transpose(channels, perm=[1, 2, 0])
|
|
tensor_dict[fields.InputDataFields.image_additional_channels] = channels
|
|
|
|
def default_groundtruth_weights():
|
|
return tf.ones(
|
|
[tf.shape(tensor_dict[fields.InputDataFields.groundtruth_boxes])[0]],
|
|
dtype=tf.float32)
|
|
|
|
tensor_dict[fields.InputDataFields.groundtruth_weights] = tf.cond(
|
|
tf.greater(
|
|
tf.shape(
|
|
tensor_dict[fields.InputDataFields.groundtruth_weights])[0],
|
|
0), lambda: tensor_dict[fields.InputDataFields.groundtruth_weights],
|
|
default_groundtruth_weights)
|
|
return tensor_dict
|
|
|
|
def _reshape_keypoints(self, keys_to_tensors):
|
|
"""Reshape keypoints.
|
|
|
|
The instance segmentation masks are reshaped to [num_instances,
|
|
num_keypoints, 2].
|
|
|
|
Args:
|
|
keys_to_tensors: a dictionary from keys to tensors.
|
|
|
|
Returns:
|
|
A 3-D float tensor of shape [num_instances, num_keypoints, 2] with values
|
|
in {0, 1}.
|
|
"""
|
|
y = keys_to_tensors['image/object/keypoint/y']
|
|
if isinstance(y, tf.SparseTensor):
|
|
y = tf.sparse_tensor_to_dense(y)
|
|
y = tf.expand_dims(y, 1)
|
|
x = keys_to_tensors['image/object/keypoint/x']
|
|
if isinstance(x, tf.SparseTensor):
|
|
x = tf.sparse_tensor_to_dense(x)
|
|
x = tf.expand_dims(x, 1)
|
|
keypoints = tf.concat([y, x], 1)
|
|
keypoints = tf.reshape(keypoints, [-1, self._num_keypoints, 2])
|
|
return keypoints
|
|
|
|
def _reshape_instance_masks(self, keys_to_tensors):
|
|
"""Reshape instance segmentation masks.
|
|
|
|
The instance segmentation masks are reshaped to [num_instances, height,
|
|
width].
|
|
|
|
Args:
|
|
keys_to_tensors: a dictionary from keys to tensors.
|
|
|
|
Returns:
|
|
A 3-D float tensor of shape [num_instances, height, width] with values
|
|
in {0, 1}.
|
|
"""
|
|
height = keys_to_tensors['image/height']
|
|
width = keys_to_tensors['image/width']
|
|
to_shape = tf.cast(tf.stack([-1, height, width]), tf.int32)
|
|
masks = keys_to_tensors['image/object/mask']
|
|
if isinstance(masks, tf.SparseTensor):
|
|
masks = tf.sparse_tensor_to_dense(masks)
|
|
masks = tf.reshape(tf.to_float(tf.greater(masks, 0.0)), to_shape)
|
|
return tf.cast(masks, tf.float32)
|
|
|
|
def _decode_png_instance_masks(self, keys_to_tensors):
|
|
"""Decode PNG instance segmentation masks and stack into dense tensor.
|
|
|
|
The instance segmentation masks are reshaped to [num_instances, height,
|
|
width].
|
|
|
|
Args:
|
|
keys_to_tensors: a dictionary from keys to tensors.
|
|
|
|
Returns:
|
|
A 3-D float tensor of shape [num_instances, height, width] with values
|
|
in {0, 1}.
|
|
"""
|
|
|
|
def decode_png_mask(image_buffer):
|
|
image = tf.squeeze(
|
|
tf.image.decode_image(image_buffer, channels=1), axis=2)
|
|
image.set_shape([None, None])
|
|
image = tf.to_float(tf.greater(image, 0))
|
|
return image
|
|
|
|
png_masks = keys_to_tensors['image/object/mask']
|
|
height = keys_to_tensors['image/height']
|
|
width = keys_to_tensors['image/width']
|
|
if isinstance(png_masks, tf.SparseTensor):
|
|
png_masks = tf.sparse_tensor_to_dense(png_masks, default_value='')
|
|
return tf.cond(
|
|
tf.greater(tf.size(png_masks), 0),
|
|
lambda: tf.map_fn(decode_png_mask, png_masks, dtype=tf.float32),
|
|
lambda: tf.zeros(tf.to_int32(tf.stack([0, height, width]))))
|