You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
359 lines
15 KiB
359 lines
15 KiB
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Abstract detection model.
|
|
|
|
This file defines a generic base class for detection models. Programs that are
|
|
designed to work with arbitrary detection models should only depend on this
|
|
class. We intend for the functions in this class to follow tensor-in/tensor-out
|
|
design, thus all functions have tensors or lists/dictionaries holding tensors as
|
|
inputs and outputs.
|
|
|
|
Abstractly, detection models predict output tensors given input images
|
|
which can be passed to a loss function at training time or passed to a
|
|
postprocessing function at eval time. The computation graphs at a high level
|
|
consequently look as follows:
|
|
|
|
Training time:
|
|
inputs (images tensor) -> preprocess -> predict -> loss -> outputs (loss tensor)
|
|
|
|
Evaluation time:
|
|
inputs (images tensor) -> preprocess -> predict -> postprocess
|
|
-> outputs (boxes tensor, scores tensor, classes tensor, num_detections tensor)
|
|
|
|
DetectionModels must thus implement four functions (1) preprocess, (2) predict,
|
|
(3) postprocess and (4) loss. DetectionModels should make no assumptions about
|
|
the input size or aspect ratio --- they are responsible for doing any
|
|
resize/reshaping necessary (see docstring for the preprocess function).
|
|
Output classes are always integers in the range [0, num_classes). Any mapping
|
|
of these integers to semantic labels is to be handled outside of this class.
|
|
|
|
Images are resized in the `preprocess` method. All of `preprocess`, `predict`,
|
|
and `postprocess` should be reentrant.
|
|
|
|
The `preprocess` method runs `image_resizer_fn` that returns resized_images and
|
|
`true_image_shapes`. Since `image_resizer_fn` can pad the images with zeros,
|
|
true_image_shapes indicate the slices that contain the image without padding.
|
|
This is useful for padding images to be a fixed size for batching.
|
|
|
|
The `postprocess` method uses the true image shapes to clip predictions that lie
|
|
outside of images.
|
|
|
|
By default, DetectionModels produce bounding box detections; However, we support
|
|
a handful of auxiliary annotations associated with each bounding box, namely,
|
|
instance masks and keypoints.
|
|
"""
|
|
import abc
|
|
|
|
from object_detection.core import standard_fields as fields
|
|
|
|
|
|
class DetectionModel(object):
|
|
"""Abstract base class for detection models."""
|
|
__metaclass__ = abc.ABCMeta
|
|
|
|
def __init__(self, num_classes):
|
|
"""Constructor.
|
|
|
|
Args:
|
|
num_classes: number of classes. Note that num_classes *does not* include
|
|
background categories that might be implicitly predicted in various
|
|
implementations.
|
|
"""
|
|
self._num_classes = num_classes
|
|
self._groundtruth_lists = {}
|
|
|
|
@property
|
|
def num_classes(self):
|
|
return self._num_classes
|
|
|
|
def groundtruth_lists(self, field):
|
|
"""Access list of groundtruth tensors.
|
|
|
|
Args:
|
|
field: a string key, options are
|
|
fields.BoxListFields.{boxes,classes,masks,keypoints} or
|
|
fields.InputDataFields.is_annotated.
|
|
|
|
Returns:
|
|
a list of tensors holding groundtruth information (see also
|
|
provide_groundtruth function below), with one entry for each image in the
|
|
batch.
|
|
Raises:
|
|
RuntimeError: if the field has not been provided via provide_groundtruth.
|
|
"""
|
|
if field not in self._groundtruth_lists:
|
|
raise RuntimeError('Groundtruth tensor {} has not been provided'.format(
|
|
field))
|
|
return self._groundtruth_lists[field]
|
|
|
|
def groundtruth_has_field(self, field):
|
|
"""Determines whether the groundtruth includes the given field.
|
|
|
|
Args:
|
|
field: a string key, options are
|
|
fields.BoxListFields.{boxes,classes,masks,keypoints} or
|
|
fields.InputDataFields.is_annotated.
|
|
|
|
Returns:
|
|
True if the groundtruth includes the given field, False otherwise.
|
|
"""
|
|
return field in self._groundtruth_lists
|
|
|
|
@abc.abstractmethod
|
|
def preprocess(self, inputs):
|
|
"""Input preprocessing.
|
|
|
|
To be overridden by implementations.
|
|
|
|
This function is responsible for any scaling/shifting of input values that
|
|
is necessary prior to running the detector on an input image.
|
|
It is also responsible for any resizing, padding that might be necessary
|
|
as images are assumed to arrive in arbitrary sizes. While this function
|
|
could conceivably be part of the predict method (below), it is often
|
|
convenient to keep these separate --- for example, we may want to preprocess
|
|
on one device, place onto a queue, and let another device (e.g., the GPU)
|
|
handle prediction.
|
|
|
|
A few important notes about the preprocess function:
|
|
+ We assume that this operation does not have any trainable variables nor
|
|
does it affect the groundtruth annotations in any way (thus data
|
|
augmentation operations such as random cropping should be performed
|
|
externally).
|
|
+ There is no assumption that the batchsize in this function is the same as
|
|
the batch size in the predict function. In fact, we recommend calling the
|
|
preprocess function prior to calling any batching operations (which should
|
|
happen outside of the model) and thus assuming that batch sizes are equal
|
|
to 1 in the preprocess function.
|
|
+ There is also no explicit assumption that the output resolutions
|
|
must be fixed across inputs --- this is to support "fully convolutional"
|
|
settings in which input images can have different shapes/resolutions.
|
|
|
|
Args:
|
|
inputs: a [batch, height_in, width_in, channels] float32 tensor
|
|
representing a batch of images with values between 0 and 255.0.
|
|
|
|
Returns:
|
|
preprocessed_inputs: a [batch, height_out, width_out, channels] float32
|
|
tensor representing a batch of images.
|
|
true_image_shapes: int32 tensor of shape [batch, 3] where each row is
|
|
of the form [height, width, channels] indicating the shapes
|
|
of true images in the resized images, as resized images can be padded
|
|
with zeros.
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def predict(self, preprocessed_inputs, true_image_shapes):
|
|
"""Predict prediction tensors from inputs tensor.
|
|
|
|
Outputs of this function can be passed to loss or postprocess functions.
|
|
|
|
Args:
|
|
preprocessed_inputs: a [batch, height, width, channels] float32 tensor
|
|
representing a batch of images.
|
|
true_image_shapes: int32 tensor of shape [batch, 3] where each row is
|
|
of the form [height, width, channels] indicating the shapes
|
|
of true images in the resized images, as resized images can be padded
|
|
with zeros.
|
|
|
|
Returns:
|
|
prediction_dict: a dictionary holding prediction tensors to be
|
|
passed to the Loss or Postprocess functions.
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def postprocess(self, prediction_dict, true_image_shapes, **params):
|
|
"""Convert predicted output tensors to final detections.
|
|
|
|
This stage typically performs a few things such as
|
|
* Non-Max Suppression to remove overlapping detection boxes.
|
|
* Score conversion and background class removal.
|
|
|
|
Outputs adhere to the following conventions:
|
|
* Classes are integers in [0, num_classes); background classes are removed
|
|
and the first non-background class is mapped to 0. If the model produces
|
|
class-agnostic detections, then no output is produced for classes.
|
|
* Boxes are to be interpreted as being in [y_min, x_min, y_max, x_max]
|
|
format and normalized relative to the image window.
|
|
* `num_detections` is provided for settings where detections are padded to a
|
|
fixed number of boxes.
|
|
* We do not specifically assume any kind of probabilistic interpretation
|
|
of the scores --- the only important thing is their relative ordering.
|
|
Thus implementations of the postprocess function are free to output
|
|
logits, probabilities, calibrated probabilities, or anything else.
|
|
|
|
Args:
|
|
prediction_dict: a dictionary holding prediction tensors.
|
|
true_image_shapes: int32 tensor of shape [batch, 3] where each row is
|
|
of the form [height, width, channels] indicating the shapes
|
|
of true images in the resized images, as resized images can be padded
|
|
with zeros.
|
|
**params: Additional keyword arguments for specific implementations of
|
|
DetectionModel.
|
|
|
|
Returns:
|
|
detections: a dictionary containing the following fields
|
|
detection_boxes: [batch, max_detections, 4]
|
|
detection_scores: [batch, max_detections]
|
|
detection_classes: [batch, max_detections]
|
|
(If a model is producing class-agnostic detections, this field may be
|
|
missing)
|
|
instance_masks: [batch, max_detections, image_height, image_width]
|
|
(optional)
|
|
keypoints: [batch, max_detections, num_keypoints, 2] (optional)
|
|
num_detections: [batch]
|
|
|
|
In addition to the above fields this stage also outputs the following
|
|
raw tensors:
|
|
|
|
raw_detection_boxes: [batch, total_detections, 4] tensor containing
|
|
all detection boxes from `prediction_dict` in the format
|
|
[ymin, xmin, ymax, xmax] and normalized co-ordinates.
|
|
raw_detection_scores: [batch, total_detections,
|
|
num_classes_with_background] tensor of class score logits for
|
|
raw detection boxes.
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def loss(self, prediction_dict, true_image_shapes):
|
|
"""Compute scalar loss tensors with respect to provided groundtruth.
|
|
|
|
Calling this function requires that groundtruth tensors have been
|
|
provided via the provide_groundtruth function.
|
|
|
|
Args:
|
|
prediction_dict: a dictionary holding predicted tensors
|
|
true_image_shapes: int32 tensor of shape [batch, 3] where each row is
|
|
of the form [height, width, channels] indicating the shapes
|
|
of true images in the resized images, as resized images can be padded
|
|
with zeros.
|
|
|
|
Returns:
|
|
a dictionary mapping strings (loss names) to scalar tensors representing
|
|
loss values.
|
|
"""
|
|
pass
|
|
|
|
def provide_groundtruth(self,
|
|
groundtruth_boxes_list,
|
|
groundtruth_classes_list,
|
|
groundtruth_masks_list=None,
|
|
groundtruth_keypoints_list=None,
|
|
groundtruth_weights_list=None,
|
|
groundtruth_confidences_list=None,
|
|
groundtruth_is_crowd_list=None,
|
|
is_annotated_list=None):
|
|
"""Provide groundtruth tensors.
|
|
|
|
Args:
|
|
groundtruth_boxes_list: a list of 2-D tf.float32 tensors of shape
|
|
[num_boxes, 4] containing coordinates of the groundtruth boxes.
|
|
Groundtruth boxes are provided in [y_min, x_min, y_max, x_max]
|
|
format and assumed to be normalized and clipped
|
|
relative to the image window with y_min <= y_max and x_min <= x_max.
|
|
groundtruth_classes_list: a list of 2-D tf.float32 one-hot (or k-hot)
|
|
tensors of shape [num_boxes, num_classes] containing the class targets
|
|
with the 0th index assumed to map to the first non-background class.
|
|
groundtruth_masks_list: a list of 3-D tf.float32 tensors of
|
|
shape [num_boxes, height_in, width_in] containing instance
|
|
masks with values in {0, 1}. If None, no masks are provided.
|
|
Mask resolution `height_in`x`width_in` must agree with the resolution
|
|
of the input image tensor provided to the `preprocess` function.
|
|
groundtruth_keypoints_list: a list of 3-D tf.float32 tensors of
|
|
shape [num_boxes, num_keypoints, 2] containing keypoints.
|
|
Keypoints are assumed to be provided in normalized coordinates and
|
|
missing keypoints should be encoded as NaN.
|
|
groundtruth_weights_list: A list of 1-D tf.float32 tensors of shape
|
|
[num_boxes] containing weights for groundtruth boxes.
|
|
groundtruth_confidences_list: A list of 2-D tf.float32 tensors of shape
|
|
[num_boxes, num_classes] containing class confidences for groundtruth
|
|
boxes.
|
|
groundtruth_is_crowd_list: A list of 1-D tf.bool tensors of shape
|
|
[num_boxes] containing is_crowd annotations
|
|
is_annotated_list: A list of scalar tf.bool tensors indicating whether
|
|
images have been labeled or not.
|
|
"""
|
|
self._groundtruth_lists[fields.BoxListFields.boxes] = groundtruth_boxes_list
|
|
self._groundtruth_lists[
|
|
fields.BoxListFields.classes] = groundtruth_classes_list
|
|
if groundtruth_weights_list:
|
|
self._groundtruth_lists[fields.BoxListFields.
|
|
weights] = groundtruth_weights_list
|
|
if groundtruth_confidences_list:
|
|
self._groundtruth_lists[fields.BoxListFields.
|
|
confidences] = groundtruth_confidences_list
|
|
if groundtruth_masks_list:
|
|
self._groundtruth_lists[
|
|
fields.BoxListFields.masks] = groundtruth_masks_list
|
|
if groundtruth_keypoints_list:
|
|
self._groundtruth_lists[
|
|
fields.BoxListFields.keypoints] = groundtruth_keypoints_list
|
|
if groundtruth_is_crowd_list:
|
|
self._groundtruth_lists[
|
|
fields.BoxListFields.is_crowd] = groundtruth_is_crowd_list
|
|
if is_annotated_list:
|
|
self._groundtruth_lists[
|
|
fields.InputDataFields.is_annotated] = is_annotated_list
|
|
|
|
@abc.abstractmethod
|
|
def regularization_losses(self):
|
|
"""Returns a list of regularization losses for this model.
|
|
|
|
Returns a list of regularization losses for this model that the estimator
|
|
needs to use during training/optimization.
|
|
|
|
Returns:
|
|
A list of regularization loss tensors.
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def restore_map(self, fine_tune_checkpoint_type='detection'):
|
|
"""Returns a map of variables to load from a foreign checkpoint.
|
|
|
|
Returns a map of variable names to load from a checkpoint to variables in
|
|
the model graph. This enables the model to initialize based on weights from
|
|
another task. For example, the feature extractor variables from a
|
|
classification model can be used to bootstrap training of an object
|
|
detector. When loading from an object detection model, the checkpoint model
|
|
should have the same parameters as this detection model with exception of
|
|
the num_classes parameter.
|
|
|
|
Args:
|
|
fine_tune_checkpoint_type: whether to restore from a full detection
|
|
checkpoint (with compatible variable names) or to restore from a
|
|
classification checkpoint for initialization prior to training.
|
|
Valid values: `detection`, `classification`. Default 'detection'.
|
|
|
|
Returns:
|
|
A dict mapping variable names (to load from a checkpoint) to variables in
|
|
the model graph.
|
|
"""
|
|
pass
|
|
|
|
@abc.abstractmethod
|
|
def updates(self):
|
|
"""Returns a list of update operators for this model.
|
|
|
|
Returns a list of update operators for this model that must be executed at
|
|
each training step. The estimator's train op needs to have a control
|
|
dependency on these updates.
|
|
|
|
Returns:
|
|
A list of update operators.
|
|
"""
|
|
pass
|