You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

674 lines
29 KiB

6 years ago
  1. # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # ==============================================================================
  15. """Classification and regression loss functions for object detection.
  16. Localization losses:
  17. * WeightedL2LocalizationLoss
  18. * WeightedSmoothL1LocalizationLoss
  19. * WeightedIOULocalizationLoss
  20. Classification losses:
  21. * WeightedSigmoidClassificationLoss
  22. * WeightedSoftmaxClassificationLoss
  23. * WeightedSoftmaxClassificationAgainstLogitsLoss
  24. * BootstrappedSigmoidClassificationLoss
  25. """
  26. from abc import ABCMeta
  27. from abc import abstractmethod
  28. import tensorflow as tf
  29. from object_detection.core import box_list
  30. from object_detection.core import box_list_ops
  31. from object_detection.utils import ops
  32. slim = tf.contrib.slim
  33. class Loss(object):
  34. """Abstract base class for loss functions."""
  35. __metaclass__ = ABCMeta
  36. def __call__(self,
  37. prediction_tensor,
  38. target_tensor,
  39. ignore_nan_targets=False,
  40. losses_mask=None,
  41. scope=None,
  42. **params):
  43. """Call the loss function.
  44. Args:
  45. prediction_tensor: an N-d tensor of shape [batch, anchors, ...]
  46. representing predicted quantities.
  47. target_tensor: an N-d tensor of shape [batch, anchors, ...] representing
  48. regression or classification targets.
  49. ignore_nan_targets: whether to ignore nan targets in the loss computation.
  50. E.g. can be used if the target tensor is missing groundtruth data that
  51. shouldn't be factored into the loss.
  52. losses_mask: A [batch] boolean tensor that indicates whether losses should
  53. be applied to individual images in the batch. For elements that
  54. are False, corresponding prediction, target, and weight tensors will not
  55. contribute to loss computation. If None, no filtering will take place
  56. prior to loss computation.
  57. scope: Op scope name. Defaults to 'Loss' if None.
  58. **params: Additional keyword arguments for specific implementations of
  59. the Loss.
  60. Returns:
  61. loss: a tensor representing the value of the loss function.
  62. """
  63. with tf.name_scope(scope, 'Loss',
  64. [prediction_tensor, target_tensor, params]) as scope:
  65. if ignore_nan_targets:
  66. target_tensor = tf.where(tf.is_nan(target_tensor),
  67. prediction_tensor,
  68. target_tensor)
  69. if losses_mask is not None:
  70. tensor_multiplier = self._get_loss_multiplier_for_tensor(
  71. prediction_tensor,
  72. losses_mask)
  73. prediction_tensor *= tensor_multiplier
  74. target_tensor *= tensor_multiplier
  75. if 'weights' in params:
  76. params['weights'] = tf.convert_to_tensor(params['weights'])
  77. weights_multiplier = self._get_loss_multiplier_for_tensor(
  78. params['weights'],
  79. losses_mask)
  80. params['weights'] *= weights_multiplier
  81. return self._compute_loss(prediction_tensor, target_tensor, **params)
  82. def _get_loss_multiplier_for_tensor(self, tensor, losses_mask):
  83. loss_multiplier_shape = tf.stack([-1] + [1] * (len(tensor.shape) - 1))
  84. return tf.cast(tf.reshape(losses_mask, loss_multiplier_shape), tf.float32)
  85. @abstractmethod
  86. def _compute_loss(self, prediction_tensor, target_tensor, **params):
  87. """Method to be overridden by implementations.
  88. Args:
  89. prediction_tensor: a tensor representing predicted quantities
  90. target_tensor: a tensor representing regression or classification targets
  91. **params: Additional keyword arguments for specific implementations of
  92. the Loss.
  93. Returns:
  94. loss: an N-d tensor of shape [batch, anchors, ...] containing the loss per
  95. anchor
  96. """
  97. pass
  98. class WeightedL2LocalizationLoss(Loss):
  99. """L2 localization loss function with anchorwise output support.
  100. Loss[b,a] = .5 * ||weights[b,a] * (prediction[b,a,:] - target[b,a,:])||^2
  101. """
  102. def _compute_loss(self, prediction_tensor, target_tensor, weights):
  103. """Compute loss function.
  104. Args:
  105. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  106. code_size] representing the (encoded) predicted locations of objects.
  107. target_tensor: A float tensor of shape [batch_size, num_anchors,
  108. code_size] representing the regression targets
  109. weights: a float tensor of shape [batch_size, num_anchors]
  110. Returns:
  111. loss: a float tensor of shape [batch_size, num_anchors] tensor
  112. representing the value of the loss function.
  113. """
  114. weighted_diff = (prediction_tensor - target_tensor) * tf.expand_dims(
  115. weights, 2)
  116. square_diff = 0.5 * tf.square(weighted_diff)
  117. return tf.reduce_sum(square_diff, 2)
  118. class WeightedSmoothL1LocalizationLoss(Loss):
  119. """Smooth L1 localization loss function aka Huber Loss..
  120. The smooth L1_loss is defined elementwise as .5 x^2 if |x| <= delta and
  121. delta * (|x|- 0.5*delta) otherwise, where x is the difference between
  122. predictions and target.
  123. See also Equation (3) in the Fast R-CNN paper by Ross Girshick (ICCV 2015)
  124. """
  125. def __init__(self, delta=1.0):
  126. """Constructor.
  127. Args:
  128. delta: delta for smooth L1 loss.
  129. """
  130. self._delta = delta
  131. def _compute_loss(self, prediction_tensor, target_tensor, weights):
  132. """Compute loss function.
  133. Args:
  134. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  135. code_size] representing the (encoded) predicted locations of objects.
  136. target_tensor: A float tensor of shape [batch_size, num_anchors,
  137. code_size] representing the regression targets
  138. weights: a float tensor of shape [batch_size, num_anchors]
  139. Returns:
  140. loss: a float tensor of shape [batch_size, num_anchors] tensor
  141. representing the value of the loss function.
  142. """
  143. return tf.reduce_sum(tf.losses.huber_loss(
  144. target_tensor,
  145. prediction_tensor,
  146. delta=self._delta,
  147. weights=tf.expand_dims(weights, axis=2),
  148. loss_collection=None,
  149. reduction=tf.losses.Reduction.NONE
  150. ), axis=2)
  151. class WeightedIOULocalizationLoss(Loss):
  152. """IOU localization loss function.
  153. Sums the IOU for corresponding pairs of predicted/groundtruth boxes
  154. and for each pair assign a loss of 1 - IOU. We then compute a weighted
  155. sum over all pairs which is returned as the total loss.
  156. """
  157. def _compute_loss(self, prediction_tensor, target_tensor, weights):
  158. """Compute loss function.
  159. Args:
  160. prediction_tensor: A float tensor of shape [batch_size, num_anchors, 4]
  161. representing the decoded predicted boxes
  162. target_tensor: A float tensor of shape [batch_size, num_anchors, 4]
  163. representing the decoded target boxes
  164. weights: a float tensor of shape [batch_size, num_anchors]
  165. Returns:
  166. loss: a float tensor of shape [batch_size, num_anchors] tensor
  167. representing the value of the loss function.
  168. """
  169. predicted_boxes = box_list.BoxList(tf.reshape(prediction_tensor, [-1, 4]))
  170. target_boxes = box_list.BoxList(tf.reshape(target_tensor, [-1, 4]))
  171. per_anchor_iou_loss = 1.0 - box_list_ops.matched_iou(predicted_boxes,
  172. target_boxes)
  173. return tf.reshape(weights, [-1]) * per_anchor_iou_loss
  174. class WeightedSigmoidClassificationLoss(Loss):
  175. """Sigmoid cross entropy classification loss function."""
  176. def _compute_loss(self,
  177. prediction_tensor,
  178. target_tensor,
  179. weights,
  180. class_indices=None):
  181. """Compute loss function.
  182. Args:
  183. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  184. num_classes] representing the predicted logits for each class
  185. target_tensor: A float tensor of shape [batch_size, num_anchors,
  186. num_classes] representing one-hot encoded classification targets
  187. weights: a float tensor of shape, either [batch_size, num_anchors,
  188. num_classes] or [batch_size, num_anchors, 1]. If the shape is
  189. [batch_size, num_anchors, 1], all the classses are equally weighted.
  190. class_indices: (Optional) A 1-D integer tensor of class indices.
  191. If provided, computes loss only for the specified class indices.
  192. Returns:
  193. loss: a float tensor of shape [batch_size, num_anchors, num_classes]
  194. representing the value of the loss function.
  195. """
  196. if class_indices is not None:
  197. weights *= tf.reshape(
  198. ops.indices_to_dense_vector(class_indices,
  199. tf.shape(prediction_tensor)[2]),
  200. [1, 1, -1])
  201. per_entry_cross_ent = (tf.nn.sigmoid_cross_entropy_with_logits(
  202. labels=target_tensor, logits=prediction_tensor))
  203. return per_entry_cross_ent * weights
  204. class SigmoidFocalClassificationLoss(Loss):
  205. """Sigmoid focal cross entropy loss.
  206. Focal loss down-weights well classified examples and focusses on the hard
  207. examples. See https://arxiv.org/pdf/1708.02002.pdf for the loss definition.
  208. """
  209. def __init__(self, gamma=2.0, alpha=0.25):
  210. """Constructor.
  211. Args:
  212. gamma: exponent of the modulating factor (1 - p_t) ^ gamma.
  213. alpha: optional alpha weighting factor to balance positives vs negatives.
  214. """
  215. self._alpha = alpha
  216. self._gamma = gamma
  217. def _compute_loss(self,
  218. prediction_tensor,
  219. target_tensor,
  220. weights,
  221. class_indices=None):
  222. """Compute loss function.
  223. Args:
  224. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  225. num_classes] representing the predicted logits for each class
  226. target_tensor: A float tensor of shape [batch_size, num_anchors,
  227. num_classes] representing one-hot encoded classification targets
  228. weights: a float tensor of shape, either [batch_size, num_anchors,
  229. num_classes] or [batch_size, num_anchors, 1]. If the shape is
  230. [batch_size, num_anchors, 1], all the classses are equally weighted.
  231. class_indices: (Optional) A 1-D integer tensor of class indices.
  232. If provided, computes loss only for the specified class indices.
  233. Returns:
  234. loss: a float tensor of shape [batch_size, num_anchors, num_classes]
  235. representing the value of the loss function.
  236. """
  237. if class_indices is not None:
  238. weights *= tf.reshape(
  239. ops.indices_to_dense_vector(class_indices,
  240. tf.shape(prediction_tensor)[2]),
  241. [1, 1, -1])
  242. per_entry_cross_ent = (tf.nn.sigmoid_cross_entropy_with_logits(
  243. labels=target_tensor, logits=prediction_tensor))
  244. prediction_probabilities = tf.sigmoid(prediction_tensor)
  245. p_t = ((target_tensor * prediction_probabilities) +
  246. ((1 - target_tensor) * (1 - prediction_probabilities)))
  247. modulating_factor = 1.0
  248. if self._gamma:
  249. modulating_factor = tf.pow(1.0 - p_t, self._gamma)
  250. alpha_weight_factor = 1.0
  251. if self._alpha is not None:
  252. alpha_weight_factor = (target_tensor * self._alpha +
  253. (1 - target_tensor) * (1 - self._alpha))
  254. focal_cross_entropy_loss = (modulating_factor * alpha_weight_factor *
  255. per_entry_cross_ent)
  256. return focal_cross_entropy_loss * weights
  257. class WeightedSoftmaxClassificationLoss(Loss):
  258. """Softmax loss function."""
  259. def __init__(self, logit_scale=1.0):
  260. """Constructor.
  261. Args:
  262. logit_scale: When this value is high, the prediction is "diffused" and
  263. when this value is low, the prediction is made peakier.
  264. (default 1.0)
  265. """
  266. self._logit_scale = logit_scale
  267. def _compute_loss(self, prediction_tensor, target_tensor, weights):
  268. """Compute loss function.
  269. Args:
  270. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  271. num_classes] representing the predicted logits for each class
  272. target_tensor: A float tensor of shape [batch_size, num_anchors,
  273. num_classes] representing one-hot encoded classification targets
  274. weights: a float tensor of shape, either [batch_size, num_anchors,
  275. num_classes] or [batch_size, num_anchors, 1]. If the shape is
  276. [batch_size, num_anchors, 1], all the classses are equally weighted.
  277. Returns:
  278. loss: a float tensor of shape [batch_size, num_anchors]
  279. representing the value of the loss function.
  280. """
  281. weights = tf.reduce_mean(weights, axis=2)
  282. num_classes = prediction_tensor.get_shape().as_list()[-1]
  283. prediction_tensor = tf.divide(
  284. prediction_tensor, self._logit_scale, name='scale_logit')
  285. per_row_cross_ent = (tf.nn.softmax_cross_entropy_with_logits(
  286. labels=tf.reshape(target_tensor, [-1, num_classes]),
  287. logits=tf.reshape(prediction_tensor, [-1, num_classes])))
  288. return tf.reshape(per_row_cross_ent, tf.shape(weights)) * weights
  289. class WeightedSoftmaxClassificationAgainstLogitsLoss(Loss):
  290. """Softmax loss function against logits.
  291. Targets are expected to be provided in logits space instead of "one hot" or
  292. "probability distribution" space.
  293. """
  294. def __init__(self, logit_scale=1.0):
  295. """Constructor.
  296. Args:
  297. logit_scale: When this value is high, the target is "diffused" and
  298. when this value is low, the target is made peakier.
  299. (default 1.0)
  300. """
  301. self._logit_scale = logit_scale
  302. def _scale_and_softmax_logits(self, logits):
  303. """Scale logits then apply softmax."""
  304. scaled_logits = tf.divide(logits, self._logit_scale, name='scale_logits')
  305. return tf.nn.softmax(scaled_logits, name='convert_scores')
  306. def _compute_loss(self, prediction_tensor, target_tensor, weights):
  307. """Compute loss function.
  308. Args:
  309. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  310. num_classes] representing the predicted logits for each class
  311. target_tensor: A float tensor of shape [batch_size, num_anchors,
  312. num_classes] representing logit classification targets
  313. weights: a float tensor of shape, either [batch_size, num_anchors,
  314. num_classes] or [batch_size, num_anchors, 1]. If the shape is
  315. [batch_size, num_anchors, 1], all the classses are equally weighted.
  316. Returns:
  317. loss: a float tensor of shape [batch_size, num_anchors]
  318. representing the value of the loss function.
  319. """
  320. weights = tf.reduce_mean(weights, axis=2)
  321. num_classes = prediction_tensor.get_shape().as_list()[-1]
  322. target_tensor = self._scale_and_softmax_logits(target_tensor)
  323. prediction_tensor = tf.divide(prediction_tensor, self._logit_scale,
  324. name='scale_logits')
  325. per_row_cross_ent = (tf.nn.softmax_cross_entropy_with_logits(
  326. labels=tf.reshape(target_tensor, [-1, num_classes]),
  327. logits=tf.reshape(prediction_tensor, [-1, num_classes])))
  328. return tf.reshape(per_row_cross_ent, tf.shape(weights)) * weights
  329. class BootstrappedSigmoidClassificationLoss(Loss):
  330. """Bootstrapped sigmoid cross entropy classification loss function.
  331. This loss uses a convex combination of training labels and the current model's
  332. predictions as training targets in the classification loss. The idea is that
  333. as the model improves over time, its predictions can be trusted more and we
  334. can use these predictions to mitigate the damage of noisy/incorrect labels,
  335. because incorrect labels are likely to be eventually highly inconsistent with
  336. other stimuli predicted to have the same label by the model.
  337. In "soft" bootstrapping, we use all predicted class probabilities, whereas in
  338. "hard" bootstrapping, we use the single class favored by the model.
  339. See also Training Deep Neural Networks On Noisy Labels with Bootstrapping by
  340. Reed et al. (ICLR 2015).
  341. """
  342. def __init__(self, alpha, bootstrap_type='soft'):
  343. """Constructor.
  344. Args:
  345. alpha: a float32 scalar tensor between 0 and 1 representing interpolation
  346. weight
  347. bootstrap_type: set to either 'hard' or 'soft' (default)
  348. Raises:
  349. ValueError: if bootstrap_type is not either 'hard' or 'soft'
  350. """
  351. if bootstrap_type != 'hard' and bootstrap_type != 'soft':
  352. raise ValueError('Unrecognized bootstrap_type: must be one of '
  353. '\'hard\' or \'soft.\'')
  354. self._alpha = alpha
  355. self._bootstrap_type = bootstrap_type
  356. def _compute_loss(self, prediction_tensor, target_tensor, weights):
  357. """Compute loss function.
  358. Args:
  359. prediction_tensor: A float tensor of shape [batch_size, num_anchors,
  360. num_classes] representing the predicted logits for each class
  361. target_tensor: A float tensor of shape [batch_size, num_anchors,
  362. num_classes] representing one-hot encoded classification targets
  363. weights: a float tensor of shape, either [batch_size, num_anchors,
  364. num_classes] or [batch_size, num_anchors, 1]. If the shape is
  365. [batch_size, num_anchors, 1], all the classses are equally weighted.
  366. Returns:
  367. loss: a float tensor of shape [batch_size, num_anchors, num_classes]
  368. representing the value of the loss function.
  369. """
  370. if self._bootstrap_type == 'soft':
  371. bootstrap_target_tensor = self._alpha * target_tensor + (
  372. 1.0 - self._alpha) * tf.sigmoid(prediction_tensor)
  373. else:
  374. bootstrap_target_tensor = self._alpha * target_tensor + (
  375. 1.0 - self._alpha) * tf.cast(
  376. tf.sigmoid(prediction_tensor) > 0.5, tf.float32)
  377. per_entry_cross_ent = (tf.nn.sigmoid_cross_entropy_with_logits(
  378. labels=bootstrap_target_tensor, logits=prediction_tensor))
  379. return per_entry_cross_ent * weights
  380. class HardExampleMiner(object):
  381. """Hard example mining for regions in a list of images.
  382. Implements hard example mining to select a subset of regions to be
  383. back-propagated. For each image, selects the regions with highest losses,
  384. subject to the condition that a newly selected region cannot have
  385. an IOU > iou_threshold with any of the previously selected regions.
  386. This can be achieved by re-using a greedy non-maximum suppression algorithm.
  387. A constraint on the number of negatives mined per positive region can also be
  388. enforced.
  389. Reference papers: "Training Region-based Object Detectors with Online
  390. Hard Example Mining" (CVPR 2016) by Srivastava et al., and
  391. "SSD: Single Shot MultiBox Detector" (ECCV 2016) by Liu et al.
  392. """
  393. def __init__(self,
  394. num_hard_examples=64,
  395. iou_threshold=0.7,
  396. loss_type='both',
  397. cls_loss_weight=0.05,
  398. loc_loss_weight=0.06,
  399. max_negatives_per_positive=None,
  400. min_negatives_per_image=0):
  401. """Constructor.
  402. The hard example mining implemented by this class can replicate the behavior
  403. in the two aforementioned papers (Srivastava et al., and Liu et al).
  404. To replicate the A2 paper (Srivastava et al), num_hard_examples is set
  405. to a fixed parameter (64 by default) and iou_threshold is set to .7 for
  406. running non-max-suppression the predicted boxes prior to hard mining.
  407. In order to replicate the SSD paper (Liu et al), num_hard_examples should
  408. be set to None, max_negatives_per_positive should be 3 and iou_threshold
  409. should be 1.0 (in order to effectively turn off NMS).
  410. Args:
  411. num_hard_examples: maximum number of hard examples to be
  412. selected per image (prior to enforcing max negative to positive ratio
  413. constraint). If set to None, all examples obtained after NMS are
  414. considered.
  415. iou_threshold: minimum intersection over union for an example
  416. to be discarded during NMS.
  417. loss_type: use only classification losses ('cls', default),
  418. localization losses ('loc') or both losses ('both').
  419. In the last case, cls_loss_weight and loc_loss_weight are used to
  420. compute weighted sum of the two losses.
  421. cls_loss_weight: weight for classification loss.
  422. loc_loss_weight: weight for location loss.
  423. max_negatives_per_positive: maximum number of negatives to retain for
  424. each positive anchor. By default, num_negatives_per_positive is None,
  425. which means that we do not enforce a prespecified negative:positive
  426. ratio. Note also that num_negatives_per_positives can be a float
  427. (and will be converted to be a float even if it is passed in otherwise).
  428. min_negatives_per_image: minimum number of negative anchors to sample for
  429. a given image. Setting this to a positive number allows sampling
  430. negatives in an image without any positive anchors and thus not biased
  431. towards at least one detection per image.
  432. """
  433. self._num_hard_examples = num_hard_examples
  434. self._iou_threshold = iou_threshold
  435. self._loss_type = loss_type
  436. self._cls_loss_weight = cls_loss_weight
  437. self._loc_loss_weight = loc_loss_weight
  438. self._max_negatives_per_positive = max_negatives_per_positive
  439. self._min_negatives_per_image = min_negatives_per_image
  440. if self._max_negatives_per_positive is not None:
  441. self._max_negatives_per_positive = float(self._max_negatives_per_positive)
  442. self._num_positives_list = None
  443. self._num_negatives_list = None
  444. def __call__(self,
  445. location_losses,
  446. cls_losses,
  447. decoded_boxlist_list,
  448. match_list=None):
  449. """Computes localization and classification losses after hard mining.
  450. Args:
  451. location_losses: a float tensor of shape [num_images, num_anchors]
  452. representing anchorwise localization losses.
  453. cls_losses: a float tensor of shape [num_images, num_anchors]
  454. representing anchorwise classification losses.
  455. decoded_boxlist_list: a list of decoded BoxList representing location
  456. predictions for each image.
  457. match_list: an optional list of matcher.Match objects encoding the match
  458. between anchors and groundtruth boxes for each image of the batch,
  459. with rows of the Match objects corresponding to groundtruth boxes
  460. and columns corresponding to anchors. Match objects in match_list are
  461. used to reference which anchors are positive, negative or ignored. If
  462. self._max_negatives_per_positive exists, these are then used to enforce
  463. a prespecified negative to positive ratio.
  464. Returns:
  465. mined_location_loss: a float scalar with sum of localization losses from
  466. selected hard examples.
  467. mined_cls_loss: a float scalar with sum of classification losses from
  468. selected hard examples.
  469. Raises:
  470. ValueError: if location_losses, cls_losses and decoded_boxlist_list do
  471. not have compatible shapes (i.e., they must correspond to the same
  472. number of images).
  473. ValueError: if match_list is specified but its length does not match
  474. len(decoded_boxlist_list).
  475. """
  476. mined_location_losses = []
  477. mined_cls_losses = []
  478. location_losses = tf.unstack(location_losses)
  479. cls_losses = tf.unstack(cls_losses)
  480. num_images = len(decoded_boxlist_list)
  481. if not match_list:
  482. match_list = num_images * [None]
  483. if not len(location_losses) == len(decoded_boxlist_list) == len(cls_losses):
  484. raise ValueError('location_losses, cls_losses and decoded_boxlist_list '
  485. 'do not have compatible shapes.')
  486. if not isinstance(match_list, list):
  487. raise ValueError('match_list must be a list.')
  488. if len(match_list) != len(decoded_boxlist_list):
  489. raise ValueError('match_list must either be None or have '
  490. 'length=len(decoded_boxlist_list).')
  491. num_positives_list = []
  492. num_negatives_list = []
  493. for ind, detection_boxlist in enumerate(decoded_boxlist_list):
  494. box_locations = detection_boxlist.get()
  495. match = match_list[ind]
  496. image_losses = cls_losses[ind]
  497. if self._loss_type == 'loc':
  498. image_losses = location_losses[ind]
  499. elif self._loss_type == 'both':
  500. image_losses *= self._cls_loss_weight
  501. image_losses += location_losses[ind] * self._loc_loss_weight
  502. if self._num_hard_examples is not None:
  503. num_hard_examples = self._num_hard_examples
  504. else:
  505. num_hard_examples = detection_boxlist.num_boxes()
  506. selected_indices = tf.image.non_max_suppression(
  507. box_locations, image_losses, num_hard_examples, self._iou_threshold)
  508. if self._max_negatives_per_positive is not None and match:
  509. (selected_indices, num_positives,
  510. num_negatives) = self._subsample_selection_to_desired_neg_pos_ratio(
  511. selected_indices, match, self._max_negatives_per_positive,
  512. self._min_negatives_per_image)
  513. num_positives_list.append(num_positives)
  514. num_negatives_list.append(num_negatives)
  515. mined_location_losses.append(
  516. tf.reduce_sum(tf.gather(location_losses[ind], selected_indices)))
  517. mined_cls_losses.append(
  518. tf.reduce_sum(tf.gather(cls_losses[ind], selected_indices)))
  519. location_loss = tf.reduce_sum(tf.stack(mined_location_losses))
  520. cls_loss = tf.reduce_sum(tf.stack(mined_cls_losses))
  521. if match and self._max_negatives_per_positive:
  522. self._num_positives_list = num_positives_list
  523. self._num_negatives_list = num_negatives_list
  524. return (location_loss, cls_loss)
  525. def summarize(self):
  526. """Summarize the number of positives and negatives after mining."""
  527. if self._num_positives_list and self._num_negatives_list:
  528. avg_num_positives = tf.reduce_mean(tf.to_float(self._num_positives_list))
  529. avg_num_negatives = tf.reduce_mean(tf.to_float(self._num_negatives_list))
  530. tf.summary.scalar('HardExampleMiner/NumPositives', avg_num_positives)
  531. tf.summary.scalar('HardExampleMiner/NumNegatives', avg_num_negatives)
  532. def _subsample_selection_to_desired_neg_pos_ratio(self,
  533. indices,
  534. match,
  535. max_negatives_per_positive,
  536. min_negatives_per_image=0):
  537. """Subsample a collection of selected indices to a desired neg:pos ratio.
  538. This function takes a subset of M indices (indexing into a large anchor
  539. collection of N anchors where M<N) which are labeled as positive/negative
  540. via a Match object (matched indices are positive, unmatched indices
  541. are negative). It returns a subset of the provided indices retaining all
  542. positives as well as up to the first K negatives, where:
  543. K=floor(num_negative_per_positive * num_positives).
  544. For example, if indices=[2, 4, 5, 7, 9, 10] (indexing into 12 anchors),
  545. with positives=[2, 5] and negatives=[4, 7, 9, 10] and
  546. num_negatives_per_positive=1, then the returned subset of indices
  547. is [2, 4, 5, 7].
  548. Args:
  549. indices: An integer tensor of shape [M] representing a collection
  550. of selected anchor indices
  551. match: A matcher.Match object encoding the match between anchors and
  552. groundtruth boxes for a given image, with rows of the Match objects
  553. corresponding to groundtruth boxes and columns corresponding to anchors.
  554. max_negatives_per_positive: (float) maximum number of negatives for
  555. each positive anchor.
  556. min_negatives_per_image: minimum number of negative anchors for a given
  557. image. Allow sampling negatives in image without any positive anchors.
  558. Returns:
  559. selected_indices: An integer tensor of shape [M'] representing a
  560. collection of selected anchor indices with M' <= M.
  561. num_positives: An integer tensor representing the number of positive
  562. examples in selected set of indices.
  563. num_negatives: An integer tensor representing the number of negative
  564. examples in selected set of indices.
  565. """
  566. positives_indicator = tf.gather(match.matched_column_indicator(), indices)
  567. negatives_indicator = tf.gather(match.unmatched_column_indicator(), indices)
  568. num_positives = tf.reduce_sum(tf.to_int32(positives_indicator))
  569. max_negatives = tf.maximum(min_negatives_per_image,
  570. tf.to_int32(max_negatives_per_positive *
  571. tf.to_float(num_positives)))
  572. topk_negatives_indicator = tf.less_equal(
  573. tf.cumsum(tf.to_int32(negatives_indicator)), max_negatives)
  574. subsampled_selection_indices = tf.where(
  575. tf.logical_or(positives_indicator, topk_negatives_indicator))
  576. num_negatives = tf.size(subsampled_selection_indices) - num_positives
  577. return (tf.reshape(tf.gather(indices, subsampled_selection_indices), [-1]),
  578. num_positives, num_negatives)