# -*- coding: utf-8 -*-
"""Developer convenience functions for ibs (detections).
TODO: need to split up into sub modules:
consistency_checks
feasibility_fixes
move the export stuff to dbio
then there are also convineience functions that need to be ordered at least
within this file
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from six.moves import zip, range
from os.path import expanduser, join, abspath
import numpy as np
import vtool as vt
import utool as ut
import cv2
from wbia.control import controller_inject
import tqdm
# Inject utool functions
(print, rrr, profile) = ut.inject2(__name__, '[other.detectfuncs]')
SAMPLES = 1000
AP_SAMPLE_POINTS = [_ / float(SAMPLES) for _ in range(0, SAMPLES + 1)]
# Must import class before injection
CLASS_INJECT_KEY, register_ibs_method = controller_inject.make_ibs_register_decorator(
__name__
)
def _resize(image, t_width=None, t_height=None, verbose=False):
if verbose:
print('RESIZING WITH t_width = %r and t_height = %r' % (t_width, t_height,))
height, width = image.shape[:2]
if t_width is None and t_height is None:
return image
elif t_width is not None and t_height is not None:
pass
elif t_width is None:
t_width = (width / height) * float(t_height)
elif t_height is None:
t_height = (height / width) * float(t_width)
t_width, t_height = float(t_width), float(t_height)
t_width, t_height = int(np.around(t_width)), int(np.around(t_height))
assert t_width > 0 and t_height > 0, 'target size too small'
assert (
t_width <= width * 10 and t_height <= height * 10
), 'target size too large (capped at 1000%)'
# interpolation = cv2.INTER_LANCZOS4
interpolation = cv2.INTER_LINEAR
return cv2.resize(image, (t_width, t_height), interpolation=interpolation)
[docs]def simple_code(label):
from wbia.constants import YAWALIAS, SPECIES_MAPPING
if label == 'ignore':
return 'IGNORE'
for key in SPECIES_MAPPING:
if key in label:
species_code, species_nice = SPECIES_MAPPING[key]
while species_code is None:
species_code, species_nice = SPECIES_MAPPING[species_nice]
assert species_code is not None
label = label.replace(key, species_code)
for key in sorted(YAWALIAS.keys(), key=len, reverse=True):
value = YAWALIAS[key]
label = label.replace(key, value)
return label
##########################################################################################
[docs]def general_precision_recall_algo(
ibs, label_list, confidence_list, category='positive', samples=SAMPLES, **kwargs
):
def errors(zipped, conf, category):
tp, tn, fp, fn = 0.0, 0.0, 0.0, 0.0
for index, (label, confidence) in enumerate(zipped):
if label == category:
if conf <= confidence:
tp += 1
else:
fn += 1
else:
if conf <= confidence:
fp += 1
else:
tn += 1
return tp, tn, fp, fn
zipped = list(zip(label_list, confidence_list))
conf_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
conf_dict = {}
for conf in conf_list:
conf_dict[conf] = errors(zipped, conf, category)
conf_list_ = [-1.0, -1.0]
pr_list = [1.0, 0.0]
re_list = [0.0, 1.0]
tpr_list = [0.0, 1.0]
fpr_list = [0.0, 1.0]
# conf_list_ = []
# pr_list = []
# re_list = []
# tpr_list = []
# fpr_list = []
for conf in sorted(conf_dict.keys(), reverse=True):
error_list = conf_dict[conf]
tp, tn, fp, fn = error_list
try:
pr = tp / (tp + fp)
re = tp / (tp + fn)
tpr = tp / (tp + fn)
fpr = fp / (fp + tn)
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
tpr_list.append(tpr)
fpr_list.append(fpr)
except ZeroDivisionError:
print(
'Zero division error (%r) - tp: %r tn: %r fp: %r fn: %r'
% (conf, tp, tn, fp, fn,)
)
return conf_list_, pr_list, re_list, tpr_list, fpr_list
[docs]def general_interpolate_precision_recall(conf_list, re_list, pr_list):
conf_list_, re_list_, pr_list_ = [], [], []
zipped = zip(re_list, conf_list, pr_list)
zipped = sorted(zipped, reverse=True)
max_pr = None
for re, conf, pr in zipped:
if max_pr is None or pr > max_pr:
if max_pr is not None:
conf_list_.append(np.nan)
re_list_.append(re)
pr_list_.append(max_pr)
max_pr = pr
if pr < max_pr:
pr = max_pr
conf_list_.append(conf)
re_list_.append(re)
pr_list_.append(pr)
return conf_list_, re_list_, pr_list_
[docs]def general_identify_operating_point(conf_list, x_list, y_list, target=(1.0, 1.0)):
best_length = np.inf
best_conf_list = []
best_x_list = []
best_y_list = []
tx, ty = target
for conf, x, y in sorted(zip(conf_list, x_list, y_list)):
x_ = x
y_ = y
x_ = x_ - tx
y_ = y_ - ty
length = np.sqrt(x_ * x_ + y_ * y_)
if length < best_length:
best_length = length
best_conf_list = [conf]
best_x_list = [x]
best_y_list = [y]
elif length == best_length:
flag_list = [abs(best_conf - conf) > 0.01 for best_conf in best_conf_list]
if False in flag_list:
continue
best_conf_list.append(conf)
best_x_list.append(x)
best_y_list.append(y)
return best_conf_list, best_x_list, best_y_list, best_length
[docs]def general_area_best_conf(
conf_list,
x_list,
y_list,
label='Unknown',
color='b',
marker='o',
plot_point=True,
interpolate=True,
target=(1.0, 1.0),
target_recall=None,
**kwargs,
):
import matplotlib.pyplot as plt
zipped = list(sorted(zip(x_list, y_list, conf_list)))
x_list = [_[0] for _ in zipped]
y_list = [_[1] for _ in zipped]
conf_list = [_[2] for _ in zipped]
if interpolate:
conf_list, x_list, y_list = general_interpolate_precision_recall(
conf_list, x_list, y_list
)
if interpolate:
ap_list = []
for AP_POINT in AP_SAMPLE_POINTS:
for re, pr in sorted(zip(x_list, y_list)):
if AP_POINT <= re:
ap_list.append(pr)
break
ap = sum(ap_list) / len(ap_list)
else:
ap = np.trapz(y_list, x=x_list)
tup1 = general_identify_operating_point(conf_list, x_list, y_list, target=target)
best_conf_list, best_x_list, best_y_list, best_length = tup1
tup2 = None
if target_recall is not None:
for x, y, conf in sorted(zip(x_list, y_list, conf_list)):
if target_recall <= x and not np.isnan(conf):
tup2 = [conf], [x], [y], None
break
if len(best_conf_list) > 1:
print('WARNING: Multiple best operating points found %r' % (best_conf_list,))
assert len(best_conf_list) > 0
best_conf = best_conf_list[0]
if interpolate:
# label = '%s [AP = %0.02f, OP = %0.02f]' % (label, ap * 100.0, best_conf)
label = '%s [AP = %0.02f]' % (label, ap * 100.0)
else:
label = '%s [AUC = %0.02f]' % (label, ap * 100.0,)
linestyle = '--' if kwargs.get('line_dotted', False) else '-'
plt.plot(x_list, y_list, color=color, linestyle=linestyle, label=label)
if plot_point:
plt.plot(best_x_list, best_y_list, color=color, marker=marker)
return ap, best_conf, tup1, tup2
[docs]def general_confusion_matrix_algo(
label_correct_list,
label_predict_list,
category_list,
category_mapping,
fig_,
axes_,
fuzzy_dict=None,
conf=None,
conf_list=None,
size=10,
**kwargs,
):
# import matplotlib.colors as colors
import matplotlib.pyplot as plt
suppressed_label = 'SUP'
if conf is not None:
assert conf_list is not None
category_list.append(suppressed_label)
index = len(category_list) - 1
category_mapping[suppressed_label] = index
if fuzzy_dict is not None:
fuzzy_dict[index] = set([])
if category_mapping is not None:
index_list = [category_mapping[category] for category in category_list]
zipped = list(sorted(zip(index_list, category_list)))
category_list = [_[1] for _ in zipped]
# Get the number of categories
num_categories = len(category_list)
# Build the confusion matrix
confusion_matrix = np.zeros((num_categories, num_categories))
zipped = zip(label_correct_list, label_predict_list)
suppressed = 0.0
suppressed_correct = 0.0
suppressed_fuzzy = 0.0
for index, (label_correct, label_predict) in enumerate(zipped):
if conf is not None:
conf_ = conf_list[index]
if conf_ < conf:
if label_correct != label_predict:
suppressed_correct += 1
if fuzzy_dict is not None:
x = category_mapping[label_correct]
y = category_mapping[label_predict]
if not (y in fuzzy_dict[x] or x in fuzzy_dict[y]):
suppressed_fuzzy += 1
label_predict = suppressed_label
suppressed += 1
# Perform any mapping that needs to be done
correct_ = category_mapping[label_correct]
predict_ = category_mapping[label_predict]
# Add to the confidence matrix
confusion_matrix[correct_][predict_] += 1
# Normalize the confusion matrix using the rows
row_normalizer = np.sum(confusion_matrix, axis=1)
confusion_normalized = np.array((confusion_matrix.T / row_normalizer).T)
# Draw the confusion matrix
res = axes_.imshow(confusion_normalized, cmap=plt.cm.jet, interpolation='nearest')
correct = suppressed_correct
fuzzy = suppressed_fuzzy
total = 0.0
for x in range(num_categories):
for y in range(num_categories):
number = int(confusion_matrix[x][y])
if x == y:
correct += number
if fuzzy_dict is not None and (y in fuzzy_dict[x] or x in fuzzy_dict[y]):
fuzzy += number
total += number
axes_.annotate(
str(number),
xy=(y, x),
horizontalalignment='center',
verticalalignment='center',
size=size,
)
cb = fig_.colorbar(res) # NOQA
cb.set_clim(0.0, 1.0)
plt.xticks(np.arange(num_categories), category_list, rotation=90)
plt.yticks(np.arange(num_categories), category_list)
margin_small = 0.1
margin_large = 0.9
plt.subplots_adjust(
left=margin_small, right=margin_large, bottom=margin_small, top=margin_large
)
correct_rate = correct / total
fuzzy_rate = fuzzy / total
return correct_rate, fuzzy_rate
[docs]def general_intersection_over_union(bbox1, bbox2):
intersection_xtl = max(bbox1['xtl'], bbox2['xtl'])
intersection_ytl = max(bbox1['ytl'], bbox2['ytl'])
intersection_xbr = min(bbox1['xbr'], bbox2['xbr'])
intersection_ybr = min(bbox1['ybr'], bbox2['ybr'])
intersection_w = intersection_xbr - intersection_xtl
intersection_h = intersection_ybr - intersection_ytl
if intersection_w <= 0 or intersection_h <= 0:
return 0.0
intersection = intersection_w * intersection_h
union = (
(bbox1['width'] * bbox1['height'])
+ (bbox2['width'] * bbox2['height'])
- intersection
)
return intersection / union
[docs]def general_overlap(gt_list, pred_list):
overlap = np.zeros((len(gt_list), len(pred_list)), dtype=np.float32)
for i, gt in enumerate(gt_list):
for j, pred in enumerate(pred_list):
overlap[i, j] = general_intersection_over_union(gt, pred)
return overlap
[docs]def general_tp_fp_fn(gt_list, pred_list, min_overlap, **kwargs):
overlap = general_overlap(gt_list, pred_list)
num_gt, num_pred = overlap.shape
if num_gt == 0:
tp = 0.0
fp = num_pred
fn = 0.0
elif num_pred == 0:
tp = 0.0
fp = 0.0
fn = num_gt
else:
pred_index_list = range(num_pred)
gt_index_list = np.argmax(overlap, axis=0)
max_overlap_list = np.max(overlap, axis=0)
confidence_list = [pred.get('confidence', None) for pred in pred_list]
assert None not in confidence_list
zipped = zip(confidence_list, max_overlap_list, pred_index_list, gt_index_list)
pred_conf_list = [
(confidence, max_overlap, pred_index, gt_index,)
for confidence, max_overlap, pred_index, gt_index in zipped
]
pred_conf_list = sorted(pred_conf_list, reverse=True)
assignment_dict = {}
for pred_conf, max_overlap, pred_index, gt_index in pred_conf_list:
if max_overlap > min_overlap:
if gt_index not in assignment_dict:
assignment_dict[gt_index] = pred_index
tp = len(assignment_dict.keys())
fp = num_pred - tp
fn = num_gt - tp
assert tp >= 0
assert fp >= 0
assert fn >= 0
return tp, fp, fn
[docs]def general_get_imageset_gids(ibs, imageset_text, unique=True, **kwargs):
imageset_id = ibs.get_imageset_imgsetids_from_text(imageset_text)
test_gid_list = ibs.get_imageset_gids(imageset_id)
if unique:
test_gid_list = list(set(test_gid_list))
return test_gid_list
[docs]def general_parse_gt_annots(
ibs, aid_list, include_parts=True, species_mapping={}, **kwargs
):
gid_list = ibs.get_annot_gids(aid_list)
species_set = set([])
gt_list = []
for gid, aid in zip(gid_list, aid_list):
width, height = ibs.get_image_sizes(gid)
bbox = ibs.get_annot_bboxes(aid)
theta = ibs.get_annot_thetas(aid)
# Transformation matrix
R = vt.rotation_around_bbox_mat3x3(theta, bbox)
# Get verticies of the annotation polygon
verts = vt.verts_from_bbox(bbox, close=True)
# Rotate and transform vertices
xyz_pts = vt.add_homogenous_coordinate(np.array(verts).T)
trans_pts = vt.remove_homogenous_coordinate(R.dot(xyz_pts))
new_verts = np.round(trans_pts).astype(np.int).T.tolist()
x_points = [pt[0] for pt in new_verts]
y_points = [pt[1] for pt in new_verts]
xtl = int(min(x_points))
xbr = int(max(x_points))
ytl = int(min(y_points))
ybr = int(max(y_points))
bbox = (xtl, ytl, xbr - xtl, ybr - ytl)
species = ibs.get_annot_species_texts(aid)
viewpoint = ibs.get_annot_viewpoints(aid)
interest = ibs.get_annot_interest(aid)
temp = {
'gid': gid,
'aid': aid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'class': species_mapping.get(species, species),
'viewpoint': viewpoint,
'interest': interest,
'confidence': 1.0,
}
species_set.add(temp['class'])
gt_list.append(temp)
part_rowid_list = ibs.get_annot_part_rowids(aid)
if include_parts:
for part_rowid in part_rowid_list:
bbox = ibs.get_part_bboxes(part_rowid)
theta = ibs.get_part_thetas(part_rowid)
# Transformation matrix
R = vt.rotation_around_bbox_mat3x3(theta, bbox)
# Get verticies of the annotation polygon
verts = vt.verts_from_bbox(bbox, close=True)
# Rotate and transform vertices
xyz_pts = vt.add_homogenous_coordinate(np.array(verts).T)
trans_pts = vt.remove_homogenous_coordinate(R.dot(xyz_pts))
new_verts = np.round(trans_pts).astype(np.int).T.tolist()
x_points = [pt[0] for pt in new_verts]
y_points = [pt[1] for pt in new_verts]
xtl = int(min(x_points))
xbr = int(max(x_points))
ytl = int(min(y_points))
ybr = int(max(y_points))
bbox = (xtl, ytl, xbr - xtl, ybr - ytl)
tag = ibs.get_part_tag_text(part_rowid)
if tag is None:
tag = species
else:
tag = '%s+%s' % (species, tag,)
temp = {
'gid': gid,
'aid': aid,
'part_id': part_rowid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'class': tag,
'viewpoint': viewpoint,
'interest': interest,
'confidence': 1.0,
}
species_set.add(temp['class'])
gt_list.append(temp)
return gt_list, species_set
[docs]def general_parse_gt(ibs, test_gid_list=None, **kwargs):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
uuid_list = ibs.get_image_uuids(test_gid_list)
gid_list = ibs.get_image_gids_from_uuid(uuid_list)
species_set = set([])
gt_dict = {}
for gid, uuid in zip(gid_list, uuid_list):
aid_list = ibs.get_image_aids(gid)
gt_list, species_set = general_parse_gt_annots(ibs, aid_list, **kwargs)
species_set = species_set | species_set
gt_dict[uuid] = gt_list
# print('General Parse GT species_set = %r' % (species_set, ))
return gt_dict
##########################################################################################
[docs]def localizer_parse_pred(ibs, test_gid_list=None, species_mapping={}, **kwargs):
depc = ibs.depc_image
if 'feature2_algo' not in kwargs:
kwargs['feature2_algo'] = 'resnet'
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
uuid_list = ibs.get_image_uuids(test_gid_list)
size_list = ibs.get_image_sizes(test_gid_list)
# Unsure, but we need to call this multiple times? Lazy loading bug?
bboxes_list = depc.get_property(
'localizations', test_gid_list, 'bboxes', config=kwargs
)
# Get actual data
bboxes_list = depc.get_property(
'localizations', test_gid_list, 'bboxes', config=kwargs
)
thetas_list = depc.get_property(
'localizations', test_gid_list, 'thetas', config=kwargs
)
confss_list = depc.get_property(
'localizations', test_gid_list, 'confs', config=kwargs
)
classs_list = depc.get_property(
'localizations', test_gid_list, 'classes', config=kwargs
)
length_list = [len(bbox_list) for bbox_list in bboxes_list]
# Establish primitives
test_gids_list = [
[test_gid] * length for test_gid, length in zip(test_gid_list, length_list)
]
sizes_list = [[size] * length for size, length in zip(size_list, length_list)]
keeps_list = [[True] * length for length in length_list]
features_list = [[None] * length for length in length_list]
features_lazy_list = [[None] * length for length in length_list]
viewpoints_list = [[None] * length for length in length_list]
interests_list = [[None] * length for length in length_list]
# Get features
if kwargs.get('features', False):
features_list = depc.get_property(
'localizations_features', test_gid_list, 'vector', config=kwargs
)
if kwargs.get('features_lazy', False):
from functools import partial
def features_lazy_func(gid, offset):
vector_list = depc.get_property(
'localizations_features', gid, 'vector', config=kwargs
)
vector = vector_list[offset]
return vector
features_lazy_list = [
[
partial(features_lazy_func, test_gid, test_offset)
for test_offset in range(length)
]
for test_gid, length in zip(test_gid_list, length_list)
]
# Get species and viewpoints labels
if kwargs.get('labels', False):
classs_list = depc.get_property(
'localizations_labeler', test_gid_list, 'species', config=kwargs
)
viewpoints_list = depc.get_property(
'localizations_labeler', test_gid_list, 'viewpoint', config=kwargs
)
# Get updated confidences for boxes
if kwargs.get('classify', False):
print('Using alternate classifications')
# depc.delete_property('localizations_classifier', test_gid_list, config=kwargs)
confss_list = depc.get_property(
'localizations_classifier', test_gid_list, 'score', config=kwargs
)
# Get updated confidences for boxes
if kwargs.get('interest', False):
print('Using alternate AoI interest flags')
interests_list = depc.get_property(
'localizations_classifier', test_gid_list, 'score', config=kwargs
)
# Reformat results for json
zipped_list_list = zip(
keeps_list,
test_gids_list,
sizes_list,
bboxes_list,
thetas_list,
confss_list,
classs_list,
viewpoints_list,
interests_list,
features_list,
features_lazy_list,
)
results_list = [
[
{
'gid': test_gid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'theta': theta,
'confidence': conf,
'class': species_mapping.get(class_, class_),
'viewpoint': viewpoint,
'interest': None if interest is None else interest >= 0.84,
'feature': feature,
'feature_lazy': feature_lazy,
}
for keep_, test_gid, (
width,
height,
), bbox, theta, conf, class_, viewpoint, interest, feature, feature_lazy in zip(
*zipped_list
)
if keep_
]
for zipped_list in zipped_list_list
]
pred_dict = {
uuid_: result_list for uuid_, result_list in zip(uuid_list, results_list)
}
return pred_dict
[docs]def localizer_precision_recall_algo(ibs, samples=SAMPLES, test_gid_list=None, **kwargs):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
test_uuid_list = ibs.get_image_uuids(test_gid_list)
print('\tGather Ground-Truth')
gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
print('\tGather Predictions')
pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
species_set = kwargs.get('species_set', None)
if species_set is not None:
# filter out any prefix ! to denote interest only
species_set_ = set([species.lstrip('!') for species in species_set])
dict_list = [
(gt_dict, 'Ground-Truth'),
(pred_dict, 'Predictions'),
]
for dict_, dict_tag in dict_list:
for image_uuid in dict_:
dict_[image_uuid] = [
val
for val in dict_[image_uuid]
if val.get('class', None) in species_set_
]
values = localizer_tp_fp(test_uuid_list, gt_dict, pred_dict, **kwargs)
conf_list, tp_list, fp_list, total = values
conf_list_ = [-1.0, -1.0]
pr_list = [1.0, 0.0]
re_list = [0.0, 1.0]
for conf, tp, fp in zip(conf_list, tp_list, fp_list):
try:
pr = tp / (tp + fp)
re = tp / total
except ZeroDivisionError:
continue
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
return conf_list_, pr_list, re_list
[docs]def localizer_assign(gt_list, pred, min_overlap):
best_overlap = min_overlap
best_index = None
for index, gt in enumerate(gt_list):
if gt['class'] != pred['class']:
continue
overlap = general_intersection_over_union(gt, pred)
if overlap < best_overlap:
continue
best_overlap = overlap
best_index = index
if best_index is None:
best_overlap = None
return best_index, best_overlap
[docs]def localizer_assignments(pred_list, gt_list, gt_list_=[], min_overlap=0.5):
pred_list = sorted(pred_list, key=lambda pred: pred['confidence'], reverse=True)
match_list = []
for pred in pred_list:
flag = False
match_index, best_overlap = localizer_assign(gt_list, pred, min_overlap)
match_index_, best_overlap_ = localizer_assign(gt_list_, pred, min_overlap)
if match_index is not None:
flag = True
del gt_list[match_index]
elif match_index_ is not None:
flag = None
if flag is not None:
match_list += [(pred['confidence'], flag, match_index, best_overlap)]
return match_list
[docs]def localizer_tp_fp(uuid_list, gt_dict, pred_dict, min_overlap=0.5, **kwargs):
total = 0.0
interest_species_set = set([])
species_set = kwargs.get('species_set', None)
if species_set is not None:
for species in species_set:
if species.startswith('!'):
species = species.lstrip('!')
interest_species_set.add(species)
match_list = []
for image_uuid in uuid_list:
gt_list = []
gt_list_ = []
pred_list = pred_dict[image_uuid]
for gt in gt_dict[image_uuid]:
species = gt['class']
interest = gt['interest']
if species in interest_species_set and not interest:
gt_list_.append(gt)
else:
gt_list.append(gt)
total += len(gt_list)
# Match predictions
match_list_ = localizer_assignments(pred_list, gt_list, gt_list_, min_overlap)
for match_ in match_list_:
match_list.append(match_)
# sort matches by confidence from high to low
match_list = sorted(match_list, key=lambda match: match[0], reverse=True)
conf_list = []
tp_list = []
fp_list = []
tp_counter = 0
fp_counter = 0
for conf, flag, index, overlap in match_list:
if flag:
tp_counter += 1
else:
fp_counter += 1
conf_list.append(conf)
tp_list.append(tp_counter)
fp_list.append(fp_counter)
# print('\t tps [:10] : %r' % (tp_list[:10], ))
# print('\t fps [:10] : %r' % (fp_list[:10], ))
# print('\t con [:10] : %r' % (conf_list[:10], ))
# print('\t tps [-10:] : %r' % (tp_list[-10:], ))
# print('\t fps [-10:] : %r' % (fp_list[-10:], ))
# print('\t con [-10:] : %r' % (conf_list[-10:], ))
# print('\t num_annotations: %r' % (total, ))
return conf_list, tp_list, fp_list, total
[docs]def localizer_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list = localizer_precision_recall_algo(ibs, **kwargs)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
def _ignore_filter_identity_func(*args, **kwargs):
return False
[docs]def localizer_iou_recall_algo(
ibs, samples=100, test_gid_list=None, ignore_filter_func=None, **kwargs
):
assert 'min_overlap' not in kwargs
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
test_uuid_list = ibs.get_image_uuids(test_gid_list)
if ignore_filter_func is None:
ignore_filter_func = _ignore_filter_identity_func
print('\tGather Ground-Truth')
gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
print('\tGather Predictions')
pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
species_set = kwargs.get('species_set', None)
if species_set is not None:
# filter out any prefix ! to denote interest only
species_set_ = set([species.lstrip('!') for species in species_set])
dict_list = [
(gt_dict, 'Ground-Truth'),
(pred_dict, 'Predictions'),
]
for dict_, dict_tag in dict_list:
for image_uuid in dict_:
temp = []
for val in dict_[image_uuid]:
if val.get('class', None) not in species_set_:
continue
if ignore_filter_func(ibs, val):
continue
temp.append(val)
dict_[image_uuid] = temp
target = (1.0, 1.0)
iou_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
conf_list_ = []
iou_list_ = []
recall_list = []
for iou in tqdm.tqdm(iou_list):
values = localizer_tp_fp(
test_uuid_list, gt_dict, pred_dict, min_overlap=iou, **kwargs
)
conf_list, tp_list, fp_list, total = values
conf_list_ = []
pr_list = []
re_list = []
for conf, tp, fp in zip(conf_list, tp_list, fp_list):
try:
pr = tp / (tp + fp)
re = tp / total
except ZeroDivisionError:
continue
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
best_tup = general_identify_operating_point(
conf_list, re_list, pr_list, target=target
)
best_conf_list, best_re_list, best_pr_list, best_length = best_tup
if len(best_conf_list) > 1:
print('WARNING: Multiple best operating points found %r' % (best_conf_list,))
assert len(best_conf_list) > 0
best_re_index = np.argmax(best_re_list)
best_re = best_re_list[best_re_index]
best_conf = best_conf_list[best_re_index]
conf_list_.append(best_conf)
iou_list_.append(iou)
recall_list.append(best_re)
return conf_list_, iou_list_, recall_list
[docs]def localizer_iou_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing IoU-Recall for: %r' % (label,))
conf_list, iou_list, recall_list = localizer_iou_recall_algo(ibs, **kwargs)
return general_area_best_conf(
conf_list, iou_list, recall_list, interpolate=False, **kwargs
)
# def localizer_iou_precision_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# print('Processing Precision-Recall for: %r' % (label, ))
# conf_list, iou_list, pr_list, re_list = localizer_iou_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, iou_list, re_list, **kwargs)
[docs]def localizer_confusion_matrix_algo_plot(
ibs, label=None, target_conf=None, test_gid_list=None, **kwargs
):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
test_uuid_list = ibs.get_image_uuids(test_gid_list)
print('\tGather Ground-Truth')
gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
print('\tGather Predictions')
pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
species_set = kwargs.get('species_set', None)
if species_set is not None:
# filter out any prefix ! to denote interest only
species_set_ = set([species.lstrip('!') for species in species_set])
dict_list = [
(gt_dict, 'Ground-Truth'),
(pred_dict, 'Predictions'),
]
for dict_, dict_tag in dict_list:
for image_uuid in dict_:
dict_[image_uuid] = [
val
for val in dict_[image_uuid]
if val.get('class', None) in species_set_
]
values = localizer_tp_fp(test_uuid_list, gt_dict, pred_dict, **kwargs)
conf_list, tp_list, fp_list, total = values
best_conf = None
best_accuracy = None
best_args = None
for conf, tp, fp in sorted(zip(conf_list, tp_list, fp_list)):
fn = total - tp
accuracy = tp / (tp + fp + fn)
if target_conf is None:
if best_accuracy is None or accuracy > best_accuracy:
best_conf = conf
best_accuracy = accuracy
best_args = (tp, fp, fn)
else:
if target_conf <= conf:
best_conf = conf
best_accuracy = accuracy
best_args = (tp, fp, fn)
break
try:
assert None not in [best_conf, best_accuracy, best_args]
except AssertionError:
ut.embed()
return np.nan, (np.nan, None)
print(
'Processing Confusion Matrix for: %r (Conf = %0.02f, Accuracy = %0.02f)'
% (label, best_conf, best_accuracy,)
)
tp, fp, fn = best_args
label_list = []
prediction_list = []
for _ in range(int(tp)):
label_list.append('positive')
prediction_list.append('positive')
for _ in range(int(fp)):
label_list.append('negative')
prediction_list.append('positive')
for _ in range(int(fn)):
label_list.append('positive')
prediction_list.append('negative')
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
values = general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, size=20, **kwargs
)
return best_conf, values
[docs]@register_ibs_method
def localizer_precision_recall(
ibs, config_dict=None, output_path=None, test_gid_list=None, **kwargs
):
if config_dict is None:
if test_gid_list is not None:
print('Using %d test gids' % (len(test_gid_list),))
# species_mapping = { # NOQA
# 'giraffe_masai' : 'giraffe',
# 'giraffe_reticulated' : 'giraffe',
# 'zebra_grevys' : 'zebra',
# 'zebra_plains' : 'zebra',
# }
config_dict = {
# 'seaturtle': (
# [
# {'label': 'Sea Turtle', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green', 'turtle_hawksbill'])},
# {'label': 'Sea Turtle Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green+head', 'turtle_hawksbill+head'])},
# {'label': 'Green', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green'])},
# {'label': 'Green Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_green+head'])},
# {'label': 'Hawksbill', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['turtle_hawksbill+head'])},
# ],
# {'BEST_INDEX': 0},
# ),
# '!seaturtle': (
# [
# {'label': '! Sea Turtle', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green', '!turtle_hawksbill'])},
# {'label': '! Sea Turtle Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green+head', '!turtle_hawksbill+head'])},
# {'label': '! Green', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green'])},
# {'label': '! Green Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_green+head'])},
# {'label': '! Hawksbill', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_hawksbill'])},
# {'label': '! Hawksbill Heads', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.2, 'species_set' : set(['!turtle_hawksbill+head'])},
# ],
# {'BEST_INDEX': 0},
# ),
# 'hawksbills': (
# [
# {'label': 'Hawksbill NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['turtle_hawksbill'])},
# {'label': 'Hawksbill NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['turtle_hawksbill'])},
# ],
# {},
# ),
# 'hawsbills+heads': (
# [
# {'label': 'Hawksbill Head NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['turtle_hawksbill+head'])},
# {'label': 'Hawksbill Head NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'seaturtle', 'weight_filepath' : 'seaturtle', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['turtle_hawksbill+head'])},
# ],
# {},
# ),
# 'hammerhead': (
# [
# {'label': 'Hammerhead NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['shark_hammerhead'])},
# ],
# {},
# ),
# '!hammerhead': (
# [
# {'label': 'Hammerhead NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!shark_hammerhead'])},
# {'label': 'Hammerhead ! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'hammerhead', 'weight_filepath' : 'hammerhead', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!shark_hammerhead'])},
# ],
# {'offset_color': 1},
# ),
# 'ggr2-giraffe-lightnet': (
# [
# {'label': 'Giraffe NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# ],
# {},
# ),
# 'ggr2-zebra-lightnet': (
# [
# {'label': 'Zebra NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# ],
# {},
# ),
# 'ggr2-!giraffe-lightnet': (
# [
# {'label': 'Giraffe ! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# ],
# {},
# ),
# 'ggr2-!zebra-lightnet': (
# [
# {'label': 'Zebra ! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'ggr2', 'weight_filepath' : 'ggr2', 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# ],
# {},
# ),
# 'ggr2-giraffe-azure': (
# [
# {'label': 'Giraffe NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# {'label': 'Giraffe NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['giraffe'])},
# ],
# {},
# ),
# 'ggr2-zebra-azure': (
# [
# {'label': 'Zebra NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# {'label': 'Zebra NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['zebra'])},
# ],
# {},
# ),
# 'ggr2-!giraffe-azure': (
# [
# {'label': 'Giraffe ! NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# {'label': 'Giraffe ! NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!giraffe'])},
# ],
# {},
# ),
# 'ggr2-!zebra-azure': (
# [
# {'label': 'Zebra ! NMS 0%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 10%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.10, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 20%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.20, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 30%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.30, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 40%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.40, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 50%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.50, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 60%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.60, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 70%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.70, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 80%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.80, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 90%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 0.90, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# {'label': 'Zebra ! NMS 100%', 'grid' : False, 'algo': 'azure', 'config_filepath' : None, 'weight_filepath' : None, 'nms': True, 'nms_thresh': 1.00, 'test_gid_list': test_gid_list, 'species_mapping': species_mapping, 'species_set': set(['!zebra'])},
# ],
# {},
# ),
# 'lynx': (
# [
# {'label': 'Lynx NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['lynx'])},
# {'label': 'Lynx NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'lynx', 'weight_filepath' : 'lynx', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['lynx'])},
# ],
# {},
# ),
# 'jaguar': (
# [
# {'label': 'Jaguar NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['jaguar'])},
# {'label': 'Jaguar NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['jaguar'])},
# ],
# {},
# ),
# '!jaguar': (
# [
# {'label': 'Jaguar NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!jaguar'])},
# {'label': 'Jaguar NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'jaguar_v2', 'weight_filepath' : 'jaguar_v2', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!jaguar'])},
# ],
# {},
# ),
# 'manta': (
# [
# {'label': 'Manta NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['manta_ray_giant'])},
# {'label': 'Manta NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['manta_ray_giant'])},
# ],
# {},
# ),
# '!manta': (
# [
# {'label': 'Manta NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!manta_ray_giant'])},
# {'label': 'Manta NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'manta', 'weight_filepath' : 'manta', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!manta_ray_giant'])},
# ],
# {},
# ),
# 'giraffe': (
# [
# {'label': 'Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_masai', 'giraffe_reticulated'])},
# {'label': 'Masai Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Masai Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_masai'])},
# {'label': 'Reticulated Giraffe NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['giraffe_reticulated'])},
# {'label': 'Reticulated Giraffe NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'giraffe_v1', 'weight_filepath' : 'giraffe_v1', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['giraffe_reticulated'])},
# ],
# {},
# ),
# 'spotted_skunk_v0': (
# [
# {'label': 'Spotted Skunk NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['skunk_spotted'])},
# ],
# {},
# ),
# '!spotted_skunk_v0': (
# [
# {'label': 'Spotted Skunk NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!skunk_spotted'])},
# {'label': 'Spotted Skunk NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_skunk_v0', 'weight_filepath' : 'spotted_skunk_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!skunk_spotted'])},
# ],
# {},
# ),
# 'nassau_grouper_v0': (
# [
# {'label': 'Nassau Grouper NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['grouper_nassau'])},
# {'label': 'Nassau Grouper NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['grouper_nassau'])},
# ],
# {},
# ),
# '!nassau_grouper_v0': (
# [
# {'label': 'Nassau Grouper! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!grouper_nassau'])},
# {'label': 'Nassau Grouper! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'nassau_grouper_v0', 'weight_filepath' : 'nassau_grouper_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!grouper_nassau'])},
# ],
# {},
# ),
# 'spotted_dolphin_v0': (
# [
# {'label': 'Spotted DolphinNMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['dolphin_spotted'])},
# {'label': 'Spotted DolphinNMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['dolphin_spotted'])},
# ],
# {},
# ),
# '!spotted_dolphin_v0': (
# [
# {'label': 'Spotted Dolphin! NMS 0%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.00, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 10%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.10, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 20%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.20, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 30%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.30, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 40%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.40, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 50%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.50, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 60%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.60, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 70%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.70, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 80%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.80, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 90%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 0.90, 'species_set' : set(['!dolphin_spotted'])},
# {'label': 'Spotted Dolphin! NMS 100%', 'grid' : False, 'algo': 'lightnet', 'config_filepath' : 'spotted_dolphin_v0', 'weight_filepath' : 'spotted_dolphin_v0', 'nms': True, 'nms_thresh': 1.00, 'species_set' : set(['!dolphin_spotted'])},
# ],
# {},
# ),
'seadragon_weedy_v1': (
[
{
'label': 'Weedy Body NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_leafy']),
},
{
'label': 'Weedy Body NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_leafy']),
},
],
{},
),
'seadragon_leafy_v1': (
[
{
'label': 'Leafy Body NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_weedy']),
},
{
'label': 'Leafy Body NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_weedy']),
},
],
{},
),
'seadragon_weedy_head_v1': (
[
{
'label': 'Weedy Head NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_leafy+head']),
},
{
'label': 'Weedy Head NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_leafy+head']),
},
],
{},
),
'seadragon_leafy_head_v1': (
[
{
'label': 'Leafy Head NMS 0%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.00,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 10%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.10,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 20%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.20,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 30%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.30,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 40%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.40,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 50%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.50,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 60%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.60,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 70%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.70,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 80%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.80,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 90%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 0.90,
'species_set': set(['seadragon_weedy+head']),
},
{
'label': 'Leafy Head NMS 100%',
'grid': False,
'algo': 'lightnet',
'config_filepath': 'seadragon_v1',
'weight_filepath': 'seadragon_v1',
'nms': True,
'nms_thresh': 1.00,
'species_set': set(['seadragon_weedy+head']),
},
],
{},
),
}
for config_key in config_dict:
config_list, config = config_dict[config_key]
for key in kwargs:
config[key] = kwargs[key]
# Backwards compatibility hack
if test_gid_list is not None:
for config_ in config_list:
if 'test_gid_list' not in config_:
config_['test_gid_list'] = test_gid_list
ibs.localizer_precision_recall_algo_display(
config_list, config_tag=config_key, output_path=output_path, **config
)
[docs]@register_ibs_method
def localizer_precision_recall_algo_display(
ibs,
config_list,
config_tag='',
min_overlap=0.5,
figsize=(40, 9),
target_recall=0.8,
BEST_INDEX=None,
offset_color=0,
write_images=False,
plot_point=True,
output_path=None,
plot_iou_recall=True,
**kwargs,
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
if output_path is None:
output_path = abspath(expanduser(join('~', 'Desktop')))
color_list_ = []
for _ in range(offset_color):
color_list_ += [(0.2, 0.2, 0.2)]
color_list = pt.distinct_colors(len(config_list) - len(color_list_), randomize=False)
color_list = color_list_ + color_list
fig_ = plt.figure(figsize=figsize, dpi=400)
######################################################################################
axes_ = plt.subplot(141)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall (Ground-Truth IOU >= %0.02f)' % (min_overlap,))
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
localizer_precision_recall_algo_plot(
ibs,
color=color,
min_overlap=min_overlap,
plot_point=plot_point,
target_recall=target_recall,
**config,
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
tup2_list = [ret[3] for ret in ret_list]
best_index = (
None if BEST_INDEX is None else BEST_INDEX
) # Match formatting of below, this is a silly conditional
best_y = 0.0
best_index_ = None
valid_best_index = []
for index, tup2 in enumerate(tup2_list):
if tup2 is None:
continue
conf_list, x_list, y_list, length = tup2
y = y_list[0]
if best_y < y:
valid_best_index.append(index)
best_index_ = index
best_y = y
# If user defined best_index is invalid, don't use it
if best_index is None:
best_index = best_index_
else:
if best_index not in valid_best_index:
best_index = None
if best_index is not None:
best_conf_list, best_x_list, best_y_list, best_length = tup2_list[best_index]
color = 'xkcd:gold'
marker = 'D'
plt.plot(best_x_list, best_y_list, color=color, marker=marker)
plt.title('Precision-Recall Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
######################################################################################
if plot_iou_recall:
axes_ = plt.subplot(142)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('IOU (Intersection / Union)')
axes_.set_ylabel('Recall')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
localizer_iou_recall_algo_plot(ibs, color=color_, plot_point=False, **config_)
for color_, config_ in zip(color_list, config_list)
]
# area_list = [ ret[0] for ret in ret_list ]
# tup2_list = [ ret[3] for ret in ret_list ]
# best_index = None if BEST_INDEX is None else BEST_INDEX # Match formatting of below, this is a silly conditional
# best_y = 0.0
# best_index_ = None
# valid_best_index = []
# for index, tup2 in enumerate(tup2_list):
# if tup2 is None:
# continue
# conf_list, x_list, y_list, length = tup2
# y = y_list[0]
# if best_y < y:
# valid_best_index.append(index)
# best_index_ = index
# best_y = y
# # If user defined best_index is invalid, don't use it
# if best_index is None:
# best_index = best_index_
# else:
# if best_index not in valid_best_index:
# best_index = None
# if best_index is not None:
# best_conf_list, best_x_list, best_y_list, best_length = tup2_list[best_index]
# color = 'xkcd:gold'
# marker = 'D'
# plt.plot(best_x_list, best_y_list, color=color, marker=marker)
plt.title('Recall-IOU Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
######################################################################################
# axes_ = plt.subplot(153)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('IOU (Intersection / Union)')
# axes_.set_ylabel('Precision')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# ret_list = [
# localizer_iou_precision_algo_plot(ibs, color=color_, plot_point=False, **config_)
# for color_, config_ in zip(color_list, config_list)
# ]
# plt.title('Precision-IOU Curves', y=1.19)
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
######################################################################################
if best_index is not None:
axes_ = plt.subplot(144)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
target_conf = best_conf_list[0]
best_config = config_list[best_index]
best_label = config_list[best_index]['label']
best_area = area_list[best_index]
values = localizer_confusion_matrix_algo_plot(
ibs,
min_overlap=min_overlap,
fig_=fig_,
axes_=axes_,
target_conf=target_conf,
**best_config,
)
best_conf, (correct_rate, _) = values
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
args = (
target_recall,
best_label,
best_area,
best_conf,
)
plt.title(
'Confusion Matrix for Recall >= %0.02f\n(Algo: %s, mAP = %0.02f, OP = %0.02f)'
% args,
y=1.26,
)
######################################################################################
axes_ = plt.subplot(143)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
best_index = np.argmax(area_list) if BEST_INDEX is None else BEST_INDEX
best_config = config_list[best_index]
best_label = config_list[best_index]['label']
best_area = area_list[best_index]
values = localizer_confusion_matrix_algo_plot(
ibs, min_overlap=min_overlap, fig_=fig_, axes_=axes_, **best_config
)
best_conf, (correct_rate, _) = values
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
args = (
best_label,
best_area,
best_conf,
)
plt.title('Confusion Matrix\n(Algo: %s, mAP = %0.02f, OP = %0.02f)' % args, y=1.26)
######################################################################################
if len(config_tag) > 0:
config_tag = '%s-' % (config_tag,)
fig_filename = '%slocalizer-precision-recall-%0.2f.png' % (config_tag, min_overlap,)
fig_path = join(output_path, fig_filename)
plt.savefig(fig_path, bbox_inches='tight')
return fig_path
[docs]@register_ibs_method
def localizer_precision_recall_algo_display_animate(ibs, config_list, **kwargs):
for value in range(10):
min_overlap = value / 10.0
print('Processing: %r' % (min_overlap,))
ibs.localizer_precision_recall_algo_display(
config_list, min_overlap=min_overlap, **kwargs
)
# def localizer_classification_tp_tn_fp_fn(gt_list, pred_list, conf, min_overlap,
# check_species=False,
# check_viewpoint=False, **kwargs):
# overlap = general_overlap(gt_list, pred_list)
# num_gt, num_pred = overlap.shape
# # Get confidences
# conf_list = [pred['confidence'] for pred in pred_list]
# pred_flag_list = [conf <= conf_ for conf_ in conf_list]
# if num_gt == 0:
# tp_list = [False] * len(pred_list)
# tn_list = [not pred_flag for pred_flag in pred_flag_list]
# fp_list = [ pred_flag for pred_flag in pred_flag_list]
# fn_list = [False] * len(pred_list)
# elif num_pred == 0:
# tp_list = []
# tn_list = []
# fp_list = []
# fn_list = []
# else:
# max_overlap = np.max(overlap, axis=0)
# gt_flag_list = min_overlap < max_overlap
# status_list = []
# for gt_flag, pred_flag in zip(gt_flag_list, pred_flag_list):
# if gt_flag and pred_flag:
# status_list.append('tp')
# elif gt_flag and not pred_flag:
# status_list.append('fn')
# elif not gt_flag and pred_flag:
# status_list.append('fp')
# elif not gt_flag and not pred_flag:
# status_list.append('tn')
# else:
# raise ValueError
# tp_list = [status == 'tp' for status in status_list]
# tn_list = [status == 'tn' for status in status_list]
# fp_list = [status == 'fp' for status in status_list]
# fn_list = [status == 'fn' for status in status_list]
# return tp_list, tn_list, fp_list, fn_list
# def localizer_classification_confusion_matrix_algo_plot(ibs, color, conf,
# label=None,
# min_overlap=0.25,
# write_images=False,
# **kwargs):
# print('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# test_uuid_list = ibs.get_image_uuids(test_gid_list)
# print('\tGather Ground-Truth')
# gt_dict = general_parse_gt(ibs, test_gid_list=test_gid_list, **kwargs)
# print('\tGather Predictions')
# pred_dict = localizer_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
# if write_images:
# output_folder = 'localizer-classification-confusion-matrix-%0.2f-%0.2f-images' % (min_overlap, conf, )
# output_path = abspath(expanduser(join('~', 'Desktop', output_folder)))
# ut.ensuredir(output_path)
# label_list = []
# prediction_list = []
# for index, (test_gid, test_uuid) in enumerate(zip(test_gid_list, test_uuid_list)):
# if test_uuid in pred_dict:
# gt_list = gt_dict[test_uuid]
# pred_list = pred_dict[test_uuid]
# values = localizer_classification_tp_tn_fp_fn(gt_list, pred_list, conf,
# min_overlap=min_overlap,
# **kwargs)
# tp_list, tn_list, fp_list, fn_list = values
# tp = tp_list.count(True)
# tn = tn_list.count(True)
# fp = fp_list.count(True)
# fn = fn_list.count(True)
# for _ in range(int(tp)):
# label_list.append('positive')
# prediction_list.append('positive')
# for _ in range(int(tn)):
# label_list.append('negative')
# prediction_list.append('negative')
# for _ in range(int(fp)):
# label_list.append('negative')
# prediction_list.append('positive')
# for _ in range(int(fn)):
# label_list.append('positive')
# prediction_list.append('negative')
# if write_images:
# test_image = ibs.get_images(test_gid)
# test_image = _resize(test_image, t_width=600, verbose=False)
# height_, width_, channels_ = test_image.shape
# for gt in gt_list:
# xtl = int(gt['xtl'] * width_)
# ytl = int(gt['ytl'] * height_)
# xbr = int(gt['xbr'] * width_)
# ybr = int(gt['ybr'] * height_)
# cv2.rectangle(test_image, (xtl, ytl), (xbr, ybr), (0, 0, 255))
# zipped = zip(pred_list, tp_list, tn_list, fp_list, fn_list)
# for pred, tp_, tn_, fp_, fn_ in zipped:
# if tp_:
# color = (0, 255, 0)
# elif fp_:
# continue
# # color = (255, 0, 0)
# elif fn_:
# color = (255, 0, 0)
# elif tn_:
# continue
# else:
# continue
# xtl = int(pred['xtl'] * width_)
# ytl = int(pred['ytl'] * height_)
# xbr = int(pred['xbr'] * width_)
# ybr = int(pred['ybr'] * height_)
# cv2.rectangle(test_image, (xtl, ytl), (xbr, ybr), color)
# status_str = 'success' if (fp + fn) == 0 else 'failure'
# status_val = tp - fp - fn
# args = (status_str, status_val, test_gid, tp, fp, fn, )
# output_filename = 'test_%s_%d_gid_%d_tp_%d_fp_%d_fn_%d.png' % args
# output_filepath = join(output_path, output_filename)
# cv2.imwrite(output_filepath, test_image)
# category_list = ['positive', 'negative']
# category_mapping = {
# 'positive': 0,
# 'negative': 1,
# }
# return general_confusion_matrix_algo(label_list, prediction_list, category_list,
# category_mapping, size=20, **kwargs)
# @register_ibs_method
# def localizer_classifications_confusion_matrix_algo_display(ibs, conf,
# min_overlap=0.25,
# figsize=(24, 7),
# write_images=False,
# target_recall=0.9,
# plot_point=True,
# masking=False,
# **kwargs):
# import matplotlib.pyplot as plt
# fig_ = plt.figure(figsize=figsize)
# config = {
# 'label' : 'WIC',
# 'algo' : '_COMBINED',
# 'species_set' : set(['zebra']),
# 'classify' : True,
# 'classifier_algo': 'svm',
# 'classifier_masking': masking,
# 'classifier_weight_filepath': '/home/jason/code/wbia/models-bootstrap/classifier.svm.image.zebra.pkl',
# }
# axes_ = plt.subplot(111)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = localizer_classification_confusion_matrix_algo_plot(ibs, None, conf,
# min_overlap=min_overlap,
# write_images=write_images,
# fig_=fig_, axes_=axes_,
# **config)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# args = (min_overlap, conf, )
# plt.title('Confusion Matrix (IoU %0.02f, Conf %0.02f)' % args, y=1.13)
# # plt.show()
# args = (min_overlap, conf, )
# fig_filename = 'localizer-classification-confusion-matrix-%0.2f-%0.2f.png' % args
# fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
# plt.savefig(fig_path, bbox_inches='tight')
# @register_ibs_method
# def localizer_classifications_confusion_matrix_algo_display_animate(ibs, total=10, **kwargs):
# for index in range(0, total + 1):
# conf = index / total
# ibs.localizer_classifications_confusion_matrix_algo_display(conf, **kwargs)
[docs]def classifier_cameratrap_precision_recall_algo(
ibs, positive_imageset_id, negative_imageset_id, **kwargs
):
depc = ibs.depc_image
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set_ = list(test_gid_set_)
positive_gid_set = set(ibs.get_imageset_gids(positive_imageset_id))
negative_gid_set = set(ibs.get_imageset_gids(negative_imageset_id))
test_gid_set = []
label_list = []
for gid in test_gid_set_:
if gid in positive_gid_set:
label = 'positive'
elif gid in negative_gid_set:
label = 'negative'
else:
# label = 'unknown'
continue
test_gid_set.append(gid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_gid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_gid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
return general_precision_recall_algo(ibs, label_list, confidence_list)
[docs]def classifier_cameratrap_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing Precision-Recall for: %r' % (label,))
(
conf_list,
pr_list,
re_list,
tpr_list,
fpr_list,
) = classifier_cameratrap_precision_recall_algo(ibs, **kwargs)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def classifier_cameratrap_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing ROC for: %r' % (label,))
(
conf_list,
pr_list,
re_list,
tpr_list,
fpr_list,
) = classifier_cameratrap_precision_recall_algo(ibs, **kwargs)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def classifier_cameratrap_confusion_matrix_algo_plot(
ibs,
label,
color,
conf,
positive_imageset_id,
negative_imageset_id,
output_cases=False,
**kwargs,
):
print('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf,))
depc = ibs.depc_image
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set_ = list(test_gid_set_)
positive_gid_set = set(ibs.get_imageset_gids(positive_imageset_id))
negative_gid_set = set(ibs.get_imageset_gids(negative_imageset_id))
test_gid_set = []
label_list = []
for gid in test_gid_set_:
if gid in positive_gid_set:
label = 'positive'
elif gid in negative_gid_set:
label = 'negative'
else:
# label = 'unknown'
continue
test_gid_set.append(gid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_gid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_gid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
prediction_list = [
'positive' if confidence >= conf else 'negative' for confidence in confidence_list
]
if output_cases:
output_path = 'cameratrap-confusion-incorrect'
output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
positive_path = join(output_path, 'positive')
negative_path = join(output_path, 'negative')
ut.delete(output_path)
ut.ensuredir(output_path)
ut.ensuredir(positive_path)
ut.ensuredir(negative_path)
interpolation = cv2.INTER_LANCZOS4
warpkw = dict(interpolation=interpolation)
for gid, label, prediction in zip(test_gid_set, label_list, prediction_list):
if label == prediction:
continue
image = ibs.get_images(gid)
image = cv2.resize(image, (192, 192), **warpkw)
# Get path
image_path = positive_path if label == 'positive' else negative_path
image_filename = 'hardidx_%d_pred_%s_case_fail.jpg' % (gid, prediction,)
image_filepath = join(image_path, image_filename)
# Save path
cv2.imwrite(image_filepath, image)
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
return general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, **kwargs
)
[docs]@register_ibs_method
def classifier_cameratrap_precision_recall_algo_display(
ibs, positive_imageset_id, negative_imageset_id, config_list=None, figsize=(20, 20)
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
fig_ = plt.figure(figsize=figsize, dpi=400)
if config_list is None:
config_list = [
# {'label': 'Initial Model (5%) - IBEIS_CNN', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'ryan.ibeis_cnn.v1'},
# {'label': 'Initial Model (5%) - DenseNet', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1'},
# {'label': 'Initial Model (5%) - DenseNet 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:0'},
# {'label': 'Initial Model (5%) - DenseNet 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:1'},
# {'label': 'Initial Model (5%) - DenseNet 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v1:2'},
{
'label': 'Initial Model (10%) - DenseNet',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'ryan_densenet_v2',
},
# {'label': 'Initial Model (10%) - DenseNet 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:0'},
# {'label': 'Initial Model (10%) - DenseNet 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:1'},
# {'label': 'Initial Model (10%) - DenseNet 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'ryan_densenet_v2:2'},
# {'label': 'Initial Model (0%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.1'},
# {'label': 'Retrained Model (1%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.2'},
# {'label': 'Retrained Model (2%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.3'},
# {'label': 'Retrained Model (3%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.4'},
# {'label': 'Retrained Model (4%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.5'},
# {'label': 'Retrained Model (5%)', 'classifier_algo': 'cnn', 'classifier_weight_filepath': 'megan2.6'},
# {'label': 'Initial Model (0%)', 'classifier_weight_filepath': 'megan1.1'},
# {'label': 'Retrained Model (1%)', 'classifier_weight_filepath': 'megan1.2'},
# {'label': 'Retrained Model (2%)', 'classifier_weight_filepath': 'megan1.3'},
# {'label': 'Retrained Model (3%)', 'classifier_weight_filepath': 'megan1.4'},
# {'label': 'Retrained Model (3.5%)', 'classifier_weight_filepath': 'megan1.5'},
# {'label': 'Retrained Model (5%)', 'classifier_weight_filepath': 'megan1.6'},
]
color_list = pt.distinct_colors(len(config_list), randomize=False)
axes_ = plt.subplot(221)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
classifier_cameratrap_precision_recall_algo_plot(
ibs,
color=color,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
**config,
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
index = np.argmax(area_list)
# index = 0
best_label1 = config_list[index]['label']
best_config1 = config_list[index]
best_color1 = color_list[index]
best_area1 = area_list[index]
best_conf1 = conf_list[index]
plt.title(
'Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label1, best_area1,),
y=1.10,
)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(222)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
classifier_cameratrap_roc_algo_plot(
ibs,
color=color,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
**config,
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
index = np.argmax(area_list)
# index = 0
best_label2 = config_list[index]['label']
best_config2 = config_list[index]
best_color2 = color_list[index]
best_area2 = area_list[index]
best_conf2 = conf_list[index]
plt.title('ROC Curve (Best: %s, AP = %0.02f)' % (best_label2, best_area2,), y=1.10)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(223)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = classifier_cameratrap_confusion_matrix_algo_plot(
ibs,
color=best_color1,
conf=best_conf1,
fig_=fig_,
axes_=axes_,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
output_cases=True,
**best_config1,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1,), y=1.12)
axes_ = plt.subplot(224)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = classifier_cameratrap_confusion_matrix_algo_plot(
ibs,
color=best_color2,
conf=best_conf2,
fig_=fig_,
axes_=axes_,
positive_imageset_id=positive_imageset_id,
negative_imageset_id=negative_imageset_id,
**best_config2,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2,), y=1.12)
fig_filename = 'classifier-cameratrap-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
# def classifier_binary_precision_recall_algo(ibs, category_set, **kwargs):
# depc = ibs.depc_image
# test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
# test_gid_set = list(test_gid_set)
# aids_list = ibs.get_image_aids(test_gid_set)
# species_set_list = [
# set(ibs.get_annot_species_texts(aid_list))
# for aid_list in aids_list
# ]
# label_list = [
# 'negative' if len(species_set & category_set) == 0 else 'positive'
# for species_set in species_set_list
# ]
# prediction_list = depc.get_property('classifier', test_gid_set, 'class', config=kwargs)
# confidence_list = depc.get_property('classifier', test_gid_set, 'score', config=kwargs)
# confidence_list = [
# confidence if prediction == 'positive' else 1.0 - confidence
# for prediction, confidence in zip(prediction_list, confidence_list)
# ]
# return general_precision_recall_algo(ibs, label_list, confidence_list)
# def classifier_binary_precision_recall_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# print('Processing Precision-Recall for: %r' % (label, ))
# conf_list, pr_list, re_list, tpr_list, fpr_list = classifier_binary_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
# def classifier_binary_roc_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# print('Processing ROC for: %r' % (label, ))
# conf_list, pr_list, re_list, tpr_list, fpr_list = classifier_binary_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, fpr_list, tpr_list, interpolate=False,
# target=(0.0, 1.0), **kwargs)
# def classifier_binary_confusion_matrix_algo_plot(ibs, label, color, conf, category_set, **kwargs):
# print('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
# depc = ibs.depc_image
# test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
# test_gid_set = list(test_gid_set)
# aids_list = ibs.get_image_aids(test_gid_set)
# species_set_list = [
# set(ibs.get_annot_species_texts(aid_list))
# for aid_list in aids_list
# ]
# label_list = [
# 'negative' if len(species_set & category_set) == 0 else 'positive'
# for species_set in species_set_list
# ]
# prediction_list = depc.get_property('classifier', test_gid_set, 'class', config=kwargs)
# confidence_list = depc.get_property('classifier', test_gid_set, 'score', config=kwargs)
# confidence_list = [
# confidence if prediction == 'positive' else 1.0 - confidence
# for prediction, confidence in zip(prediction_list, confidence_list)
# ]
# prediction_list = [
# 'positive' if confidence >= conf else 'negative'
# for confidence in confidence_list
# ]
# category_list = ['positive', 'negative']
# category_mapping = {
# 'positive': 0,
# 'negative': 1,
# }
# return general_confusion_matrix_algo(label_list, prediction_list, category_list,
# category_mapping, **kwargs)
# @register_ibs_method
# def classifier_binary_precision_recall_algo_display(ibs, figsize=(16, 16), **kwargs):
# import matplotlib.pyplot as plt
# fig_ = plt.figure(figsize=figsize)
# # label = 'V1'
# # species_list = ['zebra']
# # kwargs['classifier_weight_filepath'] = 'coco_zebra'
# label = 'V3'
# species_list = ['zebra_plains', 'zebra_grevys']
# kwargs['classifier_weight_filepath'] = 'v3_zebra'
# category_set = set(species_list)
# axes_ = plt.subplot(221)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('Recall')
# axes_.set_ylabel('Precision')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# area, best_conf1, _ = classifier_binary_precision_recall_algo_plot(ibs, label=label, color='r', category_set=category_set, **kwargs)
# plt.title('Precision-Recall Curve (AP = %0.02f)' % (area, ), y=1.10)
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
# axes_ = plt.subplot(222)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('False-Positive Rate')
# axes_.set_ylabel('True-Positive Rate')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# area, best_conf2, _ = classifier_binary_roc_algo_plot(ibs, label=label, color='r', category_set=category_set, **kwargs)
# plt.title('ROC Curve (AP = %0.02f)' % (area, ), y=1.10)
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
# axes_ = plt.subplot(223)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = classifier_binary_confusion_matrix_algo_plot(ibs, label, 'r', conf=best_conf1, fig_=fig_, axes_=axes_, category_set=category_set, **kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1, ), y=1.12)
# axes_ = plt.subplot(224)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = classifier_binary_confusion_matrix_algo_plot(ibs, label, 'r', conf=best_conf2, fig_=fig_, axes_=axes_, category_set=category_set, **kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2, ), y=1.12)
# fig_filename = 'classifier-precision-recall-roc.png'
# fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
# plt.savefig(fig_path, bbox_inches='tight')
[docs]def classifier2_precision_recall_algo(
ibs,
category,
species_mapping={},
output_path=None,
test_gid_list=None,
test_label_list=None,
**kwargs,
):
depc = ibs.depc_image
if test_gid_list is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list = list(test_gid_set)
if test_label_list is None:
aids_list = ibs.get_image_aids(test_gid_list)
species_list_list = list(map(ibs.get_annot_species_texts, aids_list))
species_set_list = []
for species_list in species_list_list:
species_set = set([])
for species in species_list:
species = species_mapping.get(species, species)
species_set.add(species)
species_set_list.append(species_set)
else:
species_set_list = [set([label]) for label in test_label_list]
label_list = [
'positive' if category in species_set_ else 'negative'
for species_set_ in species_set_list
]
confidence_dict_list = depc.get_property(
'classifier_two', test_gid_list, 'scores', config=kwargs
)
confidence_list = [
confidence_dict[category] for confidence_dict in confidence_dict_list
]
if output_path is not None:
ut.ensuredir(output_path)
config_ = {
'draw_annots': False,
'thumbsize': (192, 192),
}
thumbnail_list = depc.get_property(
'thumbnails', test_gid_list, 'img', config=config_
)
zipped = zip(
test_gid_list, thumbnail_list, species_set_list, confidence_dict_list
)
for index, (test_gid, thumbnail, species_set, confidence_dict) in enumerate(
zipped
):
print(index)
x = ';'.join(species_set)
y = []
for key in confidence_dict:
y.append('%s-%0.04f' % (key, confidence_dict[key],))
y = ';'.join(y)
output_filename = 'image-index-%s-gid-%s-gt-%s-pred-%s.png' % (
index,
test_gid,
x,
y,
)
output_filepath = join(output_path, output_filename)
cv2.imwrite(output_filepath, thumbnail)
kwargs.pop('category', None)
return general_precision_recall_algo(ibs, label_list, confidence_list)
[docs]def classifier2_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = classifier2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def classifier2_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing ROC for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = classifier2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]@register_ibs_method
def classifier2_precision_recall_algo_display(
ibs,
species_list=None,
species_mapping={},
nice_mapping={},
test_gid_list=None,
test_label_list=None,
figsize=(20, 9),
**kwargs,
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
depc = ibs.depc_image
fig_ = plt.figure(figsize=figsize, dpi=400) # NOQA
# kwargs['classifier_two_weight_filepath'] = 'v3'
# kwargs['classifier_two_weight_filepath'] = 'candidacy'
# kwargs['classifier_two_weight_filepath'] = 'ggr2'
is_labeled = test_label_list is not None
kwargs['classifier_two_algo'] = 'densenet'
kwargs['classifier_two_weight_filepath'] = 'flukebook_v1'
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set) if test_gid_list is None else test_gid_list
test_label_list_ = test_label_list if is_labeled else [None] * len(test_gid_list_)
zipped = list(zip(test_gid_list_, test_label_list_))
test_gid_list_ = []
test_label_list_ = []
for test_gid_, test_label_ in zipped:
if test_gid_ in test_gid_set:
test_gid_list_.append(test_gid_)
test_label_list_.append(test_label_)
test_gid_list = test_gid_list_
test_label_list = test_label_list_ if is_labeled else None
# depc.delete_property('classifier_two', test_gid_list, config=kwargs)
if species_list is None:
test_gid = test_gid_list[0]
confidence_dict = depc.get_property(
'classifier_two', test_gid, 'scores', config=kwargs
)
species_list = confidence_dict.keys()
category_set = sorted(species_list)
config_list = []
for category in category_set:
category_nice = nice_mapping.get(category, category)
config_dict = {
'label': category_nice,
'category': category,
}
config_dict.update(kwargs)
config_list.append(config_dict)
color_list_ = []
color_list = pt.distinct_colors(len(config_list) - len(color_list_), randomize=False)
color_list = color_list_ + color_list
axes_ = plt.subplot(121)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
for color, config in zip(color_list, config_list):
classifier2_precision_recall_algo_plot(
ibs,
color=color,
test_gid_list=test_gid_list,
test_label_list=test_label_list,
species_mapping=species_mapping,
**config,
)
plt.title('Precision-Recall Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(122)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
op_dict = {}
for color, config in zip(color_list, config_list):
values = classifier2_roc_algo_plot(
ibs,
color=color,
test_gid_list=test_gid_list,
test_label_list=test_label_list,
species_mapping=species_mapping,
**config,
)
ap, best_conf, tup1, tup2 = values
op_dict[config['category']] = best_conf
plt.title('ROC Curves', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
if is_labeled:
species_set_list = [set([label]) for label in test_label_list]
else:
aids_list = ibs.get_image_aids(test_gid_list)
species_list_list = list(map(ibs.get_annot_species_texts, aids_list))
species_set_list = []
for species_list in species_list_list:
species_set = set([])
for species in species_list:
species = species_mapping.get(species, species)
species_set.add(species)
species_set_list.append(species_set)
confidence_dict_list = depc.get_property(
'classifier_two', test_gid_list, 'scores', config=kwargs
)
correct = 0
for test_gid, confidence_dict, species_set in zip(
test_gid_list, confidence_dict_list, species_set_list
):
species_set_ = set([])
for key in confidence_dict:
if op_dict[key] <= confidence_dict[key]:
species_set_.add(key)
if len(species_set ^ species_set_) == 0:
correct += 1
else:
print(test_gid, confidence_dict, species_set)
print('Accuracy: %0.04f' % (100.0 * correct / len(test_gid_list)))
print('\t using op_dict = %r' % (op_dict,))
fig_filename = 'classifier2-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]def labeler_tp_tn_fp_fn(
ibs,
category_list,
species_mapping={},
viewpoint_mapping={},
samples=SAMPLES,
test_gid_set=None,
**kwargs,
):
def errors(zipped, conf, category):
tp, tn, fp, fn = 0.0, 0.0, 0.0, 0.0
for index, (label, confidence) in enumerate(zipped):
if label == category:
if conf <= confidence:
tp += 1
else:
fn += 1
else:
if conf <= confidence:
fp += 1
else:
tn += 1
return tp, tn, fp, fn
depc = ibs.depc_annot
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
# Get annot species and viewpoints
species_list = ibs.get_annot_species_texts(aid_list)
viewpoint_list = ibs.get_annot_viewpoints(aid_list)
# Filter aids with species of interest and undefined viewpoints
species_list = [species_mapping.get(species, species) for species in species_list]
viewpoint_list = [
viewpoint_mapping.get(species, {}).get(viewpoint, viewpoint)
for species, viewpoint in zip(species_list, viewpoint_list)
]
flag_list = [
species in category_list and viewpoint is not None
for species, viewpoint in zip(species_list, viewpoint_list)
]
if False in flag_list:
aid_list = ut.compress(aid_list, flag_list)
species_list = ut.compress(species_list, flag_list)
viewpoint_list = ut.compress(viewpoint_list, flag_list)
# Make ground-truth
label_list = [
'%s:%s' % (species, viewpoint_,)
for species, viewpoint_ in zip(species_list, viewpoint_list)
]
# Get predictions
# depc.delete_property('labeler', aid_list, config=kwargs)
probability_dict_list = depc.get_property('labeler', aid_list, 'probs', config=kwargs)
value1_list = set(label_list)
value2_list = set(probability_dict_list[0].keys())
assert len(value1_list - value2_list) == 0
assert len(value2_list - value1_list) == 0
conf_list = [_ / float(samples) for _ in range(0, int(samples) + 1)]
label_dict = {}
for key in value1_list:
print('\t%r' % (key,))
conf_dict = {}
confidence_list = [
probability_dict[key] for probability_dict in probability_dict_list
]
zipped = list(zip(label_list, confidence_list))
for conf in conf_list:
conf_dict[conf] = errors(zipped, conf, key)
label_dict[key] = conf_dict
return label_dict
[docs]def labeler_precision_recall_algo(ibs, category_list, label_dict, **kwargs):
if category_list is None:
category_list_ = label_dict.keys()
else:
category_list_ = []
for category in category_list:
for key in label_dict:
if category in key or category is None:
category_list_.append(key)
global_conf_dict = {}
for category in category_list_:
conf_dict = label_dict[category]
for conf in conf_dict:
new_list = conf_dict[conf]
if conf not in global_conf_dict:
global_conf_dict[conf] = new_list
else:
cur_list = global_conf_dict[conf]
zipped_ = zip(cur_list, new_list)
global_conf_dict[conf] = [cur + new for cur, new in zipped_]
conf_list_ = [-1.0, -1.0]
pr_list = [1.0, 0.0]
re_list = [0.0, 1.0]
tpr_list = [0.0, 1.0]
fpr_list = [0.0, 1.0]
# conf_list_ = []
# pr_list = []
# re_list = []
# tpr_list = []
# fpr_list = []
for conf in sorted(global_conf_dict.keys(), reverse=True):
error_list = global_conf_dict[conf]
tp, tn, fp, fn = error_list
try:
pr = tp / (tp + fp)
re = tp / (tp + fn)
tpr = tp / (tp + fn)
fpr = fp / (fp + tn)
conf_list_.append(conf)
pr_list.append(pr)
re_list.append(re)
tpr_list.append(tpr)
fpr_list.append(fpr)
except ZeroDivisionError:
print(
'Zero division error (%r) - tp: %r tn: %r fp: %r fn: %r'
% (conf, tp, tn, fp, fn,)
)
return conf_list_, pr_list, re_list, tpr_list, fpr_list
[docs]def labeler_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
category_list = kwargs['category_list']
print(
'Processing Precision-Recall for: %r (category_list = %r)'
% (label, category_list,)
)
conf_list, pr_list, re_list, tpr_list, fpr_list = labeler_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def labeler_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
category_list = kwargs['category_list']
print('Processing ROC for: %r (category_list = %r)' % (label, category_list,))
conf_list, pr_list, re_list, tpr_list, fpr_list = labeler_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def labeler_confusion_matrix_algo_plot(
ibs,
category_list,
species_mapping={},
viewpoint_mapping={},
category_mapping=None,
test_gid_set=None,
**kwargs,
):
print('Processing Confusion Matrix')
depc = ibs.depc_annot
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
species_list = ibs.get_annot_species_texts(aid_list)
viewpoint_list = ibs.get_annot_viewpoints(aid_list)
label_list = [
'%s:%s'
% (
species_mapping.get(species, species),
viewpoint_mapping.get(species, {}).get(viewpoint, viewpoint),
)
for species, viewpoint in zip(species_list, viewpoint_list)
]
temp_list = [
(aid, label) for aid, label in zip(aid_list, label_list) if label in category_list
]
aid_list = [_[0] for _ in temp_list]
label_list = [_[1] for _ in temp_list]
conf_list = depc.get_property('labeler', aid_list, 'score', config=kwargs)
species_list = depc.get_property('labeler', aid_list, 'species', config=kwargs)
viewpoint_list = depc.get_property('labeler', aid_list, 'viewpoint', config=kwargs)
prediction_list = [
'%s:%s' % (species, viewpoint,)
for species, viewpoint in zip(species_list, viewpoint_list)
]
category_list = list(map(simple_code, category_list))
label_list = list(map(simple_code, label_list))
prediction_list = list(map(simple_code, prediction_list))
if category_mapping is None:
category_mapping = {key: index for index, key in enumerate(category_list)}
category_mapping = {
simple_code(key): category_mapping[key] for key in category_mapping
}
return general_confusion_matrix_algo(
label_list,
prediction_list,
category_list,
category_mapping,
conf_list=conf_list,
size=8,
**kwargs,
)
[docs]@register_ibs_method
def labeler_precision_recall_algo_display(
ibs,
category_list=None,
species_mapping={},
viewpoint_mapping={},
category_mapping=None,
fuzzy_dict=None,
figsize=(30, 9),
test_gid_set=None,
use_axis_aligned_chips=False,
labeler_weight_filepath=None,
config_list=None,
**kwargs,
):
import matplotlib.pyplot as plt
import plottool as pt
if category_list is None:
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
species_list = ibs.get_annot_species_texts(aid_list)
species_list = [species_mapping.get(species, species) for species in species_list]
category_list = sorted(list(set(species_list)))
print('Compiling raw numbers...')
kwargs['labeler_algo'] = 'densenet'
if labeler_weight_filepath is None:
# kwargs['labeler_weight_filepath'] = 'zebra_v1'
# kwargs['labeler_weight_filepath'] = 'seaturtle'
# kwargs['labeler_weight_filepath'] = 'giraffe_v1'
# kwargs['labeler_weight_filepath'] = 'lynx_v3'
# kwargs['labeler_weight_filepath'] = 'seaturtle_v3'
# kwargs['labeler_weight_filepath'] = 'jaguar_v3'
# kwargs['labeler_weight_filepath'] = 'hendrik_dorsal_v2'
# kwargs['labeler_weight_filepath'] = 'spotted_skunk_v0'
# kwargs['labeler_weight_filepath'] = 'nassau_grouper_v0'
# kwargs['labeler_weight_filepath'] = 'spotted_dolphin_v0'
# kwargs['labeler_weight_filepath'] = 'seadragon_v1'
kwargs['labeler_weight_filepath'] = 'seadragon_v2'
else:
kwargs['labeler_weight_filepath'] = labeler_weight_filepath
kwargs['labeler_axis_aligned'] = use_axis_aligned_chips
label_dict = labeler_tp_tn_fp_fn(
ibs,
category_list,
species_mapping=species_mapping,
viewpoint_mapping=viewpoint_mapping,
test_gid_set=test_gid_set,
**kwargs,
)
if config_list is None:
config_list = [
# {'label': 'Giraffe', 'category_list': None},
# {'label': 'Masai Giraffe', 'category_list': ['giraffe_masai']},
# {'label': 'Reticulated Giraffe', 'category_list': ['giraffe_reticulated']},
# {'label': 'Lynx', 'category_list': ['lynx_pardinus']},
# {'label': 'Sea Turtle', 'category_list': ['turtle_sea']},
# {'label': 'Sea Turtle Head', 'category_list': ['turtle_sea+head']},
# {'label': 'Manta', 'category_list': ['manta_ray_giant']},
# {'label': 'Jaguar', 'category_list': ['jaguar']},
# {'label': 'Dorsal Fin', 'category_list': ['dolphin_bottlenose_fin']},
# {'label': 'Reticulated Giraffe', 'category_list': ['giraffe_reticulated']},
# {'label': 'Sea Turtle', 'category_list': ['turtle_sea']},
# {'label': 'Whale Fluke', 'category_list': ['whale_fluke']},
# {'label': 'Grevy\'s Zebra', 'category_list': ['zebra_grevys']},
# {'label': 'Plains Zebra', 'category_list': ['zebra_plains']},
# {'label': 'Spotted Skunk', 'category_list': ['skunk_spotted']},
# {'label': 'Nassau Grouper', 'category_list': ['grouper_nassau']},
# {'label': 'Spotted Dolphin', 'category_list': ['dolphin_spotted']},
# {'label': 'Spotted Dolphin', 'category_list': ['dolphin_spotted']},
{'label': 'Weedy SD ', 'category_list': ['seadragon_weedy']},
{'label': 'Weedy Head', 'category_list': ['seadragon_weedy+head']},
{'label': 'Leafy SD ', 'category_list': ['seadragon_leafy']},
{'label': 'Leafy Head', 'category_list': ['seadragon_leafy+head']},
]
color_list = [(0.0, 0.0, 0.0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
fig_ = plt.figure(figsize=figsize, dpi=400) # NOQA
axes_ = plt.subplot(131)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
area_list = []
for color, config in zip(color_list, config_list):
ret = labeler_precision_recall_algo_plot(
ibs, label_dict=label_dict, color=color, **config
)
area = ret[0]
area_list.append(area)
plt.title('Precision-Recall Curve', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(132)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
for color, config in zip(color_list, config_list):
labeler_roc_algo_plot(ibs, label_dict=label_dict, color=color, **config)
plt.title('ROC Curve', y=1.19)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
key_list = sorted(label_dict.keys())
fuzzy = fuzzy_dict is not None
if not fuzzy:
fuzzy_dict = {}
for index1, label1 in enumerate(key_list):
if label1 == 'ignore':
fuzzy_list = []
else:
species, viewpoint = label1.strip().split(':')
fuzzy_list = []
for index2, label2 in enumerate(key_list):
if species in label2:
fuzzy_list.append(index2)
fuzzy_dict[index1] = set(fuzzy_list)
axes_ = plt.subplot(133)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, fuzzy_rate = labeler_confusion_matrix_algo_plot(
ibs,
key_list,
species_mapping=species_mapping,
viewpoint_mapping=viewpoint_mapping,
category_mapping=category_mapping,
fig_=fig_,
axes_=axes_,
fuzzy_dict=fuzzy_dict,
test_gid_set=test_gid_set,
**kwargs,
)
if fuzzy:
axes_.set_xlabel(
'Predicted (Correct = %0.02f%%, Fuzzy = %0.02f%%)'
% (correct_rate * 100.0, fuzzy_rate * 100.0,)
)
else:
axes_.set_xlabel(
'Predicted (Correct = %0.02f%%, Species = %0.02f%%)'
% (correct_rate * 100.0, fuzzy_rate * 100.0,)
)
axes_.set_ylabel('Ground-Truth')
# area_list_ = area_list[1:]
area_list_ = area_list
mAP = sum(area_list_) / len(area_list_)
args = (mAP * 100.0,)
plt.title('Confusion Matrix\nmAP = %0.02f' % args, y=1.19)
fig_filename = 'labeler-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]def canonical_precision_recall_algo(ibs, species, **kwargs):
depc = ibs.depc_annot
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_list_ = ut.flatten(ibs.get_image_aids(test_gid_list_))
test_aid_list_ = ibs.filter_annotation_set(test_aid_list_, species=species)
test_flag_list_ = ibs.get_annot_canonical(test_aid_list_)
test_aid_set = []
label_list = []
for aid, flag in zip(test_aid_list_, test_flag_list_):
if flag:
label = 'positive'
else:
label = 'negative'
test_aid_set.append(aid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_aid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_aid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
return general_precision_recall_algo(ibs, label_list, confidence_list)
[docs]def canonical_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = canonical_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def canonical_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing ROC for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = canonical_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def canonical_confusion_matrix_algo_plot(
ibs, label, color, conf, species, output_cases=False, **kwargs
):
print('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf,))
depc = ibs.depc_annot
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_list_ = ut.flatten(ibs.get_image_aids(test_gid_list_))
test_aid_list_ = ibs.filter_annotation_set(test_aid_list_, species=species)
test_flag_list_ = ibs.get_annot_canonical(test_aid_list_)
test_aid_set = []
label_list = []
for aid, flag in zip(test_aid_list_, test_flag_list_):
if flag:
label = 'positive'
else:
label = 'negative'
test_aid_set.append(aid)
label_list.append(label)
prediction_list = depc.get_property(
'classifier', test_aid_set, 'class', config=kwargs
)
confidence_list = depc.get_property(
'classifier', test_aid_set, 'score', config=kwargs
)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
prediction_list = [
'positive' if confidence >= conf else 'negative' for confidence in confidence_list
]
if output_cases:
output_path = 'canonical-confusion-incorrect'
output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
positive_path = join(output_path, 'positive')
negative_path = join(output_path, 'negative')
ut.delete(output_path)
ut.ensuredir(output_path)
ut.ensuredir(positive_path)
ut.ensuredir(negative_path)
config = {
'dim_size': (192, 192),
'resize_dim': 'wh',
}
chip_list = ibs.depc_annot.get_property(
'chips', test_aid_set, 'img', config=config
)
zipped = zip(test_aid_set, chip_list, label_list, prediction_list)
for aid, chip, label, prediction in zipped:
if label == prediction:
continue
# Get path
image_path = positive_path if label == 'positive' else negative_path
image_filename = 'hardidx_%d_pred_%s_case_fail.jpg' % (aid, prediction,)
image_filepath = join(image_path, image_filename)
# Save path
cv2.imwrite(image_filepath, chip)
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
return general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, **kwargs
)
[docs]@register_ibs_method
def canonical_precision_recall_algo_display(ibs, figsize=(20, 20)):
import matplotlib.pyplot as plt
import plottool as pt
fig_ = plt.figure(figsize=figsize, dpi=400)
config_list = [
{
'label': 'CA V1 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v1',
'species': 'zebra_grevys',
}, # SMALLER DATASET
{
'label': 'CA V2 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v2',
'species': 'zebra_grevys',
}, # BROKEN L/R AUGMENTATION
{
'label': 'CA V3 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v3',
'species': 'zebra_grevys',
}, # LARGER DATASET, TOO HARSH AUGMENTATION
{
'label': 'CA V4 Ensemble',
'classifier_algo': 'densenet',
'classifier_weight_filepath': 'canonical_zebra_grevys_v4',
'species': 'zebra_grevys',
}, # BETTER AUGMENTATION
# {'label': 'CA V4 Model 0', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:0', 'species': 'zebra_grevys'},
# {'label': 'CA V4 Model 1', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:1', 'species': 'zebra_grevys'},
# {'label': 'CA V4 Model 2', 'classifier_algo': 'densenet', 'classifier_weight_filepath': 'canonical_zebra_grevys_v4:2', 'species': 'zebra_grevys'},
]
color_list = []
# color_list = [(0, 0, 0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
axes_ = plt.subplot(221)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
canonical_precision_recall_algo_plot(ibs, color=color, **config)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = -1
best_label1 = config_list[index]['label']
best_config1 = config_list[index]
best_color1 = color_list[index]
best_area1 = area_list[index]
best_conf1 = conf_list[index]
plt.title(
'Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label1, best_area1,),
y=1.10,
)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(222)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
canonical_roc_algo_plot(ibs, color=color, **config)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = -1
best_label2 = config_list[index]['label']
best_config2 = config_list[index]
best_color2 = color_list[index]
best_area2 = area_list[index]
best_conf2 = conf_list[index]
plt.title('ROC Curve (Best: %s, AP = %0.02f)' % (best_label2, best_area2,), y=1.10)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(223)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = canonical_confusion_matrix_algo_plot(
ibs,
color=best_color1,
conf=best_conf1,
fig_=fig_,
axes_=axes_,
output_cases=True,
**best_config1,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1,), y=1.12)
axes_ = plt.subplot(224)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = canonical_confusion_matrix_algo_plot(
ibs, color=best_color2, conf=best_conf2, fig_=fig_, axes_=axes_, **best_config2
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2,), y=1.12)
fig_filename = 'canonical-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
def _canonical_get_boxes(ibs, gid_list, species):
from wbia.web.appfuncs import CANONICAL_PART_TYPE
aid_list = ut.flatten(ibs.get_image_aids(gid_list))
aid_list = ibs.filter_annotation_set(aid_list, species=species)
flag_list = ibs.get_annot_canonical(aid_list)
part_rowids_list = ibs.get_annot_part_rowids(aid_list)
part_types_list = list(map(ibs.get_part_types, part_rowids_list))
aid_set = []
bbox_set = []
zipped = zip(aid_list, flag_list, part_rowids_list, part_types_list)
for aid, flag, part_rowid_list, part_type_list in zipped:
part_rowid_ = None
if flag:
for part_rowid, part_type in zip(part_rowid_list, part_type_list):
if part_type == CANONICAL_PART_TYPE:
assert part_rowid_ is None, 'Cannot have multiple CA for one image'
part_rowid_ = part_rowid
if part_rowid_ is not None:
axtl, aytl, aw, ah = ibs.get_annot_bboxes(aid)
axbr, aybr = axtl + aw, aytl + ah
pxtl, pytl, pw, ph = ibs.get_part_bboxes(part_rowid_)
pxbr, pybr = pxtl + pw, pytl + ph
x0 = pxtl - axtl
y0 = pytl - aytl
x1 = axbr - pxbr
y1 = aybr - pybr
x0 = max(x0 / aw, 0.0)
y0 = max(y0 / ah, 0.0)
x1 = max(x1 / aw, 0.0)
y1 = max(y1 / ah, 0.0)
assert x0 + x1 < 0.99
assert y0 + y1 < 0.99
bbox = (x0, y0, x1, y1)
aid_set.append(aid)
bbox_set.append(bbox)
return aid_set, bbox_set
[docs]def canonical_localization_deviation_plot(
ibs, attribute, color, index, label=None, species=None, marker='o', **kwargs
):
import random
import matplotlib.pyplot as plt
assert None not in [label, species]
print('Processing Deviation for: %r' % (label,))
depc = ibs.depc_annot
if attribute == 'x0':
take_index = 0
elif attribute == 'y0':
take_index = 1
elif attribute == 'x1':
take_index = 2
elif attribute == 'y1':
take_index = 3
else:
raise ValueError('attribute not valid')
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_set, test_bbox_set = _canonical_get_boxes(ibs, test_gid_list_, species)
value_list = ut.take_column(test_bbox_set, take_index)
prediction_list = depc.get_property(
'canonical', test_aid_set, attribute, config=kwargs
)
x_list = []
y_list = []
overshoot = 0.0
for value, prediction in zip(value_list, prediction_list):
x = random.uniform(index, index + 1)
y = value - prediction
if y < 0:
overshoot += 1
x_list.append(x)
y_list.append(y)
mean = np.mean(y_list)
std = np.std(y_list)
overshoot /= len(y_list)
label = '%s (Over: %0.02f, %0.02f+/-%0.02f)' % (label, overshoot, mean, std,)
plt.plot(
x_list,
y_list,
color=color,
linestyle='None',
marker=marker,
label=label,
alpha=0.5,
)
plt.plot(
[index, index + 1], [0.0, 0.0], color=(0.2, 0.2, 0.2), linestyle='-', alpha=0.3
)
if index % 4 == 3:
plt.plot(
[index + 1, index + 1],
[-1.0, 1.0],
color=(0.2, 0.2, 0.2),
linestyle='--',
alpha=0.1,
)
color = 'xkcd:gold'
marker = 'D'
plt.errorbar(
[index + 0.5],
[mean],
[std],
linestyle='None',
color=color,
marker=marker,
zorder=999,
barsabove=True,
)
# plt.plot([index + 0.5], [mean], color=color, marker=marker)
[docs]def canonical_localization_iou_plot(
ibs, color, index, label=None, species=None, marker='o', threshold=0.75, **kwargs
):
import random
import matplotlib.pyplot as plt
def _convert(bbox):
x0, y0, x1, y1 = bbox
retval = {
'xtl': x0,
'ytl': y0,
'xbr': 1.0 - x1,
'ybr': 1.0 - y1,
}
retval['width'] = retval['xbr'] - retval['xtl']
retval['height'] = retval['ybr'] - retval['ytl']
return retval
assert None not in [label, species]
print('Processing IoU for: %r' % (label,))
depc = ibs.depc_annot
test_gid_set_ = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_list_ = list(test_gid_set_)
test_aid_set, test_bbox_set = _canonical_get_boxes(ibs, test_gid_list_, species)
prediction_list = depc.get_property('canonical', test_aid_set, None, config=kwargs)
gt_list = [_convert(test_bbox) for test_bbox in test_bbox_set]
pred_list = [_convert(prediction) for prediction in prediction_list]
correct = 0.0
x_list = []
y_list = []
for gt, pred in zip(gt_list, pred_list):
overlap = general_overlap([gt], [pred])
x = random.uniform(index, index + 1)
y = overlap[0][0]
if y >= threshold:
correct += 1.0
x_list.append(x)
y_list.append(y)
accuracy = correct / len(y_list)
mean = np.mean(y_list)
std = np.std(y_list)
label = '%s (Acc: %0.02f, %0.02f+/-%0.02f)' % (label, accuracy, mean, std,)
plt.plot(
x_list,
y_list,
color=color,
linestyle='None',
marker=marker,
label=label,
alpha=0.5,
)
for y_value in [0.5, 0.75, 0.9]:
plt.plot(
[index, index + 1],
[y_value, y_value],
color=(0.2, 0.2, 0.2),
linestyle='-',
alpha=0.3,
)
if index % 4 == 3:
plt.plot(
[index + 1, index + 1],
[0.0, 1.0],
color=(0.2, 0.2, 0.2),
linestyle='--',
alpha=0.1,
)
color = 'xkcd:gold'
marker = 'D'
plt.errorbar(
[index + 0.5],
[mean],
[std],
linestyle='None',
color=color,
marker=marker,
zorder=999,
barsabove=True,
)
# plt.plot([index + 0.5], [mean], color=color, marker=marker)
return test_aid_set, test_bbox_set, prediction_list, y_list, accuracy
[docs]@register_ibs_method
def canonical_localization_iou_visualize(
ibs,
index,
test_aid_set,
test_bbox_set,
prediction_list,
overlap_list,
color_list,
label=None,
species=None,
**kwargs,
):
assert None not in [label, species]
assert len(color_list) == 4
print('Processing Renderings for: %r' % (label,))
color_list_ = []
for color in color_list:
color_ = []
for value in color:
value_ = int(np.around(255.0 * value))
color_ = [value_] + color_
color_ = tuple(color_)
color_list_.append(color_)
color_list = color_list_
output_path = expanduser(join('~', 'Desktop', 'canonical-regression-%d' % (index,)))
ut.delete(output_path)
ut.ensuredir(output_path)
config = {
'dim_size': 600,
'resize_dim': 'maxwh',
}
chip_list = ibs.depc_annot.get_property('chips', test_aid_set, 'img', config=config)
zipped = list(
zip(test_aid_set, chip_list, test_bbox_set, prediction_list, overlap_list)
)
for test_aid, chip, test_bbox, prediction, overlap in zipped:
h, w = chip.shape[:2]
chipa = chip.copy()
chipb = chip.copy()
x0a, y0a, x1a, y1a = test_bbox
x0b, y0b, x1b, y1b = prediction
x0a = int(np.around(x0a * w))
y0a = int(np.around(y0a * h))
x1a = int(np.around(x1a * w))
y1a = int(np.around(y1a * h))
x0b = int(np.around(x0b * w))
y0b = int(np.around(y0b * h))
x1b = int(np.around(x1b * w))
y1b = int(np.around(y1b * h))
x1a = w - x1a
x1b = w - x1b
y1a = h - y1a
y1b = h - y1b
chipa = cv2.line(chipa, (x0a, y0a), (x0a, y1a), color_list[0], 3)
chipa = cv2.line(chipa, (x0a, y0a), (x1a, y0a), color_list[1], 3)
chipa = cv2.line(chipa, (x1a, y0a), (x1a, y1a), color_list[2], 3)
chipa = cv2.line(chipa, (x0a, y1a), (x1a, y1a), color_list[3], 3)
chipb = cv2.line(chipb, (x0b, y0b), (x0b, y1b), color_list[0], 3)
chipb = cv2.line(chipb, (x0b, y0b), (x1b, y0b), color_list[1], 3)
chipb = cv2.line(chipb, (x1b, y0b), (x1b, y1b), color_list[2], 3)
chipb = cv2.line(chipb, (x0b, y1b), (x1b, y1b), color_list[3], 3)
canvas = np.hstack((chipa, chipb))
canvas_filepath = join(
output_path,
'canonical-regression-iou-%0.02f-aid-%s.jpg' % (overlap, test_aid,),
)
cv2.imwrite(canvas_filepath, canvas)
[docs]@register_ibs_method
def canonical_localization_precision_recall_algo_display(ibs, figsize=(20, 40)):
import matplotlib.pyplot as plt
import plottool as pt
fig_ = plt.figure(figsize=figsize, dpi=400) # NOQA
config_list = [
# {'label': 'CA V1 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V1 Model 0', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1:0', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V1 Model 1', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1:1', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V1 Model 2', 'canonical_weight_filepath': 'canonical_zebra_grevys_v1:2', 'species': 'zebra_grevys'}, # OVER = 1.0, small dataset
# {'label': 'CA V2 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V2 Model 0', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2:0', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V2 Model 1', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2:1', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V2 Model 2', 'canonical_weight_filepath': 'canonical_zebra_grevys_v2:2', 'species': 'zebra_grevys'}, # OVER = 1.0, large dataset
# {'label': 'CA V3 Ensemble', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3', 'species': 'zebra_grevys'}, # OVER = 2.0
# {'label': 'CA V3 Model 0', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3:0', 'species': 'zebra_grevys'}, # OVER = 2.0
# {'label': 'CA V3 Model 1', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3:1', 'species': 'zebra_grevys'}, # OVER = 2.0
# {'label': 'CA V3 Model 2', 'canonical_weight_filepath': 'canonical_zebra_grevys_v3:2', 'species': 'zebra_grevys'}, # OVER = 2.0
{
'label': 'CA V5-1.0 Ens.',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V5-1.0 M0',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5:0',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V5-1.0 M1',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5:1',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V5-1.0 M2',
'canonical_weight_filepath': 'canonical_zebra_grevys_v5:2',
'species': 'zebra_grevys',
}, # OVER = 1.0
{
'label': 'CA V6-2.0 Ens.',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V6-2.0 M0',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6:0',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V6-2.0 M1',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6:1',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V6-2.0 M2',
'canonical_weight_filepath': 'canonical_zebra_grevys_v6:2',
'species': 'zebra_grevys',
}, # OVER = 2.0
{
'label': 'CA V4-4.0 Ens.',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4',
'species': 'zebra_grevys',
}, # OVER = 4.0
{
'label': 'CA V4-4.0 M0',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4:0',
'species': 'zebra_grevys',
}, # OVER = 4.0
{
'label': 'CA V4-4.0 M1',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4:1',
'species': 'zebra_grevys',
}, # OVER = 4.0
{
'label': 'CA V4-4.0 M2',
'canonical_weight_filepath': 'canonical_zebra_grevys_v4:2',
'species': 'zebra_grevys',
}, # OVER = 4.0
]
color_list = []
# color_list = [(0, 0, 0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
min_, max_ = -1.0, 1.0
axes_ = plt.subplot(321)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'x0', color=color, index=index, **config
)
plt.title('X0 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(322)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'x1', color=color, index=index, **config
)
plt.title('Y0 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(323)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'y0', color=color, index=index, **config
)
plt.title('X1 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(324)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
for index, (color, config) in enumerate(zip(color_list, config_list)):
canonical_localization_deviation_plot(
ibs, 'y1', color=color, index=index, **config
)
plt.title('Y1 Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(325)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('GT - Pred Deviation (in percentages)')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([min_, max_])
axes_.fill_between([0.0, len(config_list)], -1, 0, facecolor='red', alpha=0.1)
assert len(config_list) % 4 == 0
rounds = len(config_list) // 4
colors = pt.distinct_colors(4, randomize=False)
attribute_list = []
color_list_ = []
for _ in range(rounds):
attribute_list += ['x0', 'y0', 'x1', 'y1']
color_list_ += colors
for index, (attribute, color_) in enumerate(zip(attribute_list, color_list_)):
index_ = (index // 4) * 4
config_ = config_list[index_].copy()
config_['label'] = '%s %s' % (config_['label'], attribute,)
canonical_localization_deviation_plot(
ibs, attribute, color=color_, index=index, **config_
)
plt.title('Ensemble Deviation Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(326)
axes_.grid(True, which='major')
axes_.grid(False, which='minor')
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.get_xaxis().set_ticks([])
axes_.set_ylabel('IoU')
axes_.set_xlim([0.0, len(config_list)])
axes_.set_ylim([0.0, 1.0])
for index, (color, config) in enumerate(zip(color_list, config_list)):
values_ = canonical_localization_iou_plot(ibs, color=color, index=index, **config)
if index % 4 == 0:
config_ = config_list[index]
test_aid_set, test_bbox_set, prediction_list, y_list, accuracy = values_
ibs.canonical_localization_iou_visualize(
index,
test_aid_set,
test_bbox_set,
prediction_list,
y_list,
colors,
**config_,
)
plt.title('IoU Scatter Plot')
plt.legend(
bbox_to_anchor=(0.0, 1.04, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
fig_filename = 'canonical-localization-deviance.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]@register_ibs_method
def background_accuracy_display(ibs, category_list, test_gid_set=None, output_path=None):
if output_path is None:
output_path = abspath(expanduser(join('~', 'Desktop', 'background')))
ut.ensuredir(output_path)
if test_gid_set is None:
test_gid_set = set(general_get_imageset_gids(ibs, 'TEST_SET'))
test_gid_set = list(test_gid_set)
aids_list = ibs.get_image_aids(test_gid_set)
aid_list = ut.flatten(aids_list)
species_list = ibs.get_annot_species_texts(aid_list)
aid_list = [
aid for aid, species in zip(aid_list, species_list) if species in category_list
]
species_list = ibs.get_annot_species_texts(aid_list)
gid_list = ibs.get_annot_gids(aid_list)
config2_ = {'fw_detector': 'cnn'}
hough_cpath_list = ibs.get_annot_probchip_fpath(aid_list, config2_=config2_)
image_list = [vt.imread(hough_cpath) for hough_cpath in hough_cpath_list]
chip_list = ibs.get_annot_chips(aid_list, config2_=config2_)
zipped = zip(aid_list, gid_list, species_list, image_list, chip_list)
for index, (aid, gid, species, image, chip) in enumerate(zipped):
print(index)
mask = vt.resize_mask(image, chip)
blended = vt.blend_images_multiply(chip, mask)
blended *= 255.0
blended = np.around(blended)
blended[blended < 0] = 0
blended[blended > 255] = 255
blended = blended.astype(np.uint8)
canvas = np.hstack((chip, mask, blended))
output_filepath = join(
output_path, 'background.%s.%d.%d.png' % (species, gid, aid,)
)
cv2.imwrite(output_filepath, canvas)
[docs]def aoi2_precision_recall_algo(ibs, category_list=None, test_gid_set_=None, **kwargs):
depc = ibs.depc_annot
if test_gid_set_ is None:
test_gid_set_ = general_get_imageset_gids(ibs, 'TEST_SET')
test_aid_list_ = list(set(ut.flatten(ibs.get_image_aids(test_gid_set_))))
species_list = ibs.get_annot_species_texts(test_aid_list_)
interest_list = ibs.get_annot_interest(test_aid_list_)
test_aid_list = []
label_list = []
for test_aid, species, interest in zip(test_aid_list_, species_list, interest_list):
if category_list is not None:
if species not in category_list:
continue
if interest is None:
continue
label = 'positive' if interest else 'negative'
test_aid_list.append(test_aid)
label_list.append(label)
prediction_list = depc.get_property('aoi_two', test_aid_list, 'class', config=kwargs)
confidence_list = depc.get_property('aoi_two', test_aid_list, 'score', config=kwargs)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
return general_precision_recall_algo(ibs, label_list, confidence_list, **kwargs)
[docs]def aoi2_precision_recall_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing Precision-Recall for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = aoi2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
[docs]def aoi2_roc_algo_plot(ibs, **kwargs):
label = kwargs['label']
print('Processing ROC for: %r' % (label,))
conf_list, pr_list, re_list, tpr_list, fpr_list = aoi2_precision_recall_algo(
ibs, **kwargs
)
return general_area_best_conf(
conf_list, fpr_list, tpr_list, interpolate=False, target=(0.0, 1.0), **kwargs
)
[docs]def aoi2_confusion_matrix_algo_plot(
ibs,
label,
color,
conf,
output_cases=False,
category_list=None,
test_gid_set_=None,
**kwargs,
):
print('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf,))
depc = ibs.depc_annot
if test_gid_set_ is None:
test_gid_set_ = general_get_imageset_gids(ibs, 'TEST_SET')
test_aid_list_ = list(set(ut.flatten(ibs.get_image_aids(test_gid_set_))))
species_list = ibs.get_annot_species_texts(test_aid_list_)
interest_list = ibs.get_annot_interest(test_aid_list_)
test_aid_list = []
label_list = []
for test_aid, species, interest in zip(test_aid_list_, species_list, interest_list):
if category_list is not None:
if species not in category_list:
continue
if interest is None:
continue
label = 'positive' if interest else 'negative'
test_aid_list.append(test_aid)
label_list.append(label)
prediction_list = depc.get_property('aoi_two', test_aid_list, 'class', config=kwargs)
confidence_list = depc.get_property('aoi_two', test_aid_list, 'score', config=kwargs)
confidence_list = [
confidence if prediction == 'positive' else 1.0 - confidence
for prediction, confidence in zip(prediction_list, confidence_list)
]
prediction_list = [
'positive' if confidence >= conf else 'negative' for confidence in confidence_list
]
if output_cases:
output_path = 'aoi2-confusion-incorrect'
output_path = abspath(expanduser(join('~', 'Desktop', output_path)))
ut.delete(output_path)
ut.ensuredir(output_path)
manifest_dict = {}
test_gid_list = ibs.get_annot_gids(test_aid_list)
zipped = zip(test_gid_list, test_aid_list, label_list, prediction_list)
for test_gid, test_aid, label, prediction in zipped:
if test_gid not in manifest_dict:
manifest_dict[test_gid] = {}
assert test_aid not in manifest_dict[test_gid]
manifest_dict[test_gid][test_aid] = (
label,
prediction,
)
for test_gid in manifest_dict:
image = ibs.get_images(test_gid)
w, h = ibs.get_image_sizes(test_gid)
image = _resize(image, t_width=600, verbose=False)
height_, width_, channels_ = image.shape
for test_aid in manifest_dict[test_gid]:
label, prediction = manifest_dict[test_gid][test_aid]
bbox = ibs.get_annot_bboxes(test_aid)
xtl, ytl, width, height = bbox
xbr = xtl + width
ybr = ytl + height
xtl = int(np.round((xtl / w) * width_))
ytl = int(np.round((ytl / h) * height_))
xbr = int(np.round((xbr / w) * width_))
ybr = int(np.round((ybr / h) * height_))
if label == 'positive':
color = (255, 99, 46)
else:
color = (127, 255, 127)
cv2.rectangle(image, (xtl, ytl), (xbr, ybr), color, 4)
if prediction == 'positive':
color = (255, 99, 46)
else:
color = (127, 255, 127)
cv2.rectangle(image, (xtl - 4, ytl - 4), (xbr + 4, ybr + 4), color, 4)
image_filename = 'image_%d.png' % (test_gid,)
image_filepath = join(output_path, image_filename)
cv2.imwrite(image_filepath, image)
category_list = ['positive', 'negative']
category_mapping = {
'positive': 0,
'negative': 1,
}
return general_confusion_matrix_algo(
label_list, prediction_list, category_list, category_mapping, size=20, **kwargs
)
[docs]@register_ibs_method
def aoi2_precision_recall_algo_display(
ibs, test_gid_list=None, output_cases=False, figsize=(20, 20)
):
import matplotlib.pyplot as plt
import wbia.plottool as pt
fig_ = plt.figure(figsize=figsize)
test_gid_set = None if test_gid_list is None else sorted(set(test_gid_list))
config_list = [
# {'label': 'All Species', 'aoi_two_weight_filepath': 'ggr2', 'category_list': None},
# {'label': 'Masai Giraffe', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['giraffe_masai']},
# {'label': 'Reticulated Giraffe', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['giraffe_reticulated']},
# {'label': 'Sea Turtle', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['turtle_sea']},
# {'label': 'Whale Fluke', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['whale_fluke']},
# {'label': 'Grevy\'s Zebra', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['zebra_grevys']},
# {'label': 'Plains Zebra', 'aoi_two_weight_filepath': 'ggr2', 'category_list': ['zebra_plains']},
# {'label': 'Hammerhead', 'aoi_two_weight_filepath': 'hammerhead', 'category_list': ['shark_hammerhead']},
{
'label': 'Jaguar',
'aoi_two_weight_filepath': 'jaguar',
'category_list': ['jaguar'],
},
]
color_list = [(0, 0, 0)]
color_list += pt.distinct_colors(len(config_list) - len(color_list), randomize=False)
axes_ = plt.subplot(221)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('Recall')
axes_.set_ylabel('Precision')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
aoi2_precision_recall_algo_plot(
ibs, color=color, test_gid_set_=test_gid_set, **config
)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = 0
best_label1 = config_list[index]['label']
best_config1 = config_list[index]
best_color1 = color_list[index]
best_area1 = area_list[index]
best_conf1 = conf_list[index]
plt.title(
'Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label1, best_area1,),
y=1.10,
)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
axes_ = plt.subplot(222)
axes_.set_autoscalex_on(False)
axes_.set_autoscaley_on(False)
axes_.set_xlabel('False-Positive Rate')
axes_.set_ylabel('True-Positive Rate')
axes_.set_xlim([0.0, 1.01])
axes_.set_ylim([0.0, 1.01])
ret_list = [
aoi2_roc_algo_plot(ibs, color=color, **config)
for color, config in zip(color_list, config_list)
]
area_list = [ret[0] for ret in ret_list]
conf_list = [ret[1] for ret in ret_list]
# index = np.argmax(area_list)
index = 0
best_label2 = config_list[index]['label']
best_config2 = config_list[index]
best_color2 = color_list[index]
best_area2 = area_list[index]
best_conf2 = conf_list[index]
plt.title('ROC Curve (Best: %s, AP = %0.02f)' % (best_label2, best_area2,), y=1.10)
plt.legend(
bbox_to_anchor=(0.0, 1.02, 1.0, 0.102),
loc=3,
ncol=2,
mode='expand',
borderaxespad=0.0,
)
plt.plot([0.0, 1.0], [0.0, 1.0], color=(0.5, 0.5, 0.5), linestyle='--')
axes_ = plt.subplot(223)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = aoi2_confusion_matrix_algo_plot(
ibs,
color=best_color1,
conf=best_conf1,
fig_=fig_,
axes_=axes_,
output_cases=output_cases,
test_gid_set_=test_gid_set,
**best_config1,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('P-R Confusion Matrix (OP = %0.02f)' % (best_conf1,), y=1.12)
axes_ = plt.subplot(224)
axes_.set_aspect(1)
gca_ = plt.gca()
gca_.grid(False)
correct_rate, _ = aoi2_confusion_matrix_algo_plot(
ibs,
color=best_color2,
conf=best_conf2,
fig_=fig_,
axes_=axes_,
test_gid_set_=test_gid_set,
**best_config2,
)
axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0,))
axes_.set_ylabel('Ground-Truth')
plt.title('ROC Confusion Matrix (OP = %0.02f)' % (best_conf2,), y=1.12)
fig_filename = 'aoi2-precision-recall-roc.png'
fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
plt.savefig(fig_path, bbox_inches='tight')
[docs]def detector_parse_gt(ibs, test_gid_list=None, **kwargs):
if test_gid_list is None:
test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
uuid_list = ibs.get_image_uuids(test_gid_list)
gid_list = ibs.get_image_gids_from_uuid(uuid_list)
gt_dict = {}
for gid, uuid in zip(gid_list, uuid_list):
width, height = ibs.get_image_sizes(gid)
aid_list = ibs.get_image_aids(gid)
gt_list = []
for aid in aid_list:
bbox = ibs.get_annot_bboxes(aid)
temp = {
'gid': gid,
'xtl': bbox[0] / width,
'ytl': bbox[1] / height,
'xbr': (bbox[0] + bbox[2]) / width,
'ybr': (bbox[1] + bbox[3]) / height,
'width': bbox[2] / width,
'height': bbox[3] / height,
'class': ibs.get_annot_species_texts(aid),
'viewpoint': ibs.get_annot_viewpoints(aid),
'confidence': 1.0,
}
gt_list.append(temp)
gt_dict[uuid] = gt_list
return gt_dict
# def detector_parse_pred(ibs, test_gid_list=None, **kwargs):
# depc = ibs.depc_image
# if test_gid_list is None:
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# uuid_list = ibs.get_image_uuids(test_gid_list)
# # depc.delete_property('detections', test_gid_list, config=kwargs)
# results_list = depc.get_property('detections', test_gid_list, None, config=kwargs)
# size_list = ibs.get_image_sizes(test_gid_list)
# zipped_list = zip(results_list)
# # Reformat results for json
# results_list = [
# [
# {
# 'gid' : test_gid,
# 'xtl' : bbox[0] / width,
# 'ytl' : bbox[1] / height,
# 'width' : bbox[2] / width,
# 'height' : bbox[3] / height,
# 'theta' : theta, # round(theta, 4),
# 'confidence' : conf, # round(conf, 4),
# 'class' : class_,
# 'viewpoint' : viewpoint,
# }
# for bbox, theta, class_, viewpoint, conf in zip(*zipped[0][1:])
# ]
# for zipped, (width, height), test_gid in zip(zipped_list, size_list, test_gid_list)
# ]
# pred_dict = {
# uuid_ : result_list
# for uuid_, result_list in zip(uuid_list, results_list)
# }
# # print(pred_dict)
# return pred_dict
# def detector_precision_recall_algo(ibs, samples=SAMPLES, force_serial=FORCE_SERIAL, **kwargs):
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# uuid_list = ibs.get_image_uuids(test_gid_list)
# print('\tGather Ground-Truth')
# gt_dict = detector_parse_gt(ibs, test_gid_list=test_gid_list)
# print('\tGather Predictions')
# pred_dict = detector_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
# print('\tGenerate Curves...')
# conf_list = [ _ / float(samples) for _ in range(0, int(samples) + 1) ]
# conf_list = sorted(conf_list, reverse=True)
# uuid_list_list = [ uuid_list for _ in conf_list ]
# gt_dict_list = [ gt_dict for _ in conf_list ]
# pred_dict_list = [ pred_dict for _ in conf_list ]
# kwargs_list = [ kwargs for _ in conf_list ]
# arg_iter = zip(conf_list, uuid_list_list, gt_dict_list, pred_dict_list, kwargs_list)
# pr_re_gen = ut.generate2(detector_precision_recall_algo_worker, arg_iter,
# nTasks=len(conf_list), ordered=True,
# chunksize=CHUNK_SIZE, force_serial=force_serial)
# conf_list_ = [-1.0, -1.0]
# pr_list = [1.0, 0.0]
# re_list = [0.0, 1.0]
# # conf_list_ = []
# # pr_list = []
# # re_list = []
# for conf, pr, re in pr_re_gen:
# conf_list_.append(conf)
# pr_list.append(pr)
# re_list.append(re)
# print('...complete')
# return conf_list_, pr_list, re_list
# def detector_precision_recall_algo_worker(conf, uuid_list, gt_dict, pred_dict,
# kwargs):
# tp, fp, fn = 0.0, 0.0, 0.0
# for index, uuid_ in enumerate(uuid_list):
# if uuid_ in pred_dict:
# pred_list = [
# pred
# for pred in pred_dict[uuid_]
# if pred['confidence'] >= conf
# ]
# tp_, fp_, fn_ = general_tp_fp_fn(gt_dict[uuid_], pred_list, **kwargs)
# tp += tp_
# fp += fp_
# fn += fn_
# pr = tp / (tp + fp)
# re = tp / (tp + fn)
# return (conf, pr, re)
# def detector_precision_recall_algo_plot(ibs, **kwargs):
# label = kwargs['label']
# print('Processing Precision-Recall for: %r' % (label, ))
# conf_list, pr_list, re_list = detector_precision_recall_algo(ibs, **kwargs)
# return general_area_best_conf(conf_list, re_list, pr_list, **kwargs)
# def detector_confusion_matrix_algo_plot(ibs, label, color, conf, **kwargs):
# print('Processing Confusion Matrix for: %r (Conf = %0.02f)' % (label, conf, ))
# test_gid_list = general_get_imageset_gids(ibs, 'TEST_SET', **kwargs)
# uuid_list = ibs.get_image_uuids(test_gid_list)
# print('\tGather Ground-Truth')
# gt_dict = detector_parse_gt(ibs, test_gid_list=test_gid_list)
# print('\tGather Predictions')
# pred_dict = detector_parse_pred(ibs, test_gid_list=test_gid_list, **kwargs)
# label_list = []
# prediction_list = []
# for index, uuid_ in enumerate(uuid_list):
# if uuid_ in pred_dict:
# gt_list = gt_dict[uuid_]
# pred_list = [
# pred
# for pred in pred_dict[uuid_]
# if pred['confidence'] >= conf
# ]
# tp, fp, fn = general_tp_fp_fn(gt_list, pred_list, **kwargs)
# for _ in range(int(tp)):
# label_list.append('positive')
# prediction_list.append('positive')
# for _ in range(int(fp)):
# label_list.append('negative')
# prediction_list.append('positive')
# for _ in range(int(fn)):
# label_list.append('positive')
# prediction_list.append('negative')
# category_list = ['positive', 'negative']
# category_mapping = {
# 'positive': 0,
# 'negative': 1,
# }
# return general_confusion_matrix_algo(label_list, prediction_list, category_list,
# category_mapping, **kwargs)
# @register_ibs_method
# def detector_precision_recall_algo_display(ibs, min_overlap=0.5, figsize=(24, 7), **kwargs):
# import matplotlib.pyplot as plt
# fig_ = plt.figure(figsize=figsize)
# axes_ = plt.subplot(131)
# axes_.set_autoscalex_on(False)
# axes_.set_autoscaley_on(False)
# axes_.set_xlabel('Recall (Ground-Truth IOU >= %0.02f)' % (min_overlap, ))
# axes_.set_ylabel('Precision')
# axes_.set_xlim([0.0, 1.01])
# axes_.set_ylim([0.0, 1.01])
# kwargs_list = [
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.64,
# 'localizer_grid' : False,
# 'localizer_sensitivity' : 0.16,
# 'labeler_sensitivity' : 0.42,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.64,
# 'localizer_grid' : False,
# 'localizer_sensitivity' : 0.16,
# 'labeler_sensitivity' : 0.42,
# 'check_species' : True,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.64,
# 'localizer_grid' : False,
# 'localizer_sensitivity' : 0.16,
# 'labeler_sensitivity' : 0.42,
# 'check_viewpoint' : True,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.04,
# 'localizer_grid' : True,
# 'localizer_sensitivity' : 0.05,
# 'labeler_sensitivity' : 0.39,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.04,
# 'localizer_grid' : True,
# 'localizer_sensitivity' : 0.05,
# 'labeler_sensitivity' : 0.39,
# 'check_species' : True,
# },
# {
# 'min_overlap' : min_overlap,
# 'classifier_sensitivity' : 0.04,
# 'localizer_grid' : True,
# 'localizer_sensitivity' : 0.05,
# 'labeler_sensitivity' : 0.39,
# 'check_viewpoint' : True,
# },
# ]
# label_list = [
# 'Opt L',
# 'Opt L+S',
# 'Opt L+S+V',
# 'Rec L',
# 'Rec L+S',
# 'Rec L+S+V',
# ]
# color_list = [
# 'r',
# 'b',
# 'g',
# 'k',
# 'y',
# 'c',
# ]
# ret_list = [
# detector_precision_recall_algo_plot(ibs, label=label, color=color, **kwargs_)
# for label, color, kwargs_ in zip(label_list, color_list, kwargs_list)
# ]
# area_list = [ ret[0] for ret in ret_list ]
# conf_list = [ ret[1] for ret in ret_list ]
# index = np.argmax(area_list)
# best_label = label_list[index]
# best_kwargs = kwargs_list[index]
# best_area = area_list[index]
# best_conf = conf_list[index]
# plt.title('Precision-Recall Curve (Best: %s, AP = %0.02f)' % (best_label, best_area, ), y=1.20)
# # Display graph
# plt.legend(bbox_to_anchor=(0.0, 1.02, 1.0, .102), loc=3, ncol=2, mode="expand",
# borderaxespad=0.0)
# axes_ = plt.subplot(132)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = detector_confusion_matrix_algo_plot(ibs, 'V1', 'r', conf=best_conf, fig_=fig_, axes_=axes_, **best_kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('P-R Confusion Matrix (Algo: %s, OP = %0.02f)' % (best_label, best_conf, ), y=1.26)
# best_index = None
# best_conf = None
# best_pr = 0.0
# best_re = 0.0
# tup_list = [ ret[2] for ret in ret_list ]
# for index, tup in enumerate(tup_list):
# for conf, re, pr in zip(*tup):
# if pr > best_pr:
# best_index = index
# best_conf = conf
# best_pr = pr
# best_re = re
# if best_index is not None:
# axes_ = plt.subplot(131)
# plt.plot([best_re], [best_pr], 'yo')
# best_label = label_list[best_index]
# best_kwargs = kwargs_list[best_index]
# axes_ = plt.subplot(133)
# axes_.set_aspect(1)
# gca_ = plt.gca()
# gca_.grid(False)
# correct_rate, _ = detector_confusion_matrix_algo_plot(ibs, 'V1', 'r', conf=best_conf, fig_=fig_, axes_=axes_, **best_kwargs)
# axes_.set_xlabel('Predicted (Correct = %0.02f%%)' % (correct_rate * 100.0, ))
# axes_.set_ylabel('Ground-Truth')
# plt.title('P-R Confusion Matrix (Algo: %s, OP = %0.02f)' % (best_label, best_conf, ), y=1.26)
# # plt.show()
# fig_filename = 'detector-precision-recall-%0.2f.png' % (min_overlap, )
# fig_path = abspath(expanduser(join('~', 'Desktop', fig_filename)))
# plt.savefig(fig_path, bbox_inches='tight')
# @register_ibs_method
# def detector_metric_graphs(ibs, species_list=[]):
# ibs.classifier_precision_recall_algo_display(species_list)
# ibs.localizer_precision_recall_algo_display()
# ibs.labeler_precision_recall_algo_display()
# ibs.detector_precision_recall_algo_display()
if __name__ == '__main__':
"""
CommandLine:
python -m wbia.other.detectfuncs
python -m wbia.other.detectfuncs --allexamples
python -m wbia.other.detectfuncs --allexamples --noface --nosrc
"""
import multiprocessing
multiprocessing.freeze_support() # for win32
ut.doctest_funcs()