Python fast_rcnn.bbox_transform.bbox_transform_inv() Examples

def _propagate_boxes(boxes, annot_proto, frame_id):
    pred_boxes = []
    annots = []
    for annot in annot_proto['annotations']:
        for idx, box in enumerate(annot['track']):
            if box['frame'] == frame_id and len(annot['track']) > idx + 1:
                gt1 = box['bbox']
                gt2 = annot['track'][idx+1]['bbox']
                delta = bbox_transform(np.asarray([gt1]), np.asarray([gt2]))
                annots.append((gt1, delta))
    gt1 = [annot[0] for annot in annots]
    overlaps = bbox_overlaps(np.require(boxes, dtype=np.float),
                             np.require(gt1, dtype=np.float))
    assert len(overlaps) == len(boxes)
    for gt_overlaps, box in zip(overlaps, boxes):
        max_overlap = np.max(gt_overlaps)
        max_gt = np.argmax(gt_overlaps)
        if max_overlap < 0.5:
            pred_boxes.append(box)
        else:
            delta = annots[max_gt][1]
            pred_boxes.append(bbox_transform_inv(np.asarray([box]), delta)[0].tolist())
    return pred_boxes 

def interpret_faster_rcnn(self, cls_prob, bbox_pred, rois, im_info, im_shape, nms=True, clip=True, min_score=0.0):
        # find class
        scores, inds = cls_prob.data.max(1)
        scores, inds = scores.cpu().numpy(), inds.cpu().numpy()

        keep = np.where((inds > 0) & (scores >= min_score))
        scores, inds = scores[keep], inds[keep]

        # Apply bounding-box regression deltas
        keep = keep[0]
        box_deltas = bbox_pred.data.cpu().numpy()[keep]
        box_deltas = np.asarray([
            box_deltas[i, (inds[i] * 4): (inds[i] * 4 + 4)] for i in range(len(inds))
        ], dtype=np.float)
        boxes = rois.data.cpu().numpy()[keep, 1:5] / im_info[0][2]
        pred_boxes = bbox_transform_inv(boxes, box_deltas)
        if clip:
            pred_boxes = clip_boxes(pred_boxes, im_shape)

        # nms
        if nms and pred_boxes.shape[0] > 0:
            pred_boxes, scores, inds = nms_detections(pred_boxes, scores, 0.3, inds=inds)

        return pred_boxes, scores, self.classes[inds] 

def tpn_test(vid_proto, box_proto, net, rnn_net, session, det_fun=im_detect, scheme='max', length=None,
        sample_rate=1, offset=0, cls_indices=None, batch_size=64):
    # same as roi_propagation except keep_feat is always True
    track_proto = roi_propagation(vid_proto, box_proto, net, det_fun=det_fun,
        scheme=scheme, length=length, sample_rate=sample_rate,
        offset=offset, cls_indices=cls_indices, batch_size=batch_size,
        keep_feat=True)

    print 'Running LSTM...'
    for track in track_proto['tracks']:
        feat = np.asarray([box['feature'] for box in track])
        track_length = len(track)
        expend_feat = np.zeros((rnn_net.num_steps,) + feat.shape[1:])
        expend_feat[:track_length] = feat

        # extract features
        state = session.run([rnn_net.initial_state])
        cls_scores, bbox_deltas, end_probs, state = session.run(
            [rnn_net.cls_scores, rnn_net.bbox_pred, rnn_net.end_probs,
            rnn_net.final_state],
            {rnn_net.input_data: expend_feat[np.newaxis,:,:],
             rnn_net.initial_state: state[0]})

        # process outputs
        rois = np.asarray([box['roi'] for box in track])
        bbox_pred = bbox_transform_inv(rois, bbox_deltas[:track_length,:])
        cls_pred_lstm = cls_scores[:track_length]
        end_probs = end_probs[:track_length]
        for box, cur_bbox_pred, cur_cls_pred_lstm, cur_end_prob in \
            zip(track, bbox_pred, cls_pred_lstm, end_probs):
            box['scores_lstm'] = cur_cls_pred_lstm.tolist()
            box['bbox_lstm'] = cur_bbox_pred.tolist()
            box['end_prob'] = float(cur_end_prob)
            del box['feature']
    return track_proto 

def _gt_propagate_boxes(boxes, annot_proto, frame_id, window, overlap_thres):
    pred_boxes = []
    annots = []
    for annot in annot_proto['annotations']:
        for idx, box in enumerate(annot['track']):
            if box['frame'] == frame_id:
                gt1 = box['bbox']
                deltas = []
                deltas.append(gt1)
                for offset in xrange(1, window):
                    try:
                        gt2 = annot['track'][idx+offset]['bbox']
                    except IndexError:
                        gt2 = gt1
                    delta = bbox_transform(np.asarray([gt1]), np.asarray([gt2]))
                    deltas.append(delta)
                annots.append(deltas)
    gt1s = [annot[0] for annot in annots]
    if not gt1s:
        # no grount-truth, boxes remain still
        return np.tile(np.asarray(boxes)[:,np.newaxis,:], [1,window-1,1])
    overlaps = bbox_overlaps(np.require(boxes, dtype=np.float),
                             np.require(gt1s, dtype=np.float))
    assert len(overlaps) == len(boxes)
    for gt_overlaps, box in zip(overlaps, boxes):
        max_overlap = np.max(gt_overlaps)
        max_gt = np.argmax(gt_overlaps)
        sequence_box = []
        if max_overlap < overlap_thres:
            for offset in xrange(1, window):
                sequence_box.append(box)
        else:
            for offset in xrange(1, window):
                delta = annots[max_gt][offset]
                sequence_box.append(
                    bbox_transform_inv(np.asarray([box]), delta)[0].tolist())
        pred_boxes.append((sequence_box))
    return np.asarray(pred_boxes) 

def _accuracy(track, gt):
    if len(track) < 2:
        return [], [], []
    abs_acc = []
    rel_acc = []
    ious = []
    st_frame = track[0]['frame']
    end_frame = track[-1]['frame']
    assert end_frame - st_frame + 1 == len(track)
    gt_seg = select_gt_segment(gt['track'], st_frame, end_frame)
    assert len(gt_seg) <= len(track)
    track_bbox1 = np.asarray([track[0]['bbox']])
    gt_bbox1 = np.asarray([gt_seg[0]])
    for track_box, gt_bbox in zip(track[1:len(gt_seg)], gt_seg[1:]):
        # current track box
        track_bbox = np.asarray([track_box['bbox']])
        # gt motion
        gt_delta = bbox_transform(gt_bbox1, np.asarray([gt_bbox]))
        # target is the first track_bbox with gt motion
        track_bbox_target = bbox_transform_inv(track_bbox1, gt_delta)
        abs_diff = np.abs(track_bbox - track_bbox_target)
        cur_iou = iou(track_bbox, track_bbox_target)
        width = track_bbox_target[0,2] - track_bbox_target[0,0]
        height = track_bbox_target[0,3] - track_bbox_target[0,1]
        rel_diff = abs_diff / (np.asarray([width, height, width, height]) + np.finfo(float).eps)
        abs_acc.extend(abs_diff.flatten().tolist())
        rel_acc.extend(rel_diff.flatten().tolist())
        ious.extend(cur_iou.flatten().tolist())
    return abs_acc, rel_acc, ious 

def im_detect(sess, net, inputs, im, boxes, bbox_reg, multi_iter):
    blobs, im_scales = _get_blobs(im, boxes)

    relations = []
    for i in range(boxes.shape[0]):
        for j in range(boxes.shape[0]):
            if i != j:
                relations.append([i, j])
    relations = np.array(relations, dtype=np.int32) # all possible combinations
    num_roi = blobs['rois'].shape[0]
    num_rel = relations.shape[0]

    inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)

    feed_dict = {inputs['ims']: blobs['data'],
                 inputs['rois']: blobs['rois'],
                 inputs['relations']: relations,
                 net.keep_prob: 1}

    for k in inputs_feed:
        feed_dict[inputs[k]] = inputs_feed[k]

    # compute relation rois
    feed_dict[inputs['rel_rois']] = \
        data_utils.compute_rel_rois(num_rel, blobs['rois'], relations)

    ops = {}

    ops['bbox_deltas'] = net.bbox_pred_output(multi_iter)
    ops['rel_probs'] = net.rel_pred_output(multi_iter)
    ops['cls_probs'] = net.cls_pred_output(multi_iter)

    ops_value = sess.run(ops, feed_dict=feed_dict)

    out_dict = {}
    for mi in multi_iter:
        rel_probs = None
        rel_probs_flat = ops_value['rel_probs'][mi]
        rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
        for i, rel in enumerate(relations):
            rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]

        cls_probs = ops_value['cls_probs'][mi]

        if bbox_reg:
            # Apply bounding-box regression deltas
            pred_boxes = bbox_transform_inv(boxes, ops_value['bbox_deltas'][mi])
            pred_boxes = clip_boxes(pred_boxes, im.shape)
        else:
            # Simply repeat the boxes, once for each class
            pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))

        out_dict[mi] = {'scores': cls_probs.copy(),
                        'boxes': pred_boxes.copy(),
                        'relations': rel_probs.copy()}
    return out_dict 

def tpn_caffe_test(vid_proto, box_proto, net, rnn_net, det_fun=im_detect,
        scheme='weighted', length=None,
        sample_rate=1, offset=0, cls_indices=None, batch_size=64):
    # same as roi_propagation except keep_feat is always True
    track_proto = roi_propagation(vid_proto, box_proto, net, det_fun=det_fun,
        scheme=scheme, length=length, sample_rate=sample_rate,
        offset=offset, cls_indices=cls_indices, batch_size=batch_size,
        keep_feat=True)

    print 'Running LSTM...'
    cont = np.ones((length,1))
    cont[0,:] = 0
    rnn_net.blobs['cont'].reshape(*cont.shape)
    rnn_net.blobs['cont'].data[...] = cont

    for track_idx, track in enumerate(track_proto['tracks'], start=1):
        feat = np.asarray([box['feature'] for box in track])
        track_length = len(track)
        expend_feat = np.zeros((length, 1) + feat.shape[1:])
        expend_feat[:track_length,0] = feat

        # extract features
        rnn_net.blobs['data'].reshape(*expend_feat.shape)
        rnn_net.blobs['data'].data[...] = expend_feat
        blobs_out = rnn_net.forward()
        bbox_deltas = blobs_out['bbox_pred']
        cls_scores = blobs_out['cls_prob']
        if 'end_prob' in blobs_out:
            end_probs = blobs_out['end_prob']
        else:
            end_probs = np.zeros((length,1))

        # process outputs
        rois = np.asarray([box['roi'] for box in track])
        bbox_pred = bbox_transform_inv(rois, bbox_deltas[:track_length,0,:])
        cls_pred_lstm = cls_scores[:track_length]
        end_probs = end_probs[:track_length]
        for box, cur_bbox_pred, cur_cls_pred_lstm, cur_end_prob in \
            zip(track, bbox_pred, cls_pred_lstm, end_probs):
            box['scores_lstm'] = cur_cls_pred_lstm.flatten().tolist()
            box['bbox_lstm'] = cur_bbox_pred.tolist()
            box['end_prob'] = float(cur_end_prob)
            del box['feature']
        if track_idx % 500 == 0:
            print "{} tracks processed.".format(track_idx)
    if track_idx % 500 != 0:
            print "{} tracks processed.".format(track_idx)
    return track_proto 

def forward(self, bottom, top):


        #assert bottom[0].data.shape[0] == 1, \
        #    'Only single item batches are supported'


        # the first set of _num_anchors channels are bg probs
        # the second set are the fg probs, which we want
        rpn_boxes_Actual = bottom[0].data
        box_deltas = bottom[1].data
	im_info = bottom[2].data
	scores = bottom[3].data


	#print('im_info : ', im_info)

	im_scales = im_info[0][2]

	im_shape = np.array([im_info[0][0], im_info[0][1]]) / im_scales
	#print('conv_feat : ', conv_feat.shape)
	#print('rpnBoxes : ', rpn_boxes.shape)
	

	#for idx in range(len(cls_idx)):
		#cls_boxes = final_boxes[inds, j*4:(j+1)*4]


	rpn_boxes = rpn_boxes_Actual[:, 1:5] / im_scales
	pred_boxes = bbox_transform_inv(rpn_boxes, box_deltas)
        pred_boxes = clip_boxes(pred_boxes, im_shape)

	cls_idx = np.argmax(scores, axis = 1)

	#print('cls_idx', cls_idx.shape, cls_idx )

	#cls_idx = cls_idx.reshape(len(cls_idx), 1)
	#print('cls_idx', cls_idx.shape)
	#pred_boxes = pred_boxes[:, cls_idx*4:(cls_idx+1)*4]
	temp = np.zeros((len(cls_idx), 5))

	for idx in range(len(cls_idx)):
		#print(cls_idx[idx])
		temp[idx,1:] = pred_boxes[idx, cls_idx[idx]*4:(cls_idx[idx]+1)*4]

	
	pred_boxes = temp * im_scales
	#addd = cls_idx >0
	#print('Compare :', rpn_boxes[cls_idx>0,:], temp[cls_idx>0,:])

	#rpn_boxes_Actual[:,1:5] = pred_boxes

	top[0].reshape(*(pred_boxes.shape))
        top[0].data[...] = pred_boxes