Python cntk.user_function() Examples

def create_proposal_layer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, cfg, use_native_proposal_layer=False):
    layer_config = {}
    layer_config["feat_stride"] = cfg["MODEL"].FEATURE_STRIDE
    layer_config["scales"] = cfg["DATA"].PROPOSAL_LAYER_SCALES

    layer_config["train_pre_nms_topN"] = cfg["TRAIN"].RPN_PRE_NMS_TOP_N
    layer_config["train_post_nms_topN"] = cfg["TRAIN"].RPN_POST_NMS_TOP_N
    layer_config["train_nms_thresh"] = float(cfg["TRAIN"].RPN_NMS_THRESH)
    layer_config["train_min_size"] = float(cfg["TRAIN"].RPN_MIN_SIZE)

    layer_config["test_pre_nms_topN"] = cfg["TEST"].RPN_PRE_NMS_TOP_N
    layer_config["test_post_nms_topN"] = cfg["TEST"].RPN_POST_NMS_TOP_N
    layer_config["test_nms_thresh"] = float(cfg["TEST"].RPN_NMS_THRESH)
    layer_config["test_min_size"] = float(cfg["TEST"].RPN_MIN_SIZE)

    if use_native_proposal_layer:
        cntk.ops.register_native_user_function('ProposalLayerOp',
                                               'Cntk.ProposalLayerLib-' + cntk.__version__.rstrip('+'),
                                               'CreateProposalLayer')
        rpn_rois_raw = ops.native_user_function('ProposalLayerOp', [rpn_cls_prob_reshape, rpn_bbox_pred, im_info],
                                                layer_config, 'native_proposal_layer')
    else:
        rpn_rois_raw = user_function(ProposalLayer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, layer_config))

    return alias(rpn_rois_raw, name='rpn_rois') 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def reshape(x, shape):
    if isinstance(x, C.variables.Parameter):
        return C.reshape(x, shape)
    else:
        num_dynamic_axis = _get_dynamic_axis_num(x)

        if num_dynamic_axis == 1 and len(shape) > 0 and shape[0] == -1:
            # collapse axis with batch axis
            if b_any(_ == C.InferredDimension for _ in x.shape) or b_any(
                    _ == C.FreeDimension for _ in x.shape):
                warnings.warn(
                    'Warning: CNTK backend does not support '
                    'collapse of batch axis with inferred dimension. '
                    'The reshape did not take place.')
                return x
            return C.user_function(ReshapeBatch(x, shape[1:]))
        else:
            # no collaps, then first need to padding the shape
            if num_dynamic_axis >= len(shape):
                i = 0
                while i < len(shape):
                    if shape[i] is None or shape[i] == -1:
                        i += 1
                    else:
                        break
                shape = tuple([-1 for _ in range(num_dynamic_axis - i)]) + shape

            new_shape = list(shape)
            new_shape = new_shape[num_dynamic_axis:]
            new_shape = [C.InferredDimension if _ is None else _ for _ in new_shape]
            return C.reshape(x, new_shape) 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def test_proposal_layer():
    cls_prob_shape_cntk = (18,61,61)
    cls_prob_shape_caffe = (18,61,61)
    rpn_bbox_shape = (36, 61, 61)
    dims_info_shape = (6,)
    im_info = [1000, 1000, 1]

    # Create input tensors with values
    cls_prob =  np.random.random_sample(cls_prob_shape_cntk).astype(np.float32)
    rpn_bbox_pred = np.random.random_sample(rpn_bbox_shape).astype(np.float32)
    dims_input = np.array([1000, 1000, 1000, 1000, 1000, 1000]).astype(np.float32)

    # Create CNTK layer and call forward
    cls_prob_var = input_variable(cls_prob_shape_cntk)
    rpn_bbox_var = input_variable(rpn_bbox_shape)
    dims_info_var = input_variable(dims_info_shape)

    cntk_layer = user_function(CntkProposalLayer(cls_prob_var, rpn_bbox_var, dims_info_var))
    state, cntk_output = cntk_layer.forward({cls_prob_var: [cls_prob], rpn_bbox_var: [rpn_bbox_pred], dims_info_var: dims_input})
    cntk_proposals = cntk_output[next(iter(cntk_output))][0]

    # Create Caffe layer and call forward
    cls_prob_caffe = cls_prob.reshape(cls_prob_shape_caffe)
    bottom = [np.array([cls_prob_caffe]),np.array([rpn_bbox_pred]),np.array([im_info])]
    top = None # handled through return statement in caffe layer for unit testing

    param_str = "'feat_stride': 16"
    caffe_layer = CaffeProposalLayer()
    caffe_layer.set_param_str(param_str)
    caffe_layer.setup(bottom, top)
    caffe_output = caffe_layer.forward(bottom, top)
    caffe_proposals = caffe_output[:,1:]

    # assert that results are exactly the same
    assert cntk_proposals.shape == caffe_proposals.shape
    assert np.allclose(cntk_proposals, caffe_proposals, rtol=0.0, atol=0.0)
    print("Verified ProposalLayer") 

def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, num_classes):
    '''
    Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
        Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
        "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"

    Assigns object detection proposals to ground-truth targets.
    Produces proposal classification labels and bounding-box regression targets.
    It also adds gt_boxes to candidates and samples fg and bg rois for training.

    Args:
        rpn_rois:        The proposed ROIs, e.g. from a region proposal network
        scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
        num_classes:     The number of classes in the data set

    Returns:
        rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
        label_targets - the target labels for the rois
        bbox_targets - the regression coefficient targets for the rois
        bbox_inside_weights - the weights for the regression loss
    '''

    ptl_param_string = "'num_classes': {}".format(num_classes)
    ptl = user_function(ProposalTargetLayer(rpn_rois, scaled_gt_boxes, param_str=ptl_param_string))

    # use an alias if you need to access the outputs, e.g., when cloning a trained network
    rois = alias(ptl.outputs[0], name='rpn_target_rois')
    label_targets = ptl.outputs[1]
    bbox_targets = ptl.outputs[2]
    bbox_inside_weights = ptl.outputs[3]

    return rois, label_targets, bbox_targets, bbox_inside_weights 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, num_classes):
    '''
    Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
        Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
        "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"

    Assigns object detection proposals to ground-truth targets.
    Produces proposal classification labels and bounding-box regression targets.
    It also adds gt_boxes to candidates and samples fg and bg rois for training.

    Args:
        rpn_rois:        The proposed ROIs, e.g. from a region proposal network
        scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
        num_classes:     The number of classes in the data set

    Returns:
        rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
        label_targets - the target labels for the rois
        bbox_targets - the regression coefficient targets for the rois
        bbox_inside_weights - the weights for the regression loss
    '''

    ptl_param_string = "'num_classes': {}".format(num_classes)
    ptl = user_function(ProposalTargetLayer(rpn_rois, scaled_gt_boxes, param_str=ptl_param_string))

    # use an alias if you need to access the outputs, e.g., when cloning a trained network
    rois = alias(ptl.outputs[0], name='rpn_target_rois')
    label_targets = ptl.outputs[1]
    bbox_targets = ptl.outputs[2]
    bbox_inside_weights = ptl.outputs[3]

    return rois, label_targets, bbox_targets, bbox_inside_weights 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, cfg):
    '''
    Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
        Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
        "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"

    Assigns object detection proposals to ground-truth targets.
    Produces proposal classification labels and bounding-box regression targets.
    It also adds gt_boxes to candidates and samples fg and bg rois for training.

    Args:
        rpn_rois:        The proposed ROIs, e.g. from a region proposal network
        scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
        num_classes:     The number of classes in the data set

    Returns:
        rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
        label_targets - the target labels for the rois
        bbox_targets - the regression coefficient targets for the rois
        bbox_inside_weights - the weights for the regression loss
    '''

    ptl_param_string = "'num_classes': {}".format(cfg["DATA"].NUM_CLASSES)
    ptl = user_function(ProposalTargetLayer(rpn_rois, scaled_gt_boxes,
                                            batch_size=cfg.NUM_ROI_PROPOSALS,
                                            fg_fraction=cfg["TRAIN"].FG_FRACTION,
                                            normalize_targets=cfg.BBOX_NORMALIZE_TARGETS,
                                            normalize_means=cfg.BBOX_NORMALIZE_MEANS,
                                            normalize_stds=cfg.BBOX_NORMALIZE_STDS,
                                            fg_thresh=cfg["TRAIN"].FG_THRESH,
                                            bg_thresh_hi=cfg["TRAIN"].BG_THRESH_HI,
                                            bg_thresh_lo=cfg["TRAIN"].BG_THRESH_LO,
                                            param_str=ptl_param_string))

    # use an alias if you need to access the outputs, e.g., when cloning a trained network
    rois = alias(ptl.outputs[0], name='rpn_target_rois')
    label_targets = ptl.outputs[1]
    bbox_targets = ptl.outputs[2]
    bbox_inside_weights = ptl.outputs[3]

    return rois, label_targets, bbox_targets, bbox_inside_weights 

def _reshape_batch(x, shape):
    # there is a bug in cntk 2.1's unpack_batch implementation
    if hasattr(C, 'unpack_batch') and _get_cntk_version() >= 2.2:
        const_a = C.unpack_batch(x)
        const_a = C.reshape(const_a, shape)
        return C.to_batch(const_a)
    else:
        return C.user_function(ReshapeBatch(x, shape[1:])) 

def print_tensor(x, message=''):
    return C.user_function(
        LambdaFunc(x,
                   when=lambda x: True,
                   execute=lambda x: print(message))) 

def test_proposal_layer():
    cls_prob_shape_cntk = (18,61,61)
    cls_prob_shape_caffe = (18,61,61)
    rpn_bbox_shape = (36, 61, 61)
    dims_info_shape = (6,)
    im_info = [1000, 1000, 1]

    # Create input tensors with values
    cls_prob =  np.random.random_sample(cls_prob_shape_cntk).astype(np.float32)
    rpn_bbox_pred = np.random.random_sample(rpn_bbox_shape).astype(np.float32)
    dims_input = np.array([1000, 1000, 1000, 1000, 1000, 1000]).astype(np.float32)

    # Create CNTK layer and call forward
    cls_prob_var = input_variable(cls_prob_shape_cntk)
    rpn_bbox_var = input_variable(rpn_bbox_shape)
    dims_info_var = input_variable(dims_info_shape)

    cntk_layer = user_function(CntkProposalLayer(cls_prob_var, rpn_bbox_var, dims_info_var))
    state, cntk_output = cntk_layer.forward({cls_prob_var: [cls_prob], rpn_bbox_var: [rpn_bbox_pred], dims_info_var: dims_input})
    cntk_proposals = cntk_output[next(iter(cntk_output))][0]

    # Create Caffe layer and call forward
    cls_prob_caffe = cls_prob.reshape(cls_prob_shape_caffe)
    bottom = [np.array([cls_prob_caffe]),np.array([rpn_bbox_pred]),np.array([im_info])]
    top = None # handled through return statement in caffe layer for unit testing

    param_str = "'feat_stride': 16"
    caffe_layer = CaffeProposalLayer()
    caffe_layer.set_param_str(param_str)
    caffe_layer.setup(bottom, top)
    caffe_output = caffe_layer.forward(bottom, top)
    caffe_proposals = caffe_output[:,1:]

    # assert that results are exactly the same
    assert cntk_proposals.shape == caffe_proposals.shape
    assert np.allclose(cntk_proposals, caffe_proposals, rtol=0.0, atol=0.0)
    print("Verified ProposalLayer") 

def test_anchor_target_layer():
    rpn_cls_score_shape_cntk = (1, 18, 61, 61)
    num_gt_boxes = 50
    gt_boxes_shape_cntk = (num_gt_boxes,5)
    dims_info_shape = (6,)
    im_info = [1000, 1000, 1]

    # Create input tensors with values
    rpn_cls_score_dummy = np.random.random_sample(rpn_cls_score_shape_cntk).astype(np.float32)
    dims_input = np.array([1000, 1000, 1000, 1000, 1000, 1000]).astype(np.float32)

    x1y1 = np.random.random_sample((num_gt_boxes, 2)) * 500
    wh = np.random.random_sample((num_gt_boxes, 2)) * 400
    x2y2 = x1y1 + wh + 50
    label = np.random.random_sample((num_gt_boxes, 1))
    label = (label * 17.0)
    gt_boxes = np.hstack((x1y1, x2y2, label)).astype(np.float32)

    # Create CNTK layer and call forward
    rpn_cls_score_var = input_variable(rpn_cls_score_shape_cntk)
    gt_boxes_var = input_variable(gt_boxes_shape_cntk)
    dims_info_var = input_variable(dims_info_shape)

    cntk_layer = user_function(CntkAnchorTargetLayer(rpn_cls_score_var, gt_boxes_var, dims_info_var, deterministic=True))
    state, cntk_output = cntk_layer.forward({rpn_cls_score_var: [rpn_cls_score_dummy], gt_boxes_var: [gt_boxes], dims_info_var: dims_input})

    obj_key = [k for k in cntk_output if 'objectness_target' in str(k)][0]
    bbt_key = [k for k in cntk_output if 'rpn_bbox_target' in str(k)][0]
    bbw_key = [k for k in cntk_output if 'rpn_bbox_inside_w' in str(k)][0]

    cntk_objectness_target = cntk_output[obj_key][0]
    cntk_bbox_targets = cntk_output[bbt_key][0]
    cntk_bbox_inside_w = cntk_output[bbw_key][0]

    # Create Caffe layer and call forward
    bottom = [np.array(rpn_cls_score_dummy),np.array(gt_boxes), np.array(im_info)]
    top = None # handled through return statement in caffe layer for unit testing

    param_str = "'feat_stride': 16"
    caffe_layer = CaffeAnchorTargetLayer()
    caffe_layer.set_param_str(param_str)
    caffe_layer.setup(bottom, top)
    caffe_layer.set_deterministic_mode()

    caffe_objectness_target, caffe_bbox_targets, caffe_bbox_inside_w = caffe_layer.forward(bottom, top)

    # assert that results are exactly the same
    assert cntk_objectness_target.shape == caffe_objectness_target.shape
    assert cntk_bbox_targets.shape == caffe_bbox_targets.shape
    assert cntk_bbox_inside_w.shape == caffe_bbox_inside_w.shape

    assert np.allclose(cntk_objectness_target, caffe_objectness_target, rtol=0.0, atol=0.0)
    assert np.allclose(cntk_bbox_targets, caffe_bbox_targets, rtol=0.0, atol=0.0)
    assert np.allclose(cntk_bbox_inside_w, caffe_bbox_inside_w, rtol=0.0, atol=0.0)
    print("Verified AnchorTargetLayer") 

def test_anchor_target_layer():
    rpn_cls_score_shape_cntk = (1, 18, 61, 61)
    num_gt_boxes = 50
    gt_boxes_shape_cntk = (num_gt_boxes,5)
    dims_info_shape = (6,)
    im_info = [1000, 1000, 1]

    # Create input tensors with values
    rpn_cls_score_dummy = np.random.random_sample(rpn_cls_score_shape_cntk).astype(np.float32)
    dims_input = np.array([1000, 1000, 1000, 1000, 1000, 1000]).astype(np.float32)

    x1y1 = np.random.random_sample((num_gt_boxes, 2)) * 500
    wh = np.random.random_sample((num_gt_boxes, 2)) * 400
    x2y2 = x1y1 + wh + 50
    label = np.random.random_sample((num_gt_boxes, 1))
    label = (label * 17.0)
    gt_boxes = np.hstack((x1y1, x2y2, label)).astype(np.float32)

    # Create CNTK layer and call forward
    rpn_cls_score_var = input_variable(rpn_cls_score_shape_cntk)
    gt_boxes_var = input_variable(gt_boxes_shape_cntk)
    dims_info_var = input_variable(dims_info_shape)

    cntk_layer = user_function(CntkAnchorTargetLayer(rpn_cls_score_var, gt_boxes_var, dims_info_var, deterministic=True))
    state, cntk_output = cntk_layer.forward({rpn_cls_score_var: [rpn_cls_score_dummy], gt_boxes_var: [gt_boxes], dims_info_var: dims_input})

    obj_key = [k for k in cntk_output if 'objectness_target' in str(k)][0]
    bbt_key = [k for k in cntk_output if 'rpn_bbox_target' in str(k)][0]
    bbw_key = [k for k in cntk_output if 'rpn_bbox_inside_w' in str(k)][0]

    cntk_objectness_target = cntk_output[obj_key][0]
    cntk_bbox_targets = cntk_output[bbt_key][0]
    cntk_bbox_inside_w = cntk_output[bbw_key][0]

    # Create Caffe layer and call forward
    bottom = [np.array(rpn_cls_score_dummy),np.array(gt_boxes), np.array(im_info)]
    top = None # handled through return statement in caffe layer for unit testing

    param_str = "'feat_stride': 16"
    caffe_layer = CaffeAnchorTargetLayer()
    caffe_layer.set_param_str(param_str)
    caffe_layer.setup(bottom, top)
    caffe_layer.set_deterministic_mode()

    caffe_objectness_target, caffe_bbox_targets, caffe_bbox_inside_w = caffe_layer.forward(bottom, top)

    # assert that results are exactly the same
    assert cntk_objectness_target.shape == caffe_objectness_target.shape
    assert cntk_bbox_targets.shape == caffe_bbox_targets.shape
    assert cntk_bbox_inside_w.shape == caffe_bbox_inside_w.shape

    assert np.allclose(cntk_objectness_target, caffe_objectness_target, rtol=0.0, atol=0.0)
    assert np.allclose(cntk_bbox_targets, caffe_bbox_targets, rtol=0.0, atol=0.0)
    assert np.allclose(cntk_bbox_inside_w, caffe_bbox_inside_w, rtol=0.0, atol=0.0)
    print("Verified AnchorTargetLayer")