Python steppy.base.IdentityOperation() Examples
The following are 10
code examples of steppy.base.IdentityOperation().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
steppy.base
, or try the search function
.
.
Example #1
| Source File: test_base.py From steppy with MIT License | 6 votes |
|
def test_inputs_without_conflicting_names_do_not_require_adapter(data):
step = Step(
name='test_inputs_without_conflicting_names_do_not_require_adapter_1',
transformer=IdentityOperation(),
input_data=['input_1']
)
output = step.fit_transform(data)
assert output == data['input_1']
step = Step(
name='test_inputs_without_conflicting_names_do_not_require_adapter_2',
transformer=IdentityOperation(),
input_data=['input_1', 'input_2']
)
output = step.fit_transform(data)
assert output == {**data['input_1'], **data['input_2']}
Example #2
| Source File: test_base.py From steppy with MIT License | 6 votes |
|
def test_step_with_adapted_inputs(data):
step = Step(
name='test_step_wit_adapted_inputs',
transformer=IdentityOperation(),
input_data=['input_1', 'input_3'],
adapter=Adapter({
'img': E('input_3', 'images'),
'fea': E('input_1', 'features'),
'l1': E('input_3', 'labels'),
'l2': E('input_1', 'labels'),
})
)
output = step.fit_transform(data)
expected = {
'img': data['input_3']['images'],
'fea': data['input_1']['features'],
'l1': data['input_3']['labels'],
'l2': data['input_1']['labels'],
}
assert output == expected
Example #3
| Source File: callbacks.py From open-solution-salt-identification with MIT License | 5 votes |
|
def postprocessing_pipeline_simplified(cache_dirpath, loader_mode, threshold):
if loader_mode == 'resize_and_pad':
size_adjustment_function = partial(crop_image, target_size=ORIGINAL_SIZE)
elif loader_mode == 'resize' or loader_mode == 'stacking':
size_adjustment_function = partial(resize_image, target_size=ORIGINAL_SIZE)
else:
raise NotImplementedError
mask_resize = Step(name='mask_resize',
transformer=make_apply_transformer(size_adjustment_function,
output_name='resized_images',
apply_on=['images']),
input_data=['network_output'],
adapter=Adapter({'images': E('network_output', 'mask_prediction'),
}),
experiment_directory=cache_dirpath)
binarizer = Step(name='binarizer',
transformer=make_apply_transformer(
partial(binarize, threshold=threshold),
output_name='binarized_images',
apply_on=['images']),
input_steps=[mask_resize],
adapter=Adapter({'images': E(mask_resize.name, 'resized_images'),
}),
experiment_directory=cache_dirpath)
output = Step(name='output',
transformer=IdentityOperation(),
input_steps=[binarizer],
adapter=Adapter({'y_pred': E(binarizer.name, 'binarized_images'),
}),
experiment_directory=cache_dirpath)
return output
Example #4
| Source File: pipeline_blocks.py From open-solution-home-credit with MIT License | 5 votes |
|
def oof_predictions(config, train_mode, suffix, **kwargs):
features = Step(name='oof_predictions{}'.format(suffix),
transformer=IdentityOperation(),
input_data=['oof_predictions'],
adapter=Adapter({'numerical_features': E('oof_predictions', 'X')
}),
experiment_directory=config.pipeline.experiment_directory, **kwargs)
feature_combiner = _join_features(numerical_features=[features],
categorical_features=[],
config=config, train_mode=train_mode, suffix=suffix, **kwargs)
if train_mode:
features_valid = Step(name='oof_predictions{}'.format(suffix),
transformer=IdentityOperation(),
input_data=['oof_predictions'],
adapter=Adapter({'numerical_features': E('oof_predictions', 'X_valid')
}),
experiment_directory=config.pipeline.experiment_directory, **kwargs)
feature_combiner_valid = _join_features(numerical_features=[features_valid],
categorical_features=[],
config=config, train_mode=train_mode, suffix='_valid{}'.format(suffix),
**kwargs)
return feature_combiner, feature_combiner_valid
else:
return feature_combiner
Example #5
| Source File: pipelines.py From open-solution-googleai-object-detection with MIT License | 5 votes |
|
def retinanet(config, train_mode, visualize=False):
persist_output = False
load_persisted_output = False
loader = preprocessing_generator(config, is_train=train_mode)
retinanet = Step(name='retinanet',
transformer=Retina(**config.retinanet, train_mode=train_mode),
input_steps=[loader],
experiment_directory=config.env.cache_dirpath,
persist_output=persist_output,
is_trainable=True,
load_persisted_output=load_persisted_output)
if train_mode:
return retinanet
if visualize:
return visualizer(retinanet, loader.get_step('label_encoder'), config)
postprocessor = postprocessing(retinanet, loader.get_step('label_encoder'), config)
output = Step(name='output',
transformer=IdentityOperation(),
input_steps=[postprocessor],
adapter=Adapter({'y_pred': E(postprocessor.name, 'submission')}),
experiment_directory=config.env.cache_dirpath,
persist_output=persist_output,
load_persisted_output=load_persisted_output)
return output
Example #6
| Source File: test_base.py From steppy with MIT License | 5 votes |
|
def test_inputs_with_conflicting_names_require_adapter(data):
step = Step(
name='test_inputs_with_conflicting_names_require_adapter',
transformer=IdentityOperation(),
input_data=['input_1', 'input_3']
)
with pytest.raises(StepError):
step.fit_transform(data)
Example #7
| Source File: main.py From open-solution-salt-identification with MIT License | 4 votes |
|
def network_tta(config, suffix=''):
if SECOND_LEVEL:
raise NotImplementedError('Second level does not work with TTA')
preprocessing, tta_generator = pipelines.preprocessing_inference_tta(config, model_name='network')
if USE_DEPTH:
Network = models.SegmentationModelWithDepth
else:
Network = models.SegmentationModel
network = Step(name='network{}'.format(suffix),
transformer=Network(**config.model['network']),
input_data=['callback_input'],
input_steps=[preprocessing],
is_trainable=True,
experiment_directory=config.execution.experiment_dir)
tta_aggregator = pipelines.aggregator('tta_aggregator{}'.format(suffix), network,
tta_generator=tta_generator,
experiment_directory=config.execution.experiment_dir,
config=config.tta_aggregator)
prediction_renamed = Step(name='prediction_renamed{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[tta_aggregator],
adapter=Adapter({'mask_prediction': E(tta_aggregator.name, 'aggregated_prediction')
}),
experiment_directory=config.execution.experiment_dir)
if config.general.loader_mode == 'resize_and_pad':
size_adjustment_function = partial(postprocessing.crop_image, target_size=config.general.original_size)
elif config.general.loader_mode == 'resize' or config.general.loader_mode == 'stacking':
size_adjustment_function = partial(postprocessing.resize_image, target_size=config.general.original_size)
else:
raise NotImplementedError
mask_resize = Step(name='mask_resize{}'.format(suffix),
transformer=utils.make_apply_transformer(size_adjustment_function,
output_name='resized_images',
apply_on=['images']),
input_steps=[prediction_renamed],
adapter=Adapter({'images': E(prediction_renamed.name, 'mask_prediction'),
}),
experiment_directory=config.execution.experiment_dir)
return mask_resize
# __________ ___ _______ ______ __ __ .___________. __ ______ .__ __.
# | ____\ \ / / | ____| / || | | | | || | / __ \ | \ | |
# | |__ \ V / | |__ | ,----'| | | | `---| |----`| | | | | | | \| |
# | __| > < | __| | | | | | | | | | | | | | | | . ` |
# | |____ / . \ | |____ | `----.| `--' | | | | | | `--' | | |\ |
# |_______/__/ \__\ |_______| \______| \______/ |__| |__| \______/ |__| \__|
#
Example #8
| Source File: pipeline_blocks.py From open-solution-home-credit with MIT License | 4 votes |
|
def catboost_preprocessing(features, config, train_mode, suffix, **kwargs):
if train_mode:
features, features_valid = features
fillnaer = Step(name='fillna{}'.format(suffix),
transformer=_fillna(**config.sklearn_preprocessing.fillna),
input_steps=[features],
adapter=Adapter({'X': E(features.name, 'features')}),
experiment_directory=config.pipeline.experiment_directory,
)
preprocessed = Step(name='preprocess{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[fillnaer, features],
adapter=Adapter({'features': E(fillnaer.name, 'transformed'),
'feature_names': E(features.name, 'feature_names'),
'categorical_features': E(features.name, 'categorical_features')
}),
experiment_directory=config.pipeline.experiment_directory,
**kwargs
)
if train_mode:
fillnaer_valid = Step(name='fillna_valid{}'.format(suffix),
transformer=fillnaer,
input_steps=[features_valid],
adapter=Adapter({'X': E(features_valid.name, 'features')}),
experiment_directory=config.pipeline.experiment_directory,
)
preprocessed_valid = Step(name='preprocess_valid{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[fillnaer_valid, features_valid],
adapter=Adapter({'features': E(fillnaer_valid.name, 'transformed'),
'feature_names': E(features_valid.name,
'feature_names'),
'categorical_features': E(features_valid.name,
'categorical_features')
}),
experiment_directory=config.pipeline.experiment_directory,
**kwargs
)
return preprocessed, preprocessed_valid
else:
return preprocessed
Example #9
| Source File: pipeline_blocks.py From open-solution-home-credit with MIT License | 4 votes |
|
def stacking_normalization(features, config, train_mode, suffix, **kwargs):
if train_mode:
features, features_valid = features
normalizer = Step(name='stacking_normalizer{}'.format(suffix),
transformer=Normalizer(),
input_steps=[features],
adapter=Adapter({'X': E(features.name, 'features')}),
experiment_directory=config.pipeline.experiment_directory,
)
stacking_normalized = Step(name='stacking_normalization{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[normalizer, features],
adapter=Adapter({'features': E(normalizer.name, 'X'),
'feature_names': E(features.name, 'feature_names'),
'categorical_features': E(features.name, 'categorical_features')
}),
experiment_directory=config.pipeline.experiment_directory,
**kwargs
)
if train_mode:
normalizer_valid = Step(name='stacking_normalizer_valid{}'.format(suffix),
transformer=normalizer,
input_steps=[features_valid],
adapter=Adapter({'X': E(features_valid.name, 'features')}),
experiment_directory=config.pipeline.experiment_directory,
)
stacking_normalized_valid = Step(name='stacking_normalization_valid{}'.format(suffix),
transformer=IdentityOperation(),
input_steps=[normalizer_valid, features_valid],
adapter=Adapter({'features': E(normalizer_valid.name, 'X'),
'feature_names': E(features_valid.name, 'feature_names'),
'categorical_features': E(features_valid.name,
'categorical_features')
}),
experiment_directory=config.pipeline.experiment_directory,
**kwargs
)
return stacking_normalized, stacking_normalized_valid
else:
return stacking_normalized
Example #10
| Source File: callbacks.py From open-solution-googleai-object-detection with MIT License | 4 votes |
|
def postprocessing_pipeline_simplified(cache_dirpath):
mask_resize = Step(name='mask_resize',
transformer=make_apply_transformer(post.resize_image,
output_name='resized_images',
apply_on=['images', 'target_sizes']),
input_data=['unet_output', 'callback_input'],
adapter={'images': ([('unet_output', 'multichannel_map_prediction')]),
'target_sizes': ([('callback_input', 'target_sizes')]),
},
cache_dirpath=cache_dirpath)
category_mapper = Step(name='category_mapper',
transformer=make_apply_transformer(post.categorize_image,
output_name='categorized_images'),
input_steps=[mask_resize],
adapter={'images': ([('mask_resize', 'resized_images')]),
},
cache_dirpath=cache_dirpath)
labeler = Step(name='labeler',
transformer=make_apply_transformer(post.label_multiclass_image,
output_name='labeled_images'),
input_steps=[category_mapper],
adapter={'images': ([(category_mapper.name, 'categorized_images')]),
},
cache_dirpath=cache_dirpath)
score_builder = Step(name='score_builder',
transformer=make_apply_transformer(post.build_score,
output_name='images_with_scores',
apply_on=['images', 'probabilities']),
input_steps=[labeler, mask_resize],
adapter={'images': ([(labeler.name, 'labeled_images')]),
'probabilities': ([(mask_resize.name, 'resized_images')]),
},
cache_dirpath=cache_dirpath)
output = Step(name='output',
transformer=IdentityOperation(),
input_steps=[score_builder],
adapter={'y_pred': ([(score_builder.name, 'images_with_scores')]),
},
cache_dirpath=cache_dirpath)
return output
