Python steppy.base.Step() Examples

def test_step_with_adapted_inputs(data):
    step = Step(
        name='test_step_wit_adapted_inputs',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_3'],
        adapter=Adapter({
            'img': E('input_3', 'images'),
            'fea': E('input_1', 'features'),
            'l1': E('input_3', 'labels'),
            'l2': E('input_1', 'labels'),
        })
    )
    output = step.fit_transform(data)
    expected = {
        'img': data['input_3']['images'],
        'fea': data['input_1']['features'],
        'l1': data['input_3']['labels'],
        'l2': data['input_1']['labels'],
    }
    assert output == expected 

def stacking_preprocessing_inference(config, model_name='network', suffix=''):
    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=False, **config.xy_splitter[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.ImageSegmentationLoaderStacking(train_mode=False, **config.loaders.stacking),
                  input_steps=[reader_inference],
                  adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                   'y': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir,
                  cache_output=True)
    return loader 

def stacking_preprocessing_train(config, model_name='network', suffix=''):
    reader_train = Step(name='xy_train{}'.format(suffix),
                        transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                        input_data=['input'],
                        adapter=Adapter({'meta': E('input', 'meta')}),
                        experiment_directory=config.execution.experiment_dir)

    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                            input_data=['callback_input'],
                            adapter=Adapter({'meta': E('callback_input', 'meta_valid')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.ImageSegmentationLoaderStacking(train_mode=True, **config.loaders.stacking),
                  input_steps=[reader_train, reader_inference],
                  adapter=Adapter({'X': E(reader_train.name, 'X'),
                                   'y': E(reader_train.name, 'y'),
                                   'X_valid': E(reader_inference.name, 'X'),
                                   'y_valid': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir)
    return loader 

def emptiness_preprocessing_train(config, model_name='network', suffix=''):
    reader_train = Step(name='xy_train{}'.format(suffix),
                        transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                        input_data=['input'],
                        adapter=Adapter({'meta': E('input', 'meta')}),
                        experiment_directory=config.execution.experiment_dir)

    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=True, **config.xy_splitter[model_name]),
                            input_data=['callback_input'],
                            adapter=Adapter({'meta': E('callback_input', 'meta_valid')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.EmptinessLoader(train_mode=True, **config.loaders.resize),
                  input_steps=[reader_train, reader_inference],
                  adapter=Adapter({'X': E(reader_train.name, 'X'),
                                   'y': E(reader_train.name, 'y'),
                                   'X_valid': E(reader_inference.name, 'X'),
                                   'y_valid': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir)
    return loader 

def preprocessing_inference(config, model_name='network'):
    if config.general.loader_mode == 'resize':
        loader_config = config.loaders.resize
        LOADER = loaders.ImageSegmentationLoaderResize
    else:
        raise NotImplementedError

    reader_inference = Step(name='xy_inference',
                            transformer=loaders.MetaReader(train_mode=False, **config.meta_reader[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}))

    loader = Step(name='loader',
                  transformer=LOADER(train_mode=False, **loader_config),
                  input_steps=[reader_inference],
                  adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                   'y': E(reader_inference.name, 'y'),
                                   }))
    return loader 

def preprocessing_binary_inference(config, model_name, suffix='_binary_model'):
    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.MetaReader(train_mode=True, **config.meta_reader[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}))

    transformer = OneClassImageClassificatioLoader(
        train_mode=True,
        loader_params=config.loaders.resize.loader_params,
        dataset_params=config.loaders.resize.dataset_params,
        augmentation_params=config.loaders.resize.augmentation_params
    )

    binary_loader = Step(name='loader{}'.format(suffix),
                         transformer=transformer,
                         input_steps=[reader_inference],
                         adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                          }))

    return binary_loader 

def data_cleaning_v2(config, train_mode, suffix, **kwargs):
    cleaned_data = data_cleaning_v1(config, train_mode, suffix, **kwargs)

    if train_mode:
        cleaned_data, cleaned_data_valid = cleaned_data

    impute_missing = Step(name='dummies_missing{}'.format(suffix),
                          transformer=dc.DummiesMissing(**config.dummies_missing),
                          input_steps=[cleaned_data],
                          adapter=Adapter({'X': E(cleaned_data.name, 'numerical_features')}),
                          experiment_directory=config.pipeline.experiment_directory, **kwargs)

    if train_mode:
        impute_missing_valid = Step(name='dummies_missing_valid{}'.format(suffix),
                                    transformer=impute_missing,
                                    input_steps=[cleaned_data_valid],
                                    adapter=Adapter({'X': E(cleaned_data_valid.name, 'numerical_features')}),
                                    experiment_directory=config.pipeline.experiment_directory, **kwargs)
        return impute_missing, impute_missing_valid
    else:
        return impute_missing 

def classifier_catboost(features, config, train_mode, suffix, **kwargs):
    model_name = 'catboost{}'.format(suffix)

    if train_mode:
        features_train, features_valid = features

        catboost = Step(name=model_name,
                        transformer=CatBoost(**config.catboost),
                        input_data=['main_table'],
                        input_steps=[features_train, features_valid],
                        adapter=Adapter({'X': E(features_train.name, 'features'),
                                         'y': E('main_table', 'y'),
                                         'feature_names': E(features_train.name, 'feature_names'),
                                         'categorical_features': E(features_train.name, 'categorical_features'),
                                         'X_valid': E(features_valid.name, 'features'),
                                         'y_valid': E('main_table', 'y_valid'),
                                         }),
                        experiment_directory=config.pipeline.experiment_directory, **kwargs)
    else:
        catboost = Step(name=model_name,
                        transformer=CatBoost(**config.catboost),
                        input_steps=[features],
                        adapter=Adapter({'X': E(features.name, 'features')}),
                        experiment_directory=config.pipeline.experiment_directory, **kwargs)
    return catboost 

def classifier_xgb(features, config, train_mode, suffix, **kwargs):
    if train_mode:
        features_train, features_valid = features

        xgboost = Step(name='xgboost{}'.format(suffix),
                       transformer=XGBoost(**config.xgboost),
                       input_data=['main_table'],
                       input_steps=[features_train, features_valid],
                       adapter=Adapter({'X': E(features_train.name, 'features'),
                                        'y': E('main_table', 'y'),
                                        'feature_names': E(features_train.name, 'feature_names'),
                                        'X_valid': E(features_valid.name, 'features'),
                                        'y_valid': E('main_table', 'y_valid'),
                                        }),
                       experiment_directory=config.pipeline.experiment_directory,
                       **kwargs)
    else:
        xgboost = Step(name='xgboost{}'.format(suffix),
                       transformer=XGBoost(**config.xgboost),
                       input_steps=[features],
                       adapter=Adapter({'X': E(features.name, 'features')}),
                       experiment_directory=config.pipeline.experiment_directory,
                       **kwargs)
    return xgboost 

def visualizer(model, label_encoder, config):
    label_decoder = Step(name='label_decoder',
                         transformer=GoogleAiLabelDecoder(),
                         input_steps=[label_encoder, ],
                         experiment_directory=config.env.cache_dirpath)

    decoder = Step(name='decoder',
                   transformer=DataDecoder(**config.postprocessing.data_decoder),
                   input_data=['input'],
                   input_steps=[model, ],
                   experiment_directory=config.env.cache_dirpath)

    visualize = Step(name='visualizer',
                     transformer=Visualizer(),
                     input_steps=[label_decoder, decoder],
                     input_data=['input'],
                     adapter=Adapter({'images_data': E('input', 'images_data'),
                                      'results': E(decoder.name, 'results'),
                                      'decoder_dict': E(label_decoder.name, 'inverse_mapping')}),
                     experiment_directory=config.env.cache_dirpath)

    return visualize 

def row_aggregation_features(config, train_mode, suffix, **kwargs):
    bucket_nrs = config.row_aggregations.bucket_nrs
    row_agg_features = []
    for bucket_nr in bucket_nrs:
        row_agg_feature = Step(name='row_agg_feature_bucket_nr{}{}'.format(bucket_nr, suffix),
                               transformer=fe.RowAggregationFeatures(bucket_nr=bucket_nr),
                               input_data=['input'],
                               adapter=Adapter({'X': E('input', 'X')}),
                               experiment_directory=config.pipeline.experiment_directory, **kwargs)
        row_agg_features.append(row_agg_feature)

    if train_mode:
        row_agg_features_valid = []
        for bucket_nr, row_agg_feature in zip(bucket_nrs, row_agg_features):
            row_agg_feature_valid = Step(name='row_agg_feature_bucket_nr{}_valid{}'.format(bucket_nr, suffix),
                                         transformer=row_agg_feature,
                                         input_data=['input'],
                                         adapter=Adapter({'X': E('input', 'X_valid')}),
                                         experiment_directory=config.pipeline.experiment_directory, **kwargs)
            row_agg_features_valid.append(row_agg_feature_valid)

        return row_agg_features, row_agg_features_valid
    else:
        return row_agg_features 

def postprocessing(model, label_encoder, config):
    label_decoder = Step(name='label_decoder',
                         transformer=GoogleAiLabelDecoder(),
                         input_steps=[label_encoder, ],
                         experiment_directory=config.env.cache_dirpath)

    decoder = Step(name='decoder',
                   transformer=DataDecoder(**config.postprocessing.data_decoder),
                   input_data=['input'],
                   input_steps=[model, ],
                   experiment_directory=config.env.cache_dirpath)

    submission_producer = Step(name='submission_producer',
                               transformer=PredictionFormatter(),
                               input_steps=[label_decoder, decoder],
                               input_data=['input'],
                               adapter=Adapter({'images_data': E('input', 'images_data'),
                                                'results': E(decoder.name, 'results'),
                                                'decoder_dict': E(label_decoder.name, 'inverse_mapping')}),
                               experiment_directory=config.env.cache_dirpath)
    return submission_producer 

def test_inputs_without_conflicting_names_do_not_require_adapter(data):
    step = Step(
        name='test_inputs_without_conflicting_names_do_not_require_adapter_1',
        transformer=IdentityOperation(),
        input_data=['input_1']
    )
    output = step.fit_transform(data)
    assert output == data['input_1']

    step = Step(
        name='test_inputs_without_conflicting_names_do_not_require_adapter_2',
        transformer=IdentityOperation(),
        input_data=['input_1', 'input_2']
    )
    output = step.fit_transform(data)
    assert output == {**data['input_1'], **data['input_2']} 

def emptiness_preprocessing_inference(config, model_name='network', suffix=''):
    reader_inference = Step(name='xy_inference{}'.format(suffix),
                            transformer=loaders.XYSplit(train_mode=False, **config.xy_splitter[model_name]),
                            input_data=['input'],
                            adapter=Adapter({'meta': E('input', 'meta')}),
                            experiment_directory=config.execution.experiment_dir)

    loader = Step(name='loader{}'.format(suffix),
                  transformer=loaders.EmptinessLoader(train_mode=False, **config.loaders.resize),
                  input_steps=[reader_inference],
                  adapter=Adapter({'X': E(reader_inference.name, 'X'),
                                   'y': E(reader_inference.name, 'y'),
                                   }),
                  experiment_directory=config.execution.experiment_dir,
                  cache_output=True)
    return loader 

def _installment_payments_cleaning(config, **kwargs):
    installment_payments_cleaning = Step(name='installment_payments_cleaning',
                                         transformer=dc.InstallmentPaymentsCleaning(
                                             **config.preprocessing.impute_missing),
                                         input_data=['installments_payments'],
                                         adapter=Adapter({'installments': E('installments_payments', 'X')}),
                                         experiment_directory=config.pipeline.experiment_directory,
                                         **kwargs)

    return installment_payments_cleaning 

def retinanet(config, train_mode, visualize=False):
    persist_output = False
    load_persisted_output = False

    loader = preprocessing_generator(config, is_train=train_mode)

    retinanet = Step(name='retinanet',
                     transformer=Retina(**config.retinanet, train_mode=train_mode),
                     input_steps=[loader],
                     experiment_directory=config.env.cache_dirpath,
                     persist_output=persist_output,
                     is_trainable=True,
                     load_persisted_output=load_persisted_output)

    if train_mode:
        return retinanet

    if visualize:
        return visualizer(retinanet, loader.get_step('label_encoder'), config)

    postprocessor = postprocessing(retinanet, loader.get_step('label_encoder'), config)

    output = Step(name='output',
                  transformer=IdentityOperation(),
                  input_steps=[postprocessor],
                  adapter=Adapter({'y_pred': E(postprocessor.name, 'submission')}),
                  experiment_directory=config.env.cache_dirpath,
                  persist_output=persist_output,
                  load_persisted_output=load_persisted_output)
    return output 

def _installment_payments(installment_payments_cleaned, config, **kwargs):
    installment_payments_hand_crafted = Step(name='installment_payments_hand_crafted',
                                             transformer=fe.InstallmentPaymentsFeatures(**config.installments_payments),
                                             input_steps=[installment_payments_cleaned],
                                             adapter=Adapter({'installments': E(installment_payments_cleaned.name,
                                                                                'installments')}),
                                             experiment_directory=config.pipeline.experiment_directory,
                                             **kwargs)

    return installment_payments_hand_crafted 

def preprocessing_generator(config, is_train):
    label_encoder = Step(name='label_encoder',
                         transformer=GoogleAiLabelEncoder(**config.label_encoder),
                         input_data=['metadata'],
                         adapter=Adapter({'annotations': E('metadata', 'annotations'),
                                          'annotations_human_labels': E('metadata', 'annotations_human_labels')
                                          }),
                         is_trainable=True,
                         experiment_directory=config.env.cache_dirpath)

    if is_train:
        loader = Step(name='loader',
                      transformer=ImageDetectionLoader(train_mode=True, **config.loader),
                      input_data=['input', 'validation_input'],
                      input_steps=[label_encoder],
                      adapter=Adapter({'images_data': E('input', 'images_data'),
                                       'valid_images_data': E('validation_input', 'valid_images_data'),
                                       'annotations': E(label_encoder.name, 'annotations'),
                                       'annotations_human_labels': E(label_encoder.name, 'annotations_human_labels'),
                                       }),
                      experiment_directory=config.env.cache_dirpath)

    else:
        loader = Step(name='loader',
                      transformer=ImageDetectionLoader(train_mode=False, **config.loader),
                      input_data=['input'],
                      input_steps=[label_encoder],
                      adapter=Adapter({'images_data': E('input', 'images_data'),
                                       'annotations': None,
                                       'annotations_human_labels': None,
                                       }),
                      experiment_directory=config.env.cache_dirpath)
    return loader 

def _bureau_balance(bureau_balance_cleaned, config, **kwargs):
    bureau_balance_hand_crafted = Step(name='bureau_balance_hand_crafted',
                                       transformer=fe.BureauBalanceFeatures(**config.bureau_balance),
                                       input_steps=[bureau_balance_cleaned],
                                       adapter=Adapter({'bureau_balance': E(bureau_balance_cleaned.name,
                                                                            'bureau_balance')}),
                                       experiment_directory=config.pipeline.experiment_directory, **kwargs)

    return bureau_balance_hand_crafted 

def _bureau_balance_cleaning(config, **kwargs):
    bureau_cleaning = Step(name='bureau_balance_cleaning',
                           transformer=dc.BureauBalanceCleaning(**config.preprocessing.impute_missing),
                           input_data=['bureau_balance'],
                           adapter=Adapter({'bureau_balance': E('bureau_balance', 'X')}),
                           experiment_directory=config.pipeline.experiment_directory,
                           **kwargs)

    return bureau_cleaning 

def _bureau(bureau_cleaned, config, **kwargs):
    bureau_hand_crafted = Step(name='bureau_hand_crafted',
                               transformer=fe.BureauFeatures(**config.bureau),
                               input_steps=[bureau_cleaned],
                               adapter=Adapter({'bureau': E(bureau_cleaned.name, 'bureau')}),
                               experiment_directory=config.pipeline.experiment_directory,
                               **kwargs)

    return bureau_hand_crafted 

def _application(application_cleaned, config, **kwargs):
    application_hand_crafted = Step(name='application_hand_crafted',
                                    transformer=fe.ApplicationFeatures(**config.applications),
                                    input_steps=[application_cleaned],
                                    adapter=Adapter({'application': E(application_cleaned.name, 'application')}),
                                    experiment_directory=config.pipeline.experiment_directory,
                                    **kwargs)

    return application_hand_crafted 

def _application_cleaning(config, **kwargs):
    application_cleaning = Step(name='application_cleaning',
                                transformer=dc.ApplicationCleaning(**config.preprocessing.impute_missing),
                                input_data=['application'],
                                adapter=Adapter({'application': E('application', 'X')}),
                                experiment_directory=config.pipeline.experiment_directory,
                                **kwargs)

    return application_cleaning 

def _previous_applications_groupby_agg(previous_application_cleaned, config, **kwargs):
    previous_applications_groupby_agg = Step(name='previous_applications_groupby_agg',
                                             transformer=fe.GroupbyAggregate(**config.previous_applications),
                                             input_steps=[previous_application_cleaned],
                                             adapter=Adapter({'table': E(previous_application_cleaned.name,
                                                                         'previous_application')}),
                                             experiment_directory=config.pipeline.experiment_directory, **kwargs)

    return previous_applications_groupby_agg 

def _pos_cash_balance_groupby_agg(pos_cash_balance_cleaned, config, **kwargs):
    pos_cash_balance_groupby_agg = Step(name='pos_cash_balance_groupby_agg',
                                        transformer=fe.GroupbyAggregate(**config.pos_cash_balance),
                                        input_steps=[pos_cash_balance_cleaned],
                                        adapter=Adapter({'table': E(pos_cash_balance_cleaned.name, 'pos_cash')}),
                                        experiment_directory=config.pipeline.experiment_directory,
                                        **kwargs)

    return pos_cash_balance_groupby_agg 

def _installments_payments_groupby_agg(installments_payments_cleaned, config, **kwargs):
    installments_payments_groupby_agg = Step(name='installments_payments_groupby_agg',
                                             transformer=fe.GroupbyAggregate(**config.installments_payments),
                                             input_steps=[installments_payments_cleaned],
                                             adapter=Adapter({'table': E(installments_payments_cleaned.name,
                                                                         'installments')}),
                                             experiment_directory=config.pipeline.experiment_directory,
                                             **kwargs)

    return installments_payments_groupby_agg 

def _bureau_groupby_agg(bureau_cleaned, config, **kwargs):
    bureau_groupby_agg = Step(name='bureau_groupby_agg',
                              transformer=fe.GroupbyAggregate(**config.bureau),
                              input_steps=[bureau_cleaned],
                              adapter=Adapter({'table': E(bureau_cleaned.name, 'bureau')}),
                              experiment_directory=config.pipeline.experiment_directory,
                              **kwargs)

    return bureau_groupby_agg 

def _application_groupby_agg(application_cleaned, config, **kwargs):
    application_groupby_agg = Step(name='application_groupby_agg',
                                   transformer=fe.GroupbyAggregateDiffs(**config.applications),
                                   input_steps=[application_cleaned],
                                   adapter=Adapter({'main_table': E(application_cleaned.name, 'application')}),
                                   experiment_directory=config.pipeline.experiment_directory,
                                   **kwargs)

    return application_groupby_agg 

def _previous_application_categorical_encoder(previous_application, config, **kwargs):
    categorical_encoder = Step(name='previous_application_categorical_encoder',
                               transformer=fe.CategoricalEncoder(),
                               input_steps=[previous_application],
                               adapter=Adapter({'X': E(previous_application.name, 'categorical_features'),
                                                'categorical_columns': E(previous_application.name,
                                                                         'categorical_columns')}),
                               experiment_directory=config.pipeline.experiment_directory,
                               **kwargs)

    return categorical_encoder 

def _application_previous_application_categorical_encoder(application_previous_application, config, **kwargs):
    categorical_encoder = Step(name='application_previous_application_categorical_encoder',
                               transformer=fe.CategoricalEncoder(),
                               input_steps=[application_previous_application],
                               adapter=Adapter({'X': E(application_previous_application.name, 'categorical_features'),
                                                'categorical_columns': E(application_previous_application.name,
                                                                         'categorical_columns')}),
                               experiment_directory=config.pipeline.experiment_directory,
                               **kwargs)

    return categorical_encoder