Java Code Examples for org.pentaho.di.trans.step.StepMeta#setStepMetaInterface()

private TransMeta copyCleanTransMeta( TransMeta transMeta, Repository repository, IMetaStore metaStore, VariableSpace space ) throws KettleException {

    try {
      String transMetaXml = transMeta.getXML();
      TransMeta copy = new TransMeta();
      copy.setMetaStore( metaStore );
      copy.loadXML( XMLHandler.loadXMLString( transMetaXml, TransMeta.XML_TAG ), null, metaStore, repository, true, space, null );

      for ( StepMeta stepMeta : copy.getSteps() ) {
        stepMeta.setCopies( 1 );

        DummyTransMeta dummyTransMeta = new DummyTransMeta();

        // Replace all stepMeta with a Dummy with the same name
        //
        stepMeta.setStepID( "Dummy" );
        stepMeta.setStepMetaInterface( dummyTransMeta );
      }

      return copy;

    } catch ( Exception e ) {
      throw new KettleException( "Error copying/cleaning transformation metadata", e );
    }
  }
 

/**
 * This case simulates when we do have 2 step partitioned with one same partitioner We want to get a 'swim-lanes'
 * transformation
 * 
 * @throws KettlePluginException
 */
private void prepareStepMetas_cl1_cl1() throws KettlePluginException {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );

  PartitionSchema schema = new PartitionSchema( "p1", Arrays.asList( new String[] { PID1, PID2 } ) );
  // for delayed binding StepPartitioning meta does not achieve
  // schema name when using in constructor so we have to set it
  // explicitly. See equals implementation for StepPartitioningMeta.
  StepPartitioningMeta partMeta = new StepPartitioningMeta( "Mirror to all partitions", schema );
  // that is what I am talking about:
  partMeta.setPartitionSchemaName( schema.getName() );

  dummy1.setStepPartitioningMeta( partMeta );
  dummy2.setStepPartitioningMeta( partMeta );

  chain.add( dummy1 );
  chain.add( dummy2 );
  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * This is a case when we have 2 steps, but partitioned differently
 * 
 * @throws KettlePluginException
 */
private void prepareStepMetas_cl1_cl2() throws KettlePluginException {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );

  PartitionSchema schema1 = new PartitionSchema( "p1", Arrays.asList( new String[] { PID1, PID2 } ) );
  PartitionSchema schema2 = new PartitionSchema( "p2", Arrays.asList( new String[] { PID1, PID2 } ) );

  StepPartitioningMeta partMeta1 = new StepPartitioningMeta( "Mirror to all partitions", schema1 );
  StepPartitioningMeta partMeta2 = new StepPartitioningMeta( "Mirror to all partitions", schema2 );
  partMeta1.setPartitionSchemaName( schema1.getName() );
  partMeta2.setPartitionSchemaName( schema2.getName() );

  dummy1.setStepPartitioningMeta( partMeta1 );
  dummy2.setStepPartitioningMeta( partMeta2 );

  chain.add( dummy1 );
  chain.add( dummy2 );
  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * This is a case when first step running 2 copies and next is partitioned one.
 * 
 * @throws KettlePluginException
 */
private void prepareStepMetas_x2_cl1() throws KettlePluginException {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );

  PartitionSchema schema1 = new PartitionSchema( "p1", Arrays.asList( new String[] { PID1, PID2 } ) );
  StepPartitioningMeta partMeta1 = new StepPartitioningMeta( "Mirror to all partitions", schema1 );

  dummy2.setStepPartitioningMeta( partMeta1 );
  dummy1.setCopies( 2 );

  chain.add( dummy1 );
  chain.add( dummy2 );
  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

@Test
public void testDelHop() throws Exception {

  StepMetaInterface fromStepMetaInterface = Mockito.mock( StepMetaInterface.class );
  StepMeta fromStep = new StepMeta();
  fromStep.setStepMetaInterface( fromStepMetaInterface );

  StepMetaInterface toStepMetaInterface = Mockito.mock( StepMetaInterface.class );
  StepMeta toStep = new StepMeta();
  toStep.setStepMetaInterface( toStepMetaInterface );

  TransHopMeta transHopMeta = new TransHopMeta();
  transHopMeta.setFromStep( fromStep );
  transHopMeta.setToStep( toStep );

  TransMeta transMeta = Mockito.mock( TransMeta.class );

  spoon.delHop( transMeta, transHopMeta );
  Mockito.verify( fromStepMetaInterface, times( 1 ) ).cleanAfterHopFromRemove( toStep );
  Mockito.verify( toStepMetaInterface, times( 1 ) ).cleanAfterHopToRemove( fromStep );
}
 

private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta,
                                 FactoriesHierarchy factoriesHierarchy ) throws KettleException {

  String inputSetName = inputLocation.getDataSetName();

  if ( log.isDetailed() ) {
    log.logDetailed( "Replacing step '" + stepMeta.getName() + "' with an Injector for dataset '" + inputSetName + "'" );
  }

  DataSet dataSet;
  try {
    dataSet = factoriesHierarchy.getSetFactory().loadElement( inputSetName );
  } catch ( MetaStoreException e ) {
    throw new KettleException( "Unable to load data set '" + inputSetName + "'" );
  }

  // OK, this step needs to be replaced by an Injector step...
  // Which fields do we need to use?
  //
  final RowMetaInterface stepFields = DataSetConst.getStepOutputFields( log, dataSet, inputLocation );

  if ( log.isDetailed() ) {
    log.logDetailed( "Input Data Set '" + inputSetName + "' Injector fields : '" + stepFields.toString() );
  }

  InjectorMeta injectorMeta = new InjectorMeta();
  injectorMeta.allocate( stepFields.size() );
  for ( int x = 0; x < stepFields.size(); x++ ) {
    injectorMeta.getFieldname()[ x ] = stepFields.getValueMeta( x ).getName();
    injectorMeta.getType()[ x ] = stepFields.getValueMeta( x ).getType();
    injectorMeta.getLength()[ x ] = stepFields.getValueMeta( x ).getLength();
    injectorMeta.getPrecision()[ x ] = stepFields.getValueMeta( x ).getPrecision();

    // Only the step metadata, type...
    stepMeta.setStepMetaInterface( injectorMeta );
    stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, injectorMeta ) );
  }
}
 

@Before
public void setUp() throws KettleException {
  // add variable to row generator step
  StepMetaInterface stepMetaInterface = spy( new RowGeneratorMeta() );
  ( (RowGeneratorMeta) stepMetaInterface ).setRowLimit( "${ROW_LIMIT}" );
  String[] strings = {};
  when( ( (RowGeneratorMeta) stepMetaInterface ).getFieldName() ).thenReturn( strings );

  StepMeta stepMeta = new StepMeta();
  stepMeta.setStepMetaInterface( stepMetaInterface );
  stepMeta.setName( "ROW_STEP_META" );
  StepDataInterface stepDataInterface = stepMeta.getStepMetaInterface().getStepData();

  // add variable to transformation variable space
  Map<String, String> map = new HashMap<String, String>();
  map.put( "ROW_LIMIT", "1440" );
  TransMeta transMeta = spy( new TransMeta() );
  transMeta.injectVariables( map );
  when( transMeta.findStep( anyString() ) ).thenReturn( stepMeta );

  Trans trans = spy( new Trans( transMeta, null ) );
  when( trans.getSocketRepository() ).thenReturn( null );
  when( trans.getLogChannelId() ).thenReturn( "ROW_LIMIT" );

  //prepare row generator, substitutes variable by value from transformation variable space
  rowGenerator = spy( new RowGenerator( stepMeta, stepDataInterface, 0, transMeta, trans ) );
  rowGenerator.initializeVariablesFrom( trans );
  rowGenerator.init( stepMetaInterface, stepDataInterface );
}
 

/**
 * one 'regular step' to 'step running in 2 copies'
 */
private void prepareStepMetas_1_x2() {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );
  dummy2.setCopies( 2 );
  chain.add( dummy1 );
  chain.add( dummy2 );

  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * one 'step running in 2 copies' to 'step running in 2 copies'
 */
private void prepareStepMetas_x2_x2() {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );
  dummy1.setCopies( 2 );
  dummy2.setCopies( 2 );
  chain.add( dummy1 );
  chain.add( dummy2 );

  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * many steps copies to one
 */
private void prepareStepMetas_x2_1() {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );
  dummy1.setCopies( 2 );
  chain.add( dummy1 );
  chain.add( dummy2 );

  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * This is a case when we have 1 step to 1 clustered step distribution.
 * 
 * @throws KettlePluginException
 */
private void prepareStepMetas_1_cl1() throws KettlePluginException {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );

  PartitionSchema schema = new PartitionSchema( "p1", Arrays.asList( new String[] { PID1, PID2 } ) );
  StepPartitioningMeta partMeta = new StepPartitioningMeta( "Mirror to all partitions", schema );
  dummy2.setStepPartitioningMeta( partMeta );

  chain.add( dummy1 );
  chain.add( dummy2 );
  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

private void replaceStepWithDummy( LogChannelInterface log, StepMeta stepMeta ) {
  DummyTransMeta dummyTransMeta = new DummyTransMeta();
  stepMeta.setStepMetaInterface( dummyTransMeta );
  stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, dummyTransMeta ) );
}
 

/**
 * Create a new step by loading the metadata from the specified repository.
 *
 * @param rep
 * @param stepId
 * @param databases
 * @param counters
 * @param partitionSchemas
 * @throws KettleException
 */
public StepMeta loadStepMeta( ObjectId stepId, List<DatabaseMeta> databases,
  List<PartitionSchema> partitionSchemas ) throws KettleException {
  StepMeta stepMeta = new StepMeta();
  PluginRegistry registry = PluginRegistry.getInstance();

  try {
    RowMetaAndData r = getStep( stepId );
    if ( r != null ) {
      stepMeta.setObjectId( stepId );

      stepMeta.setName( r.getString( KettleDatabaseRepository.FIELD_STEP_NAME, null ) );
      stepMeta.setDescription( r.getString( KettleDatabaseRepository.FIELD_STEP_DESCRIPTION, null ) );

      long id_step_type = r.getInteger( KettleDatabaseRepository.FIELD_STEP_ID_STEP_TYPE, -1L );
      RowMetaAndData steptyperow = getStepType( new LongObjectId( id_step_type ) );

      stepMeta.setStepID( steptyperow.getString( KettleDatabaseRepository.FIELD_STEP_TYPE_CODE, null ) );
      stepMeta.setDistributes( r.getBoolean( KettleDatabaseRepository.FIELD_STEP_DISTRIBUTE, true ) );
      int copies = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_COPIES, 1 );
      String copiesString = r.getString( KettleDatabaseRepository.FIELD_STEP_COPIES_STRING, null );
      if ( !Utils.isEmpty( copiesString ) ) {
        stepMeta.setCopiesString( copiesString );
      } else {
        stepMeta.setCopies( copies );
      }

      int x = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_X, 0 );
      int y = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_Y, 0 );
      stepMeta.setLocation( new Point( x, y ) );
      stepMeta.setDraw( r.getBoolean( KettleDatabaseRepository.FIELD_STEP_GUI_DRAW, false ) );

      // Generate the appropriate class...
      PluginInterface sp = registry.findPluginWithId( StepPluginType.class, stepMeta.getStepID() );
      if ( sp == null ) {
        stepMeta.setStepMetaInterface( new MissingTrans( stepMeta.getName(), stepMeta.getStepID() ) );
      } else {
        stepMeta.setStepMetaInterface( (StepMetaInterface) registry.loadClass( sp ) );
      }
      if ( stepMeta.getStepMetaInterface() != null ) {
        // Read the step info from the repository!
        readRepCompatibleStepMeta(
          stepMeta.getStepMetaInterface(), repository, stepMeta.getObjectId(), databases );
        stepMeta.getStepMetaInterface().readRep(
          repository, repository.metaStore, stepMeta.getObjectId(), databases );
      }

      // Get the partitioning as well...
      //
      stepMeta.setStepPartitioningMeta( loadStepPartitioningMeta( stepMeta.getObjectId() ) );
      stepMeta.getStepPartitioningMeta().setPartitionSchemaAfterLoading( partitionSchemas );

      // Get the cluster schema name
      //
      stepMeta.setClusterSchemaName( repository.getStepAttributeString( stepId, "cluster_schema" ) );

      // Are we using a custom row distribution plugin?
      //
      String rowDistributionCode = repository.getStepAttributeString( stepId, 0, "row_distribution_code" );
      RowDistributionInterface rowDistribution =
        PluginRegistry.getInstance().loadClass(
          RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class );
      stepMeta.setRowDistribution( rowDistribution );

      // Load the attribute groups map
      //
      stepMeta.setAttributesMap( loadStepAttributesMap( stepId ) );

      // Done!
      //
      return stepMeta;
    } else {
      throw new KettleException( BaseMessages.getString(
        PKG, "StepMeta.Exception.StepInfoCouldNotBeFound", String.valueOf( stepId ) ) );
    }
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleException( BaseMessages.getString( PKG, "StepMeta.Exception.StepCouldNotBeLoaded", String
      .valueOf( stepMeta.getObjectId() ) ), dbe );
  }
}
 

@Test
public void testTransListeners() {
  TransMeta TransMeta = new TransMeta();

  StepMeta oldFormStep = new StepMeta();
  oldFormStep.setName( "Generate_1" );

  StepMeta newFormStep = new StepMeta();
  newFormStep.setName( "Generate_2" );

  StepMeta toStep = new StepMeta();
  toStep.setStepMetaInterface( new MetaInjectMeta() );
  toStep.setName( "ETL Inject Metadata" );

  StepMeta deletedStep = new StepMeta();
  deletedStep.setStepMetaInterface( new MetaInjectMeta() );
  deletedStep.setName( "ETL Inject Metadata for delete" );

  // Verify add & remove listeners

  TransMeta.addStep( oldFormStep );
  TransMeta.addStep( toStep );
  TransMeta.addStep( deletedStep );

  assertEquals( TransMeta.nrStepChangeListeners(), 2 );
  TransMeta.removeStepChangeListener( (StepMetaChangeListenerInterface) deletedStep.getStepMetaInterface() );
  assertEquals( TransMeta.nrStepChangeListeners(), 1 );
  TransMeta.removeStep( 2 );

  TransHopMeta hi = new TransHopMeta( oldFormStep, toStep );
  TransMeta.addTransHop( hi );

  // Verify MetaInjectMeta.onStepChange()

  // add new TargetStepAttribute
  MetaInjectMeta toMeta = (MetaInjectMeta) toStep.getStepMetaInterface();

  Map<TargetStepAttribute, SourceStepField> sourceMapping = new HashMap<TargetStepAttribute, SourceStepField>();
  TargetStepAttribute keyTest = new TargetStepAttribute( "File", "key", true );
  SourceStepField valueTest = new SourceStepField( oldFormStep.getName(), oldFormStep.getName() );
  sourceMapping.put( keyTest, valueTest );

  toMeta.setTargetSourceMapping( sourceMapping );

  // Run all listeners
  TransMeta.notifyAllListeners( oldFormStep, newFormStep );

  // Verify changes, which listeners makes
  sourceMapping = toMeta.getTargetSourceMapping();
  for ( Entry<TargetStepAttribute, SourceStepField> entry : sourceMapping.entrySet() ) {
    SourceStepField value = entry.getValue();
    if ( !value.getStepname().equals( newFormStep.getName() ) ) {
      fail();
    }
  }

  // verify another functions
  TransMeta.addStep( deletedStep );
  assertEquals( TransMeta.nrSteps(), 3 );
  assertEquals( TransMeta.nrStepChangeListeners(), 2 );

  TransMeta.removeStep( 0 );
  assertEquals( TransMeta.nrSteps(), 2 );

}