Java Code Examples for org.pentaho.di.trans.step.StepMeta#setCopies()

private TransMeta copyCleanTransMeta( TransMeta transMeta, Repository repository, IMetaStore metaStore, VariableSpace space ) throws KettleException {

    try {
      String transMetaXml = transMeta.getXML();
      TransMeta copy = new TransMeta();
      copy.setMetaStore( metaStore );
      copy.loadXML( XMLHandler.loadXMLString( transMetaXml, TransMeta.XML_TAG ), null, metaStore, repository, true, space, null );

      for ( StepMeta stepMeta : copy.getSteps() ) {
        stepMeta.setCopies( 1 );

        DummyTransMeta dummyTransMeta = new DummyTransMeta();

        // Replace all stepMeta with a Dummy with the same name
        //
        stepMeta.setStepID( "Dummy" );
        stepMeta.setStepMetaInterface( dummyTransMeta );
      }

      return copy;

    } catch ( Exception e ) {
      throw new KettleException( "Error copying/cleaning transformation metadata", e );
    }
  }
 

/**
 * Create a copy of a step from the original transformation for use in the a slave transformation. If the step is
 * partitioned, the partitioning will be changed to "schemaName (slave)"
 *
 * @param stepMeta
 *          The step to copy / clone.
 * @return a copy of the specified step for use in a slave transformation.
 */
private StepMeta addSlaveCopy( TransMeta transMeta, StepMeta stepMeta, SlaveServer slaveServer ) {
  StepMeta copy = (StepMeta) stepMeta.clone();
  if ( copy.isPartitioned() ) {
    StepPartitioningMeta stepPartitioningMeta = copy.getStepPartitioningMeta();
    PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
    String slavePartitionSchemaName = createSlavePartitionSchemaName( partitionSchema.getName() );
    PartitionSchema slaveSchema = transMeta.findPartitionSchema( slavePartitionSchemaName );
    if ( slaveSchema != null ) {
      stepPartitioningMeta.setPartitionSchema( slaveSchema );
    }
    // Always just start a single copy on the slave server...
    // Otherwise the confusion w.r.t. to partitioning & re-partitioning would be complete.
    //
    copy.setCopies( 1 );
  }

  transMeta.addStep( copy );
  return copy;
}
 

/**
 * This is a case when first step running 2 copies and next is partitioned one.
 * 
 * @throws KettlePluginException
 */
private void prepareStepMetas_x2_cl1() throws KettlePluginException {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );

  PartitionSchema schema1 = new PartitionSchema( "p1", Arrays.asList( new String[] { PID1, PID2 } ) );
  StepPartitioningMeta partMeta1 = new StepPartitioningMeta( "Mirror to all partitions", schema1 );

  dummy2.setStepPartitioningMeta( partMeta1 );
  dummy1.setCopies( 2 );

  chain.add( dummy1 );
  chain.add( dummy2 );
  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * one 'regular step' to 'step running in 2 copies'
 */
private void prepareStepMetas_1_x2() {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );
  dummy2.setCopies( 2 );
  chain.add( dummy1 );
  chain.add( dummy2 );

  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * one 'step running in 2 copies' to 'step running in 2 copies'
 */
private void prepareStepMetas_x2_x2() {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );
  dummy1.setCopies( 2 );
  dummy2.setCopies( 2 );
  chain.add( dummy1 );
  chain.add( dummy2 );

  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * many steps copies to one
 */
private void prepareStepMetas_x2_1() {
  StepMeta dummy1 = new StepMeta( ONE, null );
  StepMeta dummy2 = new StepMeta( TWO, null );
  dummy1.setCopies( 2 );
  chain.add( dummy1 );
  chain.add( dummy2 );

  for ( StepMeta item : chain ) {
    item.setStepMetaInterface( new DummyTransMeta() );
  }
}
 

/**
 * Create a new step by loading the metadata from the specified repository.
 *
 * @param rep
 * @param stepId
 * @param databases
 * @param counters
 * @param partitionSchemas
 * @throws KettleException
 */
public StepMeta loadStepMeta( ObjectId stepId, List<DatabaseMeta> databases,
  List<PartitionSchema> partitionSchemas ) throws KettleException {
  StepMeta stepMeta = new StepMeta();
  PluginRegistry registry = PluginRegistry.getInstance();

  try {
    RowMetaAndData r = getStep( stepId );
    if ( r != null ) {
      stepMeta.setObjectId( stepId );

      stepMeta.setName( r.getString( KettleDatabaseRepository.FIELD_STEP_NAME, null ) );
      stepMeta.setDescription( r.getString( KettleDatabaseRepository.FIELD_STEP_DESCRIPTION, null ) );

      long id_step_type = r.getInteger( KettleDatabaseRepository.FIELD_STEP_ID_STEP_TYPE, -1L );
      RowMetaAndData steptyperow = getStepType( new LongObjectId( id_step_type ) );

      stepMeta.setStepID( steptyperow.getString( KettleDatabaseRepository.FIELD_STEP_TYPE_CODE, null ) );
      stepMeta.setDistributes( r.getBoolean( KettleDatabaseRepository.FIELD_STEP_DISTRIBUTE, true ) );
      int copies = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_COPIES, 1 );
      String copiesString = r.getString( KettleDatabaseRepository.FIELD_STEP_COPIES_STRING, null );
      if ( !Utils.isEmpty( copiesString ) ) {
        stepMeta.setCopiesString( copiesString );
      } else {
        stepMeta.setCopies( copies );
      }

      int x = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_X, 0 );
      int y = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_Y, 0 );
      stepMeta.setLocation( new Point( x, y ) );
      stepMeta.setDraw( r.getBoolean( KettleDatabaseRepository.FIELD_STEP_GUI_DRAW, false ) );

      // Generate the appropriate class...
      PluginInterface sp = registry.findPluginWithId( StepPluginType.class, stepMeta.getStepID() );
      if ( sp == null ) {
        stepMeta.setStepMetaInterface( new MissingTrans( stepMeta.getName(), stepMeta.getStepID() ) );
      } else {
        stepMeta.setStepMetaInterface( (StepMetaInterface) registry.loadClass( sp ) );
      }
      if ( stepMeta.getStepMetaInterface() != null ) {
        // Read the step info from the repository!
        readRepCompatibleStepMeta(
          stepMeta.getStepMetaInterface(), repository, stepMeta.getObjectId(), databases );
        stepMeta.getStepMetaInterface().readRep(
          repository, repository.metaStore, stepMeta.getObjectId(), databases );
      }

      // Get the partitioning as well...
      //
      stepMeta.setStepPartitioningMeta( loadStepPartitioningMeta( stepMeta.getObjectId() ) );
      stepMeta.getStepPartitioningMeta().setPartitionSchemaAfterLoading( partitionSchemas );

      // Get the cluster schema name
      //
      stepMeta.setClusterSchemaName( repository.getStepAttributeString( stepId, "cluster_schema" ) );

      // Are we using a custom row distribution plugin?
      //
      String rowDistributionCode = repository.getStepAttributeString( stepId, 0, "row_distribution_code" );
      RowDistributionInterface rowDistribution =
        PluginRegistry.getInstance().loadClass(
          RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class );
      stepMeta.setRowDistribution( rowDistribution );

      // Load the attribute groups map
      //
      stepMeta.setAttributesMap( loadStepAttributesMap( stepId ) );

      // Done!
      //
      return stepMeta;
    } else {
      throw new KettleException( BaseMessages.getString(
        PKG, "StepMeta.Exception.StepInfoCouldNotBeFound", String.valueOf( stepId ) ) );
    }
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleException( BaseMessages.getString( PKG, "StepMeta.Exception.StepCouldNotBeLoaded", String
      .valueOf( stepMeta.getObjectId() ) ), dbe );
  }
}
 

@Test
public void testGetCacheVersionWithIrrelevantParameters() throws Exception {
  TransMeta transMeta = new TransMeta( getClass().getResource( "one-step-trans.ktr" ).getPath() );
  int oldCacheVersion = transMeta.getCacheVersion();
  int currCacheVersion;

  transMeta.setSizeRowset( 1000 );
  currCacheVersion = transMeta.getCacheVersion();
  assertNotEquals( oldCacheVersion, currCacheVersion );

  oldCacheVersion = currCacheVersion;

  // scenarios that should not impact the cache version

  // transformation description
  transMeta.setDescription( "transformation description" );

  // transformation status
  transMeta.setTransstatus( 100 );

  // transformation log table
  transMeta.setTransLogTable( mock( TransLogTable.class ) );

  // transformation created user
  transMeta.setCreatedUser( "user" );

  // transformation modified user
  transMeta.setModifiedUser( "user" );

  // transformation created date
  transMeta.setCreatedDate( new Date() );

  // transformation modified date
  transMeta.setModifiedDate( new Date() );

  // transformation is key private flag
  transMeta.setPrivateKey( false );

  // transformation attributes
  Map<String, String> attributes = new HashMap<>();
  attributes.put( "key", "value" );
  transMeta.setAttributes( "group", attributes );

  // step description
  StepMeta stepMeta = transMeta.getStep( 0 );
  stepMeta.setDescription( "stepDescription" );

  // step position
  stepMeta.setLocation( 10, 20 );
  stepMeta.setLocation( new Point( 30, 40 ) );

  // step type id
  stepMeta.setStepID( "Dummy" );

  // step is distributed flag
  stepMeta.setDistributes( false );

  // step copies
  stepMeta.setCopies( 5 );

  // step partitioning meta
  stepMeta.setStepPartitioningMeta( mock( StepPartitioningMeta.class ) );

  // assert that nothing impacted the cache version
  assertEquals( oldCacheVersion, transMeta.getCacheVersion() );
}