Java Code Examples for org.pentaho.di.trans.step.StepMeta#setDistributes()

/**
 * Test case for hop using copy.
 *
 * The transformation is as follows: an injector step links to a dummy step, which in turn links to 2 target dummy
 * steps.
 *
 * Both dummy1 and dummy2 should get all rows.
 */
public void testCopyHops() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "hop test default" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, dummyStep );
  transMeta.addTransHop( hi );

  // THIS DETERMINES THE COPY OR DISTRIBUTE BEHAVIOUR
  dummyStep.setDistributes( false );

  //
  // Create a dummy target step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hop1 = new TransHopMeta( dummyStep, dummyStep1 );
  transMeta.addTransHop( hop1 );

  //
  // Create a dummy target step 2
  //
  String dummyStepname2 = "dummy step 2";
  DummyTransMeta dm2 = new DummyTransMeta();

  String dummyPid2 = registry.getPluginId( StepPluginType.class, dm2 );
  StepMeta dummyStep2 = new StepMeta( dummyPid2, dummyStepname2, dm2 );
  transMeta.addStep( dummyStep2 );

  TransHopMeta hop2 = new TransHopMeta( dummyStep, dummyStep2 );
  transMeta.addTransHop( hop2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si1 = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector rc1 = new RowStepCollector();
  si1.addRowListener( rc1 );

  StepInterface si2 = trans.getStepInterface( dummyStepname2, 0 );
  RowStepCollector rc2 = new RowStepCollector();
  si2.addRowListener( rc2 );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();
  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc1.getRowsWritten();
  checkRows( resultRows, inputList );
}
 

/**
 * Create a new step by loading the metadata from the specified repository.
 *
 * @param rep
 * @param stepId
 * @param databases
 * @param counters
 * @param partitionSchemas
 * @throws KettleException
 */
public StepMeta loadStepMeta( ObjectId stepId, List<DatabaseMeta> databases,
  List<PartitionSchema> partitionSchemas ) throws KettleException {
  StepMeta stepMeta = new StepMeta();
  PluginRegistry registry = PluginRegistry.getInstance();

  try {
    RowMetaAndData r = getStep( stepId );
    if ( r != null ) {
      stepMeta.setObjectId( stepId );

      stepMeta.setName( r.getString( KettleDatabaseRepository.FIELD_STEP_NAME, null ) );
      stepMeta.setDescription( r.getString( KettleDatabaseRepository.FIELD_STEP_DESCRIPTION, null ) );

      long id_step_type = r.getInteger( KettleDatabaseRepository.FIELD_STEP_ID_STEP_TYPE, -1L );
      RowMetaAndData steptyperow = getStepType( new LongObjectId( id_step_type ) );

      stepMeta.setStepID( steptyperow.getString( KettleDatabaseRepository.FIELD_STEP_TYPE_CODE, null ) );
      stepMeta.setDistributes( r.getBoolean( KettleDatabaseRepository.FIELD_STEP_DISTRIBUTE, true ) );
      int copies = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_COPIES, 1 );
      String copiesString = r.getString( KettleDatabaseRepository.FIELD_STEP_COPIES_STRING, null );
      if ( !Utils.isEmpty( copiesString ) ) {
        stepMeta.setCopiesString( copiesString );
      } else {
        stepMeta.setCopies( copies );
      }

      int x = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_X, 0 );
      int y = (int) r.getInteger( KettleDatabaseRepository.FIELD_STEP_GUI_LOCATION_Y, 0 );
      stepMeta.setLocation( new Point( x, y ) );
      stepMeta.setDraw( r.getBoolean( KettleDatabaseRepository.FIELD_STEP_GUI_DRAW, false ) );

      // Generate the appropriate class...
      PluginInterface sp = registry.findPluginWithId( StepPluginType.class, stepMeta.getStepID() );
      if ( sp == null ) {
        stepMeta.setStepMetaInterface( new MissingTrans( stepMeta.getName(), stepMeta.getStepID() ) );
      } else {
        stepMeta.setStepMetaInterface( (StepMetaInterface) registry.loadClass( sp ) );
      }
      if ( stepMeta.getStepMetaInterface() != null ) {
        // Read the step info from the repository!
        readRepCompatibleStepMeta(
          stepMeta.getStepMetaInterface(), repository, stepMeta.getObjectId(), databases );
        stepMeta.getStepMetaInterface().readRep(
          repository, repository.metaStore, stepMeta.getObjectId(), databases );
      }

      // Get the partitioning as well...
      //
      stepMeta.setStepPartitioningMeta( loadStepPartitioningMeta( stepMeta.getObjectId() ) );
      stepMeta.getStepPartitioningMeta().setPartitionSchemaAfterLoading( partitionSchemas );

      // Get the cluster schema name
      //
      stepMeta.setClusterSchemaName( repository.getStepAttributeString( stepId, "cluster_schema" ) );

      // Are we using a custom row distribution plugin?
      //
      String rowDistributionCode = repository.getStepAttributeString( stepId, 0, "row_distribution_code" );
      RowDistributionInterface rowDistribution =
        PluginRegistry.getInstance().loadClass(
          RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class );
      stepMeta.setRowDistribution( rowDistribution );

      // Load the attribute groups map
      //
      stepMeta.setAttributesMap( loadStepAttributesMap( stepId ) );

      // Done!
      //
      return stepMeta;
    } else {
      throw new KettleException( BaseMessages.getString(
        PKG, "StepMeta.Exception.StepInfoCouldNotBeFound", String.valueOf( stepId ) ) );
    }
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleException( BaseMessages.getString( PKG, "StepMeta.Exception.StepCouldNotBeLoaded", String
      .valueOf( stepMeta.getObjectId() ) ), dbe );
  }
}
 

@Test
public void testGetCacheVersionWithIrrelevantParameters() throws Exception {
  TransMeta transMeta = new TransMeta( getClass().getResource( "one-step-trans.ktr" ).getPath() );
  int oldCacheVersion = transMeta.getCacheVersion();
  int currCacheVersion;

  transMeta.setSizeRowset( 1000 );
  currCacheVersion = transMeta.getCacheVersion();
  assertNotEquals( oldCacheVersion, currCacheVersion );

  oldCacheVersion = currCacheVersion;

  // scenarios that should not impact the cache version

  // transformation description
  transMeta.setDescription( "transformation description" );

  // transformation status
  transMeta.setTransstatus( 100 );

  // transformation log table
  transMeta.setTransLogTable( mock( TransLogTable.class ) );

  // transformation created user
  transMeta.setCreatedUser( "user" );

  // transformation modified user
  transMeta.setModifiedUser( "user" );

  // transformation created date
  transMeta.setCreatedDate( new Date() );

  // transformation modified date
  transMeta.setModifiedDate( new Date() );

  // transformation is key private flag
  transMeta.setPrivateKey( false );

  // transformation attributes
  Map<String, String> attributes = new HashMap<>();
  attributes.put( "key", "value" );
  transMeta.setAttributes( "group", attributes );

  // step description
  StepMeta stepMeta = transMeta.getStep( 0 );
  stepMeta.setDescription( "stepDescription" );

  // step position
  stepMeta.setLocation( 10, 20 );
  stepMeta.setLocation( new Point( 30, 40 ) );

  // step type id
  stepMeta.setStepID( "Dummy" );

  // step is distributed flag
  stepMeta.setDistributes( false );

  // step copies
  stepMeta.setCopies( 5 );

  // step partitioning meta
  stepMeta.setStepPartitioningMeta( mock( StepPartitioningMeta.class ) );

  // assert that nothing impacted the cache version
  assertEquals( oldCacheVersion, transMeta.getCacheVersion() );
}