Java Code Examples for org.pentaho.di.trans.Trans#startThreads()

@Test
public void testGetDataFromFolderWithInvalidFieldName() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-from-previous-step-negative.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 0, trans.getSteps().get( 1 ).step.getLinesWritten() );
  assertEquals( 0, trans.getSteps().get( 1 ).step.getLinesInput() );
  // The path contains one entry of a folder
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );
}
 

@Test public void testFullCircle() throws KettleException, InterruptedException, TimeoutException,
  ExecutionException {
  TransMeta consumerMeta = new TransMeta( getClass().getResource( "/amq-consumer.ktr" ).getPath() );
  Trans consumerTrans = new Trans( consumerMeta );
  consumerTrans.prepareExecution( new String[] {} );
  consumerTrans.startThreads();

  TransMeta producerMeta = new TransMeta( getClass().getResource( "/amq-producer.ktr" ).getPath() );
  Trans producerTrans = new Trans( producerMeta );
  producerTrans.prepareExecution( new String[] {} );
  producerTrans.startThreads();
  producerTrans.waitUntilFinished();

  Future<?> future = Executors.newSingleThreadExecutor().submit( () -> {
    while ( true ) {
      if ( consumerTrans.getSteps().get( 0 ).step.getLinesWritten() == 10 ) {
        break;
      }
    }
  } );
  future.get( 5, TimeUnit.SECONDS );
  consumerTrans.safeStop();
  assertEquals( 10, consumerTrans.getSteps().get( 0 ).step.getLinesWritten() );
}
 

@Test
public void executorsInputIsStraightlyCopiedToOutput() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setExecutorsOutputStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertEquals( testInput.size(), endRc.getRowsWritten().size() );
  assertThat( asList( endRc.getRowsWritten().get( 0 ).getData() ),
    hasItem( (Object) SAMPLE_INPUT )
  );
}
 

@Test
public void subTransExecutionStatisticsIsCollected() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setExecutionTimeField( "time" );
  executorMeta.setExecutionResultTargetStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertFalse( endRc.getRowsWritten().isEmpty() );
  // execution time field
  assertNotNull( endRc.getRowsWritten().get( 0 ).getData()[ 0 ] );
}
 

protected TableModel performQueryOnTransformation( final DataRow parameters,
                                                 final int queryLimit,
                                                 final DataFactoryContext context,
                                                 final TransMeta transMeta )
  throws EvaluationException, ParseException, KettleException, ReportDataFactoryException {
  TableProducer tableProducer = null;
  Trans trans = null;

  try {
    trans = prepareTransformation( parameters, context, transMeta );

    final RowMetaInterface row = transMeta.getStepFields( getStepName() );
    tableProducer = new TableProducer( row, queryLimit, isStopOnError() );
    StepInterface targetStep = findTargetStep( trans );
    targetStep.addRowListener( tableProducer );

    currentlyRunningTransformation = trans;

    trans.startThreads();
    trans.waitUntilFinished();
  } finally {
    if ( null != trans ) {
      trans.cleanup();
    }
    currentlyRunningTransformation = null;
  }

  if ( trans.getErrors() != 0 && isStopOnError() ) {
    throw new ReportDataFactoryException( String
      .format( "Transformation reported %d records with errors and stop-on-error is true. Aborting.",
        trans.getErrors() ) );
  }

  return tableProducer.getTableModel();
}
 

@Test
public void testGetDataFromFromFiles() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-files-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

@Test
public void testRESTInput() throws Exception {
  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "restinput" );

  PluginRegistry registry = PluginRegistry.getInstance();

  StepMeta inputStep = createRestInputStep( transMeta, registry );

  RowStepCollector rowStepCollector = new RowStepCollector();
  final Trans trans = createAndTestTrans( registry, transMeta, inputStep, rowStepCollector, "limit", 5 );
  trans.startThreads();

  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = rowStepCollector.getRowsWritten();

  assertTrue( rowStepCollector.getRowsError().isEmpty() );
  assertEquals( 1, rowStepCollector.getRowsWritten().size() );

  final RowMetaAndData rowMetaAndData = resultRows.get( 0 );
  final RowMetaInterface rowMeta = rowMetaAndData.getRowMeta();

  final String[] fieldNames = rowMeta.getFieldNames();
  final Object[] data = rowMetaAndData.getData();

  assertEquals( "pageSize", fieldNames[0] );
  assertEquals( "name", fieldNames[1] );
  assertEquals( "result", fieldNames[2] );

  assertEquals( Integer.valueOf( 5 ), data[0] );
  assertEquals( "limit", data[1] );
  assertEquals( "limit:5", data[2] );
}
 

public void testMapping_WhenSharingPreviousStepWithAnother() throws Exception {
  KettleEnvironment.init();

  TransMeta transMeta = new TransMeta( "src/it/resources/org/pentaho/di/trans/steps/mapping/pdi-13435/PDI-13435-main.ktr" );
  transMeta.setTransformationType( TransMeta.TransformationType.Normal );

  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  assertEquals( 0, trans.getErrors() );
}
 

/**
 * Test case for normal table output where the table is included in the instream, but the tablename is not stored in
 * the table.
 */
public void testTableOutputJIRA897() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output JIRA897 test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table1, createSourceRowMetaInterface1() );
  createTable( database, target_table2, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTableNameInField( true );
  tom.setTableNameField( "TABLE" );
  tom.setTableNameInTable( false );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to tables on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createJIRA897DataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();

  // The name of the table should still be in here.
  List<RowMetaAndData> goldRows = createJIRA897DataRows();
  checkRows( goldRows, resultRows );
  checkResultsJIRA897( database );
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans6() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set. And no default... so the variable will be set to "". not
  // to "Variable1"
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData6();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

@Test
public void testFilterConditionRefersToNonExistingFields() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "filterrowstest" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  // Create a filter rows step
  String filterStepName = "filter rows step";
  FilterRowsMeta frm = new FilterRowsMeta();
  Condition condition = new Condition();
  String nonExistingFieldName = "non-existing-field";
  condition.setLeftValuename( nonExistingFieldName );
  condition.setFunction( 8 ); //IS NOT
  condition.setRightValuename( null );
  condition.setOperator( 0 );
  frm.setCondition( condition );

  String filterRowsStepPid = registry.getPluginId( StepPluginType.class, frm );
  StepMeta filterRowsStep = new StepMeta( filterRowsStepPid, filterStepName, frm );
  transMeta.addStep( filterRowsStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, filterRowsStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );

  // add rows
  List<RowMetaAndData> inputList = createIntegerData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 1, trans.getErrors() ); //expect errors

}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans5() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "default1", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set, so defaults should kick in.
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData5();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

public void testSortCaseSensitiveUniqueCaseSensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { true } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseSensitiveUniqueCaseSensitive(), resultRows );
}
 

public void testSortCaseSensitiveUniqueCaseInsensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { true } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { true } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseSensitiveUniqueCaseInsensitive(), resultRows );
}
 

protected void executeTrans( Trans trans ) throws KettleException {
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
}
 

public void testCaseSensitiveNoPreviousSort() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataCaseSensitiveNoPreviousSort(), resultRows );
}
 

void testOneRow(String transName, String[] referenceValues, String[] comparisonValues, Object[] goldenImageRowValues) throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( transName );
  PluginRegistry registry = PluginRegistry.getInstance();

  // Create a merge rows step
  String mergeRowsStepName = "merge rows step";
  MergeRowsMeta mergeRowsMeta = new MergeRowsMeta();

  String mergeRowsStepPid = registry.getPluginId( StepPluginType.class, mergeRowsMeta );
  StepMeta mergeRowsStep = new StepMeta( mergeRowsStepPid, mergeRowsStepName, mergeRowsMeta );
  transMeta.addStep( mergeRowsStep );

  mergeRowsMeta.setKeyFields( new String[]{ keyField } );
  mergeRowsMeta.setValueFields( new String[]{ compareField } );
  mergeRowsMeta.setFlagField( flagField );

  List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();

  //
  // create a reference stream (row generator step)
  //
  createRowGenerator(
    transMeta, registry,
    "reference row generator", referenceValues,
    mergeRowsStep, mergeRowsMeta,
    0
  );

  //
  // create a comparison stream (row generator step)
  //
  createRowGenerator(
    transMeta, registry,
    "comparison row generator", comparisonValues,
    mergeRowsStep, mergeRowsMeta,
    1
  );

  // Now execute the transformation
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( mergeRowsStepName, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();
  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData( goldenImageRowValues );
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT.
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of two (i.e. not autocommit and not the input row size)
 */
@Test
public void testExecSQLRow3() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 2 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

/**
 * Test case for normal table output case.
 */
@SuppressWarnings( "deprecation" )
public void testTableOutputNormal() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output normal test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTablename( target_table );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to table [" + target_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createNormalDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createNormalDataRows();
  checkRows( goldRows, resultRows );
  checkResultsNormal( database );
}