Java Code Examples for org.pentaho.di.trans.Trans#waitUntilFinished()

private void runTransformation(PentahoReportingOutputMeta.ProcessorType processorType) throws Exception {
  File tmp = File.createTempFile( "PentahoReportingOutputTest", "PentahoReportingOutputTest" );
  tmp.deleteOnExit();

  StepMeta outputStep = transMeta.findStep( "Data Grid" );
  DataGridMeta metaGrid = (DataGridMeta) outputStep.getStepMetaInterface();
  metaGrid.getDataLines().clear();
  metaGrid.getDataLines()
    .add( asList( "src/it/resources/org/pentaho/di/trans/steps/pentahoreporting/pdi-13434.prpt", tmp.getAbsolutePath() ) );

  StepMeta reportingStep = transMeta.findStep( "Pentaho Reporting Output" );
  PentahoReportingOutputMeta reportingMeta = (PentahoReportingOutputMeta) reportingStep.getStepMetaInterface();
  reportingMeta.setOutputProcessorType( processorType );

  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  Assert.assertEquals( 0, trans.getErrors() );
  Assert.assertTrue( tmp.length() > 0 );
  tmp.delete();
}
 

@Test
public void executorsInputIsStraightlyCopiedToOutput() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setExecutorsOutputStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertEquals( testInput.size(), endRc.getRowsWritten().size() );
  assertThat( asList( endRc.getRowsWritten().get( 0 ).getData() ),
    hasItem( (Object) SAMPLE_INPUT )
  );
}
 

@Test
public void constantFieldsAddedOnceWithCorrectFormatting() throws KettleException {
  String path = getClass().getResource( "/constants-test.ktr" ).getPath();
  TransMeta constantTrans = new TransMeta( path, new Variables() );
  Trans trans = new Trans( constantTrans );
  trans.prepareExecution( new String[] {} );
  RowStepCollector collector = new RowStepCollector();
  trans.getSteps().get( 1 ).step.addRowListener( collector );
  List<RowMetaAndData> rowsWritten = collector.getRowsWritten();
  trans.setPreview( true );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 1, rowsWritten.size() );
  Object[] data = rowsWritten.get( 0 ).getData();
  RowMetaInterface rowMeta = rowsWritten.get( 0 ).getRowMeta();
  assertEquals( 7, rowMeta.size() );
  assertEquals( "a", data[0] );
  assertEquals( "88.00", rowMeta.getValueMeta( 1 ).getString( data[1] ) );
  assertEquals( "b", data[2] );
  assertEquals( "c", data[3] );
  assertEquals( "d", data[4] );
  assertEquals( "e", data[5] );
  assertEquals( "1,234.568", rowMeta.getValueMeta( 6 ).getString( data[6] ) );
}
 

/**
 * Create, execute, and return the row listener attached to the output step with complete results from the execution.
 *
 * @param checkSumType
 *          Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
 * @param compatibilityMode
 *          Use compatibility mode for CheckSum
 * @param fieldSeparatorString
 *          The string separate multiple fields with
 * @param inputs
 *          Array of objects representing row data
 * @param inputValueMetas
 *          metas to be processed
 * @return RowListener with results.
 */
private MockRowListener executeHexTest( int checkSumType, boolean compatibilityMode, boolean oldChecksumBehaviour,
    String fieldSeparatorString, Object[] inputs, ValueMetaInterface... inputValueMetas ) throws Exception {

  String[] fieldNames = new String[inputValueMetas.length];
  RowMeta inputRowMeta = new RowMeta();
  for ( int i = 0; i < inputValueMetas.length; i++ ) {
    inputRowMeta.addValueMeta( inputValueMetas[i] );
    fieldNames[i] = inputValueMetas[i].getName();
  }

  Trans trans =
      buildHexadecimalChecksumTrans( checkSumType, compatibilityMode, oldChecksumBehaviour, fieldSeparatorString,
          fieldNames );

  trans.prepareExecution( null );

  StepInterface output = trans.getRunThread( "Output", 0 );
  MockRowListener listener = new MockRowListener();
  output.addRowListener( listener );

  RowProducer rp = trans.addRowProducer( "CheckSum", 0 );

  ( (BaseStep) trans.getRunThread( "CheckSum", 0 ) ).setInputRowMeta( inputRowMeta );

  trans.startThreads();

  rp.putRow( inputRowMeta, inputs );
  rp.finished();

  trans.waitUntilFinished();
  trans.stopAll();
  trans.cleanup();
  return listener;
}
 

public Trans runTransformationFromResource(String resource, List<TransVariable> listVaribale) {
    try {
        // load latest revision of the transformation
        // The TransMeta object is the programmatic representation of a transformation definition.
        TransMeta transMeta = new TransMeta(getClass().getResourceAsStream(resource), null, false, null, null);

        // Creating a transformation object which is the programmatic representation of a transformation 
        // A transformation object can be executed, report success, etc.
        Trans transformation = new Trans(transMeta);
        if (listVaribale != null) {
            for (TransVariable variable : listVaribale) {
                transformation.setVariable(variable.getName(), variable.getValue());
            }
        }

        // adjust the log level
        transformation.setLogLevel(LogLevel.BASIC);

        // starting the transformation, which will execute asynchronously
        transformation.execute(null);

        // waiting for the transformation to finish
        transformation.waitUntilFinished();

        return transformation;
    } catch (KettleException ex) {
        MessageInf msg = new MessageInf(MessageInf.SGN_WARNING, AppLocal.getIntString("message.syncerror"), ex);
        msg.show(this);
        return null;
    }
}
 

@Test
public void doesNotWriteRowOnTimeWhenStopped() throws KettleException, InterruptedException {
  TransMeta transMeta = new TransMeta( getClass().getResource( "safe-stop.ktr" ).getPath() );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( new String[] {} );
  trans.getSteps().get( 1 ).step.addRowListener( new RowAdapter() {
    @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
      trans.safeStop();
    }
  } );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 1, trans.getSteps().get( 1 ).step.getLinesRead() );
}
 

@Test
public void testGetDataFromFromFile() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-file-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 2, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 3, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

public void testSortCaseInsensitiveUniqueCaseSensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { false } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseInsensitiveUniqueCaseSensitive(), resultRows );
}
 

public String test( boolean compatibilityMode ) throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testJsonOutput" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testJsonOutput1", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the json output step
  // but first lets get a filename
  String jsonFileName = TestUtilities.createEmptyTempFile( "testJsonOutput1_" );
  StepMeta jsonOutputStep = createJsonOutputStep( "json output step", jsonFileName, registry );
  ( (JsonOutputMeta) jsonOutputStep.getStepMetaInterface() ).setCompatibilityMode( compatibilityMode );
  transMeta.addStep( jsonOutputStep );

  // create a TransHopMeta for jsonOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, jsonOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Create a dummy step and add it to the tranMeta
  String dummyStepName = "dummy step";
  StepMeta dummyStep = createDummyStep( dummyStepName, registry );
  transMeta.addStep( dummyStep );

  // create a TransHopMeta for the
  TransHopMeta hop_outputJson_dummyStep = new TransHopMeta( jsonOutputStep, dummyStep );
  transMeta.addTransHop( hop_outputJson_dummyStep );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( dummyStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
  trans.startThreads();
  trans.waitUntilFinished();

  // get the results and return it
  File outputFile = new File( jsonFileName + ".js" );
  String jsonStructure = FileUtils.readFileToString( outputFile );

  return jsonStructure;
}
 

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test with Like predicate.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookupWithLikePredicate() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "] by 'LIKE'";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( true );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "STRING" } );
  dbl.setKeyCondition( new String[] { "LIKE" } );
  dbl.setStreamKeyField1( new String[] { "str_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table ["
      + lookup_table + "] on database ["
      + dbInfo + "] using LIKE condition" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test case for parameters using a simple transformation. Here blocking some unwise usage of parameters.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans3() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans3" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${JAVA_HOME}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "${JAVA_HOME}", "default1", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "default2", "Parameter 2" );
  trans.setParameterValue( "${JAVA_HOME}", "param1" );
  // PARAM2 is not set

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData3();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

public void testProcessRow() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "WebServiceTest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi = new TransHopMeta( injectorStep, dummyStep1 );
  transMeta.addTransHop( hi );

  //
  // Create a String Cut step
  //
  String webServiceStepname = "web service step";
  WebServiceMeta scm = new WebServiceMeta();

  // scm.setUrl(HTTP_LOCALHOST_9998+ "wsdl");
  // scm.setOperationName("CelciusToFahrenheit");
  DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
  DocumentBuilder db = dbf.newDocumentBuilder();
  Document doc = db.parse( new InputSource( new java.io.StringReader( STEP_META ) ) );
  scm.loadXML( doc.getFirstChild(), null, (IMetaStore) null );

  String webServicePid = registry.getPluginId( StepPluginType.class, scm );
  StepMeta webServiceStep = new StepMeta( webServicePid, webServiceStepname, scm );
  transMeta.addStep( webServiceStep );

  TransHopMeta hi2 = new TransHopMeta( dummyStep1, webServiceStep );
  transMeta.addTransHop( hi2 );

  //
  // Create a dummy step 2
  //
  String dummyStepname2 = "dummy step 2";
  DummyTransMeta dm2 = new DummyTransMeta();

  String dummyPid2 = registry.getPluginId( StepPluginType.class, dm2 );
  StepMeta dummyStep2 = new StepMeta( dummyPid2, dummyStepname2, dm2 );
  transMeta.addStep( dummyStep2 );

  TransHopMeta hi3 = new TransHopMeta( webServiceStep, dummyStep2 );
  transMeta.addTransHop( hi3 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector dummyRc1 = new RowStepCollector();
  si.addRowListener( dummyRc1 );

  si = trans.getStepInterface( webServiceStepname, 0 );
  RowStepCollector webServiceRc = new RowStepCollector();
  si.addRowListener( webServiceRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData( createRowMetaInterface(), new Object[][] { new Object[] { 10 } } );
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldRows =
    createData( createOutputRowMetaInterface(), new Object[][] { new Object[] { 10,
      new BigDecimal( 20 ) } } );
  List<RowMetaAndData> resultRows2 = webServiceRc.getRowsWritten();
  assertEquals( goldRows, resultRows2 );
}
 

/**
 * Basic Test case for database lookup.
 */
@Test
public void basicDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( false );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

public void testSortCaseSensitiveUniqueCaseSensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { true } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseSensitiveUniqueCaseSensitive(), resultRows );
}
 

protected Trans createAndTestTrans( PluginRegistry registry, TransMeta transMeta, StepMeta injectorStep,
  StepMeta csvInputStep, String fileName, int numRows ) throws KettleException {
  TransHopMeta hi = new TransHopMeta( injectorStep, csvInputStep );
  transMeta.addTransHop( hi );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( csvInputStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector dummyRc1 = new RowStepCollector();
  si.addRowListener( dummyRc1 );

  RowProducer rp = trans.addRowProducer( injectorStep.getName(), 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData( fileName );
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
  List<RowMetaAndData> goldenImageRows = createResultData1();

  checkRows( goldenImageRows, resultRows, numRows );

  return trans;
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT.
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

public boolean runEngine( boolean printDescription ) throws KettleException {
  System.gc();

  KettleEnvironment.init();

  transMeta = new TransMeta( filename );
  transMeta.setVariable( "NR_OF_ROWS", Long.toString( records ) );
  if ( printDescription ) {
    printTransDescription();
  }

  // Replace the TARGET database connection settings with the one provided
  if ( targetDatabaseMeta != null ) {
    transMeta.addOrReplaceDatabase( targetDatabaseMeta );
  }

  // OK, now run this transFormation.
  Trans trans = new Trans( transMeta );
  trans.setLogLevel( logLevel );

  try {
    trans.prepareExecution( null );
  } catch ( Exception e ) {
    System.err.println( KettleLogStore.getAppender().getBuffer( trans.getLogChannelId(), true ) );

    trans.getLogChannel().logError( "Error preparing / initializing transformation", e );

    return false;
  }

  if ( !Utils.isEmpty( rowListenerStep ) ) {
    StepInterface step = trans.findRunThread( rowListenerStep );
    if ( step != null ) {
      step.addRowListener( rowListener );
    }
  }

  long startTime = System.currentTimeMillis();

  trans.startThreads();

  trans.waitUntilFinished();

  long stopTime = System.currentTimeMillis();

  result = trans.getResult();

  runTime = (double) ( stopTime - startTime ) / 1000;
  speed = records / ( runTime );

  printStats( "V3 results", records, runTime, speed );

  return true;
}
 

public void testCaseInsensitiveNoPreviousSort() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { true } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataCaseInsensitiveNoPreviousSort(), resultRows );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of two (i.e. not autocommit and not the input row size)
 */
@Test
public void testExecSQLRow3() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 2 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of zero (i.e. autocommit)
 */
@Test
public void testExecSQLRow1() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 0 ); // use Autocommit
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}