Java Code Examples for org.pentaho.di.trans.step.StepInterface#addRowListener()

public synchronized void addRowListenersToTransformation( final Trans trans ) {

    final TransDebugMeta self = this;

    // for every step in the map, add a row listener...
    //
    for ( final Map.Entry<StepMeta, StepDebugMeta> entry : stepDebugMetaMap.entrySet() ) {
      final StepMeta stepMeta = entry.getKey();
      final StepDebugMeta stepDebugMeta = entry.getValue();

      // What is the transformation thread to attach a listener to?
      //
      for ( StepInterface baseStep : trans.findBaseSteps( stepMeta.getName() ) ) {
        baseStep.addRowListener( new RowAdapter() {
          public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
            rowWrittenEventHandler( rowMeta, row, stepDebugMeta, trans, self );
          }
        } );
      }
    }
  }
 

/**
 * Create, execute, and return the row listener attached to the output step with complete results from the execution.
 *
 * @param checkSumType
 *          Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
 * @param compatibilityMode
 *          Use compatibility mode for CheckSum
 * @param fieldSeparatorString
 *          The string separate multiple fields with
 * @param inputs
 *          Array of objects representing row data
 * @param inputValueMetas
 *          metas to be processed
 * @return RowListener with results.
 */
private MockRowListener executeHexTest( int checkSumType, boolean compatibilityMode, boolean oldChecksumBehaviour,
    String fieldSeparatorString, Object[] inputs, ValueMetaInterface... inputValueMetas ) throws Exception {

  String[] fieldNames = new String[inputValueMetas.length];
  RowMeta inputRowMeta = new RowMeta();
  for ( int i = 0; i < inputValueMetas.length; i++ ) {
    inputRowMeta.addValueMeta( inputValueMetas[i] );
    fieldNames[i] = inputValueMetas[i].getName();
  }

  Trans trans =
      buildHexadecimalChecksumTrans( checkSumType, compatibilityMode, oldChecksumBehaviour, fieldSeparatorString,
          fieldNames );

  trans.prepareExecution( null );

  StepInterface output = trans.getRunThread( "Output", 0 );
  MockRowListener listener = new MockRowListener();
  output.addRowListener( listener );

  RowProducer rp = trans.addRowProducer( "CheckSum", 0 );

  ( (BaseStep) trans.getRunThread( "CheckSum", 0 ) ).setInputRowMeta( inputRowMeta );

  trans.startThreads();

  rp.putRow( inputRowMeta, inputs );
  rp.finished();

  trans.waitUntilFinished();
  trans.stopAll();
  trans.cleanup();
  return listener;
}
 

/**
 * Test case for normal table output where the table is included in the instream, but the tablename is not stored in
 * the table.
 */
public void testTableOutputJIRA897() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output JIRA897 test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table1, createSourceRowMetaInterface1() );
  createTable( database, target_table2, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTableNameInField( true );
  tom.setTableNameField( "TABLE" );
  tom.setTableNameInTable( false );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to tables on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createJIRA897DataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();

  // The name of the table should still be in here.
  List<RowMetaAndData> goldRows = createJIRA897DataRows();
  checkRows( goldRows, resultRows );
  checkResultsJIRA897( database );
}
 

/**
 * Test case for injector step... also a show case on how to use injector.
 */
public void testInjector() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "injectortest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, dummyStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  checkRows( resultRows, inputList );
}
 

public static List<RowMetaAndData> executeTestTransformation( TransMeta transMeta, String injectorStepname,
    String testStepname, String dummyStepname, List<RowMetaAndData> inputData ) throws KettleException {
  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  // Capture the rows that come out of the dummy step...
  //
  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  // Add a row producer...
  //
  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );

  // Start the steps...
  //
  trans.startThreads();

  // Inject the actual test rows...
  //
  List<RowMetaAndData> inputList = inputData;
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  // Wait until the transformation is finished...
  //
  trans.waitUntilFinished();

  // If there is an error in the result, throw an exception here...
  //
  if ( trans.getResult().getNrErrors() > 0 ) {
    throw new KettleException( "Test transformation finished with errors. Check the log." );
  }

  // Return the result from the dummy step...
  //
  return dummyRc.getRowsRead();
}
 

/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( false );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

public static Map<String, RowStepCollector> executeTestTransformationError( TransMeta transMeta,
    String injectorStepname, String testStepname, String dummyStepname, String errorStepName,
    List<RowMetaAndData> inputData ) throws KettleException {
  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  // Capture the rows that come out of the dummy step...
  //
  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  StepInterface junit = trans.getStepInterface( testStepname, 0 );
  RowStepCollector dummyJu = new RowStepCollector();
  junit.addRowListener( dummyJu );

  // add error handler
  StepInterface er = trans.getStepInterface( errorStepName, 0 );
  RowStepCollector erColl = new RowStepCollector();
  er.addRowListener( erColl );

  // Add a row producer...
  //
  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );

  // Start the steps...
  //
  trans.startThreads();

  // Inject the actual test rows...
  //
  List<RowMetaAndData> inputList = inputData;
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  // Wait until the transformation is finished...
  //
  trans.waitUntilFinished();

  // If there is an error in the result, throw an exception here...
  //
  if ( trans.getResult().getNrErrors() > 0 ) {
    throw new KettleException( "Test transformation finished with errors. Check the log." );
  }

  // Return the result from the dummy step...
  Map<String, RowStepCollector> ret = new HashMap<String, RowStepCollector>();
  ret.put( dummyStepname, dummyRc );
  ret.put( errorStepName, erColl );
  ret.put( testStepname, dummyJu );
  return ret;
}
 

/**
 * Test case for parameters using a simple transformation.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans1() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans1" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test case for javascript functionality: trans_Status and SKIP_TRANSFORMATION. Regression test case for JIRA defect
 * PDI-364.
 */
public void testTransStatus() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript trans_Status" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  // process 3 rows and skip the rest.
  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "trans_Status = CONTINUE_TRANSFORMATION;\n"
      + "if (getProcessCount(\"r\") > 3) {\n" + " \ttrans_Status = SKIP_TRANSFORMATION;\n" + "}" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] {} );
  svm.setRename( new String[] {} );
  svm.setType( new int[] {} );
  svm.setLength( new int[] {} );
  svm.setPrecision( new int[] {} );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData2();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}
 

public void testSortCaseInsensitiveUniqueCaseSensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { false } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseInsensitiveUniqueCaseSensitive(), resultRows );
}
 

/**
 * Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
 *
 * The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
 */
public void testTableInputWithParam() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTables( database );
  createData( database );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tii.setLookupFromStep( injectorStep );
  tii.setExecuteEachInputRow( true );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "ID, CODE ";
  selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setDescription( "Reads information from table [" + source_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( fromstepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

@Override
@Before
public void setUp() throws Exception {

  KettleEnvironment.init();

  /* SET UP TRANSFORMATION */

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "insert/update test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  /* SET UP DATABASE */
  // Create target table
  db = new Database( transMeta, dbInfo );
  db.connect();

  String source = db.getCreateTableStatement( TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true );
  db.execStatement( source );

  // populate target table
  for ( String sql : insertStatement ) {
    db.execStatement( sql );
  }

  /* SET UP TRANSFORMATION STEPS */

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  // create the update step...
  String updateStepName = "insert/update [" + TARGET_TABLE + "]";
  insupd = new InsertUpdateMeta();
  insupd.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  insupd.setTableName( TARGET_TABLE );

  insupd.setUpdateLookup( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdateStream( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdate( new Boolean[] { true, false } );

  String fromid = registry.getPluginId( StepPluginType.class, insupd );
  StepMeta updateStep = new StepMeta( fromid, updateStepName, insupd );
  updateStep.setDescription( "insert/update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]" );
  transMeta.addStep( updateStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, updateStep );
  transMeta.addTransHop( hi );

  /* PREPARE TRANSFORMATION EXECUTION */

  trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( updateStepName, 0 );
  rc = new RowStepCollector();
  si.addRowListener( rc );

  rp = trans.addRowProducer( injectorStepName, 0 );

}
 

public void testCaseSensitiveNoPreviousSort() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataCaseSensitiveNoPreviousSort(), resultRows );
}
 

/**
 * Process a single row. In our case, we send one row of data to a piece of transformation. In the transformation, we
 * look up the MappingInput step to send our rows to it. As a consequence, for the time being, there can only be one
 * MappingInput and one MappingOutput step in the Mapping.
 */
public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException {
  meta = (SimpleMappingMeta) smi;
  setData( (SimpleMappingData) sdi );
  SimpleMappingData simpleMappingData = getData();
  try {
    if ( first ) {
      first = false;
      simpleMappingData.wasStarted = true;

      // Rows read are injected into the one available Mapping Input step
      //
      String mappingInputStepname = simpleMappingData.mappingInput.getStepname();
      RowProducer rowProducer = simpleMappingData.mappingTrans.addRowProducer( mappingInputStepname, 0 );
      simpleMappingData.rowDataInputMapper = new RowDataInputMapper( meta.getInputMapping(), rowProducer );

      // Rows produced by the mapping are read and passed on.
      //
      String mappingOutputStepname = simpleMappingData.mappingOutput.getStepname();
      StepInterface outputStepInterface = simpleMappingData.mappingTrans.findStepInterface( mappingOutputStepname, 0 );
      RowOutputDataMapper outputDataMapper =
          new RowOutputDataMapper( meta.getInputMapping(), meta.getOutputMapping(), new PutRowInterface() {

            @Override
            public void putRow( RowMetaInterface rowMeta, Object[] rowData ) throws KettleStepException {
              SimpleMapping.this.putRow( rowMeta, rowData );
            }
          } );
      outputStepInterface.addRowListener( outputDataMapper );

      // Start the mapping/sub-transformation threads
      //
      simpleMappingData.mappingTrans.startThreads();
    }

    // The data we read we pass to the mapping
    //
    Object[] row = getRow();
    boolean rowWasPut = false;
    if ( row != null ) {
      while ( !( data.mappingTrans.isFinishedOrStopped() || rowWasPut ) ) {
        rowWasPut = data.rowDataInputMapper.putRow( getInputRowMeta(), row );
      }
    }

    if ( !rowWasPut ) {
      simpleMappingData.rowDataInputMapper.finished();
      simpleMappingData.mappingTrans.waitUntilFinished();
      setOutputDone();
      return false;
    }

    return true;
  } catch ( Throwable t ) {
    // Some unexpected situation occurred.
    // Better to stop the mapping transformation.
    //
    if ( simpleMappingData.mappingTrans != null ) {
      simpleMappingData.mappingTrans.stopAll();
    }

    // Forward the exception...
    //
    throw new KettleException( t );
  }
}
 

protected Trans createAndTestTrans( PluginRegistry registry, TransMeta transMeta, StepMeta injectorStep,
  StepMeta csvInputStep, String fileName, int numRows ) throws KettleException {
  TransHopMeta hi = new TransHopMeta( injectorStep, csvInputStep );
  transMeta.addTransHop( hi );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( csvInputStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector dummyRc1 = new RowStepCollector();
  si.addRowListener( dummyRc1 );

  RowProducer rp = trans.addRowProducer( injectorStep.getName(), 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData( fileName );
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
  List<RowMetaAndData> goldenImageRows = createResultData1();

  checkRows( goldenImageRows, resultRows, numRows );

  return trans;
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of three (i.e. not autocommit but equal to input row
 * size)
 */
@Test
public void testExecSQLRow4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 3 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

public void testCaseInsensitiveNoPreviousSort() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { true } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataCaseInsensitiveNoPreviousSort(), resultRows );
}
 

public String test( boolean compatibilityMode ) throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testJsonOutput" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testJsonOutput1", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the json output step
  // but first lets get a filename
  String jsonFileName = TestUtilities.createEmptyTempFile( "testJsonOutput1_" );
  StepMeta jsonOutputStep = createJsonOutputStep( "json output step", jsonFileName, registry );
  ( (JsonOutputMeta) jsonOutputStep.getStepMetaInterface() ).setCompatibilityMode( compatibilityMode );
  transMeta.addStep( jsonOutputStep );

  // create a TransHopMeta for jsonOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, jsonOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Create a dummy step and add it to the tranMeta
  String dummyStepName = "dummy step";
  StepMeta dummyStep = createDummyStep( dummyStepName, registry );
  transMeta.addStep( dummyStep );

  // create a TransHopMeta for the
  TransHopMeta hop_outputJson_dummyStep = new TransHopMeta( jsonOutputStep, dummyStep );
  transMeta.addTransHop( hop_outputJson_dummyStep );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( dummyStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
  trans.startThreads();
  trans.waitUntilFinished();

  // get the results and return it
  File outputFile = new File( jsonFileName + ".js" );
  String jsonStructure = FileUtils.readFileToString( outputFile );

  return jsonStructure;
}
 

public RowComparatorListener( StepInterface step, Object[]... data ) {
  this.data = data;
  step.addRowListener( this );
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans6() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set. And no default... so the variable will be set to "". not
  // to "Variable1"
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData6();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}