Java Code Examples for org.pentaho.di.core.plugins.PluginRegistry#getPluginId()

/**
 * @param stepname
 *          The name of the new step
 * @param stepMetaInterface
 *          The step metadata interface to use (TextFileInputMeta, etc)
 */
public StepMeta( String stepname, StepMetaInterface stepMetaInterface ) {
  if ( stepMetaInterface != null ) {
    PluginRegistry registry = PluginRegistry.getInstance();
    this.stepid = registry.getPluginId( StepPluginType.class, stepMetaInterface );
    setDeprecationAndSuggestedStep();
  }
  this.name = stepname;
  setStepMetaInterface( stepMetaInterface );

  selected = false;
  distributes = true;
  copiesString = "1";
  location = new Point( 0, 0 );
  drawstep = false;
  description = null;
  stepPartitioningMeta = new StepPartitioningMeta();
  // targetStepPartitioningMeta = new StepPartitioningMeta();

  clusterSchema = null; // non selected by default.

  remoteInputSteps = new ArrayList<RemoteStep>();
  remoteOutputSteps = new ArrayList<RemoteStep>();

  attributesMap = new HashMap<String, Map<String, String>>();
}
 

/**
 * Creates a row generator step for this class..
 *
 * @param name
 * @param registry
 * @return
 */
private StepMeta createRowGeneratorStep( String name, PluginRegistry registry ) {

  // Default the name if it is empty
  String testFileOutputName = ( Utils.isEmpty( name ) ? "generate rows" : name );

  // create the RowGenerator and Step Meta
  RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
  String rowGeneratorPid = registry.getPluginId( StepPluginType.class, rowGeneratorMeta );
  StepMeta generateRowsStep = new StepMeta( rowGeneratorPid, testFileOutputName, rowGeneratorMeta );

  // Set the field names, types and values
  rowGeneratorMeta.setFieldName( new String[] { "Id", "State", "City" } );
  rowGeneratorMeta.setFieldType( new String[] { "Integer", "String", "String" } );
  rowGeneratorMeta.setValue( new String[] { "1", "Orlando", "Florida" } );
  rowGeneratorMeta.setFieldLength( new int[] { -1, -1, -1 } );
  rowGeneratorMeta.setFieldPrecision( new int[] { -1, -1, -1 } );
  rowGeneratorMeta.setGroup( new String[] { "", "", "" } );
  rowGeneratorMeta.setDecimal( new String[] { "", "", "" } );
  rowGeneratorMeta.setCurrency( new String[] { "", "", "" } );
  rowGeneratorMeta.setFieldFormat( new String[] { "", "", "" } );
  rowGeneratorMeta.setRowLimit( "10" );

  // return the step meta
  return generateRowsStep;
}
 

/**
 * Creates a row generator step for this class..
 * 
 * @param name
 * @param registry
 * @return
 */
private StepMeta createRowGeneratorStep( String name, PluginRegistry registry ) {

  // Default the name if it is empty
  String testFileOutputName = ( Utils.isEmpty( name ) ? "generate rows" : name );

  // create the RowGenerator and Step Meta
  RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
  String rowGeneratorPid = registry.getPluginId( StepPluginType.class, rowGeneratorMeta );
  StepMeta generateRowsStep = new StepMeta( rowGeneratorPid, testFileOutputName, rowGeneratorMeta );

  // Set the field names, types and values
  rowGeneratorMeta.setFieldName( new String[] { "Id", "State", "City" } );
  rowGeneratorMeta.setFieldType( new String[] { "Integer", "String", "String" } );
  rowGeneratorMeta.setValue( new String[] { "1", "Florida", "Orlando" } );
  rowGeneratorMeta.setFieldLength( new int[] { -1, -1, -1 } );
  rowGeneratorMeta.setFieldPrecision( new int[] { -1, -1, -1 } );
  rowGeneratorMeta.setGroup( new String[] { "", "", "" } );
  rowGeneratorMeta.setDecimal( new String[] { "", "", "" } );
  rowGeneratorMeta.setCurrency( new String[] { "", "", "" } );
  rowGeneratorMeta.setFieldFormat( new String[] { "", "", "" } );
  rowGeneratorMeta.setRowLimit( "10" );

  // return the step meta
  return generateRowsStep;
}
 

/**
 * Create an injector step.
 *
 * @param name
 * @param registry
 * @return StepMeta
 */
public static StepMeta createInjectorStep( String name, PluginRegistry pluginRegistry ) {
  // create an injector step...
  InjectorMeta injectorMeta = new InjectorMeta();

  // Set the information of the injector
  String injectorPid = pluginRegistry.getPluginId( StepPluginType.class, injectorMeta );
  StepMeta injectorStep = new StepMeta( injectorPid, name, injectorMeta );

  return injectorStep;
}
 

/**
 * Create a dummy step for this class.
 *
 * @param name
 * @param registry
 * @return
 */
private StepMeta createDummyStep( String name, PluginRegistry registry ) {
  // Create a dummy step 1 and add it to the tranMeta
  String dummyStepName = "dummy step";
  DummyTransMeta dm1 = new DummyTransMeta();
  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep = new StepMeta( dummyPid1, dummyStepName, dm1 );

  return dummyStep;
}
 

public StepMeta createInjectorStep( TransMeta transMeta, PluginRegistry registry ) {
  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  return injectorStep;
}
 

/**
 * Test case for Combination lookup/update.
 */
public void testCombinationLookup() throws Exception {
  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta lookupDBInfo = transMeta.findDatabase( "lookup" );

  // Execute our setup SQLs in the database.
  Database lookupDatabase = new Database( transMeta, lookupDBInfo );
  lookupDatabase.connect();
  createTables( lookupDatabase );
  createData( lookupDatabase );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "lookup" ) );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "DLR_CD, DLR_NM, DLR_DESC ";
  selectSQL += "FROM " + source_table + " ORDER BY ORDNO;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setLocation( 150, 100 );
  fromstep.setDraw( true );
  fromstep.setDescription( "Reads information from table ["
    + source_table + "] on database [" + lookupDBInfo + "]" );
  transMeta.addStep( fromstep );

  //
  // create the combination lookup/update step...
  //
  String lookupstepname = "lookup from [lookup]";
  CombinationLookupMeta clm = new CombinationLookupMeta();
  String[] lookupKey = { "DLR_CD" };
  clm.setTablename( target_table );
  clm.setKeyField( lookupKey );
  clm.setKeyLookup( lookupKey );
  clm.setTechnicalKeyField( "ID" );
  clm.setTechKeyCreation( CombinationLookupMeta.CREATION_METHOD_TABLEMAX );
  clm.setDatabaseMeta( lookupDBInfo );

  String lookupstepid = registry.getPluginId( StepPluginType.class, clm );
  StepMeta lookupstep = new StepMeta( lookupstepid, lookupstepname, clm );
  lookupstep.setDescription( "Looks up information from table [lookup] on database [" + lookupDBInfo + "]" );
  transMeta.addStep( lookupstep );

  TransHopMeta hi = new TransHopMeta( fromstep, lookupstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.execute( null );

  trans.waitUntilFinished();

  checkResults( lookupDatabase );
}
 

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( true );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

@Test
public void testFilterConditionRefersToNonExistingFields() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "filterrowstest" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  // Create a filter rows step
  String filterStepName = "filter rows step";
  FilterRowsMeta frm = new FilterRowsMeta();
  Condition condition = new Condition();
  String nonExistingFieldName = "non-existing-field";
  condition.setLeftValuename( nonExistingFieldName );
  condition.setFunction( 8 ); //IS NOT
  condition.setRightValuename( null );
  condition.setOperator( 0 );
  frm.setCondition( condition );

  String filterRowsStepPid = registry.getPluginId( StepPluginType.class, frm );
  StepMeta filterRowsStep = new StepMeta( filterRowsStepPid, filterStepName, frm );
  transMeta.addStep( filterRowsStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, filterRowsStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );

  // add rows
  List<RowMetaAndData> inputList = createIntegerData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 1, trans.getErrors() ); //expect errors

}
 

/**
 * Test case for javascript functionality: ltrim(), rtrim(), trim().
 */
public void testLuhnCheck() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript LuhnCheck" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var str = string;\n" + "var bool = LuhnCheck(str);" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] { "bool" } );
  svm.setRename( new String[] { "" } );
  svm.setType( new int[] { ValueMetaInterface.TYPE_BOOLEAN } );
  svm.setLength( new int[] { -1 } );
  svm.setPrecision( new int[] { -1 } );
  svm.setReplace( new boolean[] { false } );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}
 

private StepMeta createTextFileOutputStep( String name, String textFileName, String compression,
  PluginRegistry registry ) {

  // Create a Text File Output step
  String testFileOutputName = name;
  TextFileOutputMeta textFileOutputMeta = new TextFileOutputMeta();
  String textFileInputPid = registry.getPluginId( StepPluginType.class, textFileOutputMeta );
  StepMeta textFileOutputStep = new StepMeta( textFileInputPid, testFileOutputName, textFileOutputMeta );

  // initialize the fields
  TextFileField[] fields = new TextFileField[3];
  for ( int idx = 0; idx < fields.length; idx++ ) {
    fields[idx] = new TextFileField();
  }

  // populate the fields
  // it is important that the setPosition(int)
  // is invoked with the correct position as
  // we are testing the reading of a delimited file.
  fields[0].setName( "id" );
  fields[0].setType( ValueMetaInterface.TYPE_INTEGER );
  fields[0].setFormat( "" );
  fields[0].setLength( -1 );
  fields[0].setPrecision( -1 );
  fields[0].setCurrencySymbol( "" );
  fields[0].setDecimalSymbol( "" );
  fields[0].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  // here we're swapping the order of the last 2 columns
  fields[1].setName( "city" );
  fields[1].setType( ValueMetaInterface.TYPE_STRING );
  fields[1].setFormat( "" );
  fields[1].setLength( -1 );
  fields[1].setPrecision( -1 );
  fields[1].setCurrencySymbol( "" );
  fields[1].setDecimalSymbol( "" );
  fields[1].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  fields[2].setName( "state" );
  fields[2].setType( ValueMetaInterface.TYPE_STRING );
  fields[2].setFormat( "" );
  fields[2].setLength( -1 );
  fields[2].setPrecision( -1 );
  fields[2].setCurrencySymbol( "" );
  fields[2].setDecimalSymbol( "" );
  fields[2].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );

  // call this to allocate the number of fields
  textFileOutputMeta.allocate( 3 );
  textFileOutputMeta.setOutputFields( fields );

  // set meta properties- these were determined by running Spoon
  // and setting up the transformation we are setting up here.
  // i.e. - the dialog told me what I had to set to avoid
  // NPEs during the transformation.

  // We need a file name so we will generate a temp file
  textFileOutputMeta.setFileName( textFileName );
  textFileOutputMeta.setFileNameInField( false );
  textFileOutputMeta.setExtension( EXTENSION );
  textFileOutputMeta.setEnclosure( "\"" );
  textFileOutputMeta.setFileCompression( compression );
  textFileOutputMeta.setSeparator( SEPARATOR );
  textFileOutputMeta.setFileFormat( TestUtilities.getFileFormat() );
  textFileOutputMeta.setAddToResultFiles( false );
  textFileOutputMeta.setNewline( TestUtilities.getEndOfLineCharacters() );
  textFileOutputMeta.setSeparator( ";" );
  textFileOutputMeta.setEnclosure( "\"" );

  return textFileOutputStep;

}
 

@Test
public void testEscapeCharacters() throws KettleException, IOException {
  PluginRegistry.addPluginType( ValueMetaPluginType.getInstance() );
  PluginRegistry.init( false );

  MySQLBulkLoader loader;
  MySQLBulkLoaderData ld = new MySQLBulkLoaderData();
  MySQLBulkLoaderMeta lm = new MySQLBulkLoaderMeta();

  TransMeta transMeta = new TransMeta();
  transMeta.setName( "loader" );

  PluginRegistry plugReg = PluginRegistry.getInstance();

  String loaderPid = plugReg.getPluginId( StepPluginType.class, lm );
  StepMeta stepMeta = new StepMeta( loaderPid, "loader", lm );
  Trans trans = new Trans( transMeta );
  transMeta.addStep( stepMeta );
  trans.setRunning( true );

  loader = spy( new MySQLBulkLoader( stepMeta, ld, 1, transMeta, trans ) );

  RowMeta rm = new RowMeta();
  ValueMetaString vm = new ValueMetaString( "I don't want NPE!" );
  rm.addValueMeta( vm );
  RowMeta spyRowMeta = spy( new RowMeta() );
  when( spyRowMeta.getValueMeta( anyInt() ) ).thenReturn( vm );
  loader.setInputRowMeta( spyRowMeta );

  MySQLBulkLoaderMeta smi = new MySQLBulkLoaderMeta();
  smi.setFieldStream( new String[] { "Test" } );
  smi.setFieldFormatType( new int[] { MySQLBulkLoaderMeta.FIELD_FORMAT_TYPE_STRING_ESCAPE } );
  smi.setEscapeChar( "\\" );
  smi.setEnclosure( "\"" );
  smi.setDatabaseMeta( mock( DatabaseMeta.class ) );

  MySQLBulkLoaderData sdi = new MySQLBulkLoaderData();
  sdi.keynrs = new int[1];
  sdi.keynrs[0] = 0;
  sdi.fifoStream = mock( OutputStream.class );
  loader.init( smi, sdi );
  loader.first = false;

  when( loader.getRow() ).thenReturn( new String[] { "test\"Escape\\" } );
  loader.processRow( smi, sdi );
  verify( sdi.fifoStream, times( 1 ) ).write( "test\\\"Escape\\\\".getBytes() );
}
 

@Before
public void setUp() throws KettleDatabaseException, SQLException {
  connection = DriverManager.getConnection( "jdbc:h2:mem:PERSON;" );
  connection.setAutoCommit( false );
  PreparedStatement stmt = connection.prepareStatement( "CREATE TABLE PERSON (ID INT PRIMARY KEY, personName VARCHAR(64) )" );
  stmt.execute();
  stmt.close();
  stmt = connection.prepareStatement( "INSERT INTO PERSON (ID, personName) VALUES (?, ?)" );
  for ( int i = 0; i < ROW_FOR_UPDATE + ROW_FOR_DELETE; i++ ) {
    stmt.setInt( 1, i );
    stmt.setString( 2, "personName" + i );
    stmt.addBatch();
  }
  stmt.executeBatch();
  stmt.close();
  connection.commit();

  PluginRegistry pluginRegistry = PluginRegistry.getInstance();

  transMeta = new TransMeta();
  transMeta.setName( "SynchronizeAfterMerge" );

  InjectorMeta injectorMeta = new InjectorMeta();
  String injectorPid = pluginRegistry.getPluginId( StepPluginType.class, injectorMeta );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, injectorMeta );
  transMeta.addStep( injectorStep );

  DatabaseMeta dbMeta = spy( new DatabaseMeta() );
  dbMeta.setDatabaseType( "H2" );
  when( dbMeta.getURL() ).thenReturn( "jdbc:h2:mem:PERSON;" );
  when( dbMeta.supportsErrorHandlingOnBatchUpdates() ).thenReturn( false );

  SynchronizeAfterMergeMeta synchronizeAfterMergeMeta = new SynchronizeAfterMergeMeta();
  //set commit size
  synchronizeAfterMergeMeta.setCommitSize( COMMIT_SIZE );
  synchronizeAfterMergeMeta.setDatabaseMeta( dbMeta );
  synchronizeAfterMergeMeta.setKeyCondition( new String[] { "=" } );
  synchronizeAfterMergeMeta.setKeyLookup( new String[] { "ID" } );
  synchronizeAfterMergeMeta.setKeyStream( new String[] { "personName" } );
  synchronizeAfterMergeMeta.setKeyStream2( new String[] { null } );
  synchronizeAfterMergeMeta.setUpdate( new Boolean[] { Boolean.TRUE } );
  synchronizeAfterMergeMeta.setOperationOrderField( "flag" );
  synchronizeAfterMergeMeta.setOrderDelete( DELETE_FLAG );
  synchronizeAfterMergeMeta.setOrderInsert( INSERT_FLAG );
  synchronizeAfterMergeMeta.setOrderUpdate( UPDATE_FLAG );
  synchronizeAfterMergeMeta.setPerformLookup( true );

  synchronizeAfterMergeMeta.setTableName( "PERSON" );
  synchronizeAfterMergeMeta.settablenameInField( false );
  synchronizeAfterMergeMeta.settablenameField( null );
  synchronizeAfterMergeMeta.setUseBatchUpdate( true );
  synchronizeAfterMergeMeta.setUpdateLookup( new String[] { "ID" } );
  synchronizeAfterMergeMeta.setUpdateStream( new String[] { "personName" } );

  String synchronizeAfterMergePid = pluginRegistry.getPluginId( StepPluginType.class, synchronizeAfterMergeStepname );
  StepMeta synchronizeAfterMerge = new StepMeta( synchronizeAfterMergePid, synchronizeAfterMergeStepname, synchronizeAfterMergeMeta );
  transMeta.addStep( synchronizeAfterMerge );

  String dummyResultStepName = "dummyResultStepName";
  DummyTransMeta dummyResultTransMeta = new DummyTransMeta();
  String dummyResultPid = pluginRegistry.getPluginId( StepPluginType.class, dummyResultTransMeta );
  StepMeta dummyResultStep = new StepMeta( dummyResultPid, dummyResultStepName, dummyResultTransMeta );
  transMeta.addStep( dummyResultStep );

  String dummyErrorStepName = "dummyErrorStepName";
  DummyTransMeta dummyErrorTransMeta = new DummyTransMeta();
  String dummyErrorPid = pluginRegistry.getPluginId( StepPluginType.class, dummyErrorTransMeta );
  StepMeta dummyErrorStep = new StepMeta( dummyErrorPid, dummyErrorStepName, dummyErrorTransMeta );
  transMeta.addStep( dummyErrorStep );

  StepErrorMeta stepErrorMeta = new StepErrorMeta( transMeta, synchronizeAfterMerge, dummyErrorStep );
  stepErrorMeta.setEnabled( true );
  synchronizeAfterMerge.setStepErrorMeta( stepErrorMeta );

  TransHopMeta injSynch = new TransHopMeta( injectorStep, synchronizeAfterMerge );
  transMeta.addTransHop( injSynch );

  TransHopMeta synchDummyResult = new TransHopMeta( synchronizeAfterMerge, dummyResultStep );
  transMeta.addTransHop( synchDummyResult );

  TransHopMeta synchDummyError = new TransHopMeta( synchronizeAfterMerge, dummyErrorStep );
  transMeta.addTransHop( synchDummyError );
}
 

void testOneRow(String transName, String[] referenceValues, String[] comparisonValues, Object[] goldenImageRowValues) throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( transName );
  PluginRegistry registry = PluginRegistry.getInstance();

  // Create a merge rows step
  String mergeRowsStepName = "merge rows step";
  MergeRowsMeta mergeRowsMeta = new MergeRowsMeta();

  String mergeRowsStepPid = registry.getPluginId( StepPluginType.class, mergeRowsMeta );
  StepMeta mergeRowsStep = new StepMeta( mergeRowsStepPid, mergeRowsStepName, mergeRowsMeta );
  transMeta.addStep( mergeRowsStep );

  mergeRowsMeta.setKeyFields( new String[]{ keyField } );
  mergeRowsMeta.setValueFields( new String[]{ compareField } );
  mergeRowsMeta.setFlagField( flagField );

  List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();

  //
  // create a reference stream (row generator step)
  //
  createRowGenerator(
    transMeta, registry,
    "reference row generator", referenceValues,
    mergeRowsStep, mergeRowsMeta,
    0
  );

  //
  // create a comparison stream (row generator step)
  //
  createRowGenerator(
    transMeta, registry,
    "comparison row generator", comparisonValues,
    mergeRowsStep, mergeRowsMeta,
    1
  );

  // Now execute the transformation
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( mergeRowsStepName, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();
  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData( goldenImageRowValues );
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

public void testSortCaseSensitiveUniqueCaseSensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { true } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { false } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseSensitiveUniqueCaseSensitive(), resultRows );
}
 

protected Trans createAndTestTrans( PluginRegistry registry, TransMeta transMeta, StepMeta injectorStep,
  StepMeta csvInputStep, String fileName, int numRows ) throws KettleException {
  TransHopMeta hi = new TransHopMeta( injectorStep, csvInputStep );
  transMeta.addTransHop( hi );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( csvInputStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector dummyRc1 = new RowStepCollector();
  si.addRowListener( dummyRc1 );

  RowProducer rp = trans.addRowProducer( injectorStep.getName(), 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData( fileName );
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRc1.getRowsWritten();
  List<RowMetaAndData> goldenImageRows = createResultData1();

  checkRows( goldenImageRows, resultRows, numRows );

  return trans;
}
 

@Before
public void setUp() throws Exception {
  KettleEnvironment.init();
  PluginRegistry.init( false );
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "loader" );

  lm = new MonetDBBulkLoaderMeta();
  ld = new MonetDBBulkLoaderData();

  PluginRegistry plugReg = PluginRegistry.getInstance();

  String loaderPid = plugReg.getPluginId( StepPluginType.class, lm );

  stepMeta = new StepMeta( loaderPid, "loader", lm );
  Trans trans = new Trans( transMeta );
  transMeta.addStep( stepMeta );

  loader = new MonetDBBulkLoader( stepMeta, ld, 1, transMeta, trans );

  List<String> attributes =
      Arrays.asList( "dbConnectionName", "schemaName", "tableName", "logFile", "fieldSeparator", "fieldEnclosure",
          "NULLrepresentation", "encoding", "truncate", "fullyQuoteSQL", "fieldTable", "fieldStream",
          "fieldFormatOk" );

  // Important note - the "databaseMeta" is not tested here as it's tied to the dbConnectionName. Since the loader
  // has assymetry here, we have to not test the databaseMeta, or we have to do surgery on the MonetDBBulkLoaderMeta
  // so that it's symmetric (and has no dependent variables like this).
  // MB -5/2016

  // Note - autoSchema is not persisted and neither is autoStringWidths. (Old functionality).

  FieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
      new ArrayLoadSaveValidator<String>( new StringLoadSaveValidator(), 5 );


  Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
  attrValidatorMap.put( "fieldTable", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "fieldStream", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "fieldFormatOk",
      new PrimitiveBooleanArrayLoadSaveValidator( new BooleanLoadSaveValidator(), 5 ) );

  Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

  loadSaveTester =
      new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(),
          new HashMap<String, String>(), new HashMap<String, String>(), attrValidatorMap, typeValidatorMap, this );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of three (i.e. not autocommit but equal to input row
 * size)
 */
@Test
public void testExecSQLRow4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 3 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

private StepMeta createTextFileInputStep2( String name, String fileName, PluginRegistry registry ) {

    // Create a Text File Input step
    TextFileInputMeta textFileInputMeta = new TextFileInputMeta();
    String textFileInputPid = registry.getPluginId( StepPluginType.class, textFileInputMeta );
    StepMeta textFileInputStep = new StepMeta( textFileInputPid, name, textFileInputMeta );

    // initialize the fields
    TextFileInputField[] fields = new TextFileInputField[2];
    for ( int idx = 0; idx < fields.length; idx++ ) {
      fields[idx] = new TextFileInputField();
    }

    // populate the fields
    // it is important that the setPosition(int)
    // is invoked with the correct position as
    // we are testing the reading of a delimited file.
    fields[0].setName( "a" );
    fields[0].setType( ValueMetaInterface.TYPE_INTEGER );
    fields[0].setFormat( "" );
    fields[0].setLength( 15 );
    fields[0].setPrecision( -1 );
    fields[0].setCurrencySymbol( "" );
    fields[0].setDecimalSymbol( "" );
    fields[0].setGroupSymbol( "" );
    fields[0].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );
    fields[0].setPosition( 1 );

    fields[1].setName( "b" );
    fields[1].setType( ValueMetaInterface.TYPE_INTEGER );
    fields[1].setFormat( "" );
    fields[1].setLength( 15 );
    fields[1].setPrecision( -1 );
    fields[1].setCurrencySymbol( "" );
    fields[1].setDecimalSymbol( "" );
    fields[1].setGroupSymbol( "" );
    fields[1].setTrimType( ValueMetaInterface.TRIM_TYPE_NONE );
    fields[1].setPosition( 2 );

    // call this so that we allocate the arrays
    // for files, fields and filters.
    // we are testing one file and one set
    // of fields. No filters
    textFileInputMeta.allocate( 1, 1, 0 );

    // set meta properties- these were determined by running Spoon
    // and setting up the transformation we are setting up here.
    // i.e. - the dialog told me what I had to set to avoid
    // NPEs during the transformation.
    String[] filesRequired = new String[] { "N" };
    String[] includeSubfolders = new String[] { "N" };
    textFileInputMeta.setEnclosure( "$[09]" );
    textFileInputMeta.setAddResultFile( false );
    textFileInputMeta.setFileName( new String[] { fileName } );
    textFileInputMeta.setFileFormat( TestUtilities.getFileFormat() );
    textFileInputMeta.setFileType( "CSV" );
    textFileInputMeta.setSeparator( ";" );
    textFileInputMeta.setFileRequired( filesRequired );
    textFileInputMeta.setIncludeSubFolders( includeSubfolders );
    textFileInputMeta.setInputFields( fields );
    textFileInputMeta.setHeader( false );
    textFileInputMeta.setNrHeaderLines( 0 );
    textFileInputMeta.setFileCompression( "None" );
    textFileInputMeta.setNoEmptyLines( true );
    textFileInputMeta.setRowLimit( 0 );
    textFileInputMeta.setDateFormatLocale( new Locale( "en_US" ) );
    textFileInputMeta.setIncludeFilename( false );

    return textFileInputStep;
  }
 

private StepMeta createJsonOutputStep( String name, String jsonFileName, PluginRegistry registry ) {

    // Create a Text File Output step
    String testFileOutputName = name;
    JsonOutputMeta jsonOutputMeta = new JsonOutputMeta();
    String textFileInputPid = registry.getPluginId( StepPluginType.class, jsonOutputMeta );
    StepMeta jsonOutputStep = new StepMeta( textFileInputPid, testFileOutputName, jsonOutputMeta );

    // initialize the fields
    JsonOutputField[] fields = new JsonOutputField[3];
    for ( int idx = 0; idx < fields.length; idx++ ) {
      fields[idx] = new JsonOutputField();
    }

    // populate the fields
    // it is important that the setPosition(int)
    // is invoked with the correct position as
    // we are testing the reading of a delimited file.
    fields[0].setFieldName( "id" );
    fields[0].setElementName( "id" );

    fields[1].setFieldName( "state" );
    fields[1].setElementName( "state" );

    fields[2].setFieldName( "city" );
    fields[2].setElementName( "city" );

    // call this to allocate the number of fields
    jsonOutputMeta.allocate( fields.length );
    jsonOutputMeta.setOutputFields( fields );

    // set meta properties- these were determined by running Spoon
    // and setting up the transformation we are setting up here.
    // i.e. - the dialog told me what I had to set to avoid
    // NPEs during the transformation.

    // We need a file name so we will generate a temp file
    jsonOutputMeta.setOperationType( JsonOutputMeta.OPERATION_TYPE_WRITE_TO_FILE );
    jsonOutputMeta.setOutputValue( "data" );
    jsonOutputMeta.setFileName( jsonFileName );
    jsonOutputMeta.setExtension( "js" );
    jsonOutputMeta.setNrRowsInBloc( "0" ); // a single "data" contains an array of all records
    jsonOutputMeta.setJsonBloc( "data" );

    return jsonOutputStep;
  }