Java Code Examples for org.pentaho.di.trans.TransMeta#findDatabase()

@BeforeClass
public static void createDatabase() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }
  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  database = new Database( loggingObject, dbInfo );
  database.connect();
  createTables( database );
  createData( database );
}
 

@BeforeClass
public static void createDatabase() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }
  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  database = new Database( loggingObject, dbInfo );
  database.connect();
  createTables( database );
  createData( database );
}
 

/**
 * Insert all the databases from the repository into the TransMeta object, overwriting optionally
 * 
 * @param TransMeta
 *          The transformation to load into.
 * @param overWriteShared
 *          if an object with the same name exists, overwrite
 * @throws KettleException
 */
protected void readDatabases( TransMeta transMeta, boolean overWriteShared, List<DatabaseMeta> databaseMetas ) {
  for ( DatabaseMeta databaseMeta : databaseMetas ) {
    if ( overWriteShared || transMeta.findDatabase( databaseMeta.getName() ) == null ) {
      if ( databaseMeta.getName() != null ) {
        databaseMeta.shareVariablesWith( transMeta );
        transMeta.addOrReplaceDatabase( databaseMeta );
        if ( !overWriteShared ) {
          databaseMeta.setChanged( false );
        }
      }
    }
  }
  transMeta.clearChangedDatabases();
}
 

/**
 * Read all the databases from the repository, insert into the TransMeta object, overwriting optionally
 *
 * @param transMeta
 *          The transformation to load into.
 * @param overWriteShared
 *          if an object with the same name exists, overwrite
 * @throws KettleException
 */
public void readDatabases( TransMeta transMeta, boolean overWriteShared ) throws KettleException {
  try {
    ObjectId[] dbids = repository.getDatabaseIDs( false );
    for ( int i = 0; i < dbids.length; i++ ) {
      DatabaseMeta databaseMeta = repository.loadDatabaseMeta( dbids[i], null ); // reads last version
      databaseMeta.shareVariablesWith( transMeta );

      DatabaseMeta check = transMeta.findDatabase( databaseMeta.getName() ); // Check if there already is one in the
                                                                             // transformation
      if ( check == null || overWriteShared ) { // We only add, never overwrite database connections.
        if ( databaseMeta.getName() != null ) {
          transMeta.addOrReplaceDatabase( databaseMeta );
          if ( !overWriteShared ) {
            databaseMeta.setChanged( false );
          }
        }
      }
    }
    transMeta.clearChangedDatabases();
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleException( BaseMessages.getString(
      PKG, "TransMeta.Log.UnableToReadDatabaseIDSFromRepository" ), dbe );
  } catch ( KettleException ke ) {
    throw new KettleException(
      BaseMessages.getString( PKG, "TransMeta.Log.UnableToReadDatabasesFromRepository" ), ke );
  }
}
 

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( true );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
 *
 * The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
 */
public void testTableInputWithParam() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTables( database );
  createData( database );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tii.setLookupFromStep( injectorStep );
  tii.setExecuteEachInputRow( true );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "ID, CODE ";
  selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setDescription( "Reads information from table [" + source_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( fromstepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

@Override
@Before
public void setUp() throws Exception {

  KettleEnvironment.init();

  /* SET UP TRANSFORMATION */

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "update test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  /* SET UP DATABASE */
  // Create target table
  db = new Database( transMeta, dbInfo );
  db.connect();

  String source = db.getCreateTableStatement( TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true );
  db.execStatement( source );

  // populate target table
  for ( String sql : insertStatement ) {
    db.execStatement( sql );
  }

  /* SET UP TRANSFORMATION STEPS */

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  // create the update step...
  String updateStepName = "update [" + TARGET_TABLE + "]";
  upd = new UpdateMeta();
  upd.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  upd.setTableName( TARGET_TABLE );
  upd.setUpdateLookup( new String[] { "VALUE" } );
  upd.setUpdateStream( new String[] { "VALUE" } );
  upd.setErrorIgnored( true );

  String fromid = registry.getPluginId( StepPluginType.class, upd );
  StepMeta updateStep = new StepMeta( fromid, updateStepName, upd );
  updateStep.setDescription( "update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]" );
  transMeta.addStep( updateStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, updateStep );
  transMeta.addTransHop( hi );

  /* PREPARE TRANSFORMATION EXECUTION */

  trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( updateStepName, 0 );
  rc = new RowStepCollector();
  si.addRowListener( rc );

  rp = trans.addRowProducer( injectorStepName, 0 );

}
 

/**
 * Test case for normal table output where the table is included in the instream, but the tablename is not stored in
 * the table.
 */
public void testTableOutputJIRA897() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output JIRA897 test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table1, createSourceRowMetaInterface1() );
  createTable( database, target_table2, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTableNameInField( true );
  tom.setTableNameField( "TABLE" );
  tom.setTableNameInTable( false );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to tables on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createJIRA897DataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();

  // The name of the table should still be in here.
  List<RowMetaAndData> goldRows = createJIRA897DataRows();
  checkRows( goldRows, resultRows );
  checkResultsJIRA897( database );
}
 

/**
 * Test case for normal table output case.
 */
@SuppressWarnings( "deprecation" )
public void testTableOutputNormal() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output normal test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTablename( target_table );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to table [" + target_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createNormalDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createNormalDataRows();
  checkRows( goldRows, resultRows );
  checkResultsNormal( database );
}
 

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test with Like predicate.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookupWithLikePredicate() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "] by 'LIKE'";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( true );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "STRING" } );
  dbl.setKeyCondition( new String[] { "LIKE" } );
  dbl.setStreamKeyField1( new String[] { "str_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table ["
      + lookup_table + "] on database ["
      + dbInfo + "] using LIKE condition" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( false );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Basic Test case for database lookup.
 */
@Test
public void basicDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( false );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test case for Combination lookup/update.
 */
public void testCombinationLookup() throws Exception {
  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta lookupDBInfo = transMeta.findDatabase( "lookup" );

  // Execute our setup SQLs in the database.
  Database lookupDatabase = new Database( transMeta, lookupDBInfo );
  lookupDatabase.connect();
  createTables( lookupDatabase );
  createData( lookupDatabase );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "lookup" ) );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "DLR_CD, DLR_NM, DLR_DESC ";
  selectSQL += "FROM " + source_table + " ORDER BY ORDNO;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setLocation( 150, 100 );
  fromstep.setDraw( true );
  fromstep.setDescription( "Reads information from table ["
    + source_table + "] on database [" + lookupDBInfo + "]" );
  transMeta.addStep( fromstep );

  //
  // create the combination lookup/update step...
  //
  String lookupstepname = "lookup from [lookup]";
  CombinationLookupMeta clm = new CombinationLookupMeta();
  String[] lookupKey = { "DLR_CD" };
  clm.setTablename( target_table );
  clm.setKeyField( lookupKey );
  clm.setKeyLookup( lookupKey );
  clm.setTechnicalKeyField( "ID" );
  clm.setTechKeyCreation( CombinationLookupMeta.CREATION_METHOD_TABLEMAX );
  clm.setDatabaseMeta( lookupDBInfo );

  String lookupstepid = registry.getPluginId( StepPluginType.class, clm );
  StepMeta lookupstep = new StepMeta( lookupstepid, lookupstepname, clm );
  lookupstep.setDescription( "Looks up information from table [lookup] on database [" + lookupDBInfo + "]" );
  transMeta.addStep( lookupstep );

  TransHopMeta hi = new TransHopMeta( fromstep, lookupstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.execute( null );

  trans.waitUntilFinished();

  checkResults( lookupDatabase );
}
 

@Override
@Before
public void setUp() throws Exception {

  KettleEnvironment.init();

  /* SET UP TRANSFORMATION */

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "insert/update test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  /* SET UP DATABASE */
  // Create target table
  db = new Database( transMeta, dbInfo );
  db.connect();

  String source = db.getCreateTableStatement( TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true );
  db.execStatement( source );

  // populate target table
  for ( String sql : insertStatement ) {
    db.execStatement( sql );
  }

  /* SET UP TRANSFORMATION STEPS */

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  // create the update step...
  String updateStepName = "insert/update [" + TARGET_TABLE + "]";
  insupd = new InsertUpdateMeta();
  insupd.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  insupd.setTableName( TARGET_TABLE );

  insupd.setUpdateLookup( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdateStream( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdate( new Boolean[] { true, false } );

  String fromid = registry.getPluginId( StepPluginType.class, insupd );
  StepMeta updateStep = new StepMeta( fromid, updateStepName, insupd );
  updateStep.setDescription( "insert/update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]" );
  transMeta.addStep( updateStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, updateStep );
  transMeta.addTransHop( hi );

  /* PREPARE TRANSFORMATION EXECUTION */

  trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( updateStepName, 0 );
  rc = new RowStepCollector();
  si.addRowListener( rc );

  rp = trans.addRowProducer( injectorStepName, 0 );

}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of three (i.e. not autocommit but equal to input row
 * size)
 */
@Test
public void testExecSQLRow4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 3 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of two (i.e. not autocommit and not the input row size)
 */
@Test
public void testExecSQLRow3() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 2 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of one (i.e. "simulated" autocommit)
 */
@Test
public void testExecSQLRow2() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 1 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of zero (i.e. autocommit)
 */
@Test
public void testExecSQLRow1() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 0 ); // use Autocommit
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}