Java Code Examples for org.pentaho.di.trans.Trans#getStepInterface()

The following examples show how to use org.pentaho.di.trans.Trans#getStepInterface() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SynchronizeAfterMergeIT.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up
private void processRow( TransProcessControl control ) throws Exception {
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();
  generateData( rp );
  rp.finished();
  StepInterface si = trans.getStepInterface( synchronizeAfterMergeStepname, 0 );
  switch ( control ) {
    case ITTERUPT:
      trans.stopAll();
      while (  !si.getStatus().equals( StepExecutionStatus.STATUS_STOPPED ) ) {
        //wait until transformation does not stopped
      };
      break;
    case WAIT:
    default:
      trans.waitUntilFinished();
      assertEquals( "Step still started", StepExecutionStatus.STATUS_FINISHED, si.getStatus() );
      break;
  }
  assertEquals( "Unexpected error occurred",  0, si.getErrors() );

  Field field = SynchronizeAfterMerge.class.getDeclaredField( "data" );
  field.setAccessible( true );
  SynchronizeAfterMergeData  data = (SynchronizeAfterMergeData) field.get( si );
  //should be closed and set null after finish transformation
  assertNull( data.db.getConnection() );
}
 
Example 2
Source File: TextFileOutputIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the ZIP output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput4() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testTextFileOutput4" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testTextFileOutput4", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the text file output step with ZIP compression
  // but first lets get a filename
  String textFileName = "testTextFileOutput4";
  String textFileOutputStepName = "text file output step";
  StepMeta textFileOutputStep = createTextFileOutputStep( textFileOutputStepName, textFileName, "Zip", registry );
  transMeta.addStep( textFileOutputStep );

  // create a TransHopMeta for textFileOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, textFileOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( textFileOutputStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  trans.startThreads();
  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
  Object[][] rows = new Object[10][3];
  File f = new File( textFileName + ".zip" );
  f.deleteOnExit();
  try {
    ZipFile zf = new ZipFile( f );
    int zipEntryCount = 0;
    Enumeration<? extends ZipEntry> entries = zf.entries();

    while ( entries.hasMoreElements() ) {
      ZipEntry ze = entries.nextElement();
      zipEntryCount++;
      BufferedReader input = new BufferedReader( new InputStreamReader( zf.getInputStream( ze ) ) );
      readData1Rows( rows, input );
    }

    zf.close();
    assertEquals( zipEntryCount, 1 );

  } catch ( IOException e ) {
    fail( e.getLocalizedMessage() );
  }

  List<RowMetaAndData> outFileRows = createResultDataFromObjects( rows );

  try {
    TestUtilities.checkRows( resultRows, outFileRows );
  } catch ( TestFailedException tfe ) {
    fail( tfe.getMessage() );
  }
}
 
Example 3
Source File: TableInputIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
 *
 * The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
 */
public void testTableInputWithParam() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTables( database );
  createData( database );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tii.setLookupFromStep( injectorStep );
  tii.setExecuteEachInputRow( true );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "ID, CODE ";
  selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setDescription( "Reads information from table [" + source_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( fromstepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 
Example 4
Source File: UpdateIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
@Override
@Before
public void setUp() throws Exception {

  KettleEnvironment.init();

  /* SET UP TRANSFORMATION */

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "update test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  /* SET UP DATABASE */
  // Create target table
  db = new Database( transMeta, dbInfo );
  db.connect();

  String source = db.getCreateTableStatement( TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true );
  db.execStatement( source );

  // populate target table
  for ( String sql : insertStatement ) {
    db.execStatement( sql );
  }

  /* SET UP TRANSFORMATION STEPS */

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  // create the update step...
  String updateStepName = "update [" + TARGET_TABLE + "]";
  upd = new UpdateMeta();
  upd.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  upd.setTableName( TARGET_TABLE );
  upd.setUpdateLookup( new String[] { "VALUE" } );
  upd.setUpdateStream( new String[] { "VALUE" } );
  upd.setErrorIgnored( true );

  String fromid = registry.getPluginId( StepPluginType.class, upd );
  StepMeta updateStep = new StepMeta( fromid, updateStepName, upd );
  updateStep.setDescription( "update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]" );
  transMeta.addStep( updateStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, updateStep );
  transMeta.addTransHop( hi );

  /* PREPARE TRANSFORMATION EXECUTION */

  trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( updateStepName, 0 );
  rc = new RowStepCollector();
  si.addRowListener( rc );

  rp = trans.addRowProducer( injectorStepName, 0 );

}
 
Example 5
Source File: TableOutputIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for normal table output where the table is included in the instream, but the tablename is not stored in
 * the table.
 */
public void testTableOutputJIRA897() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output JIRA897 test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table1, createSourceRowMetaInterface1() );
  createTable( database, target_table2, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTableNameInField( true );
  tom.setTableNameField( "TABLE" );
  tom.setTableNameInTable( false );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to tables on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createJIRA897DataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();

  // The name of the table should still be in here.
  List<RowMetaAndData> goldRows = createJIRA897DataRows();
  checkRows( goldRows, resultRows );
  checkResultsJIRA897( database );
}
 
Example 6
Source File: ParameterSimpleTransIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans6() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set. And no default... so the variable will be set to "". not
  // to "Variable1"
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData6();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 
Example 7
Source File: RowGeneratorIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for Row Generator step.
 */
public void testRowGenerator() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "row generatortest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a row generator step...
  //
  String rowGeneratorStepname = "row generator step";
  RowGeneratorMeta rm = new RowGeneratorMeta();

  // Set the information of the row generator.
  String rowGeneratorPid = registry.getPluginId( StepPluginType.class, rm );
  StepMeta rowGeneratorStep = new StepMeta( rowGeneratorPid, rowGeneratorStepname, rm );
  transMeta.addStep( rowGeneratorStep );

  //
  // Do the following specs 3 times.
  //
  String[] fieldName = { "string", "boolean", "integer", "timestamp" };
  String[] type = { "String", "Boolean", "Integer", "Timestamp" };
  String[] value = { "string_value", "true", "20", "1970-01-01 00:00:00.000" };
  String[] fieldFormat = { "", "", "", "" };
  String[] group = { "", "", "", "" };
  String[] decimal = { "", "", "", "" };
  String[] currency = { "", "", "", "" };
  int[] intDummies = { -1, -1, -1, -1 };
  boolean[] setEmptystring = { false, false, false, false };

  rm.setDefault();
  rm.setFieldName( fieldName );
  rm.setFieldType( type );
  rm.setValue( value );
  rm.setFieldLength( intDummies );
  rm.setFieldPrecision( intDummies );
  rm.setRowLimit( "3" );
  rm.setFieldFormat( fieldFormat );
  rm.setGroup( group );
  rm.setDecimal( decimal );
  rm.setCurrency( currency );
  rm.setEmptyString( setEmptystring );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi = new TransHopMeta( rowGeneratorStep, dummyStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  trans.startThreads();
  trans.waitUntilFinished();

  List<RowMetaAndData> checkList = createData();
  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  checkRows( resultRows, checkList );
}
 
Example 8
Source File: DatabaseLookupIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( false );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 
Example 9
Source File: TextFileOutputIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the normal output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput6() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testTextFileOutput6" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testTextFileOutput5", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the text file output step with no compression
  // but first lets get a filename
  String textFileName = "testTextFileOutput6";
  String textFileOutputStepName = "text file output step";
  StepMeta textFileOutputStep =
    createTextFileOutputStep( textFileOutputStepName, textFileName, "None", registry );
  transMeta.addStep( textFileOutputStep );

  // create a TransHopMeta for textFileOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, textFileOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( textFileOutputStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  trans.startThreads();
  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
  Object[][] rows = new Object[10][3];
  File f = new File( textFileName + "." + EXTENSION );
  f.deleteOnExit();
  try {
    FileInputStream fin = new FileInputStream( f );
    InputStreamReader xover = new InputStreamReader( fin );
    BufferedReader input = new BufferedReader( xover );

    readData1Rows( rows, input );

    fin.close();

  } catch ( IOException e ) {
    fail( e.getLocalizedMessage() );
  }

  List<RowMetaAndData> outFileRows = createResultDataFromObjects( rows );

  try {
    TestUtilities.checkRows( resultRows, outFileRows );
  } catch ( TestFailedException tfe ) {
    fail( tfe.getMessage() );
  }
}
 
Example 10
Source File: JavaScriptSpecialIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for javascript functionality: trans_Status and SKIP_TRANSFORMATION. Regression test case for JIRA defect
 * PDI-364.
 */
public void testTransStatus() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript trans_Status" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  // process 3 rows and skip the rest.
  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "trans_Status = CONTINUE_TRANSFORMATION;\n"
      + "if (getProcessCount(\"r\") > 3) {\n" + " \ttrans_Status = SKIP_TRANSFORMATION;\n" + "}" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] {} );
  svm.setRename( new String[] {} );
  svm.setType( new int[] {} );
  svm.setLength( new int[] {} );
  svm.setPrecision( new int[] {} );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData2();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}
 
Example 11
Source File: ExecSQLRowIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Basic Test case for Exec SQL Row. This tests a commit size of zero (i.e. autocommit)
 */
@Test
public void testExecSQLRow1() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 0 ); // use Autocommit
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 
Example 12
Source File: ParameterSimpleTransIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for parameters using a simple transformation. Here blocking some unwise usage of parameters.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans3() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans3" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${JAVA_HOME}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "${JAVA_HOME}", "default1", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "default2", "Parameter 2" );
  trans.setParameterValue( "${JAVA_HOME}", "param1" );
  // PARAM2 is not set

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData3();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 
Example 13
Source File: ParameterSimpleTransIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Test case for parameters using a simple transformation. Here 1 parameter is not provided as value, so the default
 * will be used.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans2() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans2" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "Param1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "default1", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "default2", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  // PARAM2 is not set

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData2();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 
Example 14
Source File: InsertUpdateIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
@Override
@Before
public void setUp() throws Exception {

  KettleEnvironment.init();

  /* SET UP TRANSFORMATION */

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "insert/update test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  /* SET UP DATABASE */
  // Create target table
  db = new Database( transMeta, dbInfo );
  db.connect();

  String source = db.getCreateTableStatement( TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true );
  db.execStatement( source );

  // populate target table
  for ( String sql : insertStatement ) {
    db.execStatement( sql );
  }

  /* SET UP TRANSFORMATION STEPS */

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  // create the update step...
  String updateStepName = "insert/update [" + TARGET_TABLE + "]";
  insupd = new InsertUpdateMeta();
  insupd.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  insupd.setTableName( TARGET_TABLE );

  insupd.setUpdateLookup( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdateStream( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdate( new Boolean[] { true, false } );

  String fromid = registry.getPluginId( StepPluginType.class, insupd );
  StepMeta updateStep = new StepMeta( fromid, updateStepName, insupd );
  updateStep.setDescription( "insert/update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]" );
  transMeta.addStep( updateStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, updateStep );
  transMeta.addTransHop( hi );

  /* PREPARE TRANSFORMATION EXECUTION */

  trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( updateStepName, 0 );
  rc = new RowStepCollector();
  si.addRowListener( rc );

  rp = trans.addRowProducer( injectorStepName, 0 );

}
 
Example 15
Source File: TextFileOutputIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the GZIP output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput5() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testTextFileOutput5" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testTextFileOutput5", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the text file output step with GZIP compression
  // but first lets get a filename
  String textFileName = "testTextFileOutput5";
  String textFileOutputStepName = "text file output step";
  StepMeta textFileOutputStep =
    createTextFileOutputStep( textFileOutputStepName, textFileName, "GZip", registry );
  transMeta.addStep( textFileOutputStep );

  // create a TransHopMeta for textFileOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, textFileOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( textFileOutputStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  trans.startThreads();
  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
  Object[][] rows = new Object[10][3];
  File f = new File( textFileName + "." + EXTENSION + ".gz" );
  f.deleteOnExit();
  try {
    FileInputStream fin = new FileInputStream( f );
    GZIPInputStream gzis = new GZIPInputStream( fin );
    InputStreamReader xover = new InputStreamReader( gzis );
    BufferedReader input = new BufferedReader( xover );

    readData1Rows( rows, input );

    fin.close();

  } catch ( IOException e ) {
    fail( e.getLocalizedMessage() );
  }

  List<RowMetaAndData> outFileRows = createResultDataFromObjects( rows );

  try {
    TestUtilities.checkRows( resultRows, outFileRows );
  } catch ( TestFailedException tfe ) {
    fail( tfe.getMessage() );
  }
}
 
Example 16
Source File: TextFileInputIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
public void testTextFileInput1() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testTextFileInput1" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // write the data that is to be read in
  // by the step we are testing
  String fileName = writeInputFile( 1 );

  // create an injector step and add it to the trans meta
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // Create a Text File Input step
  String testFileInputName = "text file input step";
  StepMeta textFileInputStep = createTextFileInputStep( testFileInputName, fileName, registry );
  transMeta.addStep( textFileInputStep );

  // create a TransHopMeta for textFileInputStep and add it to the transMeta
  TransHopMeta hopInputTextFile = new TransHopMeta( injectorStep, textFileInputStep );
  transMeta.addTransHop( hopInputTextFile );

  // Create a dummy step 1 and add it to the tranMeta
  String dummyStepName = "dummy step";
  StepMeta dummyStep = TestUtilities.createDummyStep( dummyStepName, registry );
  transMeta.addStep( dummyStep );

  // create transHopMeta for the hop from text file input to the dummy step
  TransHopMeta hop_textFileInputStep_dummyStep = new TransHopMeta( textFileInputStep, dummyStep );
  transMeta.addTransHop( hop_textFileInputStep_dummyStep );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // create a row collector and add it to a row listener for the dummy step
  StepInterface si = trans.getStepInterface( dummyStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  si.addRowListener( dummyRowCollector );

  // Create a row producer for trans
  RowProducer rowProducer = trans.addRowProducer( injectorStepName, 0 );
  trans.startThreads();

  // create the filename rows
  List<RowMetaAndData> inputList = createData( fileName );
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rowMetaAndData = it.next();
    rowProducer.putRow( rowMetaAndData.getRowMeta(), rowMetaAndData.getData() );
  }
  rowProducer.finished();

  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
  List<RowMetaAndData> goldenImageRows = createResultData1();
  try {
    TestUtilities.checkRows( goldenImageRows, resultRows, 5 );
  } catch ( TestFailedException tfe ) {
    fail( tfe.getMessage() );
  }
}
 
Example 17
Source File: MappingIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that info steps are correctly identified via StepMetaInterface.getStepIOMeta()
 */
public void testInfoStreams_single() throws Exception {
  KettleEnvironment.init();
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // Create a new transformation with a row generator that feeds a Mapping (Sub-Transformation) Step
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "Mapping Info Test" );
  StepMeta rowGenerator = buildRowGeneratorStep( registry, "Generate Rows" );
  transMeta.addStep( rowGenerator );

  String mappingName = "mapping";
  MappingMeta mappingMeta = new MappingMeta();
  mappingMeta.setSpecificationMethod( ObjectLocationSpecificationMethod.FILENAME );
  mappingMeta.setFileName( "test/org/pentaho/di/trans/steps/mapping/subtrans.ktr" );
  String mappingInputStepName = "input";
  mappingMeta.setInputMappings( Collections.singletonList( createMappingDef(
    rowGenerator.getName(), mappingInputStepName, "string", "a" ) ) );
  String mappingPid = registry.getPluginId( StepPluginType.class, mappingMeta );
  StepMeta mapping = new StepMeta( mappingPid, mappingName, mappingMeta );
  transMeta.addStep( mapping );

  TransHopMeta hopGeneratorToMapping = new TransHopMeta( rowGenerator, mapping );
  transMeta.addTransHop( hopGeneratorToMapping );

  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Mimic how a transformation is loaded and initialized from TransMeta.loadXML() or
  // KettleDatabaseRepositoryTransDelegate.loadTransformation()
  // so the StepMeta references are wired up in the MappingMeta properly
  // (Copied from TransMeta.loadXML())
  for ( int i = 0; i < transMeta.nrSteps(); i++ ) {
    StepMeta stepMeta = transMeta.getStep( i );
    StepMetaInterface sii = stepMeta.getStepMetaInterface();
    if ( sii != null ) {
      sii.searchInfoAndTargetSteps( transMeta.getSteps() );
    }
  }

  // Verify the transformation was configured properly
  assertEquals( "Transformation not initialized properly", 2, transMeta.nrSteps() );

  StepMeta meta = transMeta.getStep( 1 );
  assertTrue( "Transformation not initialized properly", meta.getStepMetaInterface() instanceof MappingMeta );

  MappingMeta loadedMappingMeta = (MappingMeta) meta.getStepMetaInterface();
  assertEquals( "Expected a single input mapping definition", 1, loadedMappingMeta.getInputMappings().size() );

  StepIOMetaInterface ioMeta = loadedMappingMeta.getStepIOMeta();
  assertEquals( "Expected a single Info Stream", 1, ioMeta.getInfoStreams().size() );
  assertEquals( "Expected a single Info Step", 1, loadedMappingMeta.getInfoSteps().length );

  // Verify the transformation can be executed
  StepInterface si = trans.getStepInterface( mappingName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  trans.startThreads();
  trans.waitUntilFinished();

  assertEquals( 1, rc.getRowsRead().size() );
  assertEquals( 1, rc.getRowsWritten().size() );
}
 
Example 18
Source File: ExecSQLRowIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Basic Test case for Exec SQL Row. This tests a commit size of three (i.e. not autocommit but equal to input row
 * size)
 */
@Test
public void testExecSQLRow4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 3 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 
Example 19
Source File: ExecSQLRowIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Basic Test case for Exec SQL Row. This tests a commit size of two (i.e. not autocommit and not the input row size)
 */
@Test
public void testExecSQLRow3() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 2 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 
Example 20
Source File: ExecSQLRowIT.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
/**
 * Basic Test case for Exec SQL Row. This tests a commit size of one (i.e. "simulated" autocommit)
 */
@Test
public void testExecSQLRow2() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 1 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}