Java Code Examples for org.pentaho.di.trans.Trans#prepareExecution()

/**
 * Tests the YAML contents generated by the GPLoad step in the transformation.
 * 
 * @param GPLoadMeta
 * @param TransMeta
 * @param expectedYAMLFilename
 *          The file name of the YAML file whose contents should match the YAML generated by the GPLoad step.
 */
private void testYAMLContents( GPLoadMeta gpLoadMeta, TransMeta transMeta, String expectedYAMLFileName ) throws KettleException, Exception {

  // create the transformation and prepare the transformation
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // get the step meta from step 1- the only step in the trans
  StepMeta gpLoadStepMeta = transMeta.getStep( 0 );

  // create a GPLoad using the transformation
  GPLoad gpLoad = new GPLoad( gpLoadStepMeta, new GPLoadData(), 0, transMeta, trans );

  // get the YAML file contents that we expect
  String expectedContents = getYamlFileContents( expectedYAMLFileName );

  // we need to the row meta interface to create the control file
  RowMetaInterface rowMetaInterface = gpLoad.getPreviewRowMeta();

  // get the file contents from the GPLoad object
  String actualContents = null;
  actualContents = gpLoad.getControlFileContents( gpLoadMeta, rowMetaInterface );

  // test that the built YAML contest are expected
  assertEquals( expectedContents, actualContents );
}
 

@Test
public void testGetDataFromAFolderRecursivelyFromPreviousStep() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-from-previous-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 1 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 1 ).step.getLinesInput() );
  // The path contains one entry of a folder
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );
}
 

@Test
public void testGetDataFromFolderRecursively() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

@Test
public void doesNotWriteRowOnTimeWhenStopped() throws KettleException, InterruptedException {
  TransMeta transMeta = new TransMeta( getClass().getResource( "safe-stop.ktr" ).getPath() );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( new String[] {} );
  trans.getSteps().get( 1 ).step.addRowListener( new RowAdapter() {
    @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
      trans.safeStop();
    }
  } );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 1, trans.getSteps().get( 1 ).step.getLinesRead() );
}
 

@Test
public void metaInjectWithFieldSplitter() throws KettleException {
  TransMeta transMeta = new TransMeta( "src/it/resources/org/pentaho/di/trans/steps/metainject/fieldsplit/PDI-15679-inject.ktr" );
  transMeta.setTransformationType( TransMeta.TransformationType.Normal );

  Trans trans = new Trans( transMeta );
  trans.setLogLevel( LogLevel.ERROR );

  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  Assert.assertEquals( 0, trans.getErrors() );
}
 

@Test
public void testPDI18776() throws KettleException, IOException {
  KettleEnvironment.init();
  String path = getClass().getResource( "Random_value.ktr" ).getPath();
  Variables variables = new Variables();
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  String childJobStep = trans.getSteps().get( 1 ).step.toString();
  assertTrue( childJobStep.contains( "Dummy" ) );
}
 

void testOneRow(String transName, String[] referenceValues, String[] comparisonValues, Object[] goldenImageRowValues) throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( transName );
  PluginRegistry registry = PluginRegistry.getInstance();

  // Create a merge rows step
  String mergeRowsStepName = "merge rows step";
  MergeRowsMeta mergeRowsMeta = new MergeRowsMeta();

  String mergeRowsStepPid = registry.getPluginId( StepPluginType.class, mergeRowsMeta );
  StepMeta mergeRowsStep = new StepMeta( mergeRowsStepPid, mergeRowsStepName, mergeRowsMeta );
  transMeta.addStep( mergeRowsStep );

  mergeRowsMeta.setKeyFields( new String[]{ keyField } );
  mergeRowsMeta.setValueFields( new String[]{ compareField } );
  mergeRowsMeta.setFlagField( flagField );

  List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();

  //
  // create a reference stream (row generator step)
  //
  createRowGenerator(
    transMeta, registry,
    "reference row generator", referenceValues,
    mergeRowsStep, mergeRowsMeta,
    0
  );

  //
  // create a comparison stream (row generator step)
  //
  createRowGenerator(
    transMeta, registry,
    "comparison row generator", comparisonValues,
    mergeRowsStep, mergeRowsMeta,
    1
  );

  // Now execute the transformation
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( mergeRowsStepName, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();
  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData( goldenImageRowValues );
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
 *
 * The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
 */
public void testTableInputWithParam() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTables( database );
  createData( database );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tii.setLookupFromStep( injectorStep );
  tii.setExecuteEachInputRow( true );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "ID, CODE ";
  selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setDescription( "Reads information from table [" + source_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( fromstepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test case for javascript functionality: trans_Status and SKIP_TRANSFORMATION. Regression test case for JIRA defect
 * PDI-364.
 */
public void testTransStatus() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript trans_Status" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  // process 3 rows and skip the rest.
  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "trans_Status = CONTINUE_TRANSFORMATION;\n"
      + "if (getProcessCount(\"r\") > 3) {\n" + " \ttrans_Status = SKIP_TRANSFORMATION;\n" + "}" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] {} );
  svm.setRename( new String[] {} );
  svm.setType( new int[] {} );
  svm.setLength( new int[] {} );
  svm.setPrecision( new int[] {} );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData2();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}
 

/**
 * Test case for parameters using a simple transformation. Here 1 parameter is not provided as value, so the default
 * will be used.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans2() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans2" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "Param1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "default1", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "default2", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  // PARAM2 is not set

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData2();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test with cache turned off but "Load All Rows" enabled (Load all rows should have no effect) See JIRA PDI-1910
 */
@Test
public void NOTCachedAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( false );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test case for janino step.
 */
@Test
public void testJaninoStep() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "janino test" );

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  //
  // create a janino step...
  //
  String stepname = "janino";
  JaninoMeta jm = new JaninoMeta();

  // Set the information of the step
  String janinoPid = registry.getPluginId( StepPluginType.class, jm );
  StepMeta janinoStep = new StepMeta( janinoPid, stepname, jm );
  transMeta.addStep( janinoStep );

  jm.setDefault();

  JaninoMetaFunction[] formulas =
  {
    new JaninoMetaFunction(
      "string", "(string==null)?null:\"string-value\"", ValueMetaInterface.TYPE_STRING, -1, -1, "string" ),
    new JaninoMetaFunction(
      "integer", "(integer==null)?null:new Long(42L)", ValueMetaInterface.TYPE_INTEGER, -1, -1, "integer" ),
    new JaninoMetaFunction(
      "number", "(number==null)?null:new Double(23.0)", ValueMetaInterface.TYPE_NUMBER, -1, -1, "number" ),
    new JaninoMetaFunction(
      "bigdecimal", "(bigdecimal==null)?null:new java.math.BigDecimal(11.0)", ValueMetaInterface.TYPE_BIGNUMBER,
      -1, -1, "bigdecimal" ),
    new JaninoMetaFunction(
      "date", "(date==null)?null:new java.util.Date(10000000)", ValueMetaInterface.TYPE_DATE, -1, -1, "date" ),
    new JaninoMetaFunction(
      "binary", "(binary==null)?null:new byte[]{1,2,3,4,5}", ValueMetaInterface.TYPE_BINARY, -1, -1, "binary" ),
    new JaninoMetaFunction(
      "bool", "(bool==null)?null:Boolean.TRUE", ValueMetaInterface.TYPE_BOOLEAN, -1, -1, "bool" ),
    new JaninoMetaFunction( "timestamp", "(timestamp==null)?null:new java.sql.Timestamp(0L)",
      ValueMetaInterface.TYPE_TIMESTAMP, -1, -1, "timestamp" ),
    new JaninoMetaFunction( "inetaddress", "(inetaddress==null)?null:java.net.InetAddress.getByAddress( new byte[]{ 127, 0, 0, 1} )",
      ValueMetaInterface.TYPE_INET, -1, -1, "inetaddress" ), };

  jm.setFormula( formulas );

  transMeta.addTransHop( new TransHopMeta( injectorStep, janinoStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi = new TransHopMeta( janinoStep, dummyStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );
  RowProducer rp = trans.addRowProducer( injectorStepName, 0 );

  trans.startThreads();

  for ( RowMetaAndData rm : createInputList() ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> checkList = createExpectedList();
  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  checkRows( resultRows, checkList );
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans5() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "default1", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set, so defaults should kick in.
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData5();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test case for normal table output case.
 */
@SuppressWarnings( "deprecation" )
public void testTableOutputNormal() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output normal test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTablename( target_table );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to table [" + target_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createNormalDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createNormalDataRows();
  checkRows( goldRows, resultRows );
  checkResultsNormal( database );
}
 

protected void executeTrans( Trans trans ) throws KettleException {
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
}
 

private static void runTransformation( Trans trans ) throws Exception {
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 0, trans.getErrors() );
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of three (i.e. not autocommit but equal to input row
 * size)
 */
@Test
public void testExecSQLRow4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 3 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

public boolean runEngine( boolean printDescription ) throws KettleException {
  System.gc();

  KettleEnvironment.init();

  transMeta = new TransMeta( filename );
  transMeta.setVariable( "NR_OF_ROWS", Long.toString( records ) );
  if ( printDescription ) {
    printTransDescription();
  }

  // Replace the TARGET database connection settings with the one provided
  if ( targetDatabaseMeta != null ) {
    transMeta.addOrReplaceDatabase( targetDatabaseMeta );
  }

  // OK, now run this transFormation.
  Trans trans = new Trans( transMeta );
  trans.setLogLevel( logLevel );

  try {
    trans.prepareExecution( null );
  } catch ( Exception e ) {
    System.err.println( KettleLogStore.getAppender().getBuffer( trans.getLogChannelId(), true ) );

    trans.getLogChannel().logError( "Error preparing / initializing transformation", e );

    return false;
  }

  if ( !Utils.isEmpty( rowListenerStep ) ) {
    StepInterface step = trans.findRunThread( rowListenerStep );
    if ( step != null ) {
      step.addRowListener( rowListener );
    }
  }

  long startTime = System.currentTimeMillis();

  trans.startThreads();

  trans.waitUntilFinished();

  long stopTime = System.currentTimeMillis();

  result = trans.getResult();

  runTime = (double) ( stopTime - startTime ) / 1000;
  speed = records / ( runTime );

  printStats( "V3 results", records, runTime, speed );

  return true;
}
 

/**
 * Basic Test case for Exec SQL Row. This tests a commit size of one (i.e. "simulated" autocommit)
 */
@Test
public void testExecSQLRow2() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );
  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the Exec SQL Row step...
  //
  String stepName = "delete from [" + execsqlrow_testtable + "]";
  ExecSQLRowMeta execsqlmeta = new ExecSQLRowMeta();
  execsqlmeta.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  execsqlmeta.setCommitSize( 1 );
  execsqlmeta.setSqlFieldName( "SQL" );

  String execSqlRowId = registry.getPluginId( StepPluginType.class, execsqlmeta );
  StepMeta execSqlRowStep = new StepMeta( execSqlRowId, stepName, execsqlmeta );
  execSqlRowStep.setDescription( "Deletes information from table ["
    + execsqlrow_testtable + "] on database [" + dbInfo + "]" );
  transMeta.addStep( execSqlRowStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, execSqlRowStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( stepName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );
}
 

/**
 * Tests the ZIP output capability of the TextFileOutput step
 *
 * @throws Exception
 */
@Test
public void testTextFileOutput4() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testTextFileOutput4" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testTextFileOutput4", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the text file output step with ZIP compression
  // but first lets get a filename
  String textFileName = "testTextFileOutput4";
  String textFileOutputStepName = "text file output step";
  StepMeta textFileOutputStep = createTextFileOutputStep( textFileOutputStepName, textFileName, "Zip", registry );
  transMeta.addStep( textFileOutputStep );

  // create a TransHopMeta for textFileOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, textFileOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( textFileOutputStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  trans.startThreads();
  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
  Object[][] rows = new Object[10][3];
  File f = new File( textFileName + ".zip" );
  f.deleteOnExit();
  try {
    ZipFile zf = new ZipFile( f );
    int zipEntryCount = 0;
    Enumeration<? extends ZipEntry> entries = zf.entries();

    while ( entries.hasMoreElements() ) {
      ZipEntry ze = entries.nextElement();
      zipEntryCount++;
      BufferedReader input = new BufferedReader( new InputStreamReader( zf.getInputStream( ze ) ) );
      readData1Rows( rows, input );
    }

    zf.close();
    assertEquals( zipEntryCount, 1 );

  } catch ( IOException e ) {
    fail( e.getLocalizedMessage() );
  }

  List<RowMetaAndData> outFileRows = createResultDataFromObjects( rows );

  try {
    TestUtilities.checkRows( resultRows, outFileRows );
  } catch ( TestFailedException tfe ) {
    fail( tfe.getMessage() );
  }
}