Java Code Examples for org.pentaho.di.trans.TransMeta#addTransHop()

@Test
public void testResolveStepMetaResources() throws KettleException, MetaStoreException {
  Variables variables = new Variables();
  TransMeta transMeta = spy( new TransMeta() );
  transMeta.setParentVariableSpace( variables );

  doReturn( transMeta ).when( transMeta ).realClone( false );

  TestMetaResolvableResource testMetaResolvableResource = spy( new TestMetaResolvableResource() );
  TestMetaResolvableResource testMetaResolvableResourceTwo = spy( new TestMetaResolvableResource() );

  StepMeta testMeta = new StepMeta( "TestMeta", testMetaResolvableResource );
  StepMeta testMetaTwo = new StepMeta( "TestMeta2", testMetaResolvableResourceTwo );

  transMeta.addStep( testMeta );
  transMeta.addStep( testMetaTwo );
  transMeta.addTransHop( new TransHopMeta( testMeta, testMetaTwo ) );
  TransMetaConverter.convert( transMeta );

  verify( testMetaResolvableResource ).resolve();
  verify( testMetaResolvableResourceTwo ).resolve();
}
 

@Test
public void testRemovingDisabledInputSteps() {
  TransMeta trans = new TransMeta();
  StepMeta inputToBeRemoved = new StepMeta( "InputToBeRemoved", stepMetaInterface );
  trans.addStep( inputToBeRemoved );
  StepMeta inputToStay = new StepMeta( "InputToStay", stepMetaInterface );
  trans.addStep( inputToStay );
  StepMeta inputReceiver1 = new StepMeta( "InputReceiver1", stepMetaInterface );
  trans.addStep( inputReceiver1 );
  StepMeta inputReceiver2 = new StepMeta( "InputReceiver2", stepMetaInterface );
  trans.addStep( inputReceiver2 );

  TransHopMeta hop1 = new TransHopMeta( inputToBeRemoved, inputReceiver1, false );
  TransHopMeta hop2 = new TransHopMeta( inputToStay, inputReceiver1 );
  TransHopMeta hop3 = new TransHopMeta( inputToBeRemoved, inputReceiver2, false );
  trans.addTransHop( hop1 );
  trans.addTransHop( hop2 );
  trans.addTransHop( hop3 );

  Transformation transformation = TransMetaConverter.convert( trans );

  List<String>
      steps =
      transformation.getOperations().stream().map( op -> op.getId() ).collect( Collectors.toList() );
  assertThat( "Only 2 ops should exist", steps.size(), is( 2 ) );
  assertThat( steps, hasItems( "InputToStay", "InputReceiver1" ) );

  List<String> hops = transformation.getHops().stream().map( hop -> hop.getId() ).collect( Collectors.toList() );
  assertThat( "Only 1 hop should exist", hops.size(), is( 1 ) );
  assertThat( hops, hasItems( "InputToStay -> InputReceiver1" ) );
}
 

void createRowGenerator(
  TransMeta transMeta, PluginRegistry registry,
  String stepName, String[] values,
  StepMeta mergeRowsStep, MergeRowsMeta mergeRowsMeta,
  int index
) {
  RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
  String rowGeneratorPid = registry.getPluginId( StepPluginType.class, rowGeneratorMeta );
  StepMeta rowGeneratorStep = new StepMeta( rowGeneratorPid, stepName, rowGeneratorMeta );
  transMeta.addStep( rowGeneratorStep );

  rowGeneratorMeta.setDefault();
  rowGeneratorMeta.setFieldName( fieldName );
  rowGeneratorMeta.setFieldType( type );
  rowGeneratorMeta.setFieldLength( intDummies );
  rowGeneratorMeta.setFieldPrecision( intDummies );
  rowGeneratorMeta.setRowLimit( "1" );
  rowGeneratorMeta.setFieldFormat( fieldFormat );
  rowGeneratorMeta.setGroup( group );
  rowGeneratorMeta.setDecimal( decimal );
  rowGeneratorMeta.setCurrency( currency );
  rowGeneratorMeta.setEmptyString( setEmptystring );

  rowGeneratorMeta.setValue( values );

  TransHopMeta hi1 = new TransHopMeta( rowGeneratorStep, mergeRowsStep );
  transMeta.addTransHop( hi1 );

  List<StreamInterface> infoStreams = mergeRowsMeta.getStepIOMeta().getInfoStreams();
  StreamInterface infoStream = infoStreams.get( index );
  infoStream.setStepMeta( transMeta.findStep( stepName ) );
}
 

protected Trans createAndTestTrans( PluginRegistry registry, TransMeta transMeta, StepMeta inputStep,
    RowStepCollector rowStepCollector, String name, int limit ) throws KettleException {
  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi3 = new TransHopMeta( inputStep, dummyStep1 );
  transMeta.addTransHop( hi3 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  si.addRowListener( rowStepCollector );

  RowProducer rp = trans.addRowProducer( inputStep.getName(), 0 );

  RowMeta rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "pageSize" ) );
  rowMeta.addValueMeta( new ValueMetaString( "name" ) );
  rp.putRow( rowMeta, new Object[] { Integer.valueOf( limit ), name } );

  rp.finished();
  return trans;
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans4() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT.
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test case for table input which is taking its input from a hop. This is a regression test case for JIRA PDI-588.
 *
 * The query in the table input step has one '?' and this parameter is filled by values read from an input hop.
 */
public void testTableInputWithParam() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTables( database );
  createData( database );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String fromstepname = "read from [" + source_table + "]";
  TableInputMeta tii = new TableInputMeta();
  tii.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tii.setLookupFromStep( injectorStep );
  tii.setExecuteEachInputRow( true );
  String selectSQL = "SELECT " + Const.CR;
  selectSQL += "ID, CODE ";
  selectSQL += "FROM " + source_table + " WHERE CODE = ? ORDER BY ID, CODE;";
  tii.setSQL( selectSQL );

  String fromstepid = registry.getPluginId( StepPluginType.class, tii );
  StepMeta fromstep = new StepMeta( fromstepid, fromstepname, tii );
  fromstep.setDescription( "Reads information from table [" + source_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( fromstepname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

/**
 * Test case for javascript functionality: ltrim(), rtrim(), trim().
 */
public void testLuhnCheck() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript LuhnCheck" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var str = string;\n" + "var bool = LuhnCheck(str);" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] { "bool" } );
  svm.setRename( new String[] { "" } );
  svm.setType( new int[] { ValueMetaInterface.TYPE_BOOLEAN } );
  svm.setLength( new int[] { -1 } );
  svm.setPrecision( new int[] { -1 } );
  svm.setReplace( new boolean[] { false } );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}
 

@Before
public void setUp() throws KettleDatabaseException, SQLException {
  connection = DriverManager.getConnection( "jdbc:h2:mem:PERSON;" );
  connection.setAutoCommit( false );
  PreparedStatement stmt = connection.prepareStatement( "CREATE TABLE PERSON (ID INT PRIMARY KEY, personName VARCHAR(64) )" );
  stmt.execute();
  stmt.close();
  stmt = connection.prepareStatement( "INSERT INTO PERSON (ID, personName) VALUES (?, ?)" );
  for ( int i = 0; i < ROW_FOR_UPDATE + ROW_FOR_DELETE; i++ ) {
    stmt.setInt( 1, i );
    stmt.setString( 2, "personName" + i );
    stmt.addBatch();
  }
  stmt.executeBatch();
  stmt.close();
  connection.commit();

  PluginRegistry pluginRegistry = PluginRegistry.getInstance();

  transMeta = new TransMeta();
  transMeta.setName( "SynchronizeAfterMerge" );

  InjectorMeta injectorMeta = new InjectorMeta();
  String injectorPid = pluginRegistry.getPluginId( StepPluginType.class, injectorMeta );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, injectorMeta );
  transMeta.addStep( injectorStep );

  DatabaseMeta dbMeta = spy( new DatabaseMeta() );
  dbMeta.setDatabaseType( "H2" );
  when( dbMeta.getURL() ).thenReturn( "jdbc:h2:mem:PERSON;" );
  when( dbMeta.supportsErrorHandlingOnBatchUpdates() ).thenReturn( false );

  SynchronizeAfterMergeMeta synchronizeAfterMergeMeta = new SynchronizeAfterMergeMeta();
  //set commit size
  synchronizeAfterMergeMeta.setCommitSize( COMMIT_SIZE );
  synchronizeAfterMergeMeta.setDatabaseMeta( dbMeta );
  synchronizeAfterMergeMeta.setKeyCondition( new String[] { "=" } );
  synchronizeAfterMergeMeta.setKeyLookup( new String[] { "ID" } );
  synchronizeAfterMergeMeta.setKeyStream( new String[] { "personName" } );
  synchronizeAfterMergeMeta.setKeyStream2( new String[] { null } );
  synchronizeAfterMergeMeta.setUpdate( new Boolean[] { Boolean.TRUE } );
  synchronizeAfterMergeMeta.setOperationOrderField( "flag" );
  synchronizeAfterMergeMeta.setOrderDelete( DELETE_FLAG );
  synchronizeAfterMergeMeta.setOrderInsert( INSERT_FLAG );
  synchronizeAfterMergeMeta.setOrderUpdate( UPDATE_FLAG );
  synchronizeAfterMergeMeta.setPerformLookup( true );

  synchronizeAfterMergeMeta.setTableName( "PERSON" );
  synchronizeAfterMergeMeta.settablenameInField( false );
  synchronizeAfterMergeMeta.settablenameField( null );
  synchronizeAfterMergeMeta.setUseBatchUpdate( true );
  synchronizeAfterMergeMeta.setUpdateLookup( new String[] { "ID" } );
  synchronizeAfterMergeMeta.setUpdateStream( new String[] { "personName" } );

  String synchronizeAfterMergePid = pluginRegistry.getPluginId( StepPluginType.class, synchronizeAfterMergeStepname );
  StepMeta synchronizeAfterMerge = new StepMeta( synchronizeAfterMergePid, synchronizeAfterMergeStepname, synchronizeAfterMergeMeta );
  transMeta.addStep( synchronizeAfterMerge );

  String dummyResultStepName = "dummyResultStepName";
  DummyTransMeta dummyResultTransMeta = new DummyTransMeta();
  String dummyResultPid = pluginRegistry.getPluginId( StepPluginType.class, dummyResultTransMeta );
  StepMeta dummyResultStep = new StepMeta( dummyResultPid, dummyResultStepName, dummyResultTransMeta );
  transMeta.addStep( dummyResultStep );

  String dummyErrorStepName = "dummyErrorStepName";
  DummyTransMeta dummyErrorTransMeta = new DummyTransMeta();
  String dummyErrorPid = pluginRegistry.getPluginId( StepPluginType.class, dummyErrorTransMeta );
  StepMeta dummyErrorStep = new StepMeta( dummyErrorPid, dummyErrorStepName, dummyErrorTransMeta );
  transMeta.addStep( dummyErrorStep );

  StepErrorMeta stepErrorMeta = new StepErrorMeta( transMeta, synchronizeAfterMerge, dummyErrorStep );
  stepErrorMeta.setEnabled( true );
  synchronizeAfterMerge.setStepErrorMeta( stepErrorMeta );

  TransHopMeta injSynch = new TransHopMeta( injectorStep, synchronizeAfterMerge );
  transMeta.addTransHop( injSynch );

  TransHopMeta synchDummyResult = new TransHopMeta( synchronizeAfterMerge, dummyResultStep );
  transMeta.addTransHop( synchDummyResult );

  TransHopMeta synchDummyError = new TransHopMeta( synchronizeAfterMerge, dummyErrorStep );
  transMeta.addTransHop( synchDummyError );
}
 

public void testProcessRow() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "WebServiceTest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi = new TransHopMeta( injectorStep, dummyStep1 );
  transMeta.addTransHop( hi );

  //
  // Create a String Cut step
  //
  String webServiceStepname = "web service step";
  WebServiceMeta scm = new WebServiceMeta();

  // scm.setUrl(HTTP_LOCALHOST_9998+ "wsdl");
  // scm.setOperationName("CelciusToFahrenheit");
  DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
  DocumentBuilder db = dbf.newDocumentBuilder();
  Document doc = db.parse( new InputSource( new java.io.StringReader( STEP_META ) ) );
  scm.loadXML( doc.getFirstChild(), null, (IMetaStore) null );

  String webServicePid = registry.getPluginId( StepPluginType.class, scm );
  StepMeta webServiceStep = new StepMeta( webServicePid, webServiceStepname, scm );
  transMeta.addStep( webServiceStep );

  TransHopMeta hi2 = new TransHopMeta( dummyStep1, webServiceStep );
  transMeta.addTransHop( hi2 );

  //
  // Create a dummy step 2
  //
  String dummyStepname2 = "dummy step 2";
  DummyTransMeta dm2 = new DummyTransMeta();

  String dummyPid2 = registry.getPluginId( StepPluginType.class, dm2 );
  StepMeta dummyStep2 = new StepMeta( dummyPid2, dummyStepname2, dm2 );
  transMeta.addStep( dummyStep2 );

  TransHopMeta hi3 = new TransHopMeta( webServiceStep, dummyStep2 );
  transMeta.addTransHop( hi3 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector dummyRc1 = new RowStepCollector();
  si.addRowListener( dummyRc1 );

  si = trans.getStepInterface( webServiceStepname, 0 );
  RowStepCollector webServiceRc = new RowStepCollector();
  si.addRowListener( webServiceRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData( createRowMetaInterface(), new Object[][] { new Object[] { 10 } } );
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldRows =
    createData( createOutputRowMetaInterface(), new Object[][] { new Object[] { 10,
      new BigDecimal( 20 ) } } );
  List<RowMetaAndData> resultRows2 = webServiceRc.getRowsWritten();
  assertEquals( goldRows, resultRows2 );
}
 

public String test( boolean compatibilityMode ) throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testJsonOutput" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testJsonOutput1", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the json output step
  // but first lets get a filename
  String jsonFileName = TestUtilities.createEmptyTempFile( "testJsonOutput1_" );
  StepMeta jsonOutputStep = createJsonOutputStep( "json output step", jsonFileName, registry );
  ( (JsonOutputMeta) jsonOutputStep.getStepMetaInterface() ).setCompatibilityMode( compatibilityMode );
  transMeta.addStep( jsonOutputStep );

  // create a TransHopMeta for jsonOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, jsonOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Create a dummy step and add it to the tranMeta
  String dummyStepName = "dummy step";
  StepMeta dummyStep = createDummyStep( dummyStepName, registry );
  transMeta.addStep( dummyStep );

  // create a TransHopMeta for the
  TransHopMeta hop_outputJson_dummyStep = new TransHopMeta( jsonOutputStep, dummyStep );
  transMeta.addTransHop( hop_outputJson_dummyStep );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( dummyStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
  trans.startThreads();
  trans.waitUntilFinished();

  // get the results and return it
  File outputFile = new File( jsonFileName + ".js" );
  String jsonStructure = FileUtils.readFileToString( outputFile );

  return jsonStructure;
}
 

@Override
@Before
public void setUp() throws Exception {

  KettleEnvironment.init();

  /* SET UP TRANSFORMATION */

  // Create a new transformation...
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "insert/update test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  /* SET UP DATABASE */
  // Create target table
  db = new Database( transMeta, dbInfo );
  db.connect();

  String source = db.getCreateTableStatement( TARGET_TABLE, getTargetTableRowMeta(), null, false, null, true );
  db.execStatement( source );

  // populate target table
  for ( String sql : insertStatement ) {
    db.execStatement( sql );
  }

  /* SET UP TRANSFORMATION STEPS */

  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step...
  String injectorStepName = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepName, im );
  transMeta.addStep( injectorStep );

  // create the update step...
  String updateStepName = "insert/update [" + TARGET_TABLE + "]";
  insupd = new InsertUpdateMeta();
  insupd.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  insupd.setTableName( TARGET_TABLE );

  insupd.setUpdateLookup( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdateStream( new String[] { "VALUE", "ROW_ORDER" } );
  insupd.setUpdate( new Boolean[] { true, false } );

  String fromid = registry.getPluginId( StepPluginType.class, insupd );
  StepMeta updateStep = new StepMeta( fromid, updateStepName, insupd );
  updateStep.setDescription( "insert/update data in table [" + TARGET_TABLE + "] on database [" + dbInfo + "]" );
  transMeta.addStep( updateStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, updateStep );
  transMeta.addTransHop( hi );

  /* PREPARE TRANSFORMATION EXECUTION */

  trans = new Trans( transMeta );
  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( updateStepName, 0 );
  rc = new RowStepCollector();
  si.addRowListener( rc );

  rp = trans.addRowProducer( injectorStepName, 0 );

}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans6() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set. And no default... so the variable will be set to "". not
  // to "Variable1"
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData6();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

public void testSortCaseSensitiveUniqueCaseInsensitive() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a sort rows step
  //
  String sortRowsStepname = "sort rows step";
  SortRowsMeta srm = new SortRowsMeta();
  srm.setFieldName( new String[] { "KEY" } );
  srm.setAscending( new boolean[] { true } );
  srm.setCaseSensitive( new boolean[] { true } );
  srm.setPreSortedField( new boolean[] { false } );
  srm.setPrefix( "SortRowsTest" );
  srm.setDirectory( "." );

  String sortRowsStepPid = registry.getPluginId( StepPluginType.class, srm );
  StepMeta sortRowsStep = new StepMeta( sortRowsStepPid, sortRowsStepname, srm );
  transMeta.addStep( sortRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, sortRowsStep ) );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { true } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( sortRowsStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataSortCaseSensitiveUniqueCaseInsensitive(), resultRows );
}
 

/**
 * Test case for parameters using a simple transformation.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans1() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans1" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "Param1", "ParamValue1" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( true );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

public void testCaseInsensitiveNoPreviousSort() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "uniquerowstest" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a unique rows step
  //
  String uniqueRowsStepname = "unique rows step";
  UniqueRowsMeta urm = new UniqueRowsMeta();
  urm.setCompareFields( new String[] { "KEY" } );
  urm.setCaseInsensitive( new boolean[] { true } );

  String uniqueRowsStepPid = registry.getPluginId( StepPluginType.class, urm );
  StepMeta uniqueRowsStep = new StepMeta( uniqueRowsStepPid, uniqueRowsStepname, urm );
  transMeta.addStep( uniqueRowsStep );

  transMeta.addTransHop( new TransHopMeta( injectorStep, uniqueRowsStep ) );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  transMeta.addTransHop( new TransHopMeta( uniqueRowsStep, dummyStep ) );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = dummyRc.getRowsWritten();
  checkRows( createResultDataCaseInsensitiveNoPreviousSort(), resultRows );
}
 

public static final TransMeta generateMergeJoinTransMeta( String transname, String inputStepname, String outputStepname, IMetaStore metaStore ) throws Exception {

    MetaStoreFactory<FileDefinition> factory = new MetaStoreFactory<>( FileDefinition.class, metaStore, PentahoDefaults.NAMESPACE );
    FileDefinition customerFileDefinition = createCustomersInputFileDefinition();
    factory.saveElement( customerFileDefinition );
    FileDefinition statePopulationFileDefinition = createStatePopulationInputFileDefinition();
    factory.saveElement( statePopulationFileDefinition );

    TransMeta transMeta = new TransMeta(  );
    transMeta.setName( transname );
    transMeta.setMetaStore( metaStore );

    // Add the left io step
    //
    BeamInputMeta leftInputMeta = new BeamInputMeta();
    leftInputMeta.setInputLocation( "/tmp/customers/io/customers-100.txt" );
    leftInputMeta.setFileDescriptionName( customerFileDefinition.getName() );
    StepMeta leftInputStepMeta = new StepMeta(inputStepname+" Left", leftInputMeta);
    leftInputStepMeta.setStepID( BeamConst.STRING_BEAM_INPUT_PLUGIN_ID );
    transMeta.addStep( leftInputStepMeta );

    BeamInputMeta rightInputMeta = new BeamInputMeta();
    rightInputMeta.setInputLocation( "/tmp/customers/io/state-data.txt" );
    rightInputMeta.setFileDescriptionName( statePopulationFileDefinition.getName() );
    StepMeta rightInputStepMeta = new StepMeta(inputStepname+" Right", rightInputMeta);
    rightInputStepMeta.setStepID( BeamConst.STRING_BEAM_INPUT_PLUGIN_ID );
    transMeta.addStep( rightInputStepMeta );


    // Add a Merge Join step
    //
    MergeJoinMeta mergeJoin = new MergeJoinMeta();
    mergeJoin.allocate( 1, 1 );
    mergeJoin.getKeyFields1()[0] = "state";
    mergeJoin.getKeyFields2()[0] = "state";
    mergeJoin.setJoinType(MergeJoinMeta.join_types[3] ); // FULL OUTER
    mergeJoin.getStepIOMeta().getInfoStreams().get(0).setStepMeta( leftInputStepMeta );
    mergeJoin.getStepIOMeta().getInfoStreams().get(1).setStepMeta( rightInputStepMeta );
    StepMeta mergeJoinStepMeta = new StepMeta("Merge Join", mergeJoin);
    transMeta.addStep( mergeJoinStepMeta );
    transMeta.addTransHop( new TransHopMeta( leftInputStepMeta, mergeJoinStepMeta ) );
    transMeta.addTransHop( new TransHopMeta( rightInputStepMeta, mergeJoinStepMeta ) );

    // Add the output step to write results
    //
    BeamOutputMeta beamOutputMeta = new BeamOutputMeta();
    beamOutputMeta.setOutputLocation( "/tmp/customers/output/" );
    beamOutputMeta.setFileDescriptionName( null );
    beamOutputMeta.setFilePrefix( "merge-join" );
    beamOutputMeta.setFileSuffix( ".csv" );
    beamOutputMeta.setWindowed( false ); // Not yet supported
    StepMeta beamOutputStepMeta = new StepMeta(outputStepname, beamOutputMeta);
    beamOutputStepMeta.setStepID( "BeamOutput" );
    transMeta.addStep( beamOutputStepMeta );
    transMeta.addTransHop(new TransHopMeta( mergeJoinStepMeta, beamOutputStepMeta ) );

    return transMeta;
  }
 

@Test
public void testDisabledHops() {
  TransMeta trans = new TransMeta();
  StepMeta start = new StepMeta( "Start", stepMetaInterface );
  trans.addStep( start );
  StepMeta withEnabledHop = new StepMeta( "WithEnabledHop", stepMetaInterface );
  trans.addStep( withEnabledHop );
  StepMeta withDisabledHop = new StepMeta( "WithDisabledHop", stepMetaInterface );
  trans.addStep( withDisabledHop );
  StepMeta shouldStay = new StepMeta( "ShouldStay", stepMetaInterface );
  trans.addStep( shouldStay );
  StepMeta shouldNotStay = new StepMeta( "ShouldNotStay", stepMetaInterface );
  trans.addStep( shouldNotStay );
  StepMeta withEnabledAndDisabledHops = new StepMeta( "WithEnabledAndDisabledHops", stepMetaInterface );
  trans.addStep( withEnabledAndDisabledHops );
  StepMeta afterEnabledDisabled = new StepMeta( "AfterEnabledDisabled", stepMetaInterface );
  trans.addStep( afterEnabledDisabled );

  trans.addTransHop( new TransHopMeta( start, withEnabledHop ) );
  trans.addTransHop( new TransHopMeta( start, withDisabledHop, false ) );
  trans.addTransHop( new TransHopMeta( withEnabledHop, shouldStay ) );
  trans.addTransHop( new TransHopMeta( withDisabledHop, shouldStay ) );
  trans.addTransHop( new TransHopMeta( withDisabledHop, shouldNotStay ) );
  trans.addTransHop( new TransHopMeta( start, withEnabledAndDisabledHops ) );
  trans.addTransHop( new TransHopMeta( withEnabledHop, withEnabledAndDisabledHops, false ) );
  trans.addTransHop( new TransHopMeta( withEnabledAndDisabledHops, afterEnabledDisabled ) );

  Transformation transformation = TransMetaConverter.convert( trans );

  List<String>
    steps =
    transformation.getOperations().stream().map( op -> op.getId() ).collect( Collectors.toList() );
  assertThat( "Only 5 ops should exist", steps.size(), is( 5 ) );
  assertThat( steps, hasItems( "Start", "WithEnabledHop", "ShouldStay", "WithEnabledAndDisabledHops",
      "AfterEnabledDisabled" ) );

  List<String> hops = transformation.getHops().stream().map( hop -> hop.getId() ).collect( Collectors.toList() );
  assertThat( "Only 4 hops should exist", hops.size(), is( 4 ) );
  assertThat( hops, hasItems( "Start -> WithEnabledHop", "WithEnabledHop -> ShouldStay",
      "Start -> WithEnabledAndDisabledHops", "WithEnabledAndDisabledHops -> AfterEnabledDisabled" ) );
}
 

/**
 * Generates a template
 * @param databaseMeta
 * @param logicalModel
 * @return
 */
public TransMeta generateDimensionTransformation(DatabaseMeta databaseMeta, LogicalTable logicalTable) {
  TransMeta transMeta = new TransMeta();

  String tableName = ConceptUtil.getName(logicalTable, locale);
  String tableDescription = ConceptUtil.getDescription(logicalTable, locale);
  DimensionType dimensionType = ConceptUtil.getDimensionType(logicalTable);

  transMeta.setName("Update dimension '"+tableName+"'");
  transMeta.setDescription(tableDescription);

  // Let's not forget to add the target database
  //
  transMeta.addDatabase(databaseMeta);

  Point location = new Point(GRAPH_LEFT, GRAPH_TOP);

  // Find all the source columns and source tables and put them into a table input step...
  //
  StepMeta inputStep = generateTableInputStepFromLogicalTable(logicalTable);
  DatabaseMeta sourceDatabaseMeta = ((TableInputMeta)inputStep.getStepMetaInterface()).getDatabaseMeta();
  if (sourceDatabaseMeta!=null) transMeta.addOrReplaceDatabase(sourceDatabaseMeta);
  inputStep.setLocation(location.x, location.y);
  nextLocation(location);
  transMeta.addStep(inputStep);
  StepMeta lastStep = inputStep;

  // Generate an dimension lookup/update step for each table
  //
  StepMeta dimensionStep;
  if (dimensionType==DimensionType.SLOWLY_CHANGING_DIMENSION) {
    dimensionStep = generateDimensionLookupStepFromLogicalTable(databaseMeta, logicalTable);
  } else {
    dimensionStep = generateCombinationLookupStepFromLogicalTable(databaseMeta, logicalTable);
  }
  dimensionStep.setLocation(location.x, location.y);
  nextLocation(location);
  transMeta.addStep(dimensionStep);

  TransHopMeta transHop = new TransHopMeta(lastStep, dimensionStep);
  transMeta.addTransHop(transHop);

  return transMeta;
}
 

/**
 * Tests output rows
 */
@Test
public void testTextFileOutput1() throws Exception {
  KettleEnvironment.init();

  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "testTextFileOutput1" );
  PluginRegistry registry = PluginRegistry.getInstance();

  // create an injector step
  String injectorStepName = "injector step";
  StepMeta injectorStep = TestUtilities.createInjectorStep( injectorStepName, registry );
  transMeta.addStep( injectorStep );

  // create a row generator step
  StepMeta rowGeneratorStep = createRowGeneratorStep( "Create rows for testTextFileOutput1", registry );
  transMeta.addStep( rowGeneratorStep );

  // create a TransHopMeta for injector and add it to the transMeta
  TransHopMeta hop_injectory_rowGenerator = new TransHopMeta( injectorStep, rowGeneratorStep );
  transMeta.addTransHop( hop_injectory_rowGenerator );

  // create the text file output step
  // but first lets get a filename
  String textFileName = TestUtilities.createEmptyTempFile( "testTextFileOutput1" );
  StepMeta textFileOutputStep =
    createTextFileOutputStep( "text file output step", textFileName, "None", registry );
  transMeta.addStep( textFileOutputStep );

  // create a TransHopMeta for textFileOutputStep and add it to the transMeta
  TransHopMeta hop_RowGenerator_outputTextFile = new TransHopMeta( rowGeneratorStep, textFileOutputStep );
  transMeta.addTransHop( hop_RowGenerator_outputTextFile );

  // Create a dummy step and add it to the tranMeta
  String dummyStepName = "dummy step";
  StepMeta dummyStep = createDummyStep( dummyStepName, registry );
  transMeta.addStep( dummyStep );

  // create a TransHopMeta for the
  TransHopMeta hop_outputTextFile_dummyStep = new TransHopMeta( textFileOutputStep, dummyStep );
  transMeta.addTransHop( hop_outputTextFile_dummyStep );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  // Create a row collector and add it to the dummy step interface
  StepInterface dummyStepInterface = trans.getStepInterface( dummyStepName, 0 );
  RowStepCollector dummyRowCollector = new RowStepCollector();
  dummyStepInterface.addRowListener( dummyRowCollector );

  // RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
  trans.startThreads();
  trans.waitUntilFinished();

  // Compare the results
  List<RowMetaAndData> resultRows = dummyRowCollector.getRowsWritten();
  List<RowMetaAndData> goldenImageRows = createResultData1();

  try {
    TestUtilities.checkRows( goldenImageRows, resultRows );
  } catch ( TestFailedException tfe ) {
    fail( tfe.getMessage() );
  }
}