org.pentaho.di.trans.step.StepInterface Java Examples

/**
 * This method runs transformations related to PDI-13545.<br/> The scenario is the following: there are two step
 * generating data, the latter of which is a Mapping step. They are followed with a Join Rows step, that has two
 * copies. The last in a row is a Dummy step, named "Last". Since both generating steps output 3 rows ([10, 20, 30]
 * and [1, 2, 3] respectively), the last step must obtain 3*3=9 rows.
 *
 * @param transPath a path to transformation file
 * @throws Exception
 */
private void runTransWhenMappingsIsFollowedByCopiedStep( String transPath ) throws Exception {
  KettleEnvironment.init();

  TransMeta transMeta = new TransMeta( transPath );
  transMeta.setTransformationType( TransMeta.TransformationType.Normal );

  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  assertEquals( 0, trans.getErrors() );

  List<StepInterface> list = trans.findBaseSteps( "Last" );
  assertEquals( 1, list.size() );
  assertEquals( 9, list.get( 0 ).getLinesRead() );
}
 

public void stopStep( StepMetaDataCombi combi, boolean safeStop ) {
  StepInterface rt = combi.step;
  rt.setStopped( true );
  rt.setSafeStopped( safeStop );
  rt.resumeRunning();

  try {
    rt.stopRunning( combi.meta, combi.data );
  } catch ( Exception e ) {
    log.logError( "Something went wrong while trying to safe stop the transformation: ", e );
  }
  combi.data.setStatus( StepExecutionStatus.STATUS_STOPPED );
  if ( safeStop ) {
    rt.setOutputDone();
  }
}
 

/**
 * Gets the mapping inputs for each step in the transformation.
 *
 * @return an array of MappingInputs
 */
public MappingInput[] findMappingInput() {
  if ( steps == null ) {
    return null;
  }

  List<MappingInput> list = new ArrayList<>();

  // Look in threads and find the MappingInput step thread...
  for ( int i = 0; i < steps.size(); i++ ) {
    StepMetaDataCombi smdc = steps.get( i );
    StepInterface step = smdc.step;
    if ( step.getStepID().equalsIgnoreCase( "MappingInput" ) ) {
      list.add( (MappingInput) step );
    }
  }
  return list.toArray( new MappingInput[ list.size() ] );
}
 

public static double getProcessCount( Context actualContext, Scriptable actualObject, Object[] ArgList,
  Function FunctionContext ) {

  if ( ArgList.length == 1 ) {
    try {
      Object scmO = actualObject.get( "_step_", actualObject );
      StepInterface scm = (StepInterface) Context.jsToJava( scmO, StepInterface.class );
      String strType = Context.toString( ArgList[0] ).toLowerCase();

      if ( strType.equals( "i" ) ) {
        return scm.getLinesInput();
      } else if ( strType.equals( "o" ) ) {
        return scm.getLinesOutput();
      } else if ( strType.equals( "r" ) ) {
        return scm.getLinesRead();
      } else if ( strType.equals( "u" ) ) {
        return scm.getLinesUpdated();
      } else if ( strType.equals( "w" ) ) {
        return scm.getLinesWritten();
      } else if ( strType.equals( "e" ) ) {
        return scm.getLinesRejected();
      } else {
        return 0;
      }
    } catch ( Exception e ) {
      // throw Context.reportRuntimeError(e.toString());
      return 0;
    }
  } else {
    throw Context.reportRuntimeError( "The function call getProcessCount requires 1 argument." );
  }
}
 

private void subscribeToOpLogging() throws KettleException {
  transformation.getOperations().stream().forEach( operation -> {
    try {
      messageEventService.addHandler( Util.getOperationLogEvent( operation.getId() ),
        new MessageEventHandler() {
          @Override
          public void execute( Message message ) throws MessageEventHandlerExecutionException {
            PDIEvent<RemoteSource, LogEntry> event = (PDIEvent<RemoteSource, LogEntry>) message;
            LogEntry logEntry = event.getData();
            StepInterface stepInterface = findStepInterface( operation.getId(), 0 );
            if ( stepInterface != null ) {
              // This is intended to put a red error (dash) on the step in PDI.
              // In order to do that 3 things are needed: errors have to be set
              // to a positive number, the state is stopped state (not finished)
              // and Error log on the step (done just below this if statement)
              if ( LogLevel.ERROR.equals( logEntry.getLogLogLevel() ) ) {
                stepInterface.setErrors( 1 );
                stepInterface.setStopped( true );
              }

              LogChannelInterface logChannel = stepInterface.getLogChannel();
              logToChannel( logChannel, logEntry );
            } else {
              // Could not find step, log at transformation level instead
              logToChannel( getLogChannel(), logEntry );
            }
          }

          @Override
          public String getIdentifier() {
            return OPERATION_LOG + operation.getKey();
          }
        } );
    } catch ( HandlerRegistrationException e ) {
      getLogChannel().logError( "Error registering message handlers", e );
    }
  } );
}
 

/**
 * Test 'Swim lines partitioning'
 * 
 * @throws KettleException
 */
@Test
public void testSwimLanesPartitioning() throws KettleException {
  prepareStepMetas_cl1_cl1();

  trans.prepareExecution( new String[] {} );
  List<RowSet> rowsets = trans.getRowsets();
  assertTrue( !rowsets.isEmpty() );
  assertEquals( "We have 2 rowsets finally", 2, rowsets.size() );
  assertEquals( "We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 4, trans.getSteps()
      .size() );

  // Ok, examine initialized steps now.
  StepInterface stepOne0 = getStepByName( SP10 );
  assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() );
  assertEquals( "1.0 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() );

  StepInterface stepOne1 = getStepByName( SP11 );
  assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() );
  assertEquals( "1.1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() );

  StepInterface stepTwo0 = getStepByName( SP20 );
  assertEquals( "2.0 step have 2 input row sets", 1, stepTwo0.getInputRowSets().size() );
  assertTrue( "2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty() );

  StepInterface stepTwo2 = getStepByName( SP21 );
  assertTrue( "2.2 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() );
  assertEquals( "2.2 step have 2 output rowsets", 1, stepTwo2.getInputRowSets().size() );
}
 

/**
 * Gets the run thread for the step at the specified index.
 *
 * @param i the index of the desired step
 * @return a StepInterface object corresponding to the run thread for the specified step
 */
public StepInterface getRunThread( int i ) {
  if ( steps == null ) {
    return null;
  }
  return steps.get( i ).step;
}
 

protected void processRows( StepInterface step, final int maxCalls ) throws Exception {
  for ( int outRowIdx = 0; outRowIdx < maxCalls; outRowIdx++ ) {
    if ( !step.processRow( helper.processRowsStepMetaInterface, helper.processRowsStepDataInterface ) ) {
      break;
    }
  }
}
 

public StepInterface getStep(StepMeta stepMeta,
    StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
    Trans trans) {

  return new CassandraInput(stepMeta, stepDataInterface, copyNr, transMeta,
      trans);
}
 

/**
 * Find the run thread for the step with the specified name.
 *
 * @param stepname the step name
 * @return a StepInterface object corresponding to the run thread for the specified step
 */
public StepInterface findRunThread( String stepname ) {
  if ( steps == null ) {
    return null;
  }

  for ( int i = 0; i < steps.size(); i++ ) {
    StepMetaDataCombi sid = steps.get( i );
    StepInterface step = sid.step;
    if ( step.getStepname().equalsIgnoreCase( stepname ) ) {
      return step;
    }
  }
  return null;
}
 

/**
 * Find the executing step copy for the step with the specified name and copy number
 *
 * @param stepname the step name
 * @param copynr
 * @return the executing step found or null if no copy could be found.
 */
public StepInterface findStepInterface( String stepname, int copyNr ) {
  if ( steps == null ) {
    return null;
  }

  for ( int i = 0; i < steps.size(); i++ ) {
    StepMetaDataCombi sid = steps.get( i );
    StepInterface stepInterface = sid.step;
    if ( stepInterface.getStepname().equalsIgnoreCase( stepname ) && sid.copy == copyNr ) {
      return stepInterface;
    }
  }
  return null;
}
 

@Before
public void setUp() throws Exception {
  stepInterface = mock( StepInterface.class );
  rowSet = mock( RowSet.class );
  rowProducer = new RowProducer( stepInterface, rowSet );
  rowMeta = mock( RowMetaInterface.class );
  rowData = new Object[]{};
}
 

@Test
@PrepareForTest( { Encode.class } )
public void testSniffStepServletEscapesHtmlWhenTransFound() throws ServletException, IOException {
  KettleLogStore.init();
  HttpServletRequest mockHttpServletRequest = mock( HttpServletRequest.class );
  HttpServletResponse mockHttpServletResponse = mock( HttpServletResponse.class );
  Trans mockTrans = mock( Trans.class );
  TransMeta mockTransMeta = mock( TransMeta.class );
  StepInterface mockStepInterface = mock( StepInterface.class );
  List<StepInterface> stepInterfaces = new ArrayList<StepInterface>();
  stepInterfaces.add( mockStepInterface );
  LogChannelInterface mockChannelInterface = mock( LogChannelInterface.class );
  StringWriter out = new StringWriter();
  PrintWriter printWriter = new PrintWriter( out );

  PowerMockito.spy( Encode.class );
  when( mockHttpServletRequest.getContextPath() ).thenReturn( SniffStepServlet.CONTEXT_PATH );
  when( mockHttpServletRequest.getParameter( anyString() ) ).thenReturn( ServletTestUtils.BAD_STRING_TO_TEST );
  when( mockHttpServletResponse.getWriter() ).thenReturn( printWriter );
  when( mockTransformationMap.getTransformation( any( CarteObjectEntry.class ) ) ).thenReturn( mockTrans );
  when( mockTrans.getLogChannel() ).thenReturn( mockChannelInterface );
  when( mockTrans.getLogChannelId() ).thenReturn( "test" );
  when( mockTrans.getTransMeta() ).thenReturn( mockTransMeta );
  when( mockTransMeta.getMaximum() ).thenReturn( new Point( 10, 10 ) );
  when( mockTrans.findBaseSteps( ServletTestUtils.BAD_STRING_TO_TEST ) ).thenReturn( stepInterfaces );

  sniffStepServlet.doGet( mockHttpServletRequest, mockHttpServletResponse );
  assertFalse( ServletTestUtils.hasBadText( ServletTestUtils.getInsideOfTag( "H1", out.toString() ) ) );

  PowerMockito.verifyStatic( atLeastOnce() );
  Encode.forHtml( anyString() );
}
 

@Test( expected = KettleException.class )
public void testVerifyTransMetaOutKeyNotDefined() throws KettleException {
  String inputStepName = "inputStepName";
  String outputStepName = "outputStepName";
  StepMeta inputStepMeta = mock( StepMeta.class );
  when( transMeta.findStep( inputStepName ) ).thenReturn( inputStepMeta );
  RowMetaInterface rowMetaInterface = mock( RowMetaInterface.class );
  when( rowMetaInterface.getFieldNames() ).thenReturn( new String[] { "key", "value" } );
  when( transMeta.getStepFields( inputStepMeta ) ).thenReturn( rowMetaInterface );
  Trans trans = mock( Trans.class );
  when( transFactory.create( transMeta ) ).thenReturn( trans );
  when( trans.getStepInterface( inputStepName, 0 ) ).thenReturn( mock( StepInterface.class ) );
  final StepMeta outputStepMeta = mock( StepMeta.class );
  StepMetaInterface stepMetaInterface = mock( StepMetaInterface.class );
  when( outputStepMeta.getStepMetaInterface() ).thenReturn( stepMetaInterface );
  when( transMeta.findStep( outputStepName ) ).thenReturn( outputStepMeta );
  RowMetaInterface outputRowMetaInterface = mock( RowMetaInterface.class );
  when( transMeta.getStepFields( outputStepMeta ) ).thenReturn( outputRowMetaInterface );
  when( outputRowMetaInterface.getFieldNames() ).thenReturn( new String[] {} );
  try {
    pentahoMapReduceJobBuilder.verifyTransMeta( transMeta, inputStepName, outputStepName );
  } catch ( KettleException e ) {
    assertEquals( BaseMessages.getString( PentahoMapReduceJobBuilderImpl.PKG,
      PentahoMapReduceJobBuilderImpl.PENTAHO_MAP_REDUCE_JOB_BUILDER_IMPL_NO_OUTPUT_KEY_ORDINAL, outputStepName ),
      e.getMessage().trim() );
    throw e;
  }
}
 

/**
 * Create, execute, and return the row listener attached to the output step with complete results from the execution.
 *
 * @param checkSumType
 *          Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
 * @param compatibilityMode
 *          Use compatibility mode for CheckSum
 * @param fieldSeparatorString
 *          The string separate multiple fields with
 * @param inputs
 *          Array of objects representing row data
 * @param inputValueMetas
 *          metas to be processed
 * @return RowListener with results.
 */
private MockRowListener executeHexTest( int checkSumType, boolean compatibilityMode, boolean oldChecksumBehaviour,
    String fieldSeparatorString, Object[] inputs, ValueMetaInterface... inputValueMetas ) throws Exception {

  String[] fieldNames = new String[inputValueMetas.length];
  RowMeta inputRowMeta = new RowMeta();
  for ( int i = 0; i < inputValueMetas.length; i++ ) {
    inputRowMeta.addValueMeta( inputValueMetas[i] );
    fieldNames[i] = inputValueMetas[i].getName();
  }

  Trans trans =
      buildHexadecimalChecksumTrans( checkSumType, compatibilityMode, oldChecksumBehaviour, fieldSeparatorString,
          fieldNames );

  trans.prepareExecution( null );

  StepInterface output = trans.getRunThread( "Output", 0 );
  MockRowListener listener = new MockRowListener();
  output.addRowListener( listener );

  RowProducer rp = trans.addRowProducer( "CheckSum", 0 );

  ( (BaseStep) trans.getRunThread( "CheckSum", 0 ) ).setInputRowMeta( inputRowMeta );

  trans.startThreads();

  rp.putRow( inputRowMeta, inputs );
  rp.finished();

  trans.waitUntilFinished();
  trans.stopAll();
  trans.cleanup();
  return listener;
}
 

/**
 * This checks transformation initialization when using many to many copies.
 * 
 * @throws KettleException
 */
@Test
public void testManyToManyCopies() throws KettleException {
  prepareStepMetas_x2_x2();

  trans.prepareExecution( new String[] {} );
  List<RowSet> rowsets = trans.getRowsets();
  assertTrue( !rowsets.isEmpty() );
  assertEquals( "We have 2 rowsets finally", 2, rowsets.size() );
  assertEquals( "We have 4 steps: 2 copies of producer and 2 copies of consumer", 4, trans.getSteps().size() );

  // Ok, examine initialized steps now.
  StepInterface stepOne0 = getStepByName( S10 );
  assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() );
  assertEquals( "1 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() );

  StepInterface stepOne1 = getStepByName( S11 );
  assertTrue( "1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() );
  assertEquals( "1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() );

  StepInterface stepTwo0 = getStepByName( S20 );
  Assert.assertEquals( "2.0 step have 1 input row sets", 1, stepTwo0.getInputRowSets().size() );
  Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() );

  StepInterface stepTwo1 = getStepByName( S21 );
  Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() );
  Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() );
}
 

private void processRow( TransProcessControl control ) throws Exception {
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();
  generateData( rp );
  rp.finished();
  StepInterface si = trans.getStepInterface( synchronizeAfterMergeStepname, 0 );
  switch ( control ) {
    case ITTERUPT:
      trans.stopAll();
      while (  !si.getStatus().equals( StepExecutionStatus.STATUS_STOPPED ) ) {
        //wait until transformation does not stopped
      };
      break;
    case WAIT:
    default:
      trans.waitUntilFinished();
      assertEquals( "Step still started", StepExecutionStatus.STATUS_FINISHED, si.getStatus() );
      break;
  }
  assertEquals( "Unexpected error occurred",  0, si.getErrors() );

  Field field = SynchronizeAfterMerge.class.getDeclaredField( "data" );
  field.setAccessible( true );
  SynchronizeAfterMergeData  data = (SynchronizeAfterMergeData) field.get( si );
  //should be closed and set null after finish transformation
  assertNull( data.db.getConnection() );
}
 

@Test
public void testStepShouldStopProcessingInput_IfUnderlyingTransitionIsStopped() throws Exception {

  MappingInput mappingInput = mock( MappingInput.class );
  when( mappingInput.getStepname() ).thenReturn( MAPPING_INPUT_STEP_NAME );
  stepMockHelper.processRowsStepDataInterface.mappingInput = mappingInput;

  RowProducer rowProducer = mock( RowProducer.class );
  when( rowProducer.putRow( any( RowMetaInterface.class ), any( Object[].class ), anyBoolean() ) )
    .thenReturn( true );

  StepInterface stepInterface = mock( StepInterface.class );

  Trans mappingTrans = mock( Trans.class );
  when( mappingTrans.addRowProducer( anyString(), anyInt() ) ).thenReturn( rowProducer );
  when( mappingTrans.findStepInterface( anyString(), anyInt() ) ).thenReturn( stepInterface );
  when( mappingTrans.isFinishedOrStopped() ).thenReturn( Boolean.FALSE ).thenReturn( Boolean.TRUE );
  stepMockHelper.processRowsStepDataInterface.mappingTrans = mappingTrans;

  MappingOutput mappingOutput = mock( MappingOutput.class );
  when( mappingOutput.getStepname() ).thenReturn( MAPPING_OUTPUT_STEP_NAME );
  stepMockHelper.processRowsStepDataInterface.mappingOutput = mappingOutput;


  smp = new SimpleMapping( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta,
    stepMockHelper.trans );
  smp.init( stepMockHelper.initStepMetaInterface, simpleMpData );
  smp.addRowSetToInputRowSets( stepMockHelper.getMockInputRowSet( new Object[] { } ) );
  smp.addRowSetToInputRowSets( stepMockHelper.getMockInputRowSet( new Object[] { } ) );

  assertTrue(
    smp.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ) );
  assertFalse(
    smp.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ) );

}
 

/**
 * Test case for javascript functionality: trans_Status and SKIP_TRANSFORMATION. Regression test case for JIRA defect
 * PDI-364.
 */
public void testTransStatus() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript trans_Status" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  // process 3 rows and skip the rest.
  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "trans_Status = CONTINUE_TRANSFORMATION;\n"
      + "if (getProcessCount(\"r\") > 3) {\n" + " \ttrans_Status = SKIP_TRANSFORMATION;\n" + "}" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] {} );
  svm.setRename( new String[] {} );
  svm.setType( new int[] {} );
  svm.setLength( new int[] {} );
  svm.setPrecision( new int[] {} );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData2();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}
 

public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  return new SalesforceUpsert( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr,
    TransMeta transMeta, Trans trans ) {
  return null;
}
 

/**
 * Test case for parameters using a simple transformation. Check whether parameters override variables.
 *
 * @throws Exception
 *           exception on any problem.
 */
public void testParameterSimpleTrans6() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "parameter_simple_trans4" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create a get variables step...
  //
  String getVariablesStepname = "get variables step";
  GetVariableMeta gvm = new GetVariableMeta();

  // Set the information of the get variables step.
  String getVariablesPid = registry.getPluginId( StepPluginType.class, gvm );
  StepMeta getVariablesStep = new StepMeta( getVariablesPid, getVariablesStepname, gvm );
  transMeta.addStep( getVariablesStep );

  //
  // Generate 1 row
  //
  String[] fieldName = { "PARAM1", "PARAM2" };
  String[] varName = { "${Param1}", "%%PARAM2%%" };
  int[] fieldType = { ValueMetaInterface.TYPE_STRING, ValueMetaInterface.TYPE_STRING };
  int[] length = { -1, -1 };
  int[] precision = { -1, -1 };
  String[] format = { "", "" };
  String[] currency = { "", "" };
  String[] decimal = { "", "" };
  String[] grouping = { "", "" };
  int[] trimType = { ValueMetaInterface.TRIM_TYPE_NONE, ValueMetaInterface.TRIM_TYPE_NONE };

  FieldDefinition[] fields = new FieldDefinition[fieldName.length];
  for ( int i = 0; i < fields.length; i++ ) {
    FieldDefinition field = new FieldDefinition();
    field.setFieldName( fieldName[i] );
    field.setVariableString( varName[i] );
    field.setFieldType( fieldType[i] );
    field.setFieldLength( length[i] );
    field.setFieldPrecision( precision[i] );
    field.setFieldFormat( format[i] );
    field.setCurrency( currency[i] );
    field.setDecimal( decimal[i] );
    field.setGroup( grouping[i] );
    field.setTrimType( trimType[i] );
    fields[i] = field;
  }
  gvm.setFieldDefinitions( fields );

  //
  // Create a dummy step 1
  //
  String dummyStepname1 = "dummy step 1";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname1, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi1 = new TransHopMeta( getVariablesStep, dummyStep1 );
  transMeta.addTransHop( hi1 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );
  trans.addParameterDefinition( "Param1", "", "Parameter 1" );
  trans.addParameterDefinition( "PARAM2", "", "Parameter 2" );
  trans.setParameterValue( "PARAM2", "PARAMVALUE2" );

  // See whether this variable overrides the parameter... it should NOT. Param1
  // is defined but not set. And no default... so the variable will be set to "". not
  // to "Variable1"
  trans.setVariable( "Param1", "Variable1" );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname1, 0 );
  RowStepCollector endRc = new RowStepCollector();
  si.addRowListener( endRc );

  trans.startThreads();

  trans.waitUntilFinished();

  // Now check whether the output is still as we expect.
  List<RowMetaAndData> goldenImageRows = createResultData6();
  List<RowMetaAndData> resultRows1 = endRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr, TransMeta tr,
  Trans trans ) {
  return new Calculator( stepMeta, stepDataInterface, cnr, tr, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  return new FieldsChangeSequence( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  return new LucidDBBulkLoader( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
                              Trans trans ) {
  return new MQTTProducer( stepMeta, stepDataInterface, copyNr, transMeta, trans );
}
 

/**
 * Test "Load All Rows" version of BasicDatabaseLookup test.
 */
@Test
public void CacheAndLoadAllRowsDatabaseLookup() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "transname" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.

  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the lookup step...
  //
  String lookupName = "look up from [" + lookup_table + "]";
  DatabaseLookupMeta dbl = new DatabaseLookupMeta();
  dbl.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  dbl.setTablename( lookup_table );
  dbl.setCached( true );
  dbl.setLoadingAllDataInCache( true );
  dbl.setEatingRowOnLookupFailure( false );
  dbl.setFailingOnMultipleResults( false );
  dbl.setOrderByClause( "" );

  dbl.setTableKeyField( new String[] { "ID" } );
  dbl.setKeyCondition( new String[] { "=" } );
  dbl.setStreamKeyField1( new String[] { "int_field" } );
  dbl.setStreamKeyField2( new String[] { "" } );

  dbl.setReturnValueField( new String[] { "CODE", "STRING" } );
  dbl.setReturnValueDefaultType( new int[] { ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_STRING } );
  dbl.setReturnValueDefault( new String[] { "-1", "UNDEF" } );
  dbl.setReturnValueNewName( new String[] { "RET_CODE", "RET_STRING" } );

  String lookupId = registry.getPluginId( StepPluginType.class, dbl );
  StepMeta lookupStep = new StepMeta( lookupId, lookupName, dbl );
  lookupStep.setDescription( "Reads information from table [" + lookup_table + "] on database [" + dbInfo + "]" );
  transMeta.addStep( lookupStep );

  TransHopMeta hi = new TransHopMeta( injectorStep, lookupStep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( lookupName, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createDataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();
  List<RowMetaAndData> goldRows = createResultDataRows();
  checkRows( goldRows, resultRows );

}
 

public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  return new MailValidator( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

@Override
public StepInterface getStep( StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr,
  TransMeta transMeta, Trans trans ) {
  return new MemoryGroupBy( stepMeta, stepDataInterface, cnr, transMeta, trans );
}
 

/**
 * Test case for javascript functionality: ltrim(), rtrim(), trim().
 */
public void testLuhnCheck() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "test javascript LuhnCheck" );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // Create a javascript step
  //
  String javaScriptStepname = "javascript step";
  ScriptValuesMetaMod svm = new ScriptValuesMetaMod();

  ScriptValuesScript[] js =
    new ScriptValuesScript[] { new ScriptValuesScript(
      ScriptValuesScript.TRANSFORM_SCRIPT, "script", "var str = string;\n" + "var bool = LuhnCheck(str);" ) };
  svm.setJSScripts( js );
  svm.setFieldname( new String[] { "bool" } );
  svm.setRename( new String[] { "" } );
  svm.setType( new int[] { ValueMetaInterface.TYPE_BOOLEAN } );
  svm.setLength( new int[] { -1 } );
  svm.setPrecision( new int[] { -1 } );
  svm.setReplace( new boolean[] { false } );
  svm.setCompatible( false );

  String javaScriptStepPid = registry.getPluginId( StepPluginType.class, svm );
  StepMeta javaScriptStep = new StepMeta( javaScriptStepPid, javaScriptStepname, svm );
  transMeta.addStep( javaScriptStep );

  TransHopMeta hi1 = new TransHopMeta( injectorStep, javaScriptStep );
  transMeta.addTransHop( hi1 );

  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm = new DummyTransMeta();

  String dummyPid = registry.getPluginId( StepPluginType.class, dm );
  StepMeta dummyStep = new StepMeta( dummyPid, dummyStepname, dm );
  transMeta.addStep( dummyStep );

  TransHopMeta hi2 = new TransHopMeta( javaScriptStep, dummyStep );
  transMeta.addTransHop( hi2 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si;

  si = trans.getStepInterface( javaScriptStepname, 0 );
  RowStepCollector javaScriptRc = new RowStepCollector();
  si.addRowListener( javaScriptRc );

  si = trans.getStepInterface( dummyStepname, 0 );
  RowStepCollector dummyRc = new RowStepCollector();
  si.addRowListener( dummyRc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createData1();
  Iterator<RowMetaAndData> it = inputList.iterator();
  while ( it.hasNext() ) {
    RowMetaAndData rm = it.next();
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> goldenImageRows = createResultData1();
  List<RowMetaAndData> resultRows1 = javaScriptRc.getRowsWritten();
  checkRows( resultRows1, goldenImageRows );

  List<RowMetaAndData> resultRows2 = dummyRc.getRowsRead();
  checkRows( resultRows2, goldenImageRows );
}