Java Code Examples for org.pentaho.di.core.variables.Variables#setVariable()

@Test
public void testVariableCondition() throws KettleException {
  Variables varSpace = new Variables();
  varSpace.setVariable( "myVarCondition", realCondition );

  JavaFilterMeta meta = new JavaFilterMeta();
  meta.setCondition( "${myVarCondition}" );

  TransMeta trans = TransTestFactory.generateTestTransformation( varSpace, meta, stepName );
  List<RowMetaAndData> input = getInputData();

  List<RowMetaAndData> result = TransTestFactory.executeTestTransformation( trans,
    TransTestFactory.INJECTOR_STEPNAME, stepName, TransTestFactory.DUMMY_STEPNAME, input );

  //Only the "Car" row should be returned, based upon the Java Filter condition
  assertTrue( result.size() == 1 );
  assertTrue( result.get( 0 ).getString( fieldName, null ).equals( "Car" ) );
}
 

@Test
public void testFilterVariables() throws Exception {

  initByFile( "default.csv" );

  Variables vars = new Variables();
  vars.setVariable( "VAR_TEST", "second" );
  data.filterProcessor =
      new TextFileFilterProcessor( new TextFileFilter[] { new TextFileFilter( 0, "${VAR_TEST}", false, false ) },
          vars );
  setFields( new BaseFileField( "f1", -1, -1 ), new BaseFileField( "f2", -1, -1 ),
    new BaseFileField( "f2", -1, -1 ) );

  process();

  check( new Object[][] { { "first", "1", "1.1" }, { "third", "3", "3.3" } } );
}
 

@Test
public void testPDI18818() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-pdi-18818.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();

  //Did we read both values?
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );

  //Did we read both files?
  assertEquals( 6, trans.getSteps().get( 1 ).step.getLinesWritten() );

  //Did we find any nulls?
  assertEquals( 0, trans.getSteps().get( 4 ).step.getLinesRead() );
}
 

@Test
public void testGetDataFromFolderWithInvalidFieldName() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-from-previous-step-negative.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 0, trans.getSteps().get( 1 ).step.getLinesWritten() );
  assertEquals( 0, trans.getSteps().get( 1 ).step.getLinesInput() );
  // The path contains one entry of a folder
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );
}
 

@Test
public void testGetDataFromListOfFilesFromPreviousStep() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-files-from-previous-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 1 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 1 ).step.getLinesInput() );
  // The path contains 7 entries containing csv file paths
  assertEquals( 7, trans.getSteps().get( 0 ).step.getLinesWritten() );
}
 

@Test
public void testGetDataFromAFolderRecursivelyFromPreviousStep() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-from-previous-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 1 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 1 ).step.getLinesInput() );
  // The path contains one entry of a folder
  assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() );
}
 

@Test
public void testResolvePasswordVariable() {
  String passwordKey = "PASS_VAR";
  String passwordVar = "${" + passwordKey + "}";
  String passwordValue = "password";
  Variables vars = new Variables();
  vars.setVariable( passwordKey, passwordValue );
  //resolvePassword gets variable
  assertSame( passwordValue, Utils.resolvePassword( vars, passwordVar ).intern() );
}
 

@Test
public void testCheckErrorsOnVariables() {
  List<CheckResultInterface> remarks = new ArrayList<>();
  Variables space = new Variables();
  space.setVariable( "something", "1000" );
  meta.setBatchSize( "${something}" );
  meta.setBatchDuration( "0" );
  meta.check( remarks, null, null, null, null, null, null, space, null, null );
  assertEquals( 0, remarks.size() );
}
 

@Test
public void withVariablesGetsNewObjectFromRegistry() throws KettleXMLException, KettleMissingPluginsException {
  String path = getClass().getResource( "/jms-consumer.ktr" ).getPath();
  TransMeta transMeta = new TransMeta( path, new Variables() );
  StepMeta step = transMeta.getStep( 0 );
  JmsConsumerMeta jmsMeta = (JmsConsumerMeta) step.getStepMetaInterface();
  assertEquals( "${testOne}", jmsMeta.jmsDelegate.amqUrl );
  Variables variables = new Variables();
  variables.setVariable( "testOne", "changedValue" );
  JmsConsumerMeta jmsMetaWithVars = (JmsConsumerMeta) jmsMeta.withVariables( variables );
  assertEquals( "changedValue", jmsMetaWithVars.jmsDelegate.amqUrl );
}
 

@Test
public void testGetDataFromFromFiles() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-files-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

@Test
public void testGetDataFromFolderRecursively() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

/**
 * The logic of this method:
 * 
 * if we have directory we return the child var space with directory used as system property
 * if we have not directory we return the child var space with directory extracted from filanme
 * if we don not have directory and filename we will return the child var space without updates
 * 
 * 
 * @param parentVariables - parent variable space which can be inherited
 * @param directory - current path which will be used as path for start trans/job
 * @param filename - is file which we use at this moment
 * @param inheritParentVar - flag which indicate should we inherit variables from parent var space to child var space
 * @return new var space if inherit was set false or child var space with updated system variables
 */
public VariableSpace resolveCurrentDirectory( VariableSpace parentVariables, RepositoryDirectoryInterface directory, String filename ) {
  Variables tmpSpace = new Variables();
  tmpSpace.setParentVariableSpace( parentVariables );
  tmpSpace.initializeVariablesFrom( parentVariables );

  if ( directory != null ) {
    tmpSpace.setVariable( Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, directory.toString() );
    tmpSpace.setVariable( Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, directory.toString() );
    tmpSpace.setVariable( Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, directory.toString() );
  } else if ( filename != null ) {
    try {
      FileObject fileObject = KettleVFS.getFileObject( filename, tmpSpace );

      if ( !fileObject.exists() ) {
        // don't set variables if the file doesn't exist
        return tmpSpace;
      }

      FileName fileName = fileObject.getName();

      // The filename of the transformation
      tmpSpace.setVariable( Const.INTERNAL_VARIABLE_JOB_FILENAME_NAME, fileName.getBaseName() );

      // The directory of the transformation
      FileName fileDir = fileName.getParent();
      tmpSpace.setVariable( Const.INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, fileDir.getURI() );
      tmpSpace.setVariable( Const.INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, fileDir.getURI() );
      tmpSpace.setVariable( Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, fileDir.getURI() );
    } catch ( Exception e ) {
    }
  }
  return tmpSpace;
}
 

/**
 * @return new var space with follow vars from parent space or just new space if parent was null
 * 
 * {@link org.pentaho.di.core.Const#INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY}
 * {@link org.pentaho.di.core.Const#INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY}
 * {@link org.pentaho.di.core.Const#INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY}
 * {@link org.pentaho.di.core.Const#INTERNAL_VARIABLE_JOB_FILENAME_NAME}
 */
private static VariableSpace getVarSpaceOnlyWithRequiredParentVars( VariableSpace parentSpace ) {
  Variables tmpSpace = new Variables();
  if ( parentSpace != null ) {
    tmpSpace.setVariable( INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, parentSpace.getVariable( INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY ) );
    tmpSpace.setVariable( INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY, parentSpace.getVariable( INTERNAL_VARIABLE_JOB_FILENAME_DIRECTORY ) );
    tmpSpace.setVariable( INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY, parentSpace.getVariable( INTERNAL_VARIABLE_TRANSFORMATION_FILENAME_DIRECTORY ) );
    tmpSpace.setVariable( INTERNAL_VARIABLE_JOB_FILENAME_NAME, parentSpace.getVariable( INTERNAL_VARIABLE_JOB_FILENAME_NAME ) );
  }
  return tmpSpace;
}
 

@Test
public void testCheckErrorsOnVariablesSubstituteError() {
  List<CheckResultInterface> remarks = new ArrayList<>();
  Variables space = new Variables();
  space.setVariable( "something", "0" );
  meta.setBatchSize( "${something}" );
  meta.setBatchDuration( "${something}" );
  meta.check( remarks, null, null, null, null, null, null, space, null, null );
  assertEquals( 1, remarks.size() );
  assertEquals( "The \"Number of records\" and \"Duration\" fields can’t both be set to 0. Please set a value of 1 "
    + "or higher for one of the fields.", remarks.get( 0 ).getText() );
  testRoundTrip( meta );
}
 

@Test
public void testParameterFolderNameOnlyFolders() throws KettleException, IOException {
  GetFileNamesMeta meta = new GetFileNamesMeta();
  meta.setDefault();

  meta.allocate( 1 );
  meta.setFileName( new String[]{ "${MY_FOLDER_PARAM}" } );
  meta.setFileMask( new String[]{ ".*" } );
  meta.setExcludeFileMask( new String[]{ "" } );
  meta.setFileRequired( new String[]{ "Y" } );
  meta.setIncludeSubFolders( new String[]{ "N" } );
  meta.setFilterFileType( 2 );

  TransMeta transMeta = TransTestFactory.generateTestTransformation( null, meta, STEPNAME );
  //Remove the Injector hop, as it's not needed for this transformation
  TransHopMeta injectHop = transMeta.findTransHop( transMeta.findStep( TransTestFactory.INJECTOR_STEPNAME ),
    transMeta.findStep( STEPNAME ) );
  transMeta.removeTransHop( transMeta.indexOfTransHop( injectHop ) );

  transMeta.addParameterDefinition( "MY_FOLDER_PARAM", "C:\\ThisFolderDoesNotExist", "The folder to look in for files" );
  Variables varSpace = new Variables();
  varSpace.setVariable( "MY_FOLDER_PARAM", tempFolder.getRoot().getAbsolutePath() );

  // Content inside selected folder
  String expectedFilename = "file.tmp";
  String expectedSubFolderName = "subfolder";
  tempFolder.newFile( expectedFilename );
  tempFolder.newFolder( expectedSubFolderName );

  List<RowMetaAndData> result =
    TransTestFactory.executeTestTransformation( transMeta, TransTestFactory.INJECTOR_STEPNAME, STEPNAME,
      TransTestFactory.DUMMY_STEPNAME, new ArrayList<RowMetaAndData>(), null, varSpace );

  // Check that the expected file was located correctly
  assertNotNull( result );
  assertEquals( 1, result.size() );
  assertTrue( result.get( 0 ).getRowMeta().indexOfValue( "short_filename" ) >= 0 );
  assertEquals( expectedSubFolderName, result.get( 0 ).getString( "short_filename", "failure" ) );
}
 

@Test
public void testParameterFolderNameWithoutWildcardAndOnlyFiles() throws KettleException, IOException {
  GetFileNamesMeta meta = new GetFileNamesMeta();
  meta.setDefault();

  meta.allocate( 1 );
  meta.setFileName( new String[]{ "${MY_FOLDER_PARAM}" } );
  meta.setFileMask( new String[]{ "" } );
  meta.setExcludeFileMask( new String[]{ "" } );
  meta.setFileRequired( new String[]{ "Y" } );
  meta.setIncludeSubFolders( new String[]{ "N" } );
  meta.setFilterFileType( 1 );

  TransMeta transMeta = TransTestFactory.generateTestTransformation( null, meta, STEPNAME );
  //Remove the Injector hop, as it's not needed for this transformation
  TransHopMeta injectHop = transMeta.findTransHop( transMeta.findStep( TransTestFactory.INJECTOR_STEPNAME ),
    transMeta.findStep( STEPNAME ) );
  transMeta.removeTransHop( transMeta.indexOfTransHop( injectHop ) );

  transMeta.addParameterDefinition( "MY_FOLDER_PARAM", "C:\\ThisFolderDoesNotExist", "The folder to look in for files" );
  Variables varSpace = new Variables();
  varSpace.setVariable( "MY_FOLDER_PARAM", tempFolder.getRoot().getAbsolutePath() );

  // Content inside selected folder
  String expectedFilename = "file.tmp";
  String expectedSubFolderName = "subfolder";
  tempFolder.newFile( expectedFilename );
  tempFolder.newFolder( expectedSubFolderName );

  List<RowMetaAndData> result =
    TransTestFactory.executeTestTransformation( transMeta, TransTestFactory.INJECTOR_STEPNAME, STEPNAME,
      TransTestFactory.DUMMY_STEPNAME, new ArrayList<RowMetaAndData>(), null, varSpace );

  // Check that the expected file was located correctly
  assertNotNull( result );
  assertEquals( 1, result.size() );
  assertTrue( result.get( 0 ).getRowMeta().indexOfValue( "short_filename" ) >= 0 );
  assertEquals( expectedFilename, result.get( 0 ).getString( "short_filename", "failure" ) );
}
 

@Test
public void testParameterFolderNameOnlyFiles() throws KettleException, IOException {
  GetFileNamesMeta meta = new GetFileNamesMeta();
  meta.setDefault();

  meta.allocate( 1 );
  meta.setFileName( new String[]{ "${MY_FOLDER_PARAM}" } );
  meta.setFileMask( new String[]{ ".*" } );
  meta.setExcludeFileMask( new String[]{ "" } );
  meta.setFileRequired( new String[]{ "Y" } );
  meta.setIncludeSubFolders( new String[]{ "N" } );
  meta.setFilterFileType( 1 );

  TransMeta transMeta = TransTestFactory.generateTestTransformation( null, meta, STEPNAME );
  //Remove the Injector hop, as it's not needed for this transformation
  TransHopMeta injectHop = transMeta.findTransHop( transMeta.findStep( TransTestFactory.INJECTOR_STEPNAME ),
    transMeta.findStep( STEPNAME ) );
  transMeta.removeTransHop( transMeta.indexOfTransHop( injectHop ) );

  transMeta.addParameterDefinition( "MY_FOLDER_PARAM", "C:\\ThisFolderDoesNotExist", "The folder to look in for files" );
  Variables varSpace = new Variables();
  varSpace.setVariable( "MY_FOLDER_PARAM", tempFolder.getRoot().getAbsolutePath() );

  // Content inside selected folder
  String expectedFilename = "file.tmp";
  String expectedSubFolderName = "subfolder";
  tempFolder.newFile( expectedFilename );
  tempFolder.newFolder( expectedSubFolderName );

  List<RowMetaAndData> result =
    TransTestFactory.executeTestTransformation( transMeta, TransTestFactory.INJECTOR_STEPNAME, STEPNAME,
      TransTestFactory.DUMMY_STEPNAME, new ArrayList<RowMetaAndData>(), null, varSpace );

  // Check that the expected file was located correctly
  assertNotNull( result );
  assertEquals( 1, result.size() );
  assertTrue( result.get( 0 ).getRowMeta().indexOfValue( "short_filename" ) >= 0 );
  assertEquals( expectedFilename, result.get( 0 ).getString( "short_filename", "failure" ) );
}
 

@Test
public void testParameterFolderNameWithoutWildcard() throws KettleException, IOException {
  GetFileNamesMeta meta = new GetFileNamesMeta();
  meta.setDefault();

  meta.allocate( 1 );
  meta.setFileName( new String[]{ "${MY_FOLDER_PARAM}" } );
  meta.setFileMask( new String[]{ "" } );
  meta.setExcludeFileMask( new String[]{ "" } );
  meta.setFileRequired( new String[]{ "Y" } );
  meta.setIncludeSubFolders( new String[]{ "N" } );

  TransMeta transMeta = TransTestFactory.generateTestTransformation( null, meta, STEPNAME );
  //Remove the Injector hop, as it's not needed for this transformation
  TransHopMeta injectHop = transMeta.findTransHop( transMeta.findStep( TransTestFactory.INJECTOR_STEPNAME ),
    transMeta.findStep( STEPNAME ) );
  transMeta.removeTransHop( transMeta.indexOfTransHop( injectHop ) );

  transMeta.addParameterDefinition( "MY_FOLDER_PARAM", "C:\\ThisFolderDoesNotExist", "The folder to look in for files" );
  Variables varSpace = new Variables();
  varSpace.setVariable( "MY_FOLDER_PARAM", tempFolder.getRoot().getAbsolutePath() );

  // Content inside selected folder
  String expectedFilename = "file.tmp";
  String expectedSubFolderName = "subfolder";
  tempFolder.newFile( expectedFilename );
  tempFolder.newFolder( expectedSubFolderName );

  List<RowMetaAndData> result =
    TransTestFactory.executeTestTransformation( transMeta, TransTestFactory.INJECTOR_STEPNAME, STEPNAME,
      TransTestFactory.DUMMY_STEPNAME, new ArrayList<RowMetaAndData>(), null, varSpace );

  // Check that the expected file was located correctly
  assertNotNull( result );
  assertEquals( 2, result.size() );
  assertTrue( result.get( 0 ).getRowMeta().indexOfValue( "short_filename" ) >= 0 );
  assertEquals( expectedFilename, result.get( 0 ).getString( "short_filename", "failure" ) );
  assertTrue( result.get( 1 ).getRowMeta().indexOfValue( "short_filename" ) >= 0 );
  assertEquals( expectedSubFolderName, result.get( 1 ).getString( "short_filename", "failure" ) );
}
 

private String reapplyCurrentDir( String oldCurrentDir, String newCurrentDir, String path ) {
  Variables vars = new Variables();
  vars.setVariable( Const.INTERNAL_VARIABLE_ENTRY_CURRENT_DIRECTORY, oldCurrentDir );
  String newPath = vars.environmentSubstitute( path );
  return getPath( newCurrentDir, newPath );
}
 

@Test
public void testPDI12897() throws KettleException, IOException {
  File tempOutputFile = File.createTempFile( "testPDI11374", ".cube" );
  tempOutputFile.deleteOnExit();

  String stepName = "Cube Output";
  CubeOutputMeta meta = new CubeOutputMeta();
  meta.setDefault();
  meta.setFilename( tempOutputFile.getAbsolutePath() );

  TransMeta transMeta = TransTestFactory.generateTestTransformation( null, meta, stepName );
  List<RowMetaAndData> inputList = getSampleData();
  TransTestFactory.executeTestTransformation( transMeta, TransTestFactory.INJECTOR_STEPNAME, stepName,
    TransTestFactory.DUMMY_STEPNAME, inputList );

  try {
    Thread.sleep( 1000 );
  } catch ( InterruptedException ignored ) {
    // Continue
  }

  // Now, check the result
  Variables varSpace = new Variables();
  varSpace.setVariable( "ROWS", "2" );

  String checkStepName = "Cube Input";
  CubeInputMeta inputMeta = new CubeInputMeta();
  inputMeta.setFilename( tempOutputFile.getAbsolutePath() );
  inputMeta.setRowLimit( "${ROWS}" );

  transMeta = TransTestFactory.generateTestTransformation( varSpace, inputMeta, checkStepName );

  //Remove the Injector hop, as it's not needed for this transformation
  TransHopMeta injectHop = transMeta.findTransHop( transMeta.findStep( TransTestFactory.INJECTOR_STEPNAME ),
    transMeta.findStep( stepName ) );
  transMeta.removeTransHop( transMeta.indexOfTransHop( injectHop ) );

  inputList = new ArrayList<RowMetaAndData>();
  List<RowMetaAndData> result =
    TransTestFactory.executeTestTransformation( transMeta, TransTestFactory.INJECTOR_STEPNAME, stepName,
      TransTestFactory.DUMMY_STEPNAME, inputList );

  assertNotNull( result );
  assertEquals( 2, result.size() );
  assertEquals( 1, result.get( 0 ).getRowMeta().size() );
  assertEquals( ValueMetaInterface.TYPE_STRING, result.get( 0 ).getValueMeta( 0 ).getType() );
  assertEquals( "col1", result.get( 0 ).getValueMeta( 0 ).getName() );
  assertEquals( "data1", result.get( 0 ).getString( 0, "fail" ) );
  assertEquals( "data2", result.get( 1 ).getString( 0, "fail" ) );
}