Java Code Examples for org.pentaho.di.core.Result#setRows()

@Test
public void testGetXML() throws KettleException {
  SlaveServerTransStatus transStatus = new SlaveServerTransStatus();
  RowMetaAndData rowMetaAndData = new RowMetaAndData();
  String testData = "testData";
  rowMetaAndData.addValue( new ValueMetaString(), testData );
  List<RowMetaAndData> rows = new ArrayList<>();
  rows.add( rowMetaAndData );
  Result result = new Result();
  result.setRows( rows );
  transStatus.setResult( result );
  //PDI-15781
  Assert.assertFalse( transStatus.getXML().contains( testData ) );
  //PDI-17061
  Assert.assertTrue( transStatus.getXML( true ).contains( testData ) );
}
 

@Test
public void testHTTPResultDefaultRows() throws IOException {
  File localFileForUpload = getInputFile( "existingFile1", ".tmp" );
  File tempFileForDownload = File.createTempFile( "downloadedFile1", ".tmp" );
  localFileForUpload.deleteOnExit();
  tempFileForDownload.deleteOnExit();

  Object[] r = new Object[] { HTTP_SERVER_BASEURL + "/uploadFile",
          localFileForUpload.getCanonicalPath(), tempFileForDownload.getCanonicalPath() };
  RowMeta rowMetaDefault = new RowMeta();
  rowMetaDefault.addValueMeta( new ValueMetaString( "URL" ) );
  rowMetaDefault.addValueMeta( new ValueMetaString( "UPLOAD" ) );
  rowMetaDefault.addValueMeta( new ValueMetaString( "DESTINATION" ) );
  List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
  rows.add( new RowMetaAndData( rowMetaDefault, r ) );
  Result previousResult = new Result();
  previousResult.setRows( rows );

  JobEntryHTTP http = new JobEntryHTTP();
  http.setParentJob( new Job() );
  http.setRunForEveryRow( true );
  http.setAddFilenameToResult( false );
  http.execute( previousResult, 0 );
  assertTrue( FileUtils.contentEquals( localFileForUpload, tempFileForDownload ) );
}
 

/**
 * This test check passing rows to sub-transformation executed on cluster
 * See PDI-10704 for details
 * @throws Exception
 */
public void runSubtransformationClustered() throws Exception {
  TransMeta transMeta =
    loadTransMetaReplaceSlavesInCluster(
      clusterGenerator, "test/org/pentaho/di/cluster/test-subtrans-clustered.ktr" );
  TransExecutionConfiguration config = createClusteredTransExecutionConfiguration();
  Result prevResult = new Result();
  prevResult.setRows( getSampleRows() );
  config.setPreviousResult( prevResult );

  TransSplitter transSplitter = Trans.executeClustered( transMeta, config );
  LogChannel logChannel = createLogChannel( "cluster unit test <runSubtransformationClustered>" );
  long nrErrors = Trans.monitorClusteredTransformation( logChannel, transSplitter, null, 1 );
  assertEquals( 0L, nrErrors );

  String result = loadFileContent( transMeta, "${java.io.tmpdir}/test-subtrans-clustered.txt" );
  assertEqualsIgnoreWhitespacesAndCase( "10", result );
}
 

@Test
public void testHTTPResultCustomRows() throws IOException {
  File localFileForUpload = getInputFile( "existingFile2", ".tmp" );
  File tempFileForDownload = File.createTempFile( "downloadedFile2", ".tmp" );
  localFileForUpload.deleteOnExit();
  tempFileForDownload.deleteOnExit();

  Object[] r = new Object[] { HTTP_SERVER_BASEURL + "/uploadFile",
          localFileForUpload.getCanonicalPath(), tempFileForDownload.getCanonicalPath() };
  RowMeta rowMetaDefault = new RowMeta();
  rowMetaDefault.addValueMeta( new ValueMetaString( "MyURL" ) );
  rowMetaDefault.addValueMeta( new ValueMetaString( "MyUpload" ) );
  rowMetaDefault.addValueMeta( new ValueMetaString( "MyDestination" ) );
  List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
  rows.add( new RowMetaAndData( rowMetaDefault, r ) );
  Result previousResult = new Result();
  previousResult.setRows( rows );

  JobEntryHTTP http = new JobEntryHTTP();
  http.setParentJob( new Job() );
  http.setRunForEveryRow( true );
  http.setAddFilenameToResult( false );
  http.setUrlFieldname( "MyURL" );
  http.setUploadFieldname( "MyUpload" );
  http.setDestinationFieldname( "MyDestination" );
  http.execute( previousResult, 0 );
  assertTrue( FileUtils.contentEquals( localFileForUpload, tempFileForDownload ) );
}
 

@Test
public void emptyResultsNotPostProcessed() throws KettleException {
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "field" ) );
  Result mockResult = new Result();
  mockResult.setRows( Arrays.asList( new RowMetaAndData( rowMeta, "queen" ), new RowMetaAndData( rowMeta, "king" ) ) );
  when( subtransExecutor.execute( any()  ) ).thenReturn( Optional.empty() );
  when( subtransExecutor.getPrefetchCount() ).thenReturn( 10 );
  AtomicInteger count = new AtomicInteger();
  FixedTimeStreamWindow<List> window =
    new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 2, 1, ( p ) -> count.set( p.getKey().get( 0 ).size() ) );
  window.buffer( Flowable.fromIterable( singletonList( asList( "v1", "v2" ) ) ) )
    .forEach( result -> assertEquals( mockResult, result ) );
  assertEquals( 0, count.get() );
}
 

@Test
public void supportsPostProcessing() throws KettleException {
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "field" ) );
  Result mockResult = new Result();
  mockResult.setRows( Arrays.asList( new RowMetaAndData( rowMeta, "queen" ), new RowMetaAndData( rowMeta, "king" ) ) );
  when( subtransExecutor.execute( any()  ) ).thenReturn( Optional.of( mockResult ) );
  when( subtransExecutor.getPrefetchCount() ).thenReturn( 10 );
  AtomicInteger count = new AtomicInteger();
  FixedTimeStreamWindow<List> window =
    new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 2, 1, ( p ) -> count.set( p.getKey().get( 0 ).size() ) );
  window.buffer( Flowable.fromIterable( singletonList( asList( "v1", "v2" ) ) ) )
    .forEach( result -> assertEquals( mockResult, result ) );
  assertEquals( 2, count.get() );
}
 

@Test
public void resultsComeBackToParent() throws KettleException {
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "field" ) );
  Result mockResult = new Result();
  mockResult.setRows( Arrays.asList( new RowMetaAndData( rowMeta, "queen" ), new RowMetaAndData( rowMeta, "king" ) ) );
  when( subtransExecutor.execute( any()  ) ).thenReturn( Optional.of( mockResult ) );
  when( subtransExecutor.getPrefetchCount() ).thenReturn( 10 );
  FixedTimeStreamWindow<List> window =
    new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 2, 1 );
  window.buffer( Flowable.fromIterable( singletonList( asList( "v1", "v2" ) ) ) )
    .forEach( result -> assertEquals( mockResult, result ) );
}
 

@Test
public void testStop() throws KettleException {
  Result result = new Result();
  result.setSafeStop( false );
  result.setRows( Collections.emptyList() );
  when( streamWindow.buffer( any() ) ).thenReturn( Collections.singletonList( result ) );

  baseStreamStep.processRow( meta, stepData );
  assertFalse( baseStreamStep.isSafeStopped() );
  verify( streamSource ).close();
}
 

@Test
public void filesPath_AreProcessed_ArgsOfPreviousMeta() throws Exception {
  jobEntry.setArgFromPrevious( true );

  Result prevMetaResult = new Result();
  List<RowMetaAndData> metaAndDataList = new ArrayList<>();

  metaAndDataList.add( constructRowMetaAndData( PATH_TO_FILE, null ) );
  prevMetaResult.setRows( metaAndDataList );

  jobEntry.execute( prevMetaResult, 0 );
  verify( jobEntry, times( metaAndDataList.size() ) ).processFile( anyString(), anyString(), any( Job.class ) );
}
 

@Test
public void filesWithNoPath_AreNotProcessed_ArgsOfPreviousMeta() throws Exception {
  jobEntry.setArgFromPrevious( true );

  Result prevMetaResult = new Result();
  List<RowMetaAndData> metaAndDataList = new ArrayList<>();

  metaAndDataList.add( constructRowMetaAndData( Const.EMPTY_STRING, null ) );
  metaAndDataList.add( constructRowMetaAndData( STRING_SPACES_ONLY, null ) );

  prevMetaResult.setRows( metaAndDataList );

  jobEntry.execute( prevMetaResult, 0 );
  verify( jobEntry, never() ).processFile( anyString(), anyString(), any( Job.class ) );
}
 

private Result generateDummyResult( int nRows ) {
  Result result = new Result();
  List<RowMetaAndData> rows = new ArrayList<>();
  for ( int i = 0; i < nRows; ++i ) {
    rows.add( new RowMetaAndData() );
  }
  result.setRows( rows );
  return result;
}
 

@Test
public void testHTTPResultDefaultRows() throws IOException {
  File localFileForUpload = getInputFile( "existingFile1", ".tmp" );
  File tempFileForDownload = File.createTempFile( "downloadedFile1", ".tmp" );
  localFileForUpload.deleteOnExit();
  tempFileForDownload.deleteOnExit();

  Object[] r = new Object[] { HTTP_SERVER_BASEURL + "/uploadFile",
    localFileForUpload.getCanonicalPath(), tempFileForDownload.getCanonicalPath() };
  RowMeta rowMetaDefault = new RowMeta();
  rowMetaDefault.addValueMeta( new ValueMetaString( "URL" ) );
  rowMetaDefault.addValueMeta( new ValueMetaString( "UPLOAD" ) );
  rowMetaDefault.addValueMeta( new ValueMetaString( "DESTINATION" ) );
  List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
  rows.add( new RowMetaAndData( rowMetaDefault, r ) );
  Result previousResult = new Result();
  previousResult.setRows( rows );

  JobEntryHTTP http = new JobEntryHTTP();
  http.setParentJob( new Job() );
  http.setRunForEveryRow( true );
  http.setAddFilenameToResult( false );
  http.setUsername( "admin" );
  http.setPassword( "password" );
  http.execute( previousResult, 0 );
  assertTrue( FileUtils.contentEquals( localFileForUpload, tempFileForDownload ) );
}
 

public Optional<Result> execute( List<RowMetaAndData> rows ) throws KettleException {
  if ( rows.isEmpty() || stopped ) {
    return Optional.empty();
  }

  Trans subtrans = this.createSubtrans();
  running.add( subtrans );
  parentTrans.addActiveSubTransformation( subTransName, subtrans );

  // Pass parameter values
  passParametersToTrans( subtrans, rows.get( 0 ) );

  Result result = new Result();
  result.setRows( rows );
  subtrans.setPreviousResult( result );

  subtrans.prepareExecution( this.parentTrans.getArguments() );
  List<RowMetaAndData> rowMetaAndData = new ArrayList<>();
  subtrans.getSteps().stream()
    .filter( c -> c.step.getStepname().equalsIgnoreCase( subStep ) )
    .findFirst()
    .ifPresent( c -> c.step.addRowListener( new RowAdapter() {
      @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) {
        rowMetaAndData.add( new RowMetaAndData( rowMeta, row ) );
      }
    } ) );
  subtrans.startThreads();

  subtrans.waitUntilFinished();
  updateStatuses( subtrans );
  running.remove( subtrans );

  Result subtransResult = subtrans.getResult();
  subtransResult.setRows( rowMetaAndData  );
  releaseBufferPermits( rows.size() );
  return Optional.of( subtransResult );
}
 

protected void updateResult( Result result ) {
  Result newResult = trans.getResult();
  result.clear(); // clear only the numbers, NOT the files or rows.
  result.add( newResult );
  if ( !Utils.isEmpty( newResult.getRows() ) || trans.isResultRowsSet() ) {
    result.setRows( newResult.getRows() );
  }
}
 

/**
 * Execute a job with previous results passed in.<br>
 * <br>
 * Execute called by JobEntryJob: don't clear the jobEntryResults.
 *
 * @param nr
 *          The job entry number
 * @param result
 *          the result of the previous execution
 * @return Result of the job execution
 * @throws KettleJobException
 */
public Result execute( int nr, Result result ) throws KettleException {
  setFinished( false );
  setActive( true );
  setInitialized( true );
  KettleEnvironment.setExecutionInformation( this, rep );

  // Where do we start?
  JobEntryCopy startpoint;

  // Perhaps there is already a list of input rows available?
  if ( getSourceRows() != null ) {
    result.setRows( getSourceRows() );
  }

  startpoint = jobMeta.findJobEntry( JobMeta.STRING_SPECIAL_START, 0, false );
  if ( startpoint == null ) {
    throw new KettleJobException( BaseMessages.getString( PKG, "Job.Log.CounldNotFindStartingPoint" ) );
  }

  JobEntrySpecial jes = (JobEntrySpecial) startpoint.getEntry();
  Result res;
  do {
    res = execute( nr, result, startpoint, null, BaseMessages.getString( PKG, "Job.Reason.StartOfJobentry" ) );
    setActive( false );
  } while ( jes.isRepeat() && !isStopped() );
  return res;
}
 

private void executeTransformation( List<String> incomingFieldValues ) throws KettleException {
  TransExecutorData transExecutorData = getData();
  // If we got 0 rows on input we don't really want to execute the transformation
  if ( transExecutorData.groupBuffer.isEmpty() ) {
    return;
  }
  transExecutorData.groupTimeStart = System.currentTimeMillis();

  if ( first ) {
    discardLogLines( transExecutorData );
  }

  Trans executorTrans = createInternalTrans();
  transExecutorData.setExecutorTrans( executorTrans );
  if ( incomingFieldValues != null ) {
    // Pass parameter values
    passParametersToTrans( incomingFieldValues );
  } else {
    List<String> lastIncomingFieldValues = getLastIncomingFieldValues();
    // incomingFieldValues == null-  There are no more rows - Last Case - pass previous values if exists
    // If not still pass the null parameter values
    passParametersToTrans( lastIncomingFieldValues != null && !lastIncomingFieldValues.isEmpty() ? lastIncomingFieldValues : incomingFieldValues );
  }


  // keep track for drill down in Spoon...
  getTrans().addActiveSubTransformation( getStepname(), executorTrans );

  Result result = new Result();
  result.setRows( transExecutorData.groupBuffer );
  executorTrans.setPreviousResult( result );

  try {
    executorTrans.prepareExecution( getTrans().getArguments() );

    // run transformation
    executorTrans.startThreads();

    // Inform the parent transformation we started something here...
    for ( DelegationListener delegationListener : getTrans().getDelegationListeners() ) {
      // TODO: copy some settings in the transformation execution configuration, not strictly needed
      // but the execution configuration information is useful in case of a transformation re-start on Carte
      delegationListener.transformationDelegationStarted( executorTrans, new TransExecutionConfiguration() );
    }

    // Wait a while until we're done with the transformation
    executorTrans.waitUntilFinished();

    result = executorTrans.getResult();
  } catch ( KettleException e ) {
    log.logError( "An error occurred executing the transformation: ", e );
    result.setResult( false );
    result.setNrErrors( 1 );
  }

  if ( result.isSafeStop() ) {
    getTrans().safeStop();
  }

  collectTransResults( result );
  collectExecutionResults( result );
  collectExecutionResultFiles( result );

  transExecutorData.groupBuffer.clear();
}
 

/**
 * Gets the result of the transformation. The Result object contains such measures as the number of errors, number of
 * lines read/written/input/output/updated/rejected, etc.
 *
 * @return the Result object containing resulting measures from execution of the transformation
 */
public Result getResult() {
  if ( steps == null ) {
    return null;
  }

  Result result = new Result();
  result.setNrErrors( errors.longValue() );
  result.setResult( errors.longValue() == 0 );
  TransLogTable transLogTable = transMeta.getTransLogTable();

  for ( int i = 0; i < steps.size(); i++ ) {
    StepMetaDataCombi sid = steps.get( i );
    StepInterface step = sid.step;

    result.setNrErrors( result.getNrErrors() + sid.step.getErrors() );
    result.getResultFiles().putAll( step.getResultFiles() );

    if ( step.isSafeStopped() ) {
      result.setSafeStop( step.isSafeStopped() );
    }

    if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_READ ) ) ) {
      result.setNrLinesRead( result.getNrLinesRead() + step.getLinesRead() );
    }
    if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_INPUT ) ) ) {
      result.setNrLinesInput( result.getNrLinesInput() + step.getLinesInput() );
    }
    if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_WRITTEN ) ) ) {
      result.setNrLinesWritten( result.getNrLinesWritten() + step.getLinesWritten() );
    }
    if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_OUTPUT ) ) ) {
      result.setNrLinesOutput( result.getNrLinesOutput() + step.getLinesOutput() );
    }
    if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_UPDATED ) ) ) {
      result.setNrLinesUpdated( result.getNrLinesUpdated() + step.getLinesUpdated() );
    }
    if ( step.getStepname().equals( transLogTable.getSubjectString( TransLogTable.ID.LINES_REJECTED ) ) ) {
      result.setNrLinesRejected( result.getNrLinesRejected() + step.getLinesRejected() );
    }
  }

  result.setRows( resultRows );
  if ( !Utils.isEmpty( resultFiles ) ) {
    result.setResultFiles( new HashMap<String, ResultFile>() );
    for ( ResultFile resultFile : resultFiles ) {
      result.getResultFiles().put( resultFile.toString(), resultFile );
    }
  }
  result.setStopped( isStopped() );
  result.setLogChannelId( log.getLogChannelId() );

  return result;
}
 

@Test
public void testSharedStreamingBatchPoolExecution() throws Exception {
  /*
  * Tests that there is only 1 thread running inside the pool at all times.
  * */

  final List<String> errors = new ArrayList<String>();

  // Only 1 thread should be present in the pool at a given time.
  System.setProperty( Const.SHARED_STREAMING_BATCH_POOL_SIZE, "1" );

  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "field" ) );
  Result mockResult = new Result();
  mockResult.setRows( Arrays.asList( new RowMetaAndData( rowMeta, "queen" ), new RowMetaAndData( rowMeta, "king" ) ) );

  FixedTimeStreamWindow<List> window1 = new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 10, 10 );
  FixedTimeStreamWindow<List> window2 = new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 10, 10 );
  Flowable flowable = Flowable.fromIterable( singletonList( asList( "v1", "v2" ) ) );

  Field field = window1.getClass().getDeclaredField( "sharedStreamingBatchPool" );
  field.setAccessible( true );
  ThreadPoolExecutor sharedStreamingBatchPool = (ThreadPoolExecutor) field.get( window1 );

  when( subtransExecutor.getPrefetchCount() ).thenReturn( 1000 );
  when( subtransExecutor.execute( any() ) ).thenAnswer( ( InvocationOnMock invocation ) -> {
    //The active count should always be 1.
    if ( sharedStreamingBatchPool.getActiveCount() != 1 ) {
      errors.add( "Error: Active count should have been 1 at all times. Current value: " + sharedStreamingBatchPool.getActiveCount() );
    }
    return Optional.of( mockResult );
  } );

  Thread bufferThread1 = new Thread( new BufferThread( window1, flowable, mockResult ) );
  bufferThread1.start();

  Thread bufferThread2 = new Thread( new BufferThread( window2, flowable, mockResult ) );
  bufferThread2.start();

  Thread.sleep( 10000 );
  assertEquals( 0, errors.size() );
}