Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#clone()

@Override
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta,
  RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository,
  IMetaStore metaStore ) throws KettleStepException {

  // Find the lookupfields...
  //
  RowMetaInterface out = prev.clone();
  getFields( out, stepMeta.getName(), new RowMetaInterface[] { info, }, null, transMeta, repository, metaStore );

  if ( out != null ) {
    for ( int i = 0; i < out.size(); i++ ) {
      ValueMetaInterface outvalue = out.getValueMeta( i );
      DatabaseImpact di =
        new DatabaseImpact(
          DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepMeta.getName(),
          databaseMeta.getDatabaseName(), "", outvalue.getName(), outvalue.getName(), stepMeta.getName(),
          transMeta.environmentSubstitute( sql ),
          BaseMessages.getString( PKG, "DatabaseJoinMeta.DatabaseImpact.Title" ) );
      impact.add( di );

    }
  }
}
 

public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta,
  RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository,
  IMetaStore metaStore ) throws KettleStepException {

  RowMetaInterface out = prev.clone();
  getFields( out, stepMeta.getName(), new RowMetaInterface[] { info, }, null, transMeta, repository, metaStore );
  if ( out != null ) {
    for ( int i = 0; i < out.size(); i++ ) {
      ValueMetaInterface outvalue = out.getValueMeta( i );
      DatabaseImpact di =
        new DatabaseImpact(
          DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepMeta.getName(), databaseMeta
            .getDatabaseName(), "", outvalue.getName(), outvalue.getName(), stepMeta.getName(), sql,
          BaseMessages.getString( PKG, "DynamicSQLRowMeta.DatabaseImpact.Title" ) );
      impact.add( di );

    }
  }
}
 

private void putRowToRowSet( RowSet rs, RowMetaInterface rowMeta, Object[] row ) {
  RowMetaInterface toBeSent;
  RowMetaInterface metaFromRs = rs.getRowMeta();
  if ( metaFromRs == null ) {
    // RowSet is not initialised so far
    toBeSent = rowMeta.clone();
  } else {
    // use the existing
    toBeSent = metaFromRs;
  }

  while ( !rs.putRow( toBeSent, row ) ) {
    if ( isStopped() && !safeStopped.get() ) {
      return;
    }
  }
}
 

/**
 * Safe mode checking.
 *
 * @param row the row
 * @throws KettleRowException the kettle row exception
 */
protected void safeModeChecking( RowMetaInterface row ) throws KettleRowException {
  if ( row == null ) {
    return;
  }

  if ( inputReferenceRow == null ) {
    inputReferenceRow = row.clone(); // copy it!

    // Check for double field names.
    //
    String[] fieldnames = row.getFieldNames();
    Arrays.sort( fieldnames );
    for ( int i = 0; i < fieldnames.length - 1; i++ ) {
      if ( fieldnames[ i ].equals( fieldnames[ i + 1 ] ) ) {
        throw new KettleRowException( BaseMessages.getString(
          PKG, "BaseStep.SafeMode.Exception.DoubleFieldnames", fieldnames[ i ] ) );
      }
    }
  } else {
    safeModeChecking( inputReferenceRow, row );
  }
}
 

@Override
public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  try {
    RowMetaInterface rowMeta = inputRowMeta.clone();
    inputRowMeta.clear();
    inputRowMeta.addRowMeta( rowMeta );

    getSelectFields( inputRowMeta, name );
    getDeleteFields( inputRowMeta );
    getMetadataFields( inputRowMeta, name );
  } catch ( Exception e ) {
    throw new KettleStepException( e );
  }
}
 

private void putToDefaultCache( Database db, List<Object[]> rows ) {
  final int keysAmount = meta.getStreamKeyField1().length;
  RowMetaInterface prototype = copyValueMetasFrom( db.getReturnRowMeta(), keysAmount );

  // Copy the data into 2 parts: key and value...
  //
  for ( Object[] row : rows ) {
    int index = 0;
    // not sure it is efficient to re-create the same on every row,
    // but this was done earlier, so I'm keeping this behaviour
    RowMetaInterface keyMeta = prototype.clone();
    Object[] keyData = new Object[ keysAmount ];
    for ( int i = 0; i < keysAmount; i++ ) {
      keyData[ i ] = row[ index++ ];
    }
    // RowMeta valueMeta = new RowMeta();
    Object[] valueData = new Object[ data.returnMeta.size() ];
    for ( int i = 0; i < data.returnMeta.size(); i++ ) {
      valueData[ i ] = row[ index++ ];
      // valueMeta.addValueMeta(returnRowMeta.getValueMeta(index++));
    }
    // Store the data...
    //
    data.cache.storeRowInCache( meta, keyMeta, keyData, valueData );
    incrementLinesInput();
  }
}
 

private Object[] constructErrorRow( RowMetaInterface rowMeta, Object[] r, String keyField,
  String referenceValue, String compareValue ) throws KettleException {

  if ( data.errorRowMeta == null ) {
    data.errorRowMeta = rowMeta.clone();
  }

  r[data.keyDescIndex] = keyField;
  r[data.valueReferenceIndex] = referenceValue;
  r[data.valueCompareIndex] = compareValue;

  return r;
}
 

public RowMetaInterface getAllFields( RowMetaInterface inputRowMeta ) {
  RowMetaInterface rowMeta = inputRowMeta.clone();

  for ( CalculatorMetaFunction fn : getCalculation() ) {
    if ( !Utils.isEmpty( fn.getFieldName() ) ) { // It's a new field!
      ValueMetaInterface v = getValueMeta( fn, null );
      rowMeta.addValueMeta( v );
    }
  }
  return rowMeta;
}
 

/**
 * Attempts to put the <code>row</code> onto the underlying <code>rowProducer</code> during its timeout period.
 * Returns <code>true</code> if the operation completed successfully and <code>false</code> otherwise.
 *
 * @param rowMeta input row's meta data
 * @param row     input row
 * @return <code>true</code> if the <code>row</code> was put successfully
 */
public boolean putRow( RowMetaInterface rowMeta, Object[] row ) {
  if ( first ) {
    first = false;
    renamedRowMeta = rowMeta.clone();

    for ( MappingValueRename valueRename : inputDefinition.getValueRenames() ) {
      ValueMetaInterface valueMeta = renamedRowMeta.searchValueMeta( valueRename.getSourceValueName() );
      if ( valueMeta != null ) {
        valueMeta.setName( valueRename.getTargetValueName() );
      }
    }
  }
  return rowProducer.putRow( renamedRowMeta, row, false );
}
 

@Test
public void testLookupValuesWhenMainFieldIsNull() throws Exception {
  FuzzyMatchData data = spy( new FuzzyMatchData() );
  FuzzyMatchMeta meta = spy( new FuzzyMatchMeta() );
  data.readLookupValues = false;
  fuzzyMatch =
          new FuzzyMatchHandler( mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta,
                  mockHelper.trans );
  fuzzyMatch.init( meta, data );
  fuzzyMatch.first = false;
  data.indexOfMainField = 1;
  Object[] inputRow = { "test input", null };
  RowSet lookupRowSet = mockHelper.getMockInputRowSet( new Object[]{ "test lookup" } );
  fuzzyMatch.addRowSetToInputRowSets( mockHelper.getMockInputRowSet( inputRow ) );
  fuzzyMatch.addRowSetToInputRowSets( lookupRowSet );
  fuzzyMatch.rowset = lookupRowSet;

  RowMetaInterface rowMetaInterface = new RowMeta();
  ValueMetaInterface valueMeta = new ValueMetaString( "field1" );
  valueMeta.setStorageMetadata( new ValueMetaString( "field1" ) );
  valueMeta.setStorageType( ValueMetaInterface.TYPE_STRING );
  rowMetaInterface.addValueMeta( valueMeta );
  when( lookupRowSet.getRowMeta() ).thenReturn( rowMetaInterface );
  fuzzyMatch.setInputRowMeta( rowMetaInterface.clone() );
  data.outputRowMeta = rowMetaInterface.clone();

  fuzzyMatch.processRow( meta, data );
  Assert.assertEquals( inputRow[0], fuzzyMatch.resultRow[0] );
  Assert.assertNull( fuzzyMatch.resultRow[1] );
  Assert.assertTrue( Arrays.stream( fuzzyMatch.resultRow, 3, fuzzyMatch.resultRow.length ).allMatch( val ->  val == null ) );
}
 

public void put( DBCacheEntry entry, RowMetaInterface fields ) {
  if ( !useCache ) {
    return;
  }

  RowMetaInterface copy = fields.clone();
  cache.put( entry, copy );
}
 

/**
 * Get the fields as a row generated by a database cache entry
 *
 * @param entry
 *          the entry to look for
 * @return the fields as a row generated by a database cache entry
 */
public RowMetaInterface get( DBCacheEntry entry ) {
  if ( !useCache ) {
    return null;
  }

  RowMetaInterface fields = cache.get( entry );
  if ( fields != null ) {
    fields = fields.clone(); // Copy it again!
  }

  return fields;
}
 

@Override
protected Map<String, RowMetaInterface> getInputRowMetaInterfaces( GetXMLDataMeta meta ) {
  Map<String, RowMetaInterface> inputRows = getInputFields( meta );
  if ( inputRows == null ) {
    inputRows = new HashMap<>();
  }
  // Get some boolean flags from the meta for easier access
  boolean isInFields = meta.isInFields();
  boolean isAFile = meta.getIsAFile();
  boolean isAUrl = meta.isReadUrl();

  // only add resource fields if we are NOT getting the xml or file from a field
  if ( !isInFields || isAFile || isAUrl ) {
    RowMetaInterface stepFields = getOutputFields( meta );
    RowMetaInterface clone = stepFields.clone();
    // if there are previous steps providing data, we should remove them from the set of "resource" fields
    for ( RowMetaInterface rowMetaInterface : inputRows.values() ) {
      for ( ValueMetaInterface valueMetaInterface : rowMetaInterface.getValueMetaList() ) {
        try {
          clone.removeValueMeta( valueMetaInterface.getName() );
        } catch ( KettleValueException e ) {
          // could not find it in the output, skip it
        }
      }
    }
    inputRows.put( RESOURCE, clone );
  }
  return inputRows;
}
 

private boolean readLookupValues() throws KettleException {
  data.infoStream = meta.getStepIOMeta().getInfoStreams().get( 0 );
  if ( data.infoStream.getStepMeta() == null ) {
    logError( BaseMessages.getString( PKG, "StreamLookup.Log.NoLookupStepSpecified" ) );
    return false;
  }
  if ( log.isDetailed() ) {
    logDetailed( BaseMessages.getString( PKG, "StreamLookup.Log.ReadingFromStream" )
      + data.infoStream.getStepname() + "]" );
  }

  int[] keyNrs = new int[meta.getKeylookup().length];
  int[] valueNrs = new int[meta.getValue().length];
  boolean firstRun = true;

  // Which row set do we read from?
  //
  RowSet rowSet = findInputRowSet( data.infoStream.getStepname() );
  Object[] rowData = getRowFrom( rowSet ); // rows are originating from "lookup_from"
  while ( rowData != null ) {
    if ( log.isRowLevel() ) {
      logRowlevel( BaseMessages.getString( PKG, "StreamLookup.Log.ReadLookupRow" )
        + rowSet.getRowMeta().getString( rowData ) );
    }

    if ( firstRun ) {
      firstRun = false;
      data.hasLookupRows = true;

      data.infoMeta = rowSet.getRowMeta().clone();
      RowMetaInterface cacheKeyMeta = new RowMeta();
      RowMetaInterface cacheValueMeta = new RowMeta();

 // Look up the keys in the source rows
      for ( int i = 0; i < meta.getKeylookup().length; i++ ) {
        keyNrs[i] = rowSet.getRowMeta().indexOfValue( meta.getKeylookup()[i] );
        if ( keyNrs[i] < 0 ) {
          throw new KettleStepException( BaseMessages.getString(
            PKG, "StreamLookup.Exception.UnableToFindField", meta.getKeylookup()[i] ) );
        }
        cacheKeyMeta.addValueMeta( rowSet.getRowMeta().getValueMeta( keyNrs[i] ) );
      }
     // Save the data types of the keys to optionally convert input rows later on...
      if ( data.keyTypes == null ) {
        data.keyTypes = cacheKeyMeta.clone();
      }

      // Cache keys are stored as normal types, not binary
      for ( int i = 0; i < keyNrs.length; i++ ) {
        cacheKeyMeta.getValueMeta( i ).setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
      }

     for ( int v = 0; v < meta.getValue().length; v++ ) {
        valueNrs[v] = rowSet.getRowMeta().indexOfValue( meta.getValue()[v] );
        if ( valueNrs[v] < 0 ) {
          throw new KettleStepException( BaseMessages.getString(
            PKG, "StreamLookup.Exception.UnableToFindField", meta.getValue()[v] ) );
        }
        cacheValueMeta.addValueMeta( rowSet.getRowMeta().getValueMeta( valueNrs[v] ) );
      }

      data.cacheKeyMeta = cacheKeyMeta;
      data.cacheValueMeta = cacheValueMeta;
    }

    Object[] keyData = new Object[keyNrs.length];
    for ( int i = 0; i < keyNrs.length; i++ ) {
      ValueMetaInterface keyMeta = data.keyTypes.getValueMeta( i );
      // Convert keys to normal storage type
      keyData[i] = keyMeta.convertToNormalStorageType( rowData[keyNrs[i]] );
    }

    Object[] valueData = new Object[valueNrs.length];
    for ( int i = 0; i < valueNrs.length; i++ ) {
      // Store value as is, avoid preliminary binary->normal storage type conversion
      valueData[i] = rowData[valueNrs[i]];
    }

    addToCache( data.cacheKeyMeta, keyData, data.cacheValueMeta, valueData );

    rowData = getRowFrom( rowSet );
  }

  return true;
}
 

public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  // First load some interesting data...

  // Then see which fields get added to the row.
  //
  TransMeta mappingTransMeta = null;
  try {
    mappingTransMeta =
      loadMappingMeta( this, repository, metaStore, space, mappingParameters.isInheritingAllVariables() );
  } catch ( KettleException e ) {
    throw new KettleStepException( BaseMessages.getString(
      PKG, "SimpleMappingMeta.Exception.UnableToLoadMappingTransformation" ), e );
  }

  // The field structure may depend on the input parameters as well (think of parameter replacements in MDX queries
  // for instance)
  if ( mappingParameters != null && mappingTransMeta != null ) {

    // Just set the variables in the transformation statically.
    // This just means: set a number of variables or parameter values:
    //
    StepWithMappingMeta.activateParams( mappingTransMeta, mappingTransMeta, space, mappingTransMeta.listParameters(),
      mappingParameters.getVariable(), mappingParameters.getInputField(), mappingParameters.isInheritingAllVariables() );
  }

  // Keep track of all the fields that need renaming...
  //
  List<MappingValueRename> inputRenameList = new ArrayList<MappingValueRename>();

  //
  // Before we ask the mapping outputs anything, we should teach the mapping
  // input steps in the sub-transformation about the data coming in...
  //

  RowMetaInterface inputRowMeta;

  // The row metadata, what we pass to the mapping input step
  // definition.getOutputStep(), is "row"
  // However, we do need to re-map some fields...
  //
  inputRowMeta = row.clone();
  if ( !inputRowMeta.isEmpty() ) {
    for ( MappingValueRename valueRename : inputMapping.getValueRenames() ) {
      ValueMetaInterface valueMeta = inputRowMeta.searchValueMeta( valueRename.getSourceValueName() );
      if ( valueMeta == null ) {
        throw new KettleStepException( BaseMessages.getString(
          PKG, "SimpleMappingMeta.Exception.UnableToFindField", valueRename.getSourceValueName() ) );
      }
      valueMeta.setName( valueRename.getTargetValueName() );
    }
  }

  // What is this mapping input step?
  //
  StepMeta mappingInputStep = mappingTransMeta.findMappingInputStep( null );

  // We're certain it's a MappingInput step...
  //
  MappingInputMeta mappingInputMeta = (MappingInputMeta) mappingInputStep.getStepMetaInterface();

  // Inform the mapping input step about what it's going to receive...
  //
  mappingInputMeta.setInputRowMeta( inputRowMeta );

  // What values are we changing names for: already done!
  //
  mappingInputMeta.setValueRenames( null );

  // Keep a list of the input rename values that need to be changed back at
  // the output
  //
  if ( inputMapping.isRenamingOnOutput() ) {
    SimpleMapping.addInputRenames( inputRenameList, inputMapping.getValueRenames() );
  }

  StepMeta mappingOutputStep = mappingTransMeta.findMappingOutputStep( null );

  // We know it's a mapping output step...
  MappingOutputMeta mappingOutputMeta = (MappingOutputMeta) mappingOutputStep.getStepMetaInterface();

  // Change a few columns.
  mappingOutputMeta.setOutputValueRenames( outputMapping.getValueRenames() );

  // Perhaps we need to change a few input columns back to the original?
  //
  mappingOutputMeta.setInputValueRenames( inputRenameList );

  // Now we know wat's going to come out of there...
  // This is going to be the full row, including all the remapping, etc.
  //
  RowMetaInterface mappingOutputRowMeta = mappingTransMeta.getStepFields( mappingOutputStep );

  row.clear();
  row.addRowMeta( mappingOutputRowMeta );
}
 

private void handlePutError( RowMetaInterface rowMeta, Object[] row, long nrErrors, String errorDescriptions,
                             String fieldNames, String errorCodes ) throws KettleStepException {
  if ( trans.isSafeModeEnabled() ) {
    if ( rowMeta.size() > row.length ) {
      throw new KettleStepException( BaseMessages.getString(
        PKG, "BaseStep.Exception.MetadataDoesntMatchDataRowSize", Integer.toString( rowMeta.size() ), Integer
          .toString( row != null ? row.length : 0 ) ) );
    }
  }

  StepErrorMeta stepErrorMeta = stepMeta.getStepErrorMeta();

  if ( errorRowMeta == null ) {
    errorRowMeta = rowMeta.clone();

    RowMetaInterface add = stepErrorMeta.getErrorRowMeta( nrErrors, errorDescriptions, fieldNames, errorCodes );
    errorRowMeta.addRowMeta( add );
  }

  Object[] errorRowData = RowDataUtil.allocateRowData( errorRowMeta.size() );
  if ( row != null ) {
    System.arraycopy( row, 0, errorRowData, 0, rowMeta.size() );
  }

  // Also add the error fields...
  stepErrorMeta.addErrorRowData(
    errorRowData, rowMeta.size(), nrErrors, errorDescriptions, fieldNames, errorCodes );

  // call all row listeners...
  for ( RowListener listener : rowListeners ) {
    listener.errorRowWrittenEvent( rowMeta, row );
  }

  if ( errorRowSet != null ) {
    while ( !errorRowSet.putRow( errorRowMeta, errorRowData ) ) {
      if ( isStopped() ) {
        break;
      }
    }
    incrementLinesRejected();
  }

  verifyRejectionRates();
}