Java Code Examples for org.pentaho.di.core.row.RowDataUtil#allocateRowData()

/**
 * Test message size is correct
 *
 * @throws Exception
 */
@Test
public void testMessageSizeIsParsed() throws Exception {
  int[] fields = { MailInputField.COLUMN_SIZE };
  MailInputField[] farr = this.getDefaultInputFields( fields );
  this.mockMailInputMeta( farr );
  try {
    mailInput.processRow( meta, data );
  } catch ( KettleException e ) {
    // don't worry about it
  }
  MessageParser underTest = mailInput.new MessageParser();
  Object[] r = RowDataUtil.allocateRowData( data.nrFields );
  underTest.parseToArray( r, message );

  Assert.assertEquals( "Message Size is correct", new Long( CNTNT_SIZE ), Long.class.cast( r[0] ) );
}
 

/**
 * Test message From can be parsed correctly
 *
 * @throws Exception
 */
@Test
public void testMessageFromIsParsed() throws Exception {
  int[] fields = { MailInputField.COLUMN_SENDER };
  MailInputField[] farr = this.getDefaultInputFields( fields );
  this.mockMailInputMeta( farr );
  try {
    mailInput.processRow( meta, data );
  } catch ( KettleException e ) {
    // don't worry about it
  }
  MessageParser underTest = mailInput.new MessageParser();
  Object[] r = RowDataUtil.allocateRowData( data.nrFields );
  underTest.parseToArray( r, message );

  // expect, that from is concatenated with ';'
  String expected = StringUtils.join( new String[] { FROM1, FROM2 }, ";" );
  Assert.assertEquals( "Message From is correct", expected, String.class.cast( r[0] ) );
}
 

/**
 * [PDI-6532] When mail header is not found returns empty String
 *
 * @throws Exception
 *
 */
@Test
public void testHeadersParsedNegative() throws Exception {
  int[] fields = { MailInputField.COLUMN_HEADER };
  MailInputField[] farr = this.getDefaultInputFields( fields );
  farr[0].setName( HDR_EX1 + "salt" );

  this.mockMailInputMeta( farr );

  try {
    mailInput.processRow( meta, data );
  } catch ( KettleException e ) {
    // don't worry about it
  }
  MessageParser underTest = mailInput.new MessageParser();
  Object[] r = RowDataUtil.allocateRowData( data.nrFields );
  underTest.parseToArray( r, message );

  Assert.assertEquals( "Header is correct", "", String.class.cast( r[0] ) );
}
 

public static final List<Object[]> getRowData( List<UnitTestResult> results ) {
  List<Object[]> rows = new ArrayList<Object[]>();
  RowMetaInterface rowMeta = getRowMeta();
  for ( UnitTestResult result : results ) {
    int index = 0;
    Object[] row = RowDataUtil.allocateRowData( rowMeta.size() );
    row[ index++ ] = result.getTransformationName();
    row[ index++ ] = result.getUnitTestName();
    row[ index++ ] = result.getDataSetName();
    row[ index++ ] = result.getStepName();
    row[ index++ ] = result.isError();
    row[ index++ ] = result.getComment();
    rows.add( row );
  }
  return rows;
}
 

private Object[] getRowForData( TableItem item, int rowNr ) throws KettleException {
  try {
    Object[] row = RowDataUtil.allocateRowData( rowMeta.size() );
    for ( int i = 0; i < rowMeta.size(); i++ ) {
      ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
      ValueMetaInterface stringValueMeta = stringRowMeta.getValueMeta( i );

      int colnr = i + 1;
      if ( GUIResource.getInstance().getColorBlue().equals( item.getForeground( colnr ) ) ) {
        row[ i ] = null; // <null> value
      } else {
        String string = item.getText( colnr );
        row[ i ] = valueMeta.convertDataFromString( string, stringValueMeta,
          null, null, ValueMetaInterface.TRIM_TYPE_NONE );
      }
    }
    return row;
  } catch ( KettleException e ) {
    throw new KettleException( BaseMessages.getString( PKG, "EditRowsDialog.Error.ErrorGettingRowForData",
      Integer.toString( rowNr ) ), e );
  }
}
 

private Object[] buildResult( Object[] r ) throws KettleValueException {
  Object[] result = null;
  if ( r != null || meta.isAlwaysGivingBackOneRow() ) {
    result = RowDataUtil.allocateRowData( data.groupnrs.length );
    if ( r != null ) {
      for ( int i = 0; i < data.groupnrs.length; i++ ) {
        result[ i ] = r[ data.groupnrs[ i ] ];
      }
    }

    result = RowDataUtil.addRowData( result, data.groupnrs.length, getAggregateResult() );
  }

  return result;
}
 

/**
 * Get a row from the resultset.
 *
 * @param rs The resultset to get the row from
 * @return one row or null if no row was found on the resultset or if an error occurred.
 */
public Object[] getRow( ResultSet rs, ResultSetMetaData dummy, RowMetaInterface rowInfo )
  throws KettleDatabaseException {
  long startTime = System.currentTimeMillis();

  try {

    int nrcols = rowInfo.size();
    Object[] data = RowDataUtil.allocateRowData( nrcols );

    if ( rs.next() ) {
      for ( int i = 0; i < nrcols; i++ ) {
        ValueMetaInterface val = rowInfo.getValueMeta( i );

        data[ i ] = databaseMeta.getValueFromResultSet( rs, val, i );
      }
    } else {
      data = null;
    }

    return data;
  } catch ( Exception ex ) {
    throw new KettleDatabaseException( "Couldn't get row from result set", ex );
  } finally {
    if ( log.isGatheringMetrics() ) {
      long time = System.currentTimeMillis() - startTime;
      log.snap( Metrics.METRIC_DATABASE_GET_ROW_SUM_TIME, databaseMeta.getName(), time );
      log.snap( Metrics.METRIC_DATABASE_GET_ROW_MIN_TIME, databaseMeta.getName(), time );
      log.snap( Metrics.METRIC_DATABASE_GET_ROW_MAX_TIME, databaseMeta.getName(), time );
      log.snap( Metrics.METRIC_DATABASE_GET_ROW_COUNT, databaseMeta.getName() );
    }
  }
}
 

public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException {
  Object[] rowData;

  if ( data.readsRows ) {
    rowData = getRow();
    if ( rowData == null ) {
      setOutputDone();
      return false;
    }
  } else {
    rowData = RowDataUtil.allocateRowData( 0 );
    incrementLinesRead();
  }

  // initialize
  if ( first && rowData != null ) {
    first = false;

    // Make output meta data
    //
    if ( data.readsRows ) {
      data.inputRowMeta = getInputRowMeta();
    } else {
      data.inputRowMeta = new RowMeta();
    }
    data.outputRowMeta = data.inputRowMeta.clone();
    meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore );

    // Create a copy of the output row metadata to do the data conversion...
    //
    data.conversionMeta = data.outputRowMeta.cloneToType( ValueMetaInterface.TYPE_STRING );

    // Add the variables to the row...
    //
    // Keep the Object[] for speed. Although this step will always be used in "small" amounts, there's always going to
    // be those cases where performance is required.
    //
    int fieldsLength = meta.getFieldDefinitions().length;
    data.extraData = new Object[fieldsLength];
    for ( int i = 0; i < fieldsLength; i++ ) {
      String newValue = environmentSubstitute( meta.getFieldDefinitions()[i].getVariableString() );
      if ( log.isDetailed() ) {
        logDetailed( "field [" + meta.getFieldDefinitions()[i].getFieldName() + "] has value [" + newValue + "]" );
      }

      // Convert the data to the desired data type...
      //
      ValueMetaInterface targetMeta = data.outputRowMeta.getValueMeta( data.inputRowMeta.size() + i );
      ValueMetaInterface sourceMeta = data.conversionMeta.getValueMeta( data.inputRowMeta.size() + i ); // String type
                                                                                                        // +
                                                                                                        // conversion
                                                                                                        // masks,
                                                                                                        // symbols,
                                                                                                        // trim type,
                                                                                                        // etc
      data.extraData[i] = targetMeta.convertData( sourceMeta, newValue );
    }
  }

  rowData = RowDataUtil.addRowData( rowData, data.inputRowMeta.size(), data.extraData );

  putRow( data.outputRowMeta, rowData );

  if ( !data.readsRows ) { // Just one row and then stop!

    setOutputDone();
    return false;
  }

  return true;
}
 

public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  if ( databaseMeta == null ) {
    return; // TODO: throw an exception here
  }

  if ( cachedRowMetaActive ) {
    row.addRowMeta( cachedRowMeta );
    return;
  }

  boolean param = false;

  Database db = getDatabase();
  super.databases = new Database[] { db }; // keep track of it for canceling purposes...

  // First try without connecting to the database... (can be S L O W)
  String sNewSQL = sql;
  if ( isVariableReplacementActive() ) {
    sNewSQL = db.environmentSubstitute( sql );
    if ( space != null ) {
      sNewSQL = space.environmentSubstitute( sNewSQL );
    }
  }

  RowMetaInterface add = null;
  try {
    add = db.getQueryFields( sNewSQL, param );
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleStepException( "Unable to get queryfields for SQL: " + Const.CR + sNewSQL, dbe );
  }

  if ( add != null ) {
    attachOrigin( add, origin );
    row.addRowMeta( add );
  } else {
    try {
      db.connect();

      RowMetaInterface paramRowMeta = null;
      Object[] paramData = null;

      StreamInterface infoStream = getStepIOMeta().getInfoStreams().get( 0 );
      if ( !Utils.isEmpty( infoStream.getStepname() ) ) {
        param = true;
        if ( info.length > 0 && info[ 0 ] != null ) {
          paramRowMeta = info[ 0 ];
          paramData = RowDataUtil.allocateRowData( paramRowMeta.size() );
        }
      }

      add = db.getQueryFields( sNewSQL, param, paramRowMeta, paramData );

      if ( add == null ) {
        return;
      }
      attachOrigin( add, origin );
      row.addRowMeta( add );
    } catch ( KettleException ke ) {
      throw new KettleStepException( "Unable to get queryfields for SQL: " + Const.CR + sNewSQL, ke );
    } finally {
      db.disconnect();
    }
  }
  if ( isLazyConversionActive() ) {
    for ( int i = 0; i < row.size(); i++ ) {
      ValueMetaInterface v = row.getValueMeta( i );
      try {
        if ( v.getType() == ValueMetaInterface.TYPE_STRING ) {
          ValueMetaInterface storageMeta = ValueMetaFactory.cloneValueMeta( v );
          storageMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
          v.setStorageMetadata( storageMeta );
          v.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING );
        }
      } catch ( KettlePluginException e ) {
        throw new KettleStepException( "Unable to clone meta for lazy conversion: " + Const.CR + v, e );
      }
    }
  }
}
 

@Override public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException {
  try {
    if ( meta.getExecuteForEachIncomingRow() && m_currentInputRowDrivingQuery == null ) {
      m_currentInputRowDrivingQuery = getRow();

      if ( m_currentInputRowDrivingQuery == null ) {
        // no more input, no more queries to make
        setOutputDone();
        return false;
      }

      if ( !first ) {
        initQuery();
      }
    }

    if ( first ) {
      data.outputRowMeta = new RowMeta();
      meta.getFields( data.outputRowMeta, getStepname(), null, null, MongoDbInput.this );

      initQuery();
      first = false;

      data.init();
    }

    boolean
        hasNext =
        ( ( meta.getQueryIsPipeline() ? data.m_pipelineResult.hasNext() : data.cursor.hasNext() ) && !isStopped() );
    if ( hasNext ) {
      DBObject nextDoc = null;
      Object[] row = null;
      if ( meta.getQueryIsPipeline() ) {
        nextDoc = data.m_pipelineResult.next();
      } else {
        nextDoc = data.cursor.next();
      }

      if ( !meta.getQueryIsPipeline() && !m_serverDetermined ) {
        ServerAddress s = data.cursor.getServerAddress();
        if ( s != null ) {
          m_serverDetermined = true;
          logBasic(
              BaseMessages.getString( PKG, "MongoDbInput.Message.QueryPulledDataFrom", s.toString() ) ); //$NON-NLS-1$
        }
      }

      if ( meta.getOutputJson() || meta.getMongoFields() == null || meta.getMongoFields().size() == 0 ) {
        String json = JSON.serialize( nextDoc );
        row = RowDataUtil.allocateRowData( data.outputRowMeta.size() );
        int index = 0;

        row[index++] = json;
        putRow( data.outputRowMeta, row );
      } else {
        Object[][] outputRows = data.mongoDocumentToKettle( nextDoc, MongoDbInput.this );

        // there may be more than one row if the paths contain an array
        // unwind
        for ( int i = 0; i < outputRows.length; i++ ) {
          putRow( data.outputRowMeta, outputRows[i] );
        }
      }
    } else {
      if ( !meta.getExecuteForEachIncomingRow() ) {
        setOutputDone();

        return false;
      } else {
        m_currentInputRowDrivingQuery = null; // finished with this row
      }
    }

    return true;
  } catch ( Exception e ) {
    if ( e instanceof KettleException ) {
      throw (KettleException) e;
    } else {
      throw new KettleException( e ); //$NON-NLS-1$
    }
  }
}
 

private void injectDataSetIntoStep( final Trans trans, final String dataSetName,
                                    final MetaStoreFactory<DataSet> dataSetFactory, final StepMeta stepMeta,
                                    TransUnitTestSetLocation inputLocation ) throws MetaStoreException, KettleException {

  final DataSet dataSet = dataSetFactory.loadElement( dataSetName );
  final LogChannelInterface log = trans.getLogChannel();

  final RowProducer rowProducer = trans.addRowProducer( stepMeta.getName(), 0 );

  // Look for the step into which we'll inject rows...
  //
  StepMetaDataCombi combi = null;
  for ( StepMetaDataCombi step : trans.getSteps() ) {
    if ( step.stepname.equals( stepMeta.getName() ) ) {
      combi = step;
      break;
    }
  }

  if ( combi != null ) {

    // Get the rows of the mapped values in the mapped order sorted as asked
    //
    final List<Object[]> dataSetRows = dataSet.getAllRows( log, inputLocation );
    RowMetaInterface dataSetRowMeta = dataSet.getMappedDataSetFieldsRowMeta( inputLocation );

    // The rows to inject are always driven by the dataset, NOT the step it replaces (!) for simplicity
    //
    RowMetaInterface injectRowMeta = new RowMeta();

    // Figure out which fields to pass
    // Only inject those mentioned in the field mappings...
    //
    int[] fieldIndexes = new int[ inputLocation.getFieldMappings().size() ];
    for ( int i = 0; i < inputLocation.getFieldMappings().size(); i++ ) {
      TransUnitTestFieldMapping fieldMapping = inputLocation.getFieldMappings().get( i );
      fieldIndexes[ i ] = dataSetRowMeta.indexOfValue( fieldMapping.getDataSetFieldName() );
      if ( fieldIndexes[ i ] < 0 ) {
        throw new KettleException( "Unable to find mapped field '" + fieldMapping.getDataSetFieldName() + "' in data set '" + dataSet.getName() + "'" );
      }
      ValueMetaInterface injectValueMeta = dataSetRowMeta.getValueMeta( fieldIndexes[ i ] ).clone();
      // Rename to the step output names though...
      //
      injectValueMeta.setName( fieldMapping.getStepFieldName() );
      injectRowMeta.addValueMeta( injectValueMeta );
    }

    log.logDetailed( "Injecting data set '" + dataSetName + "' into step '" + stepMeta.getName() + "', fields: " + Arrays.toString( injectRowMeta.getFieldNames() ) );

    // Pass rows
    //
    Runnable runnable = new Runnable() {
      @Override
      public void run() {
        try {

          for ( Object[] dataSetRow : dataSetRows ) {
            // pass the row with the external names, in the right order and with the selected columns from the data set
            //
            Object[] row = RowDataUtil.allocateRowData( injectRowMeta.size() );
            for ( int i = 0; i < fieldIndexes.length; i++ ) {
              row[ i ] = dataSetRow[ fieldIndexes[ i ] ];
            }
            rowProducer.putRow( injectRowMeta, row );
          }
          rowProducer.finished();

        } catch ( Exception e ) {
          throw new RuntimeException( "Problem injecting data set '" + dataSetName + "' row into step '" + stepMeta.getName() + "'", e );
        }
      }
    };
    Thread thread = new Thread( runnable );
    thread.start();


  }
}
 

private Object[] createNewRow( Object[] inputRowData ) {
  return inputRowData == null ? RowDataUtil.allocateRowData( data.outputRowMeta.size() ) : RowDataUtil
    .createResizedCopy( inputRowData, data.outputRowMeta.size() );
}
 

/**
 * Used for junits in DenormaliserAggregationsTest
 *
 * @param rowMeta
 * @param rowData
 * @return
 * @throws KettleValueException
 */
Object[] buildResult( RowMetaInterface rowMeta, Object[] rowData ) throws KettleValueException {
  // Deleting objects: we need to create a new object array
  // It's useless to call RowDataUtil.resizeArray
  //
  Object[] outputRowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );
  int outputIndex = 0;

  // Copy the data from the incoming row, but remove the unwanted fields in the same loop...
  //
  int removeIndex = 0;
  for ( int i = 0; i < rowMeta.size(); i++ ) {
    if ( removeIndex < data.removeNrs.length && i == data.removeNrs[removeIndex] ) {
      removeIndex++;
    } else {
      outputRowData[outputIndex++] = rowData[i];
    }
  }

  // Add the unpivoted fields...
  //
  for ( int i = 0; i < data.targetResult.length; i++ ) {
    Object resultValue = data.targetResult[i];
    DenormaliserTargetField field = meta.getDenormaliserTargetField()[i];
    switch ( field.getTargetAggregationType() ) {
      case DenormaliserTargetField.TYPE_AGGR_AVERAGE:
        long count = data.counters[i];
        Object sum = data.sum[i];
        if ( count > 0 ) {
          if ( sum instanceof Long ) {
            resultValue = (Long) sum / count;
          } else if ( sum instanceof Double ) {
            resultValue = (Double) sum / count;
          } else if ( sum instanceof BigDecimal ) {
            resultValue = ( (BigDecimal) sum ).divide( new BigDecimal( count ) );
          } else {
            resultValue = null; // TODO: perhaps throw an exception here?<
          }
        }
        break;
      case DenormaliserTargetField.TYPE_AGGR_COUNT_ALL:
        if ( resultValue == null ) {
          resultValue = Long.valueOf( 0 );
        }
        if ( field.getTargetType() != ValueMetaInterface.TYPE_INTEGER ) {
          resultValue =
              data.outputRowMeta.getValueMeta( outputIndex ).convertData(
                  new ValueMetaInteger( "num_values_aggregation" ), resultValue );
        }
        break;
      default:
        break;
    }
    if ( resultValue == null && allNullsAreZero ) {
      // PDI-9662 seems all rows for min function was nulls...
      resultValue = getZero( outputIndex );
    }
    outputRowData[outputIndex++] = resultValue;
  }

  return outputRowData;
}
 

/**
 * Build an empty row based on the meta-data.
 *
 * @return
 */
private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}
 

/**
 * Build an empty row based on the meta-data...
 *
 * @return
 */

private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}
 

/**
 * Build an empty row based on the meta-data...
 *
 * @return
 */

private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}
 

/**
 * Build an empty row based on the meta-data...
 *
 * @return
 */

private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}
 

/**
 * Build an empty row based on the meta-data...
 *
 * @return
 */

private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}
 

/**
 * Build an empty row based on the meta-data...
 *
 * @return
 */

private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}
 

/**
 * Build an empty row based on the meta-data...
 *
 * @return
 */

private Object[] buildEmptyRow() {
  Object[] rowData = RowDataUtil.allocateRowData( data.outputRowMeta.size() );

  return rowData;
}