Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#getValueMeta()

/**
 * Create a new field mapping between source and target steps.
 *
 * @param shell
 *          the shell of the parent window
 * @param sourceFields
 *          the source fields
 * @param targetFields
 *          the target fields
 * @param fieldMapping
 *          the list of source to target mappings to default to (can be empty but not null)
 * @throws KettleException
 *           in case something goes wrong during the field mapping
 */
public static final void generateFieldMapping( Shell shell, RowMetaInterface sourceFields,
  RowMetaInterface targetFields, java.util.List<SourceToTargetMapping> fieldMapping ) throws KettleException {
  // Build the mapping: let the user decide!!
  String[] source = sourceFields.getFieldNames();
  for ( int i = 0; i < source.length; i++ ) {
    ValueMetaInterface v = sourceFields.getValueMeta( i );
    source[i] += EnterMappingDialog.STRING_ORIGIN_SEPARATOR + v.getOrigin() + ")";
  }
  String[] target = targetFields.getFieldNames();

  EnterMappingDialog dialog = new EnterMappingDialog( shell, source, target, fieldMapping );
  java.util.List<SourceToTargetMapping> newMapping = dialog.open();
  if ( newMapping != null ) {
    fieldMapping.clear();
    fieldMapping.addAll( newMapping );
  }
}
 

/**
 * Create a new field mapping between source and target steps.
 *
 * @param shell        the shell of the parent window
 * @param sourceFields the source fields
 * @param targetFields the target fields
 * @param fieldMapping the list of source to target mappings to default to (can be empty but not null)
 * @throws KettleException in case something goes wrong during the field mapping
 */
public static final void generateFieldMapping( Shell shell, RowMetaInterface sourceFields,
                                               RowMetaInterface targetFields,
                                               List<SourceToTargetMapping> fieldMapping ) throws KettleException {
  // Build the mapping: let the user decide!!
  String[] source = sourceFields.getFieldNames();
  for ( int i = 0; i < source.length; i++ ) {
    ValueMetaInterface v = sourceFields.getValueMeta( i );
    source[ i ] += EnterMappingDialog.STRING_ORIGIN_SEPARATOR + v.getOrigin() + ")";
  }
  String[] target = targetFields.getFieldNames();

  EnterMappingDialog dialog = new EnterMappingDialog( shell, source, target, fieldMapping );
  List<SourceToTargetMapping> newMapping = dialog.open();
  if ( newMapping != null ) {
    fieldMapping.clear();
    fieldMapping.addAll( newMapping );
  }
}
 

public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  if ( databaseMeta == null ) {
    return; // TODO: throw an exception here
  }

  RowMetaInterface add = null;

  try {
    String mdx = getSQL();
    if ( isVariableReplacementActive() ) {
      mdx = space.environmentSubstitute( mdx );
    }
    MondrianHelper helper = new MondrianHelper( databaseMeta, catalog, mdx, space );
    add = helper.getCachedRowMeta();
    if ( add == null ) {
      helper.openQuery();
      helper.createRectangularOutput();
      add = helper.getOutputRowMeta();
    }
  } catch ( KettleDatabaseException dbe ) {
    throw new KettleStepException( "Unable to get query result for MDX query: " + Const.CR + sql, dbe );
  }

  // Set the origin
  //
  for ( int i = 0; i < add.size(); i++ ) {
    ValueMetaInterface v = add.getValueMeta( i );
    v.setOrigin( origin );
  }

  row.addRowMeta( add );
}
 

@Test
public void testRowsTransform() {

  String[] keyLookup = new String[] { "Name" };
  String[] keyStream = new String[] { "FirstName" };
  String[] updateLookup = new String[] { "SecondName", "PostAddress", "ZIP" };
  String[] updateStream = new String[] { "SurName", "Address", "ZIP" };

  RowMetaInterface prev = new RowMeta();
  prev.addValueMeta( new ValueMetaString( keyStream[0] ) );
  prev.addValueMeta( new ValueMetaString( updateStream[0] ) );
  prev.addValueMeta( new ValueMetaString( updateStream[1] ) );
  prev.addValueMeta( new ValueMetaString( updateStream[2] ) );

  try {
    RowMetaInterface result =
        RowMetaUtils.getRowMetaForUpdate( prev, keyLookup, keyStream, updateLookup, updateStream );

    ValueMetaInterface vmi = result.getValueMeta( 0 );
    assertEquals( vmi.getName(), keyLookup[0] );
    assertEquals( prev.getValueMeta( 0 ).getName(), keyStream[0] );

    assertEquals( result.getValueMeta( 1 ).getName(), updateLookup[0] );
    assertEquals( prev.getValueMeta( 1 ).getName(), updateStream[0] );

    assertEquals( result.getValueMeta( 2 ).getName(), updateLookup[1] );
    assertEquals( prev.getValueMeta( 2 ).getName(), updateStream[1] );

    assertEquals( result.getValueMeta( 3 ).getName(), updateStream[2] );
    assertEquals( prev.getValueMeta( 3 ).getName(), updateStream[2] );
  } catch ( Exception ex ) {
    ex.printStackTrace();
    fail();
  }
}
 

public int getPartition( RowMetaInterface rowMeta, Object[] row ) throws KettleException {
  init( rowMeta );

  if ( partitionColumnIndex < 0 ) {
    partitionColumnIndex = rowMeta.indexOfValue( fieldName );
    if ( partitionColumnIndex < 0 ) {
      throw new KettleStepException( "Unable to find partitioning field name ["
        + fieldName + "] in the output row..." + rowMeta );
    }
  }

  long value;

  ValueMetaInterface valueMeta = rowMeta.getValueMeta( partitionColumnIndex );
  Object valueData = row[partitionColumnIndex];

  switch ( valueMeta.getType() ) {
    case ValueMetaInterface.TYPE_INTEGER:
      Long longValue = rowMeta.getInteger( row, partitionColumnIndex );
      if ( longValue == null ) {
        value = valueMeta.hashCode( valueData );
      } else {
        value = longValue.longValue();
      }
      break;
    default:
      value = valueMeta.hashCode( valueData );
  }

  /*
   * value = rowMeta.getInteger(row, partitionColumnIndex);
   */

  int targetLocation = (int) ( Math.abs( value ) % nrPartitions );

  return targetLocation;
}
 

private void initGroupMeta( RowMetaInterface previousRowMeta ) throws KettleValueException {
  data.groupMeta = new RowMeta();
  data.entryMeta = new RowMeta();

  for ( int i = 0; i < data.groupnrs.length; i++ ) {
    ValueMetaInterface valueMeta = previousRowMeta.getValueMeta( data.groupnrs[i] );
    data.groupMeta.addValueMeta( valueMeta );

    ValueMetaInterface normalMeta = valueMeta.clone();
    normalMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
  }

  return;
}
 

protected void setupFieldsCombo() {
  // try and set up from incoming fields from previous step

  StepMeta stepMeta = transMeta.findStep(stepname);

  if (stepMeta != null) {
    try {
      RowMetaInterface row = transMeta.getPrevStepFields(stepMeta);

      if (row.size() == 0) {
        MessageDialog.openError(shell, BaseMessages.getString(PKG,
            "SSTableOutputData.Message.NoIncomingFields.Title"), BaseMessages
            .getString(PKG, "SSTableOutputData.Message.NoIncomingFields"));

        return;
      }

      m_keyFieldCombo.removeAll();
      for (int i = 0; i < row.size(); i++) {
        ValueMetaInterface vm = row.getValueMeta(i);
        m_keyFieldCombo.add(vm.getName());
      }
    } catch (KettleException ex) {
      MessageDialog.openError(shell, BaseMessages.getString(PKG,
          "SSTableOutputData.Message.NoIncomingFields.Title"), BaseMessages
          .getString(PKG, "SSTableOutputData.Message.NoIncomingFields"));
    }
  }
}
 

/**
 * Checks the fields specified for reserved words
 *
 * @param fields
 *          the list of fields to check
 * @return The nr of reserved words for this database.
 */
public int getNrReservedWords( RowMetaInterface fields ) {
  int nrReservedWords = 0;
  for ( int i = 0; i < fields.size(); i++ ) {
    ValueMetaInterface v = fields.getValueMeta( i );
    if ( isReservedWord( v.getName() ) ) {
      nrReservedWords++;
    }
  }
  return nrReservedWords;
}
 

private void setParameters( RowMetaInterface rowMeta, Object[] row, Map<String, Object> parameters, CypherParameters cypherParameters ) throws KettleValueException {
  for ( TargetParameter targetParameter : cypherParameters.getTargetParameters() ) {
    int fieldIndex = targetParameter.getInputFieldIndex();
    ValueMetaInterface valueMeta = rowMeta.getValueMeta( fieldIndex );
    Object valueData = row[ fieldIndex ];
    String parameterName = targetParameter.getParameterName();
    GraphPropertyType parameterType = targetParameter.getParameterType();

    // Convert to the neo type
    //
    Object neoObject = parameterType.convertFromKettle( valueMeta, valueData );

    parameters.put( parameterName, neoObject );
  }
}
 

public void writeLine( RowMetaInterface mi, Object[] row ) throws KettleException {
  if ( first ) {
    first = false;

    enclosure = meta.getEnclosure();

    // Setup up the fields we need to take for each of the rows
    // as this speeds up processing.
    fieldNumbers = new int[meta.getFieldStream().length];
    for ( int i = 0; i < fieldNumbers.length; i++ ) {
      fieldNumbers[i] = mi.indexOfValue( meta.getFieldStream()[i] );
      if ( fieldNumbers[i] < 0 ) {
        throw new KettleException( "Could not find field " + meta.getFieldStream()[i] + " in stream" );
      }
    }

    sdfDate = new SimpleDateFormat( "yyyy-MM-dd" );
    sdfDateTime = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss.SSS" );
  }

  // Write the data to the output
  ValueMetaInterface v = null;
  int number = 0;
  for ( int i = 0; i < fieldNumbers.length; i++ ) {
    if ( i != 0 ) {
      output.print( "," );
    }
    number = fieldNumbers[i];
    v = mi.getValueMeta( number );
    if ( row[number] == null ) {
      // TODO (SB): special check for null in case of Strings.
      output.print( enclosure );
      output.print( enclosure );
    } else {
      switch ( v.getType() ) {
        case ValueMetaInterface.TYPE_STRING:
          String s = mi.getString( row, number );
          if ( s.indexOf( enclosure ) >= 0 ) {
            s = createEscapedString( s, enclosure );
          }
          output.print( enclosure );
          output.print( s );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          Long l = mi.getInteger( row, number );
          output.print( enclosure );
          output.print( l );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          Double d = mi.getNumber( row, number );
          output.print( enclosure );
          output.print( d );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BIGNUMBER:
          BigDecimal bd = mi.getBigNumber( row, number );
          output.print( enclosure );
          output.print( bd );
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_DATE:
          Date dt = mi.getDate( row, number );
          output.print( enclosure );
          String mask = meta.getDateMask()[i];
          if ( GPBulkLoaderMeta.DATE_MASK_DATETIME.equals( mask ) ) {
            output.print( sdfDateTime.format( dt ) );
          } else {
            // Default is date format
            output.print( sdfDate.format( dt ) );
          }
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          Boolean b = mi.getBoolean( row, number );
          output.print( enclosure );
          if ( b.booleanValue() ) {
            output.print( "Y" );
          } else {
            output.print( "N" );
          }
          output.print( enclosure );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          byte[] byt = mi.getBinary( row, number );
          output.print( "<startlob>" );
          output.print( byt );
          output.print( "<endlob>" );
          break;
        default:
          throw new KettleException( "Unsupported type" );
      }
    }
  }
  output.print( Const.CR );
}
 

private String generateMatchClause( String alias, String mapName, List<String> nodeLabels, String[] nodeProps, String[] nodePropNames,
                                    GraphPropertyType[] nodePropTypes,
                                    boolean[] nodePropPrimary,
                                    RowMetaInterface rowMeta, Object[] rowData, int[] nodePropIndexes,
                                    Map<String, Object> parameters, AtomicInteger paramNr ) throws KettleValueException {
  String matchClause = "(" + alias;
  for ( int i = 0; i < nodeLabels.size(); i++ ) {
    String label = escapeProp( nodeLabels.get( i ) );
    matchClause += ":" + label;
  }
  matchClause += " {";

  boolean firstProperty = true;
  for ( int i = 0; i < nodeProps.length; i++ ) {
    if ( nodePropPrimary[ i ] ) {
      if ( firstProperty ) {
        firstProperty = false;
      } else {
        matchClause += ", ";
      }
      String propName;
      if ( StringUtils.isNotEmpty( nodePropNames[ i ] ) ) {
        propName = nodePropNames[ i ];
      } else {
        propName = nodeProps[ i ];
      }
      String parameterName = "param" + paramNr.incrementAndGet();

      if ( mapName == null ) {
        matchClause += propName + " : " + buildParameterClause( parameterName );
      } else {
        matchClause += propName + " : " + mapName + "." + parameterName;
      }

      if ( parameters != null ) {
        ValueMetaInterface valueMeta = rowMeta.getValueMeta( nodePropIndexes[ i ] );
        Object valueData = rowData[ nodePropIndexes[ i ] ];

        GraphPropertyType propertyType = nodePropTypes[ i ];
        Object neoValue = propertyType.convertFromKettle( valueMeta, valueData );

        parameters.put( parameterName, neoValue );
      }
    }
  }
  matchClause += " })";

  return matchClause;
}
 

/**
 * Ask which data set to write to
 * Ask for the mapping between the output row and the data set field
 * Start the transformation and capture the output of the step, write to the database table backing the data set.
 */
public void writeStepDataToDataSet() {
  Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );
  TransGraph transGraph = spoon.getActiveTransGraph();
  IMetaStore metaStore = spoon.getMetaStore();
  if ( transGraph == null ) {
    return;
  }
  StepMeta stepMeta = transGraph.getCurrentStep();
  TransMeta transMeta = spoon.getActiveTransformation();
  if ( stepMeta == null || transMeta == null ) {
    return;
  }

  if ( transMeta.hasChanged() ) {
    MessageBox box = new MessageBox( spoon.getShell(), SWT.OK | SWT.ICON_INFORMATION );
    box.setText( "Save transformation" );
    box.setMessage( "Please save your transformation first." );
    box.open();
    return;
  }

  try {

    FactoriesHierarchy hierarchy = getHierarchy();

    MetaStoreFactory<DataSetGroup> groupFactory = hierarchy.getGroupFactory();
    List<DatabaseMeta> databases = getAvailableDatabases( spoon.getRepository() );
    groupFactory.addNameList( DataSetConst.DATABASE_LIST_KEY, databases );
    List<DataSetGroup> groups = groupFactory.getElements();

    MetaStoreFactory<DataSet> setFactory = hierarchy.getSetFactory();
    setFactory.addNameList( DataSetConst.GROUP_LIST_KEY, groups );

    // Ask which data set to write to
    //
    List<String> setNames = setFactory.getElementNames();
    Collections.sort( setNames );
    EnterSelectionDialog esd = new EnterSelectionDialog( spoon.getShell(), setNames.toArray( new String[ setNames.size() ] ), "Select the set", "Select the data set to edit..." );
    String setName = esd.open();
    if ( setName == null ) {
      return;
    }

    DataSet dataSet = setFactory.loadElement( setName );
    String[] setFields = new String[ dataSet.getFields().size() ];
    for ( int i = 0; i < setFields.length; i++ ) {
      setFields[ i ] = dataSet.getFields().get( i ).getFieldName();
    }

    RowMetaInterface rowMeta = transMeta.getStepFields( stepMeta );
    String[] stepFields = new String[ rowMeta.size() ];
    for ( int i = 0; i < rowMeta.size(); i++ ) {
      ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
      stepFields[ i ] = valueMeta.getName();
    }

    // Ask for the mapping between the output row and the data set field
    //
    EnterMappingDialog mappingDialog = new EnterMappingDialog( spoon.getShell(), stepFields, setFields );
    List<SourceToTargetMapping> mapping = mappingDialog.open();
    if ( mapping == null ) {
      return;
    }

    // Run the transformation.  We want to use the standard Spoon runFile() method
    // So we need to leave the source to target mapping list somewhere so it can be picked up later.
    // For now we'll leave it where we need it.
    //
    WriteToDataSetExtensionPoint.stepsMap.put( transMeta.getName(), stepMeta );
    WriteToDataSetExtensionPoint.mappingsMap.put( transMeta.getName(), mapping );
    WriteToDataSetExtensionPoint.setsMap.put( transMeta.getName(), dataSet );

    // Signal to the transformation xp plugin to inject data into some data set
    //
    transMeta.setVariable( DataSetConst.VAR_WRITE_TO_DATASET, "Y" );
    spoon.runFile();

  } catch ( Exception e ) {
    new ErrorDialog( spoon.getShell(), "Error", "Error creating a new data set", e );
  }
}
 

/**
 * This method is called when a row is written to another step (even if there is no next step)
 *
 * @param rowMeta the metadata of the row
 * @param row     the data of the row
 * @throws org.pentaho.di.core.exception.KettleStepException an exception that can be thrown to hard stop the step
 */
public void rowWrittenEvent( final RowMetaInterface rowMeta, final Object[] row ) throws KettleStepException {
  if ( firstCall ) {
    this.tableModel = createTableModel( rowMeta );
    firstCall = false;
  }

  if ( queryLimit > 0 && rowsWritten > queryLimit ) {
    return;
  }

  try {
    rowsWritten += 1;

    final int count = tableModel.getColumnCount();
    final Object dataRow[] = new Object[ count ];
    for ( int columnNo = 0; columnNo < count; columnNo++ ) {
      final ValueMetaInterface valueMeta = rowMeta.getValueMeta( columnNo );

      switch( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_BIGNUMBER:
          dataRow[ columnNo ] = rowMeta.getBigNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BOOLEAN:
          dataRow[ columnNo ] = rowMeta.getBoolean( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_DATE:
          dataRow[ columnNo ] = rowMeta.getDate( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_INTEGER:
          dataRow[ columnNo ] = rowMeta.getInteger( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NONE:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          dataRow[ columnNo ] = rowMeta.getNumber( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_STRING:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
          break;
        case ValueMetaInterface.TYPE_BINARY:
          dataRow[ columnNo ] = rowMeta.getBinary( row, columnNo );
          break;
        default:
          dataRow[ columnNo ] = rowMeta.getString( row, columnNo );
      }
    }
    tableModel.addRow( dataRow );
  } catch ( final KettleValueException kve ) {
    throw new KettleStepException( kve );
  } catch ( final Exception e ) {
    throw new KettleStepException( e );
  }
}
 

protected static List<Object> rowsToCSVNew( RowMetaInterface meta, List<Object[]> rows ) throws KettleException {
  List<Object> results = new ArrayList<>( 2 );
  boolean needsBase64;
  CharsetEncoder encoder = Charset.forName( "US-ASCII" ).newEncoder();
  Charset utf8 = Charset.forName( "UTF-8" );

  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  BufferedOutputStream buf = new BufferedOutputStream( bos );

  // header row
  StringBuilder builder = new StringBuilder();
  int i = 0;
  for ( ValueMetaInterface v : meta.getValueMetaList() ) {
    String name = quote( v.getName() );
    builder.append( i > 0 ? "," : "" ).append( name );
    i++;
  }
  builder.append( "\n" );

  // We look for non-ascii characters and, if found, encode to ascii base64 first. For some reason,
  // encoding directly to utf-8 on the Java side (Mac OS X; Java 8) causes the python utf-8 decoder to hang
  // when there are non-ascii characters (such as in Mayagüez) present. Encoding to base64 first, then decoding
  // this on the python side seems to fix the issue.
  needsBase64 = !encoder.canEncode( builder.toString() );
  try {
    buf.write( builder.toString().getBytes( utf8 ) );

    for ( Object[] row : rows ) {
      builder.setLength( 0 );
      for ( i = 0; i < meta.size(); i++ ) {
        String value;
        ValueMetaInterface vm = meta.getValueMeta( i );
        if ( row[i] == null || Const.isEmpty( vm.getString( row[i] ) ) ) {
          value = "?";
        } else {
          //switch ( meta.getValueMetaList().get( i ).getType() ) {
          switch ( vm.getType() ) {
            case ValueMetaInterface.TYPE_NUMBER:
            case ValueMetaInterface.TYPE_INTEGER:
            case ValueMetaInterface.TYPE_BIGNUMBER:
              value = vm.getString( row[i] );
              break;
            case ValueMetaInterface.TYPE_DATE:
              int offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_TIMESTAMP:
              offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_BOOLEAN:
              value = "" + ( vm.getBoolean( row[i] ) ? "1" : "0" );
              break;
            // TODO throw an exception for Serializable/Binary
            default:
              value = quote( vm.getString( row[i] ) );
          }
        }
        builder.append( i > 0 ? "," : "" ).append( value );
      }
      builder.append( "\n" );

      if ( !needsBase64 ) {
        needsBase64 = !encoder.canEncode( builder.toString() );
      }
      buf.write( builder.toString().getBytes( utf8 ) );
    }

    buf.flush();
    buf.close();
  } catch ( IOException e ) {
    throw new KettleException( e );
  }

  byte[] bytes = bos.toByteArray();
  if ( needsBase64 ) {
    bytes = Base64.encodeBase64( bytes );
  }
  results.add( bytes );
  results.add( needsBase64 );

  return results;
}
 

public void storeTable() {
  CwmTable table = cwm.getTable( TEST_TABLE_NAME );
  if ( table == null ) {
    System.out.println( "Table [" + TEST_TABLE_NAME + "] not found: creating..." ); //$NON-NLS-1$ //$NON-NLS-2$
  } else {
    System.out.println( "Table [" + TEST_TABLE_NAME + "] found: overwriting..." ); //$NON-NLS-1$ //$NON-NLS-2$
  }
  cwm.beginTransaction();

  RowMetaInterface fields = new RowMeta();
  ValueMetaInterface field1 = new ValueMeta( "field1", ValueMetaInterface.TYPE_STRING ); //$NON-NLS-1$
  field1.setLength( 35 );
  field1.setOrigin( "field1 description" ); //$NON-NLS-1$
  fields.addValueMeta( field1 );
  ValueMetaInterface field2 = new ValueMeta( "field2", ValueMetaInterface.TYPE_NUMBER ); //$NON-NLS-1$
  field2.setLength( 7, 2 );
  field2.setOrigin( "field2 description" ); //$NON-NLS-1$
  fields.addValueMeta( field2 );
  ValueMetaInterface field3 = new ValueMeta( "field3", ValueMetaInterface.TYPE_INTEGER );
  field3.setLength( 5 );
  field3.setOrigin( "field3 description" );
  fields.addValueMeta( field3 );
  ValueMetaInterface field4 = new ValueMeta( "field4", ValueMetaInterface.TYPE_DATE );
  field4.setOrigin( "field4 description" );
  fields.addValueMeta( field4 );
  ValueMetaInterface field5 = new ValueMeta( "field5", ValueMetaInterface.TYPE_BIGNUMBER );
  field5.setLength( 52, 16 );
  field5.setOrigin( "field5 description" );
  fields.addValueMeta( field5 );
  ValueMetaInterface field6 = new ValueMeta( "field6", ValueMetaInterface.TYPE_BOOLEAN );
  field6.setOrigin( "field6 description" );
  fields.addValueMeta( field6 );

  table = cwm.createTable( TEST_TABLE_NAME, fields );

  // Add descriptions to table and columns...

  CwmDescription description = cwm.createDescription( "This is a table description" ); //$NON-NLS-1$
  cwm.setDescription( table, description );
  @SuppressWarnings( "unchecked" )
  Collection<CwmColumn> collection = table.getOwnedElement();
  CwmColumn[] columns = (CwmColumn[]) collection.toArray( new CwmColumn[collection.size()] );

  for ( int i = 0; i < fields.size(); i++ ) {
    ValueMetaInterface field = fields.getValueMeta( i );
    CwmColumn column = columns[i];

    // Add a description to the column
    //
    description = cwm.createDescription( field.getOrigin() );
    cwm.setDescription( column, description );
  }

  // Try to create a package here...
  CwmPackage p = cwm.createPackage( DOMAIN + " package" ); //$NON-NLS-1$
  @SuppressWarnings( "unchecked" )
  Collection<CwmTable> ca = p.getImportedElement();
  ca.add( table );
  cwm.setDescription( p, cwm.createDescription( "This is a package description for [" + DOMAIN + "]" ) ); //$NON-NLS-1$ //$NON-NLS-2$

  cwm.endTransaction();

  System.out.println( "Finished writing to table [" + TEST_TABLE_NAME + "]." ); //$NON-NLS-1$ //$NON-NLS-2$
}
 

private static Object[] createIndexes( DatabaseLookupData stepData, RowMetaInterface keysMeta, Object[][] keys ) {
  final int rowsAmount = keys.length;
  final int[] conditions = stepData.conditions;

  // it makes sense to apply restrictions in the specific order, namely, to use those, that can filter more elements
  // Index.restrictionComparator() uses heuristic "restriction power" of each index
  PriorityQueue<Index> indexes = new PriorityQueue<>( conditions.length, Index.restrictionComparator() );
  List<int[]> otherConditions = new ArrayList<>();
  for ( int i = 0, len = conditions.length; i < len; i++ ) {
    int condition = conditions[ i ];
    Index index = null;
    switch ( condition ) {
      case DatabaseLookupMeta.CONDITION_EQ:
        index = new EqIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_NE:
        index = EqIndex.nonEqualityIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_LT:
        index = new LtIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_LE:
        index = GtIndex.lessOrEqualCache( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_GT:
        index = new GtIndex( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_GE:
        index = LtIndex.greaterOrEqualCache( i, keysMeta.getValueMeta( i ), rowsAmount );
        break;
      case DatabaseLookupMeta.CONDITION_IS_NULL:
        index = new IsNullIndex( i, keysMeta.getValueMeta( i ), rowsAmount, true );
        break;
      case DatabaseLookupMeta.CONDITION_IS_NOT_NULL:
        index = new IsNullIndex( i, keysMeta.getValueMeta( i ), rowsAmount, false );
        break;
    }
    if ( index == null ) {
      otherConditions.add( new int[] { i, condition } );
    } else {
      index.performIndexingOf( keys );
      indexes.add( index );
    }
  }

  return new Object[] {
    indexes.toArray( new Index[ indexes.size() ] ),
    otherConditions.toArray( new int[ otherConditions.size() ][] )
  };
}
 

private void get() {
  boolean finished = false;

  int elementsFound = 0;

  try {
    XMLInputMeta meta = new XMLInputMeta();
    getInfo( meta );

    // check if the path is given
    if ( !checkInputPositionsFilled( meta ) ) {
      return;
    }

    EnterNumberDialog dialog =
      new EnterNumberDialog(
        shell, 1000, "Number of elements to scan", "Enter the number of elements to scan (0=all)" );
    int maxElements = dialog.open();

    // OK, let's try to walk through the complete tree
    RowMetaInterface row = new RowMeta(); // no fields found...

    // Keep the list of positions
    List<XMLInputFieldPosition> path = new ArrayList<XMLInputFieldPosition>(); // ArrayList of XMLInputFieldPosition

    FileInputList inputList = meta.getFiles( transMeta );

    for ( int f = 0; f < inputList.getFiles().size() && !finished; f++ ) {
      // Open the file...
      Node rootNode =
        XMLHandler.loadXMLFile( inputList.getFile( f ), transMeta
          .environmentSubstitute( meta.getFileBaseURI() ), meta.isIgnoreEntities(), meta.isNamespaceAware() );

      // Position to the repeating item
      for ( int p = 0; rootNode != null && p < meta.getInputPosition().length - 1; p++ ) {
        rootNode = XMLHandler.getSubNode( rootNode, meta.getInputPosition()[p] );
      }

      if ( rootNode == null ) {
        // Specified node not found: return!
        return;
      }

      if ( meta.getInputPosition().length > 1 ) {
        // Count the number of rootnodes
        String itemElement = meta.getInputPosition()[meta.getInputPosition().length - 1];
        int nrItems = XMLHandler.countNodes( rootNode, itemElement );
        for ( int i = 0; i < nrItems && !finished; i++ ) {
          Node itemNode = XMLHandler.getSubNodeByNr( rootNode, itemElement, i, false );
          if ( i >= meta.getNrRowsToSkip() ) {
            getValues( itemNode, row, path, 0 );

            elementsFound++;
            if ( elementsFound >= maxElements && maxElements > 0 ) {
              finished = true;
            }
          }
        }
      } else {
        // Only search the root node
        //
        getValues( rootNode, row, path, 0 );

        elementsFound++;
        if ( elementsFound >= maxElements && maxElements > 0 ) {
          finished = true;
        }
      }
    }

    // System.out.println("Values found: "+row);

    // add the values to the grid...
    for ( int i = 0; i < row.size(); i++ ) {
      ValueMetaInterface v = row.getValueMeta( i );
      TableItem item = new TableItem( wFields.table, SWT.NONE );
      item.setText( 1, v.getName() );
      item.setText( 2, v.getTypeDesc() );
      item.setText( 11, v.getOrigin() );
    }

    wFields.removeEmptyRows();
    wFields.setRowNums();
    wFields.optWidth( true );
  } catch ( KettleException e ) {
    new ErrorDialog(
      shell, BaseMessages.getString( PKG, "XMLInputDialog.ErrorParsingData.DialogTitle" ), BaseMessages
        .getString( PKG, "XMLInputDialog.ErrorParsingData.DialogMessage" ), e );
  }
}
 

private synchronized void AddAggregate( RowMetaInterface rowMeta, Object[] r ) throws KettleValueException {
  for ( int i = 0; i < data.fieldnrs.length; i++ ) {
    ValueMetaInterface valueMeta = rowMeta.getValueMeta( data.fieldnrs[i] );
    Object valueData = r[data.fieldnrs[i]];

    if ( !valueMeta.isNull( valueData ) ) {
      data.counts[i]++; // only count non-zero values!
      switch ( meta.getAggregateType()[i] ) {
        case AggregateRowsMeta.TYPE_AGGREGATE_SUM:
        case AggregateRowsMeta.TYPE_AGGREGATE_AVERAGE:
          Double number = valueMeta.getNumber( valueData );
          if ( data.values[i] == null ) {
            data.values[i] = number;
          } else {
            data.values[i] = new Double( ( (Double) data.values[i] ).doubleValue() + number.doubleValue() );
          }

          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_MIN:
          if ( data.values[i] == null ) {
            data.values[i] = valueData;
          } else {
            if ( valueMeta.compare( data.values[i], valueData ) < 0 ) {
              data.values[i] = valueData;
            }
          }

          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_MAX:
          if ( data.values[i] == null ) {
            data.values[i] = valueData;
          } else {
            if ( valueMeta.compare( data.values[i], valueData ) > 0 ) {
              data.values[i] = valueData;
            }
          }

          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_NONE:
        case AggregateRowsMeta.TYPE_AGGREGATE_FIRST:
          if ( data.values[i] == null ) {
            data.values[i] = valueData;
          }
          break;
        case AggregateRowsMeta.TYPE_AGGREGATE_LAST:
          data.values[i] = valueData;
          break;
        default:
          break;
      }
    }

    switch ( meta.getAggregateType()[i] ) {
      case AggregateRowsMeta.TYPE_AGGREGATE_FIRST_NULL: // First value, EVEN if it's NULL:
        if ( data.values[i] == null ) {
          data.values[i] = valueData;
        }
        break;
      case AggregateRowsMeta.TYPE_AGGREGATE_LAST_NULL: // Last value, EVEN if it's NULL:
        data.values[i] = valueData;
        break;
      default:
        break;
    }

  }
}
 

public static void getFieldsFromPrevious( RowMetaInterface row, XulTree tableView, List<Object> fields,
  StepTableDataObject field, TableItemInsertXulListener listener ) {
  if ( row == null || row.size() == 0 ) {
    return; // nothing to do
  }

  // get a list of all the non-empty keys (names)
  //
  List<String> keys = new ArrayList<String>();
  for ( Object entry : fields ) {
    keys.add( ( (StepTableDataObject) entry ).getName() );
  }

  int choice = 0;

  if ( keys.size() > 0 ) {
    // Ask what we should do with the existing data in the step.
    //
    Shell shell = ( (TableViewer) tableView.getManagedObject() ).getTable().getShell();
    MessageDialog md =
      new MessageDialog( shell,
        BaseMessages.getString( PKG, "BaseStepDialog.GetFieldsChoice.Title" ), // "Warning!"
        null,
        BaseMessages.getString( PKG, "BaseStepDialog.GetFieldsChoice.Message", "" + keys.size(), "" + row.size() ),
        MessageDialog.WARNING, new String[] {
          BaseMessages.getString( PKG, "BaseStepDialog.AddNew" ),
          BaseMessages.getString( PKG, "BaseStepDialog.Add" ),
          BaseMessages.getString( PKG, "BaseStepDialog.ClearAndAdd" ),
          BaseMessages.getString( PKG, "BaseStepDialog.Cancel" ), }, 0 );
    MessageDialog.setDefaultImage( GUIResource.getInstance().getImageSpoon() );
    int idx = md.open();
    choice = idx & 0xFF;
  }

  if ( choice == 3 || choice == 255 ) {
    return; // Cancel clicked
  }

  if ( choice == 2 ) {
    fields.clear();
  }

  for ( int i = 0; i < row.size(); i++ ) {
    ValueMetaInterface v = row.getValueMeta( i );

    if ( choice == 0 ) { // hang on, see if it's not yet in the table view

      if ( keys.indexOf( v.getName() ) >= 0 ) {
        continue;
      }
    }

    if ( listener != null && !listener.tableItemInsertedFor( v ) ) {
      continue;
    }

    StepTableDataObject newField = field.createNew( v );
    fields.add( newField );
  }
}
 

public void getFields( RowMetaInterface row, String originStepname, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  try {
    for ( int i = 0; i < fieldname.length; i++ ) {
      if ( !Utils.isEmpty( fieldname[i] ) ) {
        int valueIndex = -1;
        ValueMetaInterface v;
        if ( replace[i] ) {
          valueIndex = row.indexOfValue( fieldname[i] );
          if ( valueIndex < 0 ) {
            // The field was not found using the "name" field
            if ( Utils.isEmpty( rename[i] ) ) {
              // There is no "rename" field to try; Therefore we cannot find the
              // field to replace
              throw new KettleStepException( BaseMessages.getString(
                PKG, "ScriptValuesMetaMod.Exception.FieldToReplaceNotFound", fieldname[i] ) );
            } else {
              // Lookup the field to replace using the "rename" field
              valueIndex = row.indexOfValue( rename[i] );
              if ( valueIndex < 0 ) {
                // The field was not found using the "rename" field"; Therefore
                // we cannot find the field to replace
                //
                throw new KettleStepException( BaseMessages.getString(
                  PKG, "ScriptValuesMetaMod.Exception.FieldToReplaceNotFound", rename[i] ) );
              }
            }
          }

          // Change the data type to match what's specified...
          //
          ValueMetaInterface source = row.getValueMeta( valueIndex );
          v = ValueMetaFactory.cloneValueMeta( source, type[i] );
          row.setValueMeta( valueIndex, v );
        } else {
          if ( !Utils.isEmpty( rename[i] ) ) {
            v = ValueMetaFactory.createValueMeta( rename[i], type[i] );
          } else {
            v = ValueMetaFactory.createValueMeta( fieldname[i], type[i] );
          }
        }
        v.setLength( length[i] );
        v.setPrecision( precision[i] );
        v.setOrigin( originStepname );
        if ( !replace[i] ) {
          row.addValueMeta( v );
        }
      }
    }
  } catch ( KettleException e ) {
    throw new KettleStepException( e );
  }
}