Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#getValueMetaList()

private static void setValueFormats( RowMetaInterface rowMeta ) {
  for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
    if ( StringUtils.isEmpty( valueMeta.getConversionMask() ) ) {
      switch ( valueMeta.getType() ) {
        case ValueMetaInterface.TYPE_INTEGER:
          valueMeta.setConversionMask( "0" );
          break;
        case ValueMetaInterface.TYPE_NUMBER:
          valueMeta.setConversionMask( "0.#" );
          break;
        case ValueMetaInterface.TYPE_DATE:
          valueMeta.setConversionMask( "yyyyMMdd-HHmmss.SSS" );
          break;
        default:
          break;
      }
    }
  }
}
 

/**
 * Given a fully defined output row metadata structure, determine which of the output fields are being copied from
 * the input fields and which must be the output of the script.
 *
 * @param fullOutputRowMeta    the fully defined output row metadata structure
 * @param scriptFields         row meta that will hold script only fields
 * @param inputPresentInOutput row meta that will hold input fields being copied
 * @param infos                the array of info row metas
 * @param stepName             the name of the step
 */
protected void determineInputFieldScriptFieldSplit( RowMetaInterface fullOutputRowMeta, RowMetaInterface scriptFields,
    RowMetaInterface inputPresentInOutput, RowMetaInterface[] infos, String stepName ) {

  scriptFields.clear();
  inputPresentInOutput.clear();
  RowMetaInterface consolidatedInputFields = new RowMeta();
  for ( RowMetaInterface r : infos ) {
    consolidatedInputFields.addRowMeta( r );
  }

  for ( ValueMetaInterface vm : fullOutputRowMeta.getValueMetaList() ) {
    int index = consolidatedInputFields.indexOfValue( vm.getName() );
    if ( index >= 0 ) {
      inputPresentInOutput.addValueMeta( vm );
    } else {
      // must be a script output (either a variable name field or data frame column name
      scriptFields.addValueMeta( vm );
    }
  }
}
 

public void initializeColumns( RowMetaInterface inputRowMeta ) {
  if ( inputRowMeta == null ) {
    BaseMessages.getString( PKG, "RulesData.InitializeColumns.InputRowMetaIsNull" );
    return;
  }

  // Create objects for insertion into the rules engine
  List<ValueMetaInterface> columns = inputRowMeta.getValueMetaList();

  // This array must 1-1 match the row[] feteched by getRow()
  columnList = new Column[columns.size()];

  for ( int i = 0; i < columns.size(); i++ ) {
    ValueMetaInterface column = columns.get( i );

    Column c = new Column( true );
    c.setName( column.getName() );
    c.setType( column.getTypeDesc() );
    c.setPayload( null );

    columnList[i] = c;
  }
}
 

public RowMetaInterface updateFields( RowMetaInterface prev, MonetDBBulkLoaderData data ) {
  // update the field table from the fields coming from the previous step
  RowMetaInterface tableFields = new RowMeta();
  List<ValueMetaInterface> fields = prev.getValueMetaList();
  fieldTable = new String[fields.size()];
  fieldStream = new String[fields.size()];
  fieldFormatOk = new boolean[fields.size()];
  int idx = 0;
  for ( ValueMetaInterface field : fields ) {
    ValueMetaInterface tableField = field.clone();
    tableFields.addValueMeta( tableField );
    fieldTable[idx] = field.getName();
    fieldStream[idx] = field.getName();
    fieldFormatOk[idx] = true;
    idx++;
  }

  data.keynrs = new int[getFieldStream().length];
  for ( int i = 0; i < data.keynrs.length; i++ ) {
    data.keynrs[i] = i;
  }
  return tableFields;
}
 

/**
 * putRow is used to copy a row, to the alternate rowset(s) This should get priority over everything else!
 * (synchronized) If distribute is true, a row is copied only once to the output rowsets, otherwise copies are sent to
 * each rowset!
 *
 * @param row The row to put to the destination rowset(s).
 * @throws KettleStepException
 */
@Override
public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
  if ( rowMeta != null ) {
    if ( !allowEmptyFieldNamesAndTypes ) {
      // check row meta for empty field name (BACKLOG-18004)
      for ( ValueMetaInterface vmi : rowMeta.getValueMetaList() ) {
        if ( StringUtils.isBlank( vmi.getName() ) ) {
          throw new KettleStepException( "Please set a field name for all field(s) that have 'null'." );
        }
        if ( vmi.getType() <= 0 ) {
          throw new KettleStepException( "Please set a value for the missing field(s) type." );
        }
      }
    }
  }
  getRowHandler().putRow( rowMeta, row );
}
 

@Test
public void testGetRequiredFields() throws Exception {
  XMLOutputMeta xmlOutputMeta = new XMLOutputMeta();
  xmlOutputMeta.setDefault();
  XMLField xmlField = new XMLField();
  xmlField.setFieldName( "aField" );
  xmlField.setType( 1 );
  xmlField.setLength( 10 );
  xmlField.setPrecision( 3 );

  XMLField xmlField2 = new XMLField();
  xmlField2.setFieldName( "bField" );
  xmlField2.setType( 3 );
  xmlField2.setLength( 4 );
  xmlField2.setPrecision( 5 );
  xmlOutputMeta.setOutputFields( new XMLField[] { xmlField, xmlField2 } );
  RowMetaInterface requiredFields = xmlOutputMeta.getRequiredFields( new Variables() );
  List<ValueMetaInterface> valueMetaList = requiredFields.getValueMetaList();
  assertEquals( 2, valueMetaList.size() );
  assertEquals( "aField", valueMetaList.get( 0 ).getName() );
  assertEquals( 1, valueMetaList.get( 0 ).getType() );
  assertEquals( 10, valueMetaList.get( 0 ).getLength() );
  assertEquals( 3, valueMetaList.get( 0 ).getPrecision() );

  assertEquals( "bField", valueMetaList.get( 1 ).getName() );
  assertEquals( 3, valueMetaList.get( 1 ).getType() );
  assertEquals( 4, valueMetaList.get( 1 ).getLength() );
  assertEquals( 5, valueMetaList.get( 1 ).getPrecision() );
}
 

private void addAllFieldsToEmptyFileDefinition( RowMetaInterface rowMeta, FileDefinition fileDefinition ) throws KettleStepException {
  if ( fileDefinition.getFieldDefinitions().isEmpty() ) {
    for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
      fileDefinition.getFieldDefinitions().add( new FieldDefinition(
          valueMeta.getName(),
          valueMeta.getTypeDesc(),
          valueMeta.getLength(),
          valueMeta.getPrecision(),
          valueMeta.getConversionMask()
        )
      );
    }
  }
}
 

@Override
protected Map<String, RowMetaInterface> getInputRowMetaInterfaces( GetXMLDataMeta meta ) {
  Map<String, RowMetaInterface> inputRows = getInputFields( meta );
  if ( inputRows == null ) {
    inputRows = new HashMap<>();
  }
  // Get some boolean flags from the meta for easier access
  boolean isInFields = meta.isInFields();
  boolean isAFile = meta.getIsAFile();
  boolean isAUrl = meta.isReadUrl();

  // only add resource fields if we are NOT getting the xml or file from a field
  if ( !isInFields || isAFile || isAUrl ) {
    RowMetaInterface stepFields = getOutputFields( meta );
    RowMetaInterface clone = stepFields.clone();
    // if there are previous steps providing data, we should remove them from the set of "resource" fields
    for ( RowMetaInterface rowMetaInterface : inputRows.values() ) {
      for ( ValueMetaInterface valueMetaInterface : rowMetaInterface.getValueMetaList() ) {
        try {
          clone.removeValueMeta( valueMetaInterface.getName() );
        } catch ( KettleValueException e ) {
          // could not find it in the output, skip it
        }
      }
    }
    inputRows.put( RESOURCE, clone );
  }
  return inputRows;
}
 

private static AccessInputField[] getInputFields( List<RowMetaAndData> in ) {
  RowMetaAndData rmd = in.get( 0 );
  RowMetaInterface rm = rmd.getRowMeta();
  List<AccessInputField> retval = new ArrayList<AccessInputField>();
  for ( ValueMetaInterface value : rm.getValueMetaList() ) {
    AccessInputField field = new AccessInputField( value.getName() );
    field.setColumn( value.getName() );
    field.setType( value.getType() );
    retval.add( field );
  }
  return retval.toArray( new AccessInputField[retval.size()] );
}
 

/**
 * creates a ComboVar populated with fields from the previous step.
 * @param parentComposite - the composite in which the widget will be placed
 * @param props - PropsUI props for L&F
 * @param stepMeta - stepMeta of the current step
 * @param formData - FormData to use for placement
 */
public static ComboVar createFieldDropDown(
  Composite parentComposite, PropsUI props, BaseStepMeta stepMeta, FormData formData ) {
  TransMeta transMeta = stepMeta.getParentStepMeta().getParentTransMeta();
  ComboVar fieldDropDownCombo = new ComboVar( transMeta, parentComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
  props.setLook( fieldDropDownCombo );
  fieldDropDownCombo.addModifyListener( e -> stepMeta.setChanged() );

  fieldDropDownCombo.setLayoutData( formData );
  Listener focusListener = e -> {
    String current = fieldDropDownCombo.getText();
    fieldDropDownCombo.getCComboWidget().removeAll();
    fieldDropDownCombo.setText( current );

    try {
      RowMetaInterface rmi = transMeta.getPrevStepFields( stepMeta.getParentStepMeta().getName() );
      List ls = rmi.getValueMetaList();
      for ( Object l : ls ) {
        ValueMetaBase vmb = (ValueMetaBase) l;
        fieldDropDownCombo.add( vmb.getName() );
      }
    } catch ( KettleStepException ex ) {
      // can be ignored, since previous step may not be set yet.
      stepMeta.logDebug( ex.getMessage(), ex );
    }
  };
  fieldDropDownCombo.getCComboWidget().addListener( SWT.FocusIn, focusListener );
  return fieldDropDownCombo;
}
 

private static boolean isValidRowMeta( RowMetaInterface rowMeta ) {
  for ( ValueMetaInterface value : rowMeta.getValueMetaList() ) {
    String name = value.getName();
    if ( name == null || name.isEmpty() ) {
      return false;
    }
  }
  return true;
}
 

@Test
public void outputRowMetasAreNotSharedAmongSeveralStreams() throws Exception {
  RowSet rs1 = new SingleRowRowSet();
  RowSet rs2 = new SingleRowRowSet();

  when( mockHelper.trans.isRunning() ).thenReturn( true );
  BaseStep baseStep =
    new BaseStep( mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans );
  baseStep.setStopped( false );
  baseStep.setRepartitioning( StepPartitioningMeta.PARTITIONING_METHOD_NONE );
  baseStep.setOutputRowSets( Arrays.asList( rs1, rs2 ) );

  for ( RowSet rowSet : baseStep.getOutputRowSets() ) {
    assertNull( "RowMeta should be null, since no calls were done", rowSet.getRowMeta() );
  }

  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "string" ) );
  rowMeta.addValueMeta( new ValueMetaInteger( "integer" ) );

  baseStep.putRow( rowMeta, new Object[] { "a", 1 } );

  RowMetaInterface meta1 = rs1.getRowMeta();
  RowMetaInterface meta2 = rs2.getRowMeta();
  assertNotNull( meta1 );
  assertNotNull( meta2 );
  // content is same
  for ( ValueMetaInterface meta : meta1.getValueMetaList() ) {
    assertTrue( meta.getName(), meta2.exists( meta ) );
  }
  // whereas instances differ
  assertFalse( meta1 == meta2 );
}
 

protected Object[] writeToTable( RowMetaInterface rowMeta, Object[] r ) throws KettleException {
  // see if we need to truncate any string fields
  try {
    int index = 0;
    List<ValueMetaInterface> valueMetas = rowMeta.getValueMetaList();
    for ( ValueMetaInterface valueMeta : valueMetas ) {
      Object valueData = r[index];
      if ( valueData != null ) {
        if ( valueMeta.getType() == ValueMetaInterface.TYPE_STRING ) {
          String str = valueMeta.getString( valueData );
          int len = valueMeta.getLength();
          if ( len < 1 ) {
            len = MonetDBDatabaseMeta.DEFAULT_VARCHAR_LENGTH;
          }
          if ( str.length() > len ) {
            // TODO log this event
            str = str.substring( 0, len );
          }
          r[index] = str;
        }
      }
      index++;
    }
  } catch ( Exception e ) {
    throw new KettleException( "Error serializing rows of data to the psql command", e );
  }
  return super.writeToTable( rowMeta, r );
}
 

/**
 * Calculate the lineage for the specified step only...
 *
 * @param stepMeta
 *          The step to calculate the lineage for.
 * @throws KettleStepException
 *           In case there is an exception calculating the lineage. This is usually caused by unavailable data sources
 *           etc.
 */
private void calculateLineage( StepMeta stepMeta ) throws KettleStepException {
  RowMetaInterface outputMeta = transMeta.getStepFields( stepMeta );

  // The lineage is basically a calculation of origin for each output of a certain step.
  //
  for ( ValueMetaInterface valueMeta : outputMeta.getValueMetaList() ) {

    StepMeta originStepMeta = transMeta.findStep( valueMeta.getOrigin(), stepMeta );
    if ( originStepMeta != null ) {
      /* List<StepMeta> list = */fieldStepsMap.get( originStepMeta );
    }
  }
}
 

protected static StringBuilder rowsToCSV( RowMetaInterface meta, List<Object[]> rows ) throws KettleValueException {
  StringBuilder builder = new StringBuilder();
  // header row
  int i = 0;
  for ( ValueMetaInterface v : meta.getValueMetaList() ) {
    String name = quote( v.getName() );
    builder.append( i > 0 ? "," : "" ).append( name );
    i++;
  }
  builder.append( "\n" );
  for ( Object[] row : rows ) {
    for ( i = 0; i < meta.size(); i++ ) {
      String value;
      ValueMetaInterface vm = meta.getValueMeta( i );
      if ( row[i] == null || Const.isEmpty( vm.getString( row[i] ) ) ) {
        value = "?";
      } else {
        //switch ( meta.getValueMetaList().get( i ).getType() ) {
        switch ( vm.getType() ) {
          case ValueMetaInterface.TYPE_NUMBER:
          case ValueMetaInterface.TYPE_INTEGER:
          case ValueMetaInterface.TYPE_BIGNUMBER:
            value = vm.getString( row[i] );
            break;
          case ValueMetaInterface.TYPE_DATE:
            int offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
            value = "" + ( vm.getDate( row[i] ).getTime() + offset );
            break;
          case ValueMetaInterface.TYPE_TIMESTAMP:
            offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
            value = "" + ( vm.getDate( row[i] ).getTime() + offset );
            break;
          case ValueMetaInterface.TYPE_BOOLEAN:
            value = "" + ( vm.getBoolean( row[i] ) ? "1" : "0" );
            break;
          // TODO throw an exception for Serializable/Binary
          default:
            value = quote( vm.getString( row[i] ) );
        }
      }
      builder.append( i > 0 ? "," : "" ).append( value );
    }
    builder.append( "\n" );
  }

  return builder;
}
 

protected static List<Object> rowsToCSVNew( RowMetaInterface meta, List<Object[]> rows ) throws KettleException {
  List<Object> results = new ArrayList<>( 2 );
  boolean needsBase64;
  CharsetEncoder encoder = Charset.forName( "US-ASCII" ).newEncoder();
  Charset utf8 = Charset.forName( "UTF-8" );

  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  BufferedOutputStream buf = new BufferedOutputStream( bos );

  // header row
  StringBuilder builder = new StringBuilder();
  int i = 0;
  for ( ValueMetaInterface v : meta.getValueMetaList() ) {
    String name = quote( v.getName() );
    builder.append( i > 0 ? "," : "" ).append( name );
    i++;
  }
  builder.append( "\n" );

  // We look for non-ascii characters and, if found, encode to ascii base64 first. For some reason,
  // encoding directly to utf-8 on the Java side (Mac OS X; Java 8) causes the python utf-8 decoder to hang
  // when there are non-ascii characters (such as in Mayagüez) present. Encoding to base64 first, then decoding
  // this on the python side seems to fix the issue.
  needsBase64 = !encoder.canEncode( builder.toString() );
  try {
    buf.write( builder.toString().getBytes( utf8 ) );

    for ( Object[] row : rows ) {
      builder.setLength( 0 );
      for ( i = 0; i < meta.size(); i++ ) {
        String value;
        ValueMetaInterface vm = meta.getValueMeta( i );
        if ( row[i] == null || Const.isEmpty( vm.getString( row[i] ) ) ) {
          value = "?";
        } else {
          //switch ( meta.getValueMetaList().get( i ).getType() ) {
          switch ( vm.getType() ) {
            case ValueMetaInterface.TYPE_NUMBER:
            case ValueMetaInterface.TYPE_INTEGER:
            case ValueMetaInterface.TYPE_BIGNUMBER:
              value = vm.getString( row[i] );
              break;
            case ValueMetaInterface.TYPE_DATE:
              int offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_TIMESTAMP:
              offset = TZ.getOffset( vm.getDate( row[i] ).getTime() );
              value = "" + ( vm.getDate( row[i] ).getTime() + offset );
              break;
            case ValueMetaInterface.TYPE_BOOLEAN:
              value = "" + ( vm.getBoolean( row[i] ) ? "1" : "0" );
              break;
            // TODO throw an exception for Serializable/Binary
            default:
              value = quote( vm.getString( row[i] ) );
          }
        }
        builder.append( i > 0 ? "," : "" ).append( value );
      }
      builder.append( "\n" );

      if ( !needsBase64 ) {
        needsBase64 = !encoder.canEncode( builder.toString() );
      }
      buf.write( builder.toString().getBytes( utf8 ) );
    }

    buf.flush();
    buf.close();
  } catch ( IOException e ) {
    throw new KettleException( e );
  }

  byte[] bytes = bos.toByteArray();
  if ( needsBase64 ) {
    bytes = Base64.encodeBase64( bytes );
  }
  results.add( bytes );
  results.add( needsBase64 );

  return results;
}
 

@Override public PDone expand( PCollection<KettleRow> input ) {

    try {
      // Only initialize once on this node/vm
      //
      BeamKettle.init( stepPluginClasses, xpPluginClasses );

      // Inflate the metadata on the node where this is running...
      //
      RowMetaInterface rowMeta = JsonRowMeta.fromJson( rowMetaJson );


      // Which table do we write to?
      //
      TableReference tableReference = new TableReference();
      if ( StringUtils.isNotEmpty( projectId ) ) {
        tableReference.setProjectId( projectId );
      }
      tableReference.setDatasetId( datasetId );
      tableReference.setTableId( tableId );

      TableSchema tableSchema = new TableSchema();
      List<TableFieldSchema> schemaFields = new ArrayList<>();
      for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
        TableFieldSchema schemaField = new TableFieldSchema();
        schemaField.setName( valueMeta.getName() );
        switch(valueMeta.getType()){
          case ValueMetaInterface.TYPE_STRING: schemaField.setType( "STRING" ); break;
          case ValueMetaInterface.TYPE_INTEGER: schemaField.setType( "INTEGER" ); break;
          case ValueMetaInterface.TYPE_DATE: schemaField.setType( "DATETIME" ); break;
          case ValueMetaInterface.TYPE_BOOLEAN: schemaField.setType( "BOOLEAN" ); break;
          case ValueMetaInterface.TYPE_NUMBER: schemaField.setType( "FLOAT" ); break;
          default:
            throw new RuntimeException( "Conversion from Kettle value "+valueMeta.toString()+" to BigQuery TableRow isn't supported yet" );
        }
        schemaFields.add(schemaField);
      }
      tableSchema.setFields( schemaFields );

      SerializableFunction<KettleRow, TableRow> formatFunction = new KettleToBQTableRowFn( stepname, rowMetaJson, stepPluginClasses, xpPluginClasses );

      BigQueryIO.Write.CreateDisposition createDisposition;
      if (createIfNeeded) {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
      }  else {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_NEVER;
      }

      BigQueryIO.Write.WriteDisposition writeDisposition;
      if (truncateTable) {
        writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
      } else {
        if (failIfNotEmpty) {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_EMPTY;
        } else {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
        }
      }

      BigQueryIO.Write<KettleRow> bigQueryWrite = BigQueryIO
        .<KettleRow>write()
        .to( tableReference )
        .withSchema( tableSchema )
        .withCreateDisposition( createDisposition )
        .withWriteDisposition( writeDisposition )
        .withFormatFunction( formatFunction );

      // TODO: pass the results along the way at some point
      //
      input.apply( stepname, bigQueryWrite );

      // End of the line
      //
      return PDone.in( input.getPipeline() );

    } catch ( Exception e ) {
      numErrors.inc();
      LOG.error( "Error in Beam BigQuery output transform", e );
      throw new RuntimeException( "Error in Beam BigQuery output transform", e );
    }
  }