Java Code Examples for org.pentaho.di.core.row.RowMetaInterface#addValueMeta()

@Test public void testTopLevelArrayStructureWithPrimitives() throws Exception {
  List<MongoDbOutputMeta.MongoField> paths = asList( mf( "field1", false, "[0]" ), mf( "field2", false, "[1]" ) );

  RowMetaInterface rmi = new RowMeta();
  rmi.addValueMeta( new ValueMetaString( "field1" ) );
  rmi.addValueMeta( new ValueMetaInteger( "field2" ) );

  Object[] row = new Object[ 2 ];
  row[ 0 ] = "value1";
  row[ 1 ] = 12L;
  VariableSpace vs = new Variables();

  for ( MongoDbOutputMeta.MongoField f : paths ) {
    f.init( vs );
  }

  DBObject result = kettleRowToMongo( paths, rmi, row, MongoDbOutputData.MongoTopLevel.ARRAY, false );

  assertEquals( JSON.serialize( result ), "[ \"value1\" , 12]" );
}
 

@Test
public void abortedSubtransThrowsAnError() throws KettleException {
  Result result1 = new Result();
  result1.setNrErrors( 1 );
  when( subtransExecutor.execute( any()  ) ).thenReturn( Optional.of( result1 ) );
  when( subtransExecutor.getPrefetchCount() ).thenReturn( 10 );
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "field" ) );
  FixedTimeStreamWindow<List> window =
    new FixedTimeStreamWindow<>( subtransExecutor, rowMeta, 0, 2, 1 );
  try {
    window.buffer( Flowable.fromIterable( singletonList( asList( "v1", "v2" ) ) ) ).forEach( result -> { } );
  } catch ( Exception e ) {
    assertEquals(
      getString( BaseStreamStep.class, "FixedTimeStreamWindow.SubtransFailed" ), e.getCause().getMessage().trim() );
  }
}
 

@Test
public void testGetFields() throws Exception {
  StringOperationsMeta meta = new StringOperationsMeta();
  meta.allocate( 1 );
  meta.setFieldInStream( new String[] { "field1" } );

  RowMetaInterface rowMetaInterface = new RowMeta();
  ValueMetaInterface valueMeta = new ValueMetaString( "field1" );
  valueMeta.setStorageMetadata( new ValueMetaString( "field1" ) );
  valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING );
  rowMetaInterface.addValueMeta( valueMeta );

  VariableSpace space = mock( VariableSpace.class );
  meta.getFields( rowMetaInterface, "STRING_OPERATIONS", null, null, space, null, null );
  RowMetaInterface expectedRowMeta = new RowMeta();
  expectedRowMeta.addValueMeta( new ValueMetaString( "field1" ) );
  assertEquals( expectedRowMeta.toString(), rowMetaInterface.toString() );
}
 

@Override
public void getFields( RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  // We don't have any input fields here in "r" as they are all info fields.
  // So we just merge in the info fields.
  //
  if ( info != null ) {
    boolean found = false;
    for ( int i = 0; i < info.length && !found; i++ ) {
      if ( info[i] != null ) {
        r.mergeRowMeta( info[i], name );
        found = true;
      }
    }
  }

  if ( Utils.isEmpty( flagField ) ) {
    throw new KettleStepException( BaseMessages.getString( PKG, "MergeRowsMeta.Exception.FlagFieldNotSpecified" ) );
  }
  ValueMetaInterface flagFieldValue = new ValueMetaString( flagField );
  flagFieldValue.setOrigin( name );
  r.addValueMeta( flagFieldValue );

}
 

@SuppressWarnings( "unchecked" )
public static void getFields( boolean clearResultFields, RowMetaInterface row, String originStepname,
  RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, List<?> fields ) throws KettleStepException {
  if ( clearResultFields ) {
    row.clear();
  }
  for ( FieldInfo fi : (List<FieldInfo>) fields ) {
    try {
      ValueMetaInterface v = ValueMetaFactory.createValueMeta( fi.name, fi.type );
      v.setLength( fi.length );
      v.setPrecision( fi.precision );
      v.setOrigin( originStepname );
      row.addValueMeta( v );
    } catch ( Exception e ) {
      throw new KettleStepException( e );
    }
  }
}
 

public void getFields( RowMetaInterface row, String name, RowMetaInterface[] info, StepMeta nextStep,
    VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {

  ValueMetaInterface v = new ValueMeta( this.getValueXMLfield(), ValueMetaInterface.TYPE_STRING );
  v.setOrigin( name );

  TransMeta transMeta = (TransMeta) space;
  try {
    // Row should only include fields from the target and not the source. During the preview table generation
    // the fields from all previous steps (source and target) are included in the row so lets remove the
    // source fields.
    List<String> targetFieldNames = null;
    RowMetaInterface targetRowMeta = transMeta.getStepFields( transMeta.findStep( getTargetXMLstep() ),
      null,
      null );
    if ( targetRowMeta != null ) {
      targetFieldNames = Arrays.asList( targetRowMeta.getFieldNames() );
    }
    for ( String fieldName : transMeta.getStepFields( transMeta.findStep( getSourceXMLstep() ),
                                                                          null,
                                                                          null ).getFieldNames() ) {
      if ( targetFieldNames == null || !targetFieldNames.contains( fieldName ) ) {
        row.removeValueMeta( fieldName );
      }
    }
  } catch ( KettleValueException e ) {
    // Pass
  }

  row.addValueMeta( v );
}
 

/**
 * Creates a row meta interface for the fields that are defined by performing a getFields and by checking "Result
 * filenames - Add filenames to result from "Text File Input" dialog.
 * 
 * @return
 */
public RowMetaInterface createResultRowMetaInterface() {
  RowMetaInterface rowMetaInterface = new RowMeta();

  ValueMetaInterface[] valuesMeta =
    { new ValueMetaInteger( "Id" ), new ValueMetaString( "State" ), new ValueMetaString( "City" ) };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rowMetaInterface.addValueMeta( valuesMeta[i] );
  }

  return rowMetaInterface;
}
 

public static final RowMetaInterface getResultRowMeta() {
  RowMetaInterface rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.TransOrJob" ) ) );
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.StepDatabaseNotice" ) ) );
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.String" ) ) );
  rowMeta.addValueMeta( new ValueMetaString(
    BaseMessages.getString( PKG, "SearchResult.FieldName" ) ) );
  return rowMeta;
}
 

private ReplaceString prepareTestProcessRow( ReplaceStringData data, ReplaceStringMeta meta, String unicodeString,
                                             String matchPattern, String matchReplacer ) throws Exception {
  ReplaceString replaceString = Mockito.spy(
    new ReplaceString( stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans ) );
  RowMetaInterface inputRowMeta = new RowMeta();
  Object[] _row = new Object[] { unicodeString, null };
  doReturn( _row ).when( replaceString ).getRow();
  inputRowMeta.addValueMeta( 0, new ValueMetaString( "string" ) );

  doReturn( new String[] { "string" } ).when( meta ).getFieldInStream();
  doReturn( new String[] { "output" } ).when( meta ).getFieldOutStream();
  doReturn( new String[] { matchPattern } ).when( meta ).getReplaceString();
  doReturn( new String[] { StringUtils.EMPTY } ).when( meta ).getFieldReplaceByString();
  doReturn( new String[] { matchReplacer } ).when( meta ).getReplaceByString();
  doReturn( new boolean[] { false } ).when( meta ).isSetEmptyString();

  replaceString.init( meta, data );
  replaceString.setInputRowMeta( inputRowMeta );
  data.outputRowMeta = inputRowMeta;
  data.inputFieldsNr = 1;
  data.numFields = 2;
  data.inStreamNrs = new int[] { 0 };
  data.replaceFieldIndex = new int[] { -1 };
  data.outStreamNrs = new String[] { StringUtils.EMPTY, "1" };
  data.replaceByString = new String[] { "1" };
  data.setEmptyString = new boolean[] { false, false };

  return replaceString;
}
 

@Test public void testInsertKettleFieldThatContainsJsonIntoTopLevelRecord() throws Exception {
  List<MongoDbOutputMeta.MongoField> paths = new ArrayList<MongoDbOutputMeta.MongoField>( 3 );

  MongoDbOutputMeta.MongoField mf = mf( "field1", true, "" );
  paths.add( mf );

  mf = mf( "field2", true, "" );
  paths.add( mf );

  mf = mf( "jsonField", true, "" );
  mf.m_JSON = true;
  paths.add( mf );

  RowMetaInterface rm = new RowMeta();
  rm.addValueMeta( new ValueMetaString( "field1" ) );
  rm.addValueMeta( new ValueMetaInteger( "field2" ) );
  rm.addValueMeta( new ValueMetaString( "jsonField" ) );

  Object[] row = new Object[ 3 ];
  row[ 0 ] = "value1";
  row[ 1 ] = 12L;
  row[ 2 ] = "{\"jsonDocField1\" : \"aval\", \"jsonDocField2\" : 42}";
  VariableSpace vs = new Variables();

  for ( MongoDbOutputMeta.MongoField f : paths ) {
    f.init( vs );
  }

  DBObject result = kettleRowToMongo( paths, rm, row, MongoDbOutputData.MongoTopLevel.RECORD, false );

  assertEquals( JSON.serialize( result ),
    "{ \"field1\" : \"value1\" , \"field2\" : 12 , \"jsonField\" : { \"jsonDocField1\" : \"aval\" , "
      + "\"jsonDocField2\" : 42}}" );

}
 

@Test
public void testGetFields() throws KettleStepException {
  SalesforceDeleteMeta meta = new SalesforceDeleteMeta();
  meta.setDefault();
  RowMetaInterface r = new RowMeta();
  meta.getFields( r, "thisStep", null, null, new Variables(), null, null );
  assertEquals( 0, r.size() );

  r.clear();
  r.addValueMeta( new ValueMetaString( "testString" ) );
  meta.getFields( r, "thisStep", null, null, new Variables(), null, null );
  assertEquals( 1, r.size() );
  assertEquals( ValueMetaInterface.TYPE_STRING, r.getValueMeta( 0 ).getType() );
  assertEquals( "testString", r.getValueMeta( 0 ).getName() );
}
 

private static RowMetaInterface createRowMetaForOneField() {
  RowMetaInterface rm = new RowMeta();
  ValueMetaInterface[] valuesMeta = { new ValueMetaString( "field1" ), };
  for ( ValueMetaInterface aValuesMeta : valuesMeta ) {
    rm.addValueMeta( aValuesMeta );
  }
  return rm;
}
 

public RowMetaInterface createResultRowMetaInterface() {
  RowMetaInterface rm = new RowMeta();

  ValueMetaInterface[] valuesMeta =
  {
    new ValueMetaInteger( "a" ), new ValueMetaString( "b" ),
    new ValueMetaString( "c" ), new ValueMetaString( "filename" ), };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rm.addValueMeta( valuesMeta[i] );
  }

  return rm;
}
 

private RowMetaInterface generateRowMeta( String[] fieldNames ) {
  RowMetaInterface rowMeta = new RowMeta();

  for ( String fieldName : fieldNames ) {
    ValueMetaInterface col = new ValueMeta();
    col.setName( fieldName );
    col.setType( ValueMeta.TYPE_STRING );
    rowMeta.addValueMeta( col );
  }
  return rowMeta;
}
 

/**
 * Gets the log fields.
 *
 * @param comm the comm
 * @return the log fields
 */
public static final RowMetaInterface getLogFields( String comm ) {
  RowMetaInterface r = new RowMeta();
  ValueMetaInterface sname =
    new ValueMetaString(
      BaseMessages.getString( PKG, "BaseStep.ColumnName.Stepname" ) );
  sname.setLength( 256 );
  r.addValueMeta( sname );

  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.Copy" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesReaded" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesWritten" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesUpdated" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.LinesSkipped" ) ) );
  r.addValueMeta( new ValueMetaNumber(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.Errors" ) ) );
  r.addValueMeta( new ValueMetaDate(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.StartDate" ) ) );
  r.addValueMeta( new ValueMetaDate(
    BaseMessages.getString( PKG, "BaseStep.ColumnName.EndDate" ) ) );

  for ( int i = 0; i < r.size(); i++ ) {
    r.getValueMeta( i ).setOrigin( comm );
  }

  return r;
}
 

public RowMetaInterface createRowMetaInterface3() {
  RowMetaInterface rm = new RowMeta();

  ValueMetaInterface[] valuesMeta =
    {
      new ValueMetaInteger( "int_in" ),
      new ValueMetaNumber( "number_in" ),
      new ValueMetaString( "string_in" ), };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rm.addValueMeta( valuesMeta[ i ] );
  }

  return rm;
}
 

/**
 * PDI-11045. Here we test backwards compatibility for old ktrs that were developed before 11045. In these ktrs query
 * paths had to be specified a second time in the step in order to get them into the update/upsert object. Now we
 * assume that there will never be a situation where the user might not want the match fields present in the update
 * object
 *
 * @throws KettleException
 */
@Test public void testUpdateObjectBackwardsCompatibility() throws Exception {
  List<MongoDbOutputMeta.MongoField> paths = new ArrayList<MongoDbOutputMeta.MongoField>( 2 );

  MongoDbOutputMeta.MongoField mf = mf( "field1", true, "" );
  mf.m_updateMatchField = true;
  paths.add( mf );

  // same as previous field (but not a match condition). Prior to PDI-11045 the
  // user had to specify the match conditions a second time (but not marked in the
  // step as a match condition) in order to get them into the update/upsert object
  mf = mf( "field1", true, "" );
  mf.m_updateMatchField = false;
  paths.add( mf );

  mf = mf( "field2", true, "" );
  paths.add( mf );

  RowMetaInterface rmi = new RowMeta();
  rmi.addValueMeta( new ValueMetaString( "field1" ) );
  rmi.addValueMeta( new ValueMetaInteger( "field2" ) );

  Object[] row = new Object[ 2 ];
  row[ 0 ] = "value1";
  row[ 1 ] = 12L;
  VariableSpace vs = new Variables();

  for ( MongoDbOutputMeta.MongoField f : paths ) {
    f.init( vs );
  }

  DBObject result = kettleRowToMongo( paths, rmi, row, MongoDbOutputData.MongoTopLevel.RECORD, false );

  // here we expect that field1 does *not* occur twice in the update object
  assertEquals( JSON.serialize( result ), "{ \"field1\" : \"value1\" , \"field2\" : 12}" );
}
 

public RowMetaInterface createRowMetaInterface() {
  RowMetaInterface rm = new RowMeta();

  ValueMetaInterface[] valuesMeta = { new ValueMetaInteger( "int_field" ) };

  for ( int i = 0; i < valuesMeta.length; i++ ) {
    rm.addValueMeta( valuesMeta[i] );
  }

  return rm;
}
 

/**
 * Processing excel workbook, filling fields
 *
 * @param fields   RowMetaInterface for filling fields
 * @param info     ExcelInputMeta
 * @param workbook excel workbook for processing
 * @throws KettlePluginException
 */
private void processingWorkbook( RowMetaInterface fields, ExcelInputMeta info, KWorkbook workbook )
  throws KettlePluginException {
  int nrSheets = workbook.getNumberOfSheets();
  for ( int j = 0; j < nrSheets; j++ ) {
    KSheet sheet = workbook.getSheet( j );

    // See if it's a selected sheet:
    int sheetIndex;
    if ( info.readAllSheets() ) {
      sheetIndex = 0;
    } else {
      sheetIndex = Const.indexOfString( sheet.getName(), info.getSheetName() );
    }
    if ( sheetIndex >= 0 ) {
      // We suppose it's the complete range we're looking for...
      //
      int rownr = 0;
      int startcol = 0;

      if ( info.readAllSheets() ) {
        if ( info.getStartColumn().length == 1 ) {
          startcol = info.getStartColumn()[ 0 ];
        }
        if ( info.getStartRow().length == 1 ) {
          rownr = info.getStartRow()[ 0 ];
        }
      } else {
        rownr = info.getStartRow()[ sheetIndex ];
        startcol = info.getStartColumn()[ sheetIndex ];
      }

      boolean stop = false;
      for ( int colnr = startcol; !stop; colnr++ ) {
        try {
          String fieldname = null;
          int fieldtype = ValueMetaInterface.TYPE_NONE;

          KCell cell = sheet.getCell( colnr, rownr );
          if ( cell == null ) {
            stop = true;
          } else {
            if ( cell.getType() != KCellType.EMPTY ) {
              // We found a field.
              fieldname = cell.getContents();
            }

            // System.out.println("Fieldname = "+fieldname);

            KCell below = sheet.getCell( colnr, rownr + 1 );

            if ( below != null ) {
              if ( below.getType() == KCellType.BOOLEAN ) {
                fieldtype = ValueMetaInterface.TYPE_BOOLEAN;
              } else if ( below.getType() == KCellType.DATE ) {
                fieldtype = ValueMetaInterface.TYPE_DATE;
              } else if ( below.getType() == KCellType.LABEL ) {
                fieldtype = ValueMetaInterface.TYPE_STRING;
              } else if ( below.getType() == KCellType.NUMBER ) {
                fieldtype = ValueMetaInterface.TYPE_NUMBER;
              } else {
                fieldtype = ValueMetaInterface.TYPE_STRING;
              }
            } else {
              fieldtype = ValueMetaInterface.TYPE_STRING;
            }

            if ( Utils.isEmpty( fieldname ) ) {
              stop = true;
            } else {
              if ( fieldtype != ValueMetaInterface.TYPE_NONE ) {
                ValueMetaInterface field = ValueMetaFactory.createValueMeta( fieldname, fieldtype );
                fields.addValueMeta( field );
              }
            }
          }
        } catch ( ArrayIndexOutOfBoundsException aioobe ) {
          // System.out.println("index out of bounds at column "+colnr+" : "+aioobe.toString());
          stop = true;
        }
      }
    }
  }
}
 

public void getFields( RowMetaInterface row, String originStepname, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {
  for ( int i = 0; i < fieldname.length; i++ ) {
    if ( !Utils.isEmpty( fieldname[i] ) ) {
      String fieldName;
      int replaceIndex;
      int fieldType;

      if ( replace[i] ) {
        // Look up the field to replace...
        //
        if ( row.searchValueMeta( fieldname[i] ) == null && Utils.isEmpty( rename[i] ) ) {
          throw new KettleStepException( BaseMessages.getString(
            PKG, "ScriptMeta.Exception.FieldToReplaceNotFound", fieldname[i] ) );
        }
        replaceIndex = row.indexOfValue( rename[i] );

        // Change the data type to match what's specified...
        //
        fieldType = type[i];
        fieldName = rename[i];
      } else {
        replaceIndex = -1;
        fieldType = type[i];
        if ( rename[i] != null && rename[i].length() != 0 ) {
          fieldName = rename[i];
        } else {
          fieldName = fieldname[i];
        }
      }
      try {
        ValueMetaInterface v = ValueMetaFactory.createValueMeta( fieldName, fieldType );
        v.setLength( length[i] );
        v.setPrecision( precision[i] );
        v.setOrigin( originStepname );
        if ( replace[i] && replaceIndex >= 0 ) {
          row.setValueMeta( replaceIndex, v );
        } else {
          row.addValueMeta( v );
        }
      } catch ( KettlePluginException e ) {
        // Ignore errors
      }
    }
  }
}