Java Code Examples for org.pentaho.di.core.row.ValueMetaInterface#getType()

private void getUpdate() {
  try {
    RowMetaInterface r = transMeta.getPrevStepFields( stepname );
    if ( r != null ) {
      TableItemInsertListener listener = new TableItemInsertListener() {
        public boolean tableItemInserted( TableItem tableItem, ValueMetaInterface v ) {
          if ( v.getType() == ValueMetaInterface.TYPE_DATE ) {
            // The default is : format is OK for dates, see if this sticks later on...
            //
            tableItem.setText( 3, "Y" );
          } else {
            tableItem.setText( 3, "Y" ); // default is OK too...
          }
          return true;
        }
      };
      BaseStepDialog.getFieldsFromPrevious( r, wReturn, 1, new int[] { 1, 2 }, new int[] {}, -1, -1, listener );
    }
  } catch ( KettleException ke ) {
    new ErrorDialog( shell,
      BaseMessages.getString( PKG, "LucidDBBulkLoaderDialog.FailedToGetFields.DialogTitle" ),
      BaseMessages.getString( PKG, "LucidDBBulkLoaderDialog.FailedToGetFields.DialogMessage" ), ke );
  }
}
 

public static Object getPrimitive( ValueMetaInterface valueMeta, Object valueData ) throws KettleValueException {
  switch ( valueMeta.getType() ) {
    case ValueMetaInterface.TYPE_BIGNUMBER:
      return valueMeta.getBigNumber( valueData );
    case ValueMetaInterface.TYPE_BINARY:
      return valueMeta.getBinary( valueData );
    case ValueMetaInterface.TYPE_BOOLEAN:
      return valueMeta.getBoolean( valueData );
    case ValueMetaInterface.TYPE_DATE:
      return valueMeta.getDate( valueData );
    case ValueMetaInterface.TYPE_INTEGER:
      return valueMeta.getInteger( valueData );
    case ValueMetaInterface.TYPE_NUMBER:
      return valueMeta.getNumber( valueData );
      // case ValueMetaInterface.TYPE_SERIALIZABLE: return valueMeta.(valueData);
    case ValueMetaInterface.TYPE_STRING:
      return valueMeta.getString( valueData );
    default:
      return null;
  }
}
 

private void getUpdate() {
  try {
    RowMetaInterface r = transMeta.getPrevStepFields( stepname );
    if ( r != null ) {
      TableItemInsertListener listener = new TableItemInsertListener() {
        public boolean tableItemInserted( TableItem tableItem, ValueMetaInterface v ) {
          if ( v.getType() == ValueMetaInterface.TYPE_DATE ) {
            // The default is : format is OK for dates, see if this sticks later on...
            //
            tableItem.setText( 3, "Y" );
          } else {
            tableItem.setText( 3, "Y" ); // default is OK too...
          }
          return true;
        }
      };
      BaseStepDialog.getFieldsFromPrevious( r, wReturn, 1, new int[] { 1, 2 }, new int[] {}, -1, -1, listener );
    }
  } catch ( KettleException ke ) {
    new ErrorDialog(
      shell, BaseMessages.getString( PKG, "MySQLBulkLoaderDialog.FailedToGetFields.DialogTitle" ),
      BaseMessages.getString( PKG, "MySQLBulkLoaderDialog.FailedToGetFields.DialogMessage" ), ke );
  }
}
 

static StepMeta getInjectorStepMeta( RowMetaInterface outputRowMeta ) {
  InjectorMeta zeroMeta = new InjectorMeta();

  // Sets output fields for cases when no rows are sent to the test step, but metadata is still needed
  if ( outputRowMeta != null && outputRowMeta.size() > 0 ) {
    String[] fieldName = new String[outputRowMeta.size()];
    int[] fieldLength = new int[outputRowMeta.size()];
    int[] fieldPrecision = new int[outputRowMeta.size()];
    int[] fieldType = new int[outputRowMeta.size()];
    for ( int i = 0; i < outputRowMeta.size(); i++ ) {
      ValueMetaInterface field = outputRowMeta.getValueMeta( i );
      fieldName[i] = field.getName();
      fieldLength[i] = field.getLength();
      fieldPrecision[i] = field.getPrecision();
      fieldType[i] = field.getType();
    }
    zeroMeta.setFieldname( fieldName );
    zeroMeta.setLength( fieldLength );
    zeroMeta.setPrecision( fieldPrecision );
    zeroMeta.setType( fieldType );
  }

  StepMeta zero = new StepMeta( registry.getPluginId( StepPluginType.class, zeroMeta ), INJECTOR_STEPNAME, zeroMeta );
  zero.setLocation( 50, 50 );
  zero.setDraw( true );

  return zero;
}
 

@Override
Object doCall() throws Exception {
  Random random = new Random();
  while ( condition.get() ) {
    int acc = 0;
    for ( ValueMetaInterface meta : rowMeta.getValueMetaList() ) {
      // fake cycle to from eliminating this snippet by JIT
      acc += meta.getType() / 10;
    }
    Thread.sleep( random.nextInt( Math.max( 100, acc ) ) );
  }
  return null;
}
 

/**
 * Attempt to determine the Java {@link Class} for the {@link ValueMetaInterface} provided
 *
 * @param vmi Value Meta with type information to look up
 * @return the class that represents the type {@link ValueMetaInterface} represents; {@code null} if no type can be
 * matched.
 */
public Class<?> getJavaClass( ValueMetaInterface vmi ) {
  Class<?> metaClass = null;

  switch ( vmi.getType() ) {
    case ValueMeta.TYPE_BIGNUMBER:
      metaClass = BigDecimal.class;
      break;
    case ValueMeta.TYPE_BINARY:
      metaClass = byte[].class;
      break;
    case ValueMeta.TYPE_BOOLEAN:
      metaClass = Boolean.class;
      break;
    case ValueMeta.TYPE_DATE:
      metaClass = Date.class;
      break;
    case ValueMeta.TYPE_INTEGER:
      metaClass = Long.class;
      break;
    case ValueMeta.TYPE_NUMBER:
      metaClass = Double.class;
      break;
    case ValueMeta.TYPE_STRING:
      metaClass = String.class;
      break;
    case ValueMeta.TYPE_SERIALIZABLE:
      metaClass = Object.class;
      break;
  }

  return metaClass;
}
 

@Override
public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean useAutoinc,
                                  boolean addFieldName, boolean addCr ) {
  String retval = "";

  String fieldname = v.getName();
  int length = v.getLength();
  int precision = v.getPrecision();

  if ( addFieldName ) {
    retval += fieldname + " ";
  }

  int type = v.getType();
  switch ( type ) {
    case ValueMetaInterface.TYPE_TIMESTAMP:
      retval += "DATETIME";
      break;
    case ValueMetaInterface.TYPE_DATE:
      retval += "DATETIME YEAR to FRACTION";
      break;
    case ValueMetaInterface.TYPE_BOOLEAN:
      if ( supportsBooleanDataType() ) {
        retval += "BOOLEAN";
      } else {
        retval += "CHAR(1)";
      }
      break;
    case ValueMetaInterface.TYPE_NUMBER:
    case ValueMetaInterface.TYPE_INTEGER:
    case ValueMetaInterface.TYPE_BIGNUMBER:
      if ( fieldname.equalsIgnoreCase( tk ) || // Technical key
        fieldname.equalsIgnoreCase( pk ) // Primary key
      ) {
        if ( useAutoinc ) {
          retval += "SERIAL8";
        } else {
          retval += "INTEGER PRIMARY KEY";
        }
      } else {
        if ( ( length < 0 && precision < 0 ) || precision > 0 || length > 9 ) {
          retval += "FLOAT";
        } else { // Precision == 0 && length<=9
          retval += "INTEGER";
        }
      }
      break;
    case ValueMetaInterface.TYPE_STRING:
      if ( length >= DatabaseMeta.CLOB_LENGTH ) {
        retval += "CLOB";
      } else {
        if ( length < 256 ) {
          retval += "VARCHAR";
          if ( length > 0 ) {
            retval += "(" + length + ")";
          }
        } else {
          if ( length < 32768 ) {
            retval += "LVARCHAR";
          } else {
            retval += "TEXT";
          }
        }
      }
      break;
    default:
      retval += " UNKNOWN";
      break;
  }

  if ( addCr ) {
    retval += Const.CR;
  }

  return retval;
}
 

public void execute( Object[] row ) throws KettleException {

      // Create a new object with specified fields
      JSONObject jo = new JSONObject();

      for ( int i = 0; i < data.nrFields; i++ ) {
        JsonOutputField outputField = meta.getOutputFields()[i];

        ValueMetaInterface v = data.inputRowMeta.getValueMeta( data.fieldIndexes[i] );

        switch ( v.getType() ) {
          case ValueMetaInterface.TYPE_BOOLEAN:
            jo.put( outputField.getElementName(), data.inputRowMeta.getBoolean( row, data.fieldIndexes[i] ) );
            break;
          case ValueMetaInterface.TYPE_INTEGER:
            jo.put( outputField.getElementName(), data.inputRowMeta.getInteger( row, data.fieldIndexes[i] ) );
            break;
          case ValueMetaInterface.TYPE_NUMBER:
            jo.put( outputField.getElementName(), data.inputRowMeta.getNumber( row, data.fieldIndexes[i] ) );
            break;
          case ValueMetaInterface.TYPE_BIGNUMBER:
            jo.put( outputField.getElementName(), data.inputRowMeta.getBigNumber( row, data.fieldIndexes[i] ) );
            break;
          default:
            jo.put( outputField.getElementName(), data.inputRowMeta.getString( row, data.fieldIndexes[i] ) );
            break;
        }
      }
      data.ja.add( jo );

      data.nrRow++;

      if ( data.nrRowsInBloc > 0 ) {
        // System.out.println("data.nrRow%data.nrRowsInBloc = "+ data.nrRow%data.nrRowsInBloc);
        if ( data.nrRow % data.nrRowsInBloc == 0 ) {
          // We can now output an object
          // System.out.println("outputting the row.");
          outPutRow( row );
        }
      }
    }
 

@Override
public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean useAutoinc,
                                  boolean addFieldName, boolean addCr ) {
  StringBuilder retval = new StringBuilder( 128 );

  String fieldname = v.getName();
  int length = v.getLength();
  int precision = v.getPrecision();

  if ( addFieldName ) {
    retval.append( fieldname ).append( ' ' );
  }

  int type = v.getType();
  switch ( type ) {
    case ValueMetaInterface.TYPE_TIMESTAMP:
      if ( supportsTimestampDataType() ) {
        retval.append( "TIMESTAMP" );
      } else {
        retval.append( "DATE" );
      }
      break;
    case ValueMetaInterface.TYPE_DATE:
      retval.append( "DATE" );
      break;
    case ValueMetaInterface.TYPE_BOOLEAN:
      retval.append( "CHAR(1)" );
      break;
    case ValueMetaInterface.TYPE_NUMBER:
    case ValueMetaInterface.TYPE_BIGNUMBER:
      retval.append( "NUMBER" );
      if ( length > 0 ) {
        retval.append( '(' ).append( length );
        if ( precision > 0 ) {
          retval.append( ", " ).append( precision );
        }
        retval.append( ')' );
      }
      break;
    case ValueMetaInterface.TYPE_INTEGER:
      retval.append( "INTEGER" );
      break;
    case ValueMetaInterface.TYPE_STRING:
      if ( length >= DatabaseMeta.CLOB_LENGTH ) {
        retval.append( "CLOB" );
      } else {
        if ( length == 1 ) {
          retval.append( "CHAR(1)" );
        } else if ( length > 0 && length <= getMaxVARCHARLength() ) {
          retval.append( "VARCHAR2(" ).append( length ).append( ')' );
        } else {
          if ( length <= 0 ) {
            retval.append( "VARCHAR2(2000)" ); // We don't know, so we just use the maximum...
          } else {
            retval.append( "CLOB" );
          }
        }
      }
      break;
    case ValueMetaInterface.TYPE_BINARY: // the BLOB can contain binary data.
      retval.append( "BLOB" );
      break;
    default:
      retval.append( " UNKNOWN" );
      break;
  }

  if ( addCr ) {
    retval.append( Const.CR );
  }

  return retval.toString();
}
 

public static Object[] createObjectsForRow( RowMetaInterface rowMeta, Object[] rowData ) throws KettleValueException {
  Object[] values = new Object[rowMeta.size()];
  for ( int i = 0; i < rowMeta.size(); i++ ) {
    ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
    Object valueData = rowData[i];

    // Prevent a NullPointerException below
    if ( valueData == null || valueMeta == null ) {
      values[i] = null;
      continue;
    }

    int length = valueMeta.getLength();

    switch ( valueMeta.getType() ) {
      case ValueMetaInterface.TYPE_INTEGER:
        if ( length < 3 ) {
          values[i] = new Byte( valueMeta.getInteger( valueData ).byteValue() );
        } else {
          if ( length < 5 ) {
            values[i] = new Short( valueMeta.getInteger( valueData ).shortValue() );
          } else {
            values[i] = valueMeta.getInteger( valueData );
          }
        }
        break;
      case ValueMetaInterface.TYPE_NUMBER:
        values[i] = valueMeta.getNumber( valueData );
        break;
      case ValueMetaInterface.TYPE_DATE:
        values[i] = valueMeta.getDate( valueData );
        break;
      case ValueMetaInterface.TYPE_STRING:
        values[i] = valueMeta.getString( valueData );
        break;
      case ValueMetaInterface.TYPE_BINARY:
        values[i] = valueMeta.getBinary( valueData );
        break;
      case ValueMetaInterface.TYPE_BOOLEAN:
        values[i] = valueMeta.getBoolean( valueData );
        break;
      case ValueMetaInterface.TYPE_BIGNUMBER:
        values[i] = valueMeta.getNumber( valueData );
        break;
      default:
        break;
    }
  }
  return values;
}
 

private void checkType( KCell cell, ValueMetaInterface v ) throws KettleException {
  if ( !meta.isStrictTypes() ) {
    return;
  }
  switch ( cell.getType() ) {
    case BOOLEAN:
      if ( !( v.getType() == ValueMetaInterface.TYPE_STRING || v.getType() == ValueMetaInterface.TYPE_NONE || v
        .getType() == ValueMetaInterface.TYPE_BOOLEAN ) ) {
        throw new KettleException( BaseMessages.getString( PKG, "ExcelInput.Exception.InvalidTypeBoolean", v
          .getTypeDesc() ) );
      }
      break;

    case DATE:
      if ( !( v.getType() == ValueMetaInterface.TYPE_STRING || v.getType() == ValueMetaInterface.TYPE_NONE || v
        .getType() == ValueMetaInterface.TYPE_DATE ) ) {
        throw new KettleException( BaseMessages.getString( PKG, "ExcelInput.Exception.InvalidTypeDate", cell
          .getContents(), v.getTypeDesc() ) );
      }
      break;

    case LABEL:
      if ( v.getType() == ValueMetaInterface.TYPE_BOOLEAN
        || v.getType() == ValueMetaInterface.TYPE_DATE || v.getType() == ValueMetaInterface.TYPE_INTEGER
        || v.getType() == ValueMetaInterface.TYPE_NUMBER ) {
        throw new KettleException( BaseMessages.getString( PKG, "ExcelInput.Exception.InvalidTypeLabel", cell
          .getContents(), v.getTypeDesc() ) );
      }
      break;

    case EMPTY:
      // OK
      break;

    case NUMBER:
      if ( !( v.getType() == ValueMetaInterface.TYPE_STRING
        || v.getType() == ValueMetaInterface.TYPE_NONE || v.getType() == ValueMetaInterface.TYPE_INTEGER
        || v.getType() == ValueMetaInterface.TYPE_BIGNUMBER || v.getType() == ValueMetaInterface.TYPE_NUMBER ) ) {
        throw new KettleException( BaseMessages.getString( PKG, "ExcelInput.Exception.InvalidTypeNumber", cell
          .getContents(), v.getTypeDesc() ) );
      }
      break;

    default:
      throw new KettleException( BaseMessages.getString( PKG, "ExcelInput.Exception.UnsupportedType", cell
        .getType().getDescription(), cell.getContents() ) );
  }
}
 

public void getMetadataFields( RowMetaInterface inputRowMeta, String name, VariableSpace space ) throws KettlePluginException {
  if ( meta != null && meta.length > 0 ) {
    // METADATA mode: change the meta-data of the values mentioned...

    for ( int i = 0; i < meta.length; i++ ) {
      SelectMetadataChange metaChange = meta[i];

      int idx = inputRowMeta.indexOfValue( metaChange.getName() );
      boolean metaTypeChangeUsesNewTypeDefaults = false; // Normal behavior as of 5.x or so
      if ( space != null ) {
        metaTypeChangeUsesNewTypeDefaults = ValueMetaBase.convertStringToBoolean(
            space.getVariable( Const.KETTLE_COMPATIBILITY_SELECT_VALUES_TYPE_CHANGE_USES_TYPE_DEFAULTS, "N" ) );
      }
      if ( idx >= 0 ) { // We found the value

        // This is the value we need to change:
        ValueMetaInterface v = inputRowMeta.getValueMeta( idx );

        // Do we need to rename ?
        if ( !v.getName().equals( metaChange.getRename() ) && !Utils.isEmpty( metaChange.getRename() ) ) {
          v.setName( metaChange.getRename() );
          v.setOrigin( name );
          // need to reinsert to check name conflicts
          inputRowMeta.setValueMeta( idx, v );
        }
        // Change the type?
        if ( metaChange.getType() != ValueMetaInterface.TYPE_NONE && v.getType() != metaChange.getType() ) {
          // Fix for PDI-16388 - clone copies over the conversion mask instead of using the default for the new type
          if ( !metaTypeChangeUsesNewTypeDefaults ) {
            v = ValueMetaFactory.cloneValueMeta( v, metaChange.getType() );
          } else {
            v = ValueMetaFactory.createValueMeta( v.getName(), metaChange.getType() );
          }

          // This is now a copy, replace it in the row!
          //
          inputRowMeta.setValueMeta( idx, v );

          // This also moves the data to normal storage type
          //
          v.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
        }
        if ( metaChange.getLength() != UNDEFINED ) {
          v.setLength( metaChange.getLength() );
          v.setOrigin( name );
        }
        if ( metaChange.getPrecision() != UNDEFINED ) {
          v.setPrecision( metaChange.getPrecision() );
          v.setOrigin( name );
        }
        if ( metaChange.getStorageType() >= 0 ) {
          v.setStorageType( metaChange.getStorageType() );
          v.setOrigin( name );
        }
        if ( !Utils.isEmpty( metaChange.getConversionMask() ) ) {
          v.setConversionMask( metaChange.getConversionMask() );
          v.setOrigin( name );
        }

        v.setDateFormatLenient( metaChange.isDateFormatLenient() );
        v.setDateFormatLocale( EnvUtil.createLocale( metaChange.getDateFormatLocale() ) );
        v.setDateFormatTimeZone( EnvUtil.createTimeZone( metaChange.getDateFormatTimeZone() ) );
        v.setLenientStringToNumber( metaChange.isLenientStringToNumber() );

        if ( !Utils.isEmpty( metaChange.getEncoding() ) ) {
          v.setStringEncoding( metaChange.getEncoding() );
          v.setOrigin( name );
        }
        if ( !Utils.isEmpty( metaChange.getDecimalSymbol() ) ) {
          v.setDecimalSymbol( metaChange.getDecimalSymbol() );
          v.setOrigin( name );
        }
        if ( !Utils.isEmpty( metaChange.getGroupingSymbol() ) ) {
          v.setGroupingSymbol( metaChange.getGroupingSymbol() );
          v.setOrigin( name );
        }
        if ( !Utils.isEmpty( metaChange.getCurrencySymbol() ) ) {
          v.setCurrencySymbol( metaChange.getCurrencySymbol() );
          v.setOrigin( name );
        }
      }
    }
  }
}
 

@Override
public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean useAutoinc,
                                  boolean addFieldName, boolean addCr ) {
  String retval = "";

  String fieldname = v.getName();
  int length = v.getLength();
  // Unused in vertica
  // int precision = v.getPrecision();

  if ( addFieldName ) {
    retval += fieldname + " ";
  }

  int type = v.getType();
  switch ( type ) {
    case ValueMetaInterface.TYPE_DATE:
    case ValueMetaInterface.TYPE_TIMESTAMP:
      retval += "TIMESTAMP";
      break;
    case ValueMetaInterface.TYPE_BOOLEAN:
      retval += "BOOLEAN";
      break;
    case ValueMetaInterface.TYPE_NUMBER:
    case ValueMetaInterface.TYPE_BIGNUMBER:
      retval += "FLOAT";
      break;
    case ValueMetaInterface.TYPE_INTEGER:
      retval += "INTEGER";
      break;
    case ValueMetaInterface.TYPE_STRING:
      retval += ( length < 1 ) ? "VARCHAR" : "VARCHAR(" + length + ")";
      break;
    case ValueMetaInterface.TYPE_BINARY:
      retval += ( length < 1 ) ? "VARBINARY" : "VARBINARY(" + length + ")";
      break;
    default:
      retval += " UNKNOWN";
      break;
  }

  if ( addCr ) {
    retval += Const.CR;
  }

  return retval;
}
 

@Override
public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean useAutoinc,
                                  boolean addFieldName, boolean addCr ) {
  String retval = "";

  String fieldname = v.getName();
  int length = v.getLength();
  int precision = v.getPrecision();

  if ( addFieldName ) {
    if ( Const.indexOfString( fieldname, getReservedWords() ) >= 0 ) {
      retval += getStartQuote() + fieldname + getEndQuote();
    } else {
      retval += fieldname + " ";
    }
  }

  int type = v.getType();
  switch ( type ) {
    case ValueMetaInterface.TYPE_TIMESTAMP:
    case ValueMetaInterface.TYPE_DATE:
      retval += "TIMESTAMP";
      break;
    case ValueMetaInterface.TYPE_BOOLEAN:
      if ( supportsBooleanDataType() ) {
        retval += "BIT";
      } else {
        retval += "CHAR(1)";
      }
      break;
    case ValueMetaInterface.TYPE_NUMBER:
    case ValueMetaInterface.TYPE_INTEGER:
    case ValueMetaInterface.TYPE_BIGNUMBER:
      if ( fieldname.equalsIgnoreCase( tk ) || // Technical key
        fieldname.equalsIgnoreCase( pk ) // Primary key
      ) {
        retval += "BIGINT NOT NULL PRIMARY KEY";
      } else {
        if ( length > 0 ) {
          if ( precision > 0 || length > 18 ) {
            retval += "DECIMAL(" + length;
            if ( precision > 0 ) {
              retval += ", " + precision;
            }
            retval += ")";
          } else {
            if ( length > 9 ) {
              retval += "BIGINT";
            } else {
              if ( length < 5 ) {
                retval += "SMALLINT";
              } else {
                retval += "INTEGER";
              }
            }
          }
        } else {
          retval += "DOUBLE";
        }
      }
      break;
    case ValueMetaInterface.TYPE_STRING:
      if ( length < 32720 ) {
        retval += "VARCHAR";
        if ( length > 0 ) {
          retval += "(" + length + ")";
        } else {
          retval += "(8000)"; // Maybe use some default DB String length?
        }
      } else {
        retval += "BLOB SUB_TYPE TEXT";
      }
      break;
    case ValueMetaInterface.TYPE_BINARY:
      retval += "BLOB";
      break;
    default:
      retval += "UNKNOWN";
      break;
  }

  if ( addCr ) {
    retval += Const.CR;
  }

  return retval;
}
 

public static final List<Column> getColumns( RowMetaInterface row ) {
  List<Column> list = new ArrayList<Column>();

  for ( int i = 0; i < row.size(); i++ ) {
    ValueMetaInterface value = row.getValueMeta( i );

    Column column = new Column();
    column.setName( value.getName() );

    int length = value.getLength();

    switch ( value.getType() ) {
      case ValueMetaInterface.TYPE_INTEGER:
        if ( length < 3 ) {
          column.setType( DataType.BYTE );
          length = DataType.BYTE.getFixedSize();
        } else {
          if ( length < 5 ) {
            column.setType( DataType.INT );
            length = DataType.INT.getFixedSize();
          } else {
            column.setType( DataType.LONG );
            length = DataType.LONG.getFixedSize();
          }
        }
        break;
      case ValueMetaInterface.TYPE_NUMBER:
        column.setType( DataType.DOUBLE );
        length = DataType.DOUBLE.getFixedSize();
        break;
      case ValueMetaInterface.TYPE_DATE:
        column.setType( DataType.SHORT_DATE_TIME );
        length = DataType.SHORT_DATE_TIME.getFixedSize();
        break;
      case ValueMetaInterface.TYPE_STRING:
        if ( length < 255 ) {
          column.setType( DataType.TEXT );
          length *= DataType.TEXT.getUnitSize();
        } else {
          column.setType( DataType.MEMO );
          length *= DataType.MEMO.getUnitSize();
        }
        break;
      case ValueMetaInterface.TYPE_BINARY:
        column.setType( DataType.BINARY );
        break;
      case ValueMetaInterface.TYPE_BOOLEAN:
        column.setType( DataType.BOOLEAN );
        length = DataType.BOOLEAN.getFixedSize();
        break;
      case ValueMetaInterface.TYPE_BIGNUMBER:
        column.setType( DataType.NUMERIC );
        length = DataType.NUMERIC.getFixedSize();
        break;
      default:
        break;
    }

    if ( length >= 0 ) {
      column.setLength( (short) length );
    }
    if ( value.getPrecision() >= 1 && value.getPrecision() <= 28 ) {
      column.setPrecision( (byte) value.getPrecision() );
    }

    list.add( column );
  }

  return list;
}
 

@Override
public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean useAutoinc,
                                  boolean addFieldName, boolean addCr ) {
  String retval = "";

  String fieldname = v.getName();
  int length = v.getLength();
  int precision = v.getPrecision();

  if ( addFieldName ) {
    retval += fieldname + " ";
  }

  int type = v.getType();
  switch ( type ) {
    case ValueMetaInterface.TYPE_TIMESTAMP:
    case ValueMetaInterface.TYPE_DATE:
      retval += "TIMESTAMP";
      break;
    case ValueMetaInterface.TYPE_BOOLEAN:
      retval += "CHAR(1)";
      break;
    case ValueMetaInterface.TYPE_NUMBER:
    case ValueMetaInterface.TYPE_INTEGER:
    case ValueMetaInterface.TYPE_BIGNUMBER:
      if ( fieldname.equalsIgnoreCase( tk ) ) { // Technical & primary key : see at bottom
        retval += "DECIMAL";
      } else {
        if ( length < 0 || precision < 0 ) {
          retval += "DOUBLE";
        } else if ( precision > 0 || length > 9 ) {
          retval += "DECIMAL(" + length;
          if ( precision > 0 ) {
            retval += ", " + precision;
          }
          retval += ")";
        } else {
          // Precision == 0 && length<=9
          retval += "INT";
        }
      }
      break;
    case ValueMetaInterface.TYPE_STRING: // CLOBs are just VARCHAR in the Cache database: can be very large!
      retval += "VARCHAR";
      if ( length > 0 ) {
        retval += "(" + length + ")";
      }
      break;
    default:
      retval += " UNKNOWN";
      break;
  }

  if ( addCr ) {
    retval += Const.CR;
  }

  return retval;
}
 

@Override
public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean useAutoinc,
                                  boolean addFieldName, boolean addCr ) {
  StringBuilder retval = new StringBuilder( 128 );

  String fieldname = v.getName();
  int length = v.getLength();
  int precision = v.getPrecision();

  if ( addFieldName ) {
    retval.append( fieldname ).append( ' ' );
  }

  int type = v.getType();
  switch ( type ) {
    case ValueMetaInterface.TYPE_TIMESTAMP:
    case ValueMetaInterface.TYPE_DATE:
      retval.append( "TIMESTAMP" );
      break;
    case ValueMetaInterface.TYPE_BOOLEAN:
      if ( supportsBooleanDataType() ) {
        retval.append( "BOOLEAN" );
      } else {
        retval.append( "CHAR(1)" );
      }
      break;
    case ValueMetaInterface.TYPE_NUMBER:
    case ValueMetaInterface.TYPE_INTEGER:
    case ValueMetaInterface.TYPE_BIGNUMBER:
      if ( fieldname.equalsIgnoreCase( tk ) || // Technical key
          fieldname.equalsIgnoreCase( pk ) // Primary key
      ) {
        retval.append( "BIGINT GENERATED BY DEFAULT AS IDENTITY(START WITH 0, INCREMENT BY 1) PRIMARY KEY" );
      } else {
        if ( length > 0 ) {
          if ( precision > 0 || length > 18 ) {
            retval.append( "NUMERIC(" ).append( length ).append( ", " ).append( precision ).append( ')' );
          } else {
            if ( length > 9 ) {
              retval.append( "BIGINT" );
            } else {
              if ( length < 5 ) {
                retval.append( "SMALLINT" );
              } else {
                retval.append( "INTEGER" );
              }
            }
          }

        } else {
          retval.append( "DOUBLE PRECISION" );
        }
      }
      break;
    case ValueMetaInterface.TYPE_STRING:
      if ( length >= DatabaseMeta.CLOB_LENGTH ) {
        retval.append( "LONGVARCHAR" );
      } else {
        retval.append( "VARCHAR" );
        if ( length > 0 ) {
          retval.append( '(' ).append( length );
        } else {
          retval.append( '(' ); // Maybe use some default DB String length?
        }
        retval.append( ')' );
      }
      break;
    default:
      retval.append( " UNKNOWN" );
      break;
  }

  if ( addCr ) {
    retval.append( Const.CR );
  }

  return retval.toString();
}
 

@Override public TableRow apply( KettleRow inputRow ) {

    try {
      if ( rowMeta == null ) {
        readCounter = Metrics.counter( "read", counterName );
        outputCounter = Metrics.counter( "output", counterName );
        errorCounter = Metrics.counter( "error", counterName );

        // Initialize Kettle Beam
        //
        BeamKettle.init( stepPluginClasses, xpPluginClasses );
        rowMeta = JsonRowMeta.fromJson( rowMetaJson );

        simpleDateFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss.SSS" );
        Metrics.counter( "init", counterName ).inc();
      }

      readCounter.inc();

      TableRow tableRow = new TableRow();
      for (int i=0;i<rowMeta.size();i++) {
        ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
        Object valueData = inputRow.getRow()[i];
        if (!valueMeta.isNull( valueData )) {
          switch ( valueMeta.getType() ) {
            case ValueMetaInterface.TYPE_STRING: tableRow.put( valueMeta.getName(), valueMeta.getString( valueData ) ); break;
            case ValueMetaInterface.TYPE_INTEGER: tableRow.put( valueMeta.getName(), valueMeta.getInteger( valueData ) ); break;
            case ValueMetaInterface.TYPE_DATE:
              Date date = valueMeta.getDate( valueData );
              String formattedDate = simpleDateFormat.format( date );
              tableRow.put( valueMeta.getName(), formattedDate);
              break;
            case ValueMetaInterface.TYPE_BOOLEAN: tableRow.put( valueMeta.getName(), valueMeta.getBoolean( valueData ) ); break;
            case ValueMetaInterface.TYPE_NUMBER: tableRow.put( valueMeta.getName(), valueMeta.getNumber( valueData ) ); break;
            default:
              throw new RuntimeException( "Data type conversion from Kettle to BigQuery TableRow not supported yet: " +valueMeta.toString());
          }
        }
      }

      // Pass the row to the process context
      //
      outputCounter.inc();

      return tableRow;

    } catch ( Exception e ) {
      errorCounter.inc();
      LOG.info( "Conversion error KettleRow to BigQuery TableRow : " + e.getMessage() );
      throw new RuntimeException( "Error converting KettleRow to BigQuery TableRow", e );
    }
  }
 

@Override public PDone expand( PCollection<KettleRow> input ) {

    try {
      // Only initialize once on this node/vm
      //
      BeamKettle.init( stepPluginClasses, xpPluginClasses );

      // Inflate the metadata on the node where this is running...
      //
      RowMetaInterface rowMeta = JsonRowMeta.fromJson( rowMetaJson );


      // Which table do we write to?
      //
      TableReference tableReference = new TableReference();
      if ( StringUtils.isNotEmpty( projectId ) ) {
        tableReference.setProjectId( projectId );
      }
      tableReference.setDatasetId( datasetId );
      tableReference.setTableId( tableId );

      TableSchema tableSchema = new TableSchema();
      List<TableFieldSchema> schemaFields = new ArrayList<>();
      for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
        TableFieldSchema schemaField = new TableFieldSchema();
        schemaField.setName( valueMeta.getName() );
        switch(valueMeta.getType()){
          case ValueMetaInterface.TYPE_STRING: schemaField.setType( "STRING" ); break;
          case ValueMetaInterface.TYPE_INTEGER: schemaField.setType( "INTEGER" ); break;
          case ValueMetaInterface.TYPE_DATE: schemaField.setType( "DATETIME" ); break;
          case ValueMetaInterface.TYPE_BOOLEAN: schemaField.setType( "BOOLEAN" ); break;
          case ValueMetaInterface.TYPE_NUMBER: schemaField.setType( "FLOAT" ); break;
          default:
            throw new RuntimeException( "Conversion from Kettle value "+valueMeta.toString()+" to BigQuery TableRow isn't supported yet" );
        }
        schemaFields.add(schemaField);
      }
      tableSchema.setFields( schemaFields );

      SerializableFunction<KettleRow, TableRow> formatFunction = new KettleToBQTableRowFn( stepname, rowMetaJson, stepPluginClasses, xpPluginClasses );

      BigQueryIO.Write.CreateDisposition createDisposition;
      if (createIfNeeded) {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED;
      }  else {
        createDisposition = BigQueryIO.Write.CreateDisposition.CREATE_NEVER;
      }

      BigQueryIO.Write.WriteDisposition writeDisposition;
      if (truncateTable) {
        writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
      } else {
        if (failIfNotEmpty) {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_EMPTY;
        } else {
          writeDisposition = BigQueryIO.Write.WriteDisposition.WRITE_APPEND;
        }
      }

      BigQueryIO.Write<KettleRow> bigQueryWrite = BigQueryIO
        .<KettleRow>write()
        .to( tableReference )
        .withSchema( tableSchema )
        .withCreateDisposition( createDisposition )
        .withWriteDisposition( writeDisposition )
        .withFormatFunction( formatFunction );

      // TODO: pass the results along the way at some point
      //
      input.apply( stepname, bigQueryWrite );

      // End of the line
      //
      return PDone.in( input.getPipeline() );

    } catch ( Exception e ) {
      numErrors.inc();
      LOG.error( "Error in Beam BigQuery output transform", e );
      throw new RuntimeException( "Error in Beam BigQuery output transform", e );
    }
  }
 

@Override
public String getSQLValue( ValueMetaInterface valueMeta, Object valueData, String dateFormat ) throws KettleValueException {

  StringBuilder ins = new StringBuilder();

  if ( valueMeta.isNull( valueData ) ) {
    ins.append( "null" );
  } else {
    // Normal cases...
    //
    switch ( valueMeta.getType() ) {
      case ValueMetaInterface.TYPE_BOOLEAN:
      case ValueMetaInterface.TYPE_STRING:
        String string = valueMeta.getString( valueData );
        // Have the database dialect do the quoting.
        // This also adds the single quotes around the string (thanks to PostgreSQL)
        //
        string = quoteSQLString( string );
        ins.append( string );
        break;
      case ValueMetaInterface.TYPE_DATE:
        Date date = valueMeta.getDate( valueData );

        if ( Utils.isEmpty( dateFormat ) ) {
          ins.append( "'" + valueMeta.getString( valueData ) + "'" );
        } else {
          try {
            java.text.SimpleDateFormat formatter = new java.text.SimpleDateFormat( dateFormat );
            ins.append( "'" + formatter.format( date ) + "'" );
          } catch ( Exception e ) {
            throw new KettleValueException( "Error : ", e );
          }
        }
        break;
      default:
        ins.append( valueMeta.getString( valueData ) );
        break;
    }
  }

  return ins.toString();
}