Java Code Examples for org.apache.kudu.client.PartialRow#addString()

@Override
public CompletableFuture<?> appendPage(Page page)
{
    for (int position = 0; position < page.getPositionCount(); position++) {
        Upsert upsert = table.newUpsert();
        PartialRow row = upsert.getRow();
        int start = 0;
        if (generateUUID) {
            String id = format("%s-%08x", uuid, nextSubId++);
            row.addString(0, id);
            start = 1;
        }

        for (int channel = 0; channel < page.getChannelCount(); channel++) {
            appendColumn(row, page, position, channel, channel + start);
        }

        try {
            session.apply(upsert);
        }
        catch (KuduException e) {
            throw new RuntimeException(e);
        }
    }
    return NOT_BLOCKED;
}
 

protected void insertRowInTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final KuduTable table = client.openTable(tableName);

        final Insert insert = table.newInsert();
        final PartialRow row = insert.getRow();

        row.addInt("id", ThreadLocalRandom.current().nextInt(1, 99));
        row.addString("title", "Mr.");
        row.addString("name", "Samuel");
        row.addString("lastname", "Smith");
        row.addString("address", "4359  Plainfield Avenue");

        client.newSession().apply(insert);
    }
}
 

@Override
public CompletableFuture<?> appendPage(Page page) {
    for (int position = 0; position < page.getPositionCount(); position++) {
        Upsert upsert = table.newUpsert();
        PartialRow row = upsert.getRow();
        int start = 0;
        if (generateUUID) {
            String id = String.format("%s-%08x", uuid, nextSubId++);
            row.addString(0, id);
            start = 1;
        }

        for (int channel = 0; channel < page.getChannelCount(); channel++) {
            appendColumn(row, page, position, channel, channel + start);
        }

        try {
            session.apply(upsert);
        } catch (KuduException e) {
            throw new RuntimeException(e);
        }
    }
    return NOT_BLOCKED;
}
 

private void appendColumn(PartialRow row, Page page, int position, int channel, int destChannel) {
    Block block = page.getBlock(channel);
    Type type = columnTypes.get(destChannel);
    if (block.isNull(position)) {
        row.setNull(destChannel);
    } else if (TIMESTAMP.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position) * 1000);
    } else if (REAL.equals(type)) {
        row.addFloat(destChannel, intBitsToFloat((int) type.getLong(block, position)));
    } else if (BIGINT.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position));
    } else if (INTEGER.equals(type)) {
        row.addInt(destChannel, (int) type.getLong(block, position));
    } else if (SMALLINT.equals(type)) {
        row.addShort(destChannel, (short) type.getLong(block, position));
    } else if (TINYINT.equals(type)) {
        row.addByte(destChannel, (byte) type.getLong(block, position));
    } else if (BOOLEAN.equals(type)) {
        row.addBoolean(destChannel, type.getBoolean(block, position));
    } else if (DOUBLE.equals(type)) {
        row.addDouble(destChannel, type.getDouble(block, position));
    } else if (isVarcharType(type)) {
        Type originalType = originalColumnTypes.get(destChannel);
        if (DATE.equals(originalType)) {
            SqlDate date = (SqlDate) originalType.getObjectValue(connectorSession, block, position);
            LocalDateTime ldt = LocalDateTime.ofEpochSecond(TimeUnit.DAYS.toSeconds(date.getDays()), 0, ZoneOffset.UTC);
            byte[] bytes = ldt.format(DateTimeFormatter.ISO_LOCAL_DATE).getBytes(Charsets.UTF_8);
            row.addStringUtf8(destChannel, bytes);
        } else {
            row.addString(destChannel, type.getSlice(block, position).toStringUtf8());
        }
    } else if (VARBINARY.equals(type)) {
        row.addBinary(destChannel, type.getSlice(block, position).toByteBuffer());
    } else if (type instanceof DecimalType) {
        SqlDecimal sqlDecimal = (SqlDecimal) type.getObjectValue(connectorSession, block, position);
        row.addDecimal(destChannel, sqlDecimal.toBigDecimal());
    } else {
        throw new UnsupportedOperationException("Type is not supported: " + type);
    }
}
 

@Override
public Operation apply(TableAndRecord<Long> input) {
  Upsert upsert = input.getTable().newUpsert();
  PartialRow row = upsert.getRow();
  row.addLong(COL_ID, input.getRecord());
  row.addString(COL_NAME, input.getRecord() + ": name");
  return upsert;
}
 

public void addTestDataRows(int numRowsInEachPartition) throws Exception
{
  int intRowKeyStepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
  int splitBoundaryForIntRowKey = intRowKeyStepsize;
  int[] inputrowkeyPartitionEntries = new int[SPLIT_COUNT_FOR_INT_ROW_KEY + 1];
  // setting the int keys that will fall in the range of all partitions
  for ( int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
    inputrowkeyPartitionEntries[i] = splitBoundaryForIntRowKey + 3; // 3 to fall into the partition next to boundary
    splitBoundaryForIntRowKey += intRowKeyStepsize;
  }
  inputrowkeyPartitionEntries[SPLIT_COUNT_FOR_INT_ROW_KEY] = splitBoundaryForIntRowKey + 3;
  AbstractKuduPartitionScanner<UnitTestTablePojo,InputOperatorControlTuple> scannerForAddingRows =
      unitTestStepwiseScanInputOperator.getScanner();
  ApexKuduConnection aCurrentConnection = scannerForAddingRows.getConnectionPoolForThreads().get(0);
  KuduSession aSessionForInserts = aCurrentConnection.getKuduClient().newSession();
  KuduTable currentTable = aCurrentConnection.getKuduTable();
  long seedValueForTimestampRowKey = 0L; // constant to allow for data landing on first partition for unit tests
  for ( int i = 0; i <= SPLIT_COUNT_FOR_INT_ROW_KEY; i++) { // range key iterator
    int intRowKeyBaseValue = inputrowkeyPartitionEntries[i] + i;
    for ( int k = 0; k < 2; k++) { // hash key iterator . The table defines two hash partitions
      long timestampRowKeyValue = seedValueForTimestampRowKey + k; // to avoid spilling to another tablet
      String stringRowKeyValue = "" + timestampRowKeyValue + k; // to avoid spilling to another tablet randomly
      for ( int y = 0; y < numRowsInEachPartition; y++) {
        Upsert aNewRow = currentTable.newUpsert();
        PartialRow rowValue  = aNewRow.getRow();
        // Start assigning row keys below the current split boundary.
        rowValue.addInt("introwkey",intRowKeyBaseValue - y - 1);
        rowValue.addString("stringrowkey",stringRowKeyValue);
        rowValue.addLong("timestamprowkey",timestampRowKeyValue);
        rowValue.addLong("longdata",(seedValueForTimestampRowKey + y));
        rowValue.addString("stringdata", ("" + seedValueForTimestampRowKey + y));
        OperationResponse response = aSessionForInserts.apply(aNewRow);
      }
    }
  }
  List<OperationResponse> insertResponse = aSessionForInserts.flush();
  aSessionForInserts.close();
  Thread.sleep(2000); // Sleep to allow for scans to complete
}
 

private void appendColumn(PartialRow row, Page page, int position, int channel, int destChannel)
{
    Block block = page.getBlock(channel);
    Type type = columnTypes.get(destChannel);
    if (block.isNull(position)) {
        row.setNull(destChannel);
    }
    else if (TIMESTAMP.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position) * 1000);
    }
    else if (REAL.equals(type)) {
        row.addFloat(destChannel, intBitsToFloat(toIntExact(type.getLong(block, position))));
    }
    else if (BIGINT.equals(type)) {
        row.addLong(destChannel, type.getLong(block, position));
    }
    else if (INTEGER.equals(type)) {
        row.addInt(destChannel, toIntExact(type.getLong(block, position)));
    }
    else if (SMALLINT.equals(type)) {
        row.addShort(destChannel, Shorts.checkedCast(type.getLong(block, position)));
    }
    else if (TINYINT.equals(type)) {
        row.addByte(destChannel, SignedBytes.checkedCast(type.getLong(block, position)));
    }
    else if (BOOLEAN.equals(type)) {
        row.addBoolean(destChannel, type.getBoolean(block, position));
    }
    else if (DOUBLE.equals(type)) {
        row.addDouble(destChannel, type.getDouble(block, position));
    }
    else if (isVarcharType(type)) {
        Type originalType = originalColumnTypes.get(destChannel);
        if (DATE.equals(originalType)) {
            SqlDate date = (SqlDate) originalType.getObjectValue(connectorSession, block, position);
            LocalDateTime ldt = LocalDateTime.ofEpochSecond(TimeUnit.DAYS.toSeconds(date.getDays()), 0, ZoneOffset.UTC);
            byte[] bytes = ldt.format(DateTimeFormatter.ISO_LOCAL_DATE).getBytes(StandardCharsets.UTF_8);
            row.addStringUtf8(destChannel, bytes);
        }
        else {
            row.addString(destChannel, type.getSlice(block, position).toStringUtf8());
        }
    }
    else if (VARBINARY.equals(type)) {
        row.addBinary(destChannel, type.getSlice(block, position).toByteBuffer());
    }
    else if (type instanceof DecimalType) {
        SqlDecimal sqlDecimal = (SqlDecimal) type.getObjectValue(connectorSession, block, position);
        row.addDecimal(destChannel, sqlDecimal.toBigDecimal());
    }
    else {
        throw new UnsupportedOperationException("Type is not supported: " + type);
    }
}
 

private static void setColumnValue(PartialRow partialRow, int idx, Object obj, Type type, String name)
{
    Number n;

    switch (type) {
        case STRING:
            if (obj instanceof String) {
                partialRow.addString(idx, (String) obj);
            }
            else {
                handleInvalidValue(name, type, obj);
            }
            break;
        case INT64:
            n = toNumber(obj, type, name);
            partialRow.addLong(idx, n.longValue());
            break;
        case INT32:
            n = toNumber(obj, type, name);
            partialRow.addInt(idx, n.intValue());
            break;
        case INT16:
            n = toNumber(obj, type, name);
            partialRow.addShort(idx, n.shortValue());
            break;
        case INT8:
            n = toNumber(obj, type, name);
            partialRow.addByte(idx, n.byteValue());
            break;
        case DOUBLE:
            n = toNumber(obj, type, name);
            partialRow.addDouble(idx, n.doubleValue());
            break;
        case FLOAT:
            n = toNumber(obj, type, name);
            partialRow.addFloat(idx, n.floatValue());
            break;
        case UNIXTIME_MICROS:
            long l = toUnixTimeMicros(obj, type, name);
            partialRow.addLong(idx, l);
            break;
        case BOOL:
            boolean b = toBoolean(obj, type, name);
            partialRow.addBoolean(idx, b);
            break;
        case BINARY:
            byte[] bytes = toByteArray(obj, type, name);
            partialRow.addBinary(idx, bytes);
            break;
        default:
            handleInvalidValue(name, type, obj);
            break;
    }
}
 

private void insert(KuduTable kuduTable, Operation operation, Record record, List<String> fieldNames){
    PartialRow row = operation.getRow();
    Schema colSchema = kuduTable.getSchema();

    for (String colName : fieldNames) {
        int colIdx = this.getColumnIndex(colSchema, colName);
        if (colIdx != -1) {
            Type colType = colSchema.getColumnByIndex(colIdx).getType();

            switch (colType.getDataType()) {
                case BOOL:
                    row.addBoolean(colIdx, record.getAsBoolean(colName));
                    break;
                case FLOAT:
                    row.addFloat(colIdx, record.getAsFloat(colName));
                    break;
                case DOUBLE:
                    row.addDouble(colIdx, record.getAsDouble(colName));
                    break;
                case BINARY:
                    row.addBinary(colIdx, record.getAsString(colName).getBytes());
                    break;
                case INT8:
                case INT16:
                    short temp = (short)record.getAsInt(colName).intValue();
                    row.addShort(colIdx, temp);
                case INT32:
                    row.addInt(colIdx, record.getAsInt(colName));
                    break;
                case INT64:
                    row.addLong(colIdx, record.getAsLong(colName));
                    break;
                case STRING:
                    row.addString(colIdx, record.getAsString(colName));
                    break;
                default:
                    throw new IllegalStateException(String.format("unknown column type %s", colType));
            }
        }
    }
}
 

private void fillSchemaRow(PartialRow row, String schemaName) {
    row.addString(0, tenantPrefix);
    row.addString(1, schemaName);
}
 

private static void setColumnValue(PartialRow partialRow, int idx, Object obj, Type type, String name) {
    Number n;

    switch (type) {
        case STRING:
            if (obj instanceof String) {
                partialRow.addString(idx, (String) obj);
            } else {
                handleInvalidValue(name, type, obj);
            }
            break;
        case INT64:
            n = toNumber(obj, type, name);
            partialRow.addLong(idx, n.longValue());
            break;
        case INT32:
            n = toNumber(obj, type, name);
            partialRow.addInt(idx, n.intValue());
            break;
        case INT16:
            n = toNumber(obj, type, name);
            partialRow.addShort(idx, n.shortValue());
            break;
        case INT8:
            n = toNumber(obj, type, name);
            partialRow.addByte(idx, n.byteValue());
            break;
        case DOUBLE:
            n = toNumber(obj, type, name);
            partialRow.addDouble(idx, n.doubleValue());
            break;
        case FLOAT:
            n = toNumber(obj, type, name);
            partialRow.addFloat(idx, n.floatValue());
            break;
        case UNIXTIME_MICROS:
            long l = toUnixTimeMicros(obj, type, name);
            partialRow.addLong(idx, l);
            break;
        case BOOL:
            boolean b = toBoolean(obj, type, name);
            partialRow.addBoolean(idx, b);
            break;
        case BINARY:
            byte[] bytes = toByteArray(obj, type, name);
            partialRow.addBinary(idx, bytes);
            break;
        default:
            handleInvalidValue(name, type, obj);
            break;
    }
}
 

private void recordToRow(Record record, PartialRow row, String fieldName, String column, int operation) throws OnRecordErrorException {
  Field.Type type = columnsToFieldTypes.get(column);
  ColumnSchema columnSchema = schema.getColumn(column);
  if (record.has(fieldName)) {
    Field field = record.get(fieldName);
    if (field.getValue() == null) {
      if (!columnSchema.isNullable()) {
        throw new OnRecordErrorException(record, Errors.KUDU_06, column, fieldName);
      }
      row.setNull(column);
    } else {
      try {
        switch (type) {
          case BOOLEAN:
            row.addBoolean(column, field.getValueAsBoolean());
            break;
          case BYTE:
            row.addByte(column, field.getValueAsByte());
            break;
          case SHORT:
            row.addShort(column, field.getValueAsShort());
            break;
          case INTEGER:
            row.addInt(column, field.getValueAsInteger());
            break;
          case LONG:
            if (columnSchema.getType() == Type.UNIXTIME_MICROS) {
              // Convert millisecond to microsecond
              row.addLong(column, field.getValueAsLong() * 1000);
            } else {
              row.addLong(column, field.getValueAsLong());
            }
            break;
          case FLOAT:
            row.addFloat(column, field.getValueAsFloat());
            break;
          case DOUBLE:
            row.addDouble(column, field.getValueAsDouble());
            break;
          case STRING:
            row.addString(column, field.getValueAsString());
            break;
          case BYTE_ARRAY:
            row.addBinary(column, field.getValueAsByteArray());
            break;
          case DECIMAL:
            row.addDecimal(column, field.getValueAsDecimal());
            break;
          default:
            throw new OnRecordErrorException(record, Errors.KUDU_04, fieldName, type.name());
        }
      } catch (IllegalArgumentException e) {
        throw new OnRecordErrorException(record, Errors.KUDU_09, fieldName, type.name(), e.toString(), e);
      }
    }
  } else {
    // SDC-5816.  do not null out columns in UPDATE or UPSERT mode.
    // if the columns are not specified - they should not be changed.
    if (operation == KuduOperationType.INSERT.code) {
      if (!columnSchema.isNullable()) {
        throw new OnRecordErrorException(record, Errors.KUDU_06, column, fieldName);
      }
      row.setNull(column);
    }
  }
}
 

@Before
public void init() throws Exception {
    testRunner = TestRunners.newTestRunner(SampleProcessor.class);
    testRunner.setValidateExpressionUsage(false);
    final String tableName = "table1";

    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build());
    columns.add(new ColumnSchema
            .ColumnSchemaBuilder("decimal", Type.DECIMAL)
            .typeAttributes(DecimalUtil.typeAttributes(DecimalUtil.MAX_DECIMAL64_PRECISION, 1))
            .build()
    );
    columns.add(new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("unixtime_micros", Type.UNIXTIME_MICROS).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varchar_3", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(3).build()
    ).build());
    Schema schema = new Schema(columns);

    CreateTableOptions opts = new CreateTableOptions().setRangePartitionColumns(Collections.singletonList("string"));
    client.createTable(tableName, schema, opts);

    KuduTable table = client.openTable(tableName);
    KuduSession session = client.newSession();

    Insert insert = table.newInsert();
    PartialRow row = insert.getRow();
    row.addString("string", "string1");
    row.addBinary("binary", "binary1".getBytes());
    row.addBoolean("bool",true);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",0.2);
    row.addFloat("float",0.3f);
    row.addByte("int8", (byte) 1);
    row.addShort("int16", (short) 2);
    row.addInt("int32",3);
    row.addLong("int64",4L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis));
    row.addVarchar("varchar_3", "SFO");
    session.apply(insert);

    insert = table.newInsert();
    row = insert.getRow();
    row.addString("string", "string2");
    row.addBinary("binary", "binary2".getBytes());
    row.addBoolean("bool",false);
    row.addDecimal("decimal", BigDecimal.valueOf(0.1));
    row.addDouble("double",1.2);
    row.addFloat("float",1.3f);
    row.addByte("int8", (byte) 11);
    row.addShort("int16", (short) 12);
    row.addInt("int32",13);
    row.addLong("int64",14L);
    row.addTimestamp("unixtime_micros", new Timestamp(nowMillis+(1000L * 60 * 60 * 24 * 365))); //+ 1 year
    row.addVarchar("varchar_3", "SJC");
    session.apply(insert);

    session.close();

    kuduLookupService = new KuduLookupService();
    testRunner.addControllerService("kuduLookupService", kuduLookupService);
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_MASTERS, "testLocalHost:7051");
    testRunner.setProperty(kuduLookupService, KuduLookupService.KUDU_REPLICA_SELECTION, KuduLookupService.LEADER_ONLY);
    testRunner.setProperty(kuduLookupService, KuduLookupService.TABLE_NAME, tableName);
    kuduLookupService.kuduClient = client;
}
 

@VisibleForTesting
protected void buildPartialRow(Schema schema, PartialRow row, Record record, List<String> fieldNames, Boolean ignoreNull, Boolean lowercaseFields) {
    for (String recordFieldName : fieldNames) {
        String colName = recordFieldName;
        if (lowercaseFields) {
            colName = colName.toLowerCase();
        }
        int colIdx = this.getColumnIndex(schema, colName);
        if (colIdx != -1) {
            ColumnSchema colSchema = schema.getColumnByIndex(colIdx);
            Type colType = colSchema.getType();

            if (record.getValue(recordFieldName) == null) {
                if (schema.getColumnByIndex(colIdx).isKey()) {
                    throw new IllegalArgumentException(String.format("Can't set primary key column %s to null ", colName));
                } else if(!schema.getColumnByIndex(colIdx).isNullable()) {
                    throw new IllegalArgumentException(String.format("Can't set column %s to null ", colName));
                }

                if (!ignoreNull) {
                    row.setNull(colName);
                }
            } else {
                Object value = record.getValue(recordFieldName);
                switch (colType) {
                    case BOOL:
                        row.addBoolean(colIdx, DataTypeUtils.toBoolean(value, recordFieldName));
                        break;
                    case INT8:
                        row.addByte(colIdx, DataTypeUtils.toByte(value, recordFieldName));
                        break;
                    case INT16:
                        row.addShort(colIdx,  DataTypeUtils.toShort(value, recordFieldName));
                        break;
                    case INT32:
                        row.addInt(colIdx,  DataTypeUtils.toInteger(value, recordFieldName));
                        break;
                    case INT64:
                        row.addLong(colIdx,  DataTypeUtils.toLong(value, recordFieldName));
                        break;
                    case UNIXTIME_MICROS:
                        DataType fieldType = record.getSchema().getDataType(recordFieldName).get();
                        Timestamp timestamp = DataTypeUtils.toTimestamp(record.getValue(recordFieldName),
                                () -> DataTypeUtils.getDateFormat(fieldType.getFormat()), recordFieldName);
                        row.addTimestamp(colIdx, timestamp);
                        break;
                    case STRING:
                        row.addString(colIdx, DataTypeUtils.toString(value, recordFieldName));
                        break;
                    case BINARY:
                        row.addBinary(colIdx, DataTypeUtils.toString(value, recordFieldName).getBytes());
                        break;
                    case FLOAT:
                        row.addFloat(colIdx, DataTypeUtils.toFloat(value, recordFieldName));
                        break;
                    case DOUBLE:
                        row.addDouble(colIdx, DataTypeUtils.toDouble(value, recordFieldName));
                        break;
                    case DECIMAL:
                        row.addDecimal(colIdx, new BigDecimal(DataTypeUtils.toString(value, recordFieldName)));
                        break;
                    case VARCHAR:
                        row.addVarchar(colIdx, DataTypeUtils.toString(value, recordFieldName));
                        break;
                    default:
                        throw new IllegalStateException(String.format("unknown column type %s", colType));
                }
            }
        }
    }
}