org.apache.kudu.client.KuduTable Java Examples

public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
    try {
        String rawName = schemaEmulation.toRawName(tableMetadata.getTable());
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }

        if (!schemaEmulation.existsSchema(client, tableMetadata.getTable().getSchemaName())) {
            throw new SchemaNotFoundException(tableMetadata.getTable().getSchemaName());
        }

        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

@Override
public List<Operation> createOperations(T input, KuduTable table) {
    Optional<Operation> operationOpt = createBaseOperation(input, table);
    if (!operationOpt.isPresent()) {
        return Collections.emptyList();
    }

    Operation operation = operationOpt.get();
    PartialRow partialRow = operation.getRow();

    for (int i = 0; i < columnNames.length; i++) {
        partialRow.addObject(columnNames[i], getField(input, i));
    }

    return Collections.singletonList(operation);
}
 

@Test
public void test01() {
    List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
    KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
        .defaultSocketReadTimeoutMs(30000)
        .defaultAdminOperationTimeoutMs(60000)
        .build();
    try {
        List<String> tablesList = kuduClient.getTablesList().getTablesList();
        System.out.println(tablesList.toString());
        KuduTable web_white_list = kuduClient.openTable("web_white_list");

    } catch (KuduException e) {
        e.printStackTrace();
    }
}
 

/**
 * Scans the metadata for the kudu table that this operator is scanning for and
 * returns back the mapping for the kudu column name to the ColumnSchema metadata definition.
 * Note that the Kudu columns names are case sensitive.
 * @return A Map with Kudu column names as keys and value as the Column Definition.
 * @throws Exception
 */
private Map<String,ColumnSchema> buildColumnSchemaForTable() throws Exception
{
  if (kuduColNameToSchemaMapping == null) {
    ApexKuduConnection connectionForMetaDataScan = apexKuduConnectionInfo.build();
    KuduTable table = connectionForMetaDataScan.getKuduTable();
    List<ColumnSchema> tableColumns =  table.getSchema().getColumns();
    connectionForMetaDataScan.close();
    Map<String,ColumnSchema> columnSchemaMap = new HashMap<>();
    for (ColumnSchema aColumn: tableColumns) {
      columnSchemaMap.put(aColumn.getName(),aColumn);
    }
    kuduColNameToSchemaMapping = columnSchemaMap;
  }
  return kuduColNameToSchemaMapping;
}
 

protected void insertRowInTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final KuduTable table = client.openTable(tableName);

        final Insert insert = table.newInsert();
        final PartialRow row = insert.getRow();

        row.addInt("id", ThreadLocalRandom.current().nextInt(1, 99));
        row.addString("title", "Mr.");
        row.addString("name", "Samuel");
        row.addString("lastname", "Smith");
        row.addString("address", "4359  Plainfield Avenue");

        client.newSession().apply(insert);
    }
}
 

protected KuduDataIndexRead(
    final short adapterId,
    final byte[][] dataIds,
    final KuduTable table,
    final KuduOperations operations,
    final boolean visibilityEnabled,
    final Predicate<GeoWaveRow> filter) {
  this.adapterId = adapterId;
  this.dataIds = dataIds;
  this.table = table;
  this.schema = table.getSchema();
  this.operations = operations;
  this.visibilityEnabled = visibilityEnabled;
  this.filter = filter;
  this.results = new ArrayList<>();
}
 

@Override
public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle connectorTableHandle) {
    KuduTableHandle tableHandle = fromConnectorTableHandle(session, connectorTableHandle);

    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnSchema> columns = schema.getColumns();
    List<String> columnNames = columns.stream().map(ColumnSchema::getName).collect(toImmutableList());
    List<Type> columnTypes = columns.stream()
            .map(TypeHelper::fromKuduColumn).collect(toImmutableList());

    return new KuduInsertTableHandle(
            connectorId,
            tableHandle.getSchemaTableName(),
            columnNames,
            columnTypes,
            table);
}
 

private static RangeBoundValue buildRangePartitionBound(KuduTable table, byte[] rangeKey)
{
    if (rangeKey.length == 0) {
        return null;
    }
    else {
        Schema schema = table.getSchema();
        PartitionSchema partitionSchema = table.getPartitionSchema();
        PartitionSchema.RangeSchema rangeSchema = partitionSchema.getRangeSchema();
        List<Integer> rangeColumns = rangeSchema.getColumns();

        final int numColumns = rangeColumns.size();

        PartialRow bound = KeyEncoderAccessor.decodeRangePartitionKey(schema, partitionSchema, rangeKey);

        ArrayList<Object> list = new ArrayList<>();
        for (int i = 0; i < numColumns; i++) {
            Object obj = toValue(schema, bound, rangeColumns.get(i));
            list.add(obj);
        }
        return new RangeBoundValue(list);
    }
}
 

@Override
public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception {
  List<Row> existingForFilters = Lists.newArrayList();

  if (!filters.iterator().hasNext()) {
    return existingForFilters;
  }

  KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME));
  KuduScanner scanner = scannerForFilters(filters, table);

  long startTime = System.nanoTime();
  while (scanner.hasMoreRows()) {
    for (RowResult rowResult : scanner.nextRows()) {
      Row existing = resultAsRow(rowResult, table);

      existingForFilters.add(existing);
    }
  }
  long endTime = System.nanoTime();
  if (hasAccumulators()) {
    accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0);
  }

  return existingForFilters;
}
 

private void createAndFillSchemasTable() throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets();
    ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING)
            .key(true).build();
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.setNumReplicas(1); // TODO config
    options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            fillSchemaRow(insert.getRow(), schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}
 

private KuduPageSink(
        ConnectorSession connectorSession,
        KuduClientSession clientSession,
        KuduTable table,
        KuduTableMapping mapping)
{
    requireNonNull(clientSession, "clientSession is null");
    this.connectorSession = connectorSession;
    this.columnTypes = mapping.getColumnTypes();
    this.originalColumnTypes = mapping.getOriginalColumnTypes();
    this.generateUUID = mapping.isGenerateUUID();

    this.table = table;
    this.session = clientSession.newSession();
    this.session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
    uuid = UUID.randomUUID().toString();
}
 

/**
 * 统计kudu表数据
 *
 * @param tableName
 * @return
 */
public long countRow(String tableName) {
    this.checkClient();
    long rowCount = 0L;
    try {
        KuduTable kuduTable = kuduClient.openTable(tableName);
        // 创建scanner扫描
        KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
        // 遍历数据
        while (scanner.hasMoreRows()) {
            while (scanner.nextRows().hasNext()) {
                rowCount++;
            }
        }
    } catch (KuduException e) {
        e.printStackTrace();
    }
    return rowCount;
}
 

@Override
public CatalogTable getTable(ObjectPath tablePath) throws TableNotExistException {
    checkNotNull(tablePath);

    if (!tableExists(tablePath)) {
        throw new TableNotExistException(getName(), tablePath);
    }

    String tableName = tablePath.getObjectName();

    try {
        KuduTable kuduTable = kuduClient.openTable(tableName);

        CatalogTableImpl table = new CatalogTableImpl(
                KuduTableUtils.kuduToFlinkSchema(kuduTable.getSchema()),
                createTableProperties(tableName, kuduTable.getSchema().getPrimaryKeyColumns()),
                tableName);

        return table;
    } catch (KuduException e) {
        throw new CatalogException(e);
    }
}
 

public static Map<String, Object> toMap(KuduTable table) {
    Map<String, Object> properties = new HashMap<>();

    LinkedHashMap<String, ColumnDesign> columns = getColumns(table);

    PartitionDesign partitionDesign = getPartitionDesign(table);

    List<RangePartition> rangePartitionList = getRangePartitionList(table, DEFAULT_DEADLINE);

    try {
        String columnDesignValue = mapper.writeValueAsString(columns);
        properties.put(COLUMN_DESIGN, columnDesignValue);

        String partitionDesignValue = mapper.writeValueAsString(partitionDesign);
        properties.put(PARTITION_DESIGN, partitionDesignValue);

        String partitionRangesValue = mapper.writeValueAsString(rangePartitionList);
        properties.put(RANGE_PARTITIONS, partitionRangesValue);

        properties.put(NUM_REPLICAS, table.getNumReplicas());

        return properties;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * Opens a Kudu TS instance on a Kudu cluster.
 *
 * @param kuduMasterAddressess list of "host:port" pair master addresses
 * @param name the name of the Kudu timeseries store. Multiple instances of
 *             Kudu TS can occupy the same Kudu cluster by using a different name.
 * @return the opened {@code KuduTS}.
 * @throws Exception on error
 */
public static KuduTS open(List<String> kuduMasterAddressess, String name) throws Exception {
  AsyncKuduClient client = new AsyncKuduClient.AsyncKuduClientBuilder(kuduMasterAddressess).build();

  Deferred<KuduTable> metricsDeferred = client.openTable(Tables.metricsTableName(name));
  Deferred<KuduTable> tagsetsDeferred = client.openTable(Tables.tagsetsTableName(name));
  Deferred<KuduTable> tagsDeferred = client.openTable(Tables.tagsTableName(name));
  KuduTable metricsTable = metricsDeferred.join(client.getDefaultAdminOperationTimeoutMs());
  KuduTable tagsetsTable = tagsetsDeferred.join(client.getDefaultAdminOperationTimeoutMs());
  KuduTable tagsTable = tagsDeferred.join(client.getDefaultAdminOperationTimeoutMs());

  Tags tags = new Tags(client, tagsTable);
  Tagsets tagsets = new Tagsets(client, tags, tagsetsTable);
  Metrics metrics = new Metrics(client, metricsTable, tagsets);
  return new KuduTS(client, name, metrics, tagsets, tags);
}
 

/**
 * Builds a set of scan tokens. The list of scan tokens are generated as if the entire table is being scanned
 * i.e. a SELECT * FROM TABLE equivalent expression. This list is used to assign the partition pie assignments
 * for all of the planned partition of operators. Each operator gets a part of the PIE as if all columns were
 * selected. Subsequently when a query is to be processed, the query is used to generate the scan tokens applicable
 * for that query. Given that partition pie represents the entire data set, the scan assignments for the current
 * query will be a subset.
 * @return The list of scan tokens as if the entire table is getting scanned.
 * @throws Exception in cases when the connection to kudu cluster cannot be closed.
 */
public List<KuduScanToken> getKuduScanTokensForSelectAllColumns() throws Exception
{
  // We are not using the current query for deciding the partition strategy but a SELECT * as
  // we do not want to want to optimize on just the current query. This prevents rapid throttling of operator
  // instances when the scan patterns are erratic. On the other hand, this might result on under utilized
  // operator resources in the DAG but will be consistent at a minimum.
  ApexKuduConnection apexKuduConnection = prototypeKuduInputOperator.getApexKuduConnectionInfo().build();
  KuduClient clientHandle = apexKuduConnection.getKuduClient();
  KuduTable table = apexKuduConnection.getKuduTable();
  KuduScanToken.KuduScanTokenBuilder builder = clientHandle.newScanTokenBuilder(table);
  List<String> allColumns = new ArrayList<>();
  List<ColumnSchema> columnList = apexKuduConnection.getKuduTable().getSchema().getColumns();
  for ( ColumnSchema column : columnList) {
    allColumns.add(column.getName());
  }
  builder.setProjectedColumnNames(allColumns);
  LOG.debug("Building the partition pie assignments for the input operator");
  List<KuduScanToken> allPossibleTokens = builder.build();
  apexKuduConnection.close();
  return allPossibleTokens;
}
 

public KuduRecordCursorWithVirtualRowId(KuduScanner scanner, KuduTable table,
                                        List<Type> columnTypes,
                                        Map<Integer, Integer> fieldMapping) {
    super(scanner, columnTypes);
    this.table = table;
    this.fieldMapping = fieldMapping;
}
 

private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) {
    KuduTable table = tableHandle.getTable(clientSession);
    Schema schema = table.getSchema();

    List<ColumnMetadata> columnsMetaList = schema.getColumns().stream()
            .filter(col -> !col.isKey() || !col.getName().equals(KuduColumnHandle.ROW_ID))
            .map(col -> {
                StringBuilder extra = new StringBuilder();
                if (col.isKey()) {
                    extra.append("key, ");
                } else if (col.isNullable()) {
                    extra.append("nullable, ");
                }
                if (col.getEncoding() != null) {
                    extra.append("encoding=").append(col.getEncoding().name()).append(", ");
                }
                if (col.getCompressionAlgorithm() != null) {
                    extra.append("compression=").append(col.getCompressionAlgorithm().name()).append(", ");
                }
                if (extra.length() > 2) {
                    extra.setLength(extra.length() - 2);
                }
                Type prestoType = TypeHelper.fromKuduColumn(col);
                return new ColumnMetadata(col.getName(), prestoType, null, extra.toString(), false);
            }).collect(toImmutableList());

    Map<String, Object> properties = clientSession.getTableProperties(tableHandle);
    return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties);
}
 

@Override
public RecordCursor cursor() {
    KuduScanner scanner = clientSession.createScanner(kuduSplit);
    if (!containsVirtualRowId) {
        return new KuduRecordCursor(scanner, getColumnTypes());
    } else {
        final int primaryKeyColumnCount = kuduSplit.getPrimaryKeyColumnCount();

        Map<Integer, Integer> fieldMapping = new HashMap<>();
        int index = primaryKeyColumnCount;
        for (int i = 0; i < columns.size(); i++) {
            KuduColumnHandle handle = (KuduColumnHandle) columns.get(i);
            if (!handle.isVirtualRowId()) {
                if (handle.getOrdinalPosition() < primaryKeyColumnCount) {
                    fieldMapping.put(i, handle.getOrdinalPosition());
                } else {
                    fieldMapping.put(i, index);
                    index++;
                }
            } else {
                fieldMapping.put(i, -1);
            }
        }

        KuduTable table = getTable();
        return new KuduRecordCursorWithVirtualRowId(scanner, table, getColumnTypes(), fieldMapping);
    }
}
 

public KuduTableClient(KuduClient client, KuduTable kuduTable, KuduSession kuduSession,int batchSize) {
    this.client = client;
    this.kuduTable = kuduTable;
    this.metaTable = getMetaTable(kuduTable);
    this.kuduSession = kuduSession;
    this.batchSize = batchSize;
}
 

@Override
public KuduTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) {
    try {
        KuduTable table = clientSession.openTable(schemaTableName);
        return new KuduTableHandle(connectorId, schemaTableName, table);
    } catch (Exception e) {
        return null;
    }
}
 

@Before
public void setup() throws Exception {
  // Sample table and schema
  List<ColumnSchema> columns = new ArrayList(2);
  columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
  columns.add(new ColumnSchema.ColumnSchemaBuilder("value", Type.STRING).build());
  columns.add(new ColumnSchema.ColumnSchemaBuilder("name", Type.STRING).build());
  final Schema schema = new Schema(columns);

  // Mock KuduTable class
  KuduTable table = PowerMockito.mock(KuduTable.class);
  PowerMockito.doReturn(schema).when(table).getSchema();

  // Mock KuduSession class
  PowerMockito.suppress(PowerMockito.method(
      AsyncKuduSession.class,
      "apply",
      Operation.class
  ));
  PowerMockito.suppress(PowerMockito.method(
      AsyncKuduSession.class,
      "flush"
  ));
  PowerMockito.suppress(PowerMockito.method(
      KuduLookupProcessor.class,
      "destroy"
  ));
}
 

KuduTable getTable(String tableName) throws KuduException {
  if (tables.containsKey(tableName)) {
    return tables.get(tableName);
  }
  else {
    KuduTable table = client.openTable(tableName);
    tables.put(tableName, table);
    return table;
  }
}
 

public static PartitionDesign getPartitionDesign(KuduTable table) {
    Schema schema = table.getSchema();
    PartitionDesign partitionDesign = new PartitionDesign();
    PartitionSchema partitionSchema = table.getPartitionSchema();

    List<HashPartitionDefinition> hashPartitions = partitionSchema.getHashBucketSchemas().stream()
            .map(hashBucketSchema -> {
                HashPartitionDefinition hash = new HashPartitionDefinition();
                List<String> cols = hashBucketSchema.getColumnIds().stream()
                        .map(idx -> schema.getColumnByIndex(idx).getName()).collect(toImmutableList());
                hash.setColumns(cols);
                hash.setBuckets(hashBucketSchema.getNumBuckets());
                return hash;
            }).collect(toImmutableList());
    partitionDesign.setHash(hashPartitions);

    List<Integer> rangeColumns = partitionSchema.getRangeSchema().getColumns();
    if (!rangeColumns.isEmpty()) {
        RangePartitionDefinition definition = new RangePartitionDefinition();
        definition.setColumns(rangeColumns.stream()
                .map(i -> schema.getColumns().get(i).getName())
                .collect(ImmutableList.toImmutableList()));
        partitionDesign.setRange(definition);
    }

    return partitionDesign;
}
 

public KuduExtendedTableHandle(String connectorId, SchemaTableName schemaTableName,
                               List<String> columnNames, List<Type> columnTypes,
                               KuduTable table) {
    super(connectorId, schemaTableName, table);

    requireNonNull(columnNames, "columnNames is null");
    requireNonNull(columnTypes, "columnTypes is null");
    checkArgument(columnNames.size() == columnTypes.size(), "columnNames and columnTypes sizes don't match");
    this.columnNames = ImmutableList.copyOf(columnNames);
    this.columnTypes = ImmutableList.copyOf(columnTypes);
}
 

private KuduTable obtainTable() throws IOException {
    String tableName = tableInfo.getName();
    if (client.tableExists(tableName)) {
        return client.openTable(tableName);
    }
    if (tableInfo.getCreateTableIfNotExists()) {
        return client.createTable(tableName, tableInfo.getSchema(), tableInfo.getCreateTableOptions());
    }
    throw new UnsupportedOperationException("table not exists and is marketed to not be created");
}
 

private KuduScanner scannerForFilters(Iterable<Row> filters, KuduTable table) throws KuduException {
  List<Row> filtersList = Lists.newArrayList(filters);

  if (filtersList.size() == 0) {
    throw new RuntimeException("Kudu existing filter was not provided.");
  }
  
  if (filtersList.get(0).schema() == null) {
    throw new RuntimeException("Kudu existing filter did not contain a schema.");
  }
  
  if (hasAccumulators()) {
    accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_SCANNERS).add(1);
    accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_FILTERS_SCANNED).add(filtersList.size());
  }
  
  KuduScannerBuilder builder = getConnection().getClient().newScannerBuilder(table);

  for (String fieldName : filtersList.get(0).schema().fieldNames()) {
    ColumnSchema columnSchema = table.getSchema().getColumn(fieldName);

    List<Object> columnValues = Lists.newArrayList();
    for (Row filter : filtersList) {
      Object columnValue = filter.getAs(fieldName);
      columnValues.add(columnValue);
    }

    KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, columnValues);

    builder = builder.addPredicate(predicate);
  }

  KuduScanner scanner = builder.build();

  return scanner;
}
 

public static List<ColumnMeta> getColumnMetaList(KuduTable kuduTable) {
    List<ColumnMeta> columnMetaList = Lists.newArrayList();
    List<ColumnSchema> columnSchemas = kuduTable.getSchema().getColumns();

    columnSchemas.forEach(columnSchema -> {
        ColumnMeta columnMeta = new ColumnMeta();
        columnMeta.setType(columnSchema.getType().getName());
        columnMeta.setIsPrimaryKey(columnSchema.isKey());
        columnMeta.setName(columnSchema.getName());
        columnMeta.setLength(columnSchema.getType().getSize(columnSchema.getTypeAttributes()));
        columnMetaList.add(columnMeta);
    });
    return columnMetaList;
}
 

@Override
public KuduClient buildClient(ProcessContext context) {
    final KuduClient client = mock(KuduClient.class);

    try {
        when(client.openTable(anyString())).thenReturn(mock(KuduTable.class));
    } catch (final Exception e) {
        throw new AssertionError(e);
    }

    return client;
}
 

private synchronized StructType getTableSchema(KuduTable table) throws KuduException {
  if (tableSchemas == null) {
    tableSchemas = Maps.newHashMap();
  }

  if (tableSchemas.containsKey(table.getName())) {
    return tableSchemas.get(table.getName());
  }
  else {
    StructType tableSchema = schemaFor(table);
    tableSchemas.put(table.getName(), tableSchema);
    return tableSchema;
  }
}