org.apache.kudu.client.KuduClient Java Examples

@Override
public List<String> listSchemaNames(KuduClient client)
{
    try {
        if (rawSchemasTable == null) {
            if (!client.tableExists(rawSchemasTableName)) {
                createAndFillSchemasTable(client);
            }
            rawSchemasTable = getSchemasTable(client);
        }

        KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build();
        RowResultIterator iterator = scanner.nextRows();
        ArrayList<String> result = new ArrayList<>();
        while (iterator != null) {
            for (RowResult row : iterator) {
                result.add(row.getString(0));
            }
            iterator = scanner.nextRows();
        }
        return result;
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

@Test
public void test01() {
    List<String> masterList = Arrays.asList("10.6.36.102:7051,10.6.36.187:7051,10.6.36.229:7051".split(","));
    KuduClient kuduClient = new KuduClient.KuduClientBuilder(masterList).defaultOperationTimeoutMs(60000)
        .defaultSocketReadTimeoutMs(30000)
        .defaultAdminOperationTimeoutMs(60000)
        .build();
    try {
        List<String> tablesList = kuduClient.getTablesList().getTablesList();
        System.out.println(tablesList.toString());
        KuduTable web_white_list = kuduClient.openTable("web_white_list");

    } catch (KuduException e) {
        e.printStackTrace();
    }
}
 

protected void insertRowInTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final KuduTable table = client.openTable(tableName);

        final Insert insert = table.newInsert();
        final PartialRow row = insert.getRow();

        row.addInt("id", ThreadLocalRandom.current().nextInt(1, 99));
        row.addString("title", "Mr.");
        row.addString("name", "Samuel");
        row.addString("lastname", "Smith");
        row.addString("address", "4359  Plainfield Avenue");

        client.newSession().apply(insert);
    }
}
 

protected void createTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final List<ColumnSchema> columns = new ArrayList<>(5);
        final List<String> columnNames = Arrays.asList("id", "title", "name", "lastname", "address");

        for (int i = 0; i < columnNames.size(); i++) {
            final Type type = i == 0 ? Type.INT32 : Type.STRING;
            columns.add(
                new ColumnSchema.ColumnSchemaBuilder(columnNames.get(i), type)
                    .key(i == 0)
                    .build());
        }

        final List<String> rangeKeys = new ArrayList<>();
        rangeKeys.add("id");

        client.createTable(tableName,
            new Schema(columns),
            new CreateTableOptions().setRangePartitionColumns(rangeKeys));
    }
}
 

public static KuduClient createConnection(Map<String, Object> options) {

        String host = ConnectorOptions.extractOption(options, "host");
        String port = ConnectorOptions.extractOption(options, "port");
        if(ObjectHelper.isNotEmpty(host) && ObjectHelper.isNotEmpty(port)) {

            Long socketTimeout = ConnectorOptions.extractOptionAndMap(options, "socketTimeout", Long::valueOf, 3000L);
            Long operationTimeout = ConnectorOptions.extractOptionAndMap(options, "operationTimeout", Long::valueOf, 10000L);
            Long adminOperationTimeout = ConnectorOptions.extractOptionAndMap(options, "operationTimeout", Long::valueOf, 10000L);

            return new KuduClient.KuduClientBuilder(host + ":" + port)
                    .defaultSocketReadTimeoutMs(socketTimeout)
                    .defaultOperationTimeoutMs(operationTimeout)
                    .defaultAdminOperationTimeoutMs(adminOperationTimeout)
                    .build();
        } else {
            throw new SyndesisServerException("Port and Host parameter are mandatory");
        }
    }
 

@Test
public void testConnectionFailure() throws Exception{
  final KuduClient client = PowerMockito.mock(KuduClient.class);
  PowerMockito.when(client.getTablesList()).thenThrow(PowerMockito.mock(KuduException.class));

  final KuduTarget target = PowerMockito.spy(new KuduTarget(new KuduConfigBeanBuilder()
      .setMaster(KUDU_MASTER)
      .setTableName(tableName)
      .setDefaultOperation(KuduOperationType.INSERT)
      .setUnsupportedAction(UnsupportedOperationAction.DISCARD)
      .build()
  ));

  PowerMockito.when(target.buildKuduClient()).thenReturn(client);

  final TargetRunner targetRunner = getTargetRunner(target);

  try {
    List<Stage.ConfigIssue> issues = targetRunner.runValidateConfigs();
    Assert.assertEquals(1, issues.size());
  } catch (StageException e) {
    Assert.fail("should not throw StageException");
  }
}
 

@Singleton
@Provides
KuduClientSession createKuduClientSession(KuduClientConfig config)
{
    requireNonNull(config, "config is null");

    KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses());
    builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis());
    builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis());
    builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis());
    if (config.isDisableStatistics()) {
        builder.disableStatistics();
    }
    KuduClient client = builder.build();

    SchemaEmulation strategy;
    if (config.isSchemaEmulationEnabled()) {
        strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix());
    }
    else {
        strategy = new NoSchemaEmulation();
    }
    return new KuduClientSession(client, strategy);
}
 

private void createAndFillSchemasTable(KuduClient client)
        throws KuduException
{
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING)
            .key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    }
    finally {
        session.close();
    }
}
 

/**
 * Builds a set of scan tokens. The list of scan tokens are generated as if the entire table is being scanned
 * i.e. a SELECT * FROM TABLE equivalent expression. This list is used to assign the partition pie assignments
 * for all of the planned partition of operators. Each operator gets a part of the PIE as if all columns were
 * selected. Subsequently when a query is to be processed, the query is used to generate the scan tokens applicable
 * for that query. Given that partition pie represents the entire data set, the scan assignments for the current
 * query will be a subset.
 * @return The list of scan tokens as if the entire table is getting scanned.
 * @throws Exception in cases when the connection to kudu cluster cannot be closed.
 */
public List<KuduScanToken> getKuduScanTokensForSelectAllColumns() throws Exception
{
  // We are not using the current query for deciding the partition strategy but a SELECT * as
  // we do not want to want to optimize on just the current query. This prevents rapid throttling of operator
  // instances when the scan patterns are erratic. On the other hand, this might result on under utilized
  // operator resources in the DAG but will be consistent at a minimum.
  ApexKuduConnection apexKuduConnection = prototypeKuduInputOperator.getApexKuduConnectionInfo().build();
  KuduClient clientHandle = apexKuduConnection.getKuduClient();
  KuduTable table = apexKuduConnection.getKuduTable();
  KuduScanToken.KuduScanTokenBuilder builder = clientHandle.newScanTokenBuilder(table);
  List<String> allColumns = new ArrayList<>();
  List<ColumnSchema> columnList = apexKuduConnection.getKuduTable().getSchema().getColumns();
  for ( ColumnSchema column : columnList) {
    allColumns.add(column.getName());
  }
  builder.setProjectedColumnNames(allColumns);
  LOG.debug("Building the partition pie assignments for the input operator");
  List<KuduScanToken> allPossibleTokens = builder.build();
  apexKuduConnection.close();
  return allPossibleTokens;
}
 

public static void main(String[] args) {
    ArrayList<String> master = new ArrayList<String>();
    master.add("10.104.132.72:7051");
    master.add("10.104.132.73:7051");
    master.add("10.104.132.75:7051");
    master.add("10.104.132.223:7051");
    master.add("10.104.132.221:7051");
    KuduClient client = createClient(master);
    try {
        List<String> tablesList = client.getTablesList().getTablesList();
        System.out.println(ArrayUtils.toString(tablesList));
    } catch (KuduException e) {
        e.printStackTrace();
    } finally {
        closeClient(client);
    }


}
 

/**
 * 根据传入的MediaSourceInfo 获取所有表的元信息
 * @param info
 * @return
 */
public static List<MediaMeta> getTables(MediaSourceInfo info) throws KuduException {
    checkKudu(info);

    KuduMediaSrcParameter parameter = info.getParameterObj();
    List<String> hostList = parameter.getKuduMasterConfigs().stream().
            map(config->{return MessageFormat.format("{0}:{1}",config.getHost(),String.valueOf(config.getPort()));}).
            collect(Collectors.toList());

    KuduClient client = createClient(hostList);
   try {
       return execute(client);
   }catch (KuduException e){
        LOGGER.info("执行kudu查询时报错:",e);
        throw new RuntimeException(e);
   } finally{
       closeClient(client);
   }
}
 

@Override
public void createSchema(KuduClient client, String schemaName)
{
    if (DEFAULT_SCHEMA.equals(schemaName)) {
        throw new SchemaAlreadyExistsException(schemaName);
    }
    else {
        try {
            KuduTable schemasTable = getSchemasTable(client);
            KuduSession session = client.newSession();
            try {
                Upsert upsert = schemasTable.newUpsert();
                upsert.getRow().addString(0, schemaName);
                session.apply(upsert);
            }
            finally {
                session.close();
            }
        }
        catch (KuduException e) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
        }
    }
}
 

protected void createKuduClient(ProcessContext context) {
    kuduClientWriteLock.lock();
    try {
        if (this.kuduClient != null) {
            try {
                this.kuduClient.close();
            } catch (KuduException e) {
                getLogger().error("Couldn't close Kudu client.");
            }
        }

        if (kerberosUser != null) {
            final KerberosAction<KuduClient> kerberosAction = new KerberosAction<>(kerberosUser, () -> buildClient(context), getLogger());
            this.kuduClient = kerberosAction.execute();
        } else {
            this.kuduClient = buildClient(context);
        }
    } finally {
        kuduClientWriteLock.unlock();
    }
}
 

@Singleton
@Provides
KuduClientSession createKuduClientSession(
        KuduConnectorId connectorId,
        KuduClientConfig config) {
    requireNonNull(config, "config is null");

    KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses());
    builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis());
    builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis());
    builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis());
    if (config.isDisableStatistics()) {
        builder.disableStatistics();
    }
    KuduClient client = builder.build();
    String tenant = config.getTenant();
    return new NativeKuduClientSession(connectorId, client, tenant);
}
 

public static List<ColumnMeta> getColumns(MediaSourceInfo info, String tableName) {
    checkKudu(info);

    KuduMediaSrcParameter parameter = info.getParameterObj();
    List<String> hostList = parameter.getKuduMasterConfigs().stream().
            map(config->{return MessageFormat.format("{0}:{1}",config.getHost(),String.valueOf(config.getPort()));}).
            collect(Collectors.toList());

    KuduClient client = createClient(hostList);
    try {
        KuduTable kuduTable = client.openTable(tableName);
        return getColumnMetaList(kuduTable);
    } catch (KuduException e){
        LOGGER.info("执行kudu查询时报错:",e);
        throw new RuntimeException(e);
    } finally{
        closeClient(client);
    }
}
 

private void createKuduTable() throws KuduException {
    KuduClient client =  harness.getClient();
    List<ColumnSchema> columns = new ArrayList<>();
    columns.add(new ColumnSchema.ColumnSchemaBuilder("id", Type.INT32).key(true).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("stringval", Type.STRING).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("varcharval", Type.VARCHAR).typeAttributes(
            new ColumnTypeAttributes.ColumnTypeAttributesBuilder().length(256).build()
    ).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("num32val", Type.INT32).build());
    columns.add(new ColumnSchema.ColumnSchemaBuilder("timestampval", Type.UNIXTIME_MICROS).build());
    Schema schema = new Schema(columns);
    CreateTableOptions opts = new CreateTableOptions()
        .addHashPartitions(Collections.singletonList("id"), 4);
    client.createTable(DEFAULT_TABLE_NAME, schema, opts);
}
 

protected void executeOnKuduClient(Consumer<KuduClient> actionOnKuduClient) {
    kuduClientReadLock.lock();
    try {
        actionOnKuduClient.accept(kuduClient);
    } finally {
        kuduClientReadLock.unlock();
    }
}
 

public static void createTestTable(String tableName, KuduClient client) throws Exception
{
  List<String> rangeKeys = new ArrayList<>();
  rangeKeys.add("introwkey");
  List<String> hashPartitions = new ArrayList<>();
  hashPartitions.add("stringrowkey");
  hashPartitions.add("timestamprowkey");
  CreateTableOptions thisTableOptions = new CreateTableOptions()
      .setNumReplicas(1)
      .addHashPartitions(hashPartitions,HASH_BUCKETS_SIZE_FOR_ALL_HASH_COL)
      .setRangePartitionColumns(rangeKeys);
  int stepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
  int splitBoundary = stepsize;
  Schema schema = buildSchemaForUnitTestsTable();
  for ( int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
    PartialRow splitRowBoundary = schema.newPartialRow();
    splitRowBoundary.addInt("introwkey",splitBoundary);
    thisTableOptions = thisTableOptions.addSplitRow(splitRowBoundary);
    splitBoundary += stepsize;
  }
  try {
    client.createTable(tableName, schema,thisTableOptions);
  } catch (KuduException e) {
    LOG.error("Error while creating table for unit tests " + e.getMessage(), e);
    throw e;
  }

}
 

protected KuduClient buildClient(final ProcessContext context) {
    final String masters = context.getProperty(KUDU_MASTERS).evaluateAttributeExpressions().getValue();
    final Integer operationTimeout = context.getProperty(KUDU_OPERATION_TIMEOUT_MS).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS).intValue();
    final Integer adminOperationTimeout = context.getProperty(KUDU_KEEP_ALIVE_PERIOD_TIMEOUT_MS).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS).intValue();

    return new KuduClient.KuduClientBuilder(masters)
            .defaultOperationTimeoutMs(operationTimeout)
            .defaultSocketReadTimeoutMs(adminOperationTimeout)
            .build();
}
 

protected KuduClient buildClient(final String masters, final ConfigurationContext context) {
    final Integer operationTimeout = context.getProperty(KUDU_OPERATION_TIMEOUT_MS).asTimePeriod(TimeUnit.MILLISECONDS).intValue();

    return new KuduClient.KuduClientBuilder(masters)
            .defaultOperationTimeoutMs(operationTimeout)
            .build();
}
 

protected void createKuduClient(ConfigurationContext context) throws LoginException {
    final String kuduMasters = context.getProperty(KUDU_MASTERS).evaluateAttributeExpressions().getValue();
    final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);

    if (credentialsService != null) {
        final String keytab = credentialsService.getKeytab();
        final String principal = credentialsService.getPrincipal();
        kerberosUser = loginKerberosUser(principal, keytab);

        final KerberosAction<KuduClient> kerberosAction = new KerberosAction<>(kerberosUser, () -> buildClient(kuduMasters, context), getLogger());
        this.kuduClient = kerberosAction.execute();
    } else {
        this.kuduClient = buildClient(kuduMasters, context);
    }
}
 

private static void verifyConnection(ResultBuilder builder, Map<String, Object> parameters) {
    try {
        KuduClient c = KuduSupport.createConnection(parameters);
        c.getTablesList();
    } catch (Exception e) {
        ResultErrorBuilder errorBuilder = ResultErrorBuilder.withCodeAndDescription(
                VerificationError.StandardCode.EXCEPTION, e.getMessage());
        builder.error(errorBuilder.build());
    }
}
 

private ApexKuduConnection(ApexKuduConnectionBuilder builder)
{
  checkNotNull(builder,"Builder cannot be null to establish kudu session");
  checkArgument(builder.mastersCollection.size() > 0, "Atleast one kudu master needs to be specified");
  checkNotNull(builder.tableName,"Kudu table cannot be null");
  builderForThisConnection = builder;
  KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(builder.mastersCollection);
  if (builder.isOperationTimeOutSet) {
    kuduClientBuilder.defaultOperationTimeoutMs(builder.operationTimeOutMs);
  }
  if (builder.isBossThreadCountSet) {
    kuduClientBuilder.bossCount(builder.numBossThreads);
  }
  if (builder.isWorkerThreadsCountSet) {
    kuduClientBuilder.workerCount(builder.workerThreads);
  }
  if (builder.isSocketReadTimeOutSet) {
    kuduClientBuilder.defaultSocketReadTimeoutMs(builder.socketReadTimeOutMs);
  }
  kuduClient = kuduClientBuilder.build();
  kuduSession = kuduClient.newSession();
  if (builder.isFlushModeSet) {
    kuduSession.setFlushMode(builder.flushMode);
  }
  if (builder.isExternalConsistencyModeSet) {
    kuduSession.setExternalConsistencyMode(builder.externalConsistencyMode);
  }
  try {
    if (!kuduClient.tableExists(builder.tableName)) {
      throw new Exception("Table " + builder.tableName + " does not exist. ");
    } else {
      kuduTable = kuduClient.openTable(builder.tableName);
    }
  } catch (Exception e) {
    throw new RuntimeException("Kudu table existence could not be ascertained  " + e.getMessage(), e);
  }
}
 

@Override
public KuduClient buildClient(ProcessContext context) {
    final KuduClient client = mock(KuduClient.class);

    try {
        when(client.openTable(anyString())).thenReturn(mock(KuduTable.class));
    } catch (final Exception e) {
        throw new AssertionError(e);
    }

    return client;
}
 

public synchronized KuduClient getClient(final String kuduMaster) {
  if (kuduMaster == null) {
    LOGGER.error("Kudu Master server must be set for Kudu");
    return null;
  }
  return sessionCache.get(kuduMaster);
}
 

@Bean
public KuduClient kuduClientBean() {

    if (LOG.isDebugEnabled()) {
        LOG.debug("Creating new instance of a mocked influx db connection");
    }

    KuduClient mockedDbConnection = mock(KuduClient.class);

    assertNotNull(mockedDbConnection);
    return mockedDbConnection;
}
 

@Override
public void close(Throwable errorOrNull)
{
    try (KuduClient client = kuduClient) {
        if (kuduSession != null) {
            this.flush();
            this.kuduSession.close();
        }
    }
    catch (IOException e) {
        throwsException(e);
    }
}
 

@Override
public boolean open(long partitionId, long version)
        throws Exception
{
    this.kuduClient = new KuduClient.KuduClientBuilder(kuduHost).build();
    this.kuduSession = kuduClient.newSession();
    this.kuduTable = kuduClient.openTable(tableName);
    this.operationCreater = getOperationCreater(kuduSinkConfig.mode, kuduTable);

    kuduSession.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
    //kuduSession.setFlushInterval();
    this.kuduSession.setMutationBufferSpace(this.mutationBufferSpace); //8m
    return true;
}
 

public static void main(String[] args) {
        ArrayList<String> master = new ArrayList<String>();
        master.add("10.104.132.75:7051");
        master.add("10.104.132.73:7051");
        master.add("10.104.132.72:7051");
        master.add("10.104.132.221:7051");
        master.add("10.104.132.223:7051");
        KuduClient client = createClient(master);
        try {
            List<MediaMeta> list = execute(client);
            for(MediaMeta mediaMeta:list){
                System.out.println("tableName:"+mediaMeta.getName());
                List<ColumnMeta>  columnMetaList = mediaMeta.getColumn();
                for (ColumnMeta columnMeta:columnMetaList){
                    System.out.println(columnMeta.getName());
                }
            }

        }catch (KuduException e){
            LOGGER.info("执行kudu查询时报错:",e);
            throw new RuntimeException(e);
        } finally{
            closeClient(client);
        }
/*        KuduClient client = createClient(master);
        try {
            List<String> tablesList = client.getTablesList().getTablesList();
            System.out.println("------????------");
            System.out.println(ArrayUtils.toString(tablesList));
            KuduTable kuduTable = client.openTable(tablesList.get(0));
            System.out.println(kuduTable.getName().split("\\.")[1]+" : "+kuduTable.getName());
            System.out.println("------end------");
        } catch (KuduException e) {
            e.printStackTrace();
        } finally {
            closeClient(client);
        }*/
    }
 

public static void closeClient(KuduClient client){
    if(client != null){
        try {
            client.close();
        } catch (KuduException e) {
        }
    }
}