org.apache.kudu.client.KuduException Java Examples

public KuduTable createTable(ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
{
    try {
        String rawName = schemaEmulation.toRawName(tableMetadata.getTable());
        if (ignoreExisting) {
            if (client.tableExists(rawName)) {
                return null;
            }
        }

        if (!schemaEmulation.existsSchema(client, tableMetadata.getTable().getSchemaName())) {
            throw new SchemaNotFoundException(tableMetadata.getTable().getSchemaName());
        }

        List<ColumnMetadata> columns = tableMetadata.getColumns();
        Map<String, Object> properties = tableMetadata.getProperties();

        Schema schema = buildSchema(columns, properties);
        CreateTableOptions options = buildCreateTableOptions(schema, properties);
        return client.createTable(rawName, schema, options);
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

@Override
public void dropSchema(String schemaName) {
    if (DEFAULT_SCHEMA.equals(schemaName)) {
        throw new PrestoException(GENERIC_USER_ERROR, "Deleting default schema not allowed.");
    }
    else {
        try {
            for (SchemaTableName table : listTables(schemaName)) {
                dropTable(table);
            }
            KuduTable schemasTable = getSchemasTable();
            KuduSession session = client.newSession();
            try {
                Delete delete = schemasTable.newDelete();
                fillSchemaRow(delete.getRow(), schemaName);
                session.apply(delete);
            }
            finally {
                session.close();
            }
        }
        catch (KuduException e) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
        }
    }
}
 

@Override
public synchronized void flush() {
  try {
    session.flush();
    if (session.countPendingErrors() != 0) {
      LOGGER.error(
          "Got {} pending errors while flushing Kudu session",
          session.countPendingErrors());
      for (final RowError err : session.getPendingErrors().getRowErrors()) {
        LOGGER.error("{}", err);
      }
    }
  } catch (final KuduException e) {
    LOGGER.error("Encountered error while flushing Kudu session", e);
  }
}
 

@Override
public void deleteRows(Block rowIds) {
    Schema schema = table.getSchema();
    KuduSession session = clientSession.newSession();
    session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        try {
            for (int i = 0; i < rowIds.getPositionCount(); i++) {
                int len = rowIds.getSliceLength(i);
                Slice slice = rowIds.getSlice(i, 0, len);
                PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes());
                Delete delete = table.newDelete();
                RowHelper.copyPrimaryKey(schema, row, delete.getRow());
                session.apply(delete);
            }
        } finally {
            session.close();
        }
    } catch (KuduException e) {
        throw new RuntimeException(e);
    }
}
 

KuduConnection getConnection() throws KuduException {
  // get the thread-local connection or open a new one if expired or null
  if (connectionThreadLocal.get() == null) {
    LOG.debug("Creating new KuduConnection for thread");
    setNewConnection(KuduUtils.isSecure(config));
  } else if (TokenStoreListener.get().hasChanged(tokenAlias)) {
    LOG.debug("Kudu token has changed, refreshing KuduConnection for thread");
    lastChanged = System.currentTimeMillis();
    connectionThreadLocal.get().close();
    setNewConnection(KuduUtils.isSecure(config));
  } else if (connectionThreadLocal.get().getLastUsed() < lastChanged) {
    LOG.debug("Kudu token change detected in another thread, refreshing KuduConnection for this thread");
    connectionThreadLocal.get().close();
    setNewConnection(KuduUtils.isSecure(config));
  }

  return connectionThreadLocal.get();
}
 

protected void insertRowInTestTable(final String tableName, final String connection) throws KuduException {
    try (KuduClient client = new KuduClient.KuduClientBuilder(connection).build()) {

        final KuduTable table = client.openTable(tableName);

        final Insert insert = table.newInsert();
        final PartialRow row = insert.getRow();

        row.addInt("id", ThreadLocalRandom.current().nextInt(1, 99));
        row.addString("title", "Mr.");
        row.addString("name", "Samuel");
        row.addString("lastname", "Smith");
        row.addString("address", "4359  Plainfield Avenue");

        client.newSession().apply(insert);
    }
}
 

@Order(2)
@Test
public void insertShouldSucceed() throws KuduException {
    LOG.info("Calling insertShouldSucceed");

    RestAssured.put("/kudu/insert").then().statusCode(200);

    KuduClient client = new KuduClient.KuduClientBuilder(MASTER_RPC_AUTHORITY).build();

    List<Map<String, Object>> records = KuduUtils.doScan("TestTable", client);
    assertEquals(1, records.size());
    Map<String, Object> record = records.get(0);
    assertNotNull(record);
    assertEquals("key1", record.get("id"));
    assertEquals("Samuel", record.get("name"));
}
 

public void createTable(KuduTableInfo tableInfo, boolean ignoreIfExists) throws CatalogException, TableAlreadyExistException {
    ObjectPath path = getObjectPath(tableInfo.getName());
    if (tableExists(path)) {
        if (ignoreIfExists) {
            return;
        } else {
            throw new TableAlreadyExistException(getName(), path);
        }
    }

    try {
        kuduClient.createTable(tableInfo.getName(), tableInfo.getSchema(), tableInfo.getCreateTableOptions());
    } catch (
            KuduException e) {
        throw new CatalogException("Could not create table " + tableInfo.getName(), e);
    }
}
 

@Ignore
public void insertRow() throws KuduException, InterruptedException {
    deleteTestTable(TABLE, HOST + ":" + PORT);
    createTestTable(TABLE, HOST + ":" + PORT);

    errorEndpoint.expectedMessageCount(0);
    successEndpoint.expectedMessageCount(1);

    // Create a sample row that can be inserted in the test table
    final Map<String, Object> row = new HashMap<>();
    row.put("id", 5);
    row.put("title", "Mr.");
    row.put("name", "Samuel");
    row.put("lastname", "Smith");
    row.put("address", "4359  Plainfield Avenue");

    sendBody("direct:insert", row);

    errorEndpoint.assertIsSatisfied();
    successEndpoint.assertIsSatisfied();
}
 

@Override
public void flush() {
  try {
    session.flush();
    if (session.countPendingErrors() != 0) {
      LOGGER.error(
          "Got {} pending errors while flushing Kudu session",
          session.countPendingErrors());
      for (final RowError err : session.getPendingErrors().getRowErrors()) {
        LOGGER.error("{}", err);
      }
    }
  } catch (final KuduException e) {
    LOGGER.error("Encountered error while flushing Kudu session", e);
  }
}
 

public static void main(String[] args) {
    ArrayList<String> master = new ArrayList<String>();
    master.add("10.104.132.72:7051");
    master.add("10.104.132.73:7051");
    master.add("10.104.132.75:7051");
    master.add("10.104.132.223:7051");
    master.add("10.104.132.221:7051");
    KuduClient client = createClient(master);
    try {
        List<String> tablesList = client.getTablesList().getTablesList();
        System.out.println(ArrayUtils.toString(tablesList));
    } catch (KuduException e) {
        e.printStackTrace();
    } finally {
        closeClient(client);
    }


}
 

@Override
public List<String> listSchemaNames(KuduClient client)
{
    try {
        if (rawSchemasTable == null) {
            if (!client.tableExists(rawSchemasTableName)) {
                createAndFillSchemasTable(client);
            }
            rawSchemasTable = getSchemasTable(client);
        }

        KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build();
        RowResultIterator iterator = scanner.nextRows();
        ArrayList<String> result = new ArrayList<>();
        while (iterator != null) {
            for (RowResult row : iterator) {
                result.add(row.getString(0));
            }
            iterator = scanner.nextRows();
        }
        return result;
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 

@OnScheduled
public void OnScheduled(final ProcessContext context) {
    try {
        tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions().getValue();
        kuduMasters = context.getProperty(KUDU_MASTERS).evaluateAttributeExpressions().getValue();
        if(kuduClient == null) {
            getLogger().debug("Setting up Kudu connection...");
            kuduClient = getKuduConnection(kuduMasters);
            kuduTable = this.getKuduTable(kuduClient, tableName);
            getLogger().debug("Kudu connection successfully initialized");
        }

        operationType = DafOperationType.valueOf(context.getProperty(INSERT_OPERATION).getValue());
        batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger();
        flushMode = SessionConfiguration.FlushMode.valueOf(context.getProperty(FLUSH_MODE).getValue());
        skipHeadLine = context.getProperty(SKIP_HEAD_LINE).asBoolean();
    } catch(KuduException ex){
        //getLogger().error("Exception occurred while interacting with Kudu due to " + ex.getMessage(), ex);
    }
}
 

@Override
public void deleteRows(Block rowIds)
{
    Schema schema = table.getSchema();
    KuduSession session = clientSession.newSession();
    session.setFlushMode(FlushMode.AUTO_FLUSH_BACKGROUND);
    try {
        try {
            for (int i = 0; i < rowIds.getPositionCount(); i++) {
                int len = rowIds.getSliceLength(i);
                Slice slice = rowIds.getSlice(i, 0, len);
                PartialRow row = KeyEncoderAccessor.decodePrimaryKey(schema, slice.getBytes());
                Delete delete = table.newDelete();
                RowHelper.copyPrimaryKey(schema, row, delete.getRow());
                session.apply(delete);
            }
        }
        finally {
            session.close();
        }
    }
    catch (KuduException e) {
        throw new RuntimeException(e);
    }
}
 

protected void initScanner() {
  final Collection<SinglePartitionQueryRanges> ranges =
      readerParams.getQueryRanges().getPartitionQueryRanges();
  try {
    iterator =
        operations.getKuduRangeRead(
            readerParams.getIndex().getName(),
            readerParams.getAdapterIds(),
            ranges,
            DataStoreUtils.isMergingIteratorRequired(readerParams, visibilityEnabled),
            rowTransformer,
            new ClientVisibilityFilter(
                Sets.newHashSet(readerParams.getAdditionalAuthorizations())),
            visibilityEnabled).results();
  } catch (final KuduException e) {
    LOGGER.error("Error in initializing reader", e);
  }
}
 

/**
 * 统计kudu表数据
 *
 * @param tableName
 * @return
 */
public long countRow(String tableName) {
    this.checkClient();
    long rowCount = 0L;
    try {
        KuduTable kuduTable = kuduClient.openTable(tableName);
        // 创建scanner扫描
        KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
        // 遍历数据
        while (scanner.hasMoreRows()) {
            while (scanner.nextRows().hasNext()) {
                rowCount++;
            }
        }
    } catch (KuduException e) {
        e.printStackTrace();
    }
    return rowCount;
}
 

public static List<ColumnMeta> getColumns(MediaSourceInfo info, String tableName) {
    checkKudu(info);

    KuduMediaSrcParameter parameter = info.getParameterObj();
    List<String> hostList = parameter.getKuduMasterConfigs().stream().
            map(config->{return MessageFormat.format("{0}:{1}",config.getHost(),String.valueOf(config.getPort()));}).
            collect(Collectors.toList());

    KuduClient client = createClient(hostList);
    try {
        KuduTable kuduTable = client.openTable(tableName);
        return getColumnMetaList(kuduTable);
    } catch (KuduException e){
        LOGGER.info("执行kudu查询时报错:",e);
        throw new RuntimeException(e);
    } finally{
        closeClient(client);
    }
}
 

public <T> KuduRangeRead<T> getKuduRangeRead(
    final String indexName,
    final short[] adapterIds,
    final Collection<SinglePartitionQueryRanges> ranges,
    final boolean rowMerging,
    final GeoWaveRowIteratorTransformer<T> rowTransformer,
    final Predicate<GeoWaveRow> rowFilter,
    final boolean visibilityEnabled) throws KuduException {
  final KuduTable table = getTable(indexName);
  return new KuduRangeRead<>(
      ranges,
      adapterIds,
      table,
      this,
      visibilityEnabled,
      rowFilter,
      rowTransformer,
      rowMerging);
}
 

@Override
public void flush() {
  try {
    session.flush();
    if (session.countPendingErrors() != 0) {
      LOGGER.error(
          "Got {} pending errors while flushing Kudu session",
          session.countPendingErrors());
      for (final RowError err : session.getPendingErrors().getRowErrors()) {
        LOGGER.error("{}", err);
      }
    }
  } catch (final KuduException e) {
    LOGGER.error("Encountered error while flushing Kudu session", e);
  }
}
 

@Override
public CompletableFuture<?> appendPage(Page page) {
    for (int position = 0; position < page.getPositionCount(); position++) {
        Upsert upsert = table.newUpsert();
        PartialRow row = upsert.getRow();
        int start = 0;
        if (generateUUID) {
            String id = String.format("%s-%08x", uuid, nextSubId++);
            row.addString(0, id);
            start = 1;
        }

        for (int channel = 0; channel < page.getChannelCount(); channel++) {
            appendColumn(row, page, position, channel, channel + start);
        }

        try {
            session.apply(upsert);
        } catch (KuduException e) {
            throw new RuntimeException(e);
        }
    }
    return NOT_BLOCKED;
}
 

@Override
public CatalogTable getTable(ObjectPath tablePath) throws TableNotExistException {
    checkNotNull(tablePath);

    if (!tableExists(tablePath)) {
        throw new TableNotExistException(getName(), tablePath);
    }

    String tableName = tablePath.getObjectName();

    try {
        KuduTable kuduTable = kuduClient.openTable(tableName);

        CatalogTableImpl table = new CatalogTableImpl(
                KuduTableUtils.kuduToFlinkSchema(kuduTable.getSchema()),
                createTableProperties(tableName, kuduTable.getSchema().getPrimaryKeyColumns()),
                tableName);

        return table;
    } catch (KuduException e) {
        throw new CatalogException(e);
    }
}
 

@Override
public void openSession() throws KuduException {
  // errors are collected per session so we align session with the bundle
  session = client.newSession();
  // async flushing as per the official kudu-spark approach
  session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
}
 

@Override
public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists) throws TableNotExistException {
    String tableName = tablePath.getObjectName();
    try {
        if (tableExists(tablePath)) {
            kuduClient.alterTable(tableName, new AlterTableOptions().renameTable(newTableName));
        } else if (!ignoreIfNotExists) {
            throw new TableNotExistException(getName(), tablePath);
        }
    } catch (KuduException e) {
        throw new CatalogException("Could not rename table " + tableName, e);
    }
}
 

@OnStopped
public final void closeClient() throws KuduException {
    if (kuduClient != null) {
        getLogger().info("Closing KuduClient");
        kuduClient.close();
        kuduClient = null;
    }
}
 

@Override
public void close() throws IOException {
    if (resultIterator != null) {
        try {
            resultIterator.close();
        } catch (KuduException e) {
            e.printStackTrace();
        }
    }
    if (kuduReader != null) {
        kuduReader.close();
        kuduReader = null;
    }
}
 

public static void main(String[] args) {
        ArrayList<String> master = new ArrayList<String>();
        master.add("10.104.132.75:7051");
        master.add("10.104.132.73:7051");
        master.add("10.104.132.72:7051");
        master.add("10.104.132.221:7051");
        master.add("10.104.132.223:7051");
        KuduClient client = createClient(master);
        try {
            List<MediaMeta> list = execute(client);
            for(MediaMeta mediaMeta:list){
                System.out.println("tableName:"+mediaMeta.getName());
                List<ColumnMeta>  columnMetaList = mediaMeta.getColumn();
                for (ColumnMeta columnMeta:columnMetaList){
                    System.out.println(columnMeta.getName());
                }
            }

        }catch (KuduException e){
            LOGGER.info("执行kudu查询时报错:",e);
            throw new RuntimeException(e);
        } finally{
            closeClient(client);
        }
/*        KuduClient client = createClient(master);
        try {
            List<String> tablesList = client.getTablesList().getTablesList();
            System.out.println("------????------");
            System.out.println(ArrayUtils.toString(tablesList));
            KuduTable kuduTable = client.openTable(tablesList.get(0));
            System.out.println(kuduTable.getName().split("\\.")[1]+" : "+kuduTable.getName());
            System.out.println("------end------");
        } catch (KuduException e) {
            e.printStackTrace();
        } finally {
            closeClient(client);
        }*/
    }
 

/**
 * 更新插入事件
 *
 * @param config
 * @param dml
 */
private void upsert(KuduMappingConfig config, Dml dml) {
    KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
    String configTable = kuduMapping.getTable();
    String configDatabase = kuduMapping.getDatabase();
    String table = dml.getTable();
    String database = dml.getDatabase();
    if (configTable.equals(table) && configDatabase.equals(database)) {
        List<Map<String, Object>> data = dml.getData();
        if (data == null || data.isEmpty()) {
            return;
        }
        try {
            int idx = 1;
            boolean completed = false;
            List<Map<String, Object>> dataList = new ArrayList<>();

            for (Map<String, Object> entry : data) {
                dataList.add(entry);
                idx++;
                if (idx % kuduMapping.getCommitBatch() == 0) {
                    kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
                    dataList.clear();
                    completed = true;
                }
            }
            if (!completed) {
                kuduTemplate.upsert(kuduMapping.getTargetTable(), dataList);
            }
        } catch (KuduException e) {
            logger.error(e.getMessage());
            logger.error("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
        }
    }

}
 

@Override
public void flush()
        throws KuduException
{
    kuduSession.flush(); //真正落地
    rowNumCnt = 0;
}
 

@Override
public void close() {
    if (resultIterator != null) {
        try {
            resultIterator.close();
        } catch (KuduException e) {
            e.printStackTrace();
        }
    }
}
 

@Override
public List<BoundedSource<T>> split(long desiredBundleSizeBytes, PipelineOptions options)
    throws KuduException {
  if (serializedToken != null) {
    return Collections.singletonList(this); // we are already a split

  } else {
    Stream<BoundedSource<T>> sources =
        spec.getKuduService().createTabletScanners(spec).stream()
            .map(s -> new KuduIO.KuduSource<T>(spec, spec.getCoder(), s));
    return sources.collect(Collectors.toList());
  }
}