org.apache.kudu.client.KuduScanner Java Examples

The following examples show how to use org.apache.kudu.client.KuduScanner. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduTestBase.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
protected void validateSingleKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(1, schema.getPrimaryKeyColumnCount());
    assertEquals(2, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertFalse(schema.getColumn("second").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals("s", rows.get(0).getString("second"));
}
 
Example #2
Source File: KuduTemplate.java    From canal with Apache License 2.0 6 votes vote down vote up
/**
 * 统计kudu表数据
 *
 * @param tableName
 * @return
 */
public long countRow(String tableName) {
    this.checkClient();
    long rowCount = 0L;
    try {
        KuduTable kuduTable = kuduClient.openTable(tableName);
        // 创建scanner扫描
        KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build();
        // 遍历数据
        while (scanner.hasMoreRows()) {
            while (scanner.nextRows().hasNext()) {
                rowCount++;
            }
        }
    } catch (KuduException e) {
        e.printStackTrace();
    }
    return rowCount;
}
 
Example #3
Source File: KuduOperations.java    From geowave with Apache License 2.0 6 votes vote down vote up
public List<Delete> getDeletions(
    final KuduTable table,
    final List<KuduPredicate> predicates,
    final Function<RowResult, PersistentKuduRow> adapter) throws KuduException {
  // TODO: Kudu Java API does not support deleting with predicates, so we first perform a scan and
  // then perform individual row deletions with the full primary key. This is inefficient, because
  // we need to read in entire rows in order to perform deletions.
  final KuduScannerBuilder scannerBuilder = getScannerBuilder(table);
  for (final KuduPredicate pred : predicates) {
    scannerBuilder.addPredicate(pred);
  }
  final KuduScanner scanner = scannerBuilder.build();
  final List<RowResultIterator> allResults = new ArrayList<>();
  while (scanner.hasMoreRows()) {
    allResults.add(scanner.nextRows());
  }
  final Iterator<Delete> deletions =
      Streams.stream(Iterators.concat(allResults.iterator())).map(result -> {
        final PersistentKuduRow row = adapter.apply(result);
        final Delete delete = table.newDelete();
        row.populatePartialRowPrimaryKey(delete.getRow());
        return delete;
      }).iterator();
  return Lists.newArrayList(deletions);
}
 
Example #4
Source File: KuduClientTestCommons.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
protected void lookUpAndPopulateRecord(UnitTestTablePojo keyInfo) throws Exception
{
  KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable)
      .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("introwkey"),
      KuduPredicate.ComparisonOp.EQUAL,keyInfo.getIntrowkey()))
      .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("stringrowkey"),
      KuduPredicate.ComparisonOp.EQUAL,keyInfo.getStringrowkey()))
      .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("timestamprowkey"),
      KuduPredicate.ComparisonOp.EQUAL,keyInfo.getTimestamprowkey()))
      .build();
  RowResultIterator rowResultItr = scanner.nextRows();
  while (rowResultItr.hasNext()) {
    RowResult thisRow = rowResultItr.next();
    keyInfo.setFloatdata(thisRow.getFloat("floatdata"));
    keyInfo.setBooldata(thisRow.getBoolean("booldata"));
    keyInfo.setBinarydata(thisRow.getBinary("binarydata"));
    keyInfo.setLongdata(thisRow.getLong("longdata"));
    keyInfo.setTimestampdata(thisRow.getLong("timestampdata"));
    keyInfo.setStringdata("stringdata");
    break;
  }
}
 
Example #5
Source File: KuduInputOperatorCommons.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public long countNumRowsInTable() throws Exception
{
  List<String> allProjectedCols = new ArrayList<>(
      unitTestStepwiseScanInputOperator.getKuduColNameToSchemaMapping().keySet());
  KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable)
      .setProjectedColumnNames(allProjectedCols)
      .build();
  long counter = 0;
  while (scanner.hasMoreRows()) {
    RowResultIterator rowResultItr = scanner.nextRows();
    while (rowResultItr.hasNext()) {
      RowResult thisRow = rowResultItr.next();
      counter += 1;
    }
  }
  return counter;
}
 
Example #6
Source File: KuduServiceImpl.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public boolean start() throws IOException {
  LOG.debug("Starting Kudu reader");
  client =
      new AsyncKuduClient.AsyncKuduClientBuilder(source.spec.getMasterAddresses())
          .build()
          .syncClient();

  if (source.serializedToken != null) {
    // tokens available if the source is already split
    scanner = KuduScanToken.deserializeIntoScanner(source.serializedToken, client);
  } else {
    KuduTable table = client.openTable(source.spec.getTable());
    KuduScanner.KuduScannerBuilder builder =
        table.getAsyncClient().syncClient().newScannerBuilder(table);

    configureBuilder(source.spec, table.getSchema(), builder);
    scanner = builder.build();
  }

  return advance();
}
 
Example #7
Source File: KuduOutput.java    From envelope with Apache License 2.0 6 votes vote down vote up
@Override
public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception {
  List<Row> existingForFilters = Lists.newArrayList();

  if (!filters.iterator().hasNext()) {
    return existingForFilters;
  }

  KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME));
  KuduScanner scanner = scannerForFilters(filters, table);

  long startTime = System.nanoTime();
  while (scanner.hasMoreRows()) {
    for (RowResult rowResult : scanner.nextRows()) {
      Row existing = resultAsRow(rowResult, table);

      existingForFilters.add(existing);
    }
  }
  long endTime = System.nanoTime();
  if (hasAccumulators()) {
    accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0);
  }

  return existingForFilters;
}
 
Example #8
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
private void validateMultiKey(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(2, schema.getPrimaryKeyColumnCount());
    assertEquals(3, schema.getColumnCount());

    assertTrue(schema.getColumn("first").isKey());
    assertTrue(schema.getColumn("second").isKey());

    assertFalse(schema.getColumn("third").isKey());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals(2, rows.get(0).getInt("second"));
    assertEquals("t", rows.get(0).getString("third"));
}
 
Example #9
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsC (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTsC values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTsC");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString(0));
    assertEquals(Timestamp.valueOf("2020-01-01 12:12:12.123"), rows.get(0).getTimestamp(1));
}
 
Example #10
Source File: KuduRecordSet.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public RecordCursor cursor()
{
    KuduScanner scanner = clientSession.createScanner(kuduSplit);
    Schema projectedSchema = scanner.getProjectionSchema();
    ImmutableMap.Builder<Integer, Integer> builder = ImmutableMap.builder();
    for (int i = 0; i < columns.size(); i++) {
        KuduColumnHandle handle = (KuduColumnHandle) columns.get(i);
        if (handle.isVirtualRowId()) {
            builder.put(i, ROW_ID_POSITION);
        }
        else {
            builder.put(i, projectedSchema.getColumnIndex(handle.getName()));
        }
    }

    return new KuduRecordCursor(scanner, getTable(), getColumnTypes(), builder.build());
}
 
Example #11
Source File: SchemaEmulationByTableNameConvention.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public List<String> listSchemaNames(KuduClient client)
{
    try {
        if (rawSchemasTable == null) {
            if (!client.tableExists(rawSchemasTableName)) {
                createAndFillSchemasTable(client);
            }
            rawSchemasTable = getSchemasTable(client);
        }

        KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build();
        RowResultIterator iterator = scanner.nextRows();
        ArrayList<String> result = new ArrayList<>();
        while (iterator != null) {
            for (RowResult row : iterator) {
                result.add(row.getString(0));
            }
            iterator = scanner.nextRows();
        }
        return result;
    }
    catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 
Example #12
Source File: KuduScanCustomizerTest.java    From syndesis with Apache License 2.0 6 votes vote down vote up
@Ignore
public void testBeforeConsumer() throws Exception {
    final Map<String, Object> options = new HashMap<>();

    customizer.customize(getComponent(), options);

    final KuduTable table = connection.openTable("impala::default.syndesis_todo");

    final List<String> projectColumns = new ArrayList<>(1);
    final Iterator<ColumnSchema> columns = table.getSchema().getColumns().iterator();

    while (columns.hasNext()) {
        projectColumns.add(columns.next().getName());
    }

    final KuduScanner scanner = connection.newScannerBuilder(table)
        .setProjectedColumnNames(projectColumns)
        .build();

    final Exchange inbound = new DefaultExchange(createCamelContext());
    inbound.getIn().setBody(scanner);
    getComponent().getBeforeConsumer().process(inbound);
}
 
Example #13
Source File: KuduConsumerTest.java    From syndesis with Apache License 2.0 6 votes vote down vote up
@Ignore
public void insertRow() throws KuduException, InterruptedException {
    MockEndpoint mock = getMockEndpoint("mock:result");
    mock.expectedMessageCount(1);

    assertMockEndpointsSatisfied();

    List<Exchange> exchanges = mock.getReceivedExchanges();
    assertEquals(1, exchanges.size());

    KuduScanner scanner = exchanges.get(0).getIn().getBody(KuduScanner.class);

    RowResultIterator results = scanner.nextRows();
    RowResult result = results.next();

    ColumnSchema columnByIndex = result.getSchema().getColumnByIndex(0);
    String name = columnByIndex.getName();

    assertEquals("id", name);
    assertEquals(46, result.getInt(0));
    assertEquals(10, result.getInt(1));
}
 
Example #14
Source File: KuduTableFactoryTest.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testExistingTable() throws Exception {
    // Creating a table
    tableEnv.sqlUpdate("CREATE TABLE TestTable12 (`first` STRING, `second` STRING) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTable12', 'kudu.masters'='" + kuduMasters + "', " +
            "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");

    tableEnv.sqlUpdate("INSERT INTO TestTable12 values ('f', 's')");
    tableEnv.execute("test");

    // Then another one in SQL that refers to the previously created one
    tableEnv.sqlUpdate("CREATE TABLE TestTable12b (`first` STRING, `second` STRING) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTable12', 'kudu.masters'='" + kuduMasters + "')");
    tableEnv.sqlUpdate("INSERT INTO TestTable12b values ('f2','s2')");
    tableEnv.execute("test2");

    // Validate that both insertions were into the same table
    KuduTable kuduTable = harness.getClient().openTable("TestTable12");
    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(2, rows.size());
    assertEquals("f", rows.get(0).getString("first"));
    assertEquals("s", rows.get(0).getString("second"));
    assertEquals("f2", rows.get(1).getString("first"));
    assertEquals("s2", rows.get(1).getString("second"));
}
 
Example #15
Source File: KuduClientSession.java    From presto with Apache License 2.0 5 votes vote down vote up
public KuduScanner createScanner(KuduSplit kuduSplit)
{
    try {
        return KuduScanToken.deserializeIntoScanner(kuduSplit.getSerializedScanToken(), client);
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #16
Source File: KuduLookupService.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<Record> lookup(Map<String, Object> coordinates) {

    //Scanner
    KuduScanner.KuduScannerBuilder builder = kuduClient.newScannerBuilder(table);

    builder.setProjectedColumnNames(columnNames);
    builder.replicaSelection(replicaSelection);

    //Only expecting one match
    builder.limit(1);

    coordinates.forEach((key,value)->
            builder.addPredicate(KuduPredicate.newComparisonPredicate(tableSchema.getColumn(key), KuduPredicate.ComparisonOp.EQUAL, value))
    );

    KuduScanner kuduScanner = builder.build();

    //Run lookup
    for ( RowResult row : kuduScanner){
        final Map<String, Object> values = new HashMap<>();
        for(String columnName : columnNames){
            Object object;
            if(row.getColumnType(columnName) == Type.BINARY){
                object = Base64.getEncoder().encodeToString(row.getBinaryCopy(columnName));
            } else {
                object = row.getObject(columnName);
            }
            values.put(columnName, object);
        }
        return Optional.of(new MapRecord(resultSchema, values));
    }

    //No match
    return Optional.empty();
}
 
Example #17
Source File: KuduRecordCursor.java    From presto with Apache License 2.0 5 votes vote down vote up
public KuduRecordCursor(KuduScanner scanner, KuduTable table, List<Type> columnTypes, Map<Integer, Integer> fieldMapping)
{
    this.scanner = requireNonNull(scanner, "scanner is null");
    this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null"));
    this.table = requireNonNull(table, "table is null");
    this.fieldMapping = ImmutableMap.copyOf(requireNonNull(fieldMapping, "fieldMapping is null"));
}
 
Example #18
Source File: KuduInputOperatorCommons.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public void truncateTable() throws Exception
{
  AbstractKuduPartitionScanner<UnitTestTablePojo,InputOperatorControlTuple> scannerForDeletingRows =
      unitTestStepwiseScanInputOperator.getScanner();
  List<KuduScanToken> scansForAllTablets = unitTestStepwiseScanInputOperator
      .getPartitioner().getKuduScanTokensForSelectAllColumns();
  ApexKuduConnection aCurrentConnection = scannerForDeletingRows.getConnectionPoolForThreads().get(0);
  KuduSession aSessionForDeletes = aCurrentConnection.getKuduClient().newSession();
  KuduTable currentTable = aCurrentConnection.getKuduTable();
  for ( KuduScanToken aTabletScanToken : scansForAllTablets) {
    KuduScanner aScanner = aTabletScanToken.intoScanner(aCurrentConnection.getKuduClient());
    while ( aScanner.hasMoreRows()) {
      RowResultIterator itrForRows = aScanner.nextRows();
      while ( itrForRows.hasNext()) {
        RowResult aRow = itrForRows.next();
        int intRowKey = aRow.getInt("introwkey");
        String stringRowKey = aRow.getString("stringrowkey");
        long timestampRowKey = aRow.getLong("timestamprowkey");
        Delete aDeleteOp = currentTable.newDelete();
        aDeleteOp.getRow().addInt("introwkey",intRowKey);
        aDeleteOp.getRow().addString("stringrowkey", stringRowKey);
        aDeleteOp.getRow().addLong("timestamprowkey",timestampRowKey);
        aSessionForDeletes.apply(aDeleteOp);
      }
    }
  }
  aSessionForDeletes.close();
  Thread.sleep(2000); // Sleep to allow for scans to complete
}
 
Example #19
Source File: KuduOutput.java    From envelope with Apache License 2.0 5 votes vote down vote up
private KuduScanner scannerForFilters(Iterable<Row> filters, KuduTable table) throws KuduException {
  List<Row> filtersList = Lists.newArrayList(filters);

  if (filtersList.size() == 0) {
    throw new RuntimeException("Kudu existing filter was not provided.");
  }
  
  if (filtersList.get(0).schema() == null) {
    throw new RuntimeException("Kudu existing filter did not contain a schema.");
  }
  
  if (hasAccumulators()) {
    accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_SCANNERS).add(1);
    accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_FILTERS_SCANNED).add(filtersList.size());
  }
  
  KuduScannerBuilder builder = getConnection().getClient().newScannerBuilder(table);

  for (String fieldName : filtersList.get(0).schema().fieldNames()) {
    ColumnSchema columnSchema = table.getSchema().getColumn(fieldName);

    List<Object> columnValues = Lists.newArrayList();
    for (Row filter : filtersList) {
      Object columnValue = filter.getAs(fieldName);
      columnValues.add(columnValue);
    }

    KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, columnValues);

    builder = builder.addPredicate(predicate);
  }

  KuduScanner scanner = builder.build();

  return scanner;
}
 
Example #20
Source File: KuduRecordCursorWithVirtualRowId.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
public KuduRecordCursorWithVirtualRowId(KuduScanner scanner, KuduTable table,
                                        List<Type> columnTypes,
                                        Map<Integer, Integer> fieldMapping) {
    super(scanner, columnTypes);
    this.table = table;
    this.fieldMapping = fieldMapping;
}
 
Example #21
Source File: KuduTableFactoryTest.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestamp() throws Exception {
    // Timestamp should be bridged to sql.Timestamp
    // Test it when creating the table...
    tableEnv.sqlUpdate("CREATE TABLE TestTableTs (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "', " +
            "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')");
    tableEnv.sqlUpdate("INSERT INTO TestTableTs values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')");
    tableEnv.execute("test");

    // And also when inserting into existing table
    tableEnv.sqlUpdate("CREATE TABLE TestTableTsE (`first` STRING, `second` TIMESTAMP(3)) " +
            "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "')");

    tableEnv.sqlUpdate("INSERT INTO TestTableTsE values ('s', TIMESTAMP '2020-02-02 23:23:23')");
    tableEnv.execute("test");

    KuduTable kuduTable = harness.getClient().openTable("TestTableTs");
    assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    HashSet<Timestamp> results = new HashSet<>();
    scanner.forEach(sc -> results.add(sc.getTimestamp("second")));

    assertEquals(2, results.size());
    List<Timestamp> expected = Lists.newArrayList(
            Timestamp.valueOf("2020-01-01 12:12:12.123"),
            Timestamp.valueOf("2020-02-02 23:23:23"));
    assertEquals(new HashSet<>(expected), results);
}
 
Example #22
Source File: KuduCatalogTest.java    From bahir-flink with Apache License 2.0 5 votes vote down vote up
private void validateManyTypes(String tableName) throws Exception {
    KuduTable kuduTable = harness.getClient().openTable(tableName);
    Schema schema = kuduTable.getSchema();

    assertEquals(Type.STRING, schema.getColumn("first").getType());
    assertEquals(Type.BOOL, schema.getColumn("second").getType());
    assertEquals(Type.BINARY, schema.getColumn("third").getType());
    assertEquals(Type.INT8, schema.getColumn("fourth").getType());
    assertEquals(Type.INT16, schema.getColumn("fifth").getType());
    assertEquals(Type.INT32, schema.getColumn("sixth").getType());
    assertEquals(Type.INT64, schema.getColumn("seventh").getType());
    assertEquals(Type.FLOAT, schema.getColumn("eighth").getType());
    assertEquals(Type.DOUBLE, schema.getColumn("ninth").getType());
    assertEquals(Type.UNIXTIME_MICROS, schema.getColumn("tenth").getType());

    KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build();
    List<RowResult> rows = new ArrayList<>();
    scanner.forEach(rows::add);

    assertEquals(1, rows.size());
    assertEquals("f", rows.get(0).getString(0));
    assertEquals(false, rows.get(0).getBoolean(1));
    assertEquals(ByteBuffer.wrap("bbbb".getBytes()), rows.get(0).getBinary(2));
    assertEquals(12, rows.get(0).getByte(3));
    assertEquals(34, rows.get(0).getShort(4));
    assertEquals(56, rows.get(0).getInt(5));
    assertEquals(78, rows.get(0).getLong(6));
    assertEquals(3.14, rows.get(0).getFloat(7), 0.01);
    assertEquals(1.2345, rows.get(0).getDouble(8), 0.0001);
    assertEquals(Timestamp.valueOf("2020-04-15 12:34:56.123"), rows.get(0).getTimestamp(9));
}
 
Example #23
Source File: KuduRecordSet.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
@Override
public RecordCursor cursor() {
    KuduScanner scanner = clientSession.createScanner(kuduSplit);
    if (!containsVirtualRowId) {
        return new KuduRecordCursor(scanner, getColumnTypes());
    } else {
        final int primaryKeyColumnCount = kuduSplit.getPrimaryKeyColumnCount();

        Map<Integer, Integer> fieldMapping = new HashMap<>();
        int index = primaryKeyColumnCount;
        for (int i = 0; i < columns.size(); i++) {
            KuduColumnHandle handle = (KuduColumnHandle) columns.get(i);
            if (!handle.isVirtualRowId()) {
                if (handle.getOrdinalPosition() < primaryKeyColumnCount) {
                    fieldMapping.put(i, handle.getOrdinalPosition());
                } else {
                    fieldMapping.put(i, index);
                    index++;
                }
            } else {
                fieldMapping.put(i, -1);
            }
        }

        KuduTable table = getTable();
        return new KuduRecordCursorWithVirtualRowId(scanner, table, getColumnTypes(), fieldMapping);
    }
}
 
Example #24
Source File: KuduRecordCursor.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
public KuduRecordCursor(KuduScanner scanner, List<Type> columnTypes) {
    this.scanner = scanner;
    this.columnTypes = columnTypes;
    Field field = null;
    try {
        field = RowResult.class.getDeclaredField("rawData");
        field.setAccessible(true);
    } catch (NoSuchFieldException e) {
        // ignore
    }
    this.rowDataField = field;
}
 
Example #25
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
@Override
public KuduScanner createScanner(KuduSplit kuduSplit) {
    try {
        KuduScanner scanner = KuduScanToken.deserializeIntoScanner(kuduSplit.getPb(), client);
        return scanner;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #26
Source File: NativeKuduClientSession.java    From presto-kudu with Apache License 2.0 5 votes vote down vote up
@Override
public List<String> listSchemaNames() {
    try {
        if (rawSchemasTable == null) {
            if (!client.tableExists(rawSchemasTableName)) {
                createAndFillSchemasTable();
            }
            rawSchemasTable = getSchemasTable();
        }

        ColumnSchema tenantColumn = rawSchemasTable.getSchema().getColumnByIndex(0);
        KuduScanner scanner = client.newScannerBuilder(rawSchemasTable)
                .addPredicate(KuduPredicate.newComparisonPredicate(tenantColumn, KuduPredicate.ComparisonOp.EQUAL, tenantPrefix))
                .setProjectedColumnIndexes(ImmutableList.of(1))
                .build();
        RowResultIterator iterator = scanner.nextRows();
        ArrayList<String> result = new ArrayList<>();
        while (iterator != null) {
            for (RowResult row : iterator) {
                result.add(row.getString(0));
            }
            iterator = scanner.nextRows();
        }
        return result;
    } catch (KuduException e) {
        throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
    }
}
 
Example #27
Source File: KuduScanCustomizer.java    From syndesis with Apache License 2.0 4 votes vote down vote up
private static void processBody(Exchange exchange) throws KuduException, JsonProcessingException {
    final Message in = exchange.getIn();
    final KuduScanner scanner = in.getBody(KuduScanner.class);

    final List<String> answer = new ArrayList<>();
    while(scanner.hasMoreRows()) {
        RowResultIterator results = scanner.nextRows();

        while (results.hasNext()) {
            Map<String, Object> row = new HashMap<String, Object>();
            RowResult result = results.next();

            for (int i = 0; i < result.getSchema().getColumnCount(); i++) {
                String key = result.getSchema().getColumnByIndex(i).getName();
                Type type = result.getColumnType(i);

                switch (type.getName()) {
                    case "string":
                        row.put(key, result.getString(i));
                        break;
                    case "bool":
                        row.put(key, result.getBoolean(i));
                        break;
                    case "int8":
                    case "int16":
                    case "int32":
                        row.put(key, result.getInt(i));
                        break;
                    case "int64":
                        row.put(key, result.getLong(i));
                        break;
                    case "double":
                        row.put(key, result.getDouble(i));
                        break;
                    case "float":
                        row.put(key, result.getFloat(i));
                        break;
                    default:
                        throw new SyndesisServerException("The column schema type " + type.getName()
                                + " for column " + key
                                + " is not supported at the moment");
                }
            }
            answer.add(KuduSupport.toJSONBean(row));
        }
    }

    in.setBody(answer);
}
 
Example #28
Source File: KuduRowIterator.java    From flink-learning with Apache License 2.0 4 votes vote down vote up
public KuduRowIterator(KuduScanner scanner) throws KuduException {
    this.scanner = scanner;
    nextRows();
}
 
Example #29
Source File: KuduPartitionScannerCallable.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
@Override
public Long call() throws Exception
{
  long numRowsScanned = 0;
  KuduScanner aPartitionSpecificScanner = KuduScanToken.deserializeIntoScanner(
      kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle);
  LOG.info("Scanning the following tablet " + KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta
      .getSerializedKuduScanToken(), kuduClientHandle));
  KuduRecordWithMeta<T> beginScanRecord = new KuduRecordWithMeta<>();
  beginScanRecord.setBeginScanMarker(true);
  beginScanRecord.setTabletMetadata(kuduPartitionScanAssignmentMeta);
  bufferForTransmittingRecords.add(beginScanRecord); // Add a record entry that denotes the end of this scan.
  while ( aPartitionSpecificScanner.hasMoreRows()) {
    LOG.debug("Number of columns being returned for this read " +
        aPartitionSpecificScanner.getProjectionSchema().getColumnCount());
    RowResultIterator resultIterator = aPartitionSpecificScanner.nextRows();
    if (resultIterator == null) {
      break;
    } else {
      while (resultIterator.hasNext()) {
        KuduRecordWithMeta<T> recordWithMeta = new KuduRecordWithMeta<>();
        RowResult aRow = resultIterator.next();
        recordWithMeta.setPositionInScan(numRowsScanned);
        T payload = clazzForResultObject.newInstance();
        recordWithMeta.setThePayload(payload);
        recordWithMeta.setEndOfScanMarker(false);
        recordWithMeta.setTabletMetadata(kuduPartitionScanAssignmentMeta);
        setValuesInPOJO(aRow,payload);
        bufferForTransmittingRecords.add(recordWithMeta);
        numRowsScanned += 1;
      }
    }
  }
  aPartitionSpecificScanner.close();
  KuduRecordWithMeta<T> endScanRecord = new KuduRecordWithMeta<>();
  endScanRecord.setEndOfScanMarker(true);
  endScanRecord.setTabletMetadata(kuduPartitionScanAssignmentMeta);
  bufferForTransmittingRecords.add(endScanRecord); // Add a record entry that denotes the end of this scan.
  LOG.info(" Scanned a total of " + numRowsScanned + " for this scanner thread @tablet " +
      KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(),
      kuduClientHandle));
  return numRowsScanned;
}
 
Example #30
Source File: KuduReaderIterator.java    From bahir-flink with Apache License 2.0 4 votes vote down vote up
public KuduReaderIterator(KuduScanner scanner) throws KuduException {
    this.scanner = scanner;
    nextRows();
}