org.apache.kudu.client.KuduScanner Java Examples
The following examples show how to use
org.apache.kudu.client.KuduScanner.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KuduTestBase.java From bahir-flink with Apache License 2.0 | 6 votes |
protected void validateSingleKey(String tableName) throws Exception { KuduTable kuduTable = harness.getClient().openTable(tableName); Schema schema = kuduTable.getSchema(); assertEquals(1, schema.getPrimaryKeyColumnCount()); assertEquals(2, schema.getColumnCount()); assertTrue(schema.getColumn("first").isKey()); assertFalse(schema.getColumn("second").isKey()); KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build(); List<RowResult> rows = new ArrayList<>(); scanner.forEach(rows::add); assertEquals(1, rows.size()); assertEquals("f", rows.get(0).getString("first")); assertEquals("s", rows.get(0).getString("second")); }
Example #2
Source File: KuduTemplate.java From canal with Apache License 2.0 | 6 votes |
/** * 统计kudu表数据 * * @param tableName * @return */ public long countRow(String tableName) { this.checkClient(); long rowCount = 0L; try { KuduTable kuduTable = kuduClient.openTable(tableName); // 创建scanner扫描 KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable).build(); // 遍历数据 while (scanner.hasMoreRows()) { while (scanner.nextRows().hasNext()) { rowCount++; } } } catch (KuduException e) { e.printStackTrace(); } return rowCount; }
Example #3
Source File: KuduOperations.java From geowave with Apache License 2.0 | 6 votes |
public List<Delete> getDeletions( final KuduTable table, final List<KuduPredicate> predicates, final Function<RowResult, PersistentKuduRow> adapter) throws KuduException { // TODO: Kudu Java API does not support deleting with predicates, so we first perform a scan and // then perform individual row deletions with the full primary key. This is inefficient, because // we need to read in entire rows in order to perform deletions. final KuduScannerBuilder scannerBuilder = getScannerBuilder(table); for (final KuduPredicate pred : predicates) { scannerBuilder.addPredicate(pred); } final KuduScanner scanner = scannerBuilder.build(); final List<RowResultIterator> allResults = new ArrayList<>(); while (scanner.hasMoreRows()) { allResults.add(scanner.nextRows()); } final Iterator<Delete> deletions = Streams.stream(Iterators.concat(allResults.iterator())).map(result -> { final PersistentKuduRow row = adapter.apply(result); final Delete delete = table.newDelete(); row.populatePartialRowPrimaryKey(delete.getRow()); return delete; }).iterator(); return Lists.newArrayList(deletions); }
Example #4
Source File: KuduClientTestCommons.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
protected void lookUpAndPopulateRecord(UnitTestTablePojo keyInfo) throws Exception { KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable) .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("introwkey"), KuduPredicate.ComparisonOp.EQUAL,keyInfo.getIntrowkey())) .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("stringrowkey"), KuduPredicate.ComparisonOp.EQUAL,keyInfo.getStringrowkey())) .addPredicate(KuduPredicate.newComparisonPredicate(columnDefs.get("timestamprowkey"), KuduPredicate.ComparisonOp.EQUAL,keyInfo.getTimestamprowkey())) .build(); RowResultIterator rowResultItr = scanner.nextRows(); while (rowResultItr.hasNext()) { RowResult thisRow = rowResultItr.next(); keyInfo.setFloatdata(thisRow.getFloat("floatdata")); keyInfo.setBooldata(thisRow.getBoolean("booldata")); keyInfo.setBinarydata(thisRow.getBinary("binarydata")); keyInfo.setLongdata(thisRow.getLong("longdata")); keyInfo.setTimestampdata(thisRow.getLong("timestampdata")); keyInfo.setStringdata("stringdata"); break; } }
Example #5
Source File: KuduInputOperatorCommons.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
public long countNumRowsInTable() throws Exception { List<String> allProjectedCols = new ArrayList<>( unitTestStepwiseScanInputOperator.getKuduColNameToSchemaMapping().keySet()); KuduScanner scanner = kuduClient.newScannerBuilder(kuduTable) .setProjectedColumnNames(allProjectedCols) .build(); long counter = 0; while (scanner.hasMoreRows()) { RowResultIterator rowResultItr = scanner.nextRows(); while (rowResultItr.hasNext()) { RowResult thisRow = rowResultItr.next(); counter += 1; } } return counter; }
Example #6
Source File: KuduServiceImpl.java From beam with Apache License 2.0 | 6 votes |
@Override public boolean start() throws IOException { LOG.debug("Starting Kudu reader"); client = new AsyncKuduClient.AsyncKuduClientBuilder(source.spec.getMasterAddresses()) .build() .syncClient(); if (source.serializedToken != null) { // tokens available if the source is already split scanner = KuduScanToken.deserializeIntoScanner(source.serializedToken, client); } else { KuduTable table = client.openTable(source.spec.getTable()); KuduScanner.KuduScannerBuilder builder = table.getAsyncClient().syncClient().newScannerBuilder(table); configureBuilder(source.spec, table.getSchema(), builder); scanner = builder.build(); } return advance(); }
Example #7
Source File: KuduOutput.java From envelope with Apache License 2.0 | 6 votes |
@Override public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception { List<Row> existingForFilters = Lists.newArrayList(); if (!filters.iterator().hasNext()) { return existingForFilters; } KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME)); KuduScanner scanner = scannerForFilters(filters, table); long startTime = System.nanoTime(); while (scanner.hasMoreRows()) { for (RowResult rowResult : scanner.nextRows()) { Row existing = resultAsRow(rowResult, table); existingForFilters.add(existing); } } long endTime = System.nanoTime(); if (hasAccumulators()) { accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0); } return existingForFilters; }
Example #8
Source File: KuduCatalogTest.java From bahir-flink with Apache License 2.0 | 6 votes |
private void validateMultiKey(String tableName) throws Exception { KuduTable kuduTable = harness.getClient().openTable(tableName); Schema schema = kuduTable.getSchema(); assertEquals(2, schema.getPrimaryKeyColumnCount()); assertEquals(3, schema.getColumnCount()); assertTrue(schema.getColumn("first").isKey()); assertTrue(schema.getColumn("second").isKey()); assertFalse(schema.getColumn("third").isKey()); KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build(); List<RowResult> rows = new ArrayList<>(); scanner.forEach(rows::add); assertEquals(1, rows.size()); assertEquals("f", rows.get(0).getString("first")); assertEquals(2, rows.get(0).getInt("second")); assertEquals("t", rows.get(0).getString("third")); }
Example #9
Source File: KuduCatalogTest.java From bahir-flink with Apache License 2.0 | 6 votes |
@Test public void testTimestamp() throws Exception { tableEnv.sqlUpdate("CREATE TABLE TestTableTsC (`first` STRING, `second` TIMESTAMP(3)) " + "WITH ('kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')"); tableEnv.sqlUpdate("INSERT INTO TestTableTsC values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')"); tableEnv.execute("test"); KuduTable kuduTable = harness.getClient().openTable("TestTableTsC"); assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType()); KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build(); List<RowResult> rows = new ArrayList<>(); scanner.forEach(rows::add); assertEquals(1, rows.size()); assertEquals("f", rows.get(0).getString(0)); assertEquals(Timestamp.valueOf("2020-01-01 12:12:12.123"), rows.get(0).getTimestamp(1)); }
Example #10
Source File: KuduRecordSet.java From presto with Apache License 2.0 | 6 votes |
@Override public RecordCursor cursor() { KuduScanner scanner = clientSession.createScanner(kuduSplit); Schema projectedSchema = scanner.getProjectionSchema(); ImmutableMap.Builder<Integer, Integer> builder = ImmutableMap.builder(); for (int i = 0; i < columns.size(); i++) { KuduColumnHandle handle = (KuduColumnHandle) columns.get(i); if (handle.isVirtualRowId()) { builder.put(i, ROW_ID_POSITION); } else { builder.put(i, projectedSchema.getColumnIndex(handle.getName())); } } return new KuduRecordCursor(scanner, getTable(), getColumnTypes(), builder.build()); }
Example #11
Source File: SchemaEmulationByTableNameConvention.java From presto with Apache License 2.0 | 6 votes |
@Override public List<String> listSchemaNames(KuduClient client) { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(client); } rawSchemasTable = getSchemasTable(client); } KuduScanner scanner = client.newScannerBuilder(rawSchemasTable).build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
Example #12
Source File: KuduScanCustomizerTest.java From syndesis with Apache License 2.0 | 6 votes |
@Ignore public void testBeforeConsumer() throws Exception { final Map<String, Object> options = new HashMap<>(); customizer.customize(getComponent(), options); final KuduTable table = connection.openTable("impala::default.syndesis_todo"); final List<String> projectColumns = new ArrayList<>(1); final Iterator<ColumnSchema> columns = table.getSchema().getColumns().iterator(); while (columns.hasNext()) { projectColumns.add(columns.next().getName()); } final KuduScanner scanner = connection.newScannerBuilder(table) .setProjectedColumnNames(projectColumns) .build(); final Exchange inbound = new DefaultExchange(createCamelContext()); inbound.getIn().setBody(scanner); getComponent().getBeforeConsumer().process(inbound); }
Example #13
Source File: KuduConsumerTest.java From syndesis with Apache License 2.0 | 6 votes |
@Ignore public void insertRow() throws KuduException, InterruptedException { MockEndpoint mock = getMockEndpoint("mock:result"); mock.expectedMessageCount(1); assertMockEndpointsSatisfied(); List<Exchange> exchanges = mock.getReceivedExchanges(); assertEquals(1, exchanges.size()); KuduScanner scanner = exchanges.get(0).getIn().getBody(KuduScanner.class); RowResultIterator results = scanner.nextRows(); RowResult result = results.next(); ColumnSchema columnByIndex = result.getSchema().getColumnByIndex(0); String name = columnByIndex.getName(); assertEquals("id", name); assertEquals(46, result.getInt(0)); assertEquals(10, result.getInt(1)); }
Example #14
Source File: KuduTableFactoryTest.java From bahir-flink with Apache License 2.0 | 5 votes |
@Test public void testExistingTable() throws Exception { // Creating a table tableEnv.sqlUpdate("CREATE TABLE TestTable12 (`first` STRING, `second` STRING) " + "WITH ('connector.type'='kudu', 'kudu.table'='TestTable12', 'kudu.masters'='" + kuduMasters + "', " + "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')"); tableEnv.sqlUpdate("INSERT INTO TestTable12 values ('f', 's')"); tableEnv.execute("test"); // Then another one in SQL that refers to the previously created one tableEnv.sqlUpdate("CREATE TABLE TestTable12b (`first` STRING, `second` STRING) " + "WITH ('connector.type'='kudu', 'kudu.table'='TestTable12', 'kudu.masters'='" + kuduMasters + "')"); tableEnv.sqlUpdate("INSERT INTO TestTable12b values ('f2','s2')"); tableEnv.execute("test2"); // Validate that both insertions were into the same table KuduTable kuduTable = harness.getClient().openTable("TestTable12"); KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build(); List<RowResult> rows = new ArrayList<>(); scanner.forEach(rows::add); assertEquals(2, rows.size()); assertEquals("f", rows.get(0).getString("first")); assertEquals("s", rows.get(0).getString("second")); assertEquals("f2", rows.get(1).getString("first")); assertEquals("s2", rows.get(1).getString("second")); }
Example #15
Source File: KuduClientSession.java From presto with Apache License 2.0 | 5 votes |
public KuduScanner createScanner(KuduSplit kuduSplit) { try { return KuduScanToken.deserializeIntoScanner(kuduSplit.getSerializedScanToken(), client); } catch (IOException e) { throw new RuntimeException(e); } }
Example #16
Source File: KuduLookupService.java From nifi with Apache License 2.0 | 5 votes |
@Override public Optional<Record> lookup(Map<String, Object> coordinates) { //Scanner KuduScanner.KuduScannerBuilder builder = kuduClient.newScannerBuilder(table); builder.setProjectedColumnNames(columnNames); builder.replicaSelection(replicaSelection); //Only expecting one match builder.limit(1); coordinates.forEach((key,value)-> builder.addPredicate(KuduPredicate.newComparisonPredicate(tableSchema.getColumn(key), KuduPredicate.ComparisonOp.EQUAL, value)) ); KuduScanner kuduScanner = builder.build(); //Run lookup for ( RowResult row : kuduScanner){ final Map<String, Object> values = new HashMap<>(); for(String columnName : columnNames){ Object object; if(row.getColumnType(columnName) == Type.BINARY){ object = Base64.getEncoder().encodeToString(row.getBinaryCopy(columnName)); } else { object = row.getObject(columnName); } values.put(columnName, object); } return Optional.of(new MapRecord(resultSchema, values)); } //No match return Optional.empty(); }
Example #17
Source File: KuduRecordCursor.java From presto with Apache License 2.0 | 5 votes |
public KuduRecordCursor(KuduScanner scanner, KuduTable table, List<Type> columnTypes, Map<Integer, Integer> fieldMapping) { this.scanner = requireNonNull(scanner, "scanner is null"); this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); this.table = requireNonNull(table, "table is null"); this.fieldMapping = ImmutableMap.copyOf(requireNonNull(fieldMapping, "fieldMapping is null")); }
Example #18
Source File: KuduInputOperatorCommons.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public void truncateTable() throws Exception { AbstractKuduPartitionScanner<UnitTestTablePojo,InputOperatorControlTuple> scannerForDeletingRows = unitTestStepwiseScanInputOperator.getScanner(); List<KuduScanToken> scansForAllTablets = unitTestStepwiseScanInputOperator .getPartitioner().getKuduScanTokensForSelectAllColumns(); ApexKuduConnection aCurrentConnection = scannerForDeletingRows.getConnectionPoolForThreads().get(0); KuduSession aSessionForDeletes = aCurrentConnection.getKuduClient().newSession(); KuduTable currentTable = aCurrentConnection.getKuduTable(); for ( KuduScanToken aTabletScanToken : scansForAllTablets) { KuduScanner aScanner = aTabletScanToken.intoScanner(aCurrentConnection.getKuduClient()); while ( aScanner.hasMoreRows()) { RowResultIterator itrForRows = aScanner.nextRows(); while ( itrForRows.hasNext()) { RowResult aRow = itrForRows.next(); int intRowKey = aRow.getInt("introwkey"); String stringRowKey = aRow.getString("stringrowkey"); long timestampRowKey = aRow.getLong("timestamprowkey"); Delete aDeleteOp = currentTable.newDelete(); aDeleteOp.getRow().addInt("introwkey",intRowKey); aDeleteOp.getRow().addString("stringrowkey", stringRowKey); aDeleteOp.getRow().addLong("timestamprowkey",timestampRowKey); aSessionForDeletes.apply(aDeleteOp); } } } aSessionForDeletes.close(); Thread.sleep(2000); // Sleep to allow for scans to complete }
Example #19
Source File: KuduOutput.java From envelope with Apache License 2.0 | 5 votes |
private KuduScanner scannerForFilters(Iterable<Row> filters, KuduTable table) throws KuduException { List<Row> filtersList = Lists.newArrayList(filters); if (filtersList.size() == 0) { throw new RuntimeException("Kudu existing filter was not provided."); } if (filtersList.get(0).schema() == null) { throw new RuntimeException("Kudu existing filter did not contain a schema."); } if (hasAccumulators()) { accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_SCANNERS).add(1); accumulators.getLongAccumulators().get(ACCUMULATOR_NUMBER_OF_FILTERS_SCANNED).add(filtersList.size()); } KuduScannerBuilder builder = getConnection().getClient().newScannerBuilder(table); for (String fieldName : filtersList.get(0).schema().fieldNames()) { ColumnSchema columnSchema = table.getSchema().getColumn(fieldName); List<Object> columnValues = Lists.newArrayList(); for (Row filter : filtersList) { Object columnValue = filter.getAs(fieldName); columnValues.add(columnValue); } KuduPredicate predicate = KuduPredicate.newInListPredicate(columnSchema, columnValues); builder = builder.addPredicate(predicate); } KuduScanner scanner = builder.build(); return scanner; }
Example #20
Source File: KuduRecordCursorWithVirtualRowId.java From presto-kudu with Apache License 2.0 | 5 votes |
public KuduRecordCursorWithVirtualRowId(KuduScanner scanner, KuduTable table, List<Type> columnTypes, Map<Integer, Integer> fieldMapping) { super(scanner, columnTypes); this.table = table; this.fieldMapping = fieldMapping; }
Example #21
Source File: KuduTableFactoryTest.java From bahir-flink with Apache License 2.0 | 5 votes |
@Test public void testTimestamp() throws Exception { // Timestamp should be bridged to sql.Timestamp // Test it when creating the table... tableEnv.sqlUpdate("CREATE TABLE TestTableTs (`first` STRING, `second` TIMESTAMP(3)) " + "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "', " + "'kudu.hash-columns'='first', 'kudu.primary-key-columns'='first')"); tableEnv.sqlUpdate("INSERT INTO TestTableTs values ('f', TIMESTAMP '2020-01-01 12:12:12.123456')"); tableEnv.execute("test"); // And also when inserting into existing table tableEnv.sqlUpdate("CREATE TABLE TestTableTsE (`first` STRING, `second` TIMESTAMP(3)) " + "WITH ('connector.type'='kudu', 'kudu.table'='TestTableTs', 'kudu.masters'='" + kuduMasters + "')"); tableEnv.sqlUpdate("INSERT INTO TestTableTsE values ('s', TIMESTAMP '2020-02-02 23:23:23')"); tableEnv.execute("test"); KuduTable kuduTable = harness.getClient().openTable("TestTableTs"); assertEquals(Type.UNIXTIME_MICROS, kuduTable.getSchema().getColumn("second").getType()); KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build(); HashSet<Timestamp> results = new HashSet<>(); scanner.forEach(sc -> results.add(sc.getTimestamp("second"))); assertEquals(2, results.size()); List<Timestamp> expected = Lists.newArrayList( Timestamp.valueOf("2020-01-01 12:12:12.123"), Timestamp.valueOf("2020-02-02 23:23:23")); assertEquals(new HashSet<>(expected), results); }
Example #22
Source File: KuduCatalogTest.java From bahir-flink with Apache License 2.0 | 5 votes |
private void validateManyTypes(String tableName) throws Exception { KuduTable kuduTable = harness.getClient().openTable(tableName); Schema schema = kuduTable.getSchema(); assertEquals(Type.STRING, schema.getColumn("first").getType()); assertEquals(Type.BOOL, schema.getColumn("second").getType()); assertEquals(Type.BINARY, schema.getColumn("third").getType()); assertEquals(Type.INT8, schema.getColumn("fourth").getType()); assertEquals(Type.INT16, schema.getColumn("fifth").getType()); assertEquals(Type.INT32, schema.getColumn("sixth").getType()); assertEquals(Type.INT64, schema.getColumn("seventh").getType()); assertEquals(Type.FLOAT, schema.getColumn("eighth").getType()); assertEquals(Type.DOUBLE, schema.getColumn("ninth").getType()); assertEquals(Type.UNIXTIME_MICROS, schema.getColumn("tenth").getType()); KuduScanner scanner = harness.getClient().newScannerBuilder(kuduTable).build(); List<RowResult> rows = new ArrayList<>(); scanner.forEach(rows::add); assertEquals(1, rows.size()); assertEquals("f", rows.get(0).getString(0)); assertEquals(false, rows.get(0).getBoolean(1)); assertEquals(ByteBuffer.wrap("bbbb".getBytes()), rows.get(0).getBinary(2)); assertEquals(12, rows.get(0).getByte(3)); assertEquals(34, rows.get(0).getShort(4)); assertEquals(56, rows.get(0).getInt(5)); assertEquals(78, rows.get(0).getLong(6)); assertEquals(3.14, rows.get(0).getFloat(7), 0.01); assertEquals(1.2345, rows.get(0).getDouble(8), 0.0001); assertEquals(Timestamp.valueOf("2020-04-15 12:34:56.123"), rows.get(0).getTimestamp(9)); }
Example #23
Source File: KuduRecordSet.java From presto-kudu with Apache License 2.0 | 5 votes |
@Override public RecordCursor cursor() { KuduScanner scanner = clientSession.createScanner(kuduSplit); if (!containsVirtualRowId) { return new KuduRecordCursor(scanner, getColumnTypes()); } else { final int primaryKeyColumnCount = kuduSplit.getPrimaryKeyColumnCount(); Map<Integer, Integer> fieldMapping = new HashMap<>(); int index = primaryKeyColumnCount; for (int i = 0; i < columns.size(); i++) { KuduColumnHandle handle = (KuduColumnHandle) columns.get(i); if (!handle.isVirtualRowId()) { if (handle.getOrdinalPosition() < primaryKeyColumnCount) { fieldMapping.put(i, handle.getOrdinalPosition()); } else { fieldMapping.put(i, index); index++; } } else { fieldMapping.put(i, -1); } } KuduTable table = getTable(); return new KuduRecordCursorWithVirtualRowId(scanner, table, getColumnTypes(), fieldMapping); } }
Example #24
Source File: KuduRecordCursor.java From presto-kudu with Apache License 2.0 | 5 votes |
public KuduRecordCursor(KuduScanner scanner, List<Type> columnTypes) { this.scanner = scanner; this.columnTypes = columnTypes; Field field = null; try { field = RowResult.class.getDeclaredField("rawData"); field.setAccessible(true); } catch (NoSuchFieldException e) { // ignore } this.rowDataField = field; }
Example #25
Source File: NativeKuduClientSession.java From presto-kudu with Apache License 2.0 | 5 votes |
@Override public KuduScanner createScanner(KuduSplit kuduSplit) { try { KuduScanner scanner = KuduScanToken.deserializeIntoScanner(kuduSplit.getPb(), client); return scanner; } catch (IOException e) { throw new RuntimeException(e); } }
Example #26
Source File: NativeKuduClientSession.java From presto-kudu with Apache License 2.0 | 5 votes |
@Override public List<String> listSchemaNames() { try { if (rawSchemasTable == null) { if (!client.tableExists(rawSchemasTableName)) { createAndFillSchemasTable(); } rawSchemasTable = getSchemasTable(); } ColumnSchema tenantColumn = rawSchemasTable.getSchema().getColumnByIndex(0); KuduScanner scanner = client.newScannerBuilder(rawSchemasTable) .addPredicate(KuduPredicate.newComparisonPredicate(tenantColumn, KuduPredicate.ComparisonOp.EQUAL, tenantPrefix)) .setProjectedColumnIndexes(ImmutableList.of(1)) .build(); RowResultIterator iterator = scanner.nextRows(); ArrayList<String> result = new ArrayList<>(); while (iterator != null) { for (RowResult row : iterator) { result.add(row.getString(0)); } iterator = scanner.nextRows(); } return result; } catch (KuduException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, e); } }
Example #27
Source File: KuduScanCustomizer.java From syndesis with Apache License 2.0 | 4 votes |
private static void processBody(Exchange exchange) throws KuduException, JsonProcessingException { final Message in = exchange.getIn(); final KuduScanner scanner = in.getBody(KuduScanner.class); final List<String> answer = new ArrayList<>(); while(scanner.hasMoreRows()) { RowResultIterator results = scanner.nextRows(); while (results.hasNext()) { Map<String, Object> row = new HashMap<String, Object>(); RowResult result = results.next(); for (int i = 0; i < result.getSchema().getColumnCount(); i++) { String key = result.getSchema().getColumnByIndex(i).getName(); Type type = result.getColumnType(i); switch (type.getName()) { case "string": row.put(key, result.getString(i)); break; case "bool": row.put(key, result.getBoolean(i)); break; case "int8": case "int16": case "int32": row.put(key, result.getInt(i)); break; case "int64": row.put(key, result.getLong(i)); break; case "double": row.put(key, result.getDouble(i)); break; case "float": row.put(key, result.getFloat(i)); break; default: throw new SyndesisServerException("The column schema type " + type.getName() + " for column " + key + " is not supported at the moment"); } } answer.add(KuduSupport.toJSONBean(row)); } } in.setBody(answer); }
Example #28
Source File: KuduRowIterator.java From flink-learning with Apache License 2.0 | 4 votes |
public KuduRowIterator(KuduScanner scanner) throws KuduException { this.scanner = scanner; nextRows(); }
Example #29
Source File: KuduPartitionScannerCallable.java From attic-apex-malhar with Apache License 2.0 | 4 votes |
@Override public Long call() throws Exception { long numRowsScanned = 0; KuduScanner aPartitionSpecificScanner = KuduScanToken.deserializeIntoScanner( kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle); LOG.info("Scanning the following tablet " + KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta .getSerializedKuduScanToken(), kuduClientHandle)); KuduRecordWithMeta<T> beginScanRecord = new KuduRecordWithMeta<>(); beginScanRecord.setBeginScanMarker(true); beginScanRecord.setTabletMetadata(kuduPartitionScanAssignmentMeta); bufferForTransmittingRecords.add(beginScanRecord); // Add a record entry that denotes the end of this scan. while ( aPartitionSpecificScanner.hasMoreRows()) { LOG.debug("Number of columns being returned for this read " + aPartitionSpecificScanner.getProjectionSchema().getColumnCount()); RowResultIterator resultIterator = aPartitionSpecificScanner.nextRows(); if (resultIterator == null) { break; } else { while (resultIterator.hasNext()) { KuduRecordWithMeta<T> recordWithMeta = new KuduRecordWithMeta<>(); RowResult aRow = resultIterator.next(); recordWithMeta.setPositionInScan(numRowsScanned); T payload = clazzForResultObject.newInstance(); recordWithMeta.setThePayload(payload); recordWithMeta.setEndOfScanMarker(false); recordWithMeta.setTabletMetadata(kuduPartitionScanAssignmentMeta); setValuesInPOJO(aRow,payload); bufferForTransmittingRecords.add(recordWithMeta); numRowsScanned += 1; } } } aPartitionSpecificScanner.close(); KuduRecordWithMeta<T> endScanRecord = new KuduRecordWithMeta<>(); endScanRecord.setEndOfScanMarker(true); endScanRecord.setTabletMetadata(kuduPartitionScanAssignmentMeta); bufferForTransmittingRecords.add(endScanRecord); // Add a record entry that denotes the end of this scan. LOG.info(" Scanned a total of " + numRowsScanned + " for this scanner thread @tablet " + KuduScanToken.stringifySerializedToken(kuduPartitionScanAssignmentMeta.getSerializedKuduScanToken(), kuduClientHandle)); return numRowsScanned; }
Example #30
Source File: KuduReaderIterator.java From bahir-flink with Apache License 2.0 | 4 votes |
public KuduReaderIterator(KuduScanner scanner) throws KuduException { this.scanner = scanner; nextRows(); }