org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableMap Java Examples
The following examples show how to use
org.apache.flink.shaded.guava18.com.google.common.collect.ImmutableMap.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReaderCheckpointHookTest.java From flink-connectors with Apache License 2.0 | 6 votes |
@Test public void testRestore() throws Exception { ReaderGroup readerGroup = mock(ReaderGroup.class); Checkpoint checkpoint = mock(Checkpoint.class); CheckpointImpl checkpointImpl = mock(CheckpointImpl.class); when(checkpoint.asImpl()).thenReturn(checkpointImpl); when(checkpointImpl.getPositions()).thenReturn(ImmutableMap.<Stream, StreamCut>builder() .put(Stream.of(SCOPE, "s1"), getStreamCut("s1")) .put(Stream.of(SCOPE, "s2"), getStreamCut("s2")).build()); ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder() .disableAutomaticCheckpoints() .startFromCheckpoint(checkpoint) .build(); TestableReaderCheckpointHook hook = new TestableReaderCheckpointHook(HOOK_UID, readerGroup, Time.minutes(1), readerGroupConfig); hook.restoreCheckpoint(1L, checkpoint); verify(readerGroup).resetReaderGroup(readerGroupConfig); }
Example #2
Source File: TestSource.java From sylph with Apache License 2.0 | 6 votes |
@Override public void run(SourceContext<Row> sourceContext) throws Exception { Random random = new Random(); int numKeys = 10; long count = 1L; while (running) { long eventTime = System.currentTimeMillis() - random.nextInt(10 * 1000); //表示数据已经产生了 但是会有10秒以内的延迟 String userId = "uid_" + count; String msg = MAPPER.writeValueAsString(ImmutableMap.of("user_id", userId, "ip", "127.0.0.1", "store", 12.0)); Row row = Row.of("key" + random.nextInt(10), msg, eventTime); sourceContext.collect(row); count = count > numKeys ? 1L : count + 1; TimeUnit.MILLISECONDS.sleep(100); } }
Example #3
Source File: JsonPathUdfTest.java From sylph with Apache License 2.0 | 6 votes |
@Before public void init() throws JsonProcessingException { String json = MAPPER.writeValueAsString(ImmutableMap.of("user_id", "uid_001", "ip", "127.0.0.1", "store", 12.0, "key1", ImmutableMap.of("key2", 123) )); StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment(); execEnv.setParallelism(2); tableEnv = StreamTableEnvironment.create(execEnv); tableEnv.registerFunction("get_json_object", new UDFJson()); table = tableEnv.sqlQuery("select '" + json + "' as message"); }
Example #4
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws IOException { File jarWithNoManifest = createJarFileWithManifest(ImmutableMap.of()); File jarFile = TestJob.getTestJobJar(); JarManifestParser.JarFileWithEntryClass jarFileWithEntryClass = JarManifestParser .findOnlyEntryClass(ImmutableList.of(jarWithNoManifest, jarFile)); assertThat(jarFileWithEntryClass.getEntryClass(), is(equalTo(TestJob.class.getCanonicalName()))); }
Example #5
Source File: JarManifestParserTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassNoEntry() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of()); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertFalse(entry.isPresent()); }
Example #6
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassAssemblerClassAndMainClass() throws IOException { // We want the assembler class entry to have precedence over main class File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass", PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("AssemblerClass"))); }
Example #7
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassMainClass() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("MainClass"))); }
Example #8
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassAssemblerClass() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("AssemblerClass"))); }
Example #9
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassNoEntry() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of()); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertFalse(entry.isPresent()); }
Example #10
Source File: LeastUtilizationSlotMatchingStrategyTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void findMatchingSlot_multipleMatchingSlots_returnsSlotWithLeastUtilization() { final ResourceProfile requestedResourceProfile = ResourceProfile.fromResources(2.0, 2); final TestingTaskManagerSlotInformation leastUtilizedSlot = TestingTaskManagerSlotInformation.newBuilder() .setResourceProfile(requestedResourceProfile) .build(); final TestingTaskManagerSlotInformation tooSmallSlot = TestingTaskManagerSlotInformation.newBuilder() .setResourceProfile(ResourceProfile.fromResources(1.0, 10)) .build(); final TestingTaskManagerSlotInformation alternativeSlot = TestingTaskManagerSlotInformation.newBuilder() .setResourceProfile(requestedResourceProfile) .build(); final Collection<TestingTaskManagerSlotInformation> freeSlots = Arrays.asList( tooSmallSlot, leastUtilizedSlot, alternativeSlot); Map<InstanceID, Integer> registeredSlotPerTaskExecutor = ImmutableMap.of( leastUtilizedSlot.getInstanceId(), 1, tooSmallSlot.getInstanceId(), 1, alternativeSlot.getInstanceId(), 2); final Optional<TestingTaskManagerSlotInformation> matchingSlot = LeastUtilizationSlotMatchingStrategy.INSTANCE.findMatchingSlot( requestedResourceProfile, freeSlots, createRegisteredSlotsLookupFunction(registeredSlotPerTaskExecutor)); assertTrue(matchingSlot.isPresent()); assertThat(matchingSlot.get().getSlotId(), is(leastUtilizedSlot.getSlotId())); }
Example #11
Source File: SlotPoolImplTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCalculationOfTaskExecutorUtilization() throws Exception { try (final SlotPoolImpl slotPool = createSlotPoolImpl()) { setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor); final TaskManagerLocation firstTaskManagerLocation = new LocalTaskManagerLocation(); final TaskManagerLocation secondTaskManagerLocation = new LocalTaskManagerLocation(); final List<AllocationID> firstTaskManagersSlots = registerAndOfferSlots(firstTaskManagerLocation, slotPool, 4); final List<AllocationID> secondTaskManagersSlots = registerAndOfferSlots(secondTaskManagerLocation, slotPool, 4); slotPool.allocateAvailableSlot(new SlotRequestId(), firstTaskManagersSlots.get(0)); slotPool.allocateAvailableSlot(new SlotRequestId(), firstTaskManagersSlots.get(1)); slotPool.allocateAvailableSlot(new SlotRequestId(), secondTaskManagersSlots.get(3)); final Collection<SlotInfoWithUtilization> availableSlotsInformation = slotPool.getAvailableSlotsInformation(); final Map<TaskManagerLocation, Double> utilizationPerTaskExecutor = ImmutableMap.of( firstTaskManagerLocation, 2.0 / 4, secondTaskManagerLocation, 1.0 / 4); for (SlotInfoWithUtilization slotInfoWithUtilization : availableSlotsInformation) { final double expectedTaskExecutorUtilization = utilizationPerTaskExecutor.get(slotInfoWithUtilization.getTaskManagerLocation()); assertThat(slotInfoWithUtilization.getTaskExecutorUtilization(), is(closeTo(expectedTaskExecutorUtilization, 0.1))); } } }
Example #12
Source File: PluginConfigTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void getPluginsDir_nonExistingDirectory_returnsEmpty() { final Map<String, String> envVariables = ImmutableMap.of(ConfigConstants.ENV_FLINK_PLUGINS_DIR, new File(temporaryFolder.getRoot().getAbsoluteFile(), "should_not_exist").getAbsolutePath()); CommonTestUtils.setEnv(envVariables); assertFalse(PluginConfig.getPluginsDir().isPresent()); }
Example #13
Source File: PluginConfigTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void getPluginsDir_existingDirectory_returnsDirectoryFile() throws IOException { final File pluginsDirectory = temporaryFolder.newFolder(); final Map<String, String> envVariables = ImmutableMap.of(ConfigConstants.ENV_FLINK_PLUGINS_DIR, pluginsDirectory.getAbsolutePath()); CommonTestUtils.setEnv(envVariables); assertThat(PluginConfig.getPluginsDir().get(), is(pluginsDirectory)); }
Example #14
Source File: ParquetRecordReaderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testNestedMapGroup() throws IOException { Schema nestedMapSchema = unWrapSchema(NESTED_SCHEMA.getField("nestedMap").schema()); Preconditions.checkState(nestedMapSchema.getType().equals(Schema.Type.MAP)); Schema mapValueSchema = nestedMapSchema.getValueType(); GenericRecord mapValue = new GenericRecordBuilder(mapValueSchema) .set("type", "nested") .set("value", "nested_value").build(); ImmutableMap.Builder<String, GenericRecord> map = ImmutableMap.builder(); map.put("testKey", mapValue); GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA) .set("nestedMap", map.build()) .set("foo", 34L).build(); Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record)); MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA); ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema); InputFile inputFile = HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig); ParquetReadOptions options = ParquetReadOptions.builder().build(); ParquetFileReader fileReader = new ParquetFileReader(inputFile, options); rowReader.initialize(fileReader, testConfig); assertFalse(rowReader.reachEnd()); Row row = rowReader.nextRecord(); assertEquals(7, row.getArity()); assertEquals(34L, row.getField(0)); Map result = (Map) row.getField(5); Row nestedRow = (Row) result.get("testKey"); assertEquals("nested", nestedRow.getField(0)); assertEquals("nested_value", nestedRow.getField(1)); }
Example #15
Source File: ParquetRecordReaderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMapGroup() throws IOException { Preconditions.checkState(unWrapSchema(NESTED_SCHEMA.getField("spamMap").schema()) .getType().equals(Schema.Type.MAP)); ImmutableMap.Builder<String, String> map = ImmutableMap.builder(); map.put("testKey", "testValue"); GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA) .set("foo", 32L) .set("spamMap", map.build()) .build(); Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record)); MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA); ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema); InputFile inputFile = HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig); ParquetReadOptions options = ParquetReadOptions.builder().build(); ParquetFileReader fileReader = new ParquetFileReader(inputFile, options); rowReader.initialize(fileReader, testConfig); assertFalse(rowReader.reachEnd()); Row row = rowReader.nextRecord(); assertEquals(7, row.getArity()); assertEquals(32L, row.getField(0)); Map<?, ?> result = (Map<?, ?>) row.getField(1); assertEquals(result.get("testKey").toString(), "testValue"); assertTrue(rowReader.reachEnd()); }
Example #16
Source File: KafkaShuffleITCase.java From flink with Apache License 2.0 | 5 votes |
private Map<Integer, Collection<ConsumerRecord<byte[], byte[]>>> testKafkaShuffleProducer( String topic, StreamExecutionEnvironment env, int numberOfPartitions, int producerParallelism, int numElementsPerProducer, TimeCharacteristic timeCharacteristic) throws Exception { createTestTopic(topic, numberOfPartitions, 1); env.setParallelism(producerParallelism); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStreamTimeCharacteristic(timeCharacteristic); DataStream<Tuple3<Integer, Long, Integer>> source = env.addSource(new KafkaSourceFunction(numElementsPerProducer, false)).setParallelism(producerParallelism); DataStream<Tuple3<Integer, Long, Integer>> input = (timeCharacteristic == EventTime) ? source.assignTimestampsAndWatermarks(new PunctuatedExtractor()).setParallelism(producerParallelism) : source; Properties properties = kafkaServer.getStandardProperties(); Properties kafkaProperties = PropertiesUtil.flatten(properties); kafkaProperties.setProperty(PRODUCER_PARALLELISM, String.valueOf(producerParallelism)); kafkaProperties.setProperty(PARTITION_NUMBER, String.valueOf(numberOfPartitions)); kafkaProperties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); kafkaProperties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer"); FlinkKafkaShuffle.writeKeyBy(input, topic, kafkaProperties, 0); env.execute("Write to " + topic); ImmutableMap.Builder<Integer, Collection<ConsumerRecord<byte[], byte[]>>> results = ImmutableMap.builder(); for (int p = 0; p < numberOfPartitions; p++) { results.put(p, kafkaServer.getAllRecordsFromTopic(kafkaProperties, topic, p, 5000)); } deleteTestTopic(topic); return results.build(); }
Example #17
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassNoEntry() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of()); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertFalse(entry.isPresent()); }
Example #18
Source File: JarManifestParserTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassAssemblerClass() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("AssemblerClass"))); }
Example #19
Source File: JarManifestParserTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassMainClass() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("MainClass"))); }
Example #20
Source File: JarManifestParserTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassAssemblerClassAndMainClass() throws IOException { // We want the assembler class entry to have precedence over main class File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass", PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("AssemblerClass"))); }
Example #21
Source File: JarManifestParserTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws IOException { File jarWithNoManifest = createJarFileWithManifest(ImmutableMap.of()); File jarFile = TestJob.getTestJobJar(); JarFileWithEntryClass jarFileWithEntryClass = JarManifestParser .findOnlyEntryClass(ImmutableList.of(jarWithNoManifest, jarFile)); assertThat(jarFileWithEntryClass.getEntryClass(), is(equalTo(TestJob.class.getCanonicalName()))); }
Example #22
Source File: ParquetRecordReaderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMapGroup() throws IOException { Preconditions.checkState(unWrapSchema(NESTED_SCHEMA.getField("spamMap").schema()) .getType().equals(Schema.Type.MAP)); ImmutableMap.Builder<String, String> map = ImmutableMap.builder(); map.put("testKey", "testValue"); GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA) .set("foo", 32L) .set("spamMap", map.build()) .build(); Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record)); MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA); ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema); InputFile inputFile = HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig); ParquetReadOptions options = ParquetReadOptions.builder().build(); ParquetFileReader fileReader = new ParquetFileReader(inputFile, options); rowReader.initialize(fileReader, testConfig); assertFalse(rowReader.reachEnd()); Row row = rowReader.nextRecord(); assertEquals(7, row.getArity()); assertEquals(32L, row.getField(0)); Map<?, ?> result = (Map<?, ?>) row.getField(1); assertEquals(result.get("testKey").toString(), "testValue"); assertTrue(rowReader.reachEnd()); }
Example #23
Source File: ParquetRecordReaderTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testNestedMapGroup() throws IOException { Schema nestedMapSchema = unWrapSchema(NESTED_SCHEMA.getField("nestedMap").schema()); Preconditions.checkState(nestedMapSchema.getType().equals(Schema.Type.MAP)); Schema mapValueSchema = nestedMapSchema.getValueType(); GenericRecord mapValue = new GenericRecordBuilder(mapValueSchema) .set("type", "nested") .set("value", "nested_value").build(); ImmutableMap.Builder<String, GenericRecord> map = ImmutableMap.builder(); map.put("testKey", mapValue); GenericRecord record = new GenericRecordBuilder(NESTED_SCHEMA) .set("nestedMap", map.build()) .set("foo", 34L).build(); Path path = createTempParquetFile(tempRoot.getRoot(), NESTED_SCHEMA, Collections.singletonList(record)); MessageType readSchema = (new AvroSchemaConverter()).convert(NESTED_SCHEMA); ParquetRecordReader<Row> rowReader = new ParquetRecordReader<>(new RowReadSupport(), readSchema); InputFile inputFile = HadoopInputFile.fromPath(new org.apache.hadoop.fs.Path(path.toUri()), testConfig); ParquetReadOptions options = ParquetReadOptions.builder().build(); ParquetFileReader fileReader = new ParquetFileReader(inputFile, options); rowReader.initialize(fileReader, testConfig); assertFalse(rowReader.reachEnd()); Row row = rowReader.nextRecord(); assertEquals(7, row.getArity()); assertEquals(34L, row.getField(0)); Map result = (Map) row.getField(5); Row nestedRow = (Row) result.get("testKey"); assertEquals("nested", nestedRow.getField(0)); assertEquals("nested_value", nestedRow.getField(1)); }
Example #24
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassAssemblerClass() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("AssemblerClass"))); }
Example #25
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassMainClass() throws IOException { File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("MainClass"))); }
Example #26
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindEntryClassAssemblerClassAndMainClass() throws IOException { // We want the assembler class entry to have precedence over main class File jarFile = createJarFileWithManifest(ImmutableMap.of( PackagedProgram.MANIFEST_ATTRIBUTE_ASSEMBLER_CLASS, "AssemblerClass", PackagedProgram.MANIFEST_ATTRIBUTE_MAIN_CLASS, "MainClass")); Optional<String> entry = JarManifestParser.findEntryClass(jarFile); assertTrue(entry.isPresent()); assertThat(entry.get(), is(equalTo("AssemblerClass"))); }
Example #27
Source File: JarManifestParserTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFindOnlyEntryClassMultipleJarsWithSingleManifestEntry() throws IOException { File jarWithNoManifest = createJarFileWithManifest(ImmutableMap.of()); File jarFile = TestJob.getTestJobJar(); JarFileWithEntryClass jarFileWithEntryClass = JarManifestParser .findOnlyEntryClass(ImmutableList.of(jarWithNoManifest, jarFile)); assertThat(jarFileWithEntryClass.getEntryClass(), is(equalTo(TestJob.class.getCanonicalName()))); }
Example #28
Source File: FlinkSqlParser.java From sylph with Apache License 2.0 | 5 votes |
private RealTimeTransForm getJoinTransForm(JoinContext joinContext, CreateTable batchTable) { Map<String, Object> withConfig = batchTable.getWithConfig(); String driverOrName = (String) withConfig.get("type"); Class<?> driver = connectorStore.getConnectorDriver(driverOrName, Operator.PipelineType.transform); checkState(RealTimeTransForm.class.isAssignableFrom(driver), "batch table type driver must is RealTimeTransForm"); // instance IocFactory iocFactory = IocFactory.create(binder -> binder.bind(JoinContext.class, joinContext)); return NodeLoader.getPluginInstance(driver.asSubclass(RealTimeTransForm.class), iocFactory, ImmutableMap.copyOf(withConfig)); }
Example #29
Source File: JdbcAsyncJoin.java From sylph with Apache License 2.0 | 4 votes |
@Override public void process(Record input, Collector<Record> collector) { checkState(connection != null, " connection is null"); StringBuilder builder = new StringBuilder(); for (int index : joinOnMapping.keySet()) { builder.append(input.<Object>getField(index)).append("\u0001"); } List<Map<String, Object>> cacheData = noCatch(() -> cache.get(builder.toString(), () -> { //-- 这里进行真正的数据库查询 List<Integer> indexs = ImmutableList.copyOf(joinOnMapping.keySet()); try (PreparedStatement statement = connection.prepareStatement(sql)) { for (int i = 0; i < indexs.size(); i++) { statement.setObject(i + 1, input.getField(indexs.get(i))); } if (logger.isDebugEnabled()) { logger.debug("Thread is {}, this {}", Thread.currentThread().getId(), this); } try (ResultSet rs = statement.executeQuery()) { List<Map<String, Object>> result = JdbcUtils.resultToList(rs); if (result.isEmpty() && joinType == LEFT) { // left join and inter join return ImmutableList.of(ImmutableMap.of()); } return result; } } catch (SQLException e) { throw throwsException(e); } })); for (Map<String, Object> map : cacheData) { Object[] row = new Object[selectFieldCnt]; for (int i = 0; i < selectFieldCnt; i++) { SelectField field = selectFields.get(i); if (field.isBatchTableField()) { row[i] = map.get(field.getFieldName()); } else { row[i] = input.getField(field.getFieldIndex()); } } collector.collect(Record.of(row)); } }
Example #30
Source File: LocalExecutorITCase.java From flink with Apache License 2.0 | 4 votes |
@Test public void testGetSessionProperties() throws Exception { final Executor executor = createDefaultExecutor(clusterClient); final SessionContext session = new SessionContext("test-session", new Environment()); session.getSessionEnv().setExecution(ImmutableMap.of("result-mode", "changelog")); // Open the session and get the sessionId. String sessionId = executor.openSession(session); try { assertEquals("test-session", sessionId); assertEquals(executor.getSessionProperties(sessionId).get("execution.result-mode"), "changelog"); // modify defaults executor.setSessionProperty(sessionId, "execution.result-mode", "table"); final Map<String, String> actualProperties = executor.getSessionProperties(sessionId); final Map<String, String> expectedProperties = new HashMap<>(); expectedProperties.put("execution.planner", planner); expectedProperties.put("execution.type", "batch"); expectedProperties.put("execution.time-characteristic", "event-time"); expectedProperties.put("execution.periodic-watermarks-interval", "99"); expectedProperties.put("execution.parallelism", "1"); expectedProperties.put("execution.max-parallelism", "16"); expectedProperties.put("execution.max-idle-state-retention", "600000"); expectedProperties.put("execution.min-idle-state-retention", "1000"); expectedProperties.put("execution.result-mode", "table"); expectedProperties.put("execution.max-table-result-rows", "100"); expectedProperties.put("execution.restart-strategy.type", "failure-rate"); expectedProperties.put("execution.restart-strategy.max-failures-per-interval", "10"); expectedProperties.put("execution.restart-strategy.failure-rate-interval", "99000"); expectedProperties.put("execution.restart-strategy.delay", "1000"); expectedProperties.put("table.optimizer.join-reorder-enabled", "false"); expectedProperties.put("deployment.response-timeout", "5000"); assertEquals(expectedProperties, actualProperties); // Reset session properties executor.resetSessionProperties(sessionId); assertEquals(executor.getSessionProperties(sessionId).get("execution.result-mode"), "changelog"); } finally { executor.closeSession(sessionId); } }