org.apache.kafka.common.utils.Time Java Examples
The following examples show how to use
org.apache.kafka.common.utils.Time.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaMetricsTest.java From micrometer with Apache License 2.0 | 7 votes |
@Test void shouldKeepMetersWhenMetricsDoNotChange() { //Given Supplier<Map<MetricName, ? extends Metric>> supplier = () -> { MetricName metricName = new MetricName("a", "b", "c", new LinkedHashMap<>()); KafkaMetric metric = new KafkaMetric(this, metricName, new Value(), new MetricConfig(), Time.SYSTEM); return Collections.singletonMap(metricName, metric); }; kafkaMetrics = new KafkaMetrics(supplier); MeterRegistry registry = new SimpleMeterRegistry(); kafkaMetrics.bindTo(registry); assertThat(registry.getMeters()).hasSize(1); kafkaMetrics.checkAndBindMetrics(registry); assertThat(registry.getMeters()).hasSize(1); }
Example #2
Source File: Compatibility.java From apicurio-registry with Apache License 2.0 | 6 votes |
static Worker createWorker(String workerId, Time time, Plugins plugins, WorkerConfig config, OffsetBackingStore offsetBackingStore, Object connectorClientConfigOverridePolicy) throws ConnectException { if (CTR_WORKER_22 == null) { return new Worker(workerId, time, plugins, config, offsetBackingStore, (ConnectorClientConfigOverridePolicy)connectorClientConfigOverridePolicy); } try { return (Worker)CTR_WORKER_22.newInstance(workerId, time, plugins, config, offsetBackingStore); } catch (Throwable t) { throw new ConnectException(t); } }
Example #3
Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0 | 6 votes |
private ByteBuffer newMemoryRecordsBuffer(List<SimpleRecord> records, long producerId, short producerEpoch, boolean isTxnOffsetCommit) { TimestampType timestampType = TimestampType.CREATE_TIME; long timestamp = Time.SYSTEM.milliseconds(); ByteBuffer buffer = ByteBuffer.allocate( AbstractRecords.estimateSizeInBytes( RecordBatch.CURRENT_MAGIC_VALUE, offsetConfig.offsetsTopicCompressionType(), records ) ); MemoryRecordsBuilder builder = MemoryRecords.builder( buffer, RecordBatch.CURRENT_MAGIC_VALUE, offsetConfig.offsetsTopicCompressionType(), timestampType, 0L, timestamp, producerId, producerEpoch, 0, isTxnOffsetCommit, RecordBatch.NO_PARTITION_LEADER_EPOCH ); records.forEach(builder::append); return builder.build().buffer(); }
Example #4
Source File: DelayedOperationTest.java From kop with Apache License 2.0 | 6 votes |
@Test public void testRequestExpiry() throws Exception { long expiration = 20L; long start = Time.SYSTEM.hiResClockMs(); MockDelayedOperation r1 = new MockDelayedOperation(expiration); MockDelayedOperation r2 = new MockDelayedOperation(200000L); assertFalse( "r1 not satisfied and hence watched", purgatory.tryCompleteElseWatch(r1, Lists.newArrayList("test1"))); assertFalse( "r2 not satisfied and hence watched", purgatory.tryCompleteElseWatch(r2, Lists.newArrayList("test2"))); r1.awaitExpiration(); long elapsed = Time.SYSTEM.hiResClockMs() - start; assertTrue ("r1 completed due to expiration", r1.isCompleted()); assertFalse("r2 hasn't completed", r2.isCompleted()); assertTrue( "Time for expiration $elapsed should at least " + expiration, elapsed >= expiration); }
Example #5
Source File: GroupMetadataManager.java From kop with Apache License 2.0 | 6 votes |
GroupMetadataManager(OffsetConfig offsetConfig, ProducerBuilder<ByteBuffer> metadataTopicProducerBuilder, ReaderBuilder<ByteBuffer> metadataTopicConsumerBuilder, ScheduledExecutorService scheduler, Time time, Function<String, Integer> partitioner) { this.offsetConfig = offsetConfig; this.compressionType = offsetConfig.offsetsTopicCompressionType(); this.groupMetadataCache = new ConcurrentHashMap<>(); this.groupMetadataTopicPartitionCount = offsetConfig.offsetsTopicNumPartitions(); this.metadataTopicProducerBuilder = metadataTopicProducerBuilder; this.metadataTopicReaderBuilder = metadataTopicConsumerBuilder; this.scheduler = scheduler; this.time = time; this.partitioner = partitioner; }
Example #6
Source File: GroupMetadataManagerTest.java From kop with Apache License 2.0 | 6 votes |
private int completeTransactionalOffsetCommit(ByteBuffer buffer, long producerId, short producerEpoch, long baseOffset, boolean isCommit) { MemoryRecordsBuilder builder = MemoryRecords.builder( buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, baseOffset, Time.SYSTEM.milliseconds(), producerId, producerEpoch, 0, true, true, RecordBatch.NO_PARTITION_LEADER_EPOCH); ControlRecordType controlRecordType; if (isCommit) { controlRecordType = ControlRecordType.COMMIT; } else { controlRecordType = ControlRecordType.ABORT; } builder.appendEndTxnMarker(Time.SYSTEM.milliseconds(), new EndTransactionMarker(controlRecordType, 0)); builder.build(); return 1; }
Example #7
Source File: MultiClusterTopicManagementService.java From kafka-monitor with Apache License 2.0 | 6 votes |
void maybeElectLeader() throws Exception { if (!_preferredLeaderElectionRequested) { return; } try (KafkaZkClient zkClient = KafkaZkClient.apply(_zkConnect, JaasUtils.isZkSecurityEnabled(), com.linkedin.kmf.common.Utils.ZK_SESSION_TIMEOUT_MS, com.linkedin.kmf.common.Utils.ZK_CONNECTION_TIMEOUT_MS, Integer.MAX_VALUE, Time.SYSTEM, METRIC_GROUP_NAME, "SessionExpireListener", null)) { if (!zkClient.reassignPartitionsInProgress()) { List<TopicPartitionInfo> partitionInfoList = _adminClient .describeTopics(Collections.singleton(_topic)).all().get().get(_topic).partitions(); LOGGER.info( "MultiClusterTopicManagementService will trigger requested preferred leader election for the" + " topic {} in cluster.", _topic); triggerPreferredLeaderElection(partitionInfoList, _topic); _preferredLeaderElectionRequested = false; } } }
Example #8
Source File: KafkaTopicRepository.java From nakadi with MIT License | 6 votes |
private KafkaZkClient createZkClient() { // The calling method should make sure to close connection return new KafkaZkClient( new ZooKeeperClient( kafkaZookeeper.getZookeeperConnectionString(), zookeeperSettings.getZkSessionTimeoutMs(), zookeeperSettings.getZkConnectionTimeoutMs(), zookeeperSettings.getMaxInFlightRequests(), Time.SYSTEM, ZookeeperSettings.METRIC_GROUP, ZookeeperSettings.METRIC_TYPE ), false, Time.SYSTEM ); }
Example #9
Source File: NetworkClientProvider.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
/** * Creates a new network client with the given properties. * * @return A new network client with the given properties. */ NetworkClient createNetworkClient(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, ChannelBuilder channelBuilder, Metadata metadata, String clientId, int maxInFlightRequestsPerConnection, long reconnectBackoffMs, long reconnectBackoffMax, int socketSendBuffer, int socketReceiveBuffer, int defaultRequestTimeoutMs, boolean discoverBrokerVersions, ApiVersions apiVersions);
Example #10
Source File: WorkerBootStrap.java From DataLink with Apache License 2.0 | 6 votes |
private Keeper buildKeeper(WorkerConfig config, Worker worker, Time time, String restUrl, String bootMode) { Keeper keeper; if (BootMode.DISTRIBUTED.equals(bootMode)) { keeper = new WorkerKeeper( config, time, worker, new TaskStatusManager(DataLinkFactory.getObject(TaskStatusService.class)), new TaskConfigManager(config.getString(WorkerConfig.GROUP_ID_CONFIG), DataLinkFactory.getObject(TaskConfigService.class)), restUrl ); } else if (BootMode.STANDALONE.equals(bootMode)) { keeper = new StandaloneWorkerKeeper( config, worker, time, new TaskConfigManager(config.getString(WorkerConfig.GROUP_ID_CONFIG), DataLinkFactory.getObject(TaskConfigService.class)) ); } else { throw new DatalinkException("invalid boot mode : " + bootMode); } return keeper; }
Example #11
Source File: ConnectEmbedded.java From hello-kafka-streams with Apache License 2.0 | 6 votes |
public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception { Time time = new SystemTime(); DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig)); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config); //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random String workerId = UUID.randomUUID().toString(); worker = new Worker(workerId, time, config, offsetBackingStore); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter()); statusBackingStore.configure(config); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter()); configBackingStore.configure(config); //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, ""); this.connectorConfigs = connectorConfigs; shutdownHook = new ShutdownHook(); }
Example #12
Source File: ServiceKafkaClient.java From ranger with Apache License 2.0 | 6 votes |
private List<String> getTopicList(List<String> ignoreTopicList) throws Exception { List<String> ret = new ArrayList<String>(); int sessionTimeout = 5000; int connectionTimeout = 10000; ZooKeeperClient zookeeperClient = new ZooKeeperClient(zookeeperConnect, sessionTimeout, connectionTimeout, 1, Time.SYSTEM, "kafka.server", "SessionExpireListener", Option.empty()); try (KafkaZkClient kafkaZkClient = new KafkaZkClient(zookeeperClient, true, Time.SYSTEM)) { Iterator<String> iter = kafkaZkClient.getAllTopicsInCluster().iterator(); while (iter.hasNext()) { String topic = iter.next(); if (ignoreTopicList == null || !ignoreTopicList.contains(topic)) { ret.add(topic); } } } return ret; }
Example #13
Source File: MiniKafkaCluster.java From incubator-pinot with Apache License 2.0 | 6 votes |
@SuppressWarnings({"rawtypes", "unchecked"}) private MiniKafkaCluster(List<String> brokerIds) throws IOException, InterruptedException { this.zkServer = new EmbeddedZooKeeper(); this.tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "mini-kafka-cluster"); this.kafkaServer = new ArrayList<>(); int port = 0; for (String id : brokerIds) { port = getAvailablePort(); KafkaConfig c = new KafkaConfig(createBrokerConfig(id, port)); Seq seq = scala.collection.JavaConverters.collectionAsScalaIterableConverter(Collections.emptyList()).asScala().toSeq(); kafkaServer.add(new KafkaServer(c, Time.SYSTEM, Option.empty(), seq)); } Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:" + port); adminClient = AdminClient.create(props); }
Example #14
Source File: KafkaCruiseControl.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
/** * Package private constructor for unit tests w/o static state initialization. */ KafkaCruiseControl(KafkaCruiseControlConfig config, Time time, AnomalyDetector anomalyDetector, Executor executor, LoadMonitor loadMonitor, ExecutorService goalOptimizerExecutor, GoalOptimizer goalOptimizer) { _config = config; _time = time; _anomalyDetector = anomalyDetector; _executor = executor; _loadMonitor = loadMonitor; _goalOptimizerExecutor = goalOptimizerExecutor; _goalOptimizer = goalOptimizer; }
Example #15
Source File: ExecutionTaskManager.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
/** * The constructor of The Execution task manager. * * @param adminClient The adminClient use to query logdir information of replicas. * @param dropwizardMetricRegistry The metric registry. * @param time The time object to get the time. * @param config config object that holds all Kafka Cruise control related configs */ public ExecutionTaskManager(AdminClient adminClient, MetricRegistry dropwizardMetricRegistry, Time time, KafkaCruiseControlConfig config) { _inProgressInterBrokerReplicaMovementsByBrokerId = new HashMap<>(); _inProgressIntraBrokerReplicaMovementsByBrokerId = new HashMap<>(); _inProgressPartitionsForInterBrokerMovement = new HashSet<>(); _executionTaskTracker = new ExecutionTaskTracker(dropwizardMetricRegistry, time); _executionTaskPlanner = new ExecutionTaskPlanner(adminClient, config); _defaultInterBrokerPartitionMovementConcurrency = config.getInt(ExecutorConfig.NUM_CONCURRENT_PARTITION_MOVEMENTS_PER_BROKER_CONFIG); _defaultIntraBrokerPartitionMovementConcurrency = config.getInt(ExecutorConfig.NUM_CONCURRENT_INTRA_BROKER_PARTITION_MOVEMENTS_CONFIG); _defaultLeadershipMovementConcurrency = config.getInt(ExecutorConfig.NUM_CONCURRENT_LEADER_MOVEMENTS_CONFIG); _maxNumClusterMovementConcurrency = config.getInt(ExecutorConfig.MAX_NUM_CLUSTER_MOVEMENTS_CONFIG); _brokersToSkipConcurrencyCheck = new HashSet<>(); _isKafkaAssignerMode = false; _requestedInterBrokerPartitionMovementConcurrency = null; _requestedIntraBrokerPartitionMovementConcurrency = null; _requestedLeadershipMovementConcurrency = null; }
Example #16
Source File: KafkaNetworkClientProvider.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
@Override public NetworkClient createNetworkClient(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, ChannelBuilder channelBuilder, Metadata metadata, String clientId, int maxInFlightRequestsPerConnection, long reconnectBackoffMs, long reconnectBackoffMax, int socketSendBuffer, int socketReceiveBuffer, int defaultRequestTimeoutMs, boolean discoverBrokerVersions, ApiVersions apiVersions) { return new NetworkClient(new Selector(connectionMaxIdleMS, metrics, time, metricGrpPrefix, channelBuilder, new LogContext()), metadata, clientId, maxInFlightRequestsPerConnection, reconnectBackoffMs, reconnectBackoffMax, socketSendBuffer, socketReceiveBuffer, defaultRequestTimeoutMs, ClientDnsLookup.DEFAULT, time, discoverBrokerVersions, apiVersions, new LogContext()); }
Example #17
Source File: KafkaStore.java From data-highway with Apache License 2.0 | 6 votes |
public KafkaStore( String bootstrapServers, Serializer<K, V> serializer, String topic, Collection<StoreUpdateObserver<K, V>> observers, Time time, Map<String, Object> additionalProducerProps, Map<String, Object> additionalConsumerProps) { this.serializer = serializer; this.topic = topic; this.observers = new ArrayList<>(observers); localStore = new HashMap<>(); kafkaLog = createKafkaLog(bootstrapServers, topic, time, additionalProducerProps, additionalConsumerProps); kafkaLog.start(); }
Example #18
Source File: BrokerFailureDetectorTest.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
@Test public void testLoadFailedBrokersFromZK() throws Exception { Time mockTime = getMockTime(); Queue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator()); BrokerFailureDetector detector = createBrokerFailureDetector(anomalies, mockTime); try { detector.startDetection(); int brokerId = 0; killBroker(brokerId); long start = System.currentTimeMillis(); while (anomalies.isEmpty() && System.currentTimeMillis() < start + 30000) { // Wait for the anomalies to be drained. } assertEquals(Collections.singletonMap(brokerId, 100L), detector.failedBrokers()); // shutdown, advance the clock and create a new detector. detector.shutdown(); mockTime.sleep(100L); detector = createBrokerFailureDetector(anomalies, mockTime); // start the newly created detector and the broker down time should remain previous time. detector.startDetection(); assertEquals(Collections.singletonMap(brokerId, 100L), detector.failedBrokers()); } finally { detector.shutdown(); } }
Example #19
Source File: MinimumFileAgePredicateTest.java From kafka-connect-spooldir with Apache License 2.0 | 5 votes |
@Test public void oldEnough() throws IOException { long timestamp = 1559653835123L; this.inputFile.setLastModified(timestamp); timestamp += 5000L; Time time = time(timestamp); InputFileDequeue.MinimumFileAgePredicate predicate = new InputFileDequeue.MinimumFileAgePredicate( 1000, time ); assertTrue(predicate.test(this.inputFile), "File should be old enough"); }
Example #20
Source File: BootstrapTask.java From cruise-control with BSD 2-Clause "Simplified" License | 5 votes |
BootstrapTask(long startMs, long endMs, boolean clearMetrics, MetadataClient metadataClient, KafkaPartitionMetricSampleAggregator metricSampleAggregator, LoadMonitorTaskRunner loadMonitorTaskRunner, MetricFetcherManager metricFetcherManager, SampleStore sampleStore, int configuredNumSnapshots, long configuredSnapshotWindowMs, long samplingIntervalMs, Time time) { if (startMs < 0 || endMs < 0 || endMs <= startMs) { throw new IllegalArgumentException(String.format("Invalid bootstrap time range [%d, %d]. The bootstrap end " + "time must be non negative and the end time " + "must be greater than start time.", startMs, endMs)); } _mode = BootstrapMode.RANGE; _startMs = startMs; _endMs = endMs; _clearMetrics = clearMetrics; _metadataClient = metadataClient; _metricSampleAggregator = metricSampleAggregator; _loadMonitorTaskRunner = loadMonitorTaskRunner; _metricFetcherManager = metricFetcherManager; _sampleStore = sampleStore; _configuredNumSnapshots = configuredNumSnapshots; _configuredSnapshotWindowMs = configuredSnapshotWindowMs; _samplingIntervalMs = samplingIntervalMs; _time = time; _bootstrappedRangeStartMs = startMs; _bootstrappedRangeEndMs = startMs; }
Example #21
Source File: TimeSerializationModuleTest.java From connect-utils with Apache License 2.0 | 5 votes |
@Test public void roundtrip() throws IOException { final Time expected = mock(Time.class); when(expected.milliseconds()).thenReturn(1485910473123L); when(expected.nanoseconds()).thenReturn(1485910473123123L); final String temp = ObjectMapperFactory.INSTANCE.writeValueAsString(expected); log.trace(temp); final Time actual = ObjectMapperFactory.INSTANCE.readValue(temp, Time.class); assertNotNull(actual); assertEquals(expected.milliseconds(), actual.milliseconds(), "milliseconds() does not match."); assertEquals(expected.nanoseconds(), actual.nanoseconds(), "nanoseconds() does not match."); }
Example #22
Source File: KafkaEventChannel.java From jstarcraft-core with Apache License 2.0 | 5 votes |
@Override public void registerMonitor(Set<Class> types, EventMonitor monitor) { try { for (Class type : types) { EventManager manager = managers.get(type); String group = name + StringUtility.DOT + type.getName(); if (manager == null) { manager = new EventManager(); managers.put(type, manager); Properties properties = new Properties(); properties.put("bootstrap.servers", connections); properties.put("key.deserializer", keyDeserializer); properties.put("value.deserializer", valueDeserializer); switch (mode) { case QUEUE: { properties.put("group.id", group); properties.put("auto.offset.reset", "earliest"); break; } case TOPIC: { properties.put("group.id", group + UUID.randomUUID()); properties.put("auto.offset.reset", "latest"); break; } } KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(properties); consumer.subscribe(Collections.singleton(group)); // TODO 此处是为了防止auto.offset.reset为latest时,可能会丢失第一次poll之前的消息. updateAssignmentMetadata.invoke(consumer, Time.SYSTEM.timer(Long.MAX_VALUE)); consumers.put(type, consumer); EventThread thread = new EventThread(type, manager); thread.start(); threads.put(type, thread); } manager.attachMonitor(monitor); } } catch (Exception exception) { throw new RuntimeException(exception); } }
Example #23
Source File: SourceRecordDequeImpl.java From connect-utils with Apache License 2.0 | 5 votes |
SourceRecordDequeImpl(Time time, int maximumCapacity, int batchSize, int emptyWaitMs, int maximumCapacityWaitMs, int maximumCapacityTimeoutMs, RateLimiter writeRateLimit) { super(); this.time = time; this.batchSize = batchSize; this.maximumCapacity = maximumCapacity; this.emptyWaitMs = emptyWaitMs; this.maximumCapacityWaitMs = maximumCapacityWaitMs; this.maximumCapacityTimeoutMs = maximumCapacityTimeoutMs; this.writeRateLimit = writeRateLimit; }
Example #24
Source File: KafkaStarterUtils.java From uReplicator with Apache License 2.0 | 5 votes |
public static void createTopic(String kafkaTopic, int numOfPartitions, String zkStr, String replicatorFactor) { // TopicCommand.main() will call System.exit() finally, which will break maven-surefire-plugin try { String[] args = new String[]{"--create", "--zookeeper", zkStr, "--replication-factor", replicatorFactor, "--partitions", String.valueOf(numOfPartitions), "--topic", kafkaTopic}; KafkaZkClient zkClient = KafkaZkClient .apply(zkStr, false, 3000, 3000, Integer.MAX_VALUE, Time.SYSTEM, "kafka.server", "SessionExpireListener"); TopicCommand.TopicCommandOptions opts = new TopicCommand.TopicCommandOptions(args); TopicCommand.createTopic(zkClient, opts); } catch (TopicExistsException e) { // Catch TopicExistsException otherwise it will break maven-surefire-plugin System.out.println("Topic already existed"); } }
Example #25
Source File: KafkaNodeClient.java From feeyo-redisproxy with BSD 3-Clause "New" or "Revised" License | 5 votes |
public ClientResponse sendAndRecvice(ClientRequest clientRequest) throws IOException { // NetworkClientUtils.awaitReady(client, node, Time.SYSTEM, 5000); // ClientResponse response = NetworkClientUtils.sendAndReceive(client, clientRequest, Time.SYSTEM); return response; }
Example #26
Source File: ConnectRunner.java From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0 | 5 votes |
void start() { final Map<String, String> workerProps = new HashMap<>(); workerProps.put("bootstrap.servers", bootstrapServers); workerProps.put("offset.flush.interval.ms", Integer.toString(offsetFlushInterval)); // These don't matter much (each connector sets its own converters), but need to be filled with valid classes. workerProps.put("key.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); workerProps.put("value.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter"); workerProps.put("internal.key.converter.schemas.enable", "false"); workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter"); workerProps.put("internal.value.converter.schemas.enable", "false"); // Don't need it since we'll memory MemoryOffsetBackingStore. workerProps.put("offset.storage.file.filename", ""); workerProps.put("plugin.path", pluginDir.getPath()); final Time time = Time.SYSTEM; final String workerId = "test-worker"; final Plugins plugins = new Plugins(workerProps); final StandaloneConfig config = new StandaloneConfig(workerProps); final Worker worker = new Worker( workerId, time, plugins, config, new MemoryOffsetBackingStore()); herder = new StandaloneHerder(worker); final RestServer rest = new RestServer(config); connect = new Connect(herder, rest); connect.start(); }
Example #27
Source File: StandaloneWorkerKeeper.java From DataLink with Apache License 2.0 | 5 votes |
public StandaloneWorkerKeeper(WorkerConfig config, Worker worker, Time time, TaskConfigManager taskConfigManager ) { this.workerConfig = config; this.workerId = worker.workerId(); this.worker = worker; this.time = time; this.taskConfigManager = taskConfigManager; this.stopping = new AtomicBoolean(false); this.configState = ClusterConfigState.EMPTY; this.forwardRequestExecutor = Executors.newSingleThreadExecutor(); taskConfigManager.setUpdateListener(new ConfigUpdateListener()); }
Example #28
Source File: Worker.java From DataLink with Apache License 2.0 | 5 votes |
public Worker(String workerId, Time time, WorkerConfig config, TaskPositionManager taskPositionManager, TaskSyncStatusManager taskSyncStatusManager, ProbeManager probeManager) { this.executor = Executors.newCachedThreadPool(new NamedThreadFactory("Task-Container")); this.workerId = workerId; this.time = time; this.config = config; this.taskPositionManager = taskPositionManager; this.taskSyncStatusManager = taskSyncStatusManager; this.probeManager = probeManager; }
Example #29
Source File: ConnectStandalone.java From mongo-kafka with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") ConnectStandalone(final Properties workerProperties) { Time time = Time.SYSTEM; LOGGER.info("Kafka Connect standalone worker initializing ..."); long initStart = time.hiResClockMs(); WorkerInfo initInfo = new WorkerInfo(); initInfo.logAll(); Map<String, String> workerProps = (Map) workerProperties; LOGGER.info("Scanning for plugin classes. This might take a moment ..."); Plugins plugins = new Plugins(workerProps); plugins.compareAndSwapWithDelegatingLoader(); StandaloneConfig config = new StandaloneConfig(workerProps); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config); LOGGER.debug("Kafka cluster ID: {}", kafkaClusterId); RestServer rest = new RestServer(config); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore()); this.herder = new StandaloneHerder(worker, kafkaClusterId); connectionString = advertisedUrl.toString() + herder.kafkaClusterId(); this.connect = new Connect(herder, rest); LOGGER.info( "Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart); }
Example #30
Source File: BootstrapTask.java From cruise-control with BSD 2-Clause "Simplified" License | 5 votes |
BootstrapTask(boolean clearMetrics, MetadataClient metadataClient, KafkaPartitionMetricSampleAggregator metricSampleAggregator, LoadMonitorTaskRunner loadMonitorTaskRunner, MetricFetcherManager metricFetcherManager, SampleStore sampleStore, int configuredNumSnapshots, long configuredSnapshotWindowMs, long samplingIntervalMs, Time time) { _mode = BootstrapMode.RECENT; _startMs = -1L; _endMs = -1L; _clearMetrics = clearMetrics; _metadataClient = metadataClient; _metricSampleAggregator = metricSampleAggregator; _loadMonitorTaskRunner = loadMonitorTaskRunner; _metricFetcherManager = metricFetcherManager; _sampleStore = sampleStore; _configuredNumSnapshots = configuredNumSnapshots; _configuredSnapshotWindowMs = configuredSnapshotWindowMs; _samplingIntervalMs = samplingIntervalMs; _time = time; long now = _time.milliseconds(); _bootstrappedRangeStartMs = now; _bootstrappedRangeEndMs = now; }