org.apache.kafka.common.network.ChannelBuilder Java Examples
The following examples show how to use
org.apache.kafka.common.network.ChannelBuilder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: NetworkClientProvider.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
|
/**
* Creates a new network client with the given properties.
*
* @return A new network client with the given properties.
*/
NetworkClient createNetworkClient(long connectionMaxIdleMS,
Metrics metrics,
Time time,
String metricGrpPrefix,
ChannelBuilder channelBuilder,
Metadata metadata,
String clientId,
int maxInFlightRequestsPerConnection,
long reconnectBackoffMs,
long reconnectBackoffMax,
int socketSendBuffer,
int socketReceiveBuffer,
int defaultRequestTimeoutMs,
boolean discoverBrokerVersions,
ApiVersions apiVersions);
Example #2
| Source File: KafkaNetworkClientProvider.java From cruise-control with BSD 2-Clause "Simplified" License | 6 votes |
|
@Override
public NetworkClient createNetworkClient(long connectionMaxIdleMS,
Metrics metrics,
Time time,
String metricGrpPrefix,
ChannelBuilder channelBuilder,
Metadata metadata,
String clientId,
int maxInFlightRequestsPerConnection,
long reconnectBackoffMs,
long reconnectBackoffMax,
int socketSendBuffer,
int socketReceiveBuffer,
int defaultRequestTimeoutMs,
boolean discoverBrokerVersions,
ApiVersions apiVersions) {
return new NetworkClient(new Selector(connectionMaxIdleMS, metrics, time, metricGrpPrefix, channelBuilder, new LogContext()),
metadata, clientId, maxInFlightRequestsPerConnection, reconnectBackoffMs,
reconnectBackoffMax, socketSendBuffer, socketReceiveBuffer, defaultRequestTimeoutMs,
ClientDnsLookup.DEFAULT, time, discoverBrokerVersions, apiVersions, new LogContext());
}
Example #3
| Source File: MetadataClient.java From cruise-control with BSD 2-Clause "Simplified" License | 5 votes |
|
public MetadataClient(KafkaCruiseControlConfig config,
Metadata metadata,
long metadataTTL,
Time time) {
_metadataGeneration = new AtomicInteger(0);
_metadata = metadata;
_refreshMetadataTimeout = config.getLong(MonitorConfig.METADATA_MAX_AGE_CONFIG);
_time = time;
List<InetSocketAddress> addresses =
ClientUtils.parseAndValidateAddresses(config.getList(MonitorConfig.BOOTSTRAP_SERVERS_CONFIG),
ClientDnsLookup.DEFAULT);
Cluster bootstrapCluster = Cluster.bootstrap(addresses);
MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(bootstrapCluster.nodes(),
bootstrapCluster.clusterResource().clusterId(),
MetadataResponse.NO_CONTROLLER_ID,
Collections.emptyList());
_metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, time.milliseconds());
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time);
NetworkClientProvider provider = config.getConfiguredInstance(MonitorConfig.NETWORK_CLIENT_PROVIDER_CLASS_CONFIG,
NetworkClientProvider.class);
_networkClient = provider.createNetworkClient(config.getLong(MonitorConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG),
new Metrics(),
time,
"load-monitor",
channelBuilder,
_metadata,
config.getString(MonitorConfig.CLIENT_ID_CONFIG),
DEFAULT_MAX_IN_FLIGHT_REQUEST,
config.getLong(MonitorConfig.RECONNECT_BACKOFF_MS_CONFIG),
config.getLong(MonitorConfig.RECONNECT_BACKOFF_MS_CONFIG),
config.getInt(MonitorConfig.SEND_BUFFER_CONFIG),
config.getInt(MonitorConfig.RECEIVE_BUFFER_CONFIG),
config.getInt(MonitorConfig.REQUEST_TIMEOUT_MS_CONFIG),
true,
new ApiVersions());
_metadataTTL = metadataTTL;
}
Example #4
| Source File: KarelDbLeaderElector.java From kareldb with Apache License 2.0 | 4 votes |
|
public KarelDbLeaderElector(KarelDbConfig config, KarelDbEngine engine) throws KarelDbElectionException {
try {
this.engine = engine;
this.clientId = "kdb-" + KDB_CLIENT_ID_SEQUENCE.getAndIncrement();
this.myIdentity = findIdentity(
config.getList(KarelDbConfig.LISTENERS_CONFIG),
config.getBoolean(KarelDbConfig.LEADER_ELIGIBILITY_CONFIG));
Map<String, String> metricsTags = new LinkedHashMap<>();
metricsTags.put("client-id", clientId);
MetricConfig metricConfig = new MetricConfig().tags(metricsTags);
List<MetricsReporter> reporters = Collections.singletonList(new JmxReporter(JMX_PREFIX));
Time time = Time.SYSTEM;
ClientConfig clientConfig = new ClientConfig(config.originalsWithPrefix("kafkacache."), false);
this.metrics = new Metrics(metricConfig, reporters, time);
this.retryBackoffMs = clientConfig.getLong(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG);
String groupId = config.getString(KarelDbConfig.CLUSTER_GROUP_ID_CONFIG);
LogContext logContext = new LogContext("[KarelDB clientId=" + clientId + ", groupId="
+ groupId + "] ");
this.metadata = new Metadata(
retryBackoffMs,
clientConfig.getLong(CommonClientConfigs.METADATA_MAX_AGE_CONFIG),
logContext,
new ClusterResourceListeners()
);
List<String> bootstrapServers
= config.getList(KarelDbConfig.KAFKACACHE_BOOTSTRAP_SERVERS_CONFIG);
List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(bootstrapServers,
clientConfig.getString(CommonClientConfigs.CLIENT_DNS_LOOKUP_CONFIG));
this.metadata.bootstrap(addresses);
String metricGrpPrefix = "kareldb";
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(clientConfig, time);
long maxIdleMs = clientConfig.getLong(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG);
NetworkClient netClient = new NetworkClient(
new Selector(maxIdleMs, metrics, time, metricGrpPrefix, channelBuilder, logContext),
this.metadata,
clientId,
100, // a fixed large enough value will suffice
clientConfig.getLong(CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG),
clientConfig.getLong(CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_CONFIG),
clientConfig.getInt(CommonClientConfigs.SEND_BUFFER_CONFIG),
clientConfig.getInt(CommonClientConfigs.RECEIVE_BUFFER_CONFIG),
clientConfig.getInt(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG),
ClientDnsLookup.forConfig(clientConfig.getString(CommonClientConfigs.CLIENT_DNS_LOOKUP_CONFIG)),
time,
true,
new ApiVersions(),
logContext);
this.client = new ConsumerNetworkClient(
logContext,
netClient,
metadata,
time,
retryBackoffMs,
clientConfig.getInt(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG),
Integer.MAX_VALUE
);
this.coordinator = new KarelDbCoordinator(
logContext,
this.client,
groupId,
300000, // Default MAX_POLL_INTERVAL_MS_CONFIG
10000, // Default SESSION_TIMEOUT_MS_CONFIG)
3000, // Default HEARTBEAT_INTERVAL_MS_CONFIG
metrics,
metricGrpPrefix,
time,
retryBackoffMs,
myIdentity,
this
);
AppInfoParser.registerAppInfo(JMX_PREFIX, clientId, metrics, time.milliseconds());
initTimeout = config.getInt(KarelDbConfig.KAFKACACHE_INIT_TIMEOUT_CONFIG);
LOG.debug("Group member created");
} catch (Throwable t) {
// call close methods if internal objects are already constructed
// this is to prevent resource leak. see KAFKA-2121
stop(true);
// now propagate the exception
throw new KarelDbElectionException("Failed to construct kafka consumer", t);
}
}
Example #5
| Source File: WorkerGroupMember.java From DataLink with Apache License 2.0 | 4 votes |
|
public WorkerGroupMember(WorkerConfig config,
String restUrl,
TaskConfigManager jobTaskConfigManager,
WorkerRebalanceListener listener,
Time time) {
try {
this.time = time;
String clientIdConfig = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG);
clientId = clientIdConfig.length() <= 0 ? "datalink-worker-" + DATALINK_CLIENT_ID_SEQUENCE.getAndIncrement() : clientIdConfig;
Map<String, String> metricsTags = new LinkedHashMap<>();
metricsTags.put("client-id", clientId);
MetricConfig metricConfig = new MetricConfig().samples(config.getInt(CommonClientConfigs.METRICS_NUM_SAMPLES_CONFIG))
.timeWindow(config.getLong(CommonClientConfigs.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS)
.tags(metricsTags);
List<MetricsReporter> reporters = config.getConfiguredInstances(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, MetricsReporter.class);
reporters.add(new JmxReporter(JMX_PREFIX));
this.metrics = new Metrics(metricConfig, reporters, time);
this.retryBackoffMs = config.getLong(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG);
this.metadata = new Metadata(retryBackoffMs, config.getLong(CommonClientConfigs.METADATA_MAX_AGE_CONFIG));
List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config.getList(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG));
this.metadata.update(Cluster.bootstrap(addresses), 0);
String metricGrpPrefix = "datalink.worker";
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config.values());
NetworkClient netClient = new NetworkClient(
new Selector(config.getLong(CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder),
this.metadata,
clientId,
100, // a fixed large enough value will suffice
config.getLong(CommonClientConfigs.RECONNECT_BACKOFF_MS_CONFIG),
config.getInt(CommonClientConfigs.SEND_BUFFER_CONFIG),
config.getInt(CommonClientConfigs.RECEIVE_BUFFER_CONFIG),
config.getInt(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG), time);
this.client = new ConsumerNetworkClient(netClient, metadata, time, retryBackoffMs,
config.getInt(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG)){
@Override
public boolean awaitMetadataUpdate(long timeout) {
metadata.update(Cluster.bootstrap(addresses),time.milliseconds());
return super.awaitMetadataUpdate(timeout);
}
};
this.coordinator = new WorkerCoordinator(this.client,
config.getString(WorkerConfig.GROUP_ID_CONFIG),
config.getInt(WorkerConfig.REBALANCE_TIMEOUT_MS_CONFIG),
config.getInt(WorkerConfig.SESSION_TIMEOUT_MS_CONFIG),
config.getInt(WorkerConfig.HEARTBEAT_INTERVAL_MS_CONFIG),
metrics,
metricGrpPrefix,
this.time,
retryBackoffMs,
restUrl,
jobTaskConfigManager,
listener);
AppInfoParser.registerAppInfo(JMX_PREFIX, clientId);
log.debug("datalink worker group member created");
} catch (Throwable t) {
// call close methods if internal objects are already constructed
// this is to prevent resource leak.
stop(true);
// now propagate the errors
throw new DatalinkException("Failed to construct datalink worker", t);
}
}
