Java Code Examples for io.airlift.units.Duration#valueOf()
The following examples show how to use
io.airlift.units.Duration#valueOf() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestFileBasedNetworkTopology.java From presto with Apache License 2.0 | 6 votes |
|
@Test
public void testRefresh()
throws Exception
{
try (TempFile tempFile = new TempFile()) {
Files.copy(topologyFile, tempFile.file());
TestingTicker ticker = new TestingTicker();
FileBasedNetworkTopology topology = new FileBasedNetworkTopology(tempFile.file(), Duration.valueOf("1d"), ticker);
assertEquals(topology.locate(HostAddress.fromString("not-exist.example.com")), new NetworkLocation());
assertEquals(topology.locate(HostAddress.fromString("192.168.0.1")), new NetworkLocation("region1", "rack1", "machine1"));
assertEquals(topology.locate(HostAddress.fromString("192.168.0.2")), new NetworkLocation("region1", "rack1", "machine2"));
assertEquals(topology.locate(HostAddress.fromString("192.168.0.3")), new NetworkLocation());
assertEquals(topology.locate(HostAddress.fromString("new")), new NetworkLocation());
Files.copy(topologyNewFile, tempFile.file());
ticker.increment(1, TimeUnit.DAYS);
assertEquals(topology.locate(HostAddress.fromString("new")), new NetworkLocation("new", "rack", "machine"));
assertEquals(topology.locate(HostAddress.fromString("not-exist.example.com")), new NetworkLocation());
assertEquals(topology.locate(HostAddress.fromString("192.168.0.1")), new NetworkLocation("region1", "rack1", "machine5"));
assertEquals(topology.locate(HostAddress.fromString("192.168.0.2")), new NetworkLocation());
assertEquals(topology.locate(HostAddress.fromString("192.168.0.3")), new NetworkLocation("region1", "rack1", "machine6"));
}
}
Example 2
| Source File: KafkaConfig.java From presto with Apache License 2.0 | 5 votes |
|
@Config("kafka.connect-timeout")
@ConfigDescription("Kafka connection timeout")
public KafkaConfig setKafkaConnectTimeout(String kafkaConnectTimeout)
{
this.kafkaConnectTimeout = Duration.valueOf(kafkaConnectTimeout);
return this;
}
Example 3
| Source File: BlackHoleConnector.java From presto with Apache License 2.0 | 5 votes |
|
private static PropertyMetadata<Duration> durationProperty(String name, String description, Duration defaultValue, boolean hidden)
{
return new PropertyMetadata<>(
name,
description,
VARCHAR,
Duration.class,
defaultValue,
hidden,
value -> Duration.valueOf((String) value),
Duration::toString);
}
Example 4
| Source File: TestClusterMemoryLeakDetector.java From presto with Apache License 2.0 | 5 votes |
|
private static BasicQueryInfo createQueryInfo(String queryId, QueryState state)
{
return new BasicQueryInfo(
new QueryId(queryId),
TEST_SESSION.toSessionRepresentation(),
Optional.of(new ResourceGroupId("global")),
state,
GENERAL_POOL,
true,
URI.create("1"),
"",
Optional.empty(),
Optional.empty(),
new BasicQueryStats(
DateTime.parse("1991-09-06T05:00-05:30"),
DateTime.parse("1991-09-06T05:01-05:30"),
Duration.valueOf("8m"),
Duration.valueOf("7m"),
Duration.valueOf("34m"),
13,
14,
15,
100,
DataSize.valueOf("21GB"),
22,
23,
DataSize.valueOf("23GB"),
DataSize.valueOf("24GB"),
DataSize.valueOf("25GB"),
DataSize.valueOf("26GB"),
Duration.valueOf("23m"),
Duration.valueOf("24m"),
true,
ImmutableSet.of(WAITING_FOR_MEMORY),
OptionalDouble.of(20)),
null,
null);
}
Example 5
| Source File: AccumuloSessionProperties.java From presto with Apache License 2.0 | 5 votes |
|
private static PropertyMetadata<Duration> durationProperty(String name, String description, Duration defaultValue, boolean hidden)
{
return new PropertyMetadata<>(
name,
description,
VARCHAR,
Duration.class,
defaultValue,
hidden,
value -> Duration.valueOf((String) value),
Duration::toString);
}
Example 6
| Source File: TestOperatorAssertion.java From presto with Apache License 2.0 | 5 votes |
|
@Test
public void testToPagesWithBlockedOperator()
{
Operator operator = new BlockedOperator(Duration.valueOf("15 ms"));
List<Page> pages = OperatorAssertion.toPages(operator, emptyIterator());
Assert.assertEquals(pages, ImmutableList.of());
}
Example 7
| Source File: TestFileBasedNetworkTopology.java From presto with Apache License 2.0 | 5 votes |
|
@Test
public void testLocate()
{
NetworkTopology topology = new FileBasedNetworkTopology(topologyFile, Duration.valueOf("1d"), new TestingTicker());
assertEquals(topology.locate(HostAddress.fromString("0.0.0.0")), new NetworkLocation());
assertEquals(topology.locate(HostAddress.fromString("not-exist.example.com")), new NetworkLocation());
assertEquals(topology.locate(HostAddress.fromString("192.168.0.1")), new NetworkLocation("region1", "rack1", "machine1"));
assertEquals(topology.locate(HostAddress.fromString("192.168.0.2")), new NetworkLocation("region1", "rack1", "machine2"));
assertEquals(topology.locate(HostAddress.fromString("hdfs01.example.com")), new NetworkLocation("region2", "rack2", "machine3"));
assertEquals(topology.locate(HostAddress.fromString("192.168.0.1:8080")), new NetworkLocation("region1", "rack1", "machine1"));
}
Example 8
| Source File: FileBasedSystemAccessControl.java From presto with Apache License 2.0 | 5 votes |
|
@Override
public SystemAccessControl create(Map<String, String> config)
{
requireNonNull(config, "config is null");
String configFileName = config.get(SECURITY_CONFIG_FILE);
checkState(configFileName != null, "Security configuration must contain the '%s' property", SECURITY_CONFIG_FILE);
if (config.containsKey(SECURITY_REFRESH_PERIOD)) {
Duration refreshPeriod;
try {
refreshPeriod = Duration.valueOf(config.get(SECURITY_REFRESH_PERIOD));
}
catch (IllegalArgumentException e) {
throw invalidRefreshPeriodException(config, configFileName);
}
if (refreshPeriod.toMillis() == 0) {
throw invalidRefreshPeriodException(config, configFileName);
}
return ForwardingSystemAccessControl.of(memoizeWithExpiration(
() -> {
log.info("Refreshing system access control from %s", configFileName);
return create(configFileName);
},
refreshPeriod.toMillis(),
MILLISECONDS));
}
return create(configFileName);
}
Example 9
| Source File: PropertyMetadataUtil.java From presto with Apache License 2.0 | 5 votes |
|
public static PropertyMetadata<Duration> durationProperty(String name, String description, Duration defaultValue, boolean hidden)
{
return new PropertyMetadata<>(
name,
description,
VARCHAR,
Duration.class,
defaultValue,
hidden,
value -> Duration.valueOf((String) value),
Duration::toString);
}
Example 10
| Source File: QueryPreprocessor.java From presto with Apache License 2.0 | 5 votes |
|
public static String preprocessQuery(Terminal terminal, Optional<String> catalog, Optional<String> schema, String query)
throws QueryPreprocessorException
{
Duration timeout = DEFAULT_PREPROCESSOR_TIMEOUT;
String timeoutEnvironment = nullToEmpty(System.getenv(ENV_PREPROCESSOR_TIMEOUT)).trim();
if (!timeoutEnvironment.isEmpty()) {
timeout = Duration.valueOf(timeoutEnvironment);
}
String preprocessorCommand = System.getenv(ENV_PREPROCESSOR);
if (emptyToNull(preprocessorCommand) == null) {
return query;
}
return preprocessQuery(terminal, catalog, schema, query, ImmutableList.of("/bin/sh", "-c", preprocessorCommand), timeout);
}
Example 11
| Source File: S3SelectLineRecordReader.java From presto with Apache License 2.0 | 5 votes |
|
S3SelectLineRecordReader(
Configuration configuration,
Path path,
long start,
long length,
Properties schema,
String ionSqlQuery,
PrestoS3ClientFactory s3ClientFactory)
{
requireNonNull(configuration, "configuration is null");
requireNonNull(schema, "schema is null");
requireNonNull(path, "path is null");
requireNonNull(ionSqlQuery, "ionSqlQuery is null");
requireNonNull(s3ClientFactory, "s3ClientFactory is null");
this.lineDelimiter = (schema).getProperty(LINE_DELIM, "\n");
this.processedRecords = 0;
this.recordsFromS3 = 0;
this.start = start;
this.position = this.start;
this.end = this.start + length;
this.isFirstLine = true;
this.compressionCodecFactory = new CompressionCodecFactory(configuration);
this.selectObjectContentRequest = buildSelectObjectRequest(schema, ionSqlQuery, path);
HiveS3Config defaults = new HiveS3Config();
this.maxAttempts = configuration.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()) + 1;
this.maxBackoffTime = Duration.valueOf(configuration.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString()));
this.maxRetryTime = Duration.valueOf(configuration.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString()));
this.selectClient = new PrestoS3SelectClient(configuration, s3ClientFactory);
closer.register(selectClient);
}
Example 12
| Source File: RedisConnectorConfig.java From presto with Apache License 2.0 | 4 votes |
|
@Config("redis.connect-timeout")
public RedisConnectorConfig setRedisConnectTimeout(String redisConnectTimeout)
{
this.redisConnectTimeout = Duration.valueOf(redisConnectTimeout);
return this;
}
Example 13
| Source File: TestQueryStateInfo.java From presto with Apache License 2.0 | 4 votes |
|
private QueryInfo createQueryInfo(String queryId, QueryState state, String query)
{
return new QueryInfo(
new QueryId(queryId),
TEST_SESSION.toSessionRepresentation(),
state,
new MemoryPoolId("reserved"),
true,
URI.create("1"),
ImmutableList.of("2", "3"),
query,
Optional.empty(),
new QueryStats(
DateTime.parse("1991-09-06T05:00-05:30"),
DateTime.parse("1991-09-06T05:01-05:30"),
DateTime.parse("1991-09-06T05:02-05:30"),
DateTime.parse("1991-09-06T06:00-05:30"),
Duration.valueOf("10s"),
Duration.valueOf("8m"),
Duration.valueOf("7m"),
Duration.valueOf("34m"),
Duration.valueOf("9m"),
Duration.valueOf("10m"),
Duration.valueOf("11m"),
Duration.valueOf("12m"),
13,
14,
15,
100,
17,
18,
34,
19,
20.0,
DataSize.valueOf("21GB"),
DataSize.valueOf("22GB"),
DataSize.valueOf("23GB"),
DataSize.valueOf("24GB"),
DataSize.valueOf("25GB"),
DataSize.valueOf("30GB"),
DataSize.valueOf("26GB"),
DataSize.valueOf("27GB"),
DataSize.valueOf("28GB"),
DataSize.valueOf("29GB"),
true,
Duration.valueOf("23m"),
Duration.valueOf("24m"),
Duration.valueOf("26m"),
true,
ImmutableSet.of(WAITING_FOR_MEMORY),
DataSize.valueOf("271GB"),
281,
Duration.valueOf("26m"),
DataSize.valueOf("272GB"),
282,
DataSize.valueOf("27GB"),
28,
DataSize.valueOf("29GB"),
30,
DataSize.valueOf("31GB"),
32,
DataSize.valueOf("33GB"),
ImmutableList.of(),
ImmutableList.of()),
Optional.empty(),
Optional.empty(),
Optional.empty(),
ImmutableMap.of(),
ImmutableSet.of(),
ImmutableMap.of(),
ImmutableMap.of(),
ImmutableSet.of(),
Optional.empty(),
false,
"33",
Optional.empty(),
null,
null,
ImmutableList.of(),
ImmutableSet.of(),
Optional.empty(),
ImmutableList.of(),
ImmutableList.of(),
false,
Optional.empty());
}
Example 14
| Source File: PrestoS3ClientFactory.java From presto with Apache License 2.0 | 4 votes |
|
private AmazonS3 createS3Client(Configuration config)
{
HiveS3Config defaults = new HiveS3Config();
String userAgentPrefix = config.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());
int maxErrorRetries = config.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
boolean sslEnabled = config.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
Duration connectTimeout = Duration.valueOf(config.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));
Duration socketTimeout = Duration.valueOf(config.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString()));
int maxConnections = config.getInt(S3_SELECT_PUSHDOWN_MAX_CONNECTIONS, defaultMaxConnections);
ClientConfiguration clientConfiguration = new ClientConfiguration()
.withMaxErrorRetry(maxErrorRetries)
.withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP)
.withConnectionTimeout(toIntExact(connectTimeout.toMillis()))
.withSocketTimeout(toIntExact(socketTimeout.toMillis()))
.withMaxConnections(maxConnections)
.withUserAgentPrefix(userAgentPrefix)
.withUserAgentSuffix(enabled ? "presto-select" : "presto");
AWSCredentialsProvider awsCredentialsProvider = getAwsCredentialsProvider(config, defaults);
AmazonS3Builder<? extends AmazonS3Builder<?, ?>, ? extends AmazonS3> clientBuilder = AmazonS3Client.builder()
.withCredentials(awsCredentialsProvider)
.withClientConfiguration(clientConfiguration)
.withMetricsCollector(new PrestoS3FileSystemMetricCollector(PrestoS3FileSystem.getFileSystemStats()))
.enablePathStyleAccess();
boolean regionOrEndpointSet = false;
String endpoint = config.get(S3_ENDPOINT);
boolean pinS3ClientToCurrentRegion = config.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion());
verify(!pinS3ClientToCurrentRegion || endpoint == null,
"Invalid configuration: either endpoint can be set or S3 client can be pinned to the current region");
// use local region when running inside of EC2
if (pinS3ClientToCurrentRegion) {
clientBuilder.setRegion(getCurrentRegionFromEC2Metadata().getName());
regionOrEndpointSet = true;
}
if (!isNullOrEmpty(endpoint)) {
clientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, null));
regionOrEndpointSet = true;
}
if (!regionOrEndpointSet) {
clientBuilder.withRegion(US_EAST_1);
clientBuilder.setForceGlobalBucketAccessEnabled(true);
}
return clientBuilder.build();
}
Example 15
| Source File: PrestoS3FileSystem.java From presto with Apache License 2.0 | 4 votes |
|
@Override
public void initialize(URI uri, Configuration conf)
throws IOException
{
requireNonNull(uri, "uri is null");
requireNonNull(conf, "conf is null");
super.initialize(uri, conf);
setConf(conf);
try {
this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null);
}
catch (URISyntaxException e) {
throw new IllegalArgumentException("Invalid uri: " + uri, e);
}
this.workingDirectory = new Path(PATH_SEPARATOR).makeQualified(this.uri, new Path(PATH_SEPARATOR));
HiveS3Config defaults = new HiveS3Config();
this.stagingDirectory = new File(conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().getPath()));
this.maxAttempts = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()) + 1;
this.maxBackoffTime = Duration.valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString()));
this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString()));
int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
Duration connectTimeout = Duration.valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));
Duration socketTimeout = Duration.valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString()));
int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections());
this.multiPartUploadMinFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes());
this.multiPartUploadMinPartSize = conf.getLong(S3_MULTIPART_MIN_PART_SIZE, defaults.getS3MultipartMinPartSize().toBytes());
this.isPathStyleAccess = conf.getBoolean(S3_PATH_STYLE_ACCESS, defaults.isS3PathStyleAccess());
this.iamRole = conf.get(S3_IAM_ROLE, defaults.getS3IamRole());
this.externalId = conf.get(S3_EXTERNAL_ID, defaults.getS3ExternalId());
this.pinS3ClientToCurrentRegion = conf.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion());
verify(!pinS3ClientToCurrentRegion || conf.get(S3_ENDPOINT) == null,
"Invalid configuration: either endpoint can be set or S3 client can be pinned to the current region");
this.sseEnabled = conf.getBoolean(S3_SSE_ENABLED, defaults.isS3SseEnabled());
this.sseType = PrestoS3SseType.valueOf(conf.get(S3_SSE_TYPE, defaults.getS3SseType().name()));
this.sseKmsKeyId = conf.get(S3_SSE_KMS_KEY_ID, defaults.getS3SseKmsKeyId());
this.s3AclType = PrestoS3AclType.valueOf(conf.get(S3_ACL_TYPE, defaults.getS3AclType().name()));
String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());
this.skipGlacierObjects = conf.getBoolean(S3_SKIP_GLACIER_OBJECTS, defaults.isSkipGlacierObjects());
this.requesterPaysEnabled = conf.getBoolean(S3_REQUESTER_PAYS_ENABLED, defaults.isRequesterPaysEnabled());
this.s3StorageClass = conf.getEnum(S3_STORAGE_CLASS, defaults.getS3StorageClass());
ClientConfiguration configuration = new ClientConfiguration()
.withMaxErrorRetry(maxErrorRetries)
.withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP)
.withConnectionTimeout(toIntExact(connectTimeout.toMillis()))
.withSocketTimeout(toIntExact(socketTimeout.toMillis()))
.withMaxConnections(maxConnections)
.withUserAgentPrefix(userAgentPrefix)
.withUserAgentSuffix(S3_USER_AGENT_SUFFIX);
this.credentialsProvider = createAwsCredentialsProvider(uri, conf);
this.s3 = createAmazonS3Client(conf, configuration);
}
Example 16
| Source File: PrestoS3FileSystem.java From spliceengine with GNU Affero General Public License v3.0 | 4 votes |
|
@Override
public void initialize(URI uri, Configuration conf)
throws IOException
{
requireNonNull(uri, "uri is null");
requireNonNull(conf, "conf is null");
super.initialize(uri, conf);
setConf(conf);
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
this.workingDirectory = new Path(PATH_SEPARATOR).makeQualified(this.uri, new Path(PATH_SEPARATOR));
this.stagingDirectory = new File(conf.get(S3_STAGING_DIRECTORY, System.getProperty("java.io.tmpdir")));
this.maxAttempts = conf.getInt(S3_MAX_CLIENT_RETRIES, 3) + 1;
this.maxBackoffTime = Duration.valueOf(conf.get(S3_MAX_BACKOFF_TIME, "10m"));
this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, "10m"));
int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, 10);
boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, true);
Duration connectTimeout = Duration.valueOf(conf.get(S3_CONNECT_TIMEOUT, "5s"));
Duration socketTimeout = Duration.valueOf(conf.get(S3_SOCKET_TIMEOUT, "5s"));
int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, 500);
long minFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, 16l << 20);
long minPartSize = conf.getLong(S3_MULTIPART_MIN_PART_SIZE, 5l << 20);
this.useInstanceCredentials = conf.getBoolean(S3_USE_INSTANCE_CREDENTIALS, true);
this.pinS3ClientToCurrentRegion = conf.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, false);
this.sseEnabled = conf.getBoolean(S3_SSE_ENABLED, false);
this.sseType = PrestoS3SseType.valueOf(conf.get(S3_SSE_TYPE, PrestoS3SseType.S3.name()));
this.sseKmsKeyId = conf.get(S3_SSE_KMS_KEY_ID, null);
String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, "");
ClientConfiguration configuration = new ClientConfiguration()
.withMaxErrorRetry(maxErrorRetries)
.withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP)
.withConnectionTimeout(toIntExact(connectTimeout.toMillis()))
.withSocketTimeout(toIntExact(socketTimeout.toMillis()))
.withMaxConnections(maxConnections)
.withUserAgentPrefix(userAgentPrefix)
.withUserAgentSuffix(S3_USER_AGENT_SUFFIX);
this.s3 = createAmazonS3Client(uri, conf, configuration);
transferConfig.setMultipartUploadThreshold(minFileSize);
transferConfig.setMinimumUploadPartSize(minPartSize);
}
