org.apache.flink.util.TimeUtils Java Examples

/**
 * Validates a Java {@link Duration}. The boundaries are inclusive and in milliseconds.
 *
 * <p>The precision defines the allowed minimum unit in milliseconds (e.g. 1000 would only allow seconds).
 */
public void validateDuration(String key, boolean isOptional, int precision, long min, long max) {
	Preconditions.checkArgument(precision > 0);

	validateComparable(
		key,
		isOptional,
		min,
		max,
		"time interval (in milliseconds)",
		(value) -> {
			final long ms = TimeUtils.parseDuration(value).toMillis();
			if (ms % precision != 0) {
				throw new ValidationException(
					"Duration for key '" + key + "' must be a multiple of " + precision + " milliseconds but was: " + value);
			}
			return ms;
		}
	);
}
 

@Test
public void testMaximumRegistrationDuration() throws Exception {
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("10 ms"));

	final TaskExecutor taskExecutor = createTaskExecutor(new TaskManagerServicesBuilder().build());

	taskExecutor.start();

	try {
		final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
		assertThat(error, is(notNullValue()));
		assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));

		testingFatalErrorHandler.clearError();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 

public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(
		Configuration configuration,
		WorkerResourceSpecFactory defaultWorkerResourceSpecFactory) throws ConfigurationException {

	final String strJobTimeout = configuration.getString(ResourceManagerOptions.JOB_TIMEOUT);
	final Time jobTimeout;

	try {
		jobTimeout = Time.milliseconds(TimeUtils.parseDuration(strJobTimeout).toMillis());
	} catch (IllegalArgumentException e) {
		throw new ConfigurationException("Could not parse the resource manager's job timeout " +
			"value " + ResourceManagerOptions.JOB_TIMEOUT + '.', e);
	}

	final WorkerResourceSpec defaultWorkerResourceSpec = defaultWorkerResourceSpecFactory.createDefaultWorkerResourceSpec(configuration);
	final SlotManagerConfiguration slotManagerConfiguration =
		SlotManagerConfiguration.fromConfiguration(configuration, defaultWorkerResourceSpec);

	return new ResourceManagerRuntimeServicesConfiguration(jobTimeout, slotManagerConfiguration);
}
 

/**
 * Validates a Java {@link Duration}. The boundaries are inclusive and in milliseconds.
 *
 * <p>The precision defines the allowed minimum unit in milliseconds (e.g. 1000 would only allow seconds).
 */
public void validateDuration(String key, boolean isOptional, int precision, long min, long max) {
	Preconditions.checkArgument(precision > 0);

	validateComparable(
		key,
		isOptional,
		min,
		max,
		"time interval (in milliseconds)",
		(value) -> {
			final long ms = TimeUtils.parseDuration(value).toMillis();
			if (ms % precision != 0) {
				throw new ValidationException(
					"Duration for key '" + key + "' must be a multiple of " + precision + " milliseconds but was: " + value);
			}
			return ms;
		}
	);
}
 

@SuppressWarnings("unchecked")
private static String stringifyObject(Object value) {
	if (value instanceof String) {
		if (((String) value).isEmpty()) {
			return "(none)";
		}
		return "\"" + value + "\"";
	} else if (value instanceof Duration) {
		return TimeUtils.formatWithHighestUnit((Duration) value);
	} else if (value instanceof List) {
		return ((List<Object>) value).stream()
			.map(ConfigOptionsDocGenerator::stringifyObject)
			.collect(Collectors.joining(";"));
	} else if (value instanceof Map) {
		return ((Map<String, String>) value)
			.entrySet()
			.stream()
			.map(e -> String.format("%s:%s", e.getKey(), e.getValue()))
			.collect(Collectors.joining(","));
	}
	return value == null ? "(none)" : value.toString();
}
 

private static Duration getConfiguredIntervalOrDefault(ReporterSetup reporterSetup) {
	final Optional<String> configuredPeriod = reporterSetup.getIntervalSettings();
	Duration period = MetricOptions.REPORTER_INTERVAL.defaultValue();

	if (configuredPeriod.isPresent()) {
		try {
			period = TimeUtils.parseDuration(configuredPeriod.get());
		}
		catch (Exception e) {
			LOG.error("Cannot parse report interval from config: " + configuredPeriod +
				" - please use values like '10 SECONDS' or '500 MILLISECONDS'. " +
				"Using default reporting interval.");
		}
	}
	return period;
}
 

private Duration convertToDuration(Object o) {
	if (o.getClass() == Duration.class) {
		return (Duration) o;
	}

	return TimeUtils.parseDuration(o.toString());
}
 

@Test
public void testShouldShutdownIfRegistrationWithJobManagerFails() throws Exception {
	Configuration configuration = createConfiguration();
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("10 ms"));
	taskManagerRunner = createTaskManagerRunner(configuration);

	Integer statusCode = systemExitTrackingSecurityManager.getSystemExitFuture().get();
	assertThat(statusCode, is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
}
 

@Test
public void testShouldShutdownOnFatalError() throws Exception {
	Configuration configuration = createConfiguration();
	// very high timeout, to ensure that we don't fail because of registration timeouts
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("42 h"));
	taskManagerRunner = createTaskManagerRunner(configuration);

	taskManagerRunner.onFatalError(new RuntimeException());

	Integer statusCode = systemExitTrackingSecurityManager.getSystemExitFuture().get();
	assertThat(statusCode, is(equalTo(TaskManagerRunner.RUNTIME_FAILURE_RETURN_CODE)));
}
 

private CompletableFuture<ActorRef> resolveActorAddress(String address) {
	final ActorSelection actorSel = actorSystem.actorSelection(address);

	return actorSel.resolveOne(TimeUtils.toDuration(configuration.getTimeout()))
		.toCompletableFuture()
		.exceptionally(error -> {
			throw new CompletionException(
				new RpcConnectionException(String.format("Could not connect to rpc endpoint under address %s.", address), error));
		});
}
 

/**
 * Returns a Java {@link Duration} under the given key if it exists.
 */
public Optional<Duration> getOptionalDuration(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return TimeUtils.parseDuration(value);
		} catch (Exception e) {
			throw new ValidationException("Invalid duration value for key '" + key + "'.", e);
		}
	});
}
 

/**
 * Returns a Java {@link Duration} under the given key if it exists.
 */
public Optional<Duration> getOptionalDuration(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return TimeUtils.parseDuration(value);
		} catch (Exception e) {
			throw new ValidationException("Invalid duration value for key '" + key + "'.", e);
		}
	});
}
 

@VisibleForTesting
MetricRegistryImpl(MetricRegistryConfiguration config, Collection<ReporterSetup> reporterConfigurations, ScheduledExecutorService scheduledExecutor) {
	this.maximumFramesize = config.getQueryServiceMessageSizeLimit();
	this.scopeFormats = config.getScopeFormats();
	this.globalDelimiter = config.getDelimiter();
	this.terminationFuture = new CompletableFuture<>();
	this.isShutdown = false;

	// second, instantiate any custom configured reporters
	this.reporters = new ArrayList<>(4);

	this.executor = scheduledExecutor;

	this.queryService = null;
	this.metricQueryServiceRpcService = null;

	if (reporterConfigurations.isEmpty()) {
		// no reporters defined
		// by default, don't report anything
		LOG.info("No metrics reporter configured, no metrics will be exposed/reported.");
	} else {
		for (ReporterSetup reporterSetup : reporterConfigurations) {
			final String namedReporter = reporterSetup.getName();

			try {
				final MetricReporter reporterInstance = reporterSetup.getReporter();
				final String className = reporterInstance.getClass().getName();

				if (reporterInstance instanceof Scheduled) {
					final Duration period = getConfiguredIntervalOrDefault(reporterSetup);

					LOG.info("Periodically reporting metrics in intervals of {} for reporter {} of type {}.", TimeUtils.formatWithHighestUnit(period), namedReporter, className);

					executor.scheduleWithFixedDelay(
							new MetricRegistryImpl.ReporterTask((Scheduled) reporterInstance), period.toMillis(), period.toMillis(), TimeUnit.MILLISECONDS);
				} else {
					LOG.info("Reporting metrics for reporter {} of type {}.", namedReporter, className);
				}

				String delimiterForReporter = reporterSetup.getDelimiter().orElse(String.valueOf(globalDelimiter));
				if (delimiterForReporter.length() != 1) {
					LOG.warn("Failed to parse delimiter '{}' for reporter '{}', using global delimiter '{}'.", delimiterForReporter, namedReporter, globalDelimiter);
					delimiterForReporter = String.valueOf(globalDelimiter);
				}

				reporters.add(new ReporterAndSettings(
					reporterInstance,
					new ReporterScopedSettings(
						reporters.size(),
						delimiterForReporter.charAt(0),
						reporterSetup.getExcludedVariables())));
			}
			catch (Throwable t) {
				LOG.error("Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.", namedReporter, t);
			}
		}
	}
}
 

private List<HiveTablePartition> initAllPartitions() {
	List<HiveTablePartition> allHivePartitions = new ArrayList<>();
	// Please note that the following directly accesses Hive metastore, which is only a temporary workaround.
	// Ideally, we need to go thru Catalog API to get all info we need here, which requires some major
	// refactoring. We will postpone this until we merge Blink to Flink.
	try (HiveMetastoreClientWrapper client = HiveMetastoreClientFactory.create(new HiveConf(jobConf, HiveConf.class), hiveVersion)) {
		String dbName = tablePath.getDatabaseName();
		String tableName = tablePath.getObjectName();
		List<String> partitionColNames = catalogTable.getPartitionKeys();
		Table hiveTable = client.getTable(dbName, tableName);
		Properties tableProps = HiveReflectionUtils.getTableMetadata(hiveShim, hiveTable);
		String ttlStr = tableProps.getProperty(FileSystemOptions.LOOKUP_JOIN_CACHE_TTL.key());
		hiveTableCacheTTL = ttlStr != null ?
				TimeUtils.parseDuration(ttlStr) :
				FileSystemOptions.LOOKUP_JOIN_CACHE_TTL.defaultValue();
		if (partitionColNames != null && partitionColNames.size() > 0) {
			final String defaultPartitionName = jobConf.get(HiveConf.ConfVars.DEFAULTPARTITIONNAME.varname,
					HiveConf.ConfVars.DEFAULTPARTITIONNAME.defaultStrVal);
			List<Partition> partitions = new ArrayList<>();
			if (remainingPartitions != null) {
				for (Map<String, String> spec : remainingPartitions) {
					partitions.add(client.getPartition(dbName, tableName, partitionSpecToValues(spec, partitionColNames)));
				}
			} else {
				partitions.addAll(client.listPartitions(dbName, tableName, (short) -1));
			}
			for (Partition partition : partitions) {
				HiveTablePartition hiveTablePartition = toHiveTablePartition(
						catalogTable.getPartitionKeys(),
						catalogTable.getSchema().getFieldNames(),
						catalogTable.getSchema().getFieldDataTypes(),
						hiveShim,
						tableProps,
						defaultPartitionName,
						partition);
				allHivePartitions.add(hiveTablePartition);
			}
		} else {
			allHivePartitions.add(new HiveTablePartition(hiveTable.getSd(), tableProps));
		}
	} catch (TException e) {
		throw new FlinkHiveException("Failed to collect all partitions from hive metaStore", e);
	}
	return allHivePartitions;
}
 

private static Optional<Duration> optionalStateExpireDuration(JsonNode stateSpecNode) {
  return Selectors.optionalTextAt(stateSpecNode, StateSpecPointers.EXPIRE_DURATION)
      .map(TimeUtils::parseDuration);
}
 

private static Optional<Duration> optionalMaxRequestDuration(JsonNode functionNode) {
  return Selectors.optionalTextAt(functionNode, SpecPointers.TIMEOUT)
      .map(TimeUtils::parseDuration);
}
 

@Test
public void testMaximumRegistrationDurationAfterConnectionLoss() throws Exception {
	configuration.set(TaskManagerOptions.REGISTRATION_TIMEOUT, TimeUtils.parseDuration("100 ms"));
	final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();
	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices, new HeartbeatServices(10L, 10L));

	taskExecutor.start();

	final CompletableFuture<ResourceID> registrationFuture = new CompletableFuture<>();
	final OneShotLatch secondRegistration = new OneShotLatch();
	try {
		final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
		testingResourceManagerGateway.setRegisterTaskExecutorFunction(
			taskExecutorRegistration -> {
				if (registrationFuture.complete(taskExecutorRegistration.getResourceId())) {
					return createRegistrationResponse(testingResourceManagerGateway);
				} else {
					secondRegistration.trigger();
					return CompletableFuture.completedFuture(new Decline("Only the first registration should succeed."));
				}
			}
		);
		rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);

		resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), UUID.randomUUID());

		final ResourceID registrationResourceId = registrationFuture.get();

		assertThat(registrationResourceId, equalTo(taskManagerServices.getUnresolvedTaskManagerLocation().getResourceID()));

		secondRegistration.await();

		final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
		assertThat(error, is(notNullValue()));
		assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));

		testingFatalErrorHandler.clearError();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}