Java Code Examples for org.apache.flink.test.util.TestBaseUtils#setEnv()

@Test
public void testSourceConfig() throws Exception {
	// vector reader not available for 1.x and we're not testing orc for 2.0.x
	Assume.assumeTrue(HiveVersionTestUtil.HIVE_210_OR_LATER);
	Map<String, String> env = System.getenv();
	hiveShell.execute("create database db1");
	try {
		hiveShell.execute("create table db1.src (x int,y string) stored as orc");
		hiveShell.execute("insert into db1.src values (1,'a'),(2,'b')");
		testSourceConfig(true, true);
		testSourceConfig(false, false);
	} finally {
		TestBaseUtils.setEnv(env);
		hiveShell.execute("drop database db1 cascade");
	}
}
 

@AfterClass
public static void teardown() throws Exception {

	LOG.info("Stopping MiniYarn Cluster");
	yarnCluster.stop();

	// Unset FLINK_CONF_DIR, as it might change the behavior of other tests
	Map<String, String> map = new HashMap<>(System.getenv());
	map.remove(ConfigConstants.ENV_FLINK_CONF_DIR);
	map.remove("YARN_CONF_DIR");
	map.remove("IN_TESTS");
	TestBaseUtils.setEnv(map);

	if (tempConfPathForSecureRun != null) {
		FileUtil.fullyDelete(tempConfPathForSecureRun);
		tempConfPathForSecureRun = null;
	}

	// When we are on travis, we copy the temp files of JUnit (containing the MiniYARNCluster log files)
	// to <flinkRoot>/target/flink-yarn-tests-*.
	// The files from there are picked up by the ./tools/travis_watchdog.sh script
	// to upload them to Amazon S3.
	if (isOnTravis()) {
		File target = new File("../target" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY));
		if (!target.mkdirs()) {
			LOG.warn("Error creating dirs to {}", target);
		}
		File src = tmp.getRoot();
		LOG.info("copying the final files from {} to {}", src.getAbsolutePath(), target.getAbsolutePath());
		try {
			FileUtils.copyDirectoryToDirectory(src, target);
		} catch (IOException e) {
			LOG.warn("Error copying the final files from {} to {}: msg: {}", src.getAbsolutePath(), target.getAbsolutePath(), e.getMessage(), e);
		}
	}

}
 

@BeforeClass
public static void setup() throws Exception {

	skipIfHadoopVersionIsNotAppropriate();

	LOG.info("starting secure cluster environment for testing");

	dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());

	SecureTestEnvironment.prepare(tempFolder);

	populateSecureConfigurations();

	Configuration flinkConfig = new Configuration();
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB,
			SecureTestEnvironment.getTestKeytab());
	flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL,
			SecureTestEnvironment.getHadoopServicePrincipal());

	SecurityConfiguration ctx =
		new SecurityConfiguration(
			flinkConfig,
			Collections.singletonList(securityConfig -> new HadoopModule(securityConfig, conf)));
	try {
		TestingSecurityContext.install(ctx, SecureTestEnvironment.getClientSecurityConfigurationMap());
	} catch (Exception e) {
		throw new RuntimeException("Exception occurred while setting up secure test context. Reason: {}", e);
	}

	File hdfsSiteXML = new File(dataDir.getAbsolutePath() + "/hdfs-site.xml");

	FileWriter writer = new FileWriter(hdfsSiteXML);
	conf.writeXml(writer);
	writer.flush();
	writer.close();

	Map<String, String> map = new HashMap<String, String>(System.getenv());
	map.put("HADOOP_CONF_DIR", hdfsSiteXML.getParentFile().getAbsolutePath());
	TestBaseUtils.setEnv(map);

	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	builder.checkDataNodeAddrConfig(true);
	builder.checkDataNodeHostConfig(true);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	hdfsURI = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/";

	Configuration configuration = startSecureFlinkClusterWithRecoveryModeEnabled();

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(4)
			.build());

	miniClusterResource.before();
}
 

@AfterClass
public static void teardown() throws Exception {

	LOG.info("Stopping MiniYarn Cluster");
	yarnCluster.stop();

	// Unset FLINK_CONF_DIR, as it might change the behavior of other tests
	Map<String, String> map = new HashMap<>(System.getenv());
	map.remove(ConfigConstants.ENV_FLINK_CONF_DIR);
	map.remove("YARN_CONF_DIR");
	map.remove("IN_TESTS");
	TestBaseUtils.setEnv(map);

	if (tempConfPathForSecureRun != null) {
		FileUtil.fullyDelete(tempConfPathForSecureRun);
		tempConfPathForSecureRun = null;
	}

	if (yarnSiteXML != null) {
		yarnSiteXML.delete();
	}

	// When we are on travis, we copy the temp files of JUnit (containing the MiniYARNCluster log files)
	// to <flinkRoot>/target/flink-yarn-tests-*.
	// The files from there are picked up by the ./tools/travis_watchdog.sh script
	// to upload them to Amazon S3.
	if (isOnTravis()) {
		File target = new File("../target" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY));
		if (!target.mkdirs()) {
			LOG.warn("Error creating dirs to {}", target);
		}
		File src = tmp.getRoot();
		LOG.info("copying the final files from {} to {}", src.getAbsolutePath(), target.getAbsolutePath());
		try {
			FileUtils.copyDirectoryToDirectory(src, target);
		} catch (IOException e) {
			LOG.warn("Error copying the final files from {} to {}: msg: {}", src.getAbsolutePath(), target.getAbsolutePath(), e.getMessage(), e);
		}
	}

}
 

@AfterClass
public static void teardown() throws Exception {

	if (yarnCluster != null) {
		LOG.info("Stopping MiniYarn Cluster");
		yarnCluster.stop();
		yarnCluster = null;
	}

	if (miniDFSCluster != null) {
		LOG.info("Stopping MiniDFS Cluster");
		miniDFSCluster.shutdown();
		miniDFSCluster = null;
	}

	// Unset FLINK_CONF_DIR, as it might change the behavior of other tests
	Map<String, String> map = new HashMap<>(System.getenv());
	map.remove(ConfigConstants.ENV_FLINK_CONF_DIR);
	map.remove("YARN_CONF_DIR");
	map.remove("IN_TESTS");
	TestBaseUtils.setEnv(map);

	if (tempConfPathForSecureRun != null) {
		FileUtil.fullyDelete(tempConfPathForSecureRun);
		tempConfPathForSecureRun = null;
	}

	if (yarnSiteXML != null) {
		yarnSiteXML.delete();
	}

	if (hdfsSiteXML != null) {
		hdfsSiteXML.delete();
	}

	// When we are on CI, we copy the temp files of JUnit (containing the MiniYARNCluster log files)
	// to <flinkRoot>/target/flink-yarn-tests-*.
	// The files from there are picked up by the tools/ci/* scripts to upload them.
	if (isOnCI()) {
		File target = new File("../target" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY));
		if (!target.mkdirs()) {
			LOG.warn("Error creating dirs to {}", target);
		}
		File src = tmp.getRoot();
		LOG.info("copying the final files from {} to {}", src.getAbsolutePath(), target.getAbsolutePath());
		try {
			FileUtils.copyDirectoryToDirectory(src, target);
		} catch (IOException e) {
			LOG.warn("Error copying the final files from {} to {}: msg: {}", src.getAbsolutePath(), target.getAbsolutePath(), e.getMessage(), e);
		}
	}

}