org.apache.hadoop.hdfs.MiniDFSCluster#Builder

Source File: RollingSinkITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void setup() throws Exception {

	LOG.info("In RollingSinkITCase: Starting MiniDFSCluster ");

	dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	hdfsURI = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/";

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(4)
			.build());

	miniClusterResource.before();
}

Source File: BucketingSinkFaultToleranceITCase.java From flink with Apache License 2.0

6 votes

@Before
public void createHDFS() throws IOException {
	Configuration conf = new Configuration();

	File dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	outPath = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/string-non-rolling-out";
}

Source File: TajoTestingCluster.java From tajo with Apache License 2.0

6 votes

/**
 * Start a minidfscluster.
 * Can only create one.
 * @param servers How many DNs to start.
 * @param dir Where to home your dfs cluster.
 * @param hosts hostnames DNs to run on.
 * @throws Exception
 * @see {@link #shutdownMiniDFSCluster()}
 * @return The mini dfs cluster created.
 * @throws java.io.IOException
 */
public MiniDFSCluster startMiniDFSCluster(int servers,
                                          File dir,
                                          final String hosts[])
    throws IOException {

  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.getAbsolutePath());
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false);
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 0);

  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(new HdfsConfiguration(conf));
  builder.hosts(hosts);
  builder.numDataNodes(servers);
  builder.format(true);
  builder.waitSafeMode(true);
  this.dfsCluster = builder.build();

  // Set this just-started cluster as our filesystem.
  this.defaultFS = this.dfsCluster.getFileSystem();
  this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString());
  this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo");
  isDFSRunning = true;
  return this.dfsCluster;
}

Source File: HDFSFileStreamSourceTest.java From incubator-samoa with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
  // Start MiniDFSCluster
  config = new Configuration();
  config.set("hdfs.minidfs.basedir", "target/build/test/data/dfs");
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(config).hosts(HOSTS).numDataNodes(1)
      .format(true);
  hdfsCluster = builder.build();
  hdfsCluster.waitActive();
  hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort();

  // Construct stream source
  streamSource = new HDFSFileStreamSource();

  // General config
  config.set("fs.defaultFS", hdfsURI);
}

Source File: TestBinaryTokenFile.java From hadoop with Apache License 2.0

6 votes

@BeforeClass
public static void setUp() throws Exception {
  final Configuration conf = new Configuration();
  
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  conf.set(YarnConfiguration.RM_PRINCIPAL, "jt_id/" + SecurityUtil.HOSTNAME_PATTERN + "@APACHE.ORG");
  
  final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
  builder.checkExitOnShutdown(true);
  builder.numDataNodes(numSlaves);
  builder.format(true);
  builder.racks(null);
  dfsCluster = builder.build();
  
  mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(), noOfNMs);
  mrCluster.init(conf);
  mrCluster.start();

  NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); 
  
  FileSystem fs = dfsCluster.getFileSystem(); 
  p1 = new Path("file1");
  p1 = fs.makeQualified(p1);
}

Source File: FileSystemFunctionsTest.java From metron with Apache License 2.0

6 votes

@BeforeAll
public static void setupFS() throws IOException {
  {
    hdfsBaseDir = Files.createTempDirectory("test_hdfs").toFile().getAbsoluteFile();
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsBaseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    hdfsCluster = builder.build();
    hdfsPrefix = "/";
  }
  {
    localPrefix = "target/fsTest/";
    if (new File(localPrefix).exists()) {
      new File(localPrefix).delete();
    }
    new File(localPrefix).mkdirs();
  }
}

Source File: HDFSTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Before
public void createHDFS() {
	try {
		Configuration hdConf = new Configuration();

		File baseDir = new File("./target/hdfs/hdfsTest").getAbsoluteFile();
		FileUtil.fullyDelete(baseDir);
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";

		hdPath = new org.apache.hadoop.fs.Path("/test");
		hdfs = hdPath.getFileSystem(hdConf);
		FSDataOutputStream stream = hdfs.create(hdPath);
		for (int i = 0; i < 10; i++) {
			stream.write("Hello HDFS\n".getBytes(ConfigConstants.DEFAULT_CHARSET));
		}
		stream.close();

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}

Source File: DistributedCacheDfsTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void setup() throws Exception {
	File dataDir = TEMP_FOLDER.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	String hdfsURI = "hdfs://"
		+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
		+ "/";

	FileSystem dfs = FileSystem.get(new URI(hdfsURI));
	testFile = writeFile(dfs, dfs.getHomeDirectory(), "testFile");

	testDir = new Path(dfs.getHomeDirectory(), "testDir");
	dfs.mkdirs(testDir);
	writeFile(dfs, testDir, "testFile1");
	writeFile(dfs, testDir, "testFile2");
}

Source File: RollingSinkFaultToleranceITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@BeforeClass
public static void createHDFS() throws IOException {
	Configuration conf = new Configuration();

	File dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	outPath = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/string-non-rolling-out";
}

Source File: TestDocUseCases.java From jsr203-hadoop with Apache License 2.0

5 votes

private static MiniDFSCluster startMini(String testName) throws IOException {
	File baseDir = new File("./target/hdfs/" + testName).getAbsoluteFile();
	FileUtil.fullyDelete(baseDir);
	Configuration conf = new Configuration();
	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	MiniDFSCluster hdfsCluster = builder.clusterId(testName).build();
	hdfsCluster.waitActive();
	return hdfsCluster;
}

Source File: CreateHDFSStoreTest.java From gemfirexd-oss with Apache License 2.0

5 votes

public static MiniDFSCluster initMiniCluster(int port, int numDN, HashMap<String, String> map) throws Exception {
  System.setProperty("test.build.data", "hdfs-test-cluster");
  Configuration hconf = new HdfsConfiguration();
  for (Entry<String, String> entry : map.entrySet()) {
    hconf.set(entry.getKey(), entry.getValue());
  }

  Builder builder = new MiniDFSCluster.Builder(hconf);
  builder.numDataNodes(numDN);
  builder.nameNodePort(port);
  MiniDFSCluster cluster = builder.build();
  return cluster;
}

Source File: YarnFileStageTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@BeforeClass
public static void createHDFS() throws Exception {
	Assume.assumeTrue(!OperatingSystem.isWindows());

	final File tempDir = CLASS_TEMP_DIR.newFolder();

	org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
	hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath());

	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
	hdfsCluster = builder.build();
	hdfsRootPath = new Path(hdfsCluster.getURI());
}

Source File: TestBlockScanner.java From big-c with Apache License 2.0

5 votes

TestContext(Configuration conf, int numNameServices) throws Exception {
  this.numNameServices = numNameServices;
  MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).
      numDataNodes(1).
      storagesPerDatanode(1);
  if (numNameServices > 1) {
    bld.nnTopology(MiniDFSNNTopology.
          simpleFederatedTopology(numNameServices));
  }
  cluster = bld.build();
  cluster.waitActive();
  dfs = new DistributedFileSystem[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    dfs[i] = cluster.getFileSystem(i);
  }
  bpids = new String[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    bpids[i] = cluster.getNamesystem(i).getBlockPoolId();
  }
  datanode = cluster.getDataNodes().get(0);
  blockScanner = datanode.getBlockScanner();
  for (int i = 0; i < numNameServices; i++) {
    dfs[i].mkdirs(new Path("/test"));
  }
  data = datanode.getFSDataset();
  volumes = data.getVolumes();
}

Source File: FileStoreTestBase.java From pulsar with Apache License 2.0

5 votes

@BeforeMethod
public void start() throws Exception {
    File baseDir = Files.createTempDirectory(basePath).toFile().getAbsoluteFile();
    Configuration conf = new Configuration();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    hdfsCluster = builder.build();

    hdfsURI = "hdfs://localhost:"+ hdfsCluster.getNameNodePort() + "/";
    Properties properties = new Properties();
    fileSystemManagedLedgerOffloader = new FileSystemManagedLedgerOffloader(
            OffloadPolicies.create(properties),
            scheduler, hdfsURI, basePath);
}

Source File: HadoopRecoverableWriterOldHadoopWithNoTruncateSupportTest.java From flink with Apache License 2.0

5 votes

@BeforeClass
public static void createHDFS() throws Exception {
	final File baseDir = TEMP_FOLDER.newFolder();

	final Configuration hdConf = new Configuration();
	hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());

	final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
	hdfsCluster = builder.build();

	final org.apache.hadoop.fs.FileSystem hdfs = hdfsCluster.getFileSystem();

	fileSystem = new HadoopFileSystem(hdfs);
	basePath = new Path(hdfs.getUri() + "/tests");
}

Source File: TestPath.java From jsr203-hadoop with Apache License 2.0

5 votes

private static MiniDFSCluster startMini(String testName) throws IOException {
  File baseDir = new File("./target/hdfs/" + testName).getAbsoluteFile();
  FileUtil.fullyDelete(baseDir);
  Configuration conf = new Configuration();
  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
  MiniDFSCluster hdfsCluster = builder.clusterId(testName).build();
  hdfsCluster.waitActive();
  return hdfsCluster;
}

Source File: TestBlockScanner.java From hadoop with Apache License 2.0

5 votes

TestContext(Configuration conf, int numNameServices) throws Exception {
  this.numNameServices = numNameServices;
  MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).
      numDataNodes(1).
      storagesPerDatanode(1);
  if (numNameServices > 1) {
    bld.nnTopology(MiniDFSNNTopology.
          simpleFederatedTopology(numNameServices));
  }
  cluster = bld.build();
  cluster.waitActive();
  dfs = new DistributedFileSystem[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    dfs[i] = cluster.getFileSystem(i);
  }
  bpids = new String[numNameServices];
  for (int i = 0; i < numNameServices; i++) {
    bpids[i] = cluster.getNamesystem(i).getBlockPoolId();
  }
  datanode = cluster.getDataNodes().get(0);
  blockScanner = datanode.getBlockScanner();
  for (int i = 0; i < numNameServices; i++) {
    dfs[i].mkdirs(new Path("/test"));
  }
  data = datanode.getFSDataset();
  volumes = data.getVolumes();
}

Source File: YarnPreConfiguredMasterHaServicesTest.java From flink with Apache License 2.0

5 votes

@BeforeClass
public static void createHDFS() throws Exception {
	Assume.assumeTrue(!OperatingSystem.isWindows());

	final File tempDir = TEMP_DIR.newFolder();

	org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
	hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath());

	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
	hdfsCluster = builder.build();
	hdfsRootPath = new Path(hdfsCluster.getURI());
}

Source File: TestBalancerWithNodeGroup.java From hadoop with Apache License 2.0

4 votes

/**
 * Create a cluster with even distribution, and a new empty node is added to
 * the cluster, then test node-group locality for balancer policy.
 */
@Test(timeout=60000)
public void testBalancerWithNodeGroup() throws Exception {
  Configuration conf = createConf();
  long[] capacities = new long[]{CAPACITY, CAPACITY, CAPACITY, CAPACITY};
  String[] racks = new String[]{RACK0, RACK0, RACK1, RACK1};
  String[] nodeGroups = new String[]{NODEGROUP0, NODEGROUP0, NODEGROUP1, NODEGROUP2};
  
  int numOfDatanodes = capacities.length;
  assertEquals(numOfDatanodes, racks.length);
  assertEquals(numOfDatanodes, nodeGroups.length);
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
                              .numDataNodes(capacities.length)
                              .racks(racks)
                              .simulatedCapacities(capacities);
  MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups);
  cluster = new MiniDFSClusterWithNodeGroup(builder);
  try {
    cluster.waitActive();
    client = NameNodeProxies.createProxy(conf, 
        cluster.getFileSystem(0).getUri(),
        ClientProtocol.class).getProxy();

    long totalCapacity = TestBalancer.sum(capacities);
    // fill up the cluster to be 20% full
    long totalUsedSpace = totalCapacity * 2 / 10;
    TestBalancer.createFile(cluster, filePath, totalUsedSpace / (numOfDatanodes/2),
        (short) (numOfDatanodes/2), 0);
    
    long newCapacity = CAPACITY;
    String newRack = RACK1;
    String newNodeGroup = NODEGROUP2;
    // start up an empty node with the same capacity and on NODEGROUP2
    cluster.startDataNodes(conf, 1, true, null, new String[]{newRack},
        new long[] {newCapacity}, new String[]{newNodeGroup});

    totalCapacity += newCapacity;

    // run balancer and validate results
    runBalancer(conf, totalUsedSpace, totalCapacity);

  } finally {
    cluster.shutdown();
  }
}

Source File: MiniQJMHACluster.java From big-c with Apache License 2.0

4 votes

public MiniDFSCluster.Builder getDfsBuilder() {
  return dfsBuilder;
}

Java Code Examples for org.apache.hadoop.hdfs.MiniDFSCluster#Builder