org.apache.hadoop.fs.FileStatus Java Examples

/** Test the FileStatus obtained calling listStatus on a file */
@Test
public void testListStatusOnFile() throws IOException {
  FileStatus[] stats = fs.listStatus(file1);
  assertEquals(1, stats.length);
  FileStatus status = stats[0];
  assertFalse(file1 + " should be a file", status.isDirectory());
  assertEquals(blockSize, status.getBlockSize());
  assertEquals(1, status.getReplication());
  assertEquals(fileSize, status.getLen());
  assertEquals(file1.makeQualified(fs.getUri(), 
      fs.getWorkingDirectory()).toString(), 
      status.getPath().toString());
  
  RemoteIterator<FileStatus> itor = fc.listStatus(file1);
  status = itor.next();
  assertEquals(stats[0], status);
  assertFalse(file1 + " should be a file", status.isDirectory());
}
 

/**
 * touch file
 */
public static void touch(String filePath) throws IOException {
    checkHDFSConf();
    Path path = new Path(filePath);
    FileStatus st;
    if (fileSystem.exists(path)) {
        st = fileSystem.getFileStatus(path);
        if (st.isDirectory()) {
            throw new IOException(filePath + " is a directory");
        } else if (st.getLen() != 0) {
            throw new IOException(filePath + " must be a zero-length file");
        }
    }
    FSDataOutputStream out = null;
    try {
        out = fileSystem.create(path);
    } finally {
        if (out != null) {
            out.close();
        }
    }

}
 

private void doGetFileLinkStatusTargetNotReadable() throws Exception {
  // Try to getFileLinkStatus the link when the target is not readable
  user.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws IOException {
      FileContext myfc = FileContext.getFileContext(conf);
      FileStatus stat = myfc.getFileLinkStatus(link);
      assertEquals("Expected link's FileStatus path to match link!",
          link.makeQualified(fs.getUri(), fs.getWorkingDirectory()), stat.getPath());
      Path linkTarget = myfc.getLinkTarget(link);
      assertEquals("Expected link's target to match target!",
          target, linkTarget);
      return null;
    }
  });
}
 

private void cleanUp(boolean isAppendDictGlobal) throws IOException {
    long timestamp = System.currentTimeMillis();
    if (isAppendDictGlobal) {
        Long[] versions = listAllVersions();
        for (int i = 0; i < versions.length - maxVersions; i++) {
            if (versions[i] + versionTTL < timestamp) {
                fileSystem.delete(getVersionDir(versions[i]), true);
            }
        }
    } else {
        FileStatus[] segmentDictDirs = fileSystem.listStatus(basePath.getParent());
        for (FileStatus fileStatus : segmentDictDirs) {
            String filePath = fileStatus.getPath().getName();
            Long version = Long.parseLong(filePath.split("_")[1]);
            if (version + versionTTL < timestamp) {
                fileSystem.delete(new Path(basePath.getParent() + "/" + filePath), true);
            }
        }
    }
}
 

/**
 * stores the modification and access time for this inode. 
 * The access time is precise upto an hour. The transaction, if needed, is
 * written to the edits log but is not flushed.
 */
public synchronized void setTimes(String src, long mtime, long atime) throws IOException {
  if (!isAccessTimeSupported() && atime != -1) {
    throw new IOException("Access time for hdfs is not configured. " +
                          " Please set dfs.support.accessTime configuration parameter.");
  }
  //
  // The caller needs to have write access to set access & modification times.
  if (isPermissionEnabled) {
    checkPathAccess(src, FsAction.WRITE);
  }
  INodeFile inode = dir.getFileINode(src);
  if (inode != null) {
    dir.setTimes(src, inode, mtime, atime, true);
    if (auditLog.isInfoEnabled()) {
      final FileStatus stat = dir.getFileInfo(src);
      logAuditEvent(UserGroupInformation.getCurrentUGI(),
                    Server.getRemoteIp(),
                    "setTimes", src, null, stat);
    }
  } else {
    throw new FileNotFoundException("File " + src + " does not exist.");
  }
}
 

@Test
public void testHandleDeadWorker() throws Exception {
  Table table = TEST_UTIL.createTable(TABLE_NAME, FAMILY, TEST_UTIL.KEYS_FOR_HBA_CREATE_TABLE);
  for (int i = 0; i < 10; i++) {
    TEST_UTIL.loadTable(table, FAMILY);
  }
  HRegionServer testServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
  ProcedureExecutor<MasterProcedureEnv> masterPE = master.getMasterProcedureExecutor();
  List<FileStatus> wals = splitWALManager.getWALsToSplit(testServer.getServerName(), false);
  Assert.assertEquals(1, wals.size());
  TEST_UTIL.getHBaseCluster().killRegionServer(testServer.getServerName());
  TEST_UTIL.waitFor(30000, () -> master.getProcedures().stream()
      .anyMatch(procedure -> procedure instanceof SplitWALProcedure));
  Procedure splitWALProcedure = master.getProcedures().stream()
      .filter(procedure -> procedure instanceof SplitWALProcedure).findAny().get();
  Assert.assertNotNull(splitWALProcedure);
  TEST_UTIL.waitFor(5000, () -> ((SplitWALProcedure) splitWALProcedure).getWorker() != null);
  TEST_UTIL.getHBaseCluster()
      .killRegionServer(((SplitWALProcedure) splitWALProcedure).getWorker());
  ProcedureTestingUtility.waitProcedure(masterPE, splitWALProcedure.getProcId());
  Assert.assertTrue(splitWALProcedure.isSuccess());
  ProcedureTestingUtility.waitAllProcedures(masterPE);
}
 

private void removeFileSlicesForPartition(HoodieTimeline timeline, HoodieInstant instant, String partition,
    List<String> paths) {
  if (isPartitionAvailableInStore(partition)) {
    LOG.info("Removing file slices for partition (" + partition + ") for instant (" + instant + ")");
    FileStatus[] statuses = paths.stream().map(p -> {
      FileStatus status = new FileStatus();
      status.setPath(new Path(p));
      return status;
    }).toArray(FileStatus[]::new);
    List<HoodieFileGroup> fileGroups =
        buildFileGroups(statuses, timeline.filterCompletedAndCompactionInstants(), false);
    applyDeltaFileSlicesToPartitionView(partition, fileGroups, DeltaApplyMode.REMOVE);
  } else {
    LOG.warn("Skipping partition (" + partition + ") when syncing instant (" + instant + ") as it is not loaded");
  }
}
 

public static void setKnownSitesOnHDFS(Configuration conf, String[] val) throws IOException, URISyntaxException {
    conf.setInt(numberOfSites, val.length);
    FileSystem fs;
    for(int i = 0; i < val.length;i ++) {
        // check if dir add all files!
        fs = FileSystem.get(new URI(val[i]), conf);
        if(fs.isFile(new Path(val[i]))) {
            conf.set(sitesOnHDFSName + i, val[i]);
        } else {
            FileStatus[] files = fs.listStatus(new Path(val[i]));
            for(FileStatus file : files) {
                if (!file.isDir()) {
                    conf.set(sitesOnHDFSName + i, file.getPath().toString());
                }
            }
        }
    }
}
 

private synchronized void storeGenerations() throws IOException {
  FileSystem fileSystem = _path.getFileSystem(_configuration);
  FileStatus[] listStatus = fileSystem.listStatus(_path);
  SortedSet<FileStatus> existing = new TreeSet<FileStatus>(Arrays.asList(listStatus));
  long currentFile;
  if (!existing.isEmpty()) {
    FileStatus last = existing.last();
    currentFile = Long.parseLong(last.getPath().getName());
  } else {
    currentFile = 0;
  }
  Path path = new Path(_path, buffer(currentFile + 1));
  LOG.info("Creating new snapshot file [{0}]", path);
  FSDataOutputStream outputStream = fileSystem.create(path, false);
  Writer writer = SequenceFile.createWriter(_configuration, outputStream, Text.class, LongWritable.class,
      CompressionType.NONE, null);
  for (Entry<String, Long> e : _namesToGenerations.entrySet()) {
    writer.append(new Text(e.getKey()), new LongWritable(e.getValue()));
  }
  writer.close();
  outputStream.close();
  cleanupOldFiles(fileSystem, existing);
}
 

/**
 * Check all files in a mob column family dir.
 */
protected void checkMobColFamDir(Path cfDir) throws IOException {
  FileStatus[] statuses = null;
  try {
    statuses = fs.listStatus(cfDir); // use same filter as scanner.
  } catch (FileNotFoundException fnfe) {
    // Hadoop 0.23+ listStatus semantics throws an exception if the path does not exist.
    LOG.warn("Mob colfam Directory " + cfDir +
        " does not exist.  Likely the table is deleted. Skipping.");
    missedMobFiles.add(cfDir);
    return;
  }

  List<FileStatus> hfs = FSUtils.filterFileStatuses(statuses, new HFileFilter(fs));
  // Hadoop 1.0 listStatus does not throw an exception if the path does not exist.
  if (hfs.isEmpty() && !fs.exists(cfDir)) {
    LOG.warn("Mob colfam Directory " + cfDir +
        " does not exist.  Likely the table is deleted. Skipping.");
    missedMobFiles.add(cfDir);
    return;
  }
  for (FileStatus hfFs : hfs) {
    Path hf = hfFs.getPath();
    checkMobFile(hf);
  }
}
 

/**
 * Find suitable partitions, extract timestamp and compare it with previousTimestamp.
 */
@VisibleForTesting
static List<Tuple2<List<String>, Long>> suitablePartitions(
		Context context,
		long previousTimestamp,
		FileStatus[] statuses) {
	List<Tuple2<List<String>, Long>> partValueList = new ArrayList<>();
	for (FileStatus status : statuses) {
		List<String> partValues = extractPartitionValues(
				new org.apache.flink.core.fs.Path(status.getPath().toString()));
		long timestamp = context.extractTimestamp(
				context.partitionKeys(),
				partValues,
				// to UTC millisecond.
				() -> TimestampData.fromTimestamp(
						new Timestamp(status.getModificationTime())).getMillisecond());
		if (timestamp >= previousTimestamp) {
			partValueList.add(new Tuple2<>(partValues, timestamp));
		}
	}
	return partValueList;
}
 

@Test
public void testCustomProvider() throws Exception {
  FileSystem fs = FileSystem.get(miniDFS.getConfiguration(0));
  fs.mkdirs(new Path("/user/xxx"));
  FileStatus status = fs.getFileStatus(new Path("/user/xxx"));
  Assert.assertEquals(System.getProperty("user.name"), status.getOwner());
  Assert.assertEquals("supergroup", status.getGroup());
  Assert.assertEquals(new FsPermission((short) 0755), status.getPermission());
  fs.mkdirs(new Path("/user/authz"));
  Path p = new Path("/user/authz");
  status = fs.getFileStatus(p);
  Assert.assertEquals("foo", status.getOwner());
  Assert.assertEquals("bar", status.getGroup());
  Assert.assertEquals(new FsPermission((short) 0770), status.getPermission());
  AclStatus aclStatus = fs.getAclStatus(p);
  Assert.assertEquals(1, aclStatus.getEntries().size());
  Assert.assertEquals(AclEntryType.GROUP, aclStatus.getEntries().get(0)
          .getType());
  Assert.assertEquals("xxx", aclStatus.getEntries().get(0)
          .getName());
  Assert.assertEquals(FsAction.ALL, aclStatus.getEntries().get(0)
          .getPermission());
  Map<String, byte[]> xAttrs = fs.getXAttrs(p);
  Assert.assertTrue(xAttrs.containsKey("user.test"));
  Assert.assertEquals(2, xAttrs.get("user.test").length);
}
 

public List<LocatedFileStatus> listFilesInfo(String dir) throws IOException
{
  List<LocatedFileStatus> files = new ArrayList<>();
  Path path = new Path(dir);

  FileStatus fileStatus = fileSystem.getFileStatus(path);
  if (!fileStatus.isDirectory()) {
    throw new FileNotFoundException("Cannot read directory " + dir);
  }
  RemoteIterator<LocatedFileStatus> it = fileSystem.listFiles(path, false);
  while (it.hasNext()) {
    LocatedFileStatus lfs = it.next();
    files.add(lfs);
  }
  return files;
}
 

@Override
void checkSnapshots() throws Exception {
  for (Path snapshotFile : statusMap.keySet()) {
    FileStatus currentStatus = fs.exists(snapshotFile) ? fs
        .getFileStatus(snapshotFile) : null;
    FileStatus originalStatus = statusMap.get(snapshotFile);
    assertEquals(currentStatus, originalStatus);
    if (currentStatus != null) {
      String s = null;
      if (!currentStatus.toString().equals(originalStatus.toString())) {
        s = "FAILED: " + getClass().getSimpleName()
            + ": file="  + file + ", snapshotFile" + snapshotFile
            + "\n\n currentStatus = " + currentStatus
            +   "\noriginalStatus = " + originalStatus
            + "\n\nfile        : " + fsdir.getINode(file.toString()).toDetailString()
            + "\n\nsnapshotFile: " + fsdir.getINode(snapshotFile.toString()).toDetailString();
        
        SnapshotTestHelper.dumpTree(s, cluster);
      }
      assertEquals(s, currentStatus.toString(), originalStatus.toString());
    }
  }
}
 

public void checkPermissionRetention(Configuration conf, String ourUrl,
    Path path) throws Exception {
  CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
  // let's add a new credential and flush and check that permissions are still set to 777
  char[] cred = new char[32];
  for(int i =0; i < cred.length; ++i) {
    cred[i] = (char) i;
  }
  // create a new key
  try {
    provider.createCredentialEntry("key5", cred);
  } catch (Exception e) {
    e.printStackTrace();
    throw e;
  }
  provider.flush();
  // get a new instance of the provider to ensure it was saved correctly
  provider = CredentialProviderFactory.getProviders(conf).get(0);
  assertArrayEquals(cred, provider.getCredentialEntry("key5").getCredential());

  FileSystem fs = path.getFileSystem(conf);
  FileStatus s = fs.getFileStatus(path);
  assertTrue("Permissions should have been retained from the preexisting " +
  		"keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
}
 

@Test
public void testListDirectory() throws Exception {
  Path rootFolder = new Path("testingList");
  assertTrue(fs.mkdirs(rootFolder));
  FileStatus[] listed = fs.listStatus(rootFolder);
  assertEquals(0, listed.length);
  Path innerFolder = new Path(rootFolder, "inner");
  assertTrue(fs.mkdirs(innerFolder));
  listed = fs.listStatus(rootFolder);
  assertEquals(1, listed.length);
  assertTrue(listed[0].isDirectory());
  Path innerFile = new Path(innerFolder, "innerFile");
  writeString(innerFile, "testing");
  listed = fs.listStatus(rootFolder);
  assertEquals(1, listed.length);
  assertTrue(listed[0].isDirectory());
  listed = fs.listStatus(innerFolder);
  assertEquals(1, listed.length);
  assertFalse(listed[0].isDirectory());
  assertTrue(fs.delete(rootFolder, true));
}
 

@CliCommand(value = "su", help = "Changes current active user [*experimental*]")
    public synchronized String su(@CliOption(key = {""}, help = "su [<username>]") String newUser) throws IOException {
        if (StringUtils.isEmpty(newUser)) {
            return "No username is defined! ";
        }
//        else {
//            newUser = BashUtils.parseArguments(newUser)[0];
//        }
        final FileSystem fs = getFileSystem();
        final Path usersDir = new Path("/user");
        if (fs.exists(usersDir)) {
            final String finalNewUser = newUser;
            final boolean foundUser = Arrays.stream(fs.listStatus(usersDir)).
                    filter(FileStatus::isDirectory).
                    anyMatch(fileStatus -> fileStatus.getPath().getName().equals(finalNewUser));
            if (!foundUser) {
                return "User " + newUser + " does not exist!";
            }
        }
        System.setProperty("HADOOP_USER_NAME", newUser);
        UserGroupInformation.loginUserFromSubject(null);
        currentDir = null;
        return "";
    }
 

/**
 * groups together all the data blocks for the same HDFS block
 *
 * @param rowGroupBlocks      data blocks (row groups)
 * @param hdfsBlocksArray     hdfs blocks
 * @param fileStatus          the containing file
 * @param requestedSchema     the schema requested by the user
 * @param readSupportMetadata the metadata provided by the readSupport implementation in init
 * @param minSplitSize        the mapred.min.split.size
 * @param maxSplitSize        the mapred.max.split.size
 * @return the splits (one per HDFS block)
 * @throws IOException If hosts can't be retrieved for the HDFS block
 */
static <T> List<ParquetInputSplit> generateSplits(
        List<BlockMetaData> rowGroupBlocks,
        BlockLocation[] hdfsBlocksArray,
        FileStatus fileStatus,
        String requestedSchema,
        Map<String, String> readSupportMetadata, long minSplitSize, long maxSplitSize) throws IOException {

  List<SplitInfo> splitRowGroups =
      generateSplitInfo(rowGroupBlocks, hdfsBlocksArray, minSplitSize, maxSplitSize);

  //generate splits from rowGroups of each split
  List<ParquetInputSplit> resultSplits = new ArrayList<ParquetInputSplit>();
  for (SplitInfo splitInfo : splitRowGroups) {
    ParquetInputSplit split = splitInfo.getParquetInputSplit(fileStatus, requestedSchema, readSupportMetadata);
    resultSplits.add(split);
  }
  return resultSplits;
}
 

@Test
public void test() throws IOException, InterruptedException {
  region
    .update(r -> r.put(new Put(Bytes.toBytes(1)).addColumn(CF1, QUALIFIER, Bytes.toBytes(1))));
  region.flush(true);
  Path testDir = htu.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(htu.getConfiguration());
  // no archived wal files yet
  assertFalse(fs.exists(globalWALArchiveDir));
  region.requestRollAll();
  region.waitUntilWalRollFinished();
  // should have one
  FileStatus[] files = fs.listStatus(globalWALArchiveDir);
  assertEquals(1, files.length);
  Thread.sleep(2000);
  // should still be there
  assertTrue(fs.exists(files[0].getPath()));
  Thread.sleep(6000);
  // should have been cleaned
  assertEquals(0, fs.listStatus(globalWALArchiveDir).length);
}
 

/**
 * get all file status of a dir.
 */
public static List<FileStatus> listFileStatus(String dir) throws IOException {
    checkHDFSConf();
    List<FileStatus> fileStatusList = new ArrayList<>();
    Path path = new Path(dir);
    if (fileSystem.isFile(path)) {
        return fileStatusList;
    }
    FileStatus[] statuses = fileSystem.listStatus(path);
    for (FileStatus fileStatus : statuses) {
        if (!fileStatus.isDirectory()) {
            fileStatusList.add(fileStatus);
        }
    }
    return fileStatusList;
}
 

private Set<FileStatus> getStatuses(final Path path, final boolean recursive, final FileSystem hdfs, final PathFilter filter) throws IOException {
    final Set<FileStatus> statusSet = new HashSet<>();

    getLogger().debug("Fetching listing for {}", new Object[] {path});
    final FileStatus[] statuses = hdfs.listStatus(path, filter);

    for ( final FileStatus status : statuses ) {
        if ( status.isDirectory() ) {
            if ( recursive ) {
                try {
                    statusSet.addAll(getStatuses(status.getPath(), recursive, hdfs, filter));
                } catch (final IOException ioe) {
                    getLogger().error("Failed to retrieve HDFS listing for subdirectory {} due to {}; will continue listing others", new Object[] {status.getPath(), ioe});
                }
            }
        } else {
            statusSet.add(status);
        }
    }

    return statusSet;
}
 

/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
 

/**
 * Finds the avro file in the input folder, and returns its avro schema
 * @param inputPathDir
 * @return
 * @throws IOException
 */
public static Schema getSchema(String inputPathDir) throws IOException  {
  FileSystem fs = FileSystem.get(new Configuration());
  Schema avroSchema = null;
  for (String input : inputPathDir.split(ThirdEyeConstants.FIELD_SEPARATOR)) {
    Path inputPath = new Path(input);
    for (FileStatus fileStatus : fs.listStatus(inputPath)) {
      if (fileStatus.isFile() && fileStatus.getPath().getName().endsWith(ThirdEyeConstants.AVRO_SUFFIX)) {
        LOGGER.info("Extracting schema from {}", fileStatus.getPath());
        avroSchema = extractSchemaFromAvro(fileStatus.getPath());
        break;
      }
    }
  }
  return avroSchema;
}
 

/**
 * Make an assertion about the length of a file
 * @param fs filesystem
 * @param path path of the file
 * @param expected expected length
 * @throws IOException on File IO problems
 */
public static void assertFileHasLength(FileSystem fs, Path path,
                                       int expected) throws IOException {
  FileStatus status = fs.getFileStatus(path);
  assertEquals(
    "Wrong file length of file " + path + " status: " + status,
    expected,
    status.getLen());
}
 

/**
 * Recovers a corrupt block in a parity file to a local file.
 * 
 * The encoder generates codec.parityLength parity blocks for a source file
 * stripe. Since we want only one of the parity blocks, this function
 * creates null outputs for the blocks to be discarded.
 * 
 * @param fs
 *            The filesystem in which both srcFile and parityFile reside.
 * @param srcStat
 *            fileStatus of The source file.
 * @param blockSize
 *            The block size for the parity files.
 * @param corruptOffset
 *            The location of corruption in the parity file.
 * @param out
 *            The destination for the reovered block.
 * @param progress
 *            A reporter for progress.
 */
public void recoverParityBlockToStream(FileSystem fs, FileStatus srcStat,
		long blockSize, Path parityFile, long corruptOffset,
		OutputStream out, Progressable progress) throws IOException {
	LOG.info("Recovering parity block" + parityFile + ":" + corruptOffset);
	Path srcFile = srcStat.getPath();
	// Get the start offset of the corrupt block.
	corruptOffset = (corruptOffset / blockSize) * blockSize;
	// Output streams to each block in the parity file stripe.
	OutputStream[] outs = new OutputStream[codec.parityLength];
	long indexOfCorruptBlockInParityStripe = (corruptOffset / blockSize)
			% codec.parityLength;
	LOG.info("Index of corrupt block in parity stripe: "
			+ indexOfCorruptBlockInParityStripe);
	// Create a real output stream for the block we want to recover,
	// and create null streams for the rest.
	for (int i = 0; i < codec.parityLength; i++) {
		if (indexOfCorruptBlockInParityStripe == i) {
			outs[i] = out;
		} else {
			outs[i] = new NullOutputStream();
		}
	}
	// Get the stripe index and start offset of stripe.
	long stripeIdx = corruptOffset / (codec.parityLength * blockSize);
	StripeReader sReader = StripeReader.getStripeReader(codec, conf,
			blockSize, fs, stripeIdx, srcStat);
	// Get input streams to each block in the source file stripe.
	assert sReader.hasNext() == true;
	InputStream[] blocks = sReader.getNextStripeInputs();
	LOG.info("Starting recovery by using source stripe " + srcFile
			+ ": stripe " + stripeIdx);
	try {
		// Read the data from the blocks and write to the parity file.
		encodeStripe(blocks, blockSize, outs, progress);
	} finally {
		RaidUtils.closeStreams(blocks);
	}
}
 

private void doRetrieveMatchingResources(Path rootDir, String subPattern, Set<Resource> results) throws IOException {
    if (!this.fs.isFile(rootDir)) {
        FileStatus[] statuses = null;
        statuses = this.fs.listStatus(rootDir);
        if (!ObjectUtils.isEmpty(statuses)) {
            String root = rootDir.toUri().getPath();
            FileStatus[] var6 = statuses;
            int var7 = statuses.length;

            for(int var8 = 0; var8 < var7; ++var8) {
                FileStatus fileStatus = var6[var8];
                Path p = fileStatus.getPath();
                String location = p.toUri().getPath();
                if (location.startsWith(root)) {
                    location = location.substring(root.length());
                }

                if (fileStatus.isDir() && this.pathMatcher.matchStart(subPattern, location)) {
                    this.doRetrieveMatchingResources(p, subPattern, results);
                } else if (this.pathMatcher.match(subPattern.substring(1), location)) {
                    results.add(new HdfsResource(p, this.fs));
                }
            }
        }
    } else if (this.pathMatcher.match(subPattern, stripPrefix(rootDir.toUri().getPath()))) {
        results.add(new HdfsResource(rootDir, this.fs));
    }

}
 

public static FileStatus toFileStatus(FileStatusDTO dto) {
  if (null == dto) {
    return null;
  }

  return new FileStatus(dto.length, dto.isdir, dto.blockReplication, dto.blocksize, dto.modificationTime,
      dto.accessTime, FSPermissionDTO.fromFsPermissionDTO(dto.permission), dto.owner, dto.group,
      FilePathDTO.toPath(dto.symlink), FilePathDTO.toPath(dto.path));
}
 

/**
 * Tests that a file on the HDFS cluster contains the given parquet.
 *
 * @param path the name of the file on the HDFS cluster
 * @param expected the expected avro record in the file .
 */
public static void assertReadParquetFile(FileSystem fs, String path, Set<IndexedRecord> expected, boolean part) throws IOException {
    Path p = new Path(path);
    if (fs.isFile(p)) {
        try (AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(fs.getConf(), new Path(path))) {
            IndexedRecord record = null;
            while (null != (record = reader.read())){
                IndexedRecord eqRecord = null;
                for (IndexedRecord indexedRecord : expected) {
                    if(indexedRecord.equals(record)){
                        eqRecord = indexedRecord;
                        break;
                    }
                }
                expected.remove(eqRecord);
            }
        }
        // Check before asserting for the message.
        if (!part && expected.size() != 0)
            assertThat("Not all avro records found: " + expected.iterator().next(), expected, hasSize(0));
    } else if (fs.isDirectory(p)) {
        for (FileStatus fstatus : FileSystemUtil.listSubFiles(fs, p)) {
            assertReadParquetFile(fs, fstatus.getPath().toString(), expected, true);
        }
        // Check before asserting for the message.
        if (expected.size() != 0)
            assertThat("Not all avro records found: " + expected.iterator().next(), expected, hasSize(0));
    } else {
        fail("No such path: " + path);
    }
}
 

/**
 * Check that toString produces the expected output for a file.
 */
@Test
public void toStringFile() throws IOException {
  boolean isdir = false; 
  FileStatus fileStatus = new FileStatus(LENGTH, isdir, REPLICATION, BLKSIZE,
      MTIME, ATIME, PERMISSION, OWNER, GROUP, null, PATH);   
  validateToString(fileStatus);
}
 

@After
public void tearDown() throws Exception {
  // testAppendClose closes the FileSystem, which will prevent us from closing cleanly here.
  try {
    wals.close();
  } catch (IOException exception) {
    LOG.warn("Encountered exception while closing wal factory. If you have other errors, this" +
        " may be the cause. Message: " + exception);
    LOG.debug("Exception details for failure to close wal factory.", exception);
  }
  FileStatus[] entries = fs.listStatus(new Path("/"));
  for (FileStatus dir : entries) {
    fs.delete(dir.getPath(), true);
  }
}