org.apache.hadoop.fs.GlobFilter Java Examples

The following examples show how to use org.apache.hadoop.fs.GlobFilter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HdfsFileSystem.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
public HdfsFileSystem(String filePattern, PathMatcherMode mode, boolean processSubdirectories, FileSystem fs) {
  this.filePattern = filePattern;
  this.processSubdirectories = processSubdirectories;
  this.fs = fs;

  try {
    if (mode == GLOB) {
      filter = new GlobFilter(filePattern);
    } else if (mode == REGEX) {
      Pattern pattern = Pattern.compile(filePattern);
      filter = path -> pattern.matcher(path.toString()).matches();
    } else {
      throw new IllegalArgumentException("Unrecognized Path Matcher Mode: " + mode.getLabel());
    }
  } catch(IOException e) {
    throw new IllegalArgumentException("Can't create filter pattern: " + e.toString(), e);
  }
}
Example #2
Source File: HDFSHelper.java    From laser with Apache License 2.0 6 votes vote down vote up 
public static Path[] getFilePaths(Path filePath, String filePattern,
		FileSystem fs) throws IOException {
	if (!fs.exists(filePath)) {
		return new Path[0];
	}
	FileStatus[] hdfsFiles = fs.listStatus(filePath, new GlobFilter(
			filePattern));
	Path[] hdfsFilePaths = FileUtil.stat2Paths(hdfsFiles);
	List<Path> files = new ArrayList<Path>();
	for (Path hdfsFilePath : hdfsFilePaths) {
		FileStatus fileStatus = fs.getFileStatus(hdfsFilePath);
		if (!fileStatus.isDir()) {
			files.add(hdfsFilePath);
		}
	}
	return files.toArray(new Path[0]);
}
Example #3
Source File: Configuration.java    From laser with Apache License 2.0 6 votes vote down vote up 
public synchronized void load(Path path, FileSystem fs) throws IOException {
	final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
	FileStatus[] fileStatus = fs.listStatus(path, new GlobFilter(
			"*.properties"));
	for (FileStatus file : fileStatus) {
		if (file.isFile()) {
			Path p = file.getPath();
			FSDataInputStream in = fs.open(p);
			Collection configuration = OBJECT_MAPPER.readValue(in,
					Collection.class);
			String collection = p.getName().substring(0,
					p.getName().lastIndexOf(".properties"));
			configuration.setCollecion(collection);
			mapper.put(collection, configuration);
		}
	}
}
Example #4
Source File: SecurityUtils.java    From envelope with Apache License 2.0 5 votes vote down vote up 
/**
 * Get a list of token store files for the job. The {@value TOKENS_FILE}
 * prefix is suffixed with "*" to perform a glob search.
 * @param config application section of Envelope job config
 * @param hadoopConf Hadoop Configuration for the FileSystem
 * @return list of {@link Path}s of matching files
 * @throws IOException if the FileSystem cannot be read
 */
public static List<Path> getExistingTokenStoreFiles(Config config, Configuration hadoopConf, boolean onDriver) throws IOException {
  Path tokenFilePrefix = new Path(getTokenStoreFilePath(config, onDriver));
  FileSystem fs = FileSystem.get(hadoopConf);
  FileStatus[] matches = fs.listStatus(tokenFilePrefix.getParent(),
      new GlobFilter(tokenFilePrefix.getName() + "*"));
  List<Path> existingFiles = new ArrayList<>();
  for (FileStatus f : matches) {
    if (!f.getPath().getName().endsWith("READY")) {
      existingFiles.add(f.getPath());
    }
  }
  return existingFiles;
}
Example #5
Source File: HDFSHelper.java    From laser with Apache License 2.0 5 votes vote down vote up 
public static void deleteFiles(Path path, String filePattern, FileSystem fs)
		throws IOException {
	FileStatus[] fileStatus = fs.listStatus(path, new GlobFilter(
			filePattern));
	for (FileStatus file : fileStatus) {
		fs.delete(file.getPath(), true);
	}
}
Example #6
Source File: FSOperations.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a list-status executor.
 *
 * @param path the directory to retrieve the status of its contents.
 * @param filter glob filter to use.
 *
 * @throws IOException thrown if the filter expression is incorrect.
 */
public FSListStatus(String path, String filter) throws IOException {
  this.path = new Path(path);
  this.filter = (filter == null) ? this : new GlobFilter(filter);
}
Example #7
Source File: FSOperations.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Creates a list-status executor.
 *
 * @param path the directory to retrieve the status of its contents.
 * @param filter glob filter to use.
 *
 * @throws IOException thrown if the filter expression is incorrect.
 */
public FSListStatus(String path, String filter) throws IOException {
  this.path = new Path(path);
  this.filter = (filter == null) ? this : new GlobFilter(filter);
}