Java Code Examples for org.apache.hadoop.fs.FileSystem#isDirectory()

public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}
 

private int createTreeMergeInputDirList(Job job, Path outputReduceDir, FileSystem fs, Path fullInputList)
        throws FileNotFoundException, IOException {

    FileStatus[] dirs = listSortedOutputShardDirs(job, outputReduceDir, fs);
    int numFiles = 0;
    FSDataOutputStream out = fs.create(fullInputList);
    try {
        Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
        for (FileStatus stat : dirs) {
            LOG.debug("Adding path {}", stat.getPath());
            Path dir = new Path(stat.getPath(), "data/index");
            if (!fs.isDirectory(dir)) {
                throw new IllegalStateException("Not a directory: " + dir);
            }
            writer.write(dir.toString() + "\n");
            numFiles++;
        }
        writer.close();
    } finally {
        out.close();
    }
    return numFiles;
}
 

protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, 
		ValueType[] schema, String[] names, long rlen, long clen)
	throws IOException
{
	if( fs.isDirectory(path) ) {
		FileInputFormat.addInputPath(job, path);
		TextInputFormat informat = new TextInputFormat();
		informat.configure(job);
		InputSplit[] splits = informat.getSplits(job, 1);
		for(InputSplit split: splits)
			readTextCellFrameFromInputSplit(split, informat, job, dest);
	}
	else {
		readRawTextCellFrameFromHDFS(path, job, fs, dest, schema, names, rlen, clen);
	}
}
 

public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}
 

private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescription) {
  try {
    if(fs.exists(dir)) {
      if(! fs.isDirectory(dir) ) {
        LOG.error(dirDescription + " directory is a file, not a dir. " + dir);
        throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir);
      }
    } else if(! fs.mkdirs(dir) ) {
      LOG.error("Unable to create " + dirDescription + " directory " + dir);
      throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir);
    }
  } catch (IOException e) {
    LOG.error("Unable to create " + dirDescription + " directory " + dir, e);
    throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e);
  }
}
 

@CliCommand(value = "cd", help = "Changes current dir")
public String cd(@CliOption(key = {""}, help = "cd [<path>]") String newDir) {
    if (StringUtils.isEmpty(newDir)) {
        newDir = getHomeDir();
    }

    final Path path = (newDir.startsWith("/")) ? new Path(newDir) : new Path(getCurrentDir(), newDir);
    try {
        final FileSystem fs = getFileSystem();
        if (fs.exists(path) && fs.isDirectory(path)) {
            currentDir = path.toUri().getPath();
        } else {
            return "-shell: cd: " + newDir + " No such file or directory";
        }
    } catch (Exception e) {
        return "Change directory failed! " + e.getMessage();
    }
    return "";
}
 

private void verifiedPartition(Table table, Partition partition)
{
    Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());

    if (table.getTableType().equals(MANAGED_TABLE.name())) {
        if (!partitionMetadataDirectory.equals(new Path(partition.getStorage().getLocation()))) {
            throw new PrestoException(HIVE_METASTORE_ERROR, "Partition directory must be " + partitionMetadataDirectory);
        }
    }
    else if (table.getTableType().equals(EXTERNAL_TABLE.name())) {
        try {
            Path externalLocation = new Path(partition.getStorage().getLocation());
            FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation);
            if (!externalFileSystem.isDirectory(externalLocation)) {
                throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location does not exist");
            }
            if (isChildDirectory(catalogDirectory, externalLocation)) {
                throw new PrestoException(HIVE_METASTORE_ERROR, "External partition location cannot be inside the system metadata directory");
            }
        }
        catch (IOException e) {
            throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external partition location", e);
        }
    }
    else {
        throw new PrestoException(NOT_SUPPORTED, "Partitions cannot be added to " + table.getTableType());
    }
}
 

public static boolean hasContent(FileSystem fs, Path path)
    throws IOException {
  if (!fs.isDirectory(path)) {
    return true;
  }
  boolean content = false;
  for (FileStatus fileStatus : fs.listStatus(path)) {
    content = content || hasContent(fs, fileStatus.getPath());
    if (content) {
      break;
    }
  }
  return content;
}
 

/**
 * Renames a src {@link Path} on fs {@link FileSystem} to a dst {@link Path}. If fs is a {@link LocalFileSystem} and
 * src is a directory then {@link File#renameTo} is called directly to avoid a directory rename race condition where
 * {@link org.apache.hadoop.fs.RawLocalFileSystem#rename} copies the conflicting src directory into dst resulting in
 * an extra nested level, such as /root/a/b/c/e/e where e is repeated.
 *
 * @param fs the {@link FileSystem} where the src {@link Path} exists
 * @param src the source {@link Path} which will be renamed
 * @param dst the {@link Path} to rename to
 * @return true if rename succeeded, false if rename failed.
 * @throws IOException if rename failed for reasons other than target exists.
 */
public static boolean renamePathHandleLocalFSRace(FileSystem fs, Path src, Path dst) throws IOException {
  if (DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem && fs.isDirectory(src)) {
    LocalFileSystem localFs = (LocalFileSystem) DecoratorUtils.resolveUnderlyingObject(fs);
    File srcFile = localFs.pathToFile(src);
    File dstFile = localFs.pathToFile(dst);

    return srcFile.renameTo(dstFile);
  }
  else {
    return fs.rename(src, dst);
  }
}
 

protected String generateAndLogErrorListing(Path src, Path dst) throws
                                                                IOException {
  FileSystem fs = getFileSystem();
  getLog().error(
    "src dir " + ContractTestUtils.ls(fs, src.getParent()));
  String destDirLS = ContractTestUtils.ls(fs, dst.getParent());
  if (fs.isDirectory(dst)) {
    //include the dir into the listing
    destDirLS = destDirLS + "\n" + ContractTestUtils.ls(fs, dst);
  }
  return destDirLS;
}
 

/**
 * Get the schema of AVRO files stored in a directory
 */
public static Schema getAvroSchema(Path path, Configuration conf)
    throws IOException {
  FileSystem fs = path.getFileSystem(conf);
  Path fileToTest;
  if (fs.isDirectory(path)) {
    FileStatus[] fileStatuses = fs.listStatus(path, new PathFilter() {
      @Override
      public boolean accept(Path p) {
        String name = p.getName();
        return !name.startsWith("_") && !name.startsWith(".");
      }
    });
    if (fileStatuses.length == 0) {
      return null;
    }
    fileToTest = fileStatuses[0].getPath();
  } else {
    fileToTest = path;
  }

  SeekableInput input = new FsInput(fileToTest, conf);
  DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
  FileReader<GenericRecord> fileReader = DataFileReader.openReader(input, reader);

  Schema result = fileReader.getSchema();
  fileReader.close();
  return result;
}
 

/**
 * Tests that a file on the HDFS cluster contains the given avro.
 *
 * @param path the name of the file on the HDFS cluster
 * @param expected the expected avro record in the file .
 */
public static void assertReadAvroFile(FileSystem fs, String path, Set<IndexedRecord> expected, boolean part) throws IOException {
    Path p = new Path(path);
    if (fs.isFile(p)) {
        try (DataFileStream<GenericRecord> reader = new DataFileStream<GenericRecord>(
                new BufferedInputStream(fs.open(new Path(path))), new GenericDatumReader<GenericRecord>())) {
            IndexedRecord record = null;
            while (reader.hasNext()){
                record = reader.iterator().next();
                IndexedRecord eqRecord = null;
                for (IndexedRecord indexedRecord : expected) {
                    if(indexedRecord.equals(record)){
                        eqRecord = indexedRecord;
                        break;
                    }
                }
                expected.remove(eqRecord);
            }
        }
        // Check before asserting for the message.
        if (!part && expected.size() != 0)
            assertThat("Not all avro records found: " + expected.iterator().next(), expected, hasSize(0));
    } else if (fs.isDirectory(p)) {
        for (FileStatus fstatus : FileSystemUtil.listSubFiles(fs, p)) {
            assertReadAvroFile(fs, fstatus.getPath().toString(), expected, true);
        }
        // Check before asserting for the message.
        if (expected.size() != 0)
            assertThat("Not all avro records found: " + expected.iterator().next(), expected, hasSize(0));
    } else {
        fail("No such path: " + path);
    }
}
 

/**
 * Get avro schema of input path. There are three cases:
 * 1. if path is a file, then return its avro schema;
 * 2. if path is a first-level directory (no sub-directories), then
 * return the avro schema of one underlying file;
 * 3. if path contains sub-directories, then recursively check
 * whether all of them share the same schema and return it
 * if so or throw an exception if not.
 *
 * @param path input path
 * @param fs file system
 * @return avro schema of data
 * @throws IOException if underlying sub-directories do not share the same schema; or if input path is empty or does not exist
 */
@SuppressWarnings("deprecation")
protected Schema getAvroSchema(Path path, FileSystem fs) throws IOException {
    if (!fs.exists(path) || !AvroStorageUtils.PATH_FILTER.accept(path))
        return null;

    /* if path is first level directory or is a file */
    if (!fs.isDirectory(path)) {
        return getSchema(path, fs);
    }

    FileStatus[] ss = fs.listStatus(path, AvroStorageUtils.PATH_FILTER);
    Schema schema = null;
    if (ss.length > 0) {
        if (AvroStorageUtils.noDir(ss))
            return getSchema(path, fs);

        /*otherwise, check whether schemas of underlying directories are the same */
        for (FileStatus s : ss) {
            Schema newSchema = getAvroSchema(s.getPath(), fs);
            if (schema == null) {
                schema = newSchema;
                if(!checkSchema) {
                    System.out.println("Do not check schema; use schema of " + s.getPath());
                    return schema;
                }
            } else if (newSchema != null && !schema.equals(newSchema)) {
                throw new IOException( "Input path is " + path + ". Sub-direcotry " + s.getPath()
                                     + " contains different schema " + newSchema + " than " + schema);
            }
        }
    }

    if (schema == null)
        System.err.println("Cannot get avro schema! Input path " + path + " might be empty.");

    return schema;
}
 

@SuppressWarnings("unchecked")
private static MatrixBlock readLIBSVMMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, 
		long rlen, long clen, int blen)
	throws IOException, DMLRuntimeException
{
	//prepare file paths in alphanumeric order
	ArrayList<Path> files=new ArrayList<>();
	if(fs.isDirectory(path)) {
		for(FileStatus stat: fs.listStatus(path, IOUtilFunctions.hiddenFileFilter))
			files.add(stat.getPath());
		Collections.sort(files);
	}
	else
		files.add(path);
	
	//determine matrix size via additional pass if required
	if ( dest == null ) {
		dest = computeLIBSVMSize(files, clen, job, fs);
		clen = dest.getNumColumns();
	}
	
	//actual read of individual files
	long lnnz = 0;
	MutableInt row = new MutableInt(0);
	for(int fileNo=0; fileNo<files.size(); fileNo++) {
		lnnz += readLIBSVMMatrixFromInputStream(fs.open(files.get(fileNo)),
			path.toString(), dest, row, rlen, clen, blen);
	}
	
	//post processing
	dest.setNonZeros( lnnz );
	
	return dest;
}
 

@SuppressWarnings("unchecked")
private static MatrixBlock readCSVMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, 
		long rlen, long clen, int blen, boolean hasHeader, String delim, boolean fill, double fillValue )
	throws IOException, DMLRuntimeException
{
	//prepare file paths in alphanumeric order
	ArrayList<Path> files=new ArrayList<>();
	if(fs.isDirectory(path)) {
		for(FileStatus stat: fs.listStatus(path, IOUtilFunctions.hiddenFileFilter))
			files.add(stat.getPath());
		Collections.sort(files);
	}
	else
		files.add(path);
	
	//determine matrix size via additional pass if required
	if ( dest == null ) {
		dest = computeCSVSize(files, job, fs, hasHeader, delim, fill, fillValue);
		clen = dest.getNumColumns();
	}
	
	//actual read of individual files
	long lnnz = 0;
	MutableInt row = new MutableInt(0);
	for(int fileNo=0; fileNo<files.size(); fileNo++) {
		lnnz += readCSVMatrixFromInputStream(fs.open(files.get(fileNo)), path.toString(), dest, 
			row, rlen, clen, blen, hasHeader, delim, fill, fillValue, fileNo==0);
	}
	
	//post processing
	dest.setNonZeros( lnnz );
	
	return dest;
}
 

@SuppressWarnings("unchecked")
private static MatrixBlock readLIBSVMMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, 
		long rlen, long clen, int blen)
	throws IOException, DMLRuntimeException
{
	//prepare file paths in alphanumeric order
	ArrayList<Path> files=new ArrayList<>();
	if(fs.isDirectory(path)) {
		for(FileStatus stat: fs.listStatus(path, IOUtilFunctions.hiddenFileFilter))
			files.add(stat.getPath());
		Collections.sort(files);
	}
	else
		files.add(path);
	
	//determine matrix size via additional pass if required
	if ( dest == null ) {
		dest = computeLIBSVMSize(files, clen, job, fs);
		clen = dest.getNumColumns();
	}
	
	//actual read of individual files
	long lnnz = 0;
	MutableInt row = new MutableInt(0);
	for(int fileNo=0; fileNo<files.size(); fileNo++) {
		lnnz += readLIBSVMMatrixFromInputStream(fs.open(files.get(fileNo)),
			path.toString(), dest, row, rlen, clen, blen);
	}
	
	//post processing
	dest.setNonZeros( lnnz );
	
	return dest;
}
 

@SuppressWarnings("unchecked")
private static MatrixBlock readCSVMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, 
		long rlen, long clen, int blen, boolean hasHeader, String delim, boolean fill, double fillValue )
	throws IOException, DMLRuntimeException
{
	//prepare file paths in alphanumeric order
	ArrayList<Path> files=new ArrayList<>();
	if(fs.isDirectory(path)) {
		for(FileStatus stat: fs.listStatus(path, IOUtilFunctions.hiddenFileFilter))
			files.add(stat.getPath());
		Collections.sort(files);
	}
	else
		files.add(path);
	
	//determine matrix size via additional pass if required
	if ( dest == null ) {
		dest = computeCSVSize(files, job, fs, hasHeader, delim, fill, fillValue);
		clen = dest.getNumColumns();
	}
	
	//actual read of individual files
	long lnnz = 0;
	MutableInt row = new MutableInt(0);
	for(int fileNo=0; fileNo<files.size(); fileNo++) {
		lnnz += readCSVMatrixFromInputStream(fs.open(files.get(fileNo)), path.toString(), dest, 
			row, rlen, clen, blen, hasHeader, delim, fill, fillValue, fileNo==0);
	}
	
	//post processing
	dest.setNonZeros( lnnz );
	
	return dest;
}
 

@Override
public boolean isDirectory() {
	try {
		FileSystem fs = getFileSystem();
		return fs.isDirectory(new Path(path));
	} catch (IOException e) {
		LOG.error(String.format("An exception occurred while checking if the path '%s' is a directory.", path), e);
		return false;
	}
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager mgr = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment optimizeSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubeSegment oldSegment = optimizeSegment.getCubeInstance().getOriginalSegmentToOptimize(optimizeSegment);
    Preconditions.checkNotNull(oldSegment,
            "cannot find the original segment to be optimized by " + optimizeSegment);

    KylinConfig kylinConf = cube.getConfig();
    Configuration conf = HadoopUtil.getCurrentConfiguration();
    ResourceStore rs = ResourceStore.getStore(kylinConf);
    int averageSamplingPercentage = 0;

    try {
        //1. Add statistics from optimized segment
        Path statisticsDirPath = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()));
        FileSystem hdfs = FileSystem.get(conf);
        if (!hdfs.exists(statisticsDirPath)) {
            throw new IOException("StatisticsFilePath " + statisticsDirPath + " does not exists");
        }

        if (!hdfs.isDirectory(statisticsDirPath)) {
            throw new IOException("StatisticsFilePath " + statisticsDirPath + " is not a directory");
        }

        Path[] statisticsFiles = HadoopUtil.getFilteredPath(hdfs, statisticsDirPath,
                BatchConstants.CFG_OUTPUT_STATISTICS);
        if (statisticsFiles == null) {
            throw new IOException("fail to find the statistics file in base dir: " + statisticsDirPath);
        }

        for (Path item : statisticsFiles) {
            CubeStatsReader optimizeSegmentStatsReader = new CubeStatsReader(optimizeSegment, null,
                    optimizeSegment.getConfig(), item);
            averageSamplingPercentage += optimizeSegmentStatsReader.getSamplingPercentage();
            addFromCubeStatsReader(optimizeSegmentStatsReader);
        }

        //2. Add statistics from old segment
        CubeStatsReader oldSegmentStatsReader = new CubeStatsReader(oldSegment, null, oldSegment.getConfig());
        averageSamplingPercentage += oldSegmentStatsReader.getSamplingPercentage();
        addFromCubeStatsReader(oldSegmentStatsReader);

        logger.info("Cuboid set with stats info: " + cuboidHLLMap.keySet().toString());
        //3. Store merged statistics for recommend cuboids
        averageSamplingPercentage = averageSamplingPercentage / 2;
        Set<Long> cuboidsRecommend = cube.getCuboidsRecommend();

        Map<Long, HLLCounter> resultCuboidHLLMap = Maps.newHashMapWithExpectedSize(cuboidsRecommend.size());
        for (Long cuboid : cuboidsRecommend) {
            HLLCounter hll = cuboidHLLMap.get(cuboid);
            if (hll == null) {
                logger.warn("Cannot get the row count stats for cuboid " + cuboid);
            } else {
                resultCuboidHLLMap.put(cuboid, hll);
            }
        }

        String resultDir = CubingExecutableUtil.getMergedStatisticsPath(this.getParams());
        CubeStatsWriter.writeCuboidStatistics(conf, new Path(resultDir), resultCuboidHLLMap,
                averageSamplingPercentage, oldSegmentStatsReader.getSourceRowCount());

        try (FSDataInputStream mergedStats = hdfs
                .open(new Path(resultDir, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME))) {
            // put the statistics to metadata store
            String statisticsFileName = optimizeSegment.getStatisticsResourcePath();
            rs.putResource(statisticsFileName, mergedStats, System.currentTimeMillis());
        }

        //By default, the cube optimization will use in-memory cubing
        CubingJob cubingJob = (CubingJob) getManager()
                .getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
        StatisticsDecisionUtil.decideCubingAlgorithm(cubingJob, optimizeSegment);

        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to merge cuboid statistics", e);
        return ExecuteResult.createError(e);
    }

}
 

protected boolean isDirectory(FileSystem fs, @CheckForNull Boolean isDir, Path p) throws IOException {
  return isDir != null ? isDir : fs.isDirectory(p);
}