Java Code Examples for org.apache.hadoop.fs.FileSystem#getUri()

public HadoopFileSystem getFileSystem( NamedCluster namedCluster, URI uri ) throws IOException {
  final Configuration configuration = hadoopShim.createConfiguration( namedCluster );
  FileSystem fileSystem = (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
  if ( fileSystem instanceof LocalFileSystem ) {
    LOGGER.error( "Got a local filesystem, was expecting an hdfs connection" );
    throw new IOException( "Got a local filesystem, was expecting an hdfs connection" );
  }

  final URI finalUri = fileSystem.getUri() != null ? fileSystem.getUri() : uri;
  HadoopFileSystem hadoopFileSystem = new HadoopFileSystemImpl( () -> {
    try {
      return finalUri != null
        ? (FileSystem) hadoopShim.getFileSystem( finalUri, configuration, (NamedCluster) namedCluster ).getDelegate()
        : (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
    } catch ( IOException | InterruptedException e ) {
      LOGGER.debug( "Error looking up/creating the file system ", e );
      return null;
    }
  } );
  ( (HadoopFileSystemImpl) hadoopFileSystem ).setNamedCluster( namedCluster );

  return hadoopFileSystem;
}
 

/**
 * Executes before each test.
 * @throws Exception If failed.
 */
private void before() throws Exception {
    initSecondary();

    if (passPrimaryConfiguration) {
        Configuration primaryFsCfg = configuration(primaryCfgScheme, primaryCfgAuthority, skipEmbed, skipLocShmem);

        primaryConfFullPath = writeConfiguration(primaryFsCfg, PRIMARY_CFG_PATH);
    }
    else
        primaryConfFullPath = null;

    CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();

    fac.setConfigPaths(primaryConfFullPath);
    fac.setUri(primaryFsUriStr);

    HadoopFileSystemFactoryDelegate facDelegate = HadoopDelegateUtils.fileSystemFactoryDelegate(
        getClass().getClassLoader(), fac);

    facDelegate.start();

    primaryFs = (FileSystem)facDelegate.get(null); //provider.createFileSystem(null);

    primaryFsUri = primaryFs.getUri();
}
 

/**
 * Commits the snapshot process by moving the working snapshot
 * to the finalized filepath
 *
 * @param snapshotDir The file path of the completed snapshots
 * @param workingDir  The file path of the in progress snapshots
 * @param fs The file system of the completed snapshots
 * @param workingDirFs The file system of the in progress snapshots
 * @param conf Configuration
 *
 * @throws SnapshotCreationException if the snapshot could not be moved
 * @throws IOException the filesystem could not be reached
 */
public static void completeSnapshot(Path snapshotDir, Path workingDir, FileSystem fs,
  FileSystem workingDirFs, final Configuration conf)
  throws SnapshotCreationException, IOException {
  LOG.debug("Sentinel is done, just moving the snapshot from " + workingDir + " to "
    + snapshotDir);
  // If the working and completed snapshot directory are on the same file system, attempt
  // to rename the working snapshot directory to the completed location. If that fails,
  // or the file systems differ, attempt to copy the directory over, throwing an exception
  // if this fails
  URI workingURI = workingDirFs.getUri();
  URI rootURI = fs.getUri();
  if ((!workingURI.getScheme().equals(rootURI.getScheme()) ||
    workingURI.getAuthority() == null ||
    !workingURI.getAuthority().equals(rootURI.getAuthority()) ||
    workingURI.getUserInfo() == null ||
    !workingURI.getUserInfo().equals(rootURI.getUserInfo()) ||
    !fs.rename(workingDir, snapshotDir)) && !FileUtil.copy(workingDirFs, workingDir, fs,
    snapshotDir, true, true, conf)) {
    throw new SnapshotCreationException("Failed to copy working directory(" + workingDir
      + ") to completed directory(" + snapshotDir + ").");
  }
}
 

@Override
public HadoopFileSystem create( NamedCluster namedCluster, URI uri ) throws IOException {
  final Configuration configuration = hadoopShim.createConfiguration( namedCluster );
  FileSystem fileSystem = (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
  if ( fileSystem instanceof LocalFileSystem ) {
    LOGGER.error( "Got a local filesystem, was expecting an hdfs connection" );
    throw new IOException( "Got a local filesystem, was expecting an hdfs connection" );
  }

  final URI finalUri = fileSystem.getUri() != null ? fileSystem.getUri() : uri;
  HadoopFileSystem hadoopFileSystem = new HadoopFileSystemImpl( () -> {
    try {
      return finalUri != null
        ? (FileSystem) hadoopShim.getFileSystem( finalUri, configuration, (NamedCluster) namedCluster ).getDelegate()
        : (FileSystem) hadoopShim.getFileSystem( configuration ).getDelegate();
    } catch ( IOException | InterruptedException e ) {
      LOGGER.debug( "Error looking up/creating the file system ", e );
      return null;
    }
  } );
  ( (HadoopFileSystemImpl) hadoopFileSystem ).setNamedCluster( namedCluster );

  return hadoopFileSystem;
}
 

private static URI resolveUri(URI uri, Configuration configuration, FileSystem fileSystem) throws IOException {
  if (uri != null) {
    return uri;
  }
  if (fileSystem != null) {
    return fileSystem.getUri();
  }
  if (configuration != null) {
    return FileSystem.getDefaultUri(configuration);
  }
  throw new IOException("FileSystem URI could not be determined from available inputs.");
}
 

public static Path getAppWorkDirPathFromConfig(Config config, FileSystem fs,
    String applicationName, String applicationId) {
  if (config.hasPath(GobblinClusterConfigurationKeys.CLUSTER_WORK_DIR)) {
    return new Path(new Path(fs.getUri()), PathUtils.combinePaths(config.getString(GobblinClusterConfigurationKeys.CLUSTER_WORK_DIR),
        getAppWorkDirPath(applicationName, applicationId)));
  }
  return new Path(fs.getHomeDirectory(), getAppWorkDirPath(applicationName, applicationId));
}
 

private static DistributedFileSystem getDFS(
      Configuration conf, FileSystem dfs) throws IOException {
  Configuration clientConf = new Configuration(conf);
  clientConf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
  clientConf.setBoolean("fs.hdfs.impl.disable.cache", true);
  URI dfsUri = dfs.getUri();
  FileSystem.closeAll();
  return (DistributedFileSystem) FileSystem.get(dfsUri, clientConf);
}
 

/**
 * Add the qualified path name of jars containing the given classes
 *
 * @param fs
 *            FileSystem object
 * @param jars
 *            the resolved path names to be added to this set
 * @param classes
 *            classes to find
 */
private static void addQualifiedJarsName(FileSystem fs, Set<String> jars, Class<?>... classes) {
    URI fsUri = fs.getUri();
    Path workingDir = fs.getWorkingDirectory();
    for (Class<?> clazz : classes) {
        String jarName = findContainingJar(clazz);
        if (jarName == null) {
            log.warn("Could not find jar for class " + clazz);
            continue;
        }
        jars.add(new Path(jarName).makeQualified(fsUri, workingDir).toString());
    }
}
 

static DistributedFileSystem getDFS(Configuration conf)
    throws IOException {
  FileSystem fs = FileSystem.get(conf);
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IllegalArgumentException("FileSystem " + fs.getUri() +
        " is not an HDFS file system");
  }
  return (DistributedFileSystem)fs;
}
 

private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
  URI srcUri = srcFs.getUri();
  URI dstUri = destFs.getUri();
  if (srcUri.getScheme() == null) {
    return false;
  }
  if (!srcUri.getScheme().equals(dstUri.getScheme())) {
    return false;
  }
  String srcHost = srcUri.getHost();
  String dstHost = dstUri.getHost();
  if ((srcHost != null) && (dstHost != null)) {
    try {
      srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
      dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
    } catch (UnknownHostException ue) {
      return false;
    }
    if (!srcHost.equals(dstHost)) {
      return false;
    }
  } else if (srcHost == null && dstHost != null) {
    return false;
  } else if (srcHost != null && dstHost == null) {
    return false;
  }
  // check for ports
  if (srcUri.getPort() != dstUri.getPort()) {
    return false;
  }
  return true;
}
 

public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
  URI srcUri = srcFs.getUri();
  URI dstUri = destFs.getUri();
  if (srcUri.getScheme() == null) {
    return false;
  }
  if (!srcUri.getScheme().equals(dstUri.getScheme())) {
    return false;
  }
  String srcHost = srcUri.getHost();
  String dstHost = dstUri.getHost();
  if ((srcHost != null) && (dstHost != null)) {
    try {
      srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
      dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
    } catch(UnknownHostException ue) {
      if (LOG.isDebugEnabled())
        LOG.debug("Could not compare file-systems. Unknown host: ", ue);
      return false;
    }
    if (!srcHost.equals(dstHost)) {
      return false;
    }
  }
  else if (srcHost == null && dstHost != null) {
    return false;
  }
  else if (srcHost != null) {
    return false;
  }

  //check for ports

  return srcUri.getPort() == dstUri.getPort();
}
 

/**
 * Determines if a file system supports XAttrs by running a getXAttrs request
 * on the file system root. This method is used before distcp job submission
 * to fail fast if the user requested preserving XAttrs, but the file system
 * cannot support XAttrs.
 * 
 * @param fs FileSystem to check
 * @throws XAttrsNotSupportedException if fs does not support XAttrs
 */
public static void checkFileSystemXAttrSupport(FileSystem fs)
    throws XAttrsNotSupportedException {
  try {
    fs.getXAttrs(new Path(Path.SEPARATOR));
  } catch (Exception e) {
    throw new XAttrsNotSupportedException("XAttrs not supported for file system: "
      + fs.getUri());
  }
}
 

private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
  URI srcUri = srcFs.getUri();
  URI dstUri = destFs.getUri();
  if (srcUri.getScheme() == null) {
    return false;
  }
  if (!srcUri.getScheme().equals(dstUri.getScheme())) {
    return false;
  }
  String srcHost = srcUri.getHost();
  String dstHost = dstUri.getHost();
  if ((srcHost != null) && (dstHost != null)) {
    try {
      srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
      dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
    } catch(UnknownHostException ue) {
      return false;
    }
    if (!srcHost.equals(dstHost)) {
      return false;
    }
  }
  else if (srcHost == null && dstHost != null) {
    return false;
  }
  else if (srcHost != null && dstHost == null) {
    return false;
  }
  //check for ports
  if (srcUri.getPort() != dstUri.getPort()) {
    return false;
  }
  return true;
}
 

protected DistributedFileSystem getDFS() throws IOException {
  FileSystem fs = getFS();
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
    " is not an HDFS file system");
  }
  return (DistributedFileSystem)fs;
}
 

@Test
public void testDriverAddSingleRecordToExistingRow() throws Exception {
  FileSystem fileSystem = miniCluster.getFileSystem();
  Path root = new Path(fileSystem.getUri() + "/");

  String tableName = "testDriverAddSingleRecordToExistingRow";
  Iface client = getClient();
  creatTable(tableName, new Path(root, "tables"), true);
  addRow(client, tableName, "row1", "record1", "value1");

  Driver driver = new Driver();
  driver.setConf(conf);

  String mrIncWorkingPathStr = new Path(root, "working").toString();
  generateData(mrIncWorkingPathStr);
  String outputPathStr = new Path(root, "output").toString();
  String blurZkConnection = miniCluster.getZkConnectionString();

  assertEquals(0, driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" }));

  client.loadData(tableName, outputPathStr);

  waitUntilAllImportsAreCompleted(client, tableName);

  TableStats tableStats = client.tableStats(tableName);
  assertEquals(1, tableStats.getRowCount());
  assertEquals(2, tableStats.getRecordCount());
}
 

/** Constructor */
public DFSAdminCommand(FileSystem fs) {
  super(fs.getConf());
  if (!(fs instanceof DistributedFileSystem)) {
    throw new IllegalArgumentException("FileSystem " + fs.getUri() + 
        " is not a distributed file system");
  }
  this.dfs = (DistributedFileSystem)fs;
}
 

/**
 * Test opening and reading from an InputStream through a hdfs:// URL.
 * <p>
 * First generate a file with some content through the FileSystem API, then
 * try to open and read the file through the URL stream API.
 * 
 * @throws IOException
 */
public void testDfsUrls() throws IOException {

  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
  FileSystem fs = cluster.getFileSystem();

  // Setup our own factory
  // setURLSteramHandlerFactor is can be set at most once in the JVM
  // the new URLStreamHandler is valid for all tests cases 
  // in TestStreamHandler
  FsUrlStreamHandlerFactory factory =
      new org.apache.hadoop.fs.FsUrlStreamHandlerFactory();
  java.net.URL.setURLStreamHandlerFactory(factory);

  Path filePath = new Path("/thefile");

  try {
    byte[] fileContent = new byte[1024];
    for (int i = 0; i < fileContent.length; ++i)
      fileContent[i] = (byte) i;

    // First create the file through the FileSystem API
    OutputStream os = fs.create(filePath);
    os.write(fileContent);
    os.close();

    // Second, open and read the file content through the URL API
    URI uri = fs.getUri();
    URL fileURL =
        new URL(uri.getScheme(), uri.getHost(), uri.getPort(), filePath
            .toString());

    InputStream is = fileURL.openStream();
    assertNotNull(is);

    byte[] bytes = new byte[4096];
    assertEquals(1024, is.read(bytes));
    is.close();

    for (int i = 0; i < fileContent.length; ++i)
      assertEquals(fileContent[i], bytes[i]);

    // Cleanup: delete the file
    fs.delete(filePath, false);

  } finally {
    fs.close();
    cluster.shutdown();
  }

}
 

private void assertFsSameUri(FileSystem sourceFs, FileSystem targetFs) {
  Path source = new Path(sourceFs.getUri());
  Path target = new Path(targetFs.getUri());
  assertEquals(source, target);
}
 

/**
 * Get DFS full partition path (e.g. hdfs://ip-address:8020:/<absolute path>)
 */
public static String getDFSFullPartitionPath(FileSystem fs, Path partitionPath) {
  return fs.getUri() + partitionPath.toUri().getRawPath();
}
 

public Collection<? extends CopyEntity> testGetCopyableFilesHelper(String sourceDir, String destinationDir,
    long sourceWatermark, boolean isFilterEnabled) throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  URI local = localFs.getUri();

  Properties properties = new Properties();
  properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");
  PathFilter pathFilter = DatasetUtils.instantiatePathFilter(properties);
  boolean applyFilterToDirectories = false;
  if (isFilterEnabled) {
    properties.setProperty(DatasetUtils.CONFIGURATION_KEY_PREFIX + "path.filter.class",
        "org.apache.gobblin.util.filters.HiddenFilter");
    properties.setProperty(CopyConfiguration.APPLY_FILTER_TO_DIRECTORIES, "true");

    pathFilter = DatasetUtils.instantiatePathFilter(properties);
    applyFilterToDirectories =
        Boolean.parseBoolean(properties.getProperty(CopyConfiguration.APPLY_FILTER_TO_DIRECTORIES, "false"));
  }

  CopyConfiguration copyConfiguration =
      CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties)
          .publishDir(new Path(destinationDir))
          .preserve(PreserveAttributes.fromMnemonicString("ugp"))
          .build();

  ReplicationMetaData mockMetaData = Mockito.mock(ReplicationMetaData.class);
  Mockito.when(mockMetaData.toString()).thenReturn("Mock Meta Data");

  ReplicationConfiguration mockRC = Mockito.mock(ReplicationConfiguration.class);
  Mockito.when(mockRC.getCopyMode()).thenReturn(ReplicationCopyMode.PULL);
  Mockito.when(mockRC.getMetaData()).thenReturn(mockMetaData);
  Mockito.when(mockRC.getVersionStrategyFromConfigStore()).thenReturn(Optional.of(DataFileVersionStrategy.DEFAULT_DATA_FILE_VERSION_STRATEGY));
  Mockito.when(mockRC.getEnforceFileSizeMatchFromConfigStore()).thenReturn(Optional.absent());
  HadoopFsEndPoint copyFrom = Mockito.mock(HadoopFsEndPoint.class);
  Mockito.when(copyFrom.getDatasetPath()).thenReturn(new Path(sourceDir));
  Mockito.when(copyFrom.getFsURI()).thenReturn(local);
  ComparableWatermark sw = new LongWatermark(sourceWatermark);
  Mockito.when(copyFrom.getWatermark()).thenReturn(Optional.of(sw));
  Mockito.when(copyFrom.getFiles())
      .thenReturn(
          FileListUtils.listFilesRecursively(localFs, new Path(sourceDir), pathFilter, applyFilterToDirectories));

  HadoopFsEndPoint copyTo = Mockito.mock(HadoopFsEndPoint.class);
  Mockito.when(copyTo.getDatasetPath()).thenReturn(new Path(destinationDir));
  Mockito.when(copyTo.getFsURI()).thenReturn(local);
  Optional<ComparableWatermark> tmp = Optional.absent();
  Mockito.when(copyTo.getWatermark()).thenReturn(tmp);
  Mockito.when(copyTo.getFiles())
      .thenReturn(FileListUtils.listFilesRecursively(localFs, new Path(destinationDir), pathFilter,
          applyFilterToDirectories));

  CopyRoute route = Mockito.mock(CopyRoute.class);
  Mockito.when(route.getCopyFrom()).thenReturn(copyFrom);
  Mockito.when(route.getCopyTo()).thenReturn(copyTo);

  ConfigBasedDataset dataset = new ConfigBasedDataset(mockRC, properties, route);
  Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(localFs, copyConfiguration);
  return copyableFiles;
}