org.apache.commons.vfs2.impl.DefaultFileSystemManager Java Examples

protected void addProviders(DefaultFileSystemManager manager) throws FileSystemException {
    manager.addProvider("tmp", new TemporaryFileProvider());
    manager.addProvider("res", new ResourceFileProvider());
    manager.addProvider("ram", new RamFileProvider());
    manager.addProvider("zip", new ZipFileProvider());
    manager.addProvider("jar", new JarFileProvider());
    manager.addProvider("tar", new TarFileProvider());
    manager.addProvider("tgz", new TgzFileProvider());
    manager.addProvider("tbz2", new Tbz2FileProvider());
    manager.addProvider("gz", new GzipFileProvider());
    manager.addProvider("bz2", new Bzip2FileProvider());
    manager.addProvider("http", new HttpFileProvider());
    manager.addProvider("https", new HttpsFileProvider());
    manager.addProvider("ftp", new FtpFileProvider());
    manager.addProvider("ftps", new FtpsFileProvider());
}
 

@Override public FileSystemManager get() {
    try {
        final DefaultFileSystemManager manager = new DefaultFileSystemManager();

        manager.setFilesCache(new DefaultFilesCache());
        manager.setCacheStrategy(CacheStrategy.ON_RESOLVE);

        final String baseTmpDir = System.getProperty("java.io.tmpdir");
        final File tempDir = new File(baseTmpDir, "vfs_cache" + new Random().nextLong()).getAbsoluteFile();
        final DefaultFileReplicator replicator = new DefaultFileReplicator(tempDir);
        manager.setTemporaryFileStore(replicator);
        manager.setReplicator(replicator);

        addDefaultProvider(manager);
        addProviders(manager);
        setBaseFile(manager);

        manager.init();

        return manager;
    } catch(FileSystemException e) {
        throw new RuntimeException("Cannot initialize resource service: " + e.getMessage(), e);
    }
}
 

@Before
public void setup() throws Exception {
  connectionManager = ConnectionManager.getInstance();
  connectionManager.setMetastoreSupplier( () -> memoryMetaStore );
  addOne();

  DefaultFileSystemManager fsm = (DefaultFileSystemManager) KettleVFS.getInstance().getFileSystemManager();
  if ( !fsm.hasProvider( ConnectionFileProvider.SCHEME ) ) {
    fsm.addProvider( ConnectionFileProvider.SCHEME, new ConnectionFileProvider() );
  }
  if ( !fsm.hasProvider( TestFileProvider.SCHEME ) ) {
    fsm.addProvider( TestFileProvider.SCHEME, new TestFileProvider() );
  }
  if ( !fsm.hasProvider( TestFileWithDomainProvider.SCHEME ) ) {
    fsm.addProvider( TestFileWithDomainProvider.SCHEME, new TestFileWithDomainProvider() );
  }
}
 

/**
 * Test the on_call strategy
 */
public void testOnCallCache() throws Exception {
    final FileObject scratchFolder = getWriteFolder();
    if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class)
            || scratchFolder.getFileSystem() instanceof VirtualFileSystem) {
        // cant check ram filesystem as every manager holds its own ram filesystem data
        return;
    }

    scratchFolder.delete(Selectors.EXCLUDE_SELF);

    final DefaultFileSystemManager fs = createManager();
    fs.setCacheStrategy(CacheStrategy.ON_CALL);
    fs.init();
    final FileObject foBase2 = getBaseTestFolder(fs);

    final FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath());

    FileObject[] fos = cachedFolder.getChildren();
    assertContainsNot(fos, "file1.txt");

    scratchFolder.resolveFile("file1.txt").createFile();

    fos = cachedFolder.getChildren();
    assertContains(fos, "file1.txt");
}
 

public void testWindowsRoots() throws Exception {
    // valid URI forms of the filesystem root
    final String[] tests = new String[] { "file:///C:/", "file://C:/", "file:/C:/", "file:C:/" };

    for (final String name : tests) {
        final DefaultFileSystemManager manager = getManager();
        Assert.assertNotNull("Unexpected null manager for test " + this, manager);
        final FileName fn = manager.resolveFile(name).getName();

        // the following tests work for Windows file names only
        assertSame(WindowsFileName.class, fn.getClass());

        // all should result in the same FileName
        assertEquals("file:///C:/", fn.toString());
        assertEquals("/", fn.getPath());
        assertEquals("/", fn.getPathDecoded());
        assertEquals("file:///C:/", fn.getRootURI());
        assertEquals("file:///C:/", fn.getFriendlyURI());

        assertEquals("file:///C:/", fn.getRoot().toString());

        assertEquals("", fn.getExtension());
        assertEquals("", fn.getBaseName());
    }
}
 

@Override
public void testBasicCacheOps() throws Exception {
    final DefaultFileSystemManager manager = getManager();
    Assert.assertNotNull("This test should not have a null DefaultFileSystemManager", manager);
    // the basic test looks different for a null cache:
    final FilesCache cache = manager.getFilesCache();
    final FileObject fo = getWriteFolder().resolveFile("dir1");
    final FileName fn = fo.getName();
    final FileSystem fs = fo.getFileSystem();

    cache.clear(fs);
    assertNull(cache.getFile(fs, fn));

    cache.putFile(fo);
    assertNull(null, cache.getFile(fs, fn));

    assertFalse(cache.putFileIfAbsent(fo)); // hmmm?
    assertNull(null, cache.getFile(fs, fn));

    cache.removeFile(fs, fn);
    assertNull(cache.getFile(fs, fn));
}
 

/**
 * Check if filename starts with one of the known protocols like file: zip: ram: smb: jar: etc.
 * If yes, return true otherwise return false
 *
 * @param vfsFileName
 * @return boolean
 */
public static boolean startsWithScheme( String vfsFileName ) {
  lock.readLock().lock();
  try {

    DefaultFileSystemManager fsManager = getFileSystemManager();

    boolean found = false;
    String[] schemes = fsManager.getSchemes();
    for ( int i = 0; i < schemes.length; i++ ) {
      if ( vfsFileName.startsWith( schemes[ i ] + ":" ) ) {
        found = true;
        break;
      }
    }

    return found;
  } finally {
    lock.readLock().unlock();
  }
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    // manager.addProvider("tgz", new TgzFileProvider());
    manager.addProvider("tgz", new TarFileProvider());
    manager.addProvider("tar", new TarFileProvider());
}
 

/**
 * JUnit Fixture: Prepare a simple FSM.
 *
 * @throws FileSystemException for runtime problems
 */
@Before
public void setUp() throws FileSystemException {
    manager = new DefaultFileSystemManager();
    final FileProvider fp = new DefaultLocalFileProvider();
    manager.addProvider("file", fp);
    manager.init();
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("tgz", new TarFileProvider());
    manager.addExtensionMap("tgz", "tgz");
    manager.addProvider("tar", new TarFileProvider());
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    if (!manager.hasProvider("http5")) {
        manager.addProvider("http5", new Http5FileProvider());
    }
}
 

/**
 * Prepares the file system manager.
 *
 * Imports test data from the disk.
 *
 * @throws Exception
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    try {
        manager.addProvider("ram", new RamFileProvider());
        manager.addProvider("file", new DefaultLocalFileProvider());
    } catch (final Exception e) {
        log.error(e);
        throw e;
    }
}
 

@Before
public void setUp() throws Exception {
    manager = new DefaultFileSystemManager();
    manager.addProvider("ram", new RamFileProvider());
    manager.init();

    // File Systems Options
    RamFileSystemConfigBuilder.getInstance().setMaxSize(zeroSizedFso, 0L);
    RamFileSystemConfigBuilder.getInstance().setMaxSize(smallSizedFso, 10L);
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("zip", new ZipFileProvider());
    manager.addExtensionMap("zip", "zip");
    manager.addMimeTypeMap("application/zip", "zip");
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("zip", new ZipFileProvider());
    manager.addExtensionMap("zip", "zip");
    manager.addMimeTypeMap("application/zip", "zip");
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("zip", new ZipFileProvider());
    manager.addExtensionMap("zip", "zip");
    manager.addMimeTypeMap("application/zip", "zip");
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("zip", new ZipFileProvider());
    manager.addExtensionMap("zip", "zip");
    manager.addMimeTypeMap("application/zip", "zip");
}
 

public void testClass() {
    @SuppressWarnings("resource")
    final DefaultFileSystemManager manager = getManager();
    Assert.assertNotNull("manager", manager);
    final FilesCache filesCache = manager.getFilesCache();
    assertTrue(Objects.toString(filesCache), filesCache instanceof LRUFilesCache);
}
 

public static Test suite() throws Exception {
    return new ProviderTestSuite(new Webdav4ProviderTestCase()) {
        @Override
        protected void setUp() throws Exception {
            if (getSystemTestUriOverride() == null) {
                setUpClass();
            }
            try {
                // Since webdav4 is not registered in the standard file system configuration yet,
                // it must be registered manually here. Otherwise, HostFileNameParser#extractToPath() fails.
                final DefaultFileSystemManager manager = (DefaultFileSystemManager) VFS.getManager();
                if (!manager.hasProvider("webdav4")) {
                    manager.addProvider("webdav4", new Webdav4FileProvider());
                }
                super.setUp();
            } catch (final Exception e) {
                e.printStackTrace();
            }
        }

        @Override
        protected void tearDown() throws Exception {
            tearDownClass();
            super.tearDown();
        }
    };
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("tar", new TarFileProvider());
    manager.addExtensionMap("tar", "tar");
    manager.addMimeTypeMap("application/x-tar", "tar");
}
 

@Before
public void setUp() throws Exception {
    manager = new DefaultFileSystemManager();

    manager.setFilesCache(new SoftRefFilesCache());
    manager.setCacheStrategy(CacheStrategy.ON_RESOLVE);

    manager.addProvider("file", new DefaultLocalFileProvider());
    manager.addProvider("tgz", new TarFileProvider());
    manager.addProvider("tar", new TarFileProvider());

    new File(baseDir).mkdir(); // if test is run standalone
    createLargeFile(largeFilePath, largeFileName);
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("tbz2", new TarFileProvider());
    manager.addExtensionMap("tbz2", "tbz2");
    manager.addProvider("tar", new TarFileProvider());
}
 

/**
 * creates a new uninitialized file system manager
 *
 * @throws Exception
 */
protected DefaultFileSystemManager createManager() throws Exception {
    final DefaultFileSystemManager fs = getProviderConfig().getDefaultFileSystemManager();
    fs.setFilesCache(getProviderConfig().getFilesCache());
    getProviderConfig().prepare(fs);
    if (!fs.hasProvider("file")) {
        fs.addProvider("file", new DefaultLocalFileProvider());
    }
    return fs;
}
 

/**
 * Prepares the file system manager.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    if (!manager.hasProvider("http4")) {
        manager.addProvider("http4", new Http4FileProvider());
    }
}
 

/**
 * Prepares the file system manager. This implementation does nothing.
 */
@Override
public void prepare(final DefaultFileSystemManager manager) throws Exception {
    manager.addProvider("res", new ResourceFileProvider());
    manager.addProvider("file", new UrlFileProvider());
    manager.addProvider("jar", new JarFileProvider());
}
 

/**
 * Configures this test.
 */
public void setConfig(final DefaultFileSystemManager manager, final ProviderTestConfig providerConfig,
        final FileObject baseFolder, final FileObject readFolder, final FileObject writeFolder) {
    this.manager = manager;
    this.providerConfig = providerConfig;
    this.baseFolder = baseFolder;
    this.readFolder = readFolder;
    this.writeFolder = writeFolder;
    assertNotNull("setConfig manager", manager);
    assertNotNull("setConfig providerConfig", providerConfig);
    assertNotNull("setConfig baseFolder", baseFolder);
    assertNotNull("setConfig readFolder", readFolder);
    assertNotNull("setConfig writeFolder", writeFolder);
}
 

/**
 * Test the on_resolve strategy
 */
public void testOnResolveCache() throws Exception {
    final FileObject scratchFolder = getWriteFolder();
    if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class)
            || scratchFolder.getFileSystem() instanceof VirtualFileSystem) {
        // cant check ram filesystem as every manager holds its own ram filesystem data
        return;
    }

    scratchFolder.delete(Selectors.EXCLUDE_SELF);

    final DefaultFileSystemManager fs = createManager();
    fs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
    fs.init();
    final FileObject foBase2 = getBaseTestFolder(fs);

    FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath());

    FileObject[] fos = cachedFolder.getChildren();
    assertContainsNot(fos, "file1.txt");

    scratchFolder.resolveFile("file1.txt").createFile();

    fos = cachedFolder.getChildren();
    assertContainsNot(fos, "file1.txt");

    cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath());
    fos = cachedFolder.getChildren();
    assertContains(fos, "file1.txt");
}
 

@BeforeClass
public static void setUp() throws Exception {
    Logger.getRootLogger().setLevel(Level.ERROR);

    // Put the MiniDFSCluster directory in the target directory
    final File data = new File("target/test/hdfstestdata").getAbsoluteFile();
    data.mkdirs();
    System.setProperty("test.build.data", data.toString());
    FileUtils.cleanDirectory(data);

    // Setup HDFS
    conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, HDFS_URI);
    conf.set("hadoop.security.token.service.use_ip", "true");
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M blocksize

    setUmask(conf);

    cluster = new MiniDFSCluster(PORT, conf, 1, true, true, true, null, null, null, null);
    cluster.waitActive();

    // Set up the VFS
    manager = new DefaultFileSystemManager();
    manager.addProvider("hdfs", new HdfsFileProvider());
    manager.init();
    hdfs = cluster.getFileSystem();
}
 

/**
 * Test the manual cache strategy
 */
public void testManualCache() throws Exception {
    final FileObject scratchFolder = getWriteFolder();
    if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class)
            || scratchFolder.getFileSystem() instanceof VirtualFileSystem) {
        // cant check ram filesystem as every manager holds its own ram filesystem data
        return;
    }

    scratchFolder.delete(Selectors.EXCLUDE_SELF);

    final DefaultFileSystemManager fs = createManager();
    fs.setCacheStrategy(CacheStrategy.MANUAL);
    fs.init();
    final FileObject foBase2 = getBaseTestFolder(fs);

    final FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath());

    FileObject[] fos = cachedFolder.getChildren();
    assertContainsNot(fos, "file1.txt");

    scratchFolder.resolveFile("file1.txt").createFile();

    fos = cachedFolder.getChildren();
    assertContainsNot(fos, "file1.txt");

    cachedFolder.refresh();
    fos = cachedFolder.getChildren();
    assertContains(fos, "file1.txt");
}
 

public static DefaultFileSystemManager getFileSystemManager() {
  lock.readLock().lock();
  try {
    if ( fsm == null ) {
      try {
        fsm = createFileSystemManager();
      } catch ( Exception e ) {
        throw new RuntimeException( "Error initializing file system manager : ", e );
      }
    }
    return fsm;
  } finally {
    lock.readLock().unlock();
  }
}