com.indeed.util.io.Files Java Examples

public static <T> MMapGOV4Function<T> readFrom(final String path) throws IOException, NoSuchFieldException,
        IllegalAccessException, ClassNotFoundException {
    final GOV4Function<T> gov4Function;
    try (final InputStream file = new FileInputStream(Files.buildPath(path, "GOV4Function.bin"));
         final ObjectInput input = new ObjectInputStream(file)) {
        gov4Function = (GOV4Function<T>)(input.readObject());
    }
    final Field widthField = gov4Function.getClass().getDeclaredField("width");
    widthField.setAccessible(true);
    final int width = (Integer) widthField.get(gov4Function);
    final MMapBuffer buffer = new MMapBuffer(new File(Files.buildPath(path, "signatures.bin")),
                                             FileChannel.MapMode.READ_ONLY, ByteOrder.LITTLE_ENDIAN);
    final MMapLongBigList signaturesData = new MMapLongBigList(buffer.memory().longArray(0, buffer.memory().length() / 8), width);
    final Field dataField = gov4Function.getClass().getDeclaredField("data");
    dataField.setAccessible(true);
    dataField.set(gov4Function, signaturesData);
    return new MMapGOV4Function<>(gov4Function, buffer, width);
}
 

public static <T> void writeTo(final GOV4Function<T> gov4Function, final String path) throws NoSuchFieldException,
        IllegalAccessException, IOException {
    final Field dataField = gov4Function.getClass().getDeclaredField("data");
    dataField.setAccessible(true);
    final AbstractBitVector.LongBigListView signaturesData = (AbstractBitVector.LongBigListView) dataField.get(gov4Function);
    final Field bitVectorField = signaturesData.getClass().getDeclaredField("bitVector");
    bitVectorField.setAccessible(true);
    final BitVector bitVector = (BitVector)bitVectorField.get(signaturesData);
    try (final LittleEndianDataOutputStream outputStream = new LittleEndianDataOutputStream(
            new BufferedOutputStream(new FileOutputStream(Files.buildPath(path, "signatures.bin"))))) {
        for (final long value : bitVector.bits()) {
            outputStream.writeLong(value);
        }
    }
    dataField.set(gov4Function, null);
    try (final OutputStream outputStream = new FileOutputStream(Files.buildPath(path, "GOV4Function.bin"));
         final ObjectOutput objectOutput = new ObjectOutputStream(outputStream)) {
        objectOutput.writeObject(gov4Function);
    }
}
 

private boolean updateHosts() {
    try {
        final List<Host> newHosts = new ArrayList<Host>();
        for (String line : Files.readTextFile(hostsFile)) {
            if (line.startsWith("#")) continue;

            final String[] splitLine = line.split(":", 2);
            if (splitLine.length != 2) {
                log.error("invalid host: "+line);
                continue;
            }
            newHosts.add(new Host(splitLine[0], Integer.parseInt(splitLine[1])));
        }
        Collections.sort(newHosts);
        log.info("reloaded hosts file, new list of hosts: "+newHosts);
        if (!newHosts.equals(hosts)) {
            hosts = newHosts;
        }
        return true;
    } catch (Exception e) {
        log.error("error loading hosts file", e);
        return false;
    }
}
 

private static void checkDirectory(FileSystem fs, Path tempDir, File localTempDir, File localArchiveDir) throws IOException {
    Files.delete(localArchiveDir.getAbsolutePath());
    final SquallArchiveReader reader = new SquallArchiveReader(fs, tempDir);
    reader.copyToLocal("tmp/tempfile", localTempDir.getAbsolutePath());
    assertTrue(ByteStreams.equal(
            com.google.common.io.Files.newInputStreamSupplier(new File(localArchiveDir, "tempfile")),
            ByteStreams.newInputStreamSupplier(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
    ));

    Files.delete(localArchiveDir.getAbsolutePath());
    reader.copyAllToLocal(localTempDir.getAbsolutePath());
    assertTrue(ByteStreams.equal(
            com.google.common.io.Files.newInputStreamSupplier(new File(localArchiveDir, "tempfile")),
            ByteStreams.newInputStreamSupplier(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
    ));
}
 

private static void readAllTheFiles(FileSystem fs, Path tempDir, File localTempDir2, List<File> tempFiles) throws IOException {
    final SquallArchiveReader reader = new SquallArchiveReader(fs, tempDir);
    final List<FileMetadata> metadata = reader.readMetadata();
    assertEquals(metadata.size(), tempFiles.size());
    for (int i = 0; i < metadata.size(); ++i) {
        final FileMetadata file = metadata.get(i);
        final File tf = tempFiles.get(i);
        assertEquals(tf.getName(), file.getFilename());
        reader.copyToLocal(file, localTempDir2);
        assertTrue(com.google.common.io.Files.equal(tf, new File(localTempDir2, file.getFilename())));
    }

    for (final File f : localTempDir2.listFiles()) {
        if (!f.delete()) throw new IOException();
    }

    reader.copyToLocal("tempfile3", localTempDir2.getAbsolutePath());
    assertTrue(com.google.common.io.Files.equal(new File(localTempDir2, "tempfile3"), findTempFile3(tempFiles)));
}
 

@Test
public void testEmptyFields() throws IOException {
    final String dir = Files.getTempDirectory("flamdex-test", "foo");
    try {
        SimpleFlamdexWriter w = new SimpleFlamdexWriter(dir, 5L, true);
        w.getIntFieldWriter("if1").close();
        w.getStringFieldWriter("sf1").close();
        w.close();

        SimpleFlamdexReader r = SimpleFlamdexReader.open(dir);
        IntTermIterator it = r.getIntTermIterator("if1");
        assertFalse(it.next());
        StringTermIterator sit = r.getStringTermIterator("sf1");
        assertFalse(sit.next());
        it.close();
        sit.close();
        r.close();
    } finally {
        Files.delete(dir);
    }
}
 

@Override
protected void setUp() throws Exception {
    tempDir1 = Files.getTempDirectory("imhotep", "test");
    tempOptDir1 = Files.getTempDirectory("imhotep", "optimize.test");
    String datasetDir = new File(tempDir1, DATASET).getAbsolutePath();
    new File(datasetDir).mkdir();
    new File(datasetDir, SHARD0).mkdir();
    new File(datasetDir, SHARD1).mkdir();

    tempDir2 = Files.getTempDirectory("imhotep", "test");
    tempOptDir2 = Files.getTempDirectory("imhotep", "optimize.test");
    String datasetDir2 = new File(tempDir2, DATASET).getAbsolutePath();
    new File(datasetDir2).mkdir();
    new File(datasetDir2, SHARD1).mkdir();

    daemon1 = new ImhotepDaemonRunner(tempDir1, tempOptDir1, getFreePort());
    daemon2 = new ImhotepDaemonRunner(tempDir2, tempOptDir2, getFreePort());
}
 

@After
public void stopZK() {
    if (zk != null) {
        zk.shutdown();
    }
    Files.delete(tempDir);
}
 

@Test
public void testGetMetric() throws IOException, FlamdexOutOfMemoryException {
    final String dir = Files.getTempDirectory("flamdex-test", "foo");
    try {
        internalTestGetMetric(dir);
    } finally {
        Files.delete(dir);
    }
}
 

@Test
public void testIt() throws IOException {
    final String dir = Files.getTempDirectory("flamdex-test", "foo");
    try {
        writeAndRead(dir);
    } finally {
        Files.delete(dir);
    }
}
 

@Override
protected void tearDown() throws Exception {
    if (daemon1 != null) {
        daemon1.stop();
    }
    if (daemon2 != null) {
        daemon2.stop();
    }
    Files.delete(tempDir1);
    Files.delete(tempDir2);
}
 

protected static void buildStringBTrees(final String directory, final List<String> stringFields) throws IOException {
    for (final String stringField : stringFields) {
        final File btreeDir = new File(Files.buildPath(directory, "fld-" + stringField + ".strindex"));
        if (!btreeDir.exists()) {
            SimpleFlamdexWriter.writeStringBTree(directory, stringField, btreeDir);
        }
    }
}
 

protected static void buildIntBTrees(final String directory, final List<String> intFields) throws IOException {
    for (final String intField : intFields) {
        final File btreeDir = new File(Files.buildPath(directory, "fld-" + intField + ".intindex"));
        final File btreeDir64 = new File(Files.buildPath(directory, "fld-" + intField + ".intindex64"));
        if (!btreeDir.exists() && !btreeDir64.exists()) {
            SimpleFlamdexWriter.writeIntBTree(directory, intField, btreeDir64);
        }
    }
}
 

@Override
public StringTermIterator getStringTermIterator(String field) {
    final String termsFilename = Files.buildPath(directory, factory.getStringTermsFilename(field));
    final String docsFilename = Files.buildPath(directory, factory.getStringDocsFilename(field));
    try {
        return factory.createStringTermIterator(termsFilename, docsFilename);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public IntTermIterator getIntTermIterator(String field) {
    final String termsFilename = Files.buildPath(directory, factory.getIntTermsFilename(field));
    final String docsFilename = Files.buildPath(directory, factory.getIntDocsFilename(field));
    try {
        return factory.createIntTermIterator(termsFilename, docsFilename);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

private static void readHalfTheFiles(FileSystem fs, Path tempDir, File localTempDir2, List<File> tempFiles) throws IOException {
    final SquallArchiveReader reader = new SquallArchiveReader(fs, tempDir);
    final List<FileMetadata> metadata = reader.readMetadata();
    assertEquals(metadata.size(), tempFiles.size() / 2);
    for (int i = 0; i < metadata.size(); ++i) {
        final FileMetadata file = metadata.get(i);
        final File tf = tempFiles.get(i);
        assertEquals(tf.getName(), file.getFilename());
        reader.copyToLocal(file, localTempDir2);
        assertTrue(ByteStreams.equal(
                com.google.common.io.Files.newInputStreamSupplier(tf),
                com.google.common.io.Files.newInputStreamSupplier(new File(localTempDir2, file.getFilename()))
        ));
    }
}
 

private static void doTheTest(FileSystem fs, Path tempDir, File localTempDir, SquallArchiveCompressor compressor) throws IOException {
    final Random rand = new Random();
    final List<File> tempFiles = new ArrayList<File>();
    for (int i = 0; i < 10; ++i) {
        final File tempFile = new File(localTempDir, "tempfile" + i);
        tempFiles.add(tempFile);
        final int len = rand.nextInt(1024) + 1024;
        final OutputStream os = new FileOutputStream(tempFile);
        for (int j = 0; j < len; ++j) {
            os.write(rand.nextInt(256));
        }
        os.close();
    }

    Collections.shuffle(tempFiles);

    long bytesWritten = writeHalfTheFiles(fs, tempDir, tempFiles, compressor);
    if (compressor == NONE) {
        assertEquals(bytesWritten, getArchiveBytesWritten(fs, tempDir));
    }

    final File localTempDir2 = new File(getTempDir());
    try {
        readHalfTheFiles(fs, tempDir, localTempDir2, tempFiles);
    } finally {
        Files.delete(localTempDir2.getAbsolutePath());
    }

    bytesWritten += writeTheOtherHalf(fs, tempDir, tempFiles, compressor);
    if (compressor == NONE) {
        assertEquals(bytesWritten, getArchiveBytesWritten(fs, tempDir));
    }

    if (!localTempDir2.mkdirs()) throw new IOException();
    try {
        readAllTheFiles(fs, tempDir, localTempDir2, tempFiles);
    } finally {
        Files.delete(localTempDir2.getAbsolutePath());
    }
}
 

private GOV4Function<String> buildMphFunction() throws IOException {
    final File tempFolder = File.createTempFile("smartDictionarySerializer", ".tmp");
    if (!Files.delete(tempFolder.getAbsolutePath())) {
        throw new IOException("Can't delete tempFolder: " + tempFolder);
    }
    if (!tempFolder.mkdir()) {
        throw new IOException("Can't create tempFolder: " + tempFolder);
    }
    return new GOV4Function.Builder<String>()
            .keys(Arrays.asList(words))
            .tempDir(tempFolder)
            .transform(new TableWriter.SerializerTransformationStrategy<>(new SmartStringSerializer()))
            .build();
}
 

public static SimpleIntFieldWriter open(String outputDirectory, String field, long numDocs, boolean writeBTreesOnClose) throws FileNotFoundException {
    final OutputStream termsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getTermsFilename(field))), 65536);
    final OutputStream docsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getDocsFilename(field))), 65536);
    return new SimpleIntFieldWriter(outputDirectory, field, writeBTreesOnClose, termsOutput, docsOutput, numDocs);
}
 

@Test
    public void testCloseSessionDuringFTGS() throws ImhotepOutOfMemoryException, IOException, InterruptedException {
        String tempDir = Files.getTempDirectory("asdf", "");
        try {
            IndexWriter w = new IndexWriter(tempDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            Random rand = new Random();
            for (int i = 0; i < 1000000; ++i) {
                int numTerms = rand.nextInt(5) + 5;
                Document doc = new Document();
                for (int t = 0; t < numTerms; ++t) {
                    doc.add(new Field("sf1", Integer.toString(rand.nextInt(10000)), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                }
                w.addDocument(doc);
            }

            w.close();

            final AtomicBoolean closed = new AtomicBoolean(false);
            FlamdexReader r = new LuceneFlamdexReader(IndexReader.open(tempDir)) {
                @Override
                public void close() throws IOException {
                    super.close();
                    closed.set(true);
                }
            };
            final ExecutorService executor = Executors.newCachedThreadPool();
            try {
                ImhotepSession session =
                        new MTImhotepMultiSession(new ImhotepLocalSession[] { new ImhotepLocalSession(r) },
                                                  new MemoryReservationContext(
                                                                               new ImhotepMemoryPool(
                                                                                                     Long.MAX_VALUE)),
                                                  executor, null);
//                FTGSIterator iter = session.getFTGSIterator(new String[]{}, new String[]{"sf1"}); //TODO fix this
                session.close();
                assertTrue(closed.get());
            } finally {
                executor.shutdown();
            }
        } finally {
            Files.delete(tempDir);
        }
    }
 

public static SimpleStringFieldWriter open(String outputDirectory, String field, long numDocs, boolean writeBTreesOnClose) throws FileNotFoundException {
    final OutputStream termsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getTermsFilename(field))), 65536);
    final OutputStream docsOutput = new BufferedOutputStream(new FileOutputStream(Files.buildPath(outputDirectory, getDocsFilename(field))), 65536);
    return new SimpleStringFieldWriter(outputDirectory, field, writeBTreesOnClose, termsOutput, docsOutput, numDocs);
}
 

@Before
public void setUp() throws Exception {
    tempDir = Files.getTempDirectory("flamdex-test", "dir");
}
 

@After
public void tearDown() throws Exception {
    Files.delete(tempDir);
}
 

@Before
public void setUp() throws Exception {
    tempDir = Files.getTempDirectory("flamdex-test", "dir");
}
 

@After
public void tearDown() throws Exception {
    Files.delete(tempDir);
}
 

private static String getTempDir() {
    return com.google.common.io.Files.createTempDir().getAbsolutePath();
}
 

@Before
public void startZK() throws IOException {
    tempDir = Files.getTempDirectory("asdf", "");
    zk = new MiniZooKeeperCluster(tempDir);
}