org.apache.commons.io.LineIterator Java Examples

@Test
public void testResolvingActual() throws Exception {
    File f = Resources.asFile("data/irisSvmLight.txt");
    assertTrue(f.exists());

    //System.out.println(f.getAbsolutePath());
    int count = 0;
    try(Reader r = new BufferedReader(new FileReader(f))){
        LineIterator iter = IOUtils.lineIterator(r);
        while(iter.hasNext()){
            String line = iter.next();
            //System.out.println("LINE " + i + ": " + line);
            count++;
        }
    }

    assertEquals(12, count);        //Iris normally has 150 examples; this is subset with 12
}
 

public static int countClasses(File file) {
    int numClasses = 0;
    LineIterator iterator = null;
    try {
        iterator = FileUtils.lineIterator(file);

        while (iterator.hasNext()) {
            String line = iterator.nextLine();
            if (StringUtils.contains(line.trim(), "class ")) {
                numClasses++;
            }
        }
    } catch (IOException e) {
        LOG.error("Error determining class count for file " + file, e);
    } finally {
        LineIterator.closeQuietly(iterator);
    }

    return numClasses;
}
 

public static SentenceIterator getIterator(boolean isIntegration, File file, int linesForUnitTest) throws IOException {
    if(isIntegration){
        return new BasicLineIterator(file);
    } else {
        List<String> lines = new ArrayList<>();
        try(InputStream is = new BufferedInputStream(new FileInputStream(file))){
            LineIterator lineIter = IOUtils.lineIterator(is, StandardCharsets.UTF_8);
            try{
                for( int i=0; i<linesForUnitTest && lineIter.hasNext(); i++ ){
                    lines.add(lineIter.next());
                }
            } finally {
                lineIter.close();
            }
        }

        return new CollectionSentenceIterator(lines);
    }
}
 

public InMemoryArchiveIdentificationService addMappingsFrom(File file)
{
    try (FileInputStream inputStream = new FileInputStream(file))
    {
        LineIterator iterator = IOUtils.lineIterator(inputStream, "UTF-8");
        int lineNumber = 0;
        while (iterator.hasNext())
        {
            lineNumber++;
            String line = iterator.next();
            if (line.startsWith("#") || line.trim().isEmpty())
                continue;
            String[] parts = StringUtils.split(line, ' ');
            if (parts.length < 2)
                throw new IllegalArgumentException("Expected 'SHA1 GROUP_ID:ARTIFACT_ID:[PACKAGING:[COORDINATE:]]VERSION', but was: [" + line
                            + "] in [" + file + "] at line [" + lineNumber + "]");

            addMapping(parts[0], parts[1]);
        }
    }
    catch (IOException e)
    {
        throw new WindupException("Failed to load SHA1 to " + Coordinate.class.getSimpleName() + " definitions from [" + file + "]", e);
    }
    return this;
}
 

private Iterator<Doc> extractDocs(File file) {
    List<Doc> docs = new ArrayList<>();
    try {
        LineIterator lineIterator = FileUtils.lineIterator(file, "UTF-8");
        while (lineIterator.hasNext()) {
            String line = lineIterator.next();
            if (!line.isEmpty()) {
                docs.add(new Doc(JsonUtils.fromJsonString(Map.class, line)));
            }
        }

    } catch (Exception e) {
        throw new RuntimeException("failed to extract docs from file [" + file + "]", e);
    }

    return Iterables.cycle(docs).iterator();
}
 

public static Table<String, String, Long> loadTable(InputStream stream)
        throws IOException
{
    Table<String, String, Long> result = TreeBasedTable.create();

    LineIterator lineIterator = IOUtils.lineIterator(stream, "utf-8");
    while (lineIterator.hasNext()) {
        String line = lineIterator.next();

        System.out.println(line);

        String[] split = line.split("\t");
        String language = split[0];
        String license = split[1];
        Long documents = Long.valueOf(split[2]);
        Long tokens = Long.valueOf(split[3]);

        result.put(language, "docs " + license, documents);
        result.put(language, "tokens " + license, tokens);
    }

    return result;
}
 

public static LinkedHashMap<String, Integer> loadCorpusToRankedVocabulary(InputStream corpus)
        throws IOException
{
    LinkedHashMap<String, Integer> result = new LinkedHashMap<>();

    LineIterator lineIterator = IOUtils.lineIterator(corpus, "utf-8");
    int counter = 0;
    while (lineIterator.hasNext()) {
        String line = lineIterator.next();

        String word = line.split("\\s+")[0];

        result.put(word, counter);
        counter++;
    }

    return result;
}
 

public static List<String> firstNLines(File f, int n){
    List<String> lines = new ArrayList<>();
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        LineIterator lineIter = IOUtils.lineIterator(is, StandardCharsets.UTF_8);
        try{
            for( int i=0; i<n && lineIter.hasNext(); i++ ){
                lines.add(lineIter.next());
            }
        } finally {
            lineIter.close();
        }
        return lines;
    } catch (IOException e){
        throw new RuntimeException(e);
    }
}
 

/**
 * Import the GRISP general domains
 */
private void importDomains() throws IOException {
    domain2id = new HashMap<String, Integer>();
    id2domain = new HashMap<Integer, String>();

    LineIterator domainIterator = FileUtils.lineIterator(new File(grispDomains));
    int n = 0;
    while (domainIterator.hasNext()) {
        String line = domainIterator.next();
        final String domain = line.replace('\t', ' ').trim();
        domain2id.put(domain, new Integer(n));
        id2domain.put(new Integer(n), domain);
        n++;
    }
    LineIterator.closeQuietly(domainIterator);
}
 

private static Map<Integer, Integer> loadDictionary(String filename) {
    HashMap<Integer, Integer> result = new HashMap<>();

    InputStream stream = null;
    LineIterator iterator = null;

    try {
        stream = ChineseCharacterConverter.class.getResourceAsStream(filename);
        iterator = IOUtils.lineIterator(stream, "UTF-8");
        while (iterator.hasNext()) {
            String line = iterator.nextLine();
            String[] keyValues = line.split("\t", 2);
            Integer key = keyValues[0].codePointAt(0);
            Integer value = keyValues[1].codePointAt(0);
            result.put(key, value);
        }

        return result;
    } catch (IOException e) {
        throw new Error(e);
    } finally {
        IOUtils.closeQuietly(stream);
        if (iterator != null)
            iterator.close();
    }
}
 

private static Dictionary load(String filename) throws IOException {
    InputStream stream = null;

    try {
        HashSet<String> words = new HashSet<>(21000);
        int maxLength = 0;

        stream = StatisticalChineseAnnotator.class.getResourceAsStream(filename);

        LineIterator lines = IOUtils.lineIterator(stream, Charset.forName("UTF-8"));
        while (lines.hasNext()) {
            String line = lines.nextLine();

            words.add(line);
            maxLength = Math.max(maxLength, line.length());
        }

        return new Dictionary(maxLength, words);
    } finally {
        IOUtils.closeQuietly(stream);
    }
}
 

private void waitForStartupCompleted() throws IOException {
    for (int i = 0; i < 100; i++) {
        if (!this.process.isAlive())
            throw new IOException("Unable to start Cassandra process, more details here: " + this.logFile.getAbsolutePath());

        LineIterator lines = FileUtils.lineIterator(this.logFile, UTF8Charset.get().name());

        while (lines.hasNext()) {
            String line = lines.next();

            if (line.contains("Starting listening for CQL clients"))
                return;
        }

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            throw new IOException("Unexpected interruption", e);
        }
    }

    throw new IOException("Cassandra process startup timeout, more details here: " + this.logFile.getAbsolutePath());
}
 

/**
 * Returns the package name by parsing the content of the class.
 * 
 * @param javaClassPath the path of the class to parse
 * @return the package name if it could be parsed from the content
 */
private static Optional<String> getPackageNameFromClassContent(final Path javaClassPath) {
	try {
		final LineIterator lineIterator = FileUtils.lineIterator(javaClassPath.toFile());
		while (lineIterator.hasNext()) {
			final String line = lineIterator.next();
			if (line.startsWith("package ")) {
				return Optional.of(line.substring(8, line.indexOf(';')));
			}
		}
		LogUtil.getLogger().warning("No Package could be parsed from the Java file content: " + javaClassPath);
	} catch (IOException e) {
		LogUtil.getLogger().log(Level.SEVERE,
				"An error occurred during parsing the Java file content: " + javaClassPath, e);
	}
	return Optional.empty();
}
 

/**
 * Check if a TAB-Sep training file is in correct format before importing
 */
private boolean isTabSepFileFormatCorrect(File aFile)
{
    try {
        LineIterator it = new LineIterator(new FileReader(aFile));
        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().length() == 0) {
                continue;
            }
            if (line.split("\t").length != 2) {
                return false;
            }
        }
    }
    catch (Exception e) {
        return false;
    }
    return true;
}
 

public static void createInputFile(Configuration conf, Path file, Path targetFile,
                                   String startNode)
    throws IOException {
  FileSystem fs = file.getFileSystem(conf);

  OutputStream os = fs.create(targetFile);
  LineIterator iter = org.apache.commons.io.IOUtils
      .lineIterator(fs.open(file), "UTF8");
  while (iter.hasNext()) {
    String line = iter.nextLine();

    String[] parts = StringUtils.split(line);
    int distance = Node.INFINITE;
    if (startNode.equals(parts[0])) {
      distance = 0;
    }
    IOUtils.write(parts[0] + '\t' + String.valueOf(distance) + "\t\t",
        os);
    IOUtils.write(StringUtils.join(parts, '\t', 1, parts.length), os);
    IOUtils.write("\n", os);
  }

  os.close();
}
 

@Override
public Iterator<R> call(Iterator<String> dataSetIterator) throws Exception {
    //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately
    if(!dataSetIterator.hasNext()){
        return Collections.emptyIterator();
    }
    // here we'll be converting out Strings coming out of iterator to DataSets
    // PathSparkDataSetIterator does that for us
    //For better fault tolerance, we'll pull all paths to a local file. This way, if the Iterator<String> is backed
    // by a remote source that later goes down, we won't fail (as long as the source is still available)
    File f = SharedFlatMapPaths.toTempFile(dataSetIterator);

    LineIterator lineIter = new LineIterator(new FileReader(f));    //Buffered reader added automatically
    try {
        // iterator should be silently attached to VirtualDataSetIterator, and used appropriately
        SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachMDS(new PathSparkMultiDataSetIterator(lineIter, loader, hadoopConfig));

        // first callee will become master, others will obey and die
        SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker);

        return Collections.singletonList((R) result).iterator();
    } finally {
        lineIter.close();
        f.delete();
    }
}
 

/** Read in frequent sequences (sorted by support) */
public static SortedMap<Sequence, Integer> readFrequentSequences(final File output) throws IOException {
	final HashMap<Sequence, Integer> sequences = new HashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty()) {
			final String[] splitLine = line.split("#SUP:");
			final String[] items = splitLine[0].trim().split("-1");
			final Sequence seq = new Sequence();
			for (final String item : items)
				seq.add(Integer.parseInt(item.trim()));
			final int supp = Integer.parseInt(splitLine[1].trim());
			sequences.put(seq, supp);
		}
	}
	// Sort sequences by support
	final Ordering<Sequence> comparator = Ordering.natural().reverse().onResultOf(Functions.forMap(sequences))
			.compound(Ordering.usingToString());
	return ImmutableSortedMap.copyOf(sequences, comparator);
}
 

public static String collectResponse(InputStream response) {
    StringWriter logwriter = new StringWriter();

    try {
        LineIterator itr = IOUtils.lineIterator(response, "UTF-8");

        while (itr.hasNext()) {
            String line = (String) itr.next();
            logwriter.write(line + (itr.hasNext() ? "\n" : ""));
        }
        response.close();

        return logwriter.toString();
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        IOUtils.closeQuietly(response);
    }
}
 

public String uploadFile(String fileName) {
    File file = new File(getDataFolder(), fileName);
    if (!file.exists())
        return null;
    LineIterator it;
    String lines = "";
    try {
        it = FileUtils.lineIterator(file, "UTF-8");
        try {
            while (it.hasNext()) {
                String line = it.nextLine();
                lines += line + "\n";
            }
        } finally {
            it.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return MCDebug.paste(fileName, lines);
}
 

/** Read in SQS sequences (sorted by worth) */
public static LinkedHashMap<Sequence, Double> readSQSSequences(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty()) {
			final String[] splitLine = line.split("  ");
			final String[] items = splitLine[0].split(" ");
			final Sequence seq = new Sequence();
			for (final String item : items)
				seq.add(Integer.parseInt(item));
			final double worth = Double.parseDouble(splitLine[1].split(" ")[1]);
			sequences.put(seq, worth);
		}
	}

	return sequences;
}
 

/** Read in GoKrimp sequences (sorted by compression benefit) */
public static LinkedHashMap<Sequence, Double> readGoKrimpSequences(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty() && line.charAt(0) == '[') {
			final String[] splitLine = line.split(" ");
			final double worth = Double.parseDouble(splitLine[splitLine.length - 1]);
			final Sequence seq = new Sequence();
			for (int i = 1; i < splitLine.length - 2; i++)
				seq.add(Integer.parseInt(splitLine[i]));
			sequences.put(seq, worth);
		}
	}

	return sequences;
}
 

/**
 * Read in GOKRIMP sequences (sorted by compression benefit)
 *
 * @deprecated gives slightly different results to reference implementation
 */
@Deprecated
public static LinkedHashMap<Sequence, Double> readGoKrimpSequencesSPMF(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty()) {
			final String[] splitLine = line.split("#SUP:");
			final String[] items = splitLine[0].trim().split(" ");
			final Sequence seq = new Sequence();
			for (final String item : items)
				seq.add(Integer.parseInt(item.trim()));
			final double compressionBenefit = Double.parseDouble(splitLine[1].trim());
			sequences.put(seq, compressionBenefit);
		}
	}

	return sequences;
}
 

/** Read in frequent sequences (sorted by support) */
public static SortedMap<Sequence, Integer> readFrequentSequences(final File output) throws IOException {
	final HashMap<Sequence, Integer> sequences = new HashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty()) {
			final String[] splitLine = line.split("#SUP:");
			final String[] items = splitLine[0].trim().split("-1");
			final Sequence seq = new Sequence();
			for (final String item : items)
				seq.add(Integer.parseInt(item.trim()));
			final int supp = Integer.parseInt(splitLine[1].trim());
			sequences.put(seq, supp);
		}
	}
	// Sort sequences by support
	final Ordering<Sequence> comparator = Ordering.natural().reverse().onResultOf(Functions.forMap(sequences))
			.compound(Ordering.usingToString());
	return ImmutableSortedMap.copyOf(sequences, comparator);
}
 

/** Print useful statistics for the transaction database */
public static void printTransactionDBStats(final File dbFile) throws IOException {

	int noTransactions = 0;
	double sparsity = 0;
	final Set<Integer> singletons = new HashSet<>();
	final LineIterator it = FileUtils.lineIterator(dbFile, "UTF-8");
	while (it.hasNext()) {
		final String[] items = it.nextLine().replace("-2", "").split(" -1 ");
		for (final String item : items)
			singletons.add(Integer.parseInt(item));
		sparsity += items.length;
		noTransactions++;
	}
	LineIterator.closeQuietly(it);

	System.out.println("\nDatabase: " + dbFile);
	System.out.println("Items: " + singletons.size());
	System.out.println("Transactions: " + noTransactions);
	System.out.println("Avg. items per transaction: " + sparsity / noTransactions + "\n");

}
 

private void getSents(final Tokens alphabet) {

		LineIterator iterator = null;
		try {
			iterator = FileUtils.lineIterator(docLoc);
		} catch (final IOException e) {
			e.printStackTrace();
		}

		final ArrayList<Sentence> sents = new ArrayList<Sentence>();
		while (iterator.hasNext()) {
			final String in = iterator.nextLine().trim();
			// !! Include empty sentences so nodeID indexing is consistent !!
			// if (!in.equals("")) {
			sents.add(new Sentence(in, nsents, this, alphabet));
			nsents++;
			// }
		}

		this.sents = new Sentence[nsents];
		sents.toArray(this.sents);

		LineIterator.closeQuietly(iterator);
	}
 

/**
 * @return the original document text
 */
public String getOriginal() {

	final StringBuffer doc = new StringBuffer();
	LineIterator iterator = null;
	try {
		iterator = FileUtils.lineIterator(docLoc);
	} catch (final IOException e) {
		e.printStackTrace();
	}

	while (iterator.hasNext())
		doc.append(iterator.nextLine().trim() + "\n");
	LineIterator.closeQuietly(iterator);

	return doc.toString();
}
 

/**
 * @return The output slurped into a string.
 */
public static String asString(ClientResponse response) throws IOException {

	StringWriter out = new StringWriter();
	try {
		LineIterator itr = IOUtils.lineIterator(
				response.getEntityInputStream(), "UTF-8");
		while (itr.hasNext()) {
			String line = itr.next();
			out.write(line + (itr.hasNext() ? "\n" : ""));
		}
	} finally {
		closeQuietly(response.getEntityInputStream());
	}
	return out.toString();
}
 

@Test
public final void givenUsingApacheIo_whenStreamingThroughAFile_thenCorrect() throws IOException {
    final String path = "G:\\full\\train\\input\\" + "trainDataNegative.csv";
    // final String path = "G:\\full\\train\\input\\" + "trainDataPositive.csv";

    logMemory();

    final LineIterator it = FileUtils.lineIterator(new File(path), "UTF-8");
    try {
        while (it.hasNext()) {
            final String line = it.nextLine();
            // do something with line
        }
    } finally {
        LineIterator.closeQuietly(it);
    }

    logMemory();
}
 

@Test
public void testChangeSet() throws URISyntaxException{
  ODataClient client = ODataClientBuilder.createClient();
  URI uri = new URI("test");
  final InputStream input = getClass().getResourceAsStream("batchResponse.batch");
  Reader reader = new InputStreamReader(input);
  ODataBatchLineIterator iterator = new ODataBatchLineIteratorImpl(new LineIterator(reader ));
  ODataBatchRequest req = new ODataBatchRequestImpl(client, uri);;
  ODataChangesetResponseItem expectedResItem = new ODataChangesetResponseItem(true);
  ODataChangesetImpl change = new ODataChangesetImpl(req , expectedResItem );
  assertNotNull(change);
  ODataBatchableRequest request = new ODataInvokeRequestImpl<ClientInvokeResult>(
      client, ClientInvokeResult.class, HttpMethod.POST, uri);
  change.addRequest(request);
  assertNotNull(change.getBodyStreamWriter());
  change.close();
  change.closeItem();
  assertNotNull(change.getLastContentId());
  assertTrue(change.hasStreamedSomething());
  assertFalse(change.isOpen());
  change.streamRequestHeader(request);
  change.streamRequestHeader("1");
  
}
 

@Test
public void testErrorBatchResponse() throws URISyntaxException {
  Map<String, Collection<String>> header = new HashMap<String, Collection<String>>();
  List<String> list = new ArrayList<String>();
  list.add("multipart/mixed;boundary=changeset_12ks93js84d");
  header.put("content-type", list);
  final InputStream input = getClass().getResourceAsStream("batchResponse.batch");
  Reader reader = new InputStreamReader(input);
  ODataBatchLineIterator iterator = new ODataBatchLineIteratorImpl(new LineIterator(reader ));
  String boundary = "changeset_12ks93js84d";
  iterator.next();
  iterator.next();
  iterator.next();
  iterator.next();
  iterator.next();
  iterator.next();
  iterator.next();
  iterator.next();
  Entry<Integer, String> line = ODataBatchUtilities.readResponseLine(iterator);
  ODataBatchErrorResponse error = new ODataBatchErrorResponse
      (line, header, iterator, boundary );
  assertNotNull(error);
  assertNull(error.getETag());
 }