Java Code Examples for org.apache.commons.io.LineIterator#hasNext()

/**
 * Check if a TAB-Sep training file is in correct format before importing
 */
private boolean isTabSepFileFormatCorrect(File aFile)
{
    try {
        LineIterator it = new LineIterator(new FileReader(aFile));
        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().length() == 0) {
                continue;
            }
            if (line.split("\t").length != 2) {
                return false;
            }
        }
    }
    catch (Exception e) {
        return false;
    }
    return true;
}
 

public static int createInputFile(Path file, Path targetFile)
    throws IOException {
  Configuration conf = new Configuration();
  FileSystem fs = file.getFileSystem(conf);

  int numNodes = getNumNodes(file);
  double initialPageRank = 1.0 / (double) numNodes;

  OutputStream os = fs.create(targetFile);
  LineIterator iter = IOUtils
      .lineIterator(fs.open(file), "UTF8");

  while (iter.hasNext()) {
    String line = iter.nextLine();

    String[] parts = StringUtils.split(line);

    Node node = new Node()
        .setPageRank(initialPageRank)
        .setAdjacentNodeNames(
            Arrays.copyOfRange(parts, 1, parts.length));
    IOUtils.write(parts[0] + '\t' + node.toString() + '\n', os);
  }
  os.close();
  return numNodes;
}
 

private Iterator<Doc> extractDocs(File file) {
    List<Doc> docs = new ArrayList<>();
    try {
        LineIterator lineIterator = FileUtils.lineIterator(file, "UTF-8");
        while (lineIterator.hasNext()) {
            String line = lineIterator.next();
            if (!line.isEmpty()) {
                docs.add(new Doc(JsonUtils.fromJsonString(Map.class, line)));
            }
        }

    } catch (Exception e) {
        throw new RuntimeException("failed to extract docs from file [" + file + "]", e);
    }

    return Iterables.cycle(docs).iterator();
}
 

private static void _CheckReload(){
    if(wordfilter.lastModified() > lastModified){
        synchronized(SensitiveWord.class){
            try{
                lastModified = wordfilter.lastModified();
                LineIterator lines = FileUtils.lineIterator(wordfilter, "utf-8");
                while(lines.hasNext()){
                    String line = lines.nextLine();
                    if(StringUtils.isNotBlank(line))
                        words.add(StringUtils.trim(line).toLowerCase());
                }
            }catch(IOException e){
                e.printStackTrace();
            }
        }
    }
}
 

private void getSents(final Tokens alphabet) {

		LineIterator iterator = null;
		try {
			iterator = FileUtils.lineIterator(docLoc);
		} catch (final IOException e) {
			e.printStackTrace();
		}

		final ArrayList<Sentence> sents = new ArrayList<Sentence>();
		while (iterator.hasNext()) {
			final String in = iterator.nextLine().trim();
			// !! Include empty sentences so nodeID indexing is consistent !!
			// if (!in.equals("")) {
			sents.add(new Sentence(in, nsents, this, alphabet));
			nsents++;
			// }
		}

		this.sents = new Sentence[nsents];
		sents.toArray(this.sents);

		LineIterator.closeQuietly(iterator);
	}
 

public ObservableList<Message> getMessages(String user){
    ObservableList<Message> messages = FXCollections.observableArrayList();
    try {
        File messagesFile = new File(path.toString() + File.separator + "messages.csv");
        LineIterator lineIterator = FileUtils.lineIterator(messagesFile);
        while (lineIterator.hasNext()) {
            String line = lineIterator.nextLine();
            String[] components = splitString(line);
            if(components[0].equals(user)){
                messages.add(new Message(components[0],components[1],components[2]));
            }
        }
        return messages;
    } catch (IOException ex) {
        Logger.getLogger(Corpus.class.getName()).log(Level.SEVERE, null, ex);
    }
    return messages;
}
 

public static List<Code> fileToCodeList(String fileName) {

    List<Code> codes = new ArrayList<>();
    try {
      LineIterator it = getLineIterator(fileName);
      while (it.hasNext()) {
        String line = it.nextLine();
        if (line.equals("") || line.startsWith("#"))
          continue;
        String[] parts = line.split(";", 2);
        codes.add((new Code(parts[0], parts[1])));
      }
    } catch (IOException e) {
      e.printStackTrace();
    }

    return codes;

  }
 

public static HashMap<String, ArrayList<String>> parseSnpPerGeneFile(String snpsToTestFile) throws IOException {
	LineIterator snpGenePairIterator = FileUtils.lineIterator(new File(snpsToTestFile), "UTF-8");
	HashMap<String, ArrayList<String>> geneSnpPairs = new HashMap<String, ArrayList<String>>();
	int totalSnpsToTest = 0;
	snpGenePairIterator.next();
	while (snpGenePairIterator.hasNext()) {
		ArrayList<String> snpGeneStringVector = new ArrayList<String>(Arrays.asList(snpGenePairIterator.next().split("\t")));
		String gene = snpGeneStringVector.get(0);
		String snp = snpGeneStringVector.get(1);
		ArrayList<String> snps = geneSnpPairs.get(gene);
		if (snps==null) {
			snps = new ArrayList<String>();
			geneSnpPairs.put(gene, snps);
		}
		snps.add(snp);
		totalSnpsToTest++;
	}
	DeconvolutionLogger.log.info(String.format("SNPs to deconvolute: %d", totalSnpsToTest));

	return geneSnpPairs;
}
 

@Test
public void testResolvingActual() throws Exception {
    File f = Resources.asFile("data/irisSvmLight.txt");
    assertTrue(f.exists());

    //System.out.println(f.getAbsolutePath());
    int count = 0;
    try(Reader r = new BufferedReader(new FileReader(f))){
        LineIterator iter = IOUtils.lineIterator(r);
        while(iter.hasNext()){
            String line = iter.next();
            //System.out.println("LINE " + i + ": " + line);
            count++;
        }
    }

    assertEquals(12, count);        //Iris normally has 150 examples; this is subset with 12
}
 

/** Read in GoKrimp sequences (sorted by compression benefit) */
public static LinkedHashMap<Sequence, Double> readGoKrimpSequences(final File output) throws IOException {
	final LinkedHashMap<Sequence, Double> sequences = new LinkedHashMap<>();

	final LineIterator it = FileUtils.lineIterator(output);
	while (it.hasNext()) {
		final String line = it.nextLine();
		if (!line.trim().isEmpty() && line.charAt(0) == '[') {
			final String[] splitLine = line.split(" ");
			final double worth = Double.parseDouble(splitLine[splitLine.length - 1]);
			final Sequence seq = new Sequence();
			for (int i = 1; i < splitLine.length - 2; i++)
				seq.add(Integer.parseInt(splitLine[i]));
			sequences.put(seq, worth);
		}
	}

	return sequences;
}
 

private void doTestFile(DoubleParser parser) throws Exception {
    int nLinesToTest = 500;
    URL fileUrl = getClass().getResource("/cities-dos.txt");
    File file = new File(fileUrl.toURI());
    LineIterator lines = IOUtils.lineIterator(new FileInputStream(file), Charset.defaultCharset());
    int lineNumber = 0;
    while (lines.hasNext() && lineNumber < nLinesToTest) {
        String[] data = lines.next().split(",");
        for (int i = 0; i < data.length; i++) {
            compareParsingResult(parser, data[i]);
        }
        lineNumber ++;
    }
}
 

private static Set<String> loadIdsInputFile(String input) {
	if (input != null) {
		File inputFile = new File(input);
		if (inputFile.exists() && inputFile.isFile() && inputFile.canRead()) {
			Set<String> ids = new HashSet<String>();
			LineIterator iterator = null;
			try {
				iterator = FileUtils.lineIterator(inputFile);
				while (iterator.hasNext()) {
					String line = iterator.next();
					line = StringUtils.trimToNull(line);
					if (line != null) {
						ids.add(line);
					}
				}
				logger.info("Finished loading input file: "+input+"\n id count: "+ids.size());
				return ids;
			} catch (IOException e) {
				logger.warn("Could not load ids file: "+input, e);
			}
			finally {
				LineIterator.closeQuietly(iterator);
			}
		}
		else {
			logger.warn("Could not load ids file: "+input);
		}
	}
	return null;
}
 

@Override
public Boolean call() throws IOException, InterruptedException {
    try (Reader reader = new InputStreamReader(in)) {
        LineIterator it = IOUtils.lineIterator(reader);
        while (it.hasNext()) {
            String line = it.nextLine();
            if (outputObserver.onReadLine(line)) {
                return true; // interrupted by OutputObserver
            }
        }
    }
    return false;
}
 

public static int getTotalNumberOfLinesUsingApacheCommonsIO(String fileName) {
    int lines = 0;
    try {
        LineIterator lineIterator = FileUtils.lineIterator(new File(fileName));
        while (lineIterator.hasNext()) {
            lineIterator.nextLine();
            lines++;
        }
    } catch (IOException ioe) {
        ioe.printStackTrace();
    }
    return lines;
}
 

private static void buildTrainFile(File aBaseFile, File aTrainFile,
        List<List<String>> aPredictions)
    throws IOException
{
    LineIterator it = IOUtils.lineIterator(new FileReader(aBaseFile));
    StringBuilder trainBuffer = new StringBuilder();
    int i = 0;
    while (it.hasNext()) {
        String line = it.next();
        if (line.trim().equals("")) {
            trainBuffer.append("\n");
            continue;
        }
        StringTokenizer st = new StringTokenizer(line, " ");
        String label = "";
        String feature = "";
        // Except the last token, which is the label, maintain the line
        while (st.hasMoreTokens()) {
            feature = st.nextToken();
            if (label.equals("")) { // first time
                label = feature;
                continue;
            }
            trainBuffer.append(label).append(" ");
            label = feature;

        }
        for (List<String> prediction : aPredictions) {
            trainBuffer.append(prediction.get(i)).append(" ");
        }
        // add its own label
        trainBuffer.append(label).append("\n");
        i++;
    }
    IOUtils.write(trainBuffer.toString(), new FileOutputStream(aTrainFile));

}
 

/**
 * This method scans the input database to calculate the support of single
 * items.
 *
 * @param inputFile
 *            the input file
 * @return a multiset for storing the support of each singleton
 */
public static Multiset<Sequence> scanDatabaseToDetermineFrequencyOfSingleItems(final File inputFile)
		throws IOException {

	final Multiset<Sequence> singletons = HashMultiset.create();

	// for each line (transaction) until the end of file
	final LineIterator it = FileUtils.lineIterator(inputFile, "UTF-8");
	while (it.hasNext()) {

		final String line = it.nextLine();
		// if the line is a comment, is empty or is a
		// kind of metadata
		if (line.isEmpty() == true || line.charAt(0) == '#' || line.charAt(0) == '%' || line.charAt(0) == '@') {
			continue;
		}

		// split the line into items
		final String[] lineSplit = line.split(" ");
		// for each item
		final HashSet<Sequence> seenItems = new HashSet<>();
		for (final String itemString : lineSplit) {
			final int item = Integer.parseInt(itemString);
			if (item >= 0) { // ignore end of itemset/sequence tags
				final Sequence seq = new Sequence(item);
				PAMCore.recursiveSetOccurrence(seq, seenItems); // set
																// occurrence
				seenItems.add(seq); // add item to seen
			}
		}
		singletons.addAll(seenItems); // increase the support of the items
	}

	// close the input file
	LineIterator.closeQuietly(it);

	return singletons;
}
 

public static void main(String[] args) throws IOException {
    Properties p = CommandLineUtils.getProperties(args, helpMessage, new String[]{"-i", "-l"});
    File in = new File(p.getProperty("-i"));
    int split = 1000;
    try {
        split = Integer.parseInt(p.getProperty("-l"));
    } catch (NumberFormatException e) {
        System.exit(1);
        System.err.printf("Number of lines as given is not a number: %s\n", p.getProperty("-l"));
    }
    if (!in.exists()) {
        System.exit(1);
        System.err.printf("File %s does not exist.\n", in.getName());
    }

    // ok, let's split
    int count = 0;
    int filenum = 0;
    String filePattern = FilenameUtils.getBaseName(p.getProperty("-i")) + "%03d." + FilenameUtils.getExtension(p.getProperty("-i"));
    if (p.get("-d")!=null) {
        filePattern = p.getProperty("-d") + '/' + filePattern;
    }
    String fileExec = FilenameUtils.getBaseName(p.getProperty("-i")) + "-index.sh";
    String currentFileName = String.format(filePattern, filenum);
    System.out.printf("Writing file %s ...\n", currentFileName);
    BufferedWriter bw = new BufferedWriter(new FileWriter(currentFileName), charBufferSize);
    BufferedWriter bwExec = null;
    if (p.get("-x") != null) bwExec = new BufferedWriter(new FileWriter(fileExec));
    addToShellFile(bwExec, currentFileName);
    LineIterator lineIterator = FileUtils.lineIterator(in);
    bw.write("<add>\n");
    while (lineIterator.hasNext()) {
        String currentLine = lineIterator.next();
        if (!(currentLine.startsWith("<add>") || currentLine.startsWith("#") || currentLine.startsWith("</add>"))) {
            // check if the old file is full ...
            if (count >= split) {
                bw.write("</add>\n");
                bw.close();
                currentFileName = String.format(filePattern, ++filenum);
                System.out.printf("Writing file %s ...\n", currentFileName);
                bw = new BufferedWriter(new FileWriter(currentFileName), charBufferSize);
                bw.write("<add>\n");
                count = 0;
                addToShellFile(bwExec, currentFileName);
            }
            // write to the current file ...
            bw.write(currentLine);
            bw.write('\n');
            count++;
        }
    }
    bw.write("</add>\n");
    bw.close();
    if (bwExec != null) bwExec.close();
    System.out.println("Finished.");
}
 

private static void processRawContactInformation(String fileToRead, double minValue, ArrayList<DesiredChrContact> contactsToCheck, boolean intra) throws IOException {

        //Check if sorted version is available
        //If not make sorted available.
        if (!Gpio.exists(fileToRead + ".sorted")) {
            if (intra) {
                umcg.genetica.io.chrContacts.SortIntraChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            } else {
                umcg.genetica.io.chrContacts.SortInterChrContacts.readNonSortedWriteSorted(fileToRead, fileToRead + ".sorted");
            }

        }

        int numberToBeMatched = 0;

        LineIterator it = FileUtils.lineIterator(new File(fileToRead + ".sorted"), "UTF-8");

        try {
            while (it.hasNext()) {
                String[] parts = StringUtils.split(it.nextLine(), '\t');

                int posChr1 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[0]);
                int posChr2 = org.apache.commons.lang.math.NumberUtils.createInteger(parts[1]);

                while (numberToBeMatched < contactsToCheck.size()) {
                    if (posChr1 < contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        break;
                    } else if (posChr1 == contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        if (posChr2 < contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            break;
                        }
                        if (posChr2 == contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            double contact = org.apache.commons.lang.math.NumberUtils.createDouble(parts[2]);
                            if (contact >= minValue) {
                                contactsToCheck.get(numberToBeMatched).setContact();
                                numberToBeMatched++;
                            } else {
                                numberToBeMatched++;
                            }
                        } else if (posChr2 > contactsToCheck.get(numberToBeMatched).getChrLocationLarger()) {
                            numberToBeMatched++;
                        }
                    } else if (posChr1 > contactsToCheck.get(numberToBeMatched).getChrLocationSmaller()) {
                        numberToBeMatched++;
                    }
                }
            }
        } finally {
            LineIterator.closeQuietly(it);
        }

    }
 

private boolean ignoreLine(GraphContext context, FileLocationModel model)
{
    boolean isPropertiesFile = model.getFile() instanceof PropertiesModel;

    int lineNumber = model.getLineNumber();
    LineIterator li = null;
    try
    {
        li = FileUtils.lineIterator(model.getFile().asFile());

        int i = 0;
        while (li.hasNext())
        {
            i++;

            // read the line to memory only if it is the line of interest
            if (i == lineNumber)
            {
                String line = StringUtils.trim(li.next());
                // check that it isn't commented.
                if (isPropertiesFile && StringUtils.startsWith(line, "#"))
                    return true;
                // WINDUP-808 - Remove matches with "version" or "revision" on the same line
                else if (StringUtils.containsIgnoreCase(line, "version") || StringUtils.containsIgnoreCase(line, "revision"))
                    return true;
                else if (isMavenVersionTag(context, model))
                    return true;
                else
                    return false;
            }
            else if (i < lineNumber)
            {
                // seek
                li.next();
            }
            else if (i > lineNumber)
            {
                LOG.warning("Did not find line: " + lineNumber + " in file: " + model.getFile().getFileName());
                break;
            }
        }
    }
    catch (IOException | RuntimeException e)
    {
        LOG.log(Level.WARNING, "Exception reading properties from file: " + model.getFile().getFilePath(), e);
    }
    finally
    {
        LineIterator.closeQuietly(li);
    }

    return false;
}
 

/**
 * The main core logic for all testing.
 * 
 * @param outputFileName
 * @param function
 * @throws IOException
 */
private void testHashes(String outputFileName, StringHashFunction function) throws IOException {
	LineIterator iterator = FileUtils.lineIterator(new File(BASE_PATH + "/english-wordlist.txt"));
	LineIterator results = FileUtils.lineIterator(new File(BASE_PATH + "/" + outputFileName));
	
	int matched = 0;
	int total = 0;
	
	while(iterator.hasNext()) {
		String line = iterator.next();
		
		byte[] bytes = line.getBytes();
		String computed = function.getHash(bytes);
		String actual = results.next();
		
		if(actual.contains(",")) {
			// result has multiple values
			String[] act = actual.split(",");
			String[] com = computed.split(",");
			if(act.length == com.length) {
				boolean allMatch = true;
				for(int index = 0; index < act.length; index++) {
					allMatch = allMatch & bigMatch(act[index], com[index]);
				}
				
				if(allMatch) {
					matched++;
				}
			}
		} else {
			// result has only a single value
			if(actual.equals(computed)) {
				matched++;
			} else {
				if(bigMatch(actual, computed)) {
					matched++;
				}
			}
		}
		
		total++;
	}
	
	Assert.assertEquals("Total number of hashes did not match", total, matched);
}