Java Code Examples for org.apache.commons.io.LineIterator#close()

@Override
public Iterator<R> call(Iterator<String> dataSetIterator) throws Exception {
    //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately
    if(!dataSetIterator.hasNext()){
        return Collections.emptyIterator();
    }
    // here we'll be converting out Strings coming out of iterator to DataSets
    // PathSparkDataSetIterator does that for us
    //For better fault tolerance, we'll pull all paths to a local file. This way, if the Iterator<String> is backed
    // by a remote source that later goes down, we won't fail (as long as the source is still available)
    File f = SharedFlatMapPaths.toTempFile(dataSetIterator);

    LineIterator lineIter = new LineIterator(new FileReader(f));    //Buffered reader added automatically
    try {
        // iterator should be silently attached to VirtualDataSetIterator, and used appropriately
        SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachMDS(new PathSparkMultiDataSetIterator(lineIter, loader, hadoopConfig));

        // first callee will become master, others will obey and die
        SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker);

        return Collections.singletonList((R) result).iterator();
    } finally {
        lineIter.close();
        f.delete();
    }
}
 

@Override
public Iterator<R> call(Iterator<String> dataSetIterator) throws Exception {
    //Under some limited circumstances, we might have an empty partition. In this case, we should return immediately
    if(!dataSetIterator.hasNext()){
        return Collections.emptyIterator();
    }
    // here we'll be converting out Strings coming out of iterator to DataSets
    // PathSparkDataSetIterator does that for us
    //For better fault tolerance, we'll pull all paths to a local file. This way, if the Iterator<String> is backed
    // by a remote source that later goes down, we won't fail (as long as the source is still available)
    File f = SharedFlatMapPaths.toTempFile(dataSetIterator);

    LineIterator lineIter = new LineIterator(new FileReader(f));    //Buffered reader added automatically
    try {
        // iterator should be silently attached to VirtualDataSetIterator, and used appropriately
        SharedTrainingWrapper.getInstance(worker.getInstanceId()).attachDS(new PathSparkDataSetIterator(lineIter, loader, hadoopConfig));

        // first callee will become master, others will obey and die
        SharedTrainingResult result = SharedTrainingWrapper.getInstance(worker.getInstanceId()).run(worker);

        return Collections.singletonList((R) result).iterator();
    } finally {
        lineIter.close();
        f.delete();
    }
}
 

public static SentenceIterator getIterator(boolean isIntegration, File file, int linesForUnitTest) throws IOException {
    if(isIntegration){
        return new BasicLineIterator(file);
    } else {
        List<String> lines = new ArrayList<>();
        try(InputStream is = new BufferedInputStream(new FileInputStream(file))){
            LineIterator lineIter = IOUtils.lineIterator(is, StandardCharsets.UTF_8);
            try{
                for( int i=0; i<linesForUnitTest && lineIter.hasNext(); i++ ){
                    lines.add(lineIter.next());
                }
            } finally {
                lineIter.close();
            }
        }

        return new CollectionSentenceIterator(lines);
    }
}
 

public static List<String> firstNLines(File f, int n){
    List<String> lines = new ArrayList<>();
    try(InputStream is = new BufferedInputStream(new FileInputStream(f))){
        LineIterator lineIter = IOUtils.lineIterator(is, StandardCharsets.UTF_8);
        try{
            for( int i=0; i<n && lineIter.hasNext(); i++ ){
                lines.add(lineIter.next());
            }
        } finally {
            lineIter.close();
        }
        return lines;
    } catch (IOException e){
        throw new RuntimeException(e);
    }
}
 

private static Map<Integer, Integer> loadDictionary(String filename) {
    HashMap<Integer, Integer> result = new HashMap<>();

    InputStream stream = null;
    LineIterator iterator = null;

    try {
        stream = ChineseCharacterConverter.class.getResourceAsStream(filename);
        iterator = IOUtils.lineIterator(stream, "UTF-8");
        while (iterator.hasNext()) {
            String line = iterator.nextLine();
            String[] keyValues = line.split("\t", 2);
            Integer key = keyValues[0].codePointAt(0);
            Integer value = keyValues[1].codePointAt(0);
            result.put(key, value);
        }

        return result;
    } catch (IOException e) {
        throw new Error(e);
    } finally {
        IOUtils.closeQuietly(stream);
        if (iterator != null)
            iterator.close();
    }
}
 

public String uploadFile(String fileName) {
    File file = new File(getDataFolder(), fileName);
    if (!file.exists())
        return null;
    LineIterator it;
    String lines = "";
    try {
        it = FileUtils.lineIterator(file, "UTF-8");
        try {
            while (it.hasNext()) {
                String line = it.nextLine();
                lines += line + "\n";
            }
        } finally {
            it.close();
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return MCDebug.paste(fileName, lines);
}
 

/**
 * 切分文本编码.
 */
private String splitFileResult(File file, int splitLength, String sourceEncoding, String targetEncoding) throws IOException {
    StringBuilder rsb = new StringBuilder();
    String charset = sourceEncoding;
    if (charset.equals(sourceEncodingAuto)) {
        charset = JUniversalChardet.detectFileCharset(file, 4096);
    }
    LineIterator lineIterator = org.apache.commons.io.FileUtils.lineIterator(file, charset);
    String fileName = file.getName();
    int lastIndex = fileName.lastIndexOf(".");
    lastIndex = (lastIndex == -1 ? fileName.length() : lastIndex);
    String fileSimpleName = fileName.substring(0, lastIndex);
    String fileType = fileName.substring(lastIndex + 1, fileName.length());
    String rs;
    int fileNum = 1;
    while ((rs = loopSplitFile(lineIterator, splitLength, fileSimpleName + "_" + (fileNum++) + (fileType.length() != 0 ? "." : "") + fileType, targetEncoding)) != null) {
        if (rsb.length() != 0) {
            rsb.append("            ");
        }
        rsb.append(rs).append("\n");
    }
    lineIterator.close();
    return rsb.toString();
}
 

@Override
public void close() throws IOException {
    if (iter != null) {
        if (iter instanceof LineIterator) {
            LineIterator iter2 = (LineIterator) iter;
            iter2.close();
        }
    }
}
 

private void logErrorMessage(ClientResponse response) throws IOException {
  LOG.error("Response status={}", response.getClientResponseStatus().toString());
  LineIterator it = null;
  try {
    it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8);
    while (it.hasNext()) {
      String line = it.nextLine();
      LOG.error(line);
    }
  } finally {
    if (it != null) {
      it.close();
    }
  }
}
 

private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 

private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 

@Override
public void close() throws IOException {
	if (lineIter != null) {
        if (lineIter instanceof LineIterator) {
            LineIterator iter2 = (LineIterator) lineIter;
            iter2.close();
        }
    }
}
 

private static void generateTransactionDatabase(final String arffFile, final BiMap<String, Integer> dictionary,
		final File transactionDB) throws IOException {

	int mID = 0;
	boolean found = false;
	final PrintWriter out = new PrintWriter(transactionDB);
	final LineIterator it = FileUtils.lineIterator(new File(arffFile));
	while (it.hasNext()) {
		final String line = it.nextLine();

		if (found) {
			for (final String raw_call : line.split(",")[1].replace("\'", "").split(" ")) {
				final String call = raw_call.trim();
				if (call.isEmpty()) // skip empty strings
					continue;
				if (dictionary.containsKey(call)) {
					final int ID = dictionary.get(call);
					out.print(ID + " -1 ");
				} else {
					out.print(mID + " -1 ");
					dictionary.put(call, mID);
					mID++;
				}
			}
			out.println("-2");
		}

		if (line.contains("@data"))
			found = true;

	}
	it.close();
	out.close();
}
 

private List<List<Writable>> loadAndClose(InputStream inputStream) {
    LineIterator lineIter = null;
    try {
        lineIter = IOUtils.lineIterator(new BufferedReader(new InputStreamReader(inputStream)));
        return load(lineIter);
    } finally {
        if (lineIter != null) {
            lineIter.close();
        }
        IOUtils.closeQuietly(inputStream);
    }
}
 

@Override
public void close() throws IOException {
    if (iter != null) {
        if (iter instanceof LineIterator) {
            LineIterator iter2 = (LineIterator) iter;
            iter2.close();
        }
    }
}
 

/**
 * Searches the contents of a file
 * @param file The file to search
 * @return returns whether the search was successful
 * @throws IOException thrown if there is an error reading the file
 */
private boolean searchFile(File file) {
	LineIterator lineIterator = null;
	try {
		lineIterator = FileUtils.lineIterator(file);
	} catch (IOException e) {
		logger.error("FileGrepper.searchFile: " + e.getLocalizedMessage());
		return false;
	}
	try {
		while (lineIterator.hasNext()) {
			String line = lineIterator.nextLine();
			if (line.contains("\0")) {
				// file contains binary content
				return false;
			}
			matcher.reset(line);
			if (matcher.find()) {
				return true;
			}
		}
	} finally {
		if (lineIterator != null)
			lineIterator.close();
	}
	return false;
}
 

protected void closeIfRequired(Iterator<String> iterator) {
    if (iterator instanceof LineIterator) {
        LineIterator iter = (LineIterator) iterator;
        iter.close();
    }
}
 

/**
 * Loads an in memory cache from the given input stream (sets syn0 and the vocab).
 *
 * @param inputStream  input stream
 * @return a {@link Pair} holding the lookup table and the vocab cache.
 */
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(@NonNull InputStream inputStream) {
    AbstractCache<VocabWord> cache = new AbstractCache<>();
    LineIterator lines = null;

    try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
         BufferedReader reader = new BufferedReader(inputStreamReader)) {
        lines = IOUtils.lineIterator(reader);

        String line = null;
        boolean hasHeader = false;

        /* Check if first line is a header */
        if (lines.hasNext()) {
            line = lines.nextLine();
            hasHeader = isHeader(line, cache);
        }

        if (hasHeader) {
            log.debug("First line is a header");
            line = lines.nextLine();
        }

        List<INDArray> arrays = new ArrayList<>();
        long[] vShape = new long[]{ 1, -1 };

        do {
            String[] tokens = line.split(" ");
            String word = ReadHelper.decodeB64(tokens[0]);
            VocabWord vocabWord = new VocabWord(1.0, word);
            vocabWord.setIndex(cache.numWords());

            cache.addToken(vocabWord);
            cache.addWordToIndex(vocabWord.getIndex(), word);
            cache.putVocabWord(word);

            float[] vector = new float[tokens.length - 1];
            for (int i = 1; i < tokens.length; i++) {
                vector[i - 1] = Float.parseFloat(tokens[i]);
            }

            vShape[1] = vector.length;
            INDArray row = Nd4j.create(vector, vShape);

            arrays.add(row);

            line = lines.hasNext() ? lines.next() : null;
        } while (line != null);

        INDArray syn = Nd4j.vstack(arrays);

        InMemoryLookupTable<VocabWord> lookupTable = new InMemoryLookupTable
                .Builder<VocabWord>()
                .vectorLength(arrays.get(0).columns())
                .useAdaGrad(false)
                .cache(cache)
                .useHierarchicSoftmax(false)
                .build();

        lookupTable.setSyn0(syn);

        return new Pair<>((InMemoryLookupTable) lookupTable, (VocabCache) cache);
    } catch (IOException readeTextStreamException) {
        throw new RuntimeException(readeTextStreamException);
    } finally {
        if (lines != null) {
            lines.close();
        }
    }
}
 

protected void closeIfRequired(Iterator<String> iterator) {
    if (iterator instanceof LineIterator) {
        LineIterator iter = (LineIterator) iterator;
        iter.close();
    }
}
 

protected void closeIfRequired(Iterator<String> iterator) {
    if (iterator instanceof LineIterator) {
        LineIterator iter = (LineIterator) iterator;
        iter.close();
    }
}