Java Code Examples for org.deeplearning4j.text.sentenceiterator.SentenceIterator#hasNext()

The following examples show how to use org.deeplearning4j.text.sentenceiterator.SentenceIterator#hasNext() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BasicLineIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 6 votes vote down vote up 
public static void main(String[] args) throws IOException {
    SentenceIterator iterator = new BasicLineIterator(new ClassPathResource("raw_sentences.txt").getFile());
    int count=0;
    while(iterator.hasNext()){
       iterator.nextSentence();
       count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
    

}
Example 2
Source File: BasicLineIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 6 votes vote down vote up 
public static void main(String[] args) throws IOException {
    SentenceIterator iterator = new BasicLineIterator(new ClassPathResource("raw_sentences.txt").getFile());
    int count=0;
    while(iterator.hasNext()){
       iterator.nextSentence();
       count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
    

}
Example 3
Source File: VocabConstructorTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testVocab() throws Exception {
    File inputFile = Resources.asFile("big/raw_sentences.txt");
    SentenceIterator iter = new BasicLineIterator(inputFile);

    Set<String> set = new HashSet<>();
    int lines = 0;
    int cnt = 0;
    while (iter.hasNext()) {
        Tokenizer tok = t.create(iter.nextSentence());
        for (String token : tok.getTokens()) {
            if (token == null || token.isEmpty() || token.trim().isEmpty())
                continue;
            cnt++;

            if (!set.contains(token))
                set.add(token);
        }

        lines++;
    }

    log.info("Total number of tokens: [" + cnt + "], lines: [" + lines + "], set size: [" + set.size() + "]");
    log.info("Set:\n" + set);
}
Example 4
Source File: CollectionSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
    List<String> sentences = Arrays.asList(
            "No ,  he says now .",
            "And what did he do ?",
            "The money 's there .",
            "That was less than a year ago .",
            "But he made only the first .",
            "There 's still time for them to do it .",
            "But he should nt have .",
            " They have to come down to the people .",
            "I do nt know where that is .",
            "No , I would nt .",
            "Who Will It Be ?",
            "And no , I was not the one ."
    );
    SentenceIterator iterator = new CollectionSentenceIterator(sentences);
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
}
Example 5
Source File: FileSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
    SentenceIterator iterator = new FileSentenceIterator(new ClassPathResource("files/").getFile());
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
}
Example 6
Source File: LineSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
    SentenceIterator iterator = new LineSentenceIterator(new ClassPathResource("raw_sentences.txt").getFile());
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }

}
Example 7
Source File: UimaSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    SentenceIterator iterator = UimaSentenceIterator.createWithPath("files/");
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }

}
Example 8
Source File: CollectionSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
    List<String> sentences = Arrays.asList(
            "No ,  he says now .",
            "And what did he do ?",
            "The money 's there .",
            "That was less than a year ago .",
            "But he made only the first .",
            "There 's still time for them to do it .",
            "But he should nt have .",
            " They have to come down to the people .",
            "I do nt know where that is .",
            "No , I would nt .",
            "Who Will It Be ?",
            "And no , I was not the one ."
    );
    SentenceIterator iterator = new CollectionSentenceIterator(sentences);
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
}
Example 9
Source File: FileSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
    SentenceIterator iterator = new FileSentenceIterator(new ClassPathResource("files/").getFile());
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }
}
Example 10
Source File: LineSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws IOException {
    SentenceIterator iterator = new LineSentenceIterator(new ClassPathResource("raw_sentences.txt").getFile());
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }

}
Example 11
Source File: UimaSentenceIteratorExample.java    From Java-Deep-Learning-Cookbook with MIT License 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    SentenceIterator iterator = UimaSentenceIterator.createWithPath("files/");
    int count=0;
    while(iterator.hasNext()){
        iterator.nextSentence();
        count++;
    }
    System.out.println("count = "+count);
    iterator.reset();
    SentenceDataPreProcessor.setPreprocessor(iterator);
    while(iterator.hasNext()){
        System.out.println(iterator.nextSentence());
    }

}
Example 12
Source File: Word2VecDataSetIteratorTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
protected LabelAwareSentenceIterator getLASI(final SentenceIterator iterator, final List<String> labels) {
    iterator.reset();

    return new LabelAwareSentenceIterator() {
        private AtomicInteger cnt = new AtomicInteger(0);

        @Override
        public String currentLabel() {
            return labels.get(cnt.incrementAndGet() % labels.size());
        }

        @Override
        public List<String> currentLabels() {
            return Collections.singletonList(currentLabel());
        }

        @Override
        public String nextSentence() {
            return iterator.nextSentence();
        }

        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public void reset() {
            iterator.reset();
        }

        @Override
        public void finish() {
            iterator.finish();
        }

        @Override
        public SentencePreProcessor getPreProcessor() {
            return iterator.getPreProcessor();
        }

        @Override
        public void setPreProcessor(SentencePreProcessor preProcessor) {
            iterator.setPreProcessor(preProcessor);
        }
    };
}