org.deeplearning4j.text.documentiterator.LabelAwareIterator Java Examples

@Test
public void testLabelAwareIterator_1() throws Exception {
    val resource = new ClassPathResource("/labeled");
    val file = resource.getFile();

    val iter = (LabelAwareIterator) new FileLabelAwareIterator.Builder().addSourceFolder(file).build();

    val t = new DefaultTokenizerFactory();

    val w2v = new Word2Vec.Builder()
            .iterate(iter)
            .tokenizerFactory(t)
            .build();

    // we hope nothing is going to happen here
}
 

public BasicTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer) {
    this.iterator = iterator;
    this.allowMultithreading = false;
    this.sentenceTransformer = transformer;

    this.iterator.reset();
}
 

public ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer,
                                   boolean allowMultithreading) {
    super(new AsyncLabelAwareIterator(iterator, 512), transformer);
    //super(iterator, transformer);
    this.allowMultithreading = allowMultithreading;
    //this.stringBuffer = new LinkedBlockingQueue<>(512);

    //threads = new TokenizerThread[1];
    //threads = new TokenizerThread[allowMultithreading ? Math.max(Runtime.getRuntime().availableProcessors(), 2) : 1];
    executorService = Executors.newFixedThreadPool(allowMultithreading ? Math.max(Runtime.getRuntime().availableProcessors(), 2) : 1);

    prefetchIterator();
}
 

public void testTfIdfVectorizerFromLabelAwareIterator() throws Exception {
    LabelledDocument doc1 = new LabelledDocument();
    doc1.addLabel("dog");
    doc1.setContent("it barks like a dog");

    LabelledDocument doc2 = new LabelledDocument();
    doc2.addLabel("cat");
    doc2.setContent("it meows like a cat");

    List<LabelledDocument> docs = new ArrayList<>(2);
    docs.add(doc1);
    docs.add(doc2);
    
    LabelAwareIterator iterator = new SimpleLabelAwareIterator(docs);
    TokenizerFactory tokenizerFactory = new DefaultTokenizerFactory();

    TfidfVectorizer vectorizer = new TfidfVectorizer
        .Builder()
        .setMinWordFrequency(1)
        .setStopWords(new ArrayList<String>())
        .setTokenizerFactory(tokenizerFactory)
        .setIterator(iterator)
        .allowParallelTokenization(false)
        .build();

    vectorizer.fit();

    DataSet dataset = vectorizer.vectorize("it meows like a cat", "cat");
    assertNotNull(dataset);
    
    LabelsSource source = vectorizer.getLabelsSource();
    assertEquals(2, source.getNumberOfLabelsUsed());
    List<String> labels = source.getLabels();
    assertEquals("dog", labels.get(0));
    assertEquals("cat", labels.get(1));
}
 

private SentenceTransformer(@NonNull LabelAwareIterator iterator) {
    this.iterator = iterator;
}
 

public ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer) {
    this(iterator, transformer, true);
}
 

public Builder setIterator(@NonNull LabelAwareIterator iterator) {
    this.iterator = new LabelAwareIteratorWrapper(iterator, labelsSource);
    return this;
}
 

public Builder setIterator(@NonNull LabelAwareIterator iterator) {
    this.iterator = iterator;
    return this;
}
 

public LabelAwareConverter(@NonNull LabelAwareIterator iterator, @NonNull List<String> labels) {
    this.backingIterator = iterator;
    this.labels = labels;
}
 

/**
 * Specify how the (labelled) sentences / documents should be provided
 */
public Builder sentenceProvider(LabelAwareIterator iterator, @NonNull List<String> labels) {
    LabelAwareConverter converter = new LabelAwareConverter(iterator, labels);
    return sentenceProvider(converter);
}
 

/**
 * This test is not indicative.
 * there's no need in this test within travis, use it manually only for problems detection
 *
 * @throws Exception
 */
@Test
@Ignore
public void testParagraphVectorsReducedLabels1() throws Exception {
    val tempDir = testDir.newFolder();
    ClassPathResource resource = new ClassPathResource("/labeled");
    resource.copyDirectory(tempDir);

    LabelAwareIterator iter = new FileLabelAwareIterator.Builder().addSourceFolder(tempDir).build();

    TokenizerFactory t = new DefaultTokenizerFactory();

    /**
     * Please note: text corpus is REALLY small, and some kind of "results" could be received with HIGH epochs number, like 30.
     * But there's no reason to keep at that high
     */

    ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).epochs(3).layerSize(100)
                    .stopWords(new ArrayList<String>()).windowSize(5).iterate(iter).tokenizerFactory(t).build();

    vec.fit();

    //WordVectorSerializer.writeWordVectors(vec, "vectors.txt");

    INDArray w1 = vec.lookupTable().vector("I");
    INDArray w2 = vec.lookupTable().vector("am");
    INDArray w3 = vec.lookupTable().vector("sad.");

    INDArray words = Nd4j.create(3, vec.lookupTable().layerSize());

    words.putRow(0, w1);
    words.putRow(1, w2);
    words.putRow(2, w3);


    INDArray mean = words.isMatrix() ? words.mean(0) : words;

    log.info("Mean" + Arrays.toString(mean.dup().data().asDouble()));
    log.info("Array" + Arrays.toString(vec.lookupTable().vector("negative").dup().data().asDouble()));

    double simN = Transforms.cosineSim(mean, vec.lookupTable().vector("negative"));
    log.info("Similarity negative: " + simN);


    double simP = Transforms.cosineSim(mean, vec.lookupTable().vector("neutral"));
    log.info("Similarity neutral: " + simP);

    double simV = Transforms.cosineSim(mean, vec.lookupTable().vector("positive"));
    log.info("Similarity positive: " + simV);
}
 

/**
 * This method used to feed LabelAwareIterator, that is usually used
 *
 * @param iterator
 * @return
 */
public Builder iterate(@NonNull LabelAwareIterator iterator) {
    this.labelAwareIterator = iterator;
    return this;
}