Java Code Examples for de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token#setPos()

@Test
public void testElevatedType() throws Exception {
    JCas jcas = JCasFactory.createJCas();
    
    DocumentMetaData.create(jcas).setDocumentId("doc");
    jcas.setDocumentText("John");
    
    // Add an elevated type which is not a direct subtype of Annotation. This type not be picked
    // up by the schema analyzer but should still be serialized as the POS type which is in fact
    // picked up.
    POS_NOUN pos = new POS_NOUN(jcas, 0, 4);
    pos.setPosValue("NN");
    pos.setCoarseValue("NOUN");
    pos.addToIndexes();
    
    Token t = new Token(jcas, 0, 4);
    t.setPos(pos);
    t.addToIndexes();
    new Sentence(jcas, 0, 4).addToIndexes();
            
    writeAndAssertEquals(jcas);
}
 

/**
 * Create {@link Token} in the {@link CAS}. If the lemma and pos columns are not empty it will
 * create {@link Lemma} and {@link POS} annotations
 */
private void createToken(JCas aJCas, StringBuilder text, Map<Integer, String> tokens,
        Map<Integer, String> pos, Map<Integer, String> lemma, Map<String, Token> tokensStored)
{
    int tokenBeginPosition = 0;
    int tokenEndPosition = 0;

    for (int i = 1; i <= tokens.size(); i++) {
        tokenBeginPosition = text.indexOf(tokens.get(i), tokenBeginPosition);
        Token outToken = new Token(aJCas, tokenBeginPosition, text.indexOf(tokens.get(i),
                tokenBeginPosition) + tokens.get(i).length());
        tokenEndPosition = text.indexOf(tokens.get(i), tokenBeginPosition)
                + tokens.get(i).length();
        tokenBeginPosition = tokenEndPosition;
        outToken.addToIndexes();

        // Add pos to CAS if exist
        if (!pos.get(i).equals("_")) {
            POS outPos = new POS(aJCas, outToken.getBegin(), outToken.getEnd());
            outPos.setPosValue(pos.get(i));
            outPos.addToIndexes();
            outToken.setPos(outPos);
        }

        // Add lemma if exist
        if (!lemma.get(i).equals("_")) {
            Lemma outLemma = new Lemma(aJCas, outToken.getBegin(), outToken.getEnd());
            outLemma.setValue(lemma.get(i));
            outLemma.addToIndexes();
            outToken.setLemma(outLemma);
        }
        tokensStored.put("t_" + i, outToken);
    }
}
 

@Test
public void testTokenAttachedAnnotationsWithValues() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    
    Token t1 = tokens.get(0);
    
    Lemma l1 = new Lemma(jcas, t1.getBegin(), t1.getEnd());
    l1.setValue("lemma1");
    l1.addToIndexes();
    t1.setLemma(l1);
    
    MorphologicalFeatures m1 = new MorphologicalFeatures(jcas, t1.getBegin(), t1.getEnd());
    m1.setValue("morph");
    m1.setTense("tense1");
    m1.addToIndexes();
    t1.setMorph(m1);
    
    POS p1 = new POS(jcas, t1.getBegin(), t1.getEnd());
    p1.setPosValue("pos1");
    p1.addToIndexes();
    t1.setPos(p1);
    
    Stem s1 = new Stem(jcas, t1.getBegin(), t1.getEnd());
    s1.setValue("stem1");
    s1.addToIndexes();
    t1.setStem(s1);
    
    writeAndAssertEquals(jcas, WebannoTsv3Writer.PARAM_SPAN_LAYERS,
            asList(MorphologicalFeatures.class, POS.class, Lemma.class, Stem.class));
}
 

@Test
public void testDependencyWithValues() throws Exception 
{
    JCas jcas = makeJCasOneSentence();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    Token t1 = tokens.get(0);
    Token t2 = tokens.get(1);
    
    POS p1 = new POS(jcas, t1.getBegin(), t1.getEnd());
    p1.setPosValue("POS1");
    p1.addToIndexes();
    t1.setPos(p1);

    POS p2 = new POS(jcas, t2.getBegin(), t2.getEnd());
    p2.setPosValue("POS2");
    p2.addToIndexes();
    t2.setPos(p2);
    
    Dependency dep1 = new Dependency(jcas);
    dep1.setGovernor(t1);
    dep1.setDependent(t2);
    // WebAnno legacy conventions
    // dep1.setBegin(min(dep1.getDependent().getBegin(), dep1.getGovernor().getBegin()));
    // dep1.setEnd(max(dep1.getDependent().getEnd(), dep1.getGovernor().getEnd()));
    // DKPro Core conventions
    dep1.setBegin(dep1.getDependent().getBegin());
    dep1.setEnd(dep1.getDependent().getEnd());
    dep1.addToIndexes();
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(POS.class),
            WebannoTsv3Writer.PARAM_RELATION_LAYERS, asList(Dependency.class));
}
 

@Test
public void thatRelationCrossSentenceBehaviorOnRenderGeneratesErrors() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter adapter = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(posAnnotations.size() - 1);

    depLayer.setCrossSentence(true);
    AnnotationFS dep = adapter.add(document, username, source, target, jcas.getCas());
    
    depLayer.setCrossSentence(false);
    RelationRenderer sut = new RelationRenderer(adapter, layerSupportRegistry,
            featureSupportRegistry, asList(new RelationCrossSentenceBehavior()));
    
    VDocument vdoc = new VDocument();
    sut.render(jcas.getCas(), asList(), vdoc, 0, jcas.getDocumentText().length());
    
    assertThat(vdoc.comments())
            .usingFieldByFieldElementComparator()
            .contains(new VComment(dep, ERROR, 
                    "Crossing sentence boundaries is not permitted."));
}
 

@Test
public void thatRelationAttachmentBehaviorOnCreateWorks() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(1);

    AnnotationFS dep = sut.add(document, username, source, target, jcas.getCas());

    assertThat(FSUtil.getFeature(dep, FEAT_REL_SOURCE, Token.class)).isEqualTo(tokens.get(0));
    assertThat(FSUtil.getFeature(dep, FEAT_REL_TARGET, Token.class)).isEqualTo(tokens.get(1));
}
 

@Test
public void thatRelationCrossSentenceBehaviorOnCreateThrowsException() throws Exception
{
    depLayer.setCrossSentence(false);
    
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(posAnnotations.size() - 1);

    assertThatExceptionOfType(MultipleSentenceCoveredException.class)
            .isThrownBy(() -> sut.add(document, username, source, target, jcas.getCas()))
            .withMessageContaining("multiple sentences");
}
 

@Test
public void thatRelationCrossSentenceBehaviorOnValidateGeneratesErrors() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(posAnnotations.size() - 1);

    depLayer.setCrossSentence(true);
    sut.add(document, username, source, target, jcas.getCas());
    
    depLayer.setCrossSentence(false);
    assertThat(sut.validate(jcas.getCas()))
            .extracting(Pair::getLeft)
            .usingElementComparatorIgnoringFields("source", "message")
            .containsExactly(LogMessage.error(null, ""));
}
 

@Test
public void thatCreatingRelationWorks() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(1);

    AnnotationFS dep1 = sut.add(document, username, source, target, jcas.getCas());
    
    assertThat(FSUtil.getFeature(dep1, FEAT_REL_SOURCE, Token.class)).isEqualTo(tokens.get(0));
    assertThat(FSUtil.getFeature(dep1, FEAT_REL_TARGET, Token.class)).isEqualTo(tokens.get(1));
}
 

@Test
public void testDependencyRelation() throws Exception
{
    // Set up document with a dummy dependency relation
    jcas.setDocumentText("a b");
    Token t1 = new Token(jcas, 0, 1);
    t1.addToIndexes();
    
    POS p1 = new POS(jcas, t1.getBegin(), t1.getEnd());
    p1.setPosValue("A");
    t1.setPos(p1);
    p1.addToIndexes();

    Token t2 = new Token(jcas, 2, 3);
    t2.addToIndexes();

    POS p2 = new POS(jcas, t2.getBegin(), t2.getEnd());
    p2.setPosValue("B");
    t2.setPos(p2);
    p2.addToIndexes();
    
    Dependency d1 = new Dependency(jcas, t2.getBegin(), t2.getEnd());
    d1.setDependent(t2);
    d1.setGovernor(t1);
    d1.addToIndexes();
    
    // Set up annotation schema with POS and Dependency
    AnnotationLayer tokenLayer = new AnnotationLayer(Token.class.getName(), "Token",
            SPAN_TYPE, project, true, SINGLE_TOKEN, NO_OVERLAP);
    tokenLayer.setId(1l);
    AnnotationFeature tokenLayerPos = new AnnotationFeature(1l, tokenLayer, "pos",
            POS.class.getName());
    
    AnnotationLayer posLayer = new AnnotationLayer(POS.class.getName(), "POS",
            SPAN_TYPE, project, true, SINGLE_TOKEN, NO_OVERLAP);
    posLayer.setId(2l);
    AnnotationFeature posLayerValue = new AnnotationFeature(1l, posLayer, "PosValue",
            CAS.TYPE_NAME_STRING);
    
    AnnotationLayer depLayer = new AnnotationLayer(Dependency.class.getName(),
            "Dependency", RELATION_TYPE, project, true, SINGLE_TOKEN, NO_OVERLAP);
    depLayer.setId(3l);
    depLayer.setAttachType(tokenLayer);
    depLayer.setAttachFeature(tokenLayerPos);
    AnnotationFeature dependencyLayerGovernor = new AnnotationFeature(2l, depLayer,
            "Governor", Token.class.getName());
    AnnotationFeature dependencyLayerDependent = new AnnotationFeature(3l, depLayer,
            "Dependent", Token.class.getName());
        
    when(annotationSchemaService.listAnnotationLayer(any(Project.class)))
            .thenReturn(asList(tokenLayer, posLayer, depLayer));

    when(annotationSchemaService.getAdapter(posLayer)).thenReturn(new SpanAdapter(
        layerSupportRegistry, featureSupportRegistry, null, posLayer, 
        () -> asList(posLayerValue), null));

    when(annotationSchemaService.getAdapter(depLayer))
            .thenReturn(new RelationAdapter(
                layerSupportRegistry, featureSupportRegistry, null, depLayer,
                FEAT_REL_TARGET, FEAT_REL_SOURCE,
                () -> asList(dependencyLayerGovernor, dependencyLayerDependent),
                emptyList()));

    MtasUimaParser sut = new MtasUimaParser(
            asList(tokenLayerPos, posLayerValue, dependencyLayerGovernor,
                    dependencyLayerDependent),
            annotationSchemaService, featureIndexingSupportRegistry);
    MtasTokenCollection tc = sut.createTokenCollection(jcas.getCas());
    
    MtasUtils.print(tc);
    
    List<MtasToken> tokens = new ArrayList<>();
    tc.iterator().forEachRemaining(tokens::add);

    assertThat(tokens)
        .filteredOn(t -> t.getPrefix().startsWith("Dependency"))
        .extracting(t -> t.getPrefix() + "=" + t.getPostfix())
        .containsExactly(
                "Dependency=b", 
                "Dependency-source=a", 
                "Dependency-source.PosValue=A",
                "Dependency-target=b", 
                "Dependency-target.PosValue=B");
}
 

@Test
public void thatRelationOverlapBehaviorOnCreateWorks() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(1);

    // First annotation should work
    depLayer.setOverlapMode(ANY_OVERLAP);
    sut.add(document, username, source, target, jcas.getCas());
    
    // Adding another annotation at the same place DOES NOT work
    depLayer.setOverlapMode(NO_OVERLAP);
    assertThatExceptionOfType(AnnotationException.class)
            .isThrownBy(() -> sut.add(document, username, source, target, jcas.getCas()))
            .withMessageContaining("no overlap or stacking");

    depLayer.setOverlapMode(OverlapMode.OVERLAP_ONLY);
    assertThatExceptionOfType(AnnotationException.class)
            .isThrownBy(() -> sut.add(document, username, source, target, jcas.getCas()))
            .withMessageContaining("stacking is not allowed");
    
    // Adding another annotation at the same place DOES work
    depLayer.setOverlapMode(OverlapMode.STACKING_ONLY);
    assertThatCode(() -> sut.add(document, username, source, target, jcas.getCas()))
            .doesNotThrowAnyException();

    depLayer.setOverlapMode(OverlapMode.ANY_OVERLAP);
    assertThatCode(() -> sut.add(document, username, source, target, jcas.getCas()))
            .doesNotThrowAnyException();
}
 

@Test
public void thatRelationOverlapBehaviorOnValidateGeneratesErrors() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(1);

    // Create two annotations stacked annotations
    depLayer.setOverlapMode(ANY_OVERLAP);
    sut.add(document, username, source, target, jcas.getCas());
    AnnotationFS rel2 = sut.add(document, username, source, target, jcas.getCas());
    
    depLayer.setOverlapMode(ANY_OVERLAP);
    assertThat(sut.validate(jcas.getCas()))
            .isEmpty();

    depLayer.setOverlapMode(STACKING_ONLY);
    assertThat(sut.validate(jcas.getCas()))
            .isEmpty();

    depLayer.setOverlapMode(OVERLAP_ONLY);
    assertThat(sut.validate(jcas.getCas()))
            .extracting(Pair::getLeft)
            .usingElementComparatorIgnoringFields("source")
            .containsExactly(
                    LogMessage.error(null, "Stacked relation at [5-7]"), 
                    LogMessage.error(null, "Stacked relation at [5-7]"));

    depLayer.setOverlapMode(NO_OVERLAP);
    assertThat(sut.validate(jcas.getCas()))
            .extracting(Pair::getLeft)
            .usingElementComparatorIgnoringFields("source")
            .containsExactly(
                    LogMessage.error(null, "Stacked relation at [5-7]"), 
                    LogMessage.error(null, "Stacked relation at [5-7]"));

    // Remove the stacked annotation and introduce one that is purely overlapping
    sut.delete(document, username, jcas.getCas(), new VID(rel2));
    depLayer.setOverlapMode(ANY_OVERLAP);
    sut.add(document, username, source, posAnnotations.get(2), jcas.getCas());
    
    depLayer.setOverlapMode(NO_OVERLAP);
    assertThat(sut.validate(jcas.getCas()))
            .extracting(Pair::getLeft)
            .usingElementComparatorIgnoringFields("source")
            .containsExactly(
                    LogMessage.error(null, "Overlapping relation at [5-7]"), 
                    LogMessage.error(null, "Overlapping relation at [8-9]"));
}
 

@Test
public void testSimplePath()
    throws Exception
{
    ConstraintsGrammar parser = new ConstraintsGrammar(new FileInputStream(
            "src/test/resources/rules/10.rules"));
    Parse p = parser.Parse();

    ParsedConstraints constraints = p.accept(new ParserVisitor());

    JCas jcas = JCasFactory.createJCas();
    jcas.setDocumentText("The sun.");

    // Add token annotations
    Token t_the = new Token(jcas, 0, 3);
    t_the.addToIndexes();
    Token t_sun = new Token(jcas, 0, 3);
    t_sun.addToIndexes();

    // Add POS annotations and link them to the tokens
    POS p_the = new POS(jcas, t_the.getBegin(), t_the.getEnd());
    p_the.setPosValue("DET");
    p_the.addToIndexes();
    t_the.setPos(p_the);
    POS p_sun = new POS(jcas, t_sun.getBegin(), t_sun.getEnd());
    p_sun.setPosValue("NN");
    p_sun.addToIndexes();
    t_sun.setPos(p_sun);

    // Add dependency annotations
    Dependency dep_the_sun = new Dependency(jcas);
    dep_the_sun.setGovernor(t_sun);
    dep_the_sun.setDependent(t_the);
    dep_the_sun.setDependencyType("det");
    dep_the_sun.setBegin(dep_the_sun.getGovernor().getBegin());
    dep_the_sun.setEnd(dep_the_sun.getGovernor().getEnd());
    dep_the_sun.addToIndexes();

    Evaluator constraintsEvaluator = new ValuesGenerator();

    List<PossibleValue> possibleValues = constraintsEvaluator.generatePossibleValues(
            dep_the_sun, "DependencyType", constraints);

    List<PossibleValue> expectedOutput = new LinkedList<>();
    expectedOutput.add(new PossibleValue("det", false));

    assertEquals(expectedOutput, possibleValues);
}