Java Code Examples for de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token#getEnd()

/**
 * Returns true, if the argument component annotation ends at this token
 *
 * @param t    token
 * @param jCas jcas
 * @return boolean
 */
public static boolean argAnnotationEnds(Token t, JCas jCas)
{
    List<ArgumentComponent> argumentAnnotations = new ArrayList<>();

    argumentAnnotations
            .addAll(JCasUtil.selectCovering(jCas, Claim.class, t.getBegin(), t.getEnd()));
    argumentAnnotations
            .addAll(JCasUtil.selectCovering(jCas, Backing.class, t.getBegin(), t.getEnd()));
    argumentAnnotations
            .addAll(JCasUtil.selectCovering(jCas, Premise.class, t.getBegin(), t.getEnd()));
    argumentAnnotations
            .addAll(JCasUtil.selectCovering(jCas, Rebuttal.class, t.getBegin(), t.getEnd()));
    argumentAnnotations
            .addAll(JCasUtil.selectCovering(jCas, Refutation.class, t.getBegin(), t.getEnd()));

    return !argumentAnnotations.isEmpty() && argumentAnnotations.get(0).getEnd() == t.getEnd();
}
 

@Test
public void testTokenBoundedSpanWithFeatureValue() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    
    int n = 0;
    for (Token t : select(jcas, Token.class)) {
        Span ne = new Span(jcas, t.getBegin(), t.getEnd());
        ne.setValue("NE " + n);
        ne.addToIndexes();
        n++;
    }
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class));
}
 

private void setTokenSentenceAddress(JCas aJCas)
{
    int sentNMumber = 1;
    for (Sentence sentence : select(aJCas, Sentence.class)) {
        int lineNumber = 1;
        for (Token token : selectCovered(Token.class, sentence)) {
            AnnotationUnit unit = new AnnotationUnit(token.getBegin(), token.getEnd(), false,
                    token.getCoveredText());
            units.add(unit);
            if (lineNumber == 1) {
                sentenceUnits.put(unit, sentence.getCoveredText());
            }
            unitsLineNumber.put(unit, sentNMumber + "-" + lineNumber);
            lineNumber++;
        }
        sentNMumber++;
    }
}
 

@Test
public void testTokenBoundedStackedLookAlike() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    
    int n = 0;
    for (Token t : select(jcas, Token.class)) {
        Span ne = new Span(jcas, t.getBegin(), t.getEnd());
        ne.setValue("NOTSTACKED[" + n + "]");
        ne.addToIndexes();
        n++;
    }
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class));
}
 

@Test
public void testSingleStackedNonTokenRelationWithoutFeatureValue2() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    CAS cas = jcas.getCas();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    
    Token t1 = tokens.get(0);
    Token t2 = tokens.get(tokens.size() - 1);
    
    Span gov = new Span(jcas, t1.getBegin(), t1.getEnd());
    gov.addToIndexes();

    Span dep =  new Span(jcas, t2.getBegin(), t2.getEnd());
    dep.addToIndexes();
    new Span(jcas, t2.getBegin(), t2.getEnd()).addToIndexes();

    Type relationType = cas.getTypeSystem().getType("webanno.custom.Relation");
    
    // One at the beginning
    // WebAnno legacy conventions
    // AnnotationFS fs1 = cas.createAnnotation(relationType, 
    //         min(dep.getBegin(), gov.getBegin()),
    //         max(dep.getEnd(), gov.getEnd()));
    // DKPro Core conventions
    AnnotationFS fs1 = cas.createAnnotation(relationType, dep.getBegin(), dep.getEnd());
    FSUtil.setFeature(fs1, "Governor", gov);
    FSUtil.setFeature(fs1, "Dependent", dep);
    cas.addFsToIndexes(fs1);
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class),
            WebannoTsv3Writer.PARAM_RELATION_LAYERS, asList("webanno.custom.Relation"));
}
 

@Test
public void thatRelationAttachmentBehaviorOnCreateWorks() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(1);

    AnnotationFS dep = sut.add(document, username, source, target, jcas.getCas());

    assertThat(FSUtil.getFeature(dep, FEAT_REL_SOURCE, Token.class)).isEqualTo(tokens.get(0));
    assertThat(FSUtil.getFeature(dep, FEAT_REL_TARGET, Token.class)).isEqualTo(tokens.get(1));
}
 

@Test
public void testTokenBoundedSpanWithAsteriskFeatureValue() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    
    for (Token t : select(jcas, Token.class)) {
        Span ne = new Span(jcas, t.getBegin(), t.getEnd());
        ne.setValue("*");
        ne.addToIndexes();
    }
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class));
}
 

@Test
public void testTokenBoundedSpanWithUnderscoreFeatureValue() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    
    for (Token t : select(jcas, Token.class)) {
        Span ne = new Span(jcas, t.getBegin(), t.getEnd());
        ne.setValue("_");
        ne.addToIndexes();
    }
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class));
}
 

@Test
public void thatRelationCrossSentenceBehaviorOnValidateGeneratesErrors() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter sut = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(posAnnotations.size() - 1);

    depLayer.setCrossSentence(true);
    sut.add(document, username, source, target, jcas.getCas());
    
    depLayer.setCrossSentence(false);
    assertThat(sut.validate(jcas.getCas()))
            .extracting(Pair::getLeft)
            .usingElementComparatorIgnoringFields("source", "message")
            .containsExactly(LogMessage.error(null, ""));
}
 

@Test
public void testTokenBoundedSpanWithSpecialSymbolsValue() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    
    for (Token t : select(jcas, Token.class)) {
        Span ne = new Span(jcas, t.getBegin(), t.getEnd());
        ne.setValue("#*'\"`´\t:;{}|[ ]()\\§$%?=&_\n");
        ne.addToIndexes();
    }
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class));
}
 

@Test
public void testSingleNonTokenRelationWithoutFeature() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    CAS cas = jcas.getCas();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    
    Token t1 = tokens.get(0);
    Token t2 = tokens.get(tokens.size() - 1);
    
    Span gov = new Span(jcas, t1.getBegin(), t1.getEnd());
    gov.addToIndexes();
    Span dep =  new Span(jcas, t2.getBegin(), t2.getEnd());
    dep.addToIndexes();

    Type relationType = cas.getTypeSystem().getType("webanno.custom.SimpleRelation");
    
    // One at the beginning
    // WebAnno legacy conventions
    // AnnotationFS fs1 = cas.createAnnotation(relationType, 
    //         min(dep.getBegin(), gov.getBegin()),
    //         max(dep.getEnd(), gov.getEnd()));
    // DKPro Core conventions
    AnnotationFS fs1 = cas.createAnnotation(relationType, dep.getBegin(), dep.getEnd());
    FSUtil.setFeature(fs1, "Governor", gov);
    FSUtil.setFeature(fs1, "Dependent", dep);
    cas.addFsToIndexes(fs1);
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class),
            WebannoTsv3Writer.PARAM_RELATION_LAYERS, asList("webanno.custom.SimpleRelation"));
}
 

@Test
public void testDependencyWithValues() throws Exception 
{
    JCas jcas = makeJCasOneSentence();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    Token t1 = tokens.get(0);
    Token t2 = tokens.get(1);
    
    POS p1 = new POS(jcas, t1.getBegin(), t1.getEnd());
    p1.setPosValue("POS1");
    p1.addToIndexes();
    t1.setPos(p1);

    POS p2 = new POS(jcas, t2.getBegin(), t2.getEnd());
    p2.setPosValue("POS2");
    p2.addToIndexes();
    t2.setPos(p2);
    
    Dependency dep1 = new Dependency(jcas);
    dep1.setGovernor(t1);
    dep1.setDependent(t2);
    // WebAnno legacy conventions
    // dep1.setBegin(min(dep1.getDependent().getBegin(), dep1.getGovernor().getBegin()));
    // dep1.setEnd(max(dep1.getDependent().getEnd(), dep1.getGovernor().getEnd()));
    // DKPro Core conventions
    dep1.setBegin(dep1.getDependent().getBegin());
    dep1.setEnd(dep1.getDependent().getEnd());
    dep1.addToIndexes();
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(POS.class),
            WebannoTsv3Writer.PARAM_RELATION_LAYERS, asList(Dependency.class));
}
 

@Test
public void thatRelationCrossSentenceBehaviorOnRenderGeneratesErrors() throws Exception
{
    TokenBuilder<Token, Sentence> builder = new TokenBuilder<>(Token.class, Sentence.class);
    builder.buildTokens(jcas, "This is a test .\nThis is sentence two .");

    for (Token t : select(jcas, Token.class)) {
        POS pos = new POS(jcas, t.getBegin(), t.getEnd());
        t.setPos(pos);
        pos.addToIndexes();
    }

    RelationAdapter adapter = new RelationAdapter(layerSupportRegistry, featureSupportRegistry,
        null, depLayer, FEAT_REL_TARGET, FEAT_REL_SOURCE,
        () -> asList(dependencyLayerGovernor, dependencyLayerDependent), behaviors);

    List<POS> posAnnotations = new ArrayList<>(select(jcas, POS.class));

    POS source = posAnnotations.get(0);
    POS target = posAnnotations.get(posAnnotations.size() - 1);

    depLayer.setCrossSentence(true);
    AnnotationFS dep = adapter.add(document, username, source, target, jcas.getCas());
    
    depLayer.setCrossSentence(false);
    RelationRenderer sut = new RelationRenderer(adapter, layerSupportRegistry,
            featureSupportRegistry, asList(new RelationCrossSentenceBehavior()));
    
    VDocument vdoc = new VDocument();
    sut.render(jcas.getCas(), asList(), vdoc, 0, jcas.getDocumentText().length());
    
    assertThat(vdoc.comments())
            .usingFieldByFieldElementComparator()
            .contains(new VComment(dep, ERROR, 
                    "Crossing sentence boundaries is not permitted."));
}
 

private void addCountryDictionaryMatch(JCas jCas, List<Token> currentTokenSequence) {
    for (Token token : currentTokenSequence) {
        CountryDictionaryMatch match = new CountryDictionaryMatch(jCas,
                token.getBegin(), token.getEnd());
        match.addToIndexes();
        logger.trace("Country match added for token: " + token.getCoveredText() +
                " from sequence: " + tokenSequenceToStringLowercase(currentTokenSequence));
    }

}
 

private void addYagoDictionaryMatch(JCas jCas, List<Token> currentTokenSequence) {
    for (Token token : currentTokenSequence) {
        YagoDictionaryMatch match = new YagoDictionaryMatch(jCas,
                token.getBegin(), token.getEnd());
        match.addToIndexes();
        logger.trace("Yago match added for token: " + token.getCoveredText() +
                " from sequence: " + tokenSequenceToString(currentTokenSequence));
    }

}
 

@Test
public void testSingleStackedNonTokenRelationWithoutFeatureValue() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    CAS cas = jcas.getCas();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    
    Token t1 = tokens.get(0);
    Token t2 = tokens.get(tokens.size() - 1);
    
    Span gov = new Span(jcas, t1.getBegin(), t1.getEnd());
    gov.addToIndexes();
    new Span(jcas, t1.getBegin(), t1.getEnd()).addToIndexes();

    Span dep =  new Span(jcas, t2.getBegin(), t2.getEnd());
    dep.addToIndexes();
    new Span(jcas, t2.getBegin(), t2.getEnd()).addToIndexes();

    Type relationType = cas.getTypeSystem().getType("webanno.custom.Relation");
    
    // One at the beginning
    // WebAnno legacy conventions
    // AnnotationFS fs1 = cas.createAnnotation(relationType, 
    //         min(dep.getBegin(), gov.getBegin()),
    //         max(dep.getEnd(), gov.getEnd()));
    // DKPro Core conventions
    AnnotationFS fs1 = cas.createAnnotation(relationType, dep.getBegin(), dep.getEnd());
    FSUtil.setFeature(fs1, "Governor", gov);
    FSUtil.setFeature(fs1, "Dependent", dep);
    cas.addFsToIndexes(fs1);
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class),
            WebannoTsv3Writer.PARAM_RELATION_LAYERS, asList("webanno.custom.Relation"));
}
 

@Test
public void testSingleNonTokenRelationWithoutFeatureValue() throws Exception
{
    JCas jcas = makeJCasOneSentence();
    CAS cas = jcas.getCas();
    
    List<Token> tokens = new ArrayList<>(select(jcas, Token.class));
    
    Token t1 = tokens.get(0);
    Token t2 = tokens.get(tokens.size() - 1);
    
    Span gov = new Span(jcas, t1.getBegin(), t1.getEnd());
    gov.addToIndexes();
    Span dep =  new Span(jcas, t2.getBegin(), t2.getEnd());
    dep.addToIndexes();

    Type relationType = cas.getTypeSystem().getType("webanno.custom.Relation");
    
    // One at the beginning
    // WebAnno legacy conventions
    // AnnotationFS fs1 = cas.createAnnotation(relationType, 
    //        min(dep.getBegin(), gov.getBegin()),
    //        max(dep.getEnd(), gov.getEnd()));
    // DKPro Core conventions
    AnnotationFS fs1 = cas.createAnnotation(relationType, dep.getBegin(), dep.getEnd());
    FSUtil.setFeature(fs1, "Governor", gov);
    FSUtil.setFeature(fs1, "Dependent", dep);
    cas.addFsToIndexes(fs1);
    
    writeAndAssertEquals(jcas, 
            WebannoTsv3Writer.PARAM_SPAN_LAYERS, asList(Span.class),
            WebannoTsv3Writer.PARAM_RELATION_LAYERS, asList("webanno.custom.Relation"));
}
 

@Override
public void getNext(CAS cas) throws IOException, CollectionException {
	reader.getNext(cas);

	JCas jcas;
       try {
           jcas = cas.getJCas();
       }
       catch (CASException e) {
           throw new CollectionException(e);
       }

	for(Sentence sentence : JCasUtil.select(jcas, Sentence.class)) {
		TextClassificationSequence sequence = new TextClassificationSequence(jcas, sentence.getBegin(),
				sentence.getEnd());
		sequence.addToIndexes();

		for(Token token : JCasUtil.selectCovered(jcas, Token.class, sentence)){
			TextClassificationTarget unit = new TextClassificationTarget(jcas, token.getBegin(), token.getEnd());
			unit.setSuffix(token.getCoveredText());
			unit.addToIndexes();

			TextClassificationOutcome outcome = new TextClassificationOutcome(jcas, token.getBegin(), token.getEnd());
			outcome.setOutcome(getTextClassificationOutcome(jcas, unit));
			outcome.addToIndexes();
		}

	}
}
 

private void setTokenAnnos(CAS aCas, Map<Integer, String> aTokenAnnoMap, Type aType,
        Feature aFeature)
{
    LowLevelCAS llCas = aCas.getLowLevelCAS();
    for (AnnotationFS annoFs : CasUtil.select(aCas, aType)) {
        boolean first = true;
        boolean previous = false; // exists previous annotation, place-holed O-_ should be kept
        for (Token token : selectCovered(Token.class, annoFs)) {
            if (annoFs.getBegin() <= token.getBegin() && annoFs.getEnd() >= token.getEnd()) {
                String annotation = annoFs.getFeatureValueAsString(aFeature);
                if (annotation == null) {
                    annotation = aType.getName() + "_";
                }
                if (aTokenAnnoMap.get(llCas.ll_getFSRef(token)) == null) {
                    if (previous) {
                        if (!multipleSpans.contains(aType.getName())) {
                            aTokenAnnoMap.put(llCas.ll_getFSRef(token), annotation);
                        }
                        else {
                            aTokenAnnoMap.put(llCas.ll_getFSRef(token), "O-_|"
                                    + (first ? "B-" : "I-") + annotation);
                            first = false;
                        }
                    }
                    else {
                        if (!multipleSpans.contains(aType.getName())) {
                            aTokenAnnoMap.put(llCas.ll_getFSRef(token), annotation);
                        }
                        else {
                            aTokenAnnoMap.put(llCas.ll_getFSRef(token), (first ? "B-" : "I-")
                                    + annotation);
                            first = false;
                        }
                    }
                }
                else {
                    if (!multipleSpans.contains(aType.getName())) {
                        aTokenAnnoMap.put(llCas.ll_getFSRef(token),
                                aTokenAnnoMap.get(llCas.ll_getFSRef(token)) + "|"
                                        + annotation);
                        previous = true;
                    }
                    else {
                        aTokenAnnoMap.put(llCas.ll_getFSRef(token),
                                aTokenAnnoMap.get(llCas.ll_getFSRef(token)) + "|"
                                        + (first ? "B-" : "I-") + annotation);
                        first = false;
                        previous = true;
                    }
                }

            }
        }
    }
}
 

@Test
public void testDependencyRelation() throws Exception
{
    // Set up document with a dummy dependency relation
    jcas.setDocumentText("a b");
    Token t1 = new Token(jcas, 0, 1);
    t1.addToIndexes();
    
    POS p1 = new POS(jcas, t1.getBegin(), t1.getEnd());
    p1.setPosValue("A");
    t1.setPos(p1);
    p1.addToIndexes();

    Token t2 = new Token(jcas, 2, 3);
    t2.addToIndexes();

    POS p2 = new POS(jcas, t2.getBegin(), t2.getEnd());
    p2.setPosValue("B");
    t2.setPos(p2);
    p2.addToIndexes();
    
    Dependency d1 = new Dependency(jcas, t2.getBegin(), t2.getEnd());
    d1.setDependent(t2);
    d1.setGovernor(t1);
    d1.addToIndexes();
    
    // Set up annotation schema with POS and Dependency
    AnnotationLayer tokenLayer = new AnnotationLayer(Token.class.getName(), "Token",
            SPAN_TYPE, project, true, SINGLE_TOKEN, NO_OVERLAP);
    tokenLayer.setId(1l);
    AnnotationFeature tokenLayerPos = new AnnotationFeature(1l, tokenLayer, "pos",
            POS.class.getName());
    
    AnnotationLayer posLayer = new AnnotationLayer(POS.class.getName(), "POS",
            SPAN_TYPE, project, true, SINGLE_TOKEN, NO_OVERLAP);
    posLayer.setId(2l);
    AnnotationFeature posLayerValue = new AnnotationFeature(1l, posLayer, "PosValue",
            CAS.TYPE_NAME_STRING);
    
    AnnotationLayer depLayer = new AnnotationLayer(Dependency.class.getName(),
            "Dependency", RELATION_TYPE, project, true, SINGLE_TOKEN, NO_OVERLAP);
    depLayer.setId(3l);
    depLayer.setAttachType(tokenLayer);
    depLayer.setAttachFeature(tokenLayerPos);
    AnnotationFeature dependencyLayerGovernor = new AnnotationFeature(2l, depLayer,
            "Governor", Token.class.getName());
    AnnotationFeature dependencyLayerDependent = new AnnotationFeature(3l, depLayer,
            "Dependent", Token.class.getName());
        
    when(annotationSchemaService.listAnnotationLayer(any(Project.class)))
            .thenReturn(asList(tokenLayer, posLayer, depLayer));

    when(annotationSchemaService.getAdapter(posLayer)).thenReturn(new SpanAdapter(
        layerSupportRegistry, featureSupportRegistry, null, posLayer, 
        () -> asList(posLayerValue), null));

    when(annotationSchemaService.getAdapter(depLayer))
            .thenReturn(new RelationAdapter(
                layerSupportRegistry, featureSupportRegistry, null, depLayer,
                FEAT_REL_TARGET, FEAT_REL_SOURCE,
                () -> asList(dependencyLayerGovernor, dependencyLayerDependent),
                emptyList()));

    MtasUimaParser sut = new MtasUimaParser(
            asList(tokenLayerPos, posLayerValue, dependencyLayerGovernor,
                    dependencyLayerDependent),
            annotationSchemaService, featureIndexingSupportRegistry);
    MtasTokenCollection tc = sut.createTokenCollection(jcas.getCas());
    
    MtasUtils.print(tc);
    
    List<MtasToken> tokens = new ArrayList<>();
    tc.iterator().forEachRemaining(tokens::add);

    assertThat(tokens)
        .filteredOn(t -> t.getPrefix().startsWith("Dependency"))
        .extracting(t -> t.getPrefix() + "=" + t.getPostfix())
        .containsExactly(
                "Dependency=b", 
                "Dependency-source=a", 
                "Dependency-source.PosValue=A",
                "Dependency-target=b", 
                "Dependency-target.PosValue=B");
}