org.apache.uima.collection.CollectionReader Java Examples

public static CAS readXMI(String aPath, TypeSystemDescription aType)
    throws UIMAException, IOException
{
    CollectionReader reader = createReader(XmiReader.class, XmiReader.PARAM_SOURCE_LOCATION,
            "src/test/resources/" + aPath);
    CAS jcas;
    if (aType != null) {
        TypeSystemDescription builtInTypes = TypeSystemDescriptionFactory
                .createTypeSystemDescription();
        List<TypeSystemDescription> allTypes = new ArrayList<>();
        allTypes.add(builtInTypes);
        allTypes.add(aType);
        jcas = JCasFactory.createJCas(CasCreationUtils.mergeTypeSystems(allTypes)).getCas();
    }
    else {
        jcas = JCasFactory.createJCas().getCas();
    }

    reader.getNext(jcas);

    return jcas;
}
 

@Test
   public void test() throws Exception {

CollectionReader cr = CollectionReaderFactory.createReader(
	BioNLPGeniaEventsCollectionReader.class);

int i = 0;
while (cr.hasNext()) {
    CAS cas = CasCreationUtils.createCas(cr
	    .getProcessingResourceMetaData());
    cr.getNext(cas);

    // if (createHtml)
    // viewer.createHtml(cas.getJCas(), cas.getTypeSystem(),
    // styleMapFile, new File("target/" + i));

    i++;
}
cr.close();
assertEquals(259, i);

   }
 

@Test
public void test()
    throws Exception
{
    JCas doc = JCasFactory.createJCas();

    CollectionReader reader = createReader(LineOrientedTextReader.class,
            LineOrientedTextReader.PARAM_SOURCE_LOCATION, "LICENSE.txt");

    reader.getNext(doc.getCas());
    
    // select(doc, Sentence.class).forEach(s -> System.out.println(s.getCoveredText()));
    
    assertEquals(169, select(doc, Sentence.class).size());
    assertEquals(0, select(doc, Token.class).size());
}
 

/**
 * Constructor
 *
 * @param name Pipeline name
 * @param originalYaml The original YAML string that was used to build the pipeline
 * @param orderer The IPipelineOrderer to use to order the pipeline
 * @param collectionReader The collection reader
 * @param annotators The annotators to be ordered and used
 * @param consumers The consumers to be ordered and used
 * @throws IOException if error reading config
 * @deprecated Use {@link BaleenPipeline#BaleenPipeline(String, PipelineConfiguration,
 *     IPipelineOrderer, CollectionReader, List, List)}
 */
@Deprecated
public BaleenPipeline(
    String name,
    String originalYaml,
    IPipelineOrderer orderer,
    CollectionReader collectionReader,
    List<AnalysisEngine> annotators,
    List<AnalysisEngine> consumers)
    throws IOException {
  this(
      name,
      new YamlPipelineConfiguration(originalYaml),
      orderer,
      collectionReader,
      annotators,
      consumers);
}
 

@Test
public void testCount() throws Exception {

    CollectionReader cr = CollectionReaderFactory.createReader(
            BioNLPGeniaEventsCollectionReader.class,
            BlueUima.PARAM_INPUT_DIRECTORY, TEST_DIR);

    int i = 0;
    while (cr.hasNext()) {
        CAS cas = CasCreationUtils.createCas(cr
                .getProcessingResourceMetaData());
        cr.getNext(cas);
        LOG.debug(To.string("cas nr " + i, cas.getJCas()));
        i++;
    }
    cr.close();
    assertEquals(3, i);
}
 

@SuppressWarnings("unchecked")
@Override
public CollectionReaderDescription getReaderDescription(TypeSystemDescription aTSD)
    throws ResourceInitializationException
{
    if (!isReadable()) {
        throw new UnsupportedOperationException("The format [" + getName() + "] cannot be read");
    }
    
    Class<? extends CollectionReader> readerClazz;
    try {
        readerClazz = (Class<? extends CollectionReader>) Class.forName(readerClass);
    }
    catch (ClassNotFoundException e) {
        throw new ResourceInitializationException(e);
    }
    
    return createReaderDescription(readerClazz, aTSD);
}
 

@Test
public void testAuthors() throws Exception {

    // http://www.ncbi.nlm.nih.gov/pubmed/?term=1&report=xml&format=text
    CollectionReader cr = createReader(PubmedDatabaseCR.class,
            BlueUima.PARAM_BETWEEN, new int[] { 0, 1 },
            BlueUima.PARAM_SKIP_EMPTY_DOCS, false);

    String[] lastNames = { "Makar", "McMartin", "Palese", "Tephly" };
    String[] foreNames = { "A B", "K E", "M", "T R" };
    // AB___A B___Makar__-__KE___K
    // E___McMartin__-__M___M___Palese__-__TR___T R___Tephly
    for (JCas jCas : asList(cr)) {
        Header header = JCasUtil.selectSingle(jCas, Header.class);

        FSArray authors = header.getAuthors();
        for (int i = 0; i < authors.size(); i++) {
            AuthorInfo a = (AuthorInfo) authors.get(i);
            assertEquals(foreNames[i], a.getForeName());
            assertEquals(lastNames[i], a.getLastName());
        }

        assertEquals("1976-01-16", header.getCopyright());
    }
}
 

private List<CAS> loadData(Dataset ds, File ... files) throws UIMAException, IOException
{
    CollectionReader reader = createReader(
        Conll2002Reader.class,
        Conll2002Reader.PARAM_PATTERNS, files, 
        Conll2002Reader.PARAM_LANGUAGE, ds.getLanguage(), 
        Conll2002Reader.PARAM_COLUMN_SEPARATOR, Conll2002Reader.ColumnSeparators.TAB.getName(),
        Conll2002Reader.PARAM_HAS_TOKEN_NUMBER, true, 
        Conll2002Reader.PARAM_HAS_HEADER, true, 
        Conll2002Reader.PARAM_HAS_EMBEDDED_NAMED_ENTITY, true);

    List<CAS> casList = new ArrayList<>();
    while (reader.hasNext()) {
        JCas cas = JCasFactory.createJCas();
        reader.getNext(cas.getCas());
        casList.add(cas.getCas());
    }
    return casList;
}
 

private JCas loadNerDevelopmentData() throws IOException, UIMAException
{
    Dataset ds = loader.load("germeval2014-de", CONTINUE);
    
    CollectionReader reader = createReader(Conll2002Reader.class,
        Conll2002Reader.PARAM_PATTERNS, ds.getDefaultSplit().getDevelopmentFiles(), 
        Conll2002Reader.PARAM_LANGUAGE, ds.getLanguage(), 
        Conll2002Reader.PARAM_COLUMN_SEPARATOR, Conll2002Reader.ColumnSeparators.TAB.getName(),
        Conll2002Reader.PARAM_HAS_TOKEN_NUMBER, true, 
        Conll2002Reader.PARAM_HAS_HEADER, true, 
        Conll2002Reader.PARAM_HAS_EMBEDDED_NAMED_ENTITY, true);
    
    JCas cas = JCasFactory.createJCas();
    reader.getNext(cas.getCas());
    return cas;
}
 

@Test
public void testSerializeDeserializeXmi() throws Exception {

    // serialize
    CollectionReader cr = createReader(TextArrayReader.class,
             PARAM_INPUT, new String[] { "this is a test" });
    AnalysisEngine serializer = createEngine(XWriter.class,
            PARAM_OUTPUT_DIR, testDir + "/serdeser");
    runPipeline(cr, serializer);

    // deserialize
    cr = createReader(XCollectionReader.class, 
            PARAM_INPUT_DIRECTORY, testDir + "/serdeser");
    List<JCas> deserializedCases = asList(cr);
    assertEquals(1, deserializedCases.size());

    assertEquals("this is a test", deserializedCases.get(0)
            .getDocumentText());
}
 

private List<CAS> loadData(Dataset ds, File ... files) throws UIMAException, IOException
{
    CollectionReader reader = createReader(Conll2002Reader.class,
        Conll2002Reader.PARAM_PATTERNS, files,
        Conll2002Reader.PARAM_LANGUAGE, ds.getLanguage(),
        Conll2002Reader.PARAM_COLUMN_SEPARATOR, Conll2002Reader.ColumnSeparators.TAB.getName(),
        Conll2002Reader.PARAM_HAS_TOKEN_NUMBER, true,
        Conll2002Reader.PARAM_HAS_HEADER, true,
        Conll2002Reader.PARAM_HAS_EMBEDDED_NAMED_ENTITY, true);

    List<CAS> casList = new ArrayList<>();
    while (reader.hasNext()) {
        // Add the CasMetadata type to the CAS
        List<TypeSystemDescription> typeSystems = new ArrayList<>();
        typeSystems.add(createTypeSystemDescription());
        typeSystems.add(CasMetadataUtils.getInternalTypeSystem());
        JCas cas = JCasFactory.createJCas(mergeTypeSystems(typeSystems));
        reader.getNext(cas.getCas());
        casList.add(cas.getCas());
    }
    return casList;
}
 

public static JCas readWebAnnoTSV(String aPath, TypeSystemDescription aType)
    throws UIMAException, IOException
{
    CollectionReader reader = createReader(WebannoTsv2Reader.class,
            WebannoTsv2Reader.PARAM_SOURCE_LOCATION, "src/test/resources/" + aPath);
    JCas jcas;
    if (aType != null) {
        TypeSystemDescription builtInTypes = TypeSystemDescriptionFactory
                .createTypeSystemDescription();
        List<TypeSystemDescription> allTypes = new ArrayList<>();
        allTypes.add(builtInTypes);
        allTypes.add(aType);
        jcas = JCasFactory.createJCas(CasCreationUtils.mergeTypeSystems(allTypes));
    }
    else {
        jcas = JCasFactory.createJCas();
    }

    reader.getNext(jcas.getCas());

    return jcas;
}
 

public static JCas readXMI(String aPath, TypeSystemDescription aType)
    throws UIMAException, IOException
{
    CollectionReader reader = createReader(XmiReader.class, XmiReader.PARAM_SOURCE_LOCATION,
            "src/test/resources/" + aPath);
    JCas jcas;
    if (aType != null) {
        TypeSystemDescription builtInTypes = TypeSystemDescriptionFactory
                .createTypeSystemDescription();
        List<TypeSystemDescription> allTypes = new ArrayList<>();
        allTypes.add(builtInTypes);
        allTypes.add(aType);
        jcas = JCasFactory.createJCas(CasCreationUtils.mergeTypeSystems(allTypes));
    }
    else {
        jcas = JCasFactory.createJCas();
    }

    reader.getNext(jcas.getCas());

    return jcas;
}
 

private List<CAS> loadData(Dataset ds, File ... files) throws UIMAException, IOException
{
    CollectionReader reader = createReader(Reader.class,
            Reader.PARAM_PATTERNS, files, 
            Reader.PARAM_LANGUAGE, ds.getLanguage());

    AnalysisEngine segmenter = createEngine(BreakIteratorSegmenter.class,
            BreakIteratorSegmenter.PARAM_WRITE_SENTENCE, false);
    
    List<CAS> casList = new ArrayList<>();
    while (reader.hasNext()) {
        JCas cas = JCasFactory.createJCas();
        reader.getNext(cas.getCas());
        segmenter.process(cas);
        casList.add(cas.getCas());
    }
    return casList;
}
 

private List<CAS> loadData(Dataset ds, File ... files) throws UIMAException, IOException
{
    CollectionReader reader = createReader(Conll2002Reader.class,
        Conll2002Reader.PARAM_PATTERNS, files, 
        Conll2002Reader.PARAM_LANGUAGE, ds.getLanguage(), 
        Conll2002Reader.PARAM_COLUMN_SEPARATOR, Conll2002Reader.ColumnSeparators.TAB.getName(),
        Conll2002Reader.PARAM_HAS_TOKEN_NUMBER, true, 
        Conll2002Reader.PARAM_HAS_HEADER, true, 
        Conll2002Reader.PARAM_HAS_EMBEDDED_NAMED_ENTITY, true);

    List<CAS> casList = new ArrayList<>();
    int n = 1;
    while (reader.hasNext()) {
        JCas cas = JCasFactory.createJCas();
        reader.getNext(cas.getCas());
        casList.add(cas.getCas());
        casStorageSession.add("testDataCas" + n, EXCLUSIVE_WRITE_ACCESS, cas.getCas());
    }
    
    return casList;
}
 

@Test
   @Ignore
   // FIXME
   public void testCount() throws Exception {

CollectionReader cr = PubmedCentralCollectionReader.getCR("pmc_test_archive");

int i = 0;
while (cr.hasNext()) {
    CAS cas = CasCreationUtils.createCas(cr
	    .getProcessingResourceMetaData());
    cr.getNext(cas);
    i++;
}
cr.close();
assertEquals(6, i);
   }
 

@Test
public void testValid2() throws Exception {
  String yaml = Files.asCharSource(getFile("jobConfig2.yaml"), Charset.defaultCharset()).read();

  JobBuilder jb = new JobBuilder("Test Job", new YamlPipelineConfiguration(yaml));
  BaleenJob job = (BaleenJob) jb.createNewPipeline();

  assertEquals("Test Job", job.getName());
  assertEquals(yaml, job.originalConfig());
  assertEquals(yaml, job.orderedConfig());

  CollectionReader cr = job.collectionReader();
  assertEquals(BaleenDefaults.DEFAULT_SCHEDULER, cr.getMetaData().getName());
  assertEquals("Foo", cr.getConfigParameterValue("key"));

  List<AnalysisEngine> annotators = job.annotators();
  assertEquals(2, annotators.size());

  AnalysisEngine ann0 = annotators.get(0);
  assertEquals("uk.gov.dstl.baleen.testing.DummyTask", ann0.getMetaData().getName());
  assertEquals("Foo", ann0.getConfigParameterValue("key"));

  AnalysisEngine ann1 = annotators.get(1);
  assertEquals("uk.gov.dstl.baleen.testing.DummyTaskParams", ann1.getMetaData().getName());
  assertEquals("Bar", ann1.getConfigParameterValue("key"));

  List<AnalysisEngine> consumers = job.consumers();
  assertEquals(0, consumers.size());
}
 

private JCas loadPosDevelopmentData() throws IOException, UIMAException
{
    Dataset ds = loader.load("conll2000-en", CONTINUE);
    
    CollectionReader reader = createReader(Conll2000Reader.class,
            Conll2000Reader.PARAM_PATTERNS, ds.getDefaultSplit().getTestFiles(), 
            Conll2000Reader.PARAM_LANGUAGE, ds.getLanguage());
    
    JCas cas = JCasFactory.createJCas();
    reader.getNext(cas.getCas());
    return cas;
}
 

@Test
public void testLimit10() throws Exception {

    CollectionReader cr = createReader(PubmedDatabaseCR.class,
            BlueUima.PARAM_BETWEEN, new int[] { 0, 9 },
            BlueUima.PARAM_SKIP_EMPTY_DOCS, false);

    ArrayList<JCas> jCases = asList(cr);
    assertEquals(9, jCases.size());

    for (JCas jCas : jCases) {
        int docId = getHeaderIntDocId(jCas);
        System.err.println(docId);
    }
}
 

private static CAS loadData(File aFile) throws UIMAException, IOException
{
    CollectionReader reader = createReader(XmiReader.class,
            ConllUReader.PARAM_PATTERNS, aFile);

    List<CAS> casList = new ArrayList<>();
    while (reader.hasNext()) {
        JCas cas = JCasFactory.createJCas();
        reader.getNext(cas.getCas());
        casList.add(cas.getCas());
    }

    return casList.get(0);
}
 

@Test
public void testSerializeDeserializeXmi() throws Exception {

    CollectionReader cr = createReader(TextArrayReader.class,
             PARAM_INPUT, new String[] { "this is a test" });
    AnalysisEngine serializer = createEngine(ZipXWriter.class,
            BlueUima.PARAM_OUTPUT_DIR, testDir + "/serdeser");
    SimplePipeline.runPipeline(cr, serializer);

    cr = createReader(ZipXmiCollectionReader.class, 
            PARAM_INPUT_DIRECTORY, testDir + "/serdeser");
    CAS cas = createCas(cr.getProcessingResourceMetaData());
    cr.getNext(cas);
    assertEquals("this is a test", cas.getDocumentText());
}
 

public static List<SCAS> readFrom(CollectionReader reader) throws IOException, UIMAException {
  List<SCAS> scasList = new ArrayList<>();
  while (reader.hasNext()) {
    JCas jcas = JCasFactory.createJCas();
    CAS cas = jcas.getCas();
    reader.getNext(cas);
    scasList.add(new SCAS(cas));
  }
  return scasList;
}
 

@Test
@Ignore
public void testOnSampleForSrikanth() throws Exception {

    CollectionReader cr = createReader(PdfCollectionReader.class,
             PARAM_INPUT_DIRECTORY, "pdf_srikanth");

    AnalysisEngine dumper = createEngine(DocumentTextWriter.class,
            PARAM_OUTPUT_DIR, "/Users/richarde/Desktop/");

    SimplePipeline.runPipeline(cr, dumper);
}
 

@Test
@Ignore
public void testOnSampleForExtractionQuality() throws Exception {

    CollectionReader cr = createReader(
            PdfCollectionReader.class,
            PARAM_INPUT_DIRECTORY,
            "/Users/richarde/data_hdd/_papers_etc/pubmed/sample_pdfs_68/pdfs",
            PARAM_EXPAND_ABBREVIATIONS, true);

    AnalysisEngine dumper = createEngine(DocumentTextWriter.class,
            PARAM_OUTPUT_DIR, "/Users/richarde/Desktop/");

    runPipeline(cr, dumper);
}
 

@Test
@Ignore
public void testExtractTablesOnSample() throws Exception {

    CollectionReader cr = createReader(PdfCollectionReader.class,
             PARAM_INPUT_DIRECTORY,
            "/Users/richarde/data/_papers_etc/pmc_pdfs_sample");

    AnalysisEngine dumper = createEngine(TableWriter.class);

    SimplePipeline.runPipeline(cr, dumper);
}
 

public static CAS readConll2006(String aPath)
    throws UIMAException, IOException
{
    CollectionReader reader = createReader(Conll2006Reader.class,
            Conll2006Reader.PARAM_SOURCE_LOCATION, "src/test/resources/" + aPath);

    CAS jcas = JCasFactory.createJCas().getCas();

    reader.getNext(jcas);

    return jcas;
}
 

public static void main(String[] args) throws Exception {

        // cr
        CollectionReader cr = CollectionReaderFactory.createReader(
                PubmedWholeDatabaseCR.class, PARAM_DB_CONNECTION, new String[] {
                        "localhost", "bb_pubmed", "root", "" });

        SimplePipeline.runPipeline(
                cr,
                createEngineDescription(MyIndexer.class),
                createEngineDescription(StatsAnnotatorPlus.class,
                        PARAM_PRINT_EVERY, 50000));
    }
 

public static JCas loadWebAnnoTsv3(File aPath) throws UIMAException, IOException
{
    CollectionReader reader = createReader(WebannoTsv3XReader.class,
            WebannoTsv3XReader.PARAM_SOURCE_LOCATION, aPath);
    JCas jcas = JCasFactory.createJCas();
    reader.getNext(jcas.getCas());
    return jcas;
}
 

public static CAS read(String aPath)
    throws UIMAException, IOException
{
    CollectionReader reader = createReader(Conll2006Reader.class,
            Conll2006Reader.PARAM_SOURCE_LOCATION, "src/test/resources/" + aPath);

    CAS cas = JCasFactory.createJCas().getCas();

    reader.getNext(cas);

    return cas;
}
 

public static JCas loadWebAnnoTsv3(String aPath) throws UIMAException, IOException
{
    CollectionReader reader = createReader(WebannoTsv3XReader.class,
            WebannoTsv3XReader.PARAM_SOURCE_LOCATION, "src/test/resources/" + aPath);
    JCas jcas = JCasFactory.createJCas();
    reader.getNext(jcas.getCas());
    return jcas;
}