opennlp.tools.parser.Parse Java Examples

public double scoreStructure(String ca, String q, String passage, boolean verbose) throws InvalidFormatException, IOException{
	POSTaggerME parserModel = new POSTaggerME(new POSModel(new FileInputStream(new File("en-pos-model.bin"))));
	Tokenizer tokenizer = new TokenizerME(new TokenizerModel(new FileInputStream(new File("en-token.bin"))));
	Parser parser = ParserFactory.create(new ParserModel(new FileInputStream(new File("en-parser.bin"))));
	double score = 0;
	
	Parse[] questionParse = ParserTool.parseLine(q, parser, 1);
	Parse[] passageParse = ParserTool.parseLine(q, parser, 1);
	
	if (passage.contains(ca)) {
		for (int i =0; i < questionParse.length; i++) {
			score += matchChildren(questionParse[i],passageParse[i]);
		}
	}
	
	return score;
}
 

public static String getSubject(final Parse parse) {
	if (parse.getType().equals(LABEL_TOP)) {
		return getSubject(parse.getChildren()[0]);
	}

	if (parse.getType().equals(LABEL_SENTENCE)) {
		for (Parse child : parse.getChildren()) {
			if (child.getType().equals(LABEL_NOUN_PHRASE)) {
				return getSubject(child);
			}
		}
	}
	if (parse.getType().equals(LABEL_NOUN_PHRASE)) {
		return getFirstOccurenceForType(parse, LABEL_NAME_PREFIX);
	}

	return "";
}
 

public static String getPredicate(final Parse parse) {
	if (parse.getType().equals(LABEL_TOP)) {
		return getPredicate(parse.getChildren()[0]);
	}

	if (parse.getType().equals(LABEL_SENTENCE)) {
		for (Parse child : parse.getChildren()) {
			if (child.getType().equals(LABEL_VERBAL_PHRASE)) {
				return getPredicate(child);
			}
		}
		return "";
	}
	if (parse.getType().equals(LABEL_VERBAL_PHRASE)) {
		return getFirstOccurenceForType(parse, LABEL_VERB_PREFIX);
	}

	return "";
}
 

public void parserTest1() throws IOException {
	if (!this.modelsAreInitialized) init();
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); 
	Parse[] results = ParserTool.parseLine("Jane Austen was very modest about her own genius ."+this.q,
			parser, 1);
	Parse[] qResults = ParserTool.parseLine(this.q,parser, 1);
	Parse[] rChn = (results[0].getChildren())[0].getChildren();
	
	results[0].expandTopNode(results[0]);
	for (int i = 0; i < results.length; i++) {
		results[i].show();
	}
	for (int i = 0; i < qResults.length; i++) {
		qResults[i].show();
	}
	System.out.print("\n\n");
	for (int i = 0; i < rChn.length; i++) {
		rChn[i].show();
		System.out.print("\n");
	}
}
 

public static String getObject(final Parse parse) {
	String object = "";
	if (parse.getType().equals(LABEL_TOP)) {
		return getObject(parse.getChildren()[0]);
	}

	if (parse.getType().equals(LABEL_SENTENCE)) {
		for (Parse child : parse.getChildren()) {
			if (child.getType().equals(LABEL_VERBAL_PHRASE)) {
				object = getObject(child); 
				if (!object.isEmpty()){
					return object;
				}
			}
		}
		return object;
	}
	if (parse.getType().equals(LABEL_VERBAL_PHRASE)) {
		return getFirstOccurenceForType(parse, LABEL_NAME_PREFIX);
	}

	return object;
}
 

public static String getConstituent(final Parse parse, final String syntactic_cat,
		String lexical_cat) {
	String object = "";
	if (parse.getType().equals(LABEL_TOP)) {
		return getConstituent(parse.getChildren()[0], syntactic_cat, lexical_cat);
	}

	if (parse.getType().equals(LABEL_SENTENCE)) {
		for (Parse child : parse.getChildren()) {
			if (child.getType().equals(syntactic_cat)) {
				object = getConstituent(child, syntactic_cat, lexical_cat); 
				if (!object.isEmpty()){
					return object;
				}
			}
		}
		return object;
	}
	if (parse.getType().equals(syntactic_cat)) {
		return getFirstOccurenceForType(parse, lexical_cat);
	}

	return object;
}
 

public double compareParseType(Parse[] pa1, Parse[] pa2, boolean verbose){
	double numMatches=0;
	Map<String, String> key1 = new HashMap<String, String>();
	for (int i=0;i<pa1.length;i++){
		key1.put(pa1[i].getType(),"y");
		//pa1h.put(key[0],"y");
	}
	for (int j=0;j<pa2.length;j++){
		String key2=pa2[j].getType();
		if (key1.containsKey(key2)){ 
			numMatches++;
			if (verbose) System.out.println("\n");
			pa2[j].show();
			if (verbose) System.out.println("type: "+pa2[j].getType());
		}
	}
	if (verbose) System.out.println("numTypeMatches "+numMatches);
	return numMatches;
}
 

public CorefParse(List<Parse> parses, DiscourseEntity[] entities) {
	this.parses = parses;
	parseMap = new HashMap<Parse, Integer>();
	for (int ei = 0, en = entities.length; ei < en; ei++) {
		if (entities[ei].getNumMentions() > 1) {
			for (Iterator<MentionContext> mi = entities[ei].getMentions(); mi
					.hasNext();) {
				MentionContext mc = mi.next();
				Parse mentionParse = ((DefaultParse) mc.getParse())
						.getParse();
				parseMap.put(mentionParse, ei + 1);
				// System.err.println("CorefParse: "+mc.getParse().hashCode()+" -> "+
				// (ei+1));
			}
		}
	}
}
 

public static Parse[] parsePassageText(String p) throws InvalidFormatException{
	
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(sentenceModel);
	Parser parser = ParserFactory.create(
			parserModel,
			20, // beam size
			0.95); // advance percentage
 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);


		String sent= StringUtils.join(tks," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

public void parserTest1() throws IOException {
	if (!this.modelsAreInitialized) init();
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); 
	Parse[] results = ParserTool.parseLine("Jane Austen was very modest about her own genius ."+this.q,
			parser, 1);
	Parse[] qResults = ParserTool.parseLine(this.q,parser, 1);
	Parse[] rChn = (results[0].getChildren())[0].getChildren();
	
	results[0].expandTopNode(results[0]);
	for (int i = 0; i < results.length; i++) {
		results[i].show();
	}
	for (int i = 0; i < qResults.length; i++) {
		qResults[i].show();
	}
	System.out.print("\n\n");
	for (int i = 0; i < rChn.length; i++) {
		rChn[i].show();
		System.out.print("\n");
	}
}
 

private Parse[] getNounPhrases(Parse parse) {
  List<Parse> nps = new ArrayList<Parse>(10);
  List<Parse> parts = new ArrayList<Parse>();
  parts.add(parse);
  while (parts.size() > 0) {
    List<Parse> newParts = new ArrayList<Parse>();
    for (int pi=0,pn=parts.size();pi<pn;pi++) {
      Parse cp = parts.get(pi);
      if (cp.getType().equals("NP") && cp.isFlat()) {
        nps.add(cp);
      }
      else if (!cp.isPosTag()) {
        newParts.addAll(Arrays.asList(cp.getChildren()));
      }
    }
    parts = newParts;
  }
  return nps.toArray(new Parse[nps.size()]);
}
 

private void print(Parse p, int deep) {
	if (p.getType().length() > 1 && p.getType().substring(0, 2).equals(Parser.TOK_NODE))
		return;
	
	char[] spaces = new char[deep*2];
	Arrays.fill(spaces, ' ');
	Span span = p.getSpan();
    System.out.print(new String(spaces) + p.getType() + " -- " + p.getText().substring(span.getStart(),
			span.getEnd()));
    if (parseMap.containsKey(p)) {
		System.out.print("#" + parseMap.get(p));
	}
    System.out.print("\n");
    for (Parse child : p.getChildren()) {
    	print(child, new Integer(deep + 1));
    }
}
 

public double matchChildren(Parse pa1, Parse pa2) {
	String p1NodeLabel = pa1.getLabel();
	String p2NodeLabel = pa2.getLabel();
	Parse[] children1 = pa1.getChildren();
	Parse[] children2 = pa2.getChildren();
	double matchFound = 0;
	
	if (pa1 == null || pa2 == null) {
		return 0;
	}
	
	if (p1NodeLabel.equals(p2NodeLabel)) {
		if (pa1.getCoveredText().equals(pa2.getCoveredText())) {
			matchFound = 1;
		}
	}
	
	return matchFound + matchChildren(children1[0], children2[0]) + matchChildren(children1[1], children2[1]);
}
 

/** THIS METHOD IS NOT USED 
 * Extracts NEs from a parse tree that has been augmented with NE tags.
 * 
 * @param parse a parse tree augmented with NE tags
 * @return NEs per NE type
 */
// TODO only works with OpenNLP taggers so far
@SuppressWarnings("unchecked")
public static String[][] extractNes(Parse parse) {
	// initialize dynamic arrays
	ArrayList[] nes = new ArrayList[finders.length];
	for (int i = 0; i < nes.length; i++) nes[i] = new ArrayList();
	
	// depth-first search on the parse tree
	extractNesRec(parse, nes);
	
	// copy to static arrays
	String[][] results = new String[finders.length][];
	for (int i = 0; i < nes.length; i++)
		results[i] = (String[]) nes[i].toArray(new String[nes[i].size()]);
	
	return results;
}
 

public Parse[] parsePassageText(String p) throws InvalidFormatException{
	if (!modelsAreInitialized)init();
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(this.sentenceModel);
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); // advance percentage
	//find sentences, tokenize each, parse each, return top parse for each 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);
		//StringTokenizer st = new StringTokenizer(tks[i]); 
		//There are several tokenizers available. SimpleTokenizer works best

		String sent= StringUtils.join(tks," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

public double scoreStructureNorm(String ca, String q, String passage, boolean verbose) throws InvalidFormatException{
	double score1=0, score2=0;
	//OnlpParserTest pt= new OnlpParserTest();
	Parse[] caParse = this.parsePassageText(ca);
	Parse[] qParse = this.parsePassageText(q);
	Parse[] pasParse = this.parsePassageText(passage);
	Parse[] caParseCh = getAllChildren(caParse);
	Parse[] qParseCh = getAllChildren(qParse);
	Parse[] pasParseCh = getAllChildren(pasParse);
	score1=compareParseChunks(qParseCh, pasParseCh,verbose);
	score2=compareParseChunks(caParseCh, pasParseCh,verbose);
	return score1*score2/passage.length();
}
 

/**
	 * Identifies coreferences in an array of full parses of sentences.
	 * 
	 * @param parses array of full parses of sentences
	 */
	public static void link(Parse[] parses) {
		int sentenceNumber = 0;
		List<Mention> document = new ArrayList<Mention>();
		
		for (Parse parse : parses) {
			DefaultParse dp = new DefaultParse(parse, sentenceNumber);
			Mention[] extents =	linker.getMentionFinder().getMentions(dp);
			
			//construct new parses for mentions which do not have constituents
			for (int i = 0; i < extents.length; i++)
				if (extents[i].getParse() == null) {
					Parse snp = new Parse(parse.getText(), extents[i].getSpan(),
										  "NML", 1.0);
					parse.insert(snp);
					extents[i].setParse(new DefaultParse(snp,sentenceNumber));
				}
			
			document.addAll(Arrays.asList(extents));
			sentenceNumber++;
	    }
		
		if (document.size() > 0) {
//			Mention[] ms = document.toArray(new Mention[document.size()]);
//			DiscourseEntity[] entities = linker.getEntities(ms);
//			TODO return results in an appropriate data structure
		}
	}
 

public double scoreStructureNorm(String ca, String q, String passage, boolean verbose) throws InvalidFormatException{
	double score1=0, score2=0;
	//OnlpParserTest pt= new OnlpParserTest();
	Parse[] caParse = t.parsePassageText(ca);
	Parse[] qParse = t.parsePassageText(q);
	Parse[] pasParse = t.parsePassageText(passage);
	Parse[] caParseCh = t.getAllChildren(caParse);
	Parse[] qParseCh = t.getAllChildren(qParse);
	Parse[] pasParseCh = t.getAllChildren(pasParse);
	score1=this.compareParseType(qParseCh, pasParseCh,verbose);
	score2=this.compareParseType(caParseCh, pasParseCh,verbose);
	return score1*score2/passage.length();
}
 

public Parse[] getAllChildren(Parse[] parseAr){
	Parse[] allChildren = parseAr;
	Parse[] allChldr;		
	for (int i=0; i<parseAr.length;i++){
		Parse[] children = parseAr[i].getChildren();			
		allChldr= getAllChildren(children);
		allChildren  =ArrayUtils.addAll(allChildren, allChldr);
	}	
	return allChildren;		
}
 

public Parse[] getAllChildren(Parse[] parseAr){
	Parse[] allChildren = parseAr;
	Parse[] allChldr;		
	for (int i=0; i<parseAr.length;i++){
		Parse[] children = parseAr[i].getChildren();			
		allChldr= getAllChildren(children);
		allChildren  =ArrayUtils.addAll(allChildren, allChldr);
	}	
	return allChildren;		
}
 

public Parse[] getAllChildren(Parse parse){
	Parse[] allChildren = new Parse[1];
	allChildren[0]=parse;
	Parse[] allChldr;
	Parse[] children = parse.getChildren();			
	allChldr= getAllChildren(children);
	allChildren  =ArrayUtils.addAll(allChildren, allChldr);
	return allChildren;		
}
 

public double scoreStructure(String ca, String q, String passage, boolean verbose) throws InvalidFormatException{
	double score1=0, score2=0;
	Parse[] caParse = this.parsePassageText(ca);
	Parse[] qParse = this.parsePassageText(q);
	Parse[] pasParse = this.parsePassageText(passage);
	Parse[] caParseCh = getAllChildren(caParse);
	Parse[] qParseCh = getAllChildren(qParse);
	Parse[] pasParseCh = getAllChildren(pasParse);
	score1=compareParseChunks(qParseCh, pasParseCh,verbose);
	score2=compareParseChunks(caParseCh, pasParseCh,verbose);
	return score1*score2;
}
 

public double scoreStructureNorm(String ca, String q, String passage, boolean verbose) throws InvalidFormatException{
	double score1=0, score2=0;
	//OnlpParserTest pt= new OnlpParserTest();
	Parse[] caParse = this.parsePassageText(ca);
	Parse[] qParse = this.parsePassageText(q);
	Parse[] pasParse = this.parsePassageText(passage);
	Parse[] caParseCh = getAllChildren(caParse);
	Parse[] qParseCh = getAllChildren(qParse);
	Parse[] pasParseCh = getAllChildren(pasParse);
	score1=compareParseChunks(qParseCh, pasParseCh,verbose);
	score2=compareParseChunks(caParseCh, pasParseCh,verbose);
	return score1*score2/passage.length();
}
 

/**
 * Recursive method called by <code>extractNes(Parse)</code> to extract NEs
 * from a parse tree augmented with NE tags.
 * 
 * @param parse a node of a parse tree
 * @param nes NEs found so far
 */
private static void extractNesRec(Parse parse, ArrayList<String>[] nes) {
	String type = parse.getType();
	if (type.startsWith("NE")) {
		String text = parse.getText().substring(parse.getSpan().getStart(),
												parse.getSpan().getEnd());
		nes[getNeIds(type)[0]].add(text.trim());
	}
	
	for (Parse child : parse.getChildren())
		extractNesRec(child, nes);
}
 

private static void usingOpenNLP() {
        String fileLocation = getModelDir() + "/en-parser-chunking.bin";
        System.out.println(fileLocation);
        try (InputStream modelInputStream = new FileInputStream(fileLocation);) {
            ParserModel model = new ParserModel(modelInputStream);
            Parser parser = ParserFactory.create(model);
            String sentence = "The cow jumped over the moon";
            // Used to demonstrate difference between NER and Parser
            sentence = "He was the last person to see Fred.";

            Parse parses[] = ParserTool.parseLine(sentence, parser, 3);
            for (Parse parse : parses) {
                // First display
                parse.show();
                // Second display
//                parse.showCodeTree();
                // Third display
//                System.out.println("Children");
//                Parse children[] = parse.getChildren();
//                for (Parse parseElement : children) {
//                    System.out.println(parseElement);
//                    System.out.println(parseElement.getText());
//                    System.out.println(parseElement.getType());
//                    Parse tags[] = parseElement.getTagNodes();
//                    System.out.println("Tags");
//                    for (Parse tag : tags) {
//                        System.out.println("[" + tag + "]" + " type: " + tag.getType()
//                                + "  Probability: " + tag.getProb()
//                                + "  Label: " + tag.getLabel());
//                    }
//                }
            }
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    }
 

public Parse[] parsePassageText(String p) throws InvalidFormatException{
	if (!modelsAreInitialized)init();
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(this.sentenceModel);
	NameFinderME nameFinder = new NameFinderME(this.nerModel);
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); // advance percentage
	//find sentences, tokenize each, parse each, return top parse for each 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		//String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);
		
		//StringTokenizer st = new StringTokenizer(tks[i]); 
		//There are several tokenizers available. SimpleTokenizer works best
		Tokenizer tokenizer = SimpleTokenizer.INSTANCE;
		for (int si = 0; si < sentences.length; si++) {
	        Span[] tokenSpans = tokenizer.tokenizePos(sentences[si]);
	        String[] tokens = Span.spansToStrings(tokenSpans, sentences[si]);
	        Span[] names = nameFinder.find(tokens);
	        for (int ni = 0; ni < names.length; ni++) {
	            Span startSpan = tokenSpans[names[ni].getStart()];
	            int nameStart = startSpan.getStart();
	            Span endSpan = tokenSpans[names[ni].getEnd() - 1];
	            int nameEnd = endSpan.getEnd();
	            String name = sentences[si].substring(nameStart, nameEnd);
	            System.out.println(name);
	        }
	    }
		String sent= StringUtils.join(tokenizer," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

private void navigateTree(Parse[] currentLevel, int position, List<String> results) {
	//System.out.print("text: " + currentLevel[0].getText());			
	//System.out.print("; head: " + currentLevel[0].getHead());
	//System.out.print("; type: " + currentLevel[0].getType());
	//System.out.print("; label: " + currentLevel[0].getLabel());			
	//System.out.println();
	
	/*if (!currentLevel[position].getType().equals("TK")) {
		System.out.print(currentLevel[position].getType() + ": ");
		System.out.println(currentLevel[position].getCoveredText());
	}*/
	
	if (currentLevel[position].getType().equals("NP")) {
		//TODO: remove punctuation
		results.add(currentLevel[position].getCoveredText());
		//System.out.print(currentLevel[position].getType() + ": ");
		//System.out.println(currentLevel[position].getCoveredText());
	}
	
	if (currentLevel[position].getChildCount() < 1) {//leaf node

	} else {//recursively navigate each child
		Parse[] theChildren = currentLevel[position].getChildren();
		for (int i = 0; i < theChildren.length; i++) {	
			navigateTree(currentLevel[position].getChildren(), i, results);
		}
	}
}
 

/** Turn one tokenized sentence into one top-ranked parse tree. */
public Parse parseSentence(List<String> tokens) {
	//StringTokenizer st = new StringTokenizer(tks[i]); 
	//There are several tokenizers available. SimpleTokenizer works best
	System.out.print(";");
	String sent= StringUtils.join(tokens," ");
	return ParserTool.parseLine(sent,parser, 1)[0];
}
 

/** Enumerate all of the child parses of a parse tree */
public List<Parse> getAllChildren(List<Parse> parses){
	List<Parse> doneChildren = new ArrayList<>(parses.size());
	Deque<Parse> nextChildren = new ArrayDeque<>(100);
	nextChildren.addAll(parses);
	while (!nextChildren.isEmpty()) {
		Parse child = nextChildren.remove();
		doneChildren.add(child);
		nextChildren.addAll(Arrays.asList(child.getChildren()));
	}
	return doneChildren;		
}
 

/** Turn a tokenized paragraph into a list of parses */
public List<Parse> parseParagraph(List<List<String>> paragraph) {
	//find sentences, tokenize each, parse each, return top parse for each
	List<Parse> results = new ArrayList<>(paragraph.size());
	for (List<String> sentence : paragraph) {
		//StringTokenizer st = new StringTokenizer(tks[i]); 
		//There are several tokenizers available. SimpleTokenizer works best
		results.add(parseSentence(sentence));
	}
	return results;
}