edu.stanford.nlp.neural.rnn.RNNCoreAnnotations Java Examples

public SentimentResult getSentimentResult(String text) {
	SentimentClassification classification = new SentimentClassification();
	SentimentResult sentimentResult = new SentimentResult();
	if (text != null && text.length() > 0) {
		Annotation annotation = pipeline.process(text);
		for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
			Tree tree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
			SimpleMatrix simpleMatrix = RNNCoreAnnotations.getPredictions(tree);

			classification.setVeryNegative((double) Math.round(simpleMatrix.get(0) * 100d));
			classification.setNegative((double) Math.round(simpleMatrix.get(1) * 100d));
			classification.setNeutral((double) Math.round(simpleMatrix.get(2) * 100d));
			classification.setPositive((double) Math.round(simpleMatrix.get(3) * 100d));
			classification.setVeryPositive((double) Math.round(simpleMatrix.get(4) * 100d));

			String setimentType = sentence.get(SentimentCoreAnnotations.SentimentClass.class);
			sentimentResult.setSentimentType(setimentType);
			sentimentResult.setSentimentClass(classification);
			sentimentResult.setSentimentScore(RNNCoreAnnotations.getPredictedClass(tree));
		}
	}
	return sentimentResult;
}
 

private double getScore(List<CoreMap> sentences, double overallSentiment) {
    int matrixIndex =
            overallSentiment < -0.5  ? 0  // very negative
            : overallSentiment < 0.0 ? 1  // negative
            : overallSentiment < 0.5 ? 3  // positive
            : 4;                       // very positive
    double sum = 0;
    int numberOfSentences = 0;
    for (CoreMap sentence : sentences) {
        Tree sentiments = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
        int predictedClass = RNNCoreAnnotations.getPredictedClass(sentiments);
        if (predictedClass == 2) { // neutral
            continue;
        }
        SimpleMatrix matrix = RNNCoreAnnotations.getPredictions(sentiments);
        sum += matrix.get(matrixIndex);
        numberOfSentences++;
    }
    return sum / numberOfSentences;
}
 

public List<Pattern> run(List<Pattern> patterns) {

        Properties props = new Properties();
        props.setProperty("annotators", "tokenize, ssplit, pos, lemma, parse, sentiment");
        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);

        for (Pattern pattern : patterns) {
            Annotation annotation = pipeline.process(pattern.toSentences());
            for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
                    Tree tree = sentence.get(SentimentCoreAnnotations.AnnotatedTree.class);
                    int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
                    for (CoreLabel token : sentence.get(CoreAnnotations.TokensAnnotation.class)) {
                        String lemma = token.get(CoreAnnotations.LemmaAnnotation.class);

                    }
            }
        }
        return null;
    }
 

private double getOverallSentiment(List<CoreMap> sentences) {
    double sum = 0;
    int numberOfSentences = 0;
    for (CoreMap sentence : sentences) {
        Tree sentiments = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
        int predictedClass = RNNCoreAnnotations.getPredictedClass(sentiments);
        if (predictedClass == 2) { // neutral sentiment
            continue;
        }
        sum += predictedClass;
        numberOfSentences++;
    }
    return numberOfSentences == 0 ? 0 : (sum / numberOfSentences - 2) / 2;
}
 

/**
 * Synchronized method to obtain the sentiment of the set of documents.
 * Synchronization is fine, because the method is invoked via a scheduled job
 * and only one execution at a time is permitted.
 * That allows to optimize the loading of the model as well
 * @param documents
 * @return
 */
public synchronized SentimentResult getSentiment(Set<String> documents, TimelineMusic meta) {

    double sentimentSum = 0;
    for (String document: documents) {
        int mainSentiment = 0;
        if (document != null && document.length() > 0) {
            int longest = 0;
            try {
                Annotation annotation = pipeline.process(document);
                // mainSentiment is the sentiment of the whole document. We find
                // the whole document by comparing the length of individual
                // annotated "fragments"
                for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) {
                    Tree tree = sentence.get(SentimentCoreAnnotations.AnnotatedTree.class);
                    int sentiment = RNNCoreAnnotations.getPredictedClass(tree);
                    String partText = sentence.toString();
                    if (partText.length() > longest) {
                        mainSentiment = sentiment;
                        longest = partText.length();
                    }
                }
            } catch (Exception ex) {
                logger.error("Problem analyzing document sentiment. " + document, ex);
                continue;
            }
        }
        sentimentSum += mainSentiment;
    }

    double average = sentimentSum / documents.size();
    meta.setAverageSentiment(average);

    if (average >= 2.25) {
        return SentimentResult.POSITIVE;
    } else if (average <= 1.75) {
        return SentimentResult.NEGATIVE;
    }
    return SentimentResult.NEUTRAL;
}
 

private static void usingStanfordSentimentAnalysis() {
    String review = "An overly sentimental film with a somewhat "
            + "problematic message, but its sweetness and charm "
            + "are occasionally enough to approximate true depth "
            + "and grace. ";

    String sam = "Sam was an odd sort of fellow. Not prone to angry and "
            + "not prone to merriment. Overall, an odd fellow.";
    String mary = "Mary thought that custard pie was the best pie in the "
            + "world. However, she loathed chocolate pie.";
    Properties props = new Properties();
    props.put("annotators", "tokenize, ssplit, parse, sentiment");
    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);

    Annotation annotation = new Annotation(review);
    pipeline.annotate(annotation);

    System.out.println("---sentimentText");
    String[] sentimentText = {"Very Negative", "Negative", "Neutral",
        "Positive", "Very Positive"};
    for (CoreMap sentence : annotation.get(
            CoreAnnotations.SentencesAnnotation.class)) {
        Tree tree = sentence.get(
                SentimentCoreAnnotations.AnnotatedTree.class);
        System.out.println("---Number of children: " + tree.numChildren());
        System.out.println("[" + tree.getChild(0) + "][" + tree.getChild(1) + "]");
        tree.printLocalTree();
        int score = RNNCoreAnnotations.getPredictedClass(tree);
        System.out.println(sentimentText[score]);
    }

    // Classifer
    CRFClassifier crf
            = CRFClassifier.getClassifierNoExceptions(
                    "C:/Current Books in Progress/NLP and Java/Models"
                    + "/english.all.3class.distsim.crf.ser.gz");
    String S1 = "Good afternoon Rajat Raina, how are you today?";
    String S2 = "I go to school at Stanford University, which is located in California.";
    System.out.println(crf.classifyToString(S1));
    System.out.println(crf.classifyWithInlineXML(S2));
    System.out.println(crf.classifyToString(S2, "xml", true));

    Object classification[] = crf.classify(S2).toArray();
    for (int i = 0; i < classification.length; i++) {
        System.out.println(classification[i]);
    }
}
 

public void detectSentiment(String msgId, String txt, String source,
                            boolean useCoreNLP, boolean saveDb) {
    Annotation annotation = new Annotation(txt);
    pipeline.annotate(annotation);
    List<CoreMap> sentences = annotation.get(CoreAnnotations.SentencesAnnotation.class);
    if (sentences != null) {
        int sentNum = 0;
        for (CoreMap sentence : sentences) {
            sentNum++;
            String sentStr = sentence.toString();
            String sentiment = "Neutral";
            int sentiment_num = 2;
            double score = 0.0;
            if(useCoreNLP) {
                sentiment = sentence.get(SentimentCoreAnnotations.SentimentClass.class);
                Tree sentimentTree = sentence.get(SentimentCoreAnnotations.SentimentAnnotatedTree.class);
                // 0 = very negative, 1 = negative, 2 = neutral, 3 = positive, and 4 = very positive
                Integer predictedClass = RNNCoreAnnotations.getPredictedClass(sentimentTree);
                SimpleMatrix scoreMatrix = RNNCoreAnnotations.getPredictions(sentimentTree);
                score = scoreMatrix.get(predictedClass.intValue(), 0);
                sentiment_num = predictedClass.intValue();
            } else {
                double sentimentValue = 0.0;
                int adjectivesFound = 0;
                // check every adjective to see if it appears in sentence
                for(String adj : adjectives.keySet()) {
                    if(sentStr.matches(".*\\b" + adj + "\\b.*")) {
                        sentimentValue += adjectives.get(adj);
                        adjectivesFound++;
                    }
                }
                if(adjectivesFound > 0) {
                    sentimentValue /= adjectivesFound;
                }
                if(sentimentValue < -2) {
                    sentiment = "Very Negative";
                    sentiment_num = 0;
                } else if(sentimentValue < -0.5) {
                    sentiment = "Negative";
                    sentiment_num = 1;
                } else if(sentimentValue < 0.5) {
                    sentiment = "Neutral";
                    sentiment_num = 2;
                } else if(sentimentValue < 2) {
                    sentiment = "Positive";
                    sentiment_num = 3;
                } else {
                    sentiment = "Very Positive";
                    sentiment_num = 4;
                }
                if(adjectivesFound > 0)
                    score = 1.0;
                else
                    score = 0.0;
            }

            logger.log(Level.INFO, source + " "
                    + msgId + " - "
                    + sentiment + " (" + score + ") "
                    + sentence.toString());

            // Only save somewhat confident, non-neutral results
            if(score > 0.3 && sentiment_num != 2 && saveDb) {
                try {
                    // check if
                    String sql = "INSERT INTO sentiment"
                            + "(id,source,msg,sentiment,sentiment_num,score)"
                            + " VALUES(?,?,?,?,?,?)";
                    PreparedStatement pstmt = db.prepareStatement(sql);
                    pstmt.setString(1, msgId + "-" + sentNum);
                    pstmt.setString(2, source);
                    pstmt.setString(3, sentStr);
                    pstmt.setString(4, sentiment);
                    pstmt.setInt(5, sentiment_num);
                    pstmt.setDouble(6, score);
                    pstmt.executeUpdate();
                } catch (SQLException e) {
                    logger.log(Level.SEVERE, e.getMessage());
                }
            }
        }
    }
}
 

@Override
	public void validatedProcess(Dataset dataset, String spanTypeOfSentenceUnit) {

		Properties prop1 = new Properties();
		prop1.setProperty("annotators", "parse sentiment");
		StanfordCoreNLP pipeline = new StanfordCoreNLP(prop1, false);
		
		for (Span span : dataset.getSpans(spanTypeOfSentenceUnit)){

			
			HashMap<Integer, Word> wordIndex = new HashMap<>();
			Annotation a = CoreNLPHelper.reconstructStanfordAnnotations(span, wordIndex, true);
			pipeline.annotate(a);
			
			for (CoreMap sentence : a.get(SentencesAnnotation.class)){
				Tree sentimentTree = sentence.get(SentimentAnnotatedTree.class);
				sentimentTree.setSpans();
				sentimentTree.indexLeaves();
				sentimentTree.indexSpans();
				sentimentTree.percolateHeadIndices();
//				for (CoreLabel cl : sentimentTree.taggedLabeledYield()){
//					Main.debug(""+cl.beginPosition()+"\t"+cl.get(CharacterOffsetBeginAnnotation.class));
//					Main.debug(cl.index() + "\t" + cl.keySet());
//				}
				
//				sentimentTree.indentedListPrint();
				
//				sentence.get(TreeAnnotation.class).indentedListPrint();
				
				SimpleMatrix sm = RNNCoreAnnotations.getPredictions(sentimentTree);
				assignSentiment(span, sm, "phraseSentiment");
//				Main.debug(sm.toString());
				
//				//assign begin positions to each word in the tree because those seem to be missing
//				int order = 0;
//				ArrayList<edu.stanford.nlp.ling.Word> stanfordWords = sentimentTree.yieldWords();
//				for (Word w : span){
//					stanfordWords.get(order).setBeginPosition(w.getStartOffset());
//					order++;
//				}
				
				try {
					analyzeTree(sentimentTree, span, wordIndex,0);
				} catch (IllegalSpanException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
			
		}
	}
 

private static void analyzeTree(Tree tree, Span sentenceSpan, HashMap<Integer, Word> wordIndex, int coveredWords) throws IllegalSpanException{
		
		for (Tree t : tree.getChildrenAsList()){
			SimpleMatrix sm = RNNCoreAnnotations.getPredictions(tree);
			
//			List<edu.stanford.nlp.ling.CoreLabel> stanfordWords = t.taggedLabeledYield();
			ArrayList<edu.stanford.nlp.ling.Word> stanfordWords = t.yieldWords();
			
//			Main.debug("Words: "+stanfordWords.toString());
			Word begin = wordIndex.get( sentenceSpan.first().getOrder()+coveredWords );
			Word end = wordIndex.get( sentenceSpan.first().getOrder()+coveredWords+stanfordWords.size()-1 );
			
//			Main.debug("Words: "+begin.getWord() + " - " + end.getWord());
			if (t.isPreTerminal()){
				//this node has only the POS tag of the node below it, which is the lead word
				//add the sentiment values as an annotation to the leaf word
				assignSentiment(begin, sm, "phraseSentiment");
			}
			
			if (t.isPhrasal()){
				//try to match this phrase with one already in the dataset and get the Span
				//hopefully the parser tree and the sentiment tree line up...
				//when Span is found, add the sentiment values as an annotation to it
				String pos = t.label().toString().replaceAll("--?[\\d]*", "");
				Dataset dataset = sentenceSpan.getDataset();
				TreeSet<Span> candidateSpans = dataset.getSpans("syntacticPhrase", begin);
				boolean found = false;
				Span phraseSpan=null;
				for (Span candidate : candidateSpans){
					if (candidate.last().equals(end) && candidate.getAnnotation("pos").equals(pos)){
						phraseSpan = candidate;
						assignSentiment(candidate, sm, "phraseSentiment");
						found =true;
					}
				}
				
				if (!found){
//					Main.debug("No matching Span");
//					Main.debug("candidateSpans: "+candidateSpans);
//					Main.debug(pos + "\t" + begin.getWord() + "\t" + end.getWord());
				}
//				if (t.isPhrasal() && !t.isPrePreTerminal()){
//					if (phraseSpan != null){
//						analyzeTree(t, phraseSpan, wordIndex, coveredWords);
//					} else {
//						analyzeTree(t, parent, wordIndex, coveredWords);
//					}
//				}
			}
			if (t.isLeaf()){
				//this is a single word -> we can add the sentiment values as an annotation to it
				assignSentiment(begin, sm, "wordSentiment");
			}
			
			if (!t.isLeaf())
				analyzeTree(t, sentenceSpan, wordIndex, coveredWords);
			
			
			coveredWords += stanfordWords.size();
			
		}
	}