org.carrot2.core.Cluster Java Examples

@Override
public void process() throws ProcessingException {
  clusters = new ArrayList<>();
  if (wordsToCheck == null) {
    return;
  }

  // Test with Maltese so that the English clustering performed in other tests
  // is not affected by the test stopwords and stoplabels.
  ILexicalData lexicalData = preprocessing.lexicalDataFactory
      .getLexicalData(LanguageCode.MALTESE);

  for (String word : wordsToCheck.split(",")) {
    if (!lexicalData.isCommonWord(new MutableCharArray(word))
        && !lexicalData.isStopLabel(word)) {
      clusters.add(new Cluster(word));
    }
  }
}
 

private void storeCommitsMessagesTopics(List<Cluster> commitsMessagesTopics, VcsRepositoryDelta vcsRepositoryDelta, CommitsMessageTopicsTransMetric db) {
	db.getCommitsTopics().getDbCollection().drop();
	for (Cluster cluster : commitsMessagesTopics) {
		CommitsTopic commitsTopic = new CommitsTopic();
		db.getCommitsTopics().add(commitsTopic);
		commitsTopic.setRepository(vcsRepositoryDelta.getRepository().getUrl());
		commitsTopic.getLabels().addAll(cluster.getPhrases());
		commitsTopic.setNumberOfMessages(cluster.getAllDocuments().size());
		for(Document document : cluster.getDocuments())
		{
			String[] uid = document.getStringId().split("\t");
			commitsTopic.getCommitsMessageId().add(uid[1]);
		}
	}
	db.sync();
}
 

@Override
public void process() throws ProcessingException {
  clusters = new ArrayList<>();
  
  for (Document document : documents) {
    final Cluster cluster = new Cluster();
    cluster.addPhrases(document.getTitle(), document.getSummary());
    if (document.getLanguage() != null) {
      cluster.addPhrases(document.getLanguage().name());
    }
    for (String field : customFields.split(",")) {
      Object value = document.getField(field);
      if (value != null) {
        cluster.addPhrases(value.toString());
      }
    }
    cluster.addDocuments(document);
    clusters.add(cluster);
  }
}
 

@Override
public void measure(Project project, ProjectDelta delta, CommitsMessageTopicsTransMetric db) {
	
	VcsProjectDelta vcsd = delta.getVcsDelta();
	if (vcsd.getRepoDeltas().size()>0)
	{	
		cleanCommitsMessages(delta.getDate(),db);
		for (VcsRepositoryDelta vcsRepositoryDelta : vcsd.getRepoDeltas())
		{
			processCommitsMessages(project, vcsRepositoryDelta, db);
			List<Cluster> commitsMessagesTopics = produceCommitsMessagesTopics(db);
			storeCommitsMessagesTopics(commitsMessagesTopics, vcsRepositoryDelta, db);
		}
	}
	
}
 

public static void displayResults(ProcessingResult processingResult)
{
    final Collection<Document> documents = processingResult.getDocuments();
    final Collection<Cluster> clusters = processingResult.getClusters();
    final Map<String, Object> attributes = processingResult.getAttributes();

    // Show documents
    if (documents != null)
    {
        displayDocuments(documents);
    }

    // Show clusters
    if (clusters != null)
    {
        displayClusters(clusters);
    }

    // Show attributes other attributes
    displayAttributes(attributes);
}
 

/**
 * 对processingResult进行全面的展示,输出至控制台.
 * @author GS
 * @param processingResult
 */
public static void displayResults(ProcessingResult processingResult)
{
    final Collection<Document> documents = processingResult.getDocuments();//所有的文档
    final Collection<Cluster> clusters = processingResult.getClusters();//所有的类别
    final Map<String, Object> attributes = processingResult.getAttributes();//参数

    // Show documents
    if (documents != null)
    {
        displayDocuments(documents);//打印所有文档
    }

    // Show clusters
    if (clusters != null)
    {
        displayClusters(clusters);//打印所有分类
    }

    // Show attributes other attributes
    displayAttributes(attributes);//打印参数
}
 

public static void displayResults(ProcessingResult processingResult)
{
    final Collection<Document> documents = processingResult.getDocuments();
    final Collection<Cluster> clusters = processingResult.getClusters();
    final Map<String, Object> attributes = processingResult.getAttributes();

    // Show documents
    if (documents != null)
    {
        displayDocuments(documents);
    }

    // Show clusters
    if (clusters != null)
    {
        displayClusters(clusters);
    }

    // Show attributes other attributes
    displayAttributes(attributes);
}
 

private void storeBugTrackerTopics(List<Cluster> bugTrackerTopics, BugTrackingSystemDelta btspDelta,
		TopicsTransMetric db) {
	db.getBugTrackerTopics().getDbCollection().drop();
	for (Cluster cluster : bugTrackerTopics) {
		BugTrackerTopic bugTrackerTopic = new BugTrackerTopic();
		db.getBugTrackerTopics().add(bugTrackerTopic);
		bugTrackerTopic.setBugTrackerId(btspDelta.getBugTrackingSystem().getOSSMeterId());
		bugTrackerTopic.getLabels().addAll(cluster.getPhrases());
		bugTrackerTopic.setNumberOfDocuments(cluster.getAllDocuments().size());
		for(Document document : cluster.getAllDocuments())
		{
			String[] uid = document.getStringId().split("\t");
			bugTrackerTopic.getCommentsId().add(uid[1]+"\t"+uid[2]);
		}
	}
	db.sync();
}
 

private void storeNewsgroupTopics(List<Cluster> newsgroupTopics, CommunicationChannelDelta ccpDelta,
		TopicsTransMetric db) {
	db.getNewsgroupTopics().getDbCollection().drop();
	for (Cluster cluster : newsgroupTopics) {
		NewsgroupTopic newsgroupTopic = new NewsgroupTopic();
		db.getNewsgroupTopics().add(newsgroupTopic);
		CommunicationChannel communicationChannel = ccpDelta.getCommunicationChannel();
		newsgroupTopic.setNewsgroupName(communicationChannel.getOSSMeterId());
		newsgroupTopic.getLabels().addAll(cluster.getPhrases());
		newsgroupTopic.setNumberOfDocuments(cluster.getAllDocuments().size());
		for(Document document : cluster.getAllDocuments())
		{
			String[] uid = document.getStringId().split("\t");
			newsgroupTopic.getArticlesId().add(Long.valueOf(uid[1]));
		}
	}
	db.sync();
}
 

@Override
public void process() throws ProcessingException {
  final PreprocessingContext preprocessingContext = preprocessing.preprocess(
      documents, "", LanguageCode.ENGLISH);
  final AllTokens allTokens = preprocessingContext.allTokens;
  final AllWords allWords = preprocessingContext.allWords;
  final AllStems allStems = preprocessingContext.allStems;
  clusters = new ArrayList<>();
  for (int i = 0; i < allTokens.image.length; i++) {
    if (allTokens.wordIndex[i] >= 0) {
      clusters.add(new Cluster(new String(
          allStems.image[allWords.stemIndex[allTokens.wordIndex[i]]])));
    }
  }
}
 

private List<NamedList<Object>> clustersToNamedList(List<Cluster> carrotClusters,
                                 SolrParams solrParams) {
  List<NamedList<Object>> result = new ArrayList<>();
  clustersToNamedList(carrotClusters, result, solrParams.getBool(
          CarrotParams.OUTPUT_SUB_CLUSTERS, true), solrParams.getInt(
          CarrotParams.NUM_DESCRIPTIONS, Integer.MAX_VALUE));
  return result;
}
 

private List<Cluster> produceTopics(ArrayList<Document> documents) {
	/* A controller to manage the processing pipeline. */
	final Controller controller = ControllerFactory.createSimple();

	/*
	 * Perform clustering by topic using the Lingo algorithm. Lingo can take
	 * advantage of the original query, so we provide it along with the documents.
	 */
	final ProcessingResult byTopicClusters = controller.process(documents, null,
			LingoClusteringAlgorithm.class);
	final List<Cluster> clustersByTopic = byTopicClusters.getClusters();

	return clustersByTopic;
}
 

private List<Cluster> produceTopics(ArrayList<Document> documents) {
	/* A controller to manage the processing pipeline. */
	final Controller controller = ControllerFactory.createSimple();

	/*
	 * Perform clustering by topic using the Lingo algorithm. Lingo can take
	 * advantage of the original query, so we provide it along with the documents.
	 */
	final ProcessingResult byTopicClusters = controller.process(documents, null,
			LingoClusteringAlgorithm.class);
	final List<Cluster> clustersByTopic = byTopicClusters.getClusters();

	return clustersByTopic;
}
 

@Override
public void process() throws ProcessingException {
  final PreprocessingContext preprocessingContext = preprocessing.preprocess(
      documents, "", LanguageCode.ENGLISH);
  clusters = new ArrayList<>();
  for (char[] token : preprocessingContext.allTokens.image) {
    if (token != null) {
      clusters.add(new Cluster(new String(token)));
    }
  }
}
 

@Override
public void measure(Project project, ProjectDelta projectDelta, TopicsTransMetric db) {
	System.err.println("Started " + getIdentifier());

	// This is for indexing
	IndexPrepTransMetric indexPrepTransMetric = ((IndexPreparationTransMetricProvider) uses.get(1)).adapt(context.getProjectDB(project));
	indexPrepTransMetric.getExecutedMetricProviders().first().getMetricIdentifiers().add(getIdentifier());
	indexPrepTransMetric.sync();

	BugTrackingSystemProjectDelta bugTrackingSystemDelta = projectDelta.getBugTrackingSystemDelta();
	if (bugTrackingSystemDelta.getBugTrackingSystemDeltas().size() > 0) {
		cleanBugTrackers(projectDelta.getDate(), db);
		for (BugTrackingSystemDelta btspDelta : bugTrackingSystemDelta.getBugTrackingSystemDeltas()) {
			processBugTrackers(project, btspDelta, db);
			List<Cluster> bugTrackerTopics = produceBugTrackerTopics(db);
			System.err.println("bugTrackerTopics.size(): " + bugTrackerTopics.size());
			storeBugTrackerTopics(bugTrackerTopics, btspDelta, db);
		}
	}

	CommunicationChannelProjectDelta communicationChannelDelta = projectDelta.getCommunicationChannelDelta();
	if (communicationChannelDelta.getCommunicationChannelSystemDeltas().size() > 0) {
		cleanCommunicationChannels(projectDelta.getDate(), db);
		for (CommunicationChannelDelta ccpDelta : communicationChannelDelta.getCommunicationChannelSystemDeltas()) {
			processCommunicationChannel(project, ccpDelta, db);
			List<Cluster> newsgroupTopics = produceNewsgroupTopics(db);
			System.err.println("newsgroupTopics.size(): " + newsgroupTopics.size());
			if (newsgroupTopics.size() > 0)
				storeNewsgroupTopics(newsgroupTopics, ccpDelta, db);
			
		}
	}

}
 

private static void displayCluster(final int level, String tag, Cluster cluster,
    int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
{
    final String label = cluster.getLabel();

    // indent up to level and display this cluster's description phrase
    for (int i = 0; i < level; i++)
    {
        System.out.print("  ");
    }
    System.out.println(label + "  "
        + clusterDetailsFormatter.formatClusterDetails(cluster));

    // if this cluster has documents, display three topmost documents.
    int documentsShown = 0;
    for (final Document document : cluster.getDocuments())
    {
        if (documentsShown >= maxNumberOfDocumentsToShow)
        {
            break;
        }
        displayDocument(level + 1, document);
        documentsShown++;
    }
    if (maxNumberOfDocumentsToShow > 0
        && (cluster.getDocuments().size() > documentsShown))
    {
        System.out.println(getIndent(level + 1) + "... and "
            + (cluster.getDocuments().size() - documentsShown) + " more\n");
    }

    // finally, if this cluster has subclusters, descend into recursion.
    final int num = 1;
    for (final Cluster subcluster : cluster.getSubclusters())
    {
        displayCluster(level + 1, tag + "." + num, subcluster,
            maxNumberOfDocumentsToShow, clusterDetailsFormatter);
    }
}
 

public static void displayClusters(final Collection<Cluster> clusters,
    int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
{
    System.out.println("\n\nCreated " + clusters.size() + " clusters\n");
    int clusterNumber = 1;
    for (final Cluster cluster : clusters)
    {
        displayCluster(0, "" + clusterNumber++, cluster, maxNumberOfDocumentsToShow,
            clusterDetailsFormatter);
    }
}
 

public static void displayClusters(final Collection<Cluster> clusters,
    int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
{
    System.out.println("\n\nCreated " + clusters.size() + " clusters\n");//显示聚类数
    int clusterNumber = 1;
    for (final Cluster cluster : clusters)
    {
        displayCluster(0, "" + clusterNumber++, cluster, maxNumberOfDocumentsToShow,
            clusterDetailsFormatter);
    }
}
 

private static void displayCluster(final int level, String tag, Cluster cluster,
    int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
{
    final String label = cluster.getLabel();

    // indent up to level and display this cluster's description phrase
    for (int i = 0; i < level; i++)
    {
        System.out.print("  ");
    }
    System.out.println(label + "  "
        + clusterDetailsFormatter.formatClusterDetails(cluster));

    // if this cluster has documents, display three topmost documents.
    int documentsShown = 0;
    for (final Document document : cluster.getDocuments())
    {
        if (documentsShown >= maxNumberOfDocumentsToShow)
        {
            break;
        }
        displayDocument(level + 1, document);
        documentsShown++;
    }
    if (maxNumberOfDocumentsToShow > 0
        && (cluster.getDocuments().size() > documentsShown))
    {
        System.out.println(getIndent(level + 1) + "... and "
            + (cluster.getDocuments().size() - documentsShown) + " more\n");
    }

    // finally, if this cluster has subclusters, descend into recursion.
    final int num = 1;
    for (final Cluster subcluster : cluster.getSubclusters())
    {
        displayCluster(level + 1, tag + "." + num, subcluster,
            maxNumberOfDocumentsToShow, clusterDetailsFormatter);
    }
}
 

public static void displayClusters(final Collection<Cluster> clusters,
    int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
{
    System.out.println("\n\nCreated " + clusters.size() + " clusters\n");
    int clusterNumber = 1;
    for (final Cluster cluster : clusters)
    {
        displayCluster(0, "" + clusterNumber++, cluster, maxNumberOfDocumentsToShow,
            clusterDetailsFormatter);
    }
}
 

/**
 * 对一个类进行展示.
 * @author GS
 * @param level
 * @param tag
 * @param cluster
 * @param maxNumberOfDocumentsToShow
 * @param clusterDetailsFormatter
 */
private static void displayCluster(final int level, String tag, Cluster cluster,
    int maxNumberOfDocumentsToShow, ClusterDetailsFormatter clusterDetailsFormatter)
{
    final String label = cluster.getLabel();//当前类的标题

    // indent up to level and display this cluster's description phrase
    for (int i = 0; i < level; i++)
    {
        System.out.print("  ");
    }
    System.out.println(label + "  "
        + clusterDetailsFormatter.formatClusterDetails(cluster));

    // if this cluster has documents, display three topmost documents.
    int documentsShown = 0;
    for (final Document document : cluster.getDocuments())
    {
        if (documentsShown >= maxNumberOfDocumentsToShow)//如果达到最大展示数的话不再展示
        {
            break;
        }
        displayDocument(level + 1, document);//这个level是干嘛的?
        documentsShown++;//当前分类已经展示的文档数
    }
    if (maxNumberOfDocumentsToShow > 0
        && (cluster.getDocuments().size() > documentsShown))
    {
        System.out.println(getIndent(level + 1) + "... and "
            + (cluster.getDocuments().size() - documentsShown) + " more\n");
    }

    // finally, if this cluster has subclusters, descend into recursion.
    final int num = 1;
    for (final Cluster subcluster : cluster.getSubclusters())
    {
        displayCluster(level + 1, tag + "." + num, subcluster,
            maxNumberOfDocumentsToShow, clusterDetailsFormatter);
    }
}
 

private List<Cluster> produceBugTrackerTopics(TopicsTransMetric db) {
	final ArrayList<Document> documents = new ArrayList<Document>();
	for (BugTrackerCommentsData comment : db.getBugTrackerComments())
		documents.add(new Document(comment.getSubject(), comment.getText(), "", LanguageCode.ENGLISH, produceUID(comment)));
	return produceTopics(documents);
}
 

public static void displayClusters(final Collection<Cluster> clusters)
{
    displayClusters(clusters, Integer.MAX_VALUE);
}
 

public static void displayClusters(final Collection<Cluster> clusters,
    int maxNumberOfDocumentsToShow)
{
    displayClusters(clusters, maxNumberOfDocumentsToShow,
        ClusterDetailsFormatter.INSTANCE);
}
 

public String formatClusterDetails(Cluster cluster)//显示当前分类的详细参数,包括文档数和打分
{
    final Double score = cluster.getScore();
    return "(" + cluster.getAllDocuments().size() + " docs"
        + (score != null ? ", score: " + numberFormat.format(score) : "") + ")";
}
 

private void clustersToNamedList(List<Cluster> outputClusters,
                                 List<NamedList<Object>> parent, boolean outputSubClusters, int maxLabels) {
  for (Cluster outCluster : outputClusters) {
    NamedList<Object> cluster = new SimpleOrderedMap<>();
    parent.add(cluster);

    // Add labels
    List<String> labels = outCluster.getPhrases();
    if (labels.size() > maxLabels) {
      labels = labels.subList(0, maxLabels);
    }
    cluster.add("labels", labels);

    // Add cluster score
    final Double score = outCluster.getScore();
    if (score != null) {
      cluster.add("score", score);
    }

    // Add other topics marker
    if (outCluster.isOtherTopics()) {
      cluster.add("other-topics", outCluster.isOtherTopics());
    }

    // Add documents
    List<Document> docs = outputSubClusters ? outCluster.getDocuments() : outCluster.getAllDocuments();
    List<Object> docList = new ArrayList<>();
    cluster.add("docs", docList);
    for (Document doc : docs) {
      docList.add(doc.getField(SOLR_DOCUMENT_ID));
    }

    // Add subclusters
    if (outputSubClusters && !outCluster.getSubclusters().isEmpty()) {
      List<NamedList<Object>> subclusters = new ArrayList<>();
      cluster.add("clusters", subclusters);
      clustersToNamedList(outCluster.getSubclusters(), subclusters,
              outputSubClusters, maxLabels);
    }
  }
}
 

private List<Cluster> produceCommitsMessagesTopics(CommitsMessageTopicsTransMetric db) {
	final ArrayList<Document> documents = new ArrayList<Document>();
	for (CommitMessage commitMessage : db.getCommitsMessages())
		documents.add(new Document(commitMessage.getSubject(), commitMessage.getMessage(), "", LanguageCode.ENGLISH, produceUID(commitMessage)));
	return produceTopics(documents);
}
 

private List<Cluster> produceNewsgroupTopics(TopicsTransMetric db) {
	final ArrayList<Document> documents = new ArrayList<Document>();
	for (NewsgroupArticlesData article : db.getNewsgroupArticles())
		documents.add(new Document(article.getSubject(), article.getText(), "", LanguageCode.ENGLISH, produceUID(article)));
	return produceTopics(documents);
}
 

public String formatClusterDetails(Cluster cluster)
        {
//            final Double score = cluster.getScore();
            return "\t" + cluster.getAllDocuments().size();	// + " docs"
//                + (score != null ? ", score: " + numberFormat.format(score) : "") + ")";
        }
 

public static void displayClusters(final Collection<Cluster> clusters,
    int maxNumberOfDocumentsToShow)
{
    displayClusters(clusters, maxNumberOfDocumentsToShow,
        ClusterDetailsFormatter.INSTANCE);
}