pitt.search.semanticvectors.SearchResult Java Examples

public <T extends Comparable<T>,L extends Comparable<L>>void searchDocsUsingTermQuery(L termQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform,QueryTransform<L> termTransform,int numResults)
{
	lock.readLock().lock();
	try
	{
		String query = termTransform.toSV(termQuery); 
		LinkedList<SearchResult> results = search(query,termVecReader,docVecReader,numResults);
		for(SearchResult r : results)
		{
			String filename = r.getObjectVector().getObject().toString();
			docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
		}
	}
	finally
	{
		lock.readLock().unlock();
	}
}
 

public <T extends Comparable<T>>void recommendDocsUsingDocQuery(T docQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform,int numResults,Set<T> exclusions,T minDoc)
{
	lock.readLock().lock();
	try
	{
		String docName = docTransform.toSV(docQuery);
		Set<String> docExclusions = new HashSet<>();
		for(T i : exclusions)
			docExclusions.add(docTransform.toSV(i));
		LinkedList<SearchResult> results = recommend(docName,docVecReader,docVecReader,numResults,docExclusions,new HashSet<String>(),docTransform.toSV(minDoc));
		for(SearchResult r : results)
		{
			String filename = r.getObjectVector().getObject().toString();
			docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
		}
	}
	finally
	{
		lock.readLock().unlock();
	}
}
 

public <T extends Comparable<T>>void searchDocsUsingDocQuery(T termQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform,int numResults)
{
	lock.readLock().lock();
	try
	{
		String docName = docTransform.toSV(termQuery);
		LinkedList<SearchResult> results = search(docName,docVecReader,docVecReader,numResults);
		for(SearchResult r : results)
		{
			String filename = r.getObjectVector().getObject().toString();
			docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
		}
	}
	finally
	{
		lock.readLock().unlock();
	}
}
 

private LinkedList<SearchResult> search(String query,VectorStore queryStore,VectorStore searchStore,int numResults)
{
	 VectorSearcher vecSearcher;
	 LinkedList<SearchResult> results = new LinkedList<>();
	 try 
	 {
		 String[] queryTerms = query.split("\\s+");
		 vecSearcher =
	            new VectorSearcher.VectorSearcherCosine(queryStore,
	                                                    searchStore,
	                                                    luceneUtils,
	                                                    flagConfig,
	                                                    queryTerms);
		 results = vecSearcher.getNearestNeighbors(numResults);
		 
		 
	 } catch (pitt.search.semanticvectors.vectors.ZeroVectorException e) {
		 results = new LinkedList<>();

	}
	 return results;
}
 

private LinkedList<SearchResult> recommend(String query,VectorStore queryStore,VectorStore searchStore,int numResults,Set<String> exclusions,Set<String> inclusions,String minDoc)
{

	 LinkedList<SearchResult> results = new LinkedList<>();
	 try 
	 {
		 String[] queryTerms = query.split("\\s+");
		 VectorStoreRecommender vecSearcher =
	            new VectorStoreRecommender.VectorStoreRecommenderCosine(queryStore,
	                                                    searchStore,
	                                                    luceneUtils,
	                                                    queryTerms,
	                                                    exclusions,
	                                                    inclusions,
	                                                    minDoc);
		 results = vecSearcher.getNearestNeighbors(numResults);
		 
		 
	 } catch (ZeroVectorException zve) {
		 results = new LinkedList<>();
	 }
	 return results;
}
 

/**
 * Find similar users by querying the docstore using a query from the terms passed in
 * @param <T>
 * @param terms
 * @param lUtils : lucene utils
 * @param numResults : max number of results to return
 * @param docResult : the result list of return ids T
 * @param docTransform : the transform from document to return id type T
 */
public <T extends Comparable<T>> void findSimilarUsersFromTerms(String[] terms,LuceneUtils lUtils,int numResults,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform)
{
	List<SearchResult> results;
	try 
	{
		VectorSearcher vecSearcher =
	            new VectorSearcher.VectorSearcherCosine(termVecReader,
	                                                    docVecReader,
	                                                    luceneUtils,
	                                                    flagConfig,
	                                                    terms);
		results = vecSearcher.getNearestNeighbors(numResults);
	} 
	catch (pitt.search.semanticvectors.vectors.ZeroVectorException e) {
		results = new LinkedList<>();
	}
	for(SearchResult r : results)
	{
		String filename = r.getObjectVector().getObject().toString();
		
		docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
	}
}
 

@Test
public void makeQueryVectorTest() {
  VectorStore vectors = table.getRowVectorStore();
  Assert.assertTrue(vectors.getVector("a").measureOverlap(vectors.getVector("b")) >
      vectors.getVector("a").measureOverlap(vectors.getVector("c")));
  Assert.assertTrue(vectors.getVector("c").measureOverlap(vectors.getVector("b")) >
      vectors.getVector("a").measureOverlap(vectors.getVector("c")));

  Vector queryVector = table.makeCellVector(1, "1600");
  for (SearchResult result : table.searchRowVectors(queryVector)) {
    // System.out.println(result.getScore() + ":" + result.getObjectVector().getObject());
  }

  for (int i = 1600; i <= 1700; i += 20) {
    Vector queryVector2 = table.makeCellVector(1, "" + i);
    System.out.println(i + " : " + queryVector.measureOverlap(queryVector2));
  }
}
 

/**
 * Prints the nearest predicate for a particular flagConfig. (Please extend this comment!)
 *
 * @param flagConfig
 * @throws IOException
 */
public static void printNearestPredicate(FlagConfig flagConfig) throws IOException {
  VerbatimLogger.info("Printing predicate results.");
  Vector queryVector = VectorFactory.createZeroVector(flagConfig.vectortype(), flagConfig.dimension());
  VectorSearcher.VectorSearcherBoundProduct predicateFinder;
  try {
    predicateFinder = new VectorSearcher.VectorSearcherBoundProduct(
        VectorStoreReader.openVectorStore(flagConfig.semanticvectorfile(), flagConfig),
        VectorStoreReader.openVectorStore(flagConfig.boundvectorfile(), flagConfig),
        null, flagConfig, queryVector);
    List<SearchResult> bestPredicate = predicateFinder.getNearestNeighbors(1);
    if (bestPredicate.size() > 0) {
      String pred = bestPredicate.get(0).getObjectVector().getObject().toString();
      System.out.println(pred);
    }
  } catch (ZeroVectorException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
  }
}
 

public <T extends Comparable<T>>void searchTermsUsingTermQuery(T termQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> termTransform,int numResults)
{
	lock.readLock().lock();
	try
	{
		String query = termTransform.toSV(termQuery);
		LinkedList<SearchResult> results = search(query,termVecReader,termVecReader,numResults);
		for(SearchResult r : results)
		{
			String filename = r.getObjectVector().getObject().toString();
			docResult.add(new SemVectorResult<>(termTransform.fromSV(filename),r.getScore()));
		}
	}
	finally
	{
		lock.readLock().unlock();
	}
}
 

private LinkedList<SearchResult> search(String query,VectorStore queryStore,VectorStore searchStore,int numResults)
{
	 VectorSearcher vecSearcher;
	 LinkedList<SearchResult> results = new LinkedList<>();
	 try 
	 {
		 String[] queryTerms = query.split("\\s+");
		 vecSearcher =
	            new VectorSearcher.VectorSearcherCosine(queryStore,
	                                                    searchStore,
	                                                    luceneUtils,
	                                                    flagConfig,
	                                                    queryTerms);
		 results = vecSearcher.getNearestNeighbors(numResults);
		 
		 
	 } catch (pitt.search.semanticvectors.vectors.ZeroVectorException e) {
		 results = new LinkedList<>();

	}
	 return results;
}
 

private LinkedList<SearchResult> recommend(String query,VectorStore queryStore,VectorStore searchStore,int numResults,Set<String> exclusions,Set<String> inclusions,String minDoc)
{

	 LinkedList<SearchResult> results = new LinkedList<>();
	 try 
	 {
		 String[] queryTerms = query.split("\\s+");
		 VectorStoreRecommender vecSearcher =
	            new VectorStoreRecommender.VectorStoreRecommenderCosine(queryStore,
	                                                    searchStore,
	                                                    luceneUtils,
	                                                    queryTerms,
	                                                    exclusions,
	                                                    inclusions,
	                                                    minDoc);
		 results = vecSearcher.getNearestNeighbors(numResults);
		 
		 
	 } catch (ZeroVectorException e) {
		 results = new LinkedList<>();
	}
	 finally{}
	 return results;
}
 

/**
 * Find similar users by querying the docstore using a query from the terms passed in
 * @param <T>
 * @param terms
 * @param lUtils : lucene utils
 * @param numResults : max number of results to return
 * @param docResult : the result list of return ids T
 * @param docTransform : the transform from document to return id type T
 */
public <T extends Comparable<T>> void findSimilarUsersFromTerms(String[] terms,LuceneUtils lUtils,int numResults,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform)
{
	List<SearchResult> results;
	try 
	{
		VectorSearcher vecSearcher =
	            new VectorSearcher.VectorSearcherCosine(termVecReader,
	                                                    docVecReader,
	                                                    luceneUtils,
	                                                    flagConfig,
	                                                    terms);
		results = vecSearcher.getNearestNeighbors(numResults);
	} 
	catch (pitt.search.semanticvectors.vectors.ZeroVectorException e) {
		results = new LinkedList<>();
	}
	for(SearchResult r : results)
	{
		String filename = r.getObjectVector().getObject().toString();
		
		docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
	}
}
 

public <T extends Comparable<T>>void searchDocsUsingDocQuery(T termQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform,int numResults)
{
	String docName = docTransform.toSV(termQuery);
	LinkedList<SearchResult> results = search(docName,docVecReader,docVecReader,numResults);
	for(SearchResult r : results)
	{
		String filename = r.getObjectVector().getObject().toString();
		docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
	}
}
 

private static void outputSuggestions(String query) throws Exception  {
  String[] args = new String[] {
      "-queryvectorfile", "termvectors.bin", "-numsearchresults", "10",
      "-luceneindexpath", "permutation_index",
      query };
  List<SearchResult> results = Search.runSearch(FlagConfig.getFlagConfig(args));

  if (results.size() > 0) {
    for (SearchResult result: results) {
      String suggestion = ((ObjectVector)result.getObjectVector()).getObject().toString();
      logger.finest("query:"+query + " suggestion:" + suggestion + " score:" + result.getScore());
    }
  }
}
 

/** Experiment in querying for a particular president's name. */
private static void queryForName(Table table, String name) {
  System.out.println("Querying for name: '" + name + "'");
  Vector queryVector = table.getRowVectorStore().getVector(name);
  for (SearchResult result : table.searchRowVectors(queryVector)) {
    System.out.println(result.toTexTableString(20));
  }
}
 

public <T extends Comparable<T>>void recommendDocsUsingDocQuery(T docQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform,int numResults,Set<T> exclusions,T minDoc)
{
	String docName = docTransform.toSV(docQuery);
	Set<String> docExclusions = new HashSet<>();
	for(T i : exclusions)
		docExclusions.add(docTransform.toSV(i));
	LinkedList<SearchResult> results = recommend(docName,docVecReader,docVecReader,numResults,docExclusions,new HashSet<String>(),docTransform.toSV(minDoc));
	for(SearchResult r : results)
	{
		String filename = r.getObjectVector().getObject().toString();
		docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
	}
}
 

public <T extends Comparable<T>,L extends Comparable<L>>void searchDocsUsingTermQuery(L termQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> docTransform,QueryTransform<L> termTransform,int numResults)
{
	String query = termTransform.toSV(termQuery); 
	LinkedList<SearchResult> results = search(query,termVecReader,docVecReader,numResults);
	for(SearchResult r : results)
	{
		String filename = r.getObjectVector().getObject().toString();
		docResult.add(new SemVectorResult<>(docTransform.fromSV(filename),r.getScore()));
	}
}
 

public <T extends Comparable<T>>void searchTermsUsingTermQuery(T termQuery,ArrayList<SemVectorResult<T>> docResult,QueryTransform<T> termTransform,int numResults)
{
	String query = termTransform.toSV(termQuery);
	LinkedList<SearchResult> results = search(query,termVecReader,termVecReader,numResults);
	for(SearchResult r : results)
	{
		String filename = r.getObjectVector().getObject().toString();
		docResult.add(new SemVectorResult<>(termTransform.fromSV(filename),r.getScore()));
	}
}
 

/**
	 * Method to process a single analogy
	 * Questions will be skipped if any of the four terms are not present
	 * @return
	 */
	
	
	public void processAnalogy()
	{
		
	String[] queryTerms = inLine.toLowerCase().split(" ");
    
	//not a proportional analogy
	if (queryTerms.length < 4) 
	{
		System.out.println(threadno+": "+inLine+": Skipping line");
		return;
	}
    	
	Vector aTermVector = null;
	Vector bTermVector = null;
	Vector cTermVector = null;

	
	String missingTerms = "";
	if (!termVectors.containsVector(queryTerms[0])) missingTerms = missingTerms + queryTerms[0]+"; ";
	if (!termVectors.containsVector(queryTerms[1])) missingTerms = missingTerms + queryTerms[1]+"; ";
	if (!termVectors.containsVector(queryTerms[2])) missingTerms = missingTerms + queryTerms[2]+"; ";
	if (!termVectors.containsVector(queryTerms[3])) missingTerms = missingTerms + queryTerms[3]+"; ";
	
	if (!missingTerms.isEmpty())
	{
		System.out.println(threadno+": "+"Missing terms "+missingTerms);
		return;
	}
	
	aTermVector = termVectors.getVector(queryTerms[0]);
	bTermVector = termVectors.getVector(queryTerms[1]);
	cTermVector = termVectors.getVector(queryTerms[2]);

	Vector cueVector = bTermVector.copy();
	cueVector.superpose(aTermVector, -1, null);
	cueVector.superpose(cTermVector, +1, null);
	cueVector.normalize();
		
	int rank = 1;
	String object = "";
	
	
	try {
		VectorSearcher.VectorSearcherCosine analogySearcher
		 	 = new VectorSearcher.VectorSearcherCosine(termVectors,termVectors,null, flagConfig, cueVector);
		
		//get top 1000 results plus the three query terms
		LinkedList<SearchResult> results = analogySearcher.getNearestNeighbors(1003);
		
		
		for (SearchResult sr:results)
		{
			object = sr.getObjectVector().getObject().toString();
		
			//result found 
			if (object.equals(queryTerms[3]))
				break;
			
			//ignore query terms
			if (!(object.equals(queryTerms[0])
					|| object.equals(queryTerms[1])
					  || object.equals(queryTerms[2]))
						)
				rank++;
			
		}
		} 	catch (ZeroVectorException e) {
	// TODO Auto-generated catch block
	e.printStackTrace();
	System.out.println(threadno+": "+"Error on example "+ inLine);
}
	
	//calculate reciprocal rank as a more granular metric than accuracy
	double reciprank = 1 / (double) rank;
	if (reciprank < 0.001) reciprank = 0;
	
	examplessubset.incrementAndGet();
	exampletot.incrementAndGet();;
	recipsubset.add(reciprank);
	reciptot.add(reciprank);
	
	//correct result (top ranked other than query terms)
	if (reciprank == 1)
		{
			accsubset.incrementAndGet();;
			acctot.incrementAndGet();;
		}
	
	System.out.println(threadno+": "+inLine +" --> "+object+" "+rank+" "+reciprank);
	
		
}
 

/**
 * This nearest neighbor search is implemented in the abstract
 * VectorSearcher class itself: this enables all subclasses to reuse
 * the search whatever scoring method they implement.  Since query
 * expressions are built into the VectorSearcher,
 * getNearestNeighbors no longer takes a query vector as an
 * argument.
 * @param numResults the number of results / length of the result list.
 */
public LinkedList<SearchResult> getNearestNeighbors(int numResults) {
  LinkedList<SearchResult> results = new LinkedList<>();
  double score, threshold = -1;
  int duplicatesRemoved = 0;
  Enumeration<ObjectVector> vecEnum = searchVecStore.getAllVectors();

  while (vecEnum.hasMoreElements()) {
  	
  	ObjectVector testElement = vecEnum.nextElement();
  	
  	// ignore excluded items
  	if (exclusions.contains(testElement.getObject().toString()))
  		continue;
  	
  	//only allow includions if specified
  	if (inclusions != null && inclusions.size()>0 && !inclusions.contains(testElement.getObject().toString()))
  		continue;
  	
  	// ignore items greater than minDoc id (assume doc ids string ordering is useful)
  	if (minDoc != null && testElement.getObject().toString().compareTo(minDoc) < 0)
  		continue;
  	
  	
    // Initialize result list if just starting.
    if (results.size() == 0) {
      score = getScore(testElement.getVector());
      results.add(new SearchResult(score, testElement));
      continue;
    }

    // Test this element.

    score = getScore(testElement.getVector());

    // This is a way of using the Lucene Index to get term and
    // document frequency information to reweight all results. It
    // seems to be good at moving excessively common terms further
    // down the results. Note that using this means that scores
    // returned are no longer just cosine similarities.
    if (this.luceneUtils != null) {
      score = score *
          luceneUtils.getGlobalTermWeightFromString((String) testElement.getObject());
    }

    if (score > threshold) 
    {
      boolean added = false;
      boolean duplicate = false;
      for (int i = 0; i < results.size() && !added && !duplicate; ++i) 
      {
      	SearchResult r = results.get(i);
      	
      	if (score == r.getScore())
      	{
      		ObjectVector rVec = (ObjectVector) r.getObjectVector();
      		double overlap = getScore(rVec.getVector(),testElement.getVector());
      		double epsilon = Math.abs(overlap-1);
      		if (epsilon < 0.000001)
      		{
      			duplicatesRemoved++;
      			duplicate = true;
      			continue;
      		}
      	}
      	// Add to list if this is right place.
      	if (score > r.getScore() && added == false) 
      	{
      		results.add(i, new SearchResult(score, testElement));
      		added = true;
      	}
      }
      // Prune list if there are already numResults.
      if (results.size() > numResults) 
      {
        results.removeLast();
        threshold = results.getLast().getScore();
      } 
      else 
      {
        if (results.size() < numResults && !added && !duplicate) 
        {
          results.add(new SearchResult(score, testElement));
        }
      }
    }
  }
  if (duplicatesRemoved > 0)
  	logger.info("removed "+duplicatesRemoved+" duplicates");
  return results;
}
 

public void testQuery(FlagConfig flagConfig, String searchfile, String indexfile, String query )
{
	VectorSearcher vs;
	LuceneUtils lUtils = null;
	CloseableVectorStore queryVecReader, searchVecReader;
	LinkedList<SearchResult> results;
	int numResults = 20;

	BeagleUtils utils = BeagleUtils.getInstance();
	utils.setFFTCacheSize(100);

	try
	{
		queryVecReader = VectorStoreReader.openVectorStore(indexfile, flagConfig);
		searchVecReader = VectorStoreReader.openVectorStore(searchfile, flagConfig);

		//BeagleCompoundVecBuilder bcb = new BeagleCompoundVecBuilder ();

		String[] queryTerms = query.split(" ");

		// Create VectorSearcher and search for nearest neighbors.
		vs = new BeagleVectorSearcher( queryVecReader, searchVecReader, lUtils, flagConfig, queryTerms);
		System.err.print("Searching term vectors, searchtype BEAGLE ... ");
		queryVecReader.close();
		searchVecReader.close();

		results = vs.getNearestNeighbors(numResults);

	}
	catch (Exception e)
	{
		System.err.println(e.getMessage());
		results = new LinkedList<SearchResult>();
	}

	// Print out results.
	if (results.size() > 0) {
		System.err.println("Search output follows ...\n");
		for (SearchResult result: results) {
			System.out.println(result.getScore() + ":" +
                                                  ((ObjectVector)result.getObjectVector()).getObject().toString());
		}
	} else {
		System.err.println("No search output.");
	}
}
 

public List<Passage> query(Question question) {
	List<Passage> passages = new ArrayList<>();
	VectorSearcher[] sv_searchers;
	try {
		sv_searchers = new VectorSearcher[]{
				new VectorSearcher.VectorSearcherCosine( 
				        queryVecReader, resultsVecReader, luceneUtils, 
				        fconfig, question.getTokens().toArray(new String[]{})),
		        /*new VectorSearcher.VectorSearcherLucene(luceneUtils, 
				        fconfig, question.getTokens().toArray(new String[]{})),
		        new VectorSearcher.VectorSearcherMaxSim( 
				        queryVecReader, resultsVecReader, luceneUtils, 
				        fconfig, question.getTokens().toArray(new String[]{})),*/
		        new VectorSearcher.VectorSearcherMinSim(
				        queryVecReader, resultsVecReader, luceneUtils, 
				        fconfig, question.getTokens().toArray(new String[]{})),
		        /*new VectorSearcher.VectorSearcherSubspaceSim(
				        queryVecReader, resultsVecReader, luceneUtils, 
				        fconfig, question.getTokens().toArray(new String[]{})),*/
		};
	
		System.out.println("sv_searchers = " + sv_searchers);
		for (VectorSearcher sv_searcher : sv_searchers)
		if (sv_searcher != null) {
			List<SearchResult> results = sv_searcher.getNearestNeighbors(10);
			System.out.println("result = " + results);
			int rank = 0;
			for (SearchResult result: results) {
				passages.add(new Passage(
						"semvec", 											// Engine
						"",	// Title
						"",	// Text
						result.getObjectVector().getObject().toString())													// Reference
						.score("SEMVEC_RANK", (double) rank++)				// Rank
						.score("SEMVEC_SCORE", (double) result.getScore())	// Score
						.score("SEMVEC_PRESENT", 1.0)
						);
			}
		}
		/*sv_searcher = new VectorSearcher.VectorSearcherCosine( 
        queryVecReader, resultsVecReader, luceneUtils, 
        fconfig, question.tokens.toArray(new String[]{}));*/
	} catch (ZeroVectorException e) {
	// TODO: Under what circumstances does this happen?
	e.printStackTrace();
	}
	return fillFromSources(passages);
}