com.googlecode.javaewah.EWAHCompressedBitmap Java Examples

@Test
@Ignore("This test requires an external resource. This can lead to false positive failures.")
public void bitmapTest() throws OWLOntologyCreationException, OBOFormatParserException, IOException {
	Set<Integer> ixs = new HashSet<Integer>();
	ixs.add(99);
	ixs.add(2000);
	ixs.add(7777);
	EWAHCompressedBitmap bm = bm(ixs);
	Set<Integer> ixs2 = new HashSet<Integer>();
	for (int i : bm.toArray()) {
		System.out.println(i);
		ixs.add(i);
	}
	assert(ixs.equals(ixs2));
	
}
 

public EWAHCompressedBitmap ancsProperBitmapCachedModifiable(OWLClass c) {
	if (properSuperclassBitmapMap != null && properSuperclassBitmapMap.containsKey(c)) {
		return properSuperclassBitmapMap.get(c);
	}
	
	Set<Integer> ancsInts = new HashSet<Integer>();
	for (Node<OWLClass> anc : reasoner.getSuperClasses(c, false)) {
		// TODO - verify robust for non-Rep elements
		OWLClass ac = anc.getRepresentativeElement();
		if (ac.equals(thing))
			continue;
		Integer ix = classIndex.get(ac);
		if (ix == null) {
			msg("??"+anc);
		}
		ancsInts.add(ix.intValue());
	}


	//msg(c + " ancs = "+caints.size());
	EWAHCompressedBitmap bm = bm(ancsInts);
	if (properSuperclassBitmapMap == null)
		properSuperclassBitmapMap = new HashMap<OWLClass,EWAHCompressedBitmap>();
	properSuperclassBitmapMap.put(c, bm);
	return bm;		
}
 

private EWAHCompressedBitmap getElementsForAttributeAsBitmao(OWLClass c) throws UnknownOWLClassException {
	if (classToElementBitmapMap == null)
		classToElementBitmapMap = new HashMap<OWLClass,EWAHCompressedBitmap>();
	if (classToElementBitmapMap.containsKey(c)) {
		return classToElementBitmapMap.get(c);
	}
	Set<OWLNamedIndividual> inds = getElementsForAttribute(c);
	Set<Integer> indInts = new HashSet<Integer>();
	makeIndividualIndex();
	for (OWLNamedIndividual i : inds) {
		indInts.add(individualIndex.get(i));
	}
	EWAHCompressedBitmap bm = convertIntsToBitmap(indInts);
	classToElementBitmapMap.put(c, bm);
	return bm;
}
 

@Override
public AttributePairScores getPairwiseSimilarity(OWLClass c, OWLClass d)
		throws UnknownOWLClassException {
	AttributePairScores s = new AttributePairScores(c,d);
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(c);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(d);

	s.simjScore =  bmc.andCardinality(bmd) / (double) bmc.orCardinality(bmd);
	s.asymmetricSimjScore =  bmc.andCardinality(bmd) / (double) bmd.cardinality();
	s.inverseAsymmetricSimjScore =  bmc.andCardinality(bmd) / (double) bmc.cardinality();

	ScoreAttributeSetPair sap = getLowestCommonSubsumerWithIC(c, d);
	s.lcsIC = sap.score;
	s.lcsSet = sap.attributeClassSet;
	return s;
}
 

private EWAHCompressedBitmap ancsProperBitmapCachedModifiable(OWLClass c) {
	if (properSuperclassBitmapMap != null && properSuperclassBitmapMap.containsKey(c)) {
		return properSuperclassBitmapMap.get(c);
	}

	Set<Integer> ancsInts = new HashSet<Integer>();
	for (Node<OWLClass> anc : reasoner.getSuperClasses(c, false)) {
		// TODO - verify robust for non-Rep elements
		OWLClass ac = getDeterministicRepresentative(anc);
		if (ac.equals(thing))
			continue;
		ancsInts.add(classIndex.get(ac));
	}

	EWAHCompressedBitmap bm = convertIntsToBitmap(ancsInts);
	if (properSuperclassBitmapMap == null)
		properSuperclassBitmapMap = new HashMap<OWLClass,EWAHCompressedBitmap>();
	properSuperclassBitmapMap.put(c, bm);
	return bm;		
}
 

/**
 * Creates a new SparseBitmap that is the logical <code>or</code> of all the given bitmaps.
 *
 * @param bitmaps Bitmaps to compute the logical <code>or</code> for
 * @return SparseBitmap that has the values set according to the <code>or</code> of the given bitmaps.
 */
public static SparseBitmap batchOr(final SparseBitmap... bitmaps) {
	if (bitmaps.length < 1) {
		return SparseBitmap.createFromUnsortedData();
	}

	if (bitmaps.length < 2) {
		return bitmaps[0];
	}

	EWAHCompressedBitmap firstMap = bitmaps[0].bitmap;

	for (int index = 1; index < bitmaps.length; ++index) {
		firstMap = firstMap.or(bitmaps[index].bitmap);
	}

	return new SparseBitmap(firstMap);
}
 

@Override
protected void trainInternal(List<Item> items, List<ClickData> transactions) {
	//iterate over all newly published items
	for (Item item : items) {
		//if the keywords of the item are not null, create a keyword map an save it
		if(item.keywords!=null){
			//create an empty bitmap of keyword IDs
			EWAHCompressedBitmap itemKeywords = new EWAHCompressedBitmap();
			itemKeywordMap.put(item.id, itemKeywords);
			//iterate over the item's keywords
			for (String keyword : item.keywords.keySet()) {				
				//if the keyword is unknown, generate a new INT id for it
				if(!keywordMap.containsKey(keyword)){
					keywordMap.put(keyword, ++currentKeywordCounter);
					keywordItemMap.put(currentKeywordCounter, new LongOpenHashSet());
				}
				//extract the id of the keyword
				int keywordInt = keywordMap.get(keyword);
				//set the bit for this keyword in the item's bitmap
				itemKeywords.set(keywordInt);
				//also set the item in the keyword's reverse lookup map
				keywordItemMap.get(keywordInt).add(item.id);
			}
		}
	}
}
 

/**
 * Creates a new SparseBitmap that is the logical <code>or</code> of all the given bitmaps.
 *
 * @param bitmaps Bitmaps to compute the logical <code>or</code> for
 * @return SparseBitmap that has the values set according to the <code>or</code> of the given
 * bitmaps.
 */
public static SparseBitmap batchOr(final SparseBitmap... bitmaps) {
    if (bitmaps.length < 1) {
        return SparseBitmap.createFromUnsortedData();
    }

    if (bitmaps.length < 2) {
        return bitmaps[0];
    }

    EWAHCompressedBitmap firstMap = bitmaps[0].bitmap;

    for (int index = 1; index < bitmaps.length; ++index) {
        firstMap = firstMap.or(bitmaps[index].bitmap);
    }

    return new SparseBitmap(firstMap);
}
 

private double getAttributeGraphInformationContentSimilarity(
		int cix, int dix) throws UnknownOWLClassException {
	long t = System.currentTimeMillis();
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(cix);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(dix);
	EWAHCompressedBitmap cad = bmc.and(bmd);
	EWAHCompressedBitmap cud = bmc.or(bmd);

	double sumICboth = 0;
	double sumICunion = 0;
	// faster than translating to integer list
	IntIterator it = cud.intIterator();
	while (it.hasNext()) {
		int x = it.next();
		double ic = getInformationContentForAttribute(x);
		// TODO - we can avoid doing this twice by using xor in the bitmap
		sumICunion += ic;

		if (cad.get(x)) {
			sumICboth += ic;
		}
	}
	totalTimeGIC += tdelta(t);
	this.totalCallsGIC++;
	return sumICboth / sumICunion;
}
 

@Override
public double getAttributeJaccardSimilarity(OWLClass c, OWLClass d) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(c);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(d);

	return bmc.andCardinality(bmd) / (double) bmc.orCardinality(bmd);
}
 

private Set<Node<OWLClass>> getNamedCommonSubsumers(int cix, int dix) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(cix);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(dix);
	EWAHCompressedBitmap cad = bmc.and(bmd);
	Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
	for (int ix : cad.toArray()) {
		OWLClassNode node = new OWLClassNode(classArray[ix]);
		nodes.add(node);
	}
	return nodes;
}
 

@Override
public Set<Node<OWLClass>> getNamedCommonSubsumers(OWLNamedIndividual i,
		OWLNamedIndividual j) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);
	EWAHCompressedBitmap cad = bmc.and(bmd);
	Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
	for (int ix : cad.toArray()) {
		OWLClassNode node = new OWLClassNode(classArray[ix]);
		nodes.add(node);
	}
	return nodes;
}
 

private Set<Node<OWLClass>> getNamedUnionSubsumers(OWLNamedIndividual i,
		OWLNamedIndividual j) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);
	EWAHCompressedBitmap cad = bmc.or(bmd);
	Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
	for (int ix : cad.toArray()) {
		OWLClassNode node = new OWLClassNode(classArray[ix]);
		nodes.add(node);
	}
	return nodes;
}
 

@Override
public Set<Node<OWLClass>> getNamedLowestCommonSubsumers(OWLClass c,
		OWLClass d) throws UnknownOWLClassException {
	EWAHCompressedBitmap cad = getNamedLowestCommonSubsumersAsBitmap(c, d);
	Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
	// TODO - optimize this & ensure all elements of an equivalence set are included
	for (int ix : cad.toArray()) {
		OWLClassNode node = new OWLClassNode(classArray[ix]);
		nodes.add(node);
	}
	return nodes;

}
 

private double computeSimJwithBM(EWAHCompressedBitmap iAttsBM, EWAHCompressedBitmap jAttsBM) {
	int cadSize = iAttsBM.andCardinality(jAttsBM);
	int cudSize = iAttsBM.orCardinality(jAttsBM);
	double simJPct = 0;
	if (cudSize != 0) {
		simJPct = (cadSize * 100) / cudSize;
	}
	return simJPct;
}
 

private Set<Node<OWLClass>> getNamedLowestCommonSubsumers(int cix, int dix) throws UnknownOWLClassException {
	EWAHCompressedBitmap cad = getNamedLowestCommonSubsumersAsBitmap(cix, dix);
	Set<Node<OWLClass>> nodes = new HashSet<Node<OWLClass>>();
	// TODO - optimize this & ensure all elements of an equivalence set are included
	for (int ix : cad.toArray()) {
		OWLClassNode node = new OWLClassNode(classArray[ix]);
		nodes.add(node);
	}
	return nodes;

}
 

private EWAHCompressedBitmap getNamedLowestCommonSubsumersAsBitmap(OWLClass c,
		OWLClass d) throws UnknownOWLClassException  {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(c);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(d);
	EWAHCompressedBitmap cad = bmc.and(bmd);	
	int[] csInts = cad.toArray();
	for (int ix : csInts) {
		cad = cad.andNot(ancsProperBitmapCachedModifiable(classArray[ix]));
	}
	return cad;
}
 

private EWAHCompressedBitmap getNamedLowestCommonSubsumersAsBitmap(int cix, int dix)
		throws UnknownOWLClassException  {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(cix);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(dix);
	EWAHCompressedBitmap cad = bmc.and(bmd);	
	int[] csInts = cad.toArray();
	for (int ix : csInts) {
		cad = cad.andNot(ancsProperBitmapCachedModifiable(classArray[ix]));
	}
	return cad;
}
 

private ScoreAttributeSetPair getLowestCommonSubsumerWithICNoCache(int cix, int dix)
		throws UnknownOWLClassException {
	long t = System.currentTimeMillis();
	EWAHCompressedBitmap cad = getNamedLowestCommonSubsumersAsBitmap(cix, dix);

	Set<OWLClass> lcsClasses = new HashSet<OWLClass>();
	double maxScore = 0.0;
	for (int ix : cad.toArray()) {
		// TODO: use getAttributeTriadScore here
		double score = 
				getInformationContentForAttribute(ix);
		double sdiff = score - maxScore;
		if (sdiff >= 0) {
			if (sdiff > 0.01) {
				lcsClasses= new HashSet<OWLClass>(Collections.singleton(classArray[ix]));
				maxScore = score;					
			}
			else {
				lcsClasses.add(classArray[ix]);
				maxScore = score;					
			}
		}
		//			if (score == maxScore) {
		//				lcsClasses.add(classArray[ix]);
		//				maxScore = score;
		//			}
		//			else if (score >= maxScore) {
		//				lcsClasses= new HashSet<OWLClass>(Collections.singleton(classArray[ix]));
		//				maxScore = score;
		//			}
	}
	if (lcsClasses.size() == 0) {
		// TODO - remove obsoletes
		//LOG.warn("Hmmmm "+c+" "+d+" "+lcs);
	}
	totalTimeLCSIC += tdelta(t);
	this.totalCallsLCSIC++;

	return new ScoreAttributeSetPair(maxScore, lcsClasses);
}
 

@Override
public int getAttributeJaccardSimilarityAsPercent(OWLClass c,
		OWLClass d) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(c);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(d);

	return (bmc.andCardinality(bmd) * 100) / bmc.orCardinality(bmd);
}
 

@Override
public double getElementJaccardSimilarity(OWLNamedIndividual i,
		OWLNamedIndividual j) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);

	return bmc.andCardinality(bmd) / (double) bmc.orCardinality(bmd);
}
 

@Override
public int getElementJaccardSimilarityAsPercent(OWLNamedIndividual i,
		OWLNamedIndividual j) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);

	return (bmc.andCardinality(bmd) * 100) / bmc.orCardinality(bmd);
}
 

@Override
public double getAsymmetricAttributeJaccardSimilarity(OWLClass c, OWLClass d) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(c);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(d);

	return bmc.andCardinality(bmd) / (double) bmd.cardinality();
}
 

public double getAsymmetricElementJaccardSimilarity(OWLNamedIndividual i,
		OWLNamedIndividual j) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);

	return bmc.andCardinality(bmd) / (double) bmd.cardinality();
}
 

public int getAsymmetricElementJaccardSimilarityAsPercent(OWLNamedIndividual i,
		OWLNamedIndividual j) throws UnknownOWLClassException {
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);

	return (bmc.andCardinality(bmd) * 100) / bmd.cardinality();
}
 

@Override
public double getElementGraphInformationContentSimilarity(
		OWLNamedIndividual i, OWLNamedIndividual j) throws UnknownOWLClassException {
	// TODO - optimize
	long t = System.currentTimeMillis();
	EWAHCompressedBitmap bmc = ancsBitmapCachedModifiable(i);
	EWAHCompressedBitmap bmd = ancsBitmapCachedModifiable(j);
	EWAHCompressedBitmap cad = bmc.and(bmd);
	EWAHCompressedBitmap cud = bmc.or(bmd);


	//Set<Node<OWLClass>> ci = getNamedCommonSubsumers(i, j);
	//Set<Node<OWLClass>> cu = getNamedUnionSubsumers(i, j);
	double sumICboth = 0;
	double sumICunion = 0;

	// faster than translating to integer list
	IntIterator it = cud.intIterator();
	while (it.hasNext()) {
		int x = it.next();
		double ic = getInformationContentForAttribute(x);
		// TODO - we can avoid doing this twice by using xor in the bitmap
		sumICunion += ic;

		if (cad.get(x)) {
			sumICboth += ic;
		}
	}

	totalTimeGIC += tdelta(t);
	this.totalCallsGIC++;

	return sumICboth / sumICunion;
}
 

public EWAHCompressedBitmap ancsBitmapCachedModifiable(OWLClass c) {
	if (superclassBitmapMap != null && superclassBitmapMap.containsKey(c)) {
		return superclassBitmapMap.get(c);
	}
	Set<Integer> caints = ancsIntsCachedModifiable(c);
	//msg(c + " ancs = "+caints.size());
	EWAHCompressedBitmap bm = bm(caints);
	if (superclassBitmapMap == null)
		superclassBitmapMap = new HashMap<OWLClass,EWAHCompressedBitmap>();
	superclassBitmapMap.put(c, bm);
	return bm;		
}
 

@Override
public BitmapAggregator priorityQueueOrAggregator() {
  return new BitmapAggregator() {
    @Override
    public Bitmap aggregate(final Iterable<Bitmap> bitmaps) {
      Iterator<EWAHCompressedBitmap> iterator = new Iterator<EWAHCompressedBitmap>() {
        final Iterator<Bitmap> i = bitmaps.iterator();

        @Override
        public boolean hasNext() {
          return i.hasNext();
        }

        @Override
        public EWAHCompressedBitmap next() {
          return ((EwahBitmapWrapper) i.next()).bitmap;
        }

        @Override
        public void remove() {
          throw new UnsupportedOperationException();
        }
      };
      return new EwahBitmapWrapper(FastAggregation.or(iterator));
    }
  };
}
 

/**
 * Calculates the Jaccard similarity of two items in terms of their keywords
 * @param itemKeywords Keywords of the first item
 * @param anotherItemsKeywords Keywords of the other item 
 * @return the Jaccard similarity
 */
protected double similarity(EWAHCompressedBitmap itemKeywords, EWAHCompressedBitmap anotherItemsKeywords) {
	int intersection = itemKeywords.andCardinality(anotherItemsKeywords);
	if (intersection == 0) {
		// if the intersection is 0 -> return 0
		return 0;
	}
	// otherwise calculate the jaccard
	int union = itemKeywords.orCardinality(anotherItemsKeywords);
	return intersection * 1d / union * 1d;
}
 

/**
 * Calculates the Jaccard similarity of two items in terms of the sets of users 
 * that clicked on them respectively
 * @param itemClicks Users that clicked on the first item
 * @param anotherItemsClicks Users that clicked on the other item 
 * @return the Jaccard similarity
 */
protected double similarity(EWAHCompressedBitmap itemClicks, EWAHCompressedBitmap anotherItemsClicks) {
	int intersection = itemClicks.andCardinality(anotherItemsClicks);
	if (intersection == 0) {
		// if the intersection is 0 -> return 0
		return 0;
	}
	// otherwise calculate the jaccard
	int union = itemClicks.orCardinality(anotherItemsClicks);
	return intersection * 1d / union * 1d;
}