Java Code Examples for it.unimi.dsi.fastutil.ints.IntArrayList#size()

protected static List<PositionListIndex> fetchPositionListIndexesStatic(List<HashMap<String, IntArrayList>> clusterMaps, boolean isNullEqualNull) {
	List<PositionListIndex> clustersPerAttribute = new ArrayList<>();
	for (int columnId = 0; columnId < clusterMaps.size(); columnId++) {
		List<IntArrayList> clusters = new ArrayList<>();
		HashMap<String, IntArrayList> clusterMap = clusterMaps.get(columnId);
		
		if (!isNullEqualNull)
			clusterMap.remove(null);
		
		for (IntArrayList cluster : clusterMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
		
		clustersPerAttribute.add(new PositionListIndex(columnId, clusters));
	}
	return clustersPerAttribute;
}
 

public static int findViolationsFor(final IntArrayList cluster, final int[] invertedRhsPLI) {
	int maxSize = 0;
	Int2IntAVLTreeMap clusterMap = new Int2IntAVLTreeMap();
	clusterMap.defaultReturnValue(0);
	for (int tuple : cluster) {
		int clusterId = invertedRhsPLI[tuple];
		if (clusterId == -1) {
			// single-tuple cluster, skip
			continue;
		}
		int count = clusterMap.get(clusterId);
		count += 1;
		clusterMap.put(clusterId, count);
		if (count > maxSize) {
			maxSize = count;
		}
	}
	if (maxSize > 0) {
		return cluster.size() - maxSize;
	}
	// if there is no cluster in the list, there are only single-tuple clusters on the rhs and thus, only one keeper
	return cluster.size() - 1;
}
 

protected static List<PositionListIndex> fetchPositionListIndexesStatic(List<HashMap<String, IntArrayList>> clusterMaps, boolean isNullEqualNull) {
	List<PositionListIndex> clustersPerAttribute = new ArrayList<>();
	for (int columnId = 0; columnId < clusterMaps.size(); columnId++) {
		List<IntArrayList> clusters = new ArrayList<>();
		HashMap<String, IntArrayList> clusterMap = clusterMaps.get(columnId);
		
		if (!isNullEqualNull)
			clusterMap.remove(null);
		
		for (IntArrayList cluster : clusterMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
		
		clustersPerAttribute.add(new PositionListIndex(columnId, clusters));
	}
	return clustersPerAttribute;
}
 

private void fetchNonFdsWindowingOverClusters(Set<OpenBitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tMoving window over small clusters ...");
	for (PositionListIndex pli : plis) {
		boolean selectSmallClustersOnly = pli.getClusters().size() < this.attributeThreshold; 	// If there are too few clusters, then the clusters are large and we have already executed sufficient comparisons between the records of these clusters
		
		for (IntArrayList cluster : pli.getClusters()) {
			if (selectSmallClustersOnly && (cluster.size() > this.windowSize))					// But if the current cluster is very small, we should still use it for comparisons (the other cluster(s) must be very large)
				continue;
			
			for (int recordIndex = 0; recordIndex < cluster.size(); recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = recordIndex + 1; partnerRecordIndex < Math.min(recordIndex + this.windowSize, cluster.size()); partnerRecordIndex++) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}
 

public boolean refines(int[][] lhsInvertedPlis, int[] rhs) {
	for (IntArrayList cluster : this.clusters) {
		Object2IntOpenHashMap<IntArrayList> clustersMap = new Object2IntOpenHashMap<>(cluster.size());
		
		// Check if all subclusters of this cluster point into the same other clusters
		for (int recordId : cluster) {
			IntArrayList additionalLhsCluster = this.buildClusterIdentifier(recordId, lhsInvertedPlis);
			if (additionalLhsCluster == null)
				continue;
			
			if (clustersMap.containsKey(additionalLhsCluster)) {
				if ((rhs[recordId] == -1) || (clustersMap.getInt(additionalLhsCluster) != rhs[recordId]))
					return false;
			}
			else {
				clustersMap.put(additionalLhsCluster, rhs[recordId]);
			}
		}
	}
	return true;
}
 

private void fetchNonFdsFromClustersTopsAndBottoms(Set<BitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tComparing window on clusters tops and bottoms ...");
	for (PositionListIndex pli : plis) {
		for (IntArrayList cluster : pli.getClusters()) {
			if (cluster.size() < this.windowSize)
				continue;
			
			for (int recordIndex = 0; recordIndex < this.windowSize; recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = cluster.size() - 1; partnerRecordIndex > cluster.size() - this.windowSize; partnerRecordIndex--) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					if (recordId == partnerRecordId)
						continue;
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}
 

protected List<PositionListIndex> fetchPositionListIndexes(List<HashMap<String, IntArrayList>> clusterMaps, boolean isNullEqualNull) {
	List<PositionListIndex> clustersPerAttribute = new ArrayList<>();
	for (int columnId = 0; columnId < clusterMaps.size(); columnId++) {
		List<IntArrayList> clusters = new ArrayList<>();
		HashMap<String, IntArrayList> clusterMap = clusterMaps.get(columnId);
		
		if (!isNullEqualNull)
			clusterMap.remove(null);
		
		for (IntArrayList cluster : clusterMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
		
		clustersPerAttribute.add(new PositionListIndex(columnId, clusters));
	}
	return clustersPerAttribute;
}
 

public boolean refines(int[][] lhsInvertedPlis, int[] rhs) {
	for (IntArrayList cluster : this.clusters) {
		Object2IntOpenHashMap<IntArrayList> clustersMap = new Object2IntOpenHashMap<>(cluster.size());
		
		// Check if all subclusters of this cluster point into the same other clusters
		for (int recordId : cluster) {
			IntArrayList additionalLhsCluster = this.buildClusterIdentifier(recordId, lhsInvertedPlis);
			if (additionalLhsCluster == null)
				continue;
			
			if (clustersMap.containsKey(additionalLhsCluster)) {
				if ((rhs[recordId] == -1) || (clustersMap.getInt(additionalLhsCluster) != rhs[recordId]))
					return false;
			}
			else {
				clustersMap.put(additionalLhsCluster, rhs[recordId]);
			}
		}
	}
	return true;
}
 

public PositionListIndex intersect(PositionListIndex otherPLI) {
	Int2IntOpenHashMap hashedPLI = otherPLI.asHashMap();
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(this, hashedPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}
 

public PositionListIndex intersect(int[] otherPLI) {
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(otherPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}
 

private boolean sendSyncs() {
  boolean sent = false;
  IntArrayList sendingGroup = sendingGroupsWorkers.get(sendGroupIndex);
  for (int j = 0; j < sendingGroup.size(); j++) {
    int worker = sendingGroup.getInt(j);
    IntArrayList workerTargets = workerToTargets.get(worker);
    int targetIndex = sendWorkerTaskIndex.get(worker);

    for (int i = targetIndex; i < workerTargets.size(); i++) {
      int target = workerTargets.getInt(i);
      RoutingParameters parameters = targetRoutes.get(target);
      byte[] message = new byte[1];
      int flags = MessageFlags.SYNC_EMPTY;
      if (delegate.sendMessage(representSource, message, target, flags, parameters)) {
        sent = true;
        // advance the index
        targetIndex++;
        syncSent.add(target);
        sendWorkerTaskIndex.put(worker, targetIndex);
      } else {
        // no point in going further
        return false;
      }
    }
  }


  if (sent) {
    if (syncSent.size() == targetsArray.length) {
      for (int source : thisWorkerSources) {
        sourceStates.put(source, ReceiverState.SYNCED);
      }
    }
    finishedSendGroups.add(sendGroupIndex);
  }
  return true;
}
 

private void fetchNonFdsFromClustersTopsAndBottomsProgressive(Set<BitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tComparing window on clusters tops and bottoms ...");
	for (PositionListIndex pli : plis) {
		int currentWindowDistance = 1;
		int newNonFDs = 0;
		do {
			newNonFDs = 0;
			
			for (IntArrayList cluster : pli.getClusters()) {
				int recordIndex = currentWindowDistance;
				int partnerRecordIndex = cluster.size() - currentWindowDistance; 
				
				if ((recordIndex >= cluster.size()) || (partnerRecordIndex < 0) || (recordIndex == partnerRecordIndex))
					continue;
				
				int recordId = cluster.getInt(recordIndex);
				int partnerRecordId = cluster.getInt(partnerRecordIndex);
				
				if (negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId])))
					newNonFDs++;
			}
			
			currentWindowDistance++;
		}
		while (newNonFDs > this.progressiveThreshold);
	}
}
 

public void runNext(UCCList newNonUCCs, int[][] compressedRecords) {
	this.windowDistance++;
	int numNewNonFds = 0;
	int numComparisons = 0;
	BitSet equalAttrs = new BitSet(this.posCover.getNumAttributes());
	
	int previousNewNonFdsSize = newNonUCCs.size();
	Iterator<IntArrayList> clusterIterator = this.clusters.iterator();
	while (clusterIterator.hasNext()) {
		IntArrayList cluster = clusterIterator.next();
		
		if (cluster.size() <= this.windowDistance) {
			clusterIterator.remove();
			continue;
		}
		
		for (int recordIndex = 0; recordIndex < (cluster.size() - this.windowDistance); recordIndex++) {
			int recordId = cluster.getInt(recordIndex);
			int partnerRecordId = cluster.getInt(recordIndex + this.windowDistance);
			
			this.sampler.match(equalAttrs, compressedRecords[recordId], compressedRecords[partnerRecordId]);
			
			if (!this.negCover.contains(equalAttrs)) {
				BitSet equalAttrsCopy = (BitSet) equalAttrs.clone();
				this.negCover.add(equalAttrsCopy);
				newNonUCCs.add(equalAttrsCopy);
				
				this.memoryGuardian.memoryChanged(1);
				this.memoryGuardian.match(this.negCover, this.posCover, newNonUCCs);
			}
			numComparisons++;
		}
	}
	numNewNonFds = newNonUCCs.size() - previousNewNonFdsSize;
	
	this.numNewNonFds.add(numNewNonFds);
	this.numComparisons.add(numComparisons);
}
 

public PositionListIndex intersect(int[] otherPLI) {
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(otherPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}
 

public int removeUnsupportedUniqueEdgeOnly(){
//HashSet<Integer> readHash;
//ArrayList<Integer> removalList = new ArrayList<Integer>();
CustomHashMap readHash;
IntArrayList removalList = new IntArrayList();

for(int i=0; i<this.paths.size(); i++){//Path p: this.paths){
    Path p = this.paths.get(i);
    ArrayList<CustomWeightedEdge> eList = p.getOrderedEdgeList();
    boolean inited = false;
    readHash = null;
    int numUnique = 0;
    for(CustomWeightedEdge e : eList){
	HLA.log.append("|" + e.getNumActivePath() + "|");
	if(e.isUniqueEdge()){ //we only care about uniq edges
	    
	    numUnique++;
	    if(!inited){
		readHash = e.getReadHashSet().clone();//e.getReadHashSetDeepCopy();
		inited = true;
	    }else{
		//update with union after checking intersection
		//if null, we remove this path
		if(e.unionAfterCheckingIntersection(readHash) == null){
		    removalList.add(i);//new Integer(i));
		    break;
		}
	    }
	}
    }
    HLA.log.append("[" + numUnique +  "]");
}
for(int i=removalList.size() - 1; i >= 0; i--){
    this.paths.get(removalList.getInt(i)).excludePath();
    this.paths.remove(removalList.getInt(i));
    
}
return removalList.size();
   }
 

public boolean runNext(Set<OpenBitSet> negCover, int[][] compressedRecords) {
	this.windowDistance++;
	int numNewNonFds = 0;
	int numComparisons = 0;
	
	int previousNegCoverSize = negCover.size();
	Iterator<IntArrayList> clusterIterator = this.clusters.iterator();
	while (clusterIterator.hasNext()) {
		IntArrayList cluster = clusterIterator.next();
		
		if (cluster.size() <= this.windowDistance) {
			clusterIterator.remove();
			continue;
		}
		
		for (int recordIndex = 0; recordIndex < (cluster.size() - this.windowDistance); recordIndex++) {
			int recordId = cluster.getInt(recordIndex);
			int partnerRecordId = cluster.getInt(recordIndex + this.windowDistance);
			negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
			numComparisons++;
		}
	}
	numNewNonFds = negCover.size() - previousNegCoverSize;
	
	this.numNewNonFds.add(numNewNonFds);
	this.numComparisons.add(numComparisons);
	
	if (numComparisons == 0)
		return false;
	return true;
}
 

public PositionListIndex intersect(int[] otherPLI) {
	Int2ObjectMap<Int2ObjectMap<IntArrayList>> intersectMap = this.buildIntersectMap(otherPLI);
	
	List<IntArrayList> clusters = new ArrayList<>();
	for (Int2ObjectMap<IntArrayList> cluster1 : intersectMap.values())
		for (IntArrayList cluster2 : cluster1.values())
			if (cluster2.size() > 1)
				clusters.add(cluster2);
	
	return new PositionListIndex(-1, clusters);
}
 

@Override
public synchronized void accept(Page page)
{
    // reset the assignment lists
    for (IntList partitionAssignment : partitionAssignments) {
        partitionAssignment.clear();
    }

    // assign each row to a partition
    for (int position = 0; position < page.getPositionCount(); position++) {
        int partition = partitionGenerator.getPartition(page, position);
        partitionAssignments[partition].add(position);
    }

    // build a page for each partition
    Block[] outputBlocks = new Block[page.getChannelCount()];
    for (int partition = 0; partition < buffers.size(); partition++) {
        IntArrayList positions = partitionAssignments[partition];
        if (!positions.isEmpty()) {
            for (int i = 0; i < page.getChannelCount(); i++) {
                outputBlocks[i] = page.getBlock(i).copyPositions(positions.elements(), 0, positions.size());
            }

            Page pageSplit = new Page(positions.size(), outputBlocks);
            memoryManager.updateMemoryUsage(pageSplit.getRetainedSizeInBytes());
            buffers.get(partition).accept(new PageReference(pageSplit, 1, () -> memoryManager.updateMemoryUsage(-pageSplit.getRetainedSizeInBytes())));
        }
    }
}
 

@Override
public void onCandidateIteration(CandidateInfo candidateInfo, Object candidate, int iteration) {
    double score;
    long numParams;
    int numLayers;
    String modelConfigJson;
    int totalNumUpdates;
    if(candidate instanceof MultiLayerNetwork){
        MultiLayerNetwork m = (MultiLayerNetwork)candidate;
        score = m.score();
        numParams = m.numParams();
        numLayers = m.getnLayers();
        modelConfigJson = m.getLayerWiseConfigurations().toJson();
        totalNumUpdates = m.getLayerWiseConfigurations().getIterationCount();
    } else if(candidate instanceof ComputationGraph) {
        ComputationGraph cg = (ComputationGraph)candidate;
        score = cg.score();
        numParams = cg.numParams();
        numLayers = cg.getNumLayers();
        modelConfigJson = cg.getConfiguration().toJson();
        totalNumUpdates = cg.getConfiguration().getIterationCount();
    } else {
        score = 0;
        numParams = 0;
        numLayers = 0;
        totalNumUpdates = 0;
        modelConfigJson = "";
    }

    int idx = candidateInfo.getIndex();

    Pair<IntArrayList, FloatArrayList> pair = candidateScoreVsIter.computeIfAbsent(idx, k -> new Pair<>(new IntArrayList(), new FloatArrayList()));

    IntArrayList iter = pair.getFirst();
    FloatArrayList scores = pair.getSecond();

    //Do we need subsampling to avoid having too many data points?
    int subsamplingFreq = candidateScoreVsIterSubsampleFreq.computeIfAbsent(idx, k -> 1);
    if(iteration / subsamplingFreq > MAX_SCORE_VS_ITER_PTS){
        //Double subsampling frequency and re-parse data
        subsamplingFreq *= 2;
        candidateScoreVsIterSubsampleFreq.put(idx, subsamplingFreq);

        IntArrayList newIter = new IntArrayList();
        FloatArrayList newScores = new FloatArrayList();
        for( int i=0; i<iter.size(); i++ ){
            int it = iter.get(i);
            if(it % subsamplingFreq == 0){
                newIter.add(it);
                newScores.add(scores.get(i));
            }
        }

        iter = newIter;
        scores = newScores;
        candidateScoreVsIter.put(idx, new Pair<>(iter, scores));
    }

    if(iteration % subsamplingFreq == 0) {
        iter.add(iteration);
        scores.add((float) score);
    }


    int[] iters = iter.toIntArray();
    float[] fScores = new float[iters.length];
    for( int i=0; i<iters.length; i++ ){
        fScores[i] = scores.get(i);
    }

    ModelInfoPersistable p = new ModelInfoPersistable.Builder()
            .timestamp(candidateInfo.getCreatedTime())
            .sessionId(sessionId)
            .workerId(String.valueOf(candidateInfo.getIndex()))
            .modelIdx(candidateInfo.getIndex())
            .score(candidateInfo.getScore())
            .status(candidateInfo.getCandidateStatus())
            .scoreVsIter(iters, fScores)
            .lastUpdateTime(System.currentTimeMillis())
            .numParameters(numParams)
            .numLayers(numLayers)
            .totalNumUpdates(totalNumUpdates)
            .paramSpaceValues(candidateInfo.getFlatParams())
            .modelConfigJson(modelConfigJson)
            .exceptionStackTrace(candidateInfo.getExceptionStackTrace())
            .build();


    lastModelInfoPersistable.put(candidateInfo.getIndex(), p);
    statsStorage.putUpdate(p);
}
 

protected int countNonUniqueValuesIn(List<IntArrayList> clusters) {
	int numNonUniqueValues = 0;
	for (IntArrayList cluster : clusters)
		numNonUniqueValues += cluster.size();
	return numNonUniqueValues;
}