it.unimi.dsi.fastutil.ints.IntArrayList Java Examples

private synchronized IntArrayList partitionPage(Page page, IntPredicate spillPartitionMask)
{
    IntArrayList unspilledPositions = new IntArrayList();

    for (int position = 0; position < page.getPositionCount(); position++) {
        int partition = partitionFunction.getPartition(page, position);

        if (!spillPartitionMask.test(partition)) {
            unspilledPositions.add(position);
            continue;
        }

        spilledPartitions.add(partition);
        PageBuilder pageBuilder = pageBuilders.get(partition);
        pageBuilder.declarePosition();
        for (int channel = 0; channel < types.size(); channel++) {
            Type type = types.get(channel);
            type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
        }
    }

    return unspilledPositions;
}
 

public ColumnChunk readPrimitive(Field field)
{
    IntList definitionLevels = new IntArrayList();
    IntList repetitionLevels = new IntArrayList();
    seek();
    BlockBuilder blockBuilder = field.getType().createBlockBuilder(null, nextBatchSize);
    int valueCount = 0;
    while (valueCount < nextBatchSize) {
        if (page == null) {
            readNextPage();
        }
        int valuesToRead = Math.min(remainingValueCountInPage, nextBatchSize - valueCount);
        readValues(blockBuilder, valuesToRead, field.getType(), definitionLevels, repetitionLevels);
        valueCount += valuesToRead;
    }
    checkArgument(valueCount == nextBatchSize, "valueCount %s not equals to batchSize %s", valueCount, nextBatchSize);

    readOffset = 0;
    nextBatchSize = 0;
    return new ColumnChunk(blockBuilder.build(), definitionLevels.toIntArray(), repetitionLevels.toIntArray());
}
 

public static String concat(IntArrayList integers, String separator) {
	if (integers == null)
		return "";
	
	StringBuilder buffer = new StringBuilder();
	
	for (int integer : integers) {
		buffer.append(integer);
		buffer.append(separator);
	}
	
	if (buffer.length() > separator.length())
		buffer.delete(buffer.length() - separator.length(), buffer.length());
	
	return buffer.toString();
}
 

/**
 * Given a list of lists, return all the combinations between the lists (i.e. their indices). For example, suppose we
 * have the list of lists: [[1, 2, 3], [4, 5], [6, 7, 8]]. Then, this function will return:
 * [[0, 1], [1, 0], [0, 2], [2, 0], [1, 2], [2, 1], 
 *  [0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 1, 0], [2, 0, 1]]
 * @param lists: list of lists
 * @return
 */
public static <T> ObjectArrayList<IntArrayList> getListsCombinationIndices(ObjectArrayList<ObjectArrayList<T>> lists){
    ObjectArrayList<IntArrayList> combinationsInd = new ObjectArrayList<>();
    ObjectArrayList<IntArrayList> result = new ObjectArrayList<>();
    int[][] combinations;
    
    for (int k = 2; k <= lists.size(); k++){
        result.clear();
        combinations = null;
        
        combinations = getCombinations(k, lists.size());
        
        for (int i = 0; i < combinations.length; i++) {
            IntArrayList indices = new IntArrayList();
            for (int j = 0; j < combinations[i].length; j++) {
                indices.add(combinations[i][j]);
            }
            permute(indices, 0, result);
        }
        combinationsInd.addAll(result);
    }
    return combinationsInd;
}
 

private void fetchNonFdsWindowingOverClusters(Set<BitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tMoving window over small clusters ...");
	for (PositionListIndex pli : plis) {
		boolean selectSmallClustersOnly = pli.getClusters().size() < this.attributeThreshold; 	// If there are too few clusters, then the clusters are large and we have already executed sufficient comparisons between the records of these clusters
		
		for (IntArrayList cluster : pli.getClusters()) {
			if (selectSmallClustersOnly && (cluster.size() > this.windowSize))					// But if the current cluster is very small, we should still use it for comparisons (the other cluster(s) must be very large)
				continue;
			
			for (int recordIndex = 0; recordIndex < cluster.size(); recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = recordIndex + 1; partnerRecordIndex < Math.min(recordIndex + this.windowSize, cluster.size()); partnerRecordIndex++) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}
 

@Test
public void testGetChildPatternsFromExistingPattern() {
    int[][] values = new int[][] {
            new int[] {0, 1, 2},
            new int[] {0, 3, 4}
    };
    Pattern pattern = new Pattern(createAttributeMap(new int[] {0, -1, -1}));

    List<IntArrayList> cover = new ArrayList<>();
    cover.add(new IntArrayList(new int[] {0}));
    cover.add(new IntArrayList(new int[] {1}));
    pattern.setCover(cover);

    ConstantPatternExpansionStrategy out = new ConstantPatternExpansionStrategy(values);
    List<Pattern> children = out.getChildPatterns(pattern);

    Assert.assertEquals(4, children.size());
    Assert.assertTrue(containsPattern(children, new Pattern(createAttributeMap(new int[] {0, 1, -1}))));
    Assert.assertTrue(containsPattern(children, new Pattern(createAttributeMap(new int[] {0, -1, 2}))));
    Assert.assertTrue(containsPattern(children, new Pattern(createAttributeMap(new int[] {0, 3, -1}))));
    Assert.assertTrue(containsPattern(children, new Pattern(createAttributeMap(new int[] {0, -1, 4}))));
}
 

/**
 * Given a sequence of words and a pivot-word index, return the "chained words" from the left and from the right
 * of the pivot word. "Chained words" are a list of words, which all of them share the same POS tag and have no 
 * NE types.
 * 
 * @param sequence: a sequence of words (list of IndexedWord)
 * @param wordInd: the index of the pivot word
 * @return a list of chained words to the left and the right of the pivot word (the pivot word is included)
 */
public static ObjectArrayList<IndexedWord> getChainedTagNoNER(ObjectArrayList<IndexedWord> sequence, int wordInd){
    IntArrayList chainedPosWordsInd = new IntArrayList();
    
    // Get the chained nouns from left and right
    IntArrayList chainedPosWordsLeft = getChainedTagsFromLeftNoNER(sequence, chainedPosWordsInd.clone(), wordInd);
    IntArrayList chainedPosWordsRight = getChainedTagsFromRightNoNER(sequence, chainedPosWordsInd.clone(), wordInd);
    
    // Add all the words to the chained nouns
    chainedPosWordsInd.addAll(chainedPosWordsLeft);
    chainedPosWordsInd.add(wordInd);
    chainedPosWordsInd.addAll(chainedPosWordsRight);
    
    // IndexedWord chained nouns
    ObjectArrayList<IndexedWord> iChainedNouns = new ObjectArrayList<IndexedWord>();
    for (int i: FastUtil.sort(chainedPosWordsInd)){
        iChainedNouns.add(sequence.get(i));
    }
    
    return iChainedNouns;
}
 

@Test
public void testAssignmentOnlyTargetAttributeMixedSerial() {
	ILabeledDataset<ILabeledInstance> dataset = this.createToyDatasetMixed();
	Integer[] attributeIndices = { 2 };
	AttributeBasedStratiAmountSelectorAndAssigner selectorAndAssigner = new AttributeBasedStratiAmountSelectorAndAssigner(Arrays.asList(attributeIndices),
			DiscretizationStrategy.EQUAL_SIZE, 2);
	selectorAndAssigner.setLoggerName(GeneralAlgorithmTester.TESTEDALGORITHM_LOGGERNAME);
	selectorAndAssigner.setNumCPUs(1);
	selectorAndAssigner.init(dataset);
	IntList stratiAssignment = new IntArrayList();
	for (ILabeledInstance i : dataset) {
		stratiAssignment.add(selectorAndAssigner.assignToStrati(i));
	}
	// Number of strati must be 2
	assertEquals(2, new HashSet<>(stratiAssignment).size());

	assertTrue("Instances 1 and 3 need to be in the same stratum", stratiAssignment.getInt(0) == stratiAssignment.getInt(2));

	assertTrue("Instances 1 and 4 need to be in the same stratum", stratiAssignment.getInt(0) == stratiAssignment.getInt(3));

	assertFalse("Instances 1 and 2 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(1));

	assertFalse("Instances 1 and 5 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(4));

	assertFalse("Instances 1 and 6 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(5));
}
 

private ColumnChunk readMap(GroupField field)
        throws IOException
{
    List<Type> parameters = field.getType().getTypeParameters();
    checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %s", parameters.size());
    Block[] blocks = new Block[parameters.size()];

    ColumnChunk columnChunk = readColumnChunk(field.getChildren().get(0).get());
    blocks[0] = columnChunk.getBlock();
    blocks[1] = readColumnChunk(field.getChildren().get(1).get()).getBlock();
    IntList offsets = new IntArrayList();
    BooleanList valueIsNull = new BooleanArrayList();
    calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    Block mapBlock = ((MapType) field.getType()).createBlockFromKeyValue(Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), blocks[0], blocks[1]);
    return new ColumnChunk(mapBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}
 

public boolean refines(int[][] lhsInvertedPlis, int[] rhs) {
	for (IntArrayList cluster : this.clusters) {
		Object2IntOpenHashMap<IntArrayList> clustersMap = new Object2IntOpenHashMap<>(cluster.size());
		
		// Check if all subclusters of this cluster point into the same other clusters
		for (int recordId : cluster) {
			IntArrayList additionalLhsCluster = this.buildClusterIdentifier(recordId, lhsInvertedPlis);
			if (additionalLhsCluster == null)
				continue;
			
			if (clustersMap.containsKey(additionalLhsCluster)) {
				if ((rhs[recordId] == -1) || (clustersMap.getInt(additionalLhsCluster) != rhs[recordId]))
					return false;
			}
			else {
				clustersMap.put(additionalLhsCluster, rhs[recordId]);
			}
		}
	}
	return true;
}
 

private void executePara(int attribute) throws CouldNotReceiveResultException, ColumnNameMismatchException {

        for (BitSet lhs : this.lhss.get(attribute)) {
            if (lhs.get(attribute)) {
                continue;
            }
            IntList bits = new IntArrayList();
            int lastIndex = lhs.nextSetBit(0);
            while (lastIndex != -1) {
                bits.add(lastIndex);
                lastIndex = lhs.nextSetBit(lastIndex + 1);
            }

            FunctionalDependencyGroup2 fdg = new FunctionalDependencyGroup2(attribute, bits);
            this.fdrr.receiveResult((fdg.buildDependency(this.relationName, this.columns)));
            this.result.add(fdg);
        }
    }
 

@Override
public IntColumn lag(int n) {
  final int srcPos = n >= 0 ? 0 : 0 - n;
  final int[] dest = new int[size()];
  final int destPos = n <= 0 ? 0 : n;
  final int length = n >= 0 ? size() - n : size() + n;

  for (int i = 0; i < size(); i++) {
    dest[i] = IntColumnType.missingValueIndicator();
  }

  int[] array = data.toIntArray();

  System.arraycopy(array, srcPos, dest, destPos, length);
  return new IntColumn(name() + " lag(" + n + ")", new IntArrayList(dest));
}
 

@Override
public void end() {
  if (elementClass == boolean.class) {
    parent.add(((BooleanArrayList) container).toBooleanArray());
  } else if (elementClass == byte.class) {
    parent.add(((ByteArrayList) container).toByteArray());
  } else if (elementClass == char.class) {
    parent.add(((CharArrayList) container).toCharArray());
  } else if (elementClass == short.class) {
    parent.add(((ShortArrayList) container).toShortArray());
  } else if (elementClass == int.class) {
    parent.add(((IntArrayList) container).toIntArray());
  } else if (elementClass == long.class) {
    parent.add(((LongArrayList) container).toLongArray());
  } else if (elementClass == float.class) {
    parent.add(((FloatArrayList) container).toFloatArray());
  } else if (elementClass == double.class) {
    parent.add(((DoubleArrayList) container).toDoubleArray());
  } else {
    parent.add(((ArrayList) container).toArray());
  }
}
 

private void enrichCompressedRecords(int[][] compressedRecords, final List<List<IntArrayList>> enrichedPLIs) {
	for (int tupleId = 0; tupleId < compressedRecords.length; tupleId += 1) {
		int[] tuple = compressedRecords[tupleId];
		for (int attr = 0; attr < tuple.length; attr += 1) {
			if (tuple[attr] == -1) {
				List<IntArrayList> clusters = enrichedPLIs.get(attr);
				for (int clusterId = clusters.size() - 1; clusterId >= 0; clusterId -= 1) {
					IntArrayList cluster = clusters.get(clusterId);
					if (cluster.getInt(0) == tupleId) {
						tuple[attr] = clusterId;
						break;
					}
				}
			}
		}
	}
}
 

public Selection lessThan(long value) {
  Selection selection = new BitmapBackedSelection();
  Long2ObjectSortedMap<IntArrayList> head =
      index.headMap(value); // we add 1 to get values equal to the arg
  for (IntArrayList keys : head.values()) {
    addAllToSelection(keys, selection);
  }
  return selection;
}
 

protected List<HashMap<String, IntArrayList>> calculateClusterMaps(RelationalInput relationalInput, int numAttributes) throws InputIterationException {
	List<HashMap<String, IntArrayList>> clusterMaps = new ArrayList<>();
	for (int i = 0; i < numAttributes; i++)
		clusterMaps.add(new HashMap<String, IntArrayList>());
	
	this.numRecords = 0;
	while (relationalInput.hasNext() && (this.inputRowLimit <= 0 || this.inputRowLimit != this.numRecords)) {
		List<String> record = relationalInput.next();
		
		int attributeId = 0;
		for (String value : record) {
			HashMap<String, IntArrayList> clusterMap = clusterMaps.get(attributeId);
			
			if (clusterMap.containsKey(value)) {
				clusterMap.get(value).add(this.numRecords);
			}
			else {
				IntArrayList newCluster = new IntArrayList();
				newCluster.add(this.numRecords);
				clusterMap.put(value, newCluster);
			}
			
			attributeId++;
		}
		this.numRecords++;
		if (this.numRecords == Integer.MAX_VALUE - 1)
			throw new RuntimeException("PLI encoding into integer based PLIs is not possible, because the number of records in the dataset exceeds Integer.MAX_VALUE. Use long based plis instead! (NumRecords = " + this.numRecords + " and Integer.MAX_VALUE = " + Integer.MAX_VALUE);
	}
	
	return clusterMaps;
}
 

private String concat(BitSet bits, int length, String separator) {
	if (bits == null)
		return "";
	
	IntArrayList ints = new IntArrayList(length);
	for (int bit = 0; bit < length; bit++)
		ints.add(bits.get(bit) ? 1 : 0);
	
	return CollectionUtils.concat(ints, separator);
}
 

@Override
public String toString() {
	StringBuilder builder = new StringBuilder("{ ");
	for (IntArrayList cluster : this.clusters) {
		builder.append("{");
		builder.append(CollectionUtils.concat(cluster, ","));
		builder.append("} ");
	}
	builder.append("}");
	return builder.toString();
}
 

protected List<IntOpenHashSet> convertClustersToSets(List<IntArrayList> listCluster) {
	List<IntOpenHashSet> setClusters = new LinkedList<>();
	for (IntArrayList cluster : listCluster) {
		setClusters.add(new IntOpenHashSet(cluster));
	}

	return setClusters;
}
 

public void retainAll(IntArrayList otherList) {
	this.initialize();
	
	Element previous = null;
	Element current = this.first;
	while (current != null) {
		if (otherList.contains(current.value)) {
			previous = current;
			current = current.next;
		}
		else {
			if (previous == null) {
				this.first = current.next;
				current.next = null;
				if (this.first == null)
					current = null;
				else
					current = this.first;
				
			}
			else {
				previous.next = current.next;
				current.next = null;
				current = previous.next;
			}
		}
	}
}
 

public Selection greaterThan(float value) {
  Selection selection = new BitmapBackedSelection();
  Float2ObjectSortedMap<IntArrayList> tail = index.tailMap(value + 0.000001f);
  for (IntArrayList keys : tail.values()) {
    addAllToSelection(keys, selection);
  }
  return selection;
}
 

public PositionListIndex intersect(int[]... plis) {
	List<IntArrayList> clusters = new ArrayList<>();
	for (IntArrayList pivotCluster : this.clusters) {
		HashMap<IntArrayList, IntArrayList> clustersMap = new HashMap<IntArrayList, IntArrayList>(pivotCluster.size());
		
		for (int recordId : pivotCluster) {
			IntArrayList subClusters = new IntArrayList(plis.length);
			
			boolean isUnique = false;
			for (int i = 0; i < plis.length; i++) {
				if (plis[i][recordId] == -1) {
					isUnique = true;
					break;
				}	
				subClusters.add(plis[i][recordId]);
			}
			if (isUnique)
				continue;
			
			if (!clustersMap.containsKey(subClusters))
				clustersMap.put(subClusters, new IntArrayList());
			
			clustersMap.get(subClusters).add(recordId);
		}
		
		for (IntArrayList cluster : clustersMap.values())
			if (cluster.size() > 1)
				clusters.add(cluster);
	}
	return new PositionListIndex(-1, clusters);
}
 

/**
 * Given a list of lists, and a list of lists of integers, which is a combination of indices between the elements of 
 * "lists", get the set of all elements' combinations. For example, if we have a list of the list 'combinationsInd' which is
 * [1, 2], and the list of lists 'lists' is [[1, 2, 3], [4, 5], [6, 7, 8]], then this function will add the following lists 
 * to the result: [[4, 6], [4, 7], [4, 8], [5, 7], [5, 8]] 
 * @param combinationsInd: list of indices of the lists to be combined
 * @param lists: list of lists
 * @return
 */
public static <T> ObjectOpenHashSet<ObjectArrayList<T>> getListsElementsCombinationSet(
                        ObjectArrayList<IntArrayList> combinationsInd, ObjectArrayList<ObjectArrayList<T>> lists){
    ObjectOpenHashSet<ObjectArrayList<T>> combinationSets = new ObjectOpenHashSet<>();
    ObjectArrayList<ObjectArrayList<T>> tempLists = new ObjectArrayList<>();

    for (IntArrayList indList: combinationsInd){
        tempLists.clear();
        for (int index: indList){
            tempLists.add(lists.get(index));
        }
        combinationSets.addAll(getElementsCombinations(tempLists));
    }
    return combinationSets;
}
 

public Selection lessThan(short value) {
  Selection selection = new BitmapBackedSelection();
  Short2ObjectSortedMap<IntArrayList> head =
      index.headMap(value); // we add 1 to get values equal to the arg
  for (IntArrayList keys : head.values()) {
    addAllToSelection(keys, selection);
  }
  return selection;
}
 

/**
 * Returns a bitmap containing row numbers of all cells matching the given int
 *
 * @param value This is a 'key' from the index perspective, meaning it is a value from the
 *     standpoint of the column
 */
public Selection get(double value) {
  Selection selection = new BitmapBackedSelection();
  IntArrayList list = index.get(value);
  if (list != null) {
    addAllToSelection(list, selection);
  }
  return selection;
}
 

/**
 * Constructor.
 *
 * @param prefs preference data
 * @param rnd random number generator
 */
public BPRPreferenceFMData(FastPreferenceData<?, ?> prefs, Random rnd) {
    this.prefs = prefs;
    this.rnd = rnd;

    this.uidxs = new IntArrayList();
    prefs.getUidxWithPreferences().forEach(uidxs::add);

    this.iidxs = new IntArrayList();
    prefs.getIidxWithPreferences().forEach(iidxs::add);
}
 

@Override
public DateColumn unique() {
  IntSet ints = new IntOpenHashSet(data.size());
  for (int i = 0; i < size(); i++) {
    ints.add(data.getInt(i));
  }
  DateColumn copy = emptyCopy(ints.size());
  copy.setName(name() + " Unique values");
  copy.data = IntArrayList.wrap(ints.toIntArray());
  return copy;
}
 

protected boolean probe(int[][] compressedRecords, int rhsAttr, IntArrayList cluster) {
	int rhsClusterId = compressedRecords[cluster.getInt(0)][rhsAttr];
	
	// If otherClusterId < 0, then this cluster must point into more than one other clusters
	if (rhsClusterId == -1)
		return false;
	
	// Check if all records of this cluster point into the same other cluster
	for (int recordId : cluster)
		if (compressedRecords[recordId][rhsAttr] != rhsClusterId)
			return false;
	
	return true;
}
 

public static IntList convertToIntList(BitSet set) {
    IntList bits = new IntArrayList();
    int lastIndex = set.nextSetBit(0);
    while (lastIndex != -1) {
        bits.add(lastIndex);
        lastIndex = set.nextSetBit(lastIndex + 1);
    }
    return bits;
}
 

protected List<IntOpenHashSet> convertClustersToSets(List<IntArrayList> listCluster) {
	List<IntOpenHashSet> setClusters = new LinkedList<>();
	for (IntArrayList cluster : listCluster) {
		setClusters.add(new IntOpenHashSet(cluster));
	}

	return setClusters;
}