Java Code Examples for it.unimi.dsi.fastutil.ints.IntList#add()

/**
 * @return a IntList from the input JsonNode with the given name, or throw an BadRequestException
 */
public static IntList getRequiredListOfInteger(JsonNode json, String name) throws BadRequestException {
    final JsonNode node = json.get(name);

    if (node == null || !(node.isArray())) {
        throw new BadRequestException("json is missing required List: " + name);
    }

    final IntList list = new IntArrayList(node.size());
    for (JsonNode innerNode : node) {
        if (!innerNode.canConvertToInt()) {
            throw new BadRequestException("json is not an int: " + innerNode.toString());
        }
        list.add(innerNode.asInt());
    }
    return list;
}
 

@Test
public void testAssignmentOnlyTargetAttributeMixedSerial() {
	ILabeledDataset<ILabeledInstance> dataset = this.createToyDatasetMixed();
	Integer[] attributeIndices = { 2 };
	AttributeBasedStratiAmountSelectorAndAssigner selectorAndAssigner = new AttributeBasedStratiAmountSelectorAndAssigner(Arrays.asList(attributeIndices),
			DiscretizationStrategy.EQUAL_SIZE, 2);
	selectorAndAssigner.setLoggerName(GeneralAlgorithmTester.TESTEDALGORITHM_LOGGERNAME);
	selectorAndAssigner.setNumCPUs(1);
	selectorAndAssigner.init(dataset);
	IntList stratiAssignment = new IntArrayList();
	for (ILabeledInstance i : dataset) {
		stratiAssignment.add(selectorAndAssigner.assignToStrati(i));
	}
	// Number of strati must be 2
	assertEquals(2, new HashSet<>(stratiAssignment).size());

	assertTrue("Instances 1 and 3 need to be in the same stratum", stratiAssignment.getInt(0) == stratiAssignment.getInt(2));

	assertTrue("Instances 1 and 4 need to be in the same stratum", stratiAssignment.getInt(0) == stratiAssignment.getInt(3));

	assertFalse("Instances 1 and 2 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(1));

	assertFalse("Instances 1 and 5 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(4));

	assertFalse("Instances 1 and 6 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(5));
}
 

private void handleList(IntList list, Long2ObjectMap<Set<IntList>> maxSets, boolean firstStep) {

        for (int i = 0; i < list.size(); i++) {
            int removedElement = list.removeInt(i);
            if (maxSets.containsKey(list.size()) && maxSets.get(list.size()).contains(list))
                maxSets.get(list.size()).remove(list);
            else {
                if (list.size() > 2) {
                    this.handleList(list, maxSets, false);
                }
            }
            list.add(i, removedElement);
        }

        if (firstStep)
            maxSets.get(list.size()).add(list);
    }
 

private void handleList(IntList list, Int2ObjectMap<Set<IntList>> maxSets, boolean firstStep) {

        for (int i = 0; i < list.size(); i++) {
        	int removedElement = list.removeInt(i);
            if (maxSets.containsKey(list.size()) && maxSets.get(list.size()).contains(list))
                maxSets.get(list.size()).remove(list);
            else {
                if (list.size() > 2) {
                    this.handleList(list, maxSets, false);
                }
            }
            list.add(i, removedElement);
        }

        if (firstStep)
            maxSets.get(list.size()).add(list);
    }
 

public FeatureKnnModel buildModel() {
    List<String> features = Lists.newArrayList(svdFeature.getFactorFeatures(minSupport).keySet());
    IntList svdIndices = new IntArrayList();
    IntList simIndices = new IntArrayList(features.size());
    for (int i=0; i<features.size(); i++) {
        String feature = features.get(i);
        Map<String, String> attrVals = FeatureExtractorUtilities.decomposeKey(feature);
        boolean ifModel = true;
        if (attrVals.size() == feaAttrs.size()) {
            for (String attr : feaAttrs) {
                if (!attrVals.containsKey(attr)) {
                    ifModel = false;
                }
            }
        } else {
            ifModel = false;
        }
        if (ifModel) {
            svdIndices.add(i);
            ensureKey(feature);
            simIndices.add(getIndexByKey(feature));
        } else {
            simIndices.add(-1);
        }
    }
    Logger.info("Total number of items to compute similarity model {}: {}",
            modelName, svdIndices.size());
    svdIndices.parallelStream().forEach(curIdx -> {
        int simIdx = simIndices.getInt(curIdx);
        List<double[]> neighbors = getNeighbors(curIdx, svdIndices, svdFeature, features);
        RealVector sims = getIndexVector(simIdx);
        for (int j=0; j<neighbors.size(); j++) {
            double[] neighbor = neighbors.get(j);
            sims.setEntry(j*2, simIndices.getInt((int)neighbor[0]));
            sims.setEntry(j*2+1, neighbor[1]);
        }
        setIndexVector(simIdx, sims);
    });
    return this;
}
 

private IntList getSampledIndices(IntSet trues, int maxVal) {
    IntList samples = new IntArrayList();
    int num = trues.size();
    if (maxNumSample != null) {
        num = maxNumSample;
    }
    for (int i=0; i<num; i++) {
        int dice = new Random().nextInt(maxVal);
        if (!trues.contains(dice)) {
            samples.add(dice);
        }
    }
    return samples;
}
 

private IntList getWinVector() {
	IntList wins = new IntArrayList();
	for (short t = 0; t < this.numObjects; t ++) {
		int w = 0;
		for (ShortList ranking : this.getInput().getRankings()) {
			if (ranking.indexOf(t) < ranking.size() - 1) {
				w ++;
			}
		}
		wins.add(w);
	}
	return wins;
}
 

@Test
public void testAssignmentOnlyTargetAttributeCategoricalParallel() {
	ILabeledDataset<ILabeledInstance> dataset = this.createToyDatasetOnlyCategorical();
	Integer[] attributeIndices = { 2 };
	AttributeBasedStratiAmountSelectorAndAssigner selectorAndAssigner = new AttributeBasedStratiAmountSelectorAndAssigner(Arrays.asList(attributeIndices));
	selectorAndAssigner.setLoggerName(GeneralAlgorithmTester.TESTEDALGORITHM_LOGGERNAME);
	selectorAndAssigner.setNumCPUs(4);
	selectorAndAssigner.init(dataset);
	IntList stratiAssignment = new IntArrayList();
	for (ILabeledInstance i : dataset) {
		stratiAssignment.add(selectorAndAssigner.assignToStrati(i));
	}
	// Number of strati must be 3
	assertEquals(3, new HashSet<>(stratiAssignment).size());

	assertTrue("Instances 1 and 3 need to be in the same stratum", stratiAssignment.getInt(0) == stratiAssignment.getInt(2));

	assertFalse("Instances 1 and 2 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(1));

	assertFalse("Instances 1 and 4 need to be in the different strati", stratiAssignment.getInt(0) == stratiAssignment.getInt(3));

	assertFalse("Instances 2 and 3 need to be in the different strati", stratiAssignment.getInt(1) == stratiAssignment.getInt(2));

	assertFalse("Instances 2 and 4 need to be in the different strati", stratiAssignment.getInt(1) == stratiAssignment.getInt(3));

	assertFalse("Instances 3 and 4 need to be in the different strati", stratiAssignment.getInt(2) == stratiAssignment.getInt(3));
}
 

public IntList getApplicableChanceList(Block block, int blockMD, int level) {
    IntList chance = new IntArrayList();
    for (TreasureFinderEntry tfe : entries) {
        if (tfe.minimumLevel <= level) {
            if (((tfe.getBlock() == null) && ((block == Blocks.DIRT) || (block == Blocks.GRASS))) || ((tfe.getBlock() == block) && (tfe.blockMetadata == blockMD))) {
                chance.add(tfe.chance);
            }
        }
    }
    return chance;
}
 

@Test(dataProvider = "testValueInTransformFunction")
public void testValueInTransformFunction(String expressionStr) {
  ExpressionContext expression = QueryContextConverterUtils.getExpression(expressionStr);
  TransformFunction transformFunction = TransformFunctionFactory.get(expression, _dataSourceMap);
  Assert.assertTrue(transformFunction instanceof ValueInTransformFunction);
  Assert.assertEquals(transformFunction.getName(), ValueInTransformFunction.FUNCTION_NAME);
  Assert.assertTrue(transformFunction.getResultMetadata().hasDictionary());
  int[][] dictIds = transformFunction.transformToDictIdsMV(_projectionBlock);
  int[][] intValues = transformFunction.transformToIntValuesMV(_projectionBlock);
  long[][] longValues = transformFunction.transformToLongValuesMV(_projectionBlock);
  float[][] floatValues = transformFunction.transformToFloatValuesMV(_projectionBlock);
  double[][] doubleValues = transformFunction.transformToDoubleValuesMV(_projectionBlock);
  String[][] stringValues = transformFunction.transformToStringValuesMV(_projectionBlock);

  Dictionary dictionary = transformFunction.getDictionary();
  for (int i = 0; i < NUM_ROWS; i++) {
    IntList expectedList = new IntArrayList();
    for (int value : _intMVValues[i]) {
      if (value == 1 || value == 2 || value == 9 || value == 5) {
        expectedList.add(value);
      }
    }
    int[] expectedValues = expectedList.toIntArray();

    int numValues = expectedValues.length;
    for (int j = 0; j < numValues; j++) {
      int expected = expectedValues[j];
      Assert.assertEquals(dictIds[i][j], dictionary.indexOf(Integer.toString(expected)));
      Assert.assertEquals(intValues[i][j], expected);
      Assert.assertEquals(longValues[i][j], (long) expected);
      Assert.assertEquals(floatValues[i][j], (float) expected);
      Assert.assertEquals(doubleValues[i][j], (double) expected);
      Assert.assertEquals(stringValues[i][j], Integer.toString(expected));
    }
  }
}
 

/**
 * Each collection (Array or Map) has four variants of presence:
 * 1) Collection is not defined, because one of it's optional parent fields is null
 * 2) Collection is null
 * 3) Collection is defined but empty
 * 4) Collection is defined and not empty. In this case offset value is increased by the number of elements in that collection
 */
public static void calculateCollectionOffsets(Field field, IntList offsets, BooleanList collectionIsNull, int[] definitionLevels, int[] repetitionLevels)
{
    int maxDefinitionLevel = field.getDefinitionLevel();
    int maxElementRepetitionLevel = field.getRepetitionLevel() + 1;
    boolean required = field.isRequired();
    int offset = 0;
    offsets.add(offset);
    for (int i = 0; i < definitionLevels.length; i = getNextCollectionStartIndex(repetitionLevels, maxElementRepetitionLevel, i)) {
        if (ParquetTypeUtils.isValueNull(required, definitionLevels[i], maxDefinitionLevel)) {
            // Collection is null
            collectionIsNull.add(true);
            offsets.add(offset);
        }
        else if (definitionLevels[i] == maxDefinitionLevel) {
            // Collection is defined but empty
            collectionIsNull.add(false);
            offsets.add(offset);
        }
        else if (definitionLevels[i] > maxDefinitionLevel) {
            // Collection is defined and not empty
            collectionIsNull.add(false);
            offset += getCollectionSize(repetitionLevels, maxElementRepetitionLevel, i + 1);
            offsets.add(offset);
        }
    }
}
 

private void doRecusiveCrap(int currentAttribute, IntList currentOrdering, List<DifferenceSet> setsNotCovered,
                            IntList currentPath, List<DifferenceSet> originalDiffSet, List<FunctionalDependencyGroup2> result)
        throws CouldNotReceiveResultException, ColumnNameMismatchException {

    // Basic Case
    // FIXME
    if (!currentOrdering.isEmpty() && /* BUT */setsNotCovered.isEmpty()) {
        if (this.debugSysout)
            System.out.println("no FDs here");
        return;
    }

    if (setsNotCovered.isEmpty()) {

        List<BitSet> subSets = this.generateSubSets(currentPath);
        if (this.noOneCovers(subSets, originalDiffSet)) {
            FunctionalDependencyGroup2 fdg = new FunctionalDependencyGroup2(currentAttribute, currentPath);
            this.addFdToReceivers(fdg);
            result.add(fdg);
        } else {
            if (this.debugSysout) {
                System.out.println("FD not minimal");
                System.out.println(new FunctionalDependencyGroup2(currentAttribute, currentPath));
            }
        }

        return;
    }

    // Recusive Case
    for (int i = 0; i < currentOrdering.size(); i++) {

        List<DifferenceSet> next = this.generateNextNotCovered(currentOrdering.getInt(i), setsNotCovered);
        IntList nextOrdering = this.generateNextOrdering(next, currentOrdering, currentOrdering.getInt(i));
        IntList currentPathCopy = new IntArrayList(currentPath);
        currentPathCopy.add(currentOrdering.getInt(i));
        this.doRecusiveCrap(currentAttribute, nextOrdering, next, currentPathCopy, originalDiffSet, result);
    }

}
 

public static IntList convertToIntList(BitSet set) {
    IntList bits = new IntArrayList();
    int lastIndex = set.nextSetBit(0);
    while (lastIndex != -1) {
        bits.add(lastIndex);
        lastIndex = set.nextSetBit(lastIndex + 1);
    }
    return bits;
}
 

private void increaseMeasure(final IntList measureList, final int level) {
  if (measureList.size() > level - 1) {
    final int prev = measureList.getInt(level - 1);
    measureList.set(level - 1, prev + 1);
  } else {
    final int skippedLevels = level - measureList.size() - 1;
    // add 0 for no application of the pruning rule in the skipped levels
    for (int i = 0; i < skippedLevels; i++) {
      measureList.add(0);
    }
    // add 1 for one application of the pruning rule in the new level
    measureList.add(1);
  }
}
 

@Override
protected IntList getRight(MathVector vector) {
    IntList recommendList = new IntArrayList(vector.getElementSize());
    for (VectorScalar scalar : vector) {
        if (RandomUtility.randomFloat(1F) < 0.5F) {
            recommendList.add(scalar.getIndex());
        }
    }
    return recommendList;
}
 

static OffsetIndex filterOffsetIndex(OffsetIndex offsetIndex, RowRanges rowRanges, long totalRowCount) {
  IntList indexMap = new IntArrayList();
  for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) {
    long from = offsetIndex.getFirstRowIndex(i);
    if (rowRanges.isOverlapping(from, offsetIndex.getLastRowIndex(i, totalRowCount))) {
      indexMap.add(i);
    }
  }
  return new FilteredOffsetIndex(offsetIndex, indexMap.toIntArray());
}
 

private IntList generateNextOrdering(List<DifferenceSet> next, IntList currentOrdering, int attribute) {

        IntList result = new IntArrayList();

        Int2IntMap counting = new Int2IntArrayMap();
        boolean seen = false;
        for (int i = 0; i < currentOrdering.size(); i++) {

            if (!seen) {
                if (currentOrdering.getInt(i) == attribute) {
                    seen = true;
                }
            } else {

                counting.put(currentOrdering.getInt(i), 0);
                for (DifferenceSet ds : next) {

                    if (ds.getAttributes().get(currentOrdering.getInt(i))) {
                        counting.put(currentOrdering.getInt(i), counting.get(currentOrdering.getInt(i)) + 1);
                    }
                }
            }
        }

        // TODO: Comperator und TreeMap --> Tommy
        while (true) {

            if (counting.size() == 0) {
                break;
            }

            int biggestAttribute = -1;
            int numberOfOcc = 0;
            for (int attr : counting.keySet()) {

                if (biggestAttribute < 0) {
                    biggestAttribute = attr;
                    numberOfOcc = counting.get(attr);
                    continue;
                }

                int tempOcc = counting.get(attr);
                if (tempOcc > numberOfOcc) {
                    numberOfOcc = tempOcc;
                    biggestAttribute = attr;
                } else if (tempOcc == numberOfOcc) {
                    if (biggestAttribute > attr) {
                        biggestAttribute = attr;
                    }
                }
            }

            if (numberOfOcc == 0) {
                break;
            }

            result.add(biggestAttribute);
            counting.remove(biggestAttribute);
        }

        return result;
    }
 

@Override
protected AnchorTernaryTrie parseSet() throws IOException
{
	
	File indexDir = RepositoryDirs.ANCHORS.getDir(lang);
	long indexSize = FileUtils.sizeOfDirectory(indexDir);
	long maxMemory = Runtime.getRuntime().maxMemory();
	
	IndexReader anchors;
	if (indexSize < maxMemory * 0.8){
		
		log.info("MaxMemory is enough, loading Anchor index...");
		anchors = IndexReader.open(new RAMDirectory(new SimpleFSDirectory(indexDir)), true);
		log.info("Anchor index loaded.");
		
	} else {
		log.info("Not enough memory ["+maxMemory/1000000+"Mb] to load Anchor index (about "+indexSize/1000000+"Mb)");
		anchors = Indexes.getReader(RepositoryDirs.ANCHORS.getPath(lang));
	}

	
	AnchorTernaryTrie trie = new AnchorTernaryTrie();
	
	int maxdoc = anchors.maxDoc();
	
	IntList doclist = new IntArrayList();
	for(int i=0;i<maxdoc;i++) doclist.add(i);
	Random rnd = new Random(System.currentTimeMillis());
	
	PLogger plog = new PLogger(log, Step.TEN_MINUTES, "anchors", "skipped", "duplicates");
	plog.setEnd(0, maxdoc);
	plog.start("Inserting in to trie...");
	while(!doclist.isEmpty())
	{
		int docID = doclist.removeInt(rnd.nextInt(doclist.size()));
		
		plog.update(0);
		Document doc = anchors.document(docID);
		if (doc == null){
			plog.update(1);
			continue;
		}
		
		String anchorText = doc.get(AnchorIndexer.FIELD_TEXT);
		String serial = doc.get(AnchorIndexer.FIELD_OBJECT);
		Anchor anchorObj = Anchor.deserialize(serial);
		
		if (anchorObj == null){
			plog.update(1);
			continue;
		}
		
		boolean added = trie.add(anchorText, anchorObj);
		
		if (!added) plog.update(2);
	}
	plog.stop();
	
	return trie;
}
 

private IntList generateInitialOrdering(List<DifferenceSet> tempDiffSet) {

        IntList result = new IntArrayList();

        Int2IntMap counting = new Int2IntArrayMap();
        for (DifferenceSet ds : tempDiffSet) {

            int lastIndex = ds.getAttributes().nextSetBit(0);

            while (lastIndex != -1) {
                if (!counting.containsKey(lastIndex)) {
                    counting.put(lastIndex, 1);
                } else {
                    counting.put(lastIndex, counting.get(lastIndex) + 1);
                }
                lastIndex = ds.getAttributes().nextSetBit(lastIndex + 1);
            }
        }

        // TODO: Comperator und TreeMap --> Tommy
        while (true) {

            if (counting.size() == 0) {
                break;
            }

            int biggestAttribute = -1;
            int numberOfOcc = 0;
            for (int attr : counting.keySet()) {

                if (biggestAttribute < 0) {
                    biggestAttribute = attr;
                    numberOfOcc = counting.get(attr);
                    continue;
                }

                int tempOcc = counting.get(attr);
                if (tempOcc > numberOfOcc) {
                    numberOfOcc = tempOcc;
                    biggestAttribute = attr;
                } else if (tempOcc == numberOfOcc) {
                    if (biggestAttribute > attr) {
                        biggestAttribute = attr;
                    }
                }
            }

            if (numberOfOcc == 0) {
                break;
            }

            result.add(biggestAttribute);
            counting.remove(biggestAttribute);
        }

        return result;
    }
 

public static UpgradeData fixChunk(WorldChunk chunk) {
    IntList centerIndicesToUpgrade = new IntArrayList();
    int sidesToUpgrade = 0;

    BlockPos.Mutable otherPos = new BlockPos.Mutable();
    for (BlockPos pos : BlockPos.iterate(chunk.getPos().getStartX(), 0, chunk.getPos().getStartZ(),
            chunk.getPos().getEndX(), chunk.getHighestNonEmptySectionYOffset() + 15, chunk.getPos().getEndZ())) {
        BlockState state = chunk.getBlockState(pos);
        Block block = state.getBlock();
        inPlaceFix(chunk, state, pos, otherPos);

        int blockId = Registry.BLOCK.getRawId(block) & 4095;
        if (ChunkPalettedStorageFixAccessor.getBlocksNeedingSideUpdate().get(blockId)) {
            boolean west = (pos.getX() & 15) == 0;
            boolean east = (pos.getX() & 15) == 15;
            boolean north = (pos.getZ() & 15) == 0;
            boolean south = (pos.getZ() & 15) == 15;
            if (north) {
                if (east) {
                    sidesToUpgrade |= 2;
                } else if (west) {
                    sidesToUpgrade |= 128;
                } else {
                    sidesToUpgrade |= 1;
                }
            } else if (south) {
                if (west) {
                    sidesToUpgrade |= 32;
                } else if (east) {
                    sidesToUpgrade |= 8;
                } else {
                    sidesToUpgrade |= 16;
                }
            } else if (east) {
                sidesToUpgrade |= 4;
            } else if (west) {
                sidesToUpgrade |= 64;
            } else {
                centerIndicesToUpgrade.add(pos.getY() << 8 | (pos.getZ() & 15) << 4 | (pos.getX() & 15));
            }
        }
    }

    if (centerIndicesToUpgrade.isEmpty() && sidesToUpgrade == 0)
        return null;

    CompoundTag upgradeData = new CompoundTag();
    upgradeData.putInt("Sides", sidesToUpgrade);
    CompoundTag centerIndices = new CompoundTag();
    centerIndicesToUpgrade.forEach((IntConsumer) index -> {
        int low = index & 4095;
        int high = index >>> 12;
        Tag tag = centerIndices.get(String.valueOf(high));
        if (tag == null)
            centerIndices.put(String.valueOf(high), tag = new ListTag());
        ((ListTag) tag).add(IntTag.of(low));
    });
    for (String key : centerIndices.getKeys()) {
        //noinspection ConstantConditions
        centerIndices.put(key, new IntArrayTag(((ListTag) centerIndices.get(key)).stream().mapToInt(val -> ((IntTag) val).getInt()).toArray()));
    }
    upgradeData.put("Indices", centerIndices);
    return new UpgradeData(upgradeData);
}